From d94b227906a67d6f283c890f4080259e0706160d Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Mon, 14 Sep 2026 12:27:24 +0000
Subject: [PATCH 001/160] fix(mcp): stable ordering for MCP servers list in
 Admin UI

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../mcp_management_endpoints.py               |  9 ++-
 .../test_mcp_management_endpoints.py          | 70 +++++++++++++++++++
 .../_components/mcp_servers.test.tsx          | 29 +++++++-
 .../mcp-servers/_components/mcp_servers.tsx   | 49 ++++++-------
 4 files changed, 127 insertions(+), 30 deletions(-)

diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py
index 4c97bbaf5de..1dac2fd3a77 100644
--- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py
+++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py
@@ -1024,6 +1024,9 @@ if MCP_AVAILABLE:
         return {"servers": registry_servers}
 
     ## FastAPI Routes
+    def _mcp_server_display_order(server: LiteLLM_MCPServerTable) -> tuple[str, str]:
+        return ((server.server_name or server.alias or server.server_id).lower(), server.server_id)
+
     def _get_user_mcp_management_mode() -> UserMCPManagementMode:
         from litellm.proxy.proxy_server import (
             general_settings as proxy_general_settings,
@@ -1174,10 +1177,12 @@ if MCP_AVAILABLE:
                         detail="You do not have permission to view MCP servers for this team.",
                     )
 
-            redacted_mcp_servers = await _get_team_scoped_mcp_server_list(sanitized_team_id)
+            redacted_mcp_servers = sorted(
+                await _get_team_scoped_mcp_server_list(sanitized_team_id), key=_mcp_server_display_order
+            )
         else:
             servers: Final = await _resolve_accessible_mcp_servers(user_api_key_dict)
-            redacted_mcp_servers = _redact_mcp_credentials_list(servers)
+            redacted_mcp_servers = sorted(_redact_mcp_credentials_list(servers), key=_mcp_server_display_order)
 
         if connected_app_view is True and is_ui_session_credential(user_api_key_dict):
             reachable_ids: Final = await _connected_app_reachable_server_ids(user_api_key_dict)
diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py
index 5e00e7d75be..96bb0e35f0f 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py
@@ -1535,6 +1535,76 @@ class TestTeamScopedMCPServerAccess:
             result = await fetch_all_mcp_servers(user_api_key_dict=mock_user_auth, team_id="any-team-id")
             assert len(result) == 1
 
+    @pytest.mark.asyncio
+    async def test_team_scoped_list_is_sorted_by_display_name(self):
+        """Set-derived resolution order must not leak to the client."""
+        mock_user_auth = generate_mock_user_api_key_auth(
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+            user_id="admin_user",
+        )
+        unsorted = [
+            generate_mock_mcp_server_db_record(server_id="s-zeta", alias="zeta"),
+            generate_mock_mcp_server_db_record(server_id="s-alpha", alias="Alpha"),
+            generate_mock_mcp_server_db_record(server_id="s-mid", alias="mid"),
+        ]
+
+        with (
+            patch(
+                "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view",
+                return_value=True,
+            ),
+            patch(
+                "litellm.proxy.management_endpoints.mcp_management_endpoints._get_team_scoped_mcp_server_list",
+                AsyncMock(return_value=unsorted),
+            ),
+        ):
+            from litellm.proxy.management_endpoints.mcp_management_endpoints import (
+                fetch_all_mcp_servers,
+            )
+
+            result = await fetch_all_mcp_servers(user_api_key_dict=mock_user_auth, team_id="any-team-id")
+            assert [s.server_id for s in result] == ["s-alpha", "s-mid", "s-zeta"]
+
+
+class TestFetchAllMCPServersOrdering:
+    @pytest.mark.asyncio
+    async def test_list_is_sorted_by_display_name_regardless_of_resolution_order(self):
+        """The registry resolves ids through a set, so the response must impose its own order."""
+        mock_user_auth = generate_mock_user_api_key_auth(
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+            user_id="admin_user",
+        )
+        first_order = [
+            generate_mock_mcp_server_db_record(server_id="s-zeta", alias="zeta"),
+            generate_mock_mcp_server_db_record(server_id="s-alpha", alias="Alpha"),
+            generate_mock_mcp_server_db_record(server_id="s-mid", alias="mid"),
+        ]
+        second_order = list(reversed(first_order))
+
+        for resolved in (first_order, second_order):
+            mock_manager = MagicMock()
+            mock_manager.get_all_allowed_mcp_servers = AsyncMock(return_value=resolved)
+            with (
+                patch(
+                    "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager",
+                    mock_manager,
+                ),
+                patch(
+                    "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view",
+                    return_value=True,
+                ),
+                patch(
+                    "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts",
+                    AsyncMock(return_value=[mock_user_auth]),
+                ),
+            ):
+                from litellm.proxy.management_endpoints.mcp_management_endpoints import (
+                    fetch_all_mcp_servers,
+                )
+
+                result = await fetch_all_mcp_servers(user_api_key_dict=mock_user_auth)
+                assert [s.server_id for s in result] == ["s-alpha", "s-mid", "s-zeta"]
+
     @pytest.mark.asyncio
     async def test_restricted_virtual_key_cannot_use_team_id_filter(self):
         """Restricted virtual keys must not bypass access limits via team_id."""
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx
index 8abb8855e3d..390c42f1ea3 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx
@@ -3,7 +3,8 @@ import { render, waitFor, screen, act, within } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import { describe, it, expect, vi, beforeEach } from "vitest";
 import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
-import MCPServers from "./mcp_servers";
+import MCPServers, { compareServers } from "./mcp_servers";
+import type { MCPServer } from "@/components/mcp_tools/types";
 import * as networking from "@/components/networking";
 
 // Mock the networking module
@@ -29,6 +30,32 @@ const createQueryClient = () =>
     },
   });
 
+describe("compareServers", () => {
+  const server = (server_id: string, name: string, created_at = ""): MCPServer =>
+    ({ server_id, server_name: name, created_at, updated_at: created_at }) as MCPServer;
+
+  const shuffled = [server("c", "github"), server("a", "slack"), server("b", "Jira")];
+
+  it("orders servers without timestamps by name so config.yaml servers render in a stable order", () => {
+    const byCreated = [...shuffled].sort((a, b) => compareServers(a, b, "created_desc")).map((s) => s.server_id);
+    const byUpdated = [...shuffled].sort((a, b) => compareServers(a, b, "updated_desc")).map((s) => s.server_id);
+    const byHealth = [...shuffled].sort((a, b) => compareServers(a, b, "health")).map((s) => s.server_id);
+
+    expect(byCreated).toEqual(["c", "b", "a"]);
+    expect(byUpdated).toEqual(["c", "b", "a"]);
+    expect(byHealth).toEqual(["c", "b", "a"]);
+  });
+
+  it("keeps newest-first when timestamps differ", () => {
+    const newest = server("new", "zzz", "2026-02-01T00:00:00Z");
+    const oldest = server("old", "aaa", "2026-01-01T00:00:00Z");
+    expect([oldest, newest].sort((a, b) => compareServers(a, b, "created_desc")).map((s) => s.server_id)).toEqual([
+      "new",
+      "old",
+    ]);
+  });
+});
+
 describe("MCPServers", () => {
   const defaultProps = {
     accessToken: "123",
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx
index e6148d5d997..902e8811996 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx
@@ -45,7 +45,7 @@ import { TOOLS_OAUTH_UI_STATE_KEY } from "@/hooks/mcpOAuthUtils";
 import UserEnvVarsModal from "./UserEnvVarsModal";
 import { listMCPUserEnvVarStatus } from "@/components/networking";
 
-type SortKey = "created_desc" | "updated_desc" | "name_asc" | "health";
+export type SortKey = "created_desc" | "updated_desc" | "name_asc" | "health";
 
 const SORT_OPTIONS: { value: SortKey; label: string }[] = [
   { value: "created_desc", label: "Recently created" },
@@ -60,32 +60,33 @@ const HEALTH_RANK: Record<string, number> = {
   healthy: 2,
 };
 
-const compareServers = (a: MCPServer, b: MCPServer, sort: SortKey): number => {
+const compareByName = (a: MCPServer, b: MCPServer): number => {
+  const nameA = (a.server_name || a.alias || a.server_id).toLowerCase();
+  const nameB = (b.server_name || b.alias || b.server_id).toLowerCase();
+  return nameA.localeCompare(nameB) || a.server_id.localeCompare(b.server_id);
+};
+
+const compareByTimestampDesc = (a: string | null | undefined, b: string | null | undefined): number => {
+  const ta = a ? new Date(a).getTime() : 0;
+  const tb = b ? new Date(b).getTime() : 0;
+  return tb - ta;
+};
+
+export const compareServers = (a: MCPServer, b: MCPServer, sort: SortKey): number => {
   switch (sort) {
-    case "name_asc": {
-      const nameA = (a.server_name || a.alias || a.server_id).toLowerCase();
-      const nameB = (b.server_name || b.alias || b.server_id).toLowerCase();
-      return nameA.localeCompare(nameB);
-    }
-    case "updated_desc": {
-      const ta = a.updated_at ? new Date(a.updated_at).getTime() : 0;
-      const tb = b.updated_at ? new Date(b.updated_at).getTime() : 0;
-      return tb - ta;
-    }
+    case "name_asc":
+      return compareByName(a, b);
+    case "updated_desc":
+      return compareByTimestampDesc(a.updated_at, b.updated_at) || compareByName(a, b);
     case "health": {
       const ra = HEALTH_RANK[a.status ?? "unknown"] ?? 1;
       const rb = HEALTH_RANK[b.status ?? "unknown"] ?? 1;
       if (ra !== rb) return ra - rb;
-      const ta = a.created_at ? new Date(a.created_at).getTime() : 0;
-      const tb = b.created_at ? new Date(b.created_at).getTime() : 0;
-      return tb - ta;
+      return compareByTimestampDesc(a.created_at, b.created_at) || compareByName(a, b);
     }
     case "created_desc":
-    default: {
-      const ta = a.created_at ? new Date(a.created_at).getTime() : 0;
-      const tb = b.created_at ? new Date(b.created_at).getTime() : 0;
-      return tb - ta;
-    }
+    default:
+      return compareByTimestampDesc(a.created_at, b.created_at) || compareByName(a, b);
   }
 };
 
@@ -297,13 +298,7 @@ const MCPServers: React.FC<MCPServerProps> = ({ accessToken, userRole, userID })
           server.mcp_access_groups?.some((g: any) => (typeof g === "string" ? g === group : g && g.name === group)),
         );
       }
-      const sorted = [...filtered].sort((a, b) => {
-        if (!a.created_at && !b.created_at) return 0;
-        if (!a.created_at) return 1;
-        if (!b.created_at) return -1;
-        return new Date(b.created_at).getTime() - new Date(a.created_at).getTime();
-      });
-      setFilteredServers(sorted);
+      setFilteredServers(filtered);
     },
     [serversWithHealth],
   );

From aec592b734a91c3d068f89f0977561b733bc926f Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Mon, 14 Sep 2026 12:43:50 +0000
Subject: [PATCH 002/160] test(mcp): drop internal patches from ordering tests
 and regenerate API types

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../test_mcp_management_endpoints.py          | 47 ++++++-------------
 ui/litellm-dashboard/src/lib/http/schema.d.ts |  2 -
 2 files changed, 14 insertions(+), 35 deletions(-)

diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py
index 96bb0e35f0f..6381bb7b9fc 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py
@@ -1535,38 +1535,19 @@ class TestTeamScopedMCPServerAccess:
             result = await fetch_all_mcp_servers(user_api_key_dict=mock_user_auth, team_id="any-team-id")
             assert len(result) == 1
 
-    @pytest.mark.asyncio
-    async def test_team_scoped_list_is_sorted_by_display_name(self):
-        """Set-derived resolution order must not leak to the client."""
-        mock_user_auth = generate_mock_user_api_key_auth(
-            user_role=LitellmUserRoles.PROXY_ADMIN,
-            user_id="admin_user",
-        )
-        unsorted = [
-            generate_mock_mcp_server_db_record(server_id="s-zeta", alias="zeta"),
-            generate_mock_mcp_server_db_record(server_id="s-alpha", alias="Alpha"),
-            generate_mock_mcp_server_db_record(server_id="s-mid", alias="mid"),
-        ]
-
-        with (
-            patch(
-                "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view",
-                return_value=True,
-            ),
-            patch(
-                "litellm.proxy.management_endpoints.mcp_management_endpoints._get_team_scoped_mcp_server_list",
-                AsyncMock(return_value=unsorted),
-            ),
-        ):
-            from litellm.proxy.management_endpoints.mcp_management_endpoints import (
-                fetch_all_mcp_servers,
-            )
-
-            result = await fetch_all_mcp_servers(user_api_key_dict=mock_user_auth, team_id="any-team-id")
-            assert [s.server_id for s in result] == ["s-alpha", "s-mid", "s-zeta"]
-
 
 class TestFetchAllMCPServersOrdering:
+    def test_display_order_is_case_insensitive_name_then_id(self):
+        servers = [
+            generate_mock_mcp_server_db_record(server_id="s-2", alias="github"),
+            generate_mock_mcp_server_db_record(server_id="s-1", alias="github"),
+            generate_mock_mcp_server_db_record(server_id="s-0", alias="Slack"),
+            generate_mock_mcp_server_db_record(server_id="s-3", alias="confluence"),
+        ]
+
+        ordered = sorted(servers, key=mgmt_endpoints._mcp_server_display_order)
+        assert [s.server_id for s in ordered] == ["s-3", "s-1", "s-2", "s-0"]
+
     @pytest.mark.asyncio
     async def test_list_is_sorted_by_display_name_regardless_of_resolution_order(self):
         """The registry resolves ids through a set, so the response must impose its own order."""
@@ -1585,15 +1566,15 @@ class TestFetchAllMCPServersOrdering:
             mock_manager = MagicMock()
             mock_manager.get_all_allowed_mcp_servers = AsyncMock(return_value=resolved)
             with (
-                patch(
+                patch(  # test-quality-ok: the route reads a module-global manager with no injection seam
                     "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager",
                     mock_manager,
                 ),
-                patch(
+                patch(  # test-quality-ok: admin view is derived from module-global proxy settings
                     "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view",
                     return_value=True,
                 ),
-                patch(
+                patch(  # test-quality-ok: auth contexts need a live prisma client
                     "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts",
                     AsyncMock(return_value=[mock_user_auth]),
                 ),
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 7eadaa6c991..839aa52fa84 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -16781,7 +16781,6 @@ export interface paths {
          *     - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
          *     - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
          *     - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
-         *     - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests.
          *     - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
          *     - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
          *     - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
@@ -16887,7 +16886,6 @@ export interface paths {
          *         - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
          *         - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
          *         - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
-         *         - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests.
          *         - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
          *         - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
          *         - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)

From 067514bc76794fc3f9a66b9083f1cf52e6728427 Mon Sep 17 00:00:00 2001
From: Tejas Chopra <tejas@Tejass-MacBook-Pro.local>
Date: Wed, 16 Sep 2026 21:33:26 -0700
Subject: [PATCH 003/160] fix(responses): patch custom_tool_call_output in
 place on guardrail write-back

---
 .../guardrail_translation/handler.py          |  2 +-
 ...test_openai_responses_guardrail_handler.py | 50 +++++++++++++++++++
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py
index 1ab4811b8a0..9739ca95089 100644
--- a/litellm/llms/openai/responses/guardrail_translation/handler.py
+++ b/litellm/llms/openai/responses/guardrail_translation/handler.py
@@ -209,7 +209,7 @@ _TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | f
 )
 _OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"})
 _PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
-    {"function_call_output": "output", "message": "content"}
+    {"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"}
 )
 
 _EMPTY_RESPONSES_REQUEST: Final[ResponsesAPIOptionalRequestParams] = {}
diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
index c714b5d378a..ea58a48a3d5 100644
--- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
+++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
@@ -2223,6 +2223,56 @@ class TestStructuredMessagesWriteBack:
         }
         assert result["input"][3] == {"role": "user", "content": "What is the codename?"}
 
+    @pytest.mark.asyncio
+    async def test_codex_custom_tool_items_survive_tool_output_compression(self):
+        handler = OpenAIResponsesHandler()
+        additional_tools_item = {
+            "type": "additional_tools",
+            "tools": [{"type": "custom", "name": "exec", "description": "Run a JavaScript snippet"}],
+        }
+        reasoning_item = {
+            "id": "rs_456",
+            "type": "reasoning",
+            "summary": [],
+            "encrypted_content": "gAAAAA-signed-reasoning",
+        }
+        custom_tool_call_item = {
+            "id": "ctc_456",
+            "type": "custom_tool_call",
+            "call_id": "call_exec",
+            "name": "exec",
+            "input": 'const r = await tools.exec_command({"cmd": "cat memo.txt"});\ntext(r.output);',
+            "status": "completed",
+        }
+        data = {
+            "model": "gpt-5.6",
+            "input": [
+                additional_tools_item,
+                {"role": "user", "content": "What is the codename?"},
+                reasoning_item,
+                custom_tool_call_item,
+                {
+                    "type": "custom_tool_call_output",
+                    "call_id": "call_exec",
+                    "output": [
+                        {"type": "input_text", "text": "Script completed\nOutput:\n"},
+                        {"type": "input_text", "text": "memo " * 400},
+                    ],
+                },
+            ],
+        }
+
+        result = await handler.process_input_messages(data, ToolOutputRewriteGuardrail())
+
+        assert result["input"][0] is additional_tools_item
+        assert result["input"][1] == {"role": "user", "content": "What is the codename?"}
+        assert result["input"][2] is reasoning_item
+        assert result["input"][3] is custom_tool_call_item
+        assert result["input"][4]["type"] == "custom_tool_call_output"
+        assert result["input"][4]["call_id"] == "call_exec"
+        assert COMPRESSED_MARKER in str(result["input"][4]["output"])
+        assert len(result["input"]) == 5
+
     @pytest.mark.asyncio
     async def test_web_search_call_item_preserved_verbatim(self):
         handler = OpenAIResponsesHandler()

From b84f8b6a772d859a6ad762e8429549b86b877ca0 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Thu, 17 Sep 2026 19:24:14 +0000
Subject: [PATCH 004/160] feat(agents): attach access groups to agents and
 enforce them for models, MCP servers and agent calls

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../migration.sql                             |   2 +
 .../litellm_proxy_extras/schema.prisma        |   1 +
 .../mcp_server/auth/user_api_key_auth_mcp.py  |  37 ++-
 litellm/proxy/_lazy_openapi_snapshot.json     |  42 +++
 litellm/proxy/_types.py                       |   9 +-
 .../proxy/agent_endpoints/agent_registry.py   |  15 +-
 .../auth/agent_access_groups.py               |  80 ++++++
 .../auth/agent_permission_handler.py          |  33 ++-
 litellm/proxy/auth/auth_checks.py             |  37 ++-
 .../access_group_endpoints.py                 |  72 +++++
 litellm/proxy/schema.prisma                   |   1 +
 litellm/types/agents.py                       |   3 +
 schema.prisma                                 |   1 +
 .../auth/test_user_api_key_auth_mcp.py        |  59 ++++
 .../auth/test_agent_access_groups.py          | 130 +++++++++
 .../auth/test_agent_permission_handler.py     |  83 ++++++
 .../agent_endpoints/test_agent_registry.py    | 133 +++++++++
 .../proxy/auth/test_auth_checks.py            |  91 ++++++
 .../test_access_group_endpoints.py            | 271 ++++++++----------
 .../agents/_components/AgentFormKit.tsx       |   2 +
 .../add_agent_form.integration.test.tsx       |   3 +
 .../_components/add_agent_form.test.tsx       |  30 ++
 .../agents/_components/add_agent_form.tsx     |  21 ++
 .../agents/_components/agent_config.ts        |   5 +
 .../agent_info.integration.test.tsx           |   7 +
 .../agents/_components/agent_info.test.tsx    |  70 +++++
 .../agents/_components/agent_info.tsx         |  53 +++-
 .../src/components/agents/types.ts            |   1 +
 .../src/components/networking.tsx             |   1 +
 ui/litellm-dashboard/src/lib/http/schema.d.ts |   6 +
 30 files changed, 1138 insertions(+), 161 deletions(-)
 create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260917000000_add_agent_access_group_ids/migration.sql
 create mode 100644 litellm/proxy/agent_endpoints/auth/agent_access_groups.py
 create mode 100644 tests/test_litellm/proxy/agent_endpoints/auth/test_agent_access_groups.py

diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260917000000_add_agent_access_group_ids/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260917000000_add_agent_access_group_ids/migration.sql
new file mode 100644
index 00000000000..d594b0056df
--- /dev/null
+++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260917000000_add_agent_access_group_ids/migration.sql
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN IF NOT EXISTS "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[];
diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
index 139fb031671..e0b52dd77de 100644
--- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
@@ -71,6 +71,7 @@ model LiteLLM_AgentsTable {
   static_headers       Json?          @default("{}")
   extra_headers        String[]       @default([])
   agent_access_groups String[]       @default([])
+  access_group_ids     String[]       @default([])
   object_permission_id String?
   object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
   spend      Float    @default(0.0)
diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
index b0d57cb6228..bbb3d30864f 100644
--- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
+++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
@@ -1549,10 +1549,18 @@ class MCPRequestHandler:
                 allowed_mcp_servers_for_agent: Final = await MCPRequestHandler._get_allowed_mcp_servers_for_agent(
                     user_api_key_auth
                 )
-                if len(allowed_mcp_servers_for_agent) > 0:
+                agent_access_group_servers: Final = await MCPRequestHandler._get_agent_access_group_server_ceiling(
+                    user_api_key_auth
+                )
+                if len(allowed_mcp_servers_for_agent) > 0 or agent_access_group_servers is not None:
                     has_lower_level_mcp_restrictions = True
-                    # Intersect: agent can only use servers allowed by BOTH key/team AND agent config
-                    allowed_mcp_servers = [s for s in allowed_mcp_servers if s in allowed_mcp_servers_for_agent]
+                    # Intersect: agent can only use servers allowed by key/team AND agent config AND agent access groups
+                    allowed_mcp_servers = [
+                        s
+                        for s in allowed_mcp_servers
+                        if (len(allowed_mcp_servers_for_agent) == 0 or s in allowed_mcp_servers_for_agent)
+                        and (agent_access_group_servers is None or s in agent_access_group_servers)
+                    ]
                     verbose_logger.debug(
                         "Applied agent intersection filter. Final allowed servers: %s", allowed_mcp_servers
                     )
@@ -3137,6 +3145,29 @@ class MCPRequestHandler:
             verbose_logger.warning("Failed to get allowed MCP servers for agent: %s", e)
             return []
 
+    @staticmethod
+    async def _get_agent_access_group_server_ceiling(
+        user_api_key_auth: UserAPIKeyAuth,
+    ) -> frozenset[str] | None:
+        """
+        Server IDs the agent's attached unified access groups (``LiteLLM_AgentsTable.access_group_ids``)
+        allow, or None when the agent has none attached. Unlike the object_permission path above, an
+        attached group set that names no servers is an empty ceiling and denies every server.
+        """
+        from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
+            global_mcp_server_manager,
+        )
+        from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
+            resolve_agent_access_group_ceiling,
+        )
+
+        if not user_api_key_auth.agent_id:
+            return None
+        ceiling: Final = await resolve_agent_access_group_ceiling(user_api_key_auth.agent_id)
+        if ceiling is None:
+            return None
+        return frozenset(global_mcp_server_manager.expand_permission_list(sorted(ceiling.mcp_server_ids)))
+
     @staticmethod
     async def _get_agent_tool_permissions_for_server(
         server_id: str,
diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json
index 74f38b3ca6d..cf547dc89d5 100644
--- a/litellm/proxy/_lazy_openapi_snapshot.json
+++ b/litellm/proxy/_lazy_openapi_snapshot.json
@@ -2357,6 +2357,20 @@
         },
         "AgentConfig": {
           "properties": {
+            "access_group_ids": {
+              "anyOf": [
+                {
+                  "items": {
+                    "type": "string"
+                  },
+                  "type": "array"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Access Group Ids"
+            },
             "agent_card_params": {
               "$ref": "#/components/schemas/AgentCard"
             },
@@ -2683,6 +2697,20 @@
         },
         "AgentResponse": {
           "properties": {
+            "access_group_ids": {
+              "anyOf": [
+                {
+                  "items": {
+                    "type": "string"
+                  },
+                  "type": "array"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Access Group Ids"
+            },
             "agent_card_params": {
               "additionalProperties": true,
               "title": "Agent Card Params",
@@ -3471,6 +3499,20 @@
         },
         "PatchAgentRequest": {
           "properties": {
+            "access_group_ids": {
+              "anyOf": [
+                {
+                  "items": {
+                    "type": "string"
+                  },
+                  "type": "array"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Access Group Ids"
+            },
             "agent_card_params": {
               "$ref": "#/components/schemas/AgentCard"
             },
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 680c63393e8..3a7125acd1f 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -4127,6 +4127,11 @@ class ProxyErrorTypes(str, enum.Enum):
     Project does not have access to the model
     """
 
+    agent_model_access_denied = "agent_model_access_denied"
+    """
+    The agent behind the key does not have access to the model
+    """
+
     model_cost_map_missing = "model_cost_map_missing"
 
     expired_key = "expired_key"
@@ -4201,7 +4206,7 @@ class ProxyErrorTypes(str, enum.Enum):
 
     @classmethod
     def get_model_access_error_type_for_object(
-        cls, object_type: Literal["key", "user", "team", "org", "project"]
+        cls, object_type: Literal["key", "user", "team", "org", "project", "agent"]
     ) -> "ProxyErrorTypes":
         """
         Get the model access error type for object_type
@@ -4216,6 +4221,8 @@ class ProxyErrorTypes(str, enum.Enum):
             return cls.org_model_access_denied
         elif object_type == "project":
             return cls.project_model_access_denied
+        elif object_type == "agent":
+            return cls.agent_model_access_denied
 
     @classmethod
     def get_vector_store_access_error_type_for_object(
diff --git a/litellm/proxy/agent_endpoints/agent_registry.py b/litellm/proxy/agent_endpoints/agent_registry.py
index c7b6bca72cf..c8948d8d70e 100644
--- a/litellm/proxy/agent_endpoints/agent_registry.py
+++ b/litellm/proxy/agent_endpoints/agent_registry.py
@@ -7,6 +7,7 @@ from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, NamedTuple, Protocol, TypedDict
 
 from pydantic import TypeAdapter, ValidationError
+from typing_extensions import ReadOnly
 
 import litellm
 from litellm.constants import REDACTED_BY_LITELM_STRING
@@ -37,6 +38,7 @@ class AgentRecordDump(TypedDict):
     agent_card_params: dict[str, object]
     static_headers: dict[str, str] | None
     extra_headers: list[str] | None
+    access_group_ids: ReadOnly[Sequence[str] | None]
     object_permission: dict[str, object] | None
     spend: float
     tpm_limit: int | None
@@ -284,6 +286,12 @@ def _resolved_agent_param_value(
     return _MISSING_AGENT_PARAM
 
 
+def _patched_access_group_ids(agent: PatchAgentRequest) -> Mapping[str, object]:
+    if "access_group_ids" not in agent:
+        return MappingProxyType({})
+    return MappingProxyType({"access_group_ids": tuple(dict.fromkeys(agent.get("access_group_ids") or ()))})
+
+
 def _restore_redacted_litellm_params(
     incoming: Mapping[str, object],
     existing: Mapping[str, object],
@@ -516,6 +524,7 @@ class AgentRegistry:
             static_headers_val: Final[str | None] = safe_dumps(dict(static_headers_obj)) if static_headers_obj else None
 
             extra_headers_val: Final = agent.get("extra_headers")
+            access_group_ids_val: Final = agent.get("access_group_ids")
 
             create_data: Final[dict[str, object]] = {
                 "agent_name": agent_name,
@@ -532,6 +541,8 @@ class AgentRegistry:
                 create_data["static_headers"] = static_headers_val
             if extra_headers_val is not None:
                 create_data["extra_headers"] = extra_headers_val
+            if access_group_ids_val is not None:
+                create_data["access_group_ids"] = tuple(dict.fromkeys(access_group_ids_val))
             if object_permission_id is not None:
                 create_data["object_permission_id"] = object_permission_id
 
@@ -601,7 +612,7 @@ class AgentRegistry:
             existing_agent: Final[Mapping[str, object]] = dict(existing_record)
 
             augment_agent: Final = {**existing_agent, **agent}
-            update_data: Final[dict[str, object]] = {}
+            update_data: Final[dict[str, object]] = {**_patched_access_group_ids(agent)}
             if augment_agent.get("agent_name"):
                 update_data["agent_name"] = augment_agent.get("agent_name")
             if "litellm_params" in agent:
@@ -703,6 +714,7 @@ class AgentRegistry:
                 safe_dumps(dict(static_headers_obj_u)) if static_headers_obj_u is not None else safe_dumps({})
             )
             extra_headers_val_u: Final = agent.get("extra_headers") or []
+            access_group_ids_val_u: Final = tuple(dict.fromkeys(agent.get("access_group_ids") or ()))
 
             update_data: Final[dict[str, object]] = {
                 "agent_name": agent_name,
@@ -710,6 +722,7 @@ class AgentRegistry:
                 "agent_card_params": agent_card_params,
                 "static_headers": static_headers_val_u,
                 "extra_headers": extra_headers_val_u,
+                "access_group_ids": access_group_ids_val_u,
                 "updated_by": updated_by,
                 "updated_at": datetime.now(timezone.utc),
             }
diff --git a/litellm/proxy/agent_endpoints/auth/agent_access_groups.py b/litellm/proxy/agent_endpoints/auth/agent_access_groups.py
new file mode 100644
index 00000000000..4a579de679e
--- /dev/null
+++ b/litellm/proxy/agent_endpoints/auth/agent_access_groups.py
@@ -0,0 +1,80 @@
+"""
+Ceiling that an agent's attached access groups place on requests made with that agent's key.
+
+Keys and teams use access groups as grants. An agent uses them the way it already uses its
+``object_permission``: the union of the attached groups caps what the agent's key can reach,
+on top of whatever the key and team allow. A group that cannot be loaded contributes nothing,
+so a missing or unreadable group can only narrow the agent, never widen it.
+"""
+
+import asyncio
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from typing import Final, TypeAlias
+
+from fastapi import HTTPException
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy._types import LiteLLM_AccessGroupTable
+from litellm.types.agents import AgentResponse
+
+AgentLoader: TypeAlias = Callable[[str], Awaitable[AgentResponse | None]]
+AccessGroupLoader: TypeAlias = Callable[[str], Awaitable[LiteLLM_AccessGroupTable | None]]
+
+
+@dataclass(frozen=True, slots=True)
+class AgentAccessGroupCeiling:
+    """Everything the agent's attached access groups allow. An empty set denies that resource kind."""
+
+    access_group_ids: tuple[str, ...]
+    models: frozenset[str]
+    mcp_server_ids: frozenset[str]
+    agent_ids: frozenset[str]
+
+
+async def _load_agent(agent_id: str) -> AgentResponse | None:
+    from litellm.proxy.common_utils.registry_read_through import get_agent_with_read_through
+
+    return await get_agent_with_read_through(agent_id)
+
+
+async def _load_access_group(access_group_id: str) -> LiteLLM_AccessGroupTable | None:
+    from litellm.proxy.auth.auth_checks import get_access_object
+    from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
+
+    if prisma_client is None:
+        verbose_proxy_logger.warning("Agent access group %s cannot be loaded without a DB", access_group_id)
+        return None
+    try:
+        return await get_access_object(
+            access_group_id=access_group_id,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+    except HTTPException as e:
+        verbose_proxy_logger.warning(
+            "Agent access group %s could not be loaded, treating it as empty: %s", access_group_id, e.detail
+        )
+        return None
+
+
+async def resolve_agent_access_group_ceiling(
+    agent_id: str,
+    load_agent: AgentLoader = _load_agent,
+    load_access_group: AccessGroupLoader = _load_access_group,
+) -> AgentAccessGroupCeiling | None:
+    """``None`` when the agent has no access groups attached, so nothing is capped."""
+    agent: Final = await load_agent(agent_id)
+    access_group_ids: Final = tuple(agent.access_group_ids or ()) if agent is not None else ()
+    if not access_group_ids:
+        return None
+
+    loaded: Final = await asyncio.gather(*(load_access_group(group_id) for group_id in access_group_ids))
+    groups: Final = tuple(group for group in loaded if group is not None)
+    return AgentAccessGroupCeiling(
+        access_group_ids=access_group_ids,
+        models=frozenset(model for group in groups for model in group.access_model_names),
+        mcp_server_ids=frozenset(server_id for group in groups for server_id in group.access_mcp_server_ids),
+        agent_ids=frozenset(target_id for group in groups for target_id in group.access_agent_ids),
+    )
diff --git a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
index e4dd77e2f82..11d2a68072c 100644
--- a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
+++ b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
@@ -66,9 +66,24 @@ class AgentRequestHandler:
         Resolve the agents the given user/key may reach.
 
         ``UnrestrictedAgentAccess`` is only returned when neither the key nor its team
-        carries any grant. Grants that intersect to nothing stay restricted, so
-        narrowing a caller can never widen what it reaches.
+        carries any grant and the agent behind the key has no access groups attached.
+        Grants that intersect to nothing stay restricted, so narrowing a caller can
+        never widen what it reaches.
         """
+        key_team_access: Final = await AgentRequestHandler._resolve_key_team_agent_access(user_api_key_auth)
+        agent_ceiling: Final = await AgentRequestHandler._agent_access_group_ceiling(user_api_key_auth)
+        if agent_ceiling is None:
+            return key_team_access
+        match key_team_access:
+            case UnrestrictedAgentAccess():
+                return RestrictedAgentAccess(agent_ceiling)
+            case RestrictedAgentAccess(key_team_ids):
+                return RestrictedAgentAccess(key_team_ids & agent_ceiling)
+
+    @staticmethod
+    async def _resolve_key_team_agent_access(
+        user_api_key_auth: UserAPIKeyAuth | None,
+    ) -> AgentAccess:
         try:
             key_access: Final = await AgentRequestHandler._get_allowed_agents_for_key(user_api_key_auth)
             team_access: Final = await AgentRequestHandler._get_allowed_agents_for_team(user_api_key_auth)
@@ -86,6 +101,20 @@ class AgentRequestHandler:
             verbose_logger.warning("Failed to get allowed agents: %s", e)
             return UnrestrictedAgentAccess()
 
+    @staticmethod
+    async def _agent_access_group_ceiling(
+        user_api_key_auth: UserAPIKeyAuth | None,
+    ) -> frozenset[str] | None:
+        """Stable IDs of the agents the calling agent's attached access groups allow; None when none attached."""
+        from litellm.proxy.agent_endpoints.auth.agent_access_groups import resolve_agent_access_group_ceiling
+
+        if user_api_key_auth is None or not user_api_key_auth.agent_id:
+            return None
+        ceiling: Final = await resolve_agent_access_group_ceiling(user_api_key_auth.agent_id)
+        if ceiling is None:
+            return None
+        return _to_stable_ids(ceiling.agent_ids)
+
     @staticmethod
     async def is_agent_allowed(
         agent_id: str,
diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py
index 3dd2e2d8eb2..5c53ee49717 100644
--- a/litellm/proxy/auth/auth_checks.py
+++ b/litellm/proxy/auth/auth_checks.py
@@ -1003,6 +1003,9 @@ async def common_checks(
                         code=status.HTTP_400_BAD_REQUEST,
                     )
 
+    # 2.4 If the agent behind the key has access groups attached, they cap the models it can call
+    await _check_agent_access_group_model_access(model=_model, valid_token=valid_token, llm_router=llm_router)
+
     ## 2.1 If user can call model (if personal key)
     if _model and team_object is None and user_object is not None:
         with tracer.trace("litellm.proxy.auth.common_checks.can_user_call_model"):
@@ -4126,7 +4129,7 @@ def _can_object_call_model(
     models: list[str],
     team_model_aliases: dict[str, str] | None = None,
     team_id: str | None = None,
-    object_type: Literal["user", "team", "key", "org", "project"] = "user",
+    object_type: Literal["user", "team", "key", "org", "project", "agent"] = "user",
     fallback_depth: int = 0,
 ) -> Literal[True]:
     """
@@ -4192,6 +4195,38 @@ def _can_object_call_model(
     )
 
 
+async def _check_agent_access_group_model_access(
+    model: str | list[str] | None,  # mutable-ok: _can_object_call_model and the client message helper take list[str]
+    valid_token: UserAPIKeyAuth | None,
+    llm_router: Router | None,
+) -> Literal[True]:
+    """Raises when the key's agent has access groups attached and none of them names the model.
+    Attached groups that name no model deny every model; ``_can_object_call_model`` would read
+    an empty allowlist as unrestricted."""
+    from litellm.proxy.agent_endpoints.auth.agent_access_groups import resolve_agent_access_group_ceiling
+
+    if not model or valid_token is None or not valid_token.agent_id:
+        return True
+    ceiling: Final = await resolve_agent_access_group_ceiling(valid_token.agent_id)
+    if ceiling is None:
+        return True
+    if not ceiling.models:
+        raise ModelAccessDeniedProxyException(
+            message=model_access_denied_client_message(model=model),
+            internal_message=f"agent {valid_token.agent_id} access groups {ceiling.access_group_ids} grant no models",
+            type=ProxyErrorTypes.agent_model_access_denied,
+            param="model",
+            code=status.HTTP_403_FORBIDDEN,
+        )
+    return _can_object_call_model(
+        model=model,
+        llm_router=llm_router,
+        models=sorted(ceiling.models),
+        team_id=valid_token.team_id,
+        object_type="agent",
+    )
+
+
 def _model_in_team_aliases(model: str, team_model_aliases: dict[str, str] | None = None) -> bool:
     """
     Returns True if `model` being accessed is an alias of a team model
diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py
index a6cc5140b15..b4923b0a2dc 100644
--- a/litellm/proxy/management_endpoints/access_group_endpoints.py
+++ b/litellm/proxy/management_endpoints/access_group_endpoints.py
@@ -5,6 +5,7 @@ from types import MappingProxyType
 from typing import Final, Protocol
 
 from fastapi import APIRouter, Depends, HTTPException, status
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager
@@ -109,6 +110,36 @@ class _KeyTable(Protocol):
     async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> object: ...
 
 
+class _AgentRecord(Protocol):
+    @property
+    def agent_id(self) -> str: ...
+
+    @property
+    def access_group_ids(self) -> Sequence[str] | None: ...
+
+
+class _AgentTable(Protocol):
+    async def find_many(self, where: Mapping[str, object]) -> Sequence[_AgentRecord]: ...
+
+    async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> object: ...
+
+
+class _HasSomeFilter(TypedDict):
+    hasSome: ReadOnly[Sequence[str]]
+
+
+class _AgentAccessGroupsWhere(TypedDict):
+    access_group_ids: ReadOnly[_HasSomeFilter]
+
+
+class _AgentIdWhere(TypedDict):
+    agent_id: ReadOnly[str]
+
+
+class _AgentAccessGroupsData(TypedDict):
+    access_group_ids: ReadOnly[Sequence[str]]
+
+
 class _AccessGroupTx(Protocol):
     @property
     def litellm_accessgrouptable(self) -> _AccessGroupTable: ...
@@ -119,6 +150,9 @@ class _AccessGroupTx(Protocol):
     @property
     def litellm_verificationtoken(self) -> _KeyTable: ...
 
+    @property
+    def litellm_agentstable(self) -> _AgentTable: ...
+
 
 def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None:
     if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
@@ -324,6 +358,41 @@ async def _sync_remove_access_group_from_keys(tx: _AccessGroupTx, key_tokens: li
             )
 
 
+def _without_access_group(access_group_ids: Sequence[str] | None, access_group_id: str) -> tuple[str, ...]:
+    return tuple(ag for ag in (access_group_ids or ()) if ag != access_group_id)
+
+
+async def _detach_access_group_from_agents(tx: _AccessGroupTx, access_group_id: str) -> tuple[str, ...]:
+    agents_with_group: Final = await tx.litellm_agentstable.find_many(
+        where=_AgentAccessGroupsWhere(access_group_ids=_HasSomeFilter(hasSome=(access_group_id,)))
+    )
+    for agent in agents_with_group:
+        await tx.litellm_agentstable.update(
+            where=_AgentIdWhere(agent_id=agent.agent_id),
+            data=_AgentAccessGroupsData(
+                access_group_ids=_without_access_group(agent.access_group_ids, access_group_id)
+            ),
+        )
+    return tuple(agent.agent_id for agent in agents_with_group)
+
+
+def _detach_access_group_from_agent_registry(agent_ids: Sequence[str], access_group_id: str) -> None:
+    registered: Final = tuple(
+        agent
+        for agent in (global_agent_registry.get_agent_by_id(agent_id) for agent_id in agent_ids)
+        if agent is not None
+    )
+    for agent in registered:
+        global_agent_registry.deregister_agent(agent_name=agent.agent_name)
+        global_agent_registry.register_agent(
+            agent_config=agent.model_copy(
+                update=_AgentAccessGroupsData(
+                    access_group_ids=_without_access_group(agent.access_group_ids, access_group_id)
+                )
+            )
+        )
+
+
 # ---------------------------------------------------------------------------
 # Cache patch helpers
 # ---------------------------------------------------------------------------
@@ -705,11 +774,14 @@ async def delete_access_group(
             out_of_sync_key_tokens: Final = set(existing.assigned_key_ids or []) - {k.token for k in keys_with_group}
             await _sync_remove_access_group_from_keys(tx, list(out_of_sync_key_tokens), access_group_id)
 
+            detached_agent_ids: Final = await _detach_access_group_from_agents(tx, access_group_id)
+
             await tx.litellm_accessgrouptable.delete(where={"access_group_id": access_group_id})
 
         from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache
 
         await invalidate_access_group_cache(access_group_id)
+        _detach_access_group_from_agent_registry(detached_agent_ids, access_group_id)
         await _patch_team_caches_remove_access_group(
             affected_team_ids, access_group_id, user_api_key_cache, proxy_logging_obj
         )
diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma
index 139fb031671..e0b52dd77de 100644
--- a/litellm/proxy/schema.prisma
+++ b/litellm/proxy/schema.prisma
@@ -71,6 +71,7 @@ model LiteLLM_AgentsTable {
   static_headers       Json?          @default("{}")
   extra_headers        String[]       @default([])
   agent_access_groups String[]       @default([])
+  access_group_ids     String[]       @default([])
   object_permission_id String?
   object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
   spend      Float    @default(0.0)
diff --git a/litellm/types/agents.py b/litellm/types/agents.py
index dbaaab62d86..12e60352a97 100644
--- a/litellm/types/agents.py
+++ b/litellm/types/agents.py
@@ -189,6 +189,7 @@ class AgentConfig(TypedDict, total=False):
     session_rpm_limit: int | None
     static_headers: dict[str, str] | None
     extra_headers: list[str] | None
+    access_group_ids: ReadOnly[Sequence[str] | None]
 
 
 class PatchAgentRequest(TypedDict, total=False):
@@ -202,6 +203,7 @@ class PatchAgentRequest(TypedDict, total=False):
     session_rpm_limit: int | None
     static_headers: dict[str, str] | None
     extra_headers: list[str] | None
+    access_group_ids: ReadOnly[Sequence[str] | None]
 
 
 # Request/Response models for CRUD endpoints
@@ -226,6 +228,7 @@ class AgentResponse(BaseModel):
     session_rpm_limit: int | None = None
     static_headers: dict[str, str] | None = None
     extra_headers: list[str] | None = None
+    access_group_ids: Sequence[str] | None = None
     keys: list[AgentKeySummary] | None = None
     search_score: float | None = None
     created_at: datetime | None = None
diff --git a/schema.prisma b/schema.prisma
index 139fb031671..e0b52dd77de 100644
--- a/schema.prisma
+++ b/schema.prisma
@@ -71,6 +71,7 @@ model LiteLLM_AgentsTable {
   static_headers       Json?          @default("{}")
   extra_headers        String[]       @default([])
   agent_access_groups String[]       @default([])
+  access_group_ids     String[]       @default([])
   object_permission_id String?
   object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
   spend      Float    @default(0.0)
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py
index 90ce821d62e..2f8e3d1cb82 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py
@@ -4208,6 +4208,65 @@ class TestAgentMCPPermissions:
                     assert sorted(result) == ["server_1", "server_2"]
                     mock_agent.assert_called_once_with(user_api_key_auth)
 
+    @pytest.mark.parametrize(
+        ("group_ceiling", "expected"),
+        [
+            (frozenset({"server_1"}), ["server_1"]),
+            (frozenset({"server_1", "server_2", "server_3"}), ["server_1", "server_2"]),
+            (frozenset(), []),
+        ],
+    )
+    async def test_get_allowed_mcp_servers_agent_access_group_ceiling(self, group_ceiling, expected):
+        """The agent's attached access groups cap the key/team servers; groups naming no server deny all."""
+        user_api_key_auth = UserAPIKeyAuth(api_key="test-key", user_id="test-user", agent_id="agent-ag")
+        with (
+            patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_key", return_value=["server_1", "server_2"]),
+            patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_team", return_value=[]),
+            patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_agent", return_value=[]),
+            patch.object(MCPRequestHandler, "_get_agent_access_group_server_ceiling", return_value=group_ceiling),
+        ):
+            access = await MCPRequestHandler.get_mcp_server_access(user_api_key_auth=user_api_key_auth)
+            assert sorted(access.server_ids) == expected
+            assert access.scope == "scoped"
+
+    async def test_get_allowed_mcp_servers_agent_without_access_groups_is_uncapped(self):
+        user_api_key_auth = UserAPIKeyAuth(api_key="test-key", user_id="test-user", agent_id="agent-ag")
+        with (
+            patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_key", return_value=["server_1", "server_2"]),
+            patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_team", return_value=[]),
+            patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_agent", return_value=[]),
+            patch.object(MCPRequestHandler, "_get_agent_access_group_server_ceiling", return_value=None),
+        ):
+            result = await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth=user_api_key_auth)
+            assert sorted(result) == ["server_1", "server_2"]
+
+    async def test_agent_access_group_server_ceiling_expands_group_servers(self):
+        from litellm.proxy.agent_endpoints.auth.agent_access_groups import AgentAccessGroupCeiling
+
+        ceiling = AgentAccessGroupCeiling(
+            access_group_ids=("ag-1",),
+            models=frozenset(),
+            mcp_server_ids=frozenset({"server_1"}),
+            agent_ids=frozenset(),
+        )
+        with (
+            patch(
+                "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
+                new=AsyncMock(return_value=ceiling),
+            ),
+            patch(
+                "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager"
+            ) as mock_manager,
+        ):
+            mock_manager.expand_permission_list.return_value = ["server_1"]
+            result = await MCPRequestHandler._get_agent_access_group_server_ceiling(
+                UserAPIKeyAuth(api_key="test-key", agent_id="agent-ag")
+            )
+            assert result == frozenset({"server_1"})
+            mock_manager.expand_permission_list.assert_called_once_with(["server_1"])
+
+        assert await MCPRequestHandler._get_agent_access_group_server_ceiling(UserAPIKeyAuth(api_key="k")) is None
+
     async def test_get_allowed_mcp_servers_key_team_agent_intersection(self):
         """Key allows [1, 2], agent allows [2, 3]. Result = [2]."""
         user_api_key_auth = UserAPIKeyAuth(
diff --git a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_access_groups.py b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_access_groups.py
new file mode 100644
index 00000000000..8c31c9428e7
--- /dev/null
+++ b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_access_groups.py
@@ -0,0 +1,130 @@
+from typing import Final
+
+import pytest
+from fastapi import HTTPException
+
+from litellm.models.access_group import LiteLLM_AccessGroupTable
+from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
+    AgentAccessGroupCeiling,
+    resolve_agent_access_group_ceiling,
+)
+from litellm.types.agents import AgentResponse
+
+_CARD: Final = {"name": "agent", "url": "http://localhost:9999", "version": "1.0.0"}
+
+
+def _agent(access_group_ids: list[str] | None) -> AgentResponse:
+    return AgentResponse(
+        agent_id="agent-1", agent_name="agent", agent_card_params=_CARD, access_group_ids=access_group_ids
+    )
+
+
+def _group(
+    group_id: str,
+    models: tuple[str, ...] = (),
+    mcp_servers: tuple[str, ...] = (),
+    agents: tuple[str, ...] = (),
+) -> LiteLLM_AccessGroupTable:
+    return LiteLLM_AccessGroupTable(
+        access_group_id=group_id,
+        access_group_name=group_id,
+        access_model_names=list(models),
+        access_mcp_server_ids=list(mcp_servers),
+        access_agent_ids=list(agents),
+    )
+
+
+def _loaders(agent: AgentResponse | None, groups: dict[str, LiteLLM_AccessGroupTable]):
+    async def load_agent(agent_id: str) -> AgentResponse | None:
+        return agent
+
+    async def load_group(group_id: str) -> LiteLLM_AccessGroupTable | None:
+        return groups.get(group_id)
+
+    return load_agent, load_group
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("access_group_ids", [None, []])
+async def test_agent_without_access_groups_has_no_ceiling(access_group_ids: list[str] | None):
+    load_agent, load_group = _loaders(_agent(access_group_ids), {"g1": _group("g1", models=("gpt-5",))})
+
+    assert await resolve_agent_access_group_ceiling("agent-1", load_agent, load_group) is None
+
+
+@pytest.mark.asyncio
+async def test_unknown_agent_has_no_ceiling():
+    load_agent, load_group = _loaders(None, {})
+
+    assert await resolve_agent_access_group_ceiling("missing", load_agent, load_group) is None
+
+
+@pytest.mark.asyncio
+async def test_ceiling_is_the_union_of_every_attached_group():
+    load_agent, load_group = _loaders(
+        _agent(["g1", "g2"]),
+        {
+            "g1": _group("g1", models=("gpt-5",), mcp_servers=("mcp-a",), agents=("agent-b",)),
+            "g2": _group("g2", models=("claude-sonnet",), mcp_servers=("mcp-b",), agents=("agent-c",)),
+        },
+    )
+
+    ceiling: Final = await resolve_agent_access_group_ceiling("agent-1", load_agent, load_group)
+
+    assert ceiling == AgentAccessGroupCeiling(
+        access_group_ids=("g1", "g2"),
+        models=frozenset({"gpt-5", "claude-sonnet"}),
+        mcp_server_ids=frozenset({"mcp-a", "mcp-b"}),
+        agent_ids=frozenset({"agent-b", "agent-c"}),
+    )
+
+
+@pytest.mark.asyncio
+async def test_unloadable_group_contributes_nothing_but_the_ceiling_still_applies():
+    load_agent, load_group = _loaders(_agent(["g1", "gone"]), {"g1": _group("g1", models=("gpt-5",))})
+
+    ceiling: Final = await resolve_agent_access_group_ceiling("agent-1", load_agent, load_group)
+
+    assert ceiling == AgentAccessGroupCeiling(
+        access_group_ids=("g1", "gone"),
+        models=frozenset({"gpt-5"}),
+        mcp_server_ids=frozenset(),
+        agent_ids=frozenset(),
+    )
+
+
+@pytest.mark.asyncio
+async def test_only_unloadable_groups_is_an_empty_ceiling_not_unrestricted():
+    load_agent, load_group = _loaders(_agent(["gone"]), {})
+
+    ceiling: Final = await resolve_agent_access_group_ceiling("agent-1", load_agent, load_group)
+
+    assert ceiling is not None
+    assert ceiling.models == frozenset()
+    assert ceiling.mcp_server_ids == frozenset()
+    assert ceiling.agent_ids == frozenset()
+
+
+@pytest.mark.asyncio
+async def test_default_loader_treats_a_missing_group_as_unreadable(monkeypatch: pytest.MonkeyPatch):
+    from litellm.proxy import proxy_server
+    from litellm.proxy.agent_endpoints.auth.agent_access_groups import _load_access_group
+    from litellm.proxy.auth import auth_checks
+
+    async def missing_group(**_: object) -> LiteLLM_AccessGroupTable:
+        raise HTTPException(status_code=404, detail={"error": "Access group doesn't exist in db."})
+
+    monkeypatch.setattr(proxy_server, "prisma_client", object())
+    monkeypatch.setattr(auth_checks, "get_access_object", missing_group)
+
+    assert await _load_access_group("gone") is None
+
+
+@pytest.mark.asyncio
+async def test_default_loader_returns_nothing_without_a_db(monkeypatch: pytest.MonkeyPatch):
+    from litellm.proxy import proxy_server
+    from litellm.proxy.agent_endpoints.auth.agent_access_groups import _load_access_group
+
+    monkeypatch.setattr(proxy_server, "prisma_client", None)
+
+    assert await _load_access_group("ag-1") is None
diff --git a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py
index 383b72e5c58..a8a55d332b6 100644
--- a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py
+++ b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py
@@ -13,6 +13,7 @@ import pytest
 from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
 from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
 from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry
+from litellm.proxy.agent_endpoints.auth.agent_access_groups import AgentAccessGroupCeiling
 from litellm.proxy.agent_endpoints.auth.agent_permission_handler import (
     AgentAccess,
     AgentRequestHandler,
@@ -157,6 +158,88 @@ class TestAgentRequestHandler:
                         is False
                     ), agent_id
 
+    @staticmethod
+    def _ceiling(agent_ids: frozenset[str]) -> AgentAccessGroupCeiling:
+        return AgentAccessGroupCeiling(
+            access_group_ids=("ag-1",),
+            models=frozenset(),
+            mcp_server_ids=frozenset(),
+            agent_ids=agent_ids,
+        )
+
+    async def test_agent_access_groups_cap_an_otherwise_unrestricted_key(self):
+        """A key with no agent grant of its own may still only reach the agents its
+        agent's attached access groups name."""
+        agent_key: Final = UserAPIKeyAuth(api_key="test-key", user_id="test-user", agent_id="caller-agent")
+
+        with (
+            patch.object(AgentRequestHandler, "_get_allowed_agents_for_key", return_value=UnrestrictedAgentAccess()),
+            patch.object(AgentRequestHandler, "_get_allowed_agents_for_team", return_value=UnrestrictedAgentAccess()),
+            patch(
+                "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
+                new=AsyncMock(return_value=self._ceiling(frozenset({"agent-beta"}))),
+            ) as mock_ceiling,
+        ):
+            assert await AgentRequestHandler.resolve_agent_access(agent_key) == RestrictedAgentAccess(
+                frozenset({"agent-beta"})
+            )
+            assert await AgentRequestHandler.is_agent_allowed("agent-beta", agent_key) is True
+            assert await AgentRequestHandler.is_agent_allowed("agent-alpha", agent_key) is False
+            mock_ceiling.assert_called_with("caller-agent")
+
+    async def test_agent_access_groups_intersect_with_key_and_team_grants(self):
+        agent_key: Final = UserAPIKeyAuth(
+            api_key="test-key", user_id="test-user", team_id="test-team", agent_id="caller-agent"
+        )
+
+        with (
+            patch.object(
+                AgentRequestHandler,
+                "_get_allowed_agents_for_key",
+                return_value=RestrictedAgentAccess(frozenset({"agent-alpha", "agent-beta"})),
+            ),
+            patch.object(
+                AgentRequestHandler,
+                "_get_allowed_agents_for_team",
+                return_value=RestrictedAgentAccess(frozenset({"agent-alpha", "agent-beta", "agent-gamma"})),
+            ),
+            patch(
+                "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
+                new=AsyncMock(return_value=self._ceiling(frozenset({"agent-beta", "agent-gamma"}))),
+            ),
+        ):
+            assert await AgentRequestHandler.resolve_agent_access(agent_key) == RestrictedAgentAccess(
+                frozenset({"agent-beta"})
+            )
+
+    async def test_agent_access_groups_naming_no_agent_deny_every_agent(self):
+        agent_key: Final = UserAPIKeyAuth(api_key="test-key", user_id="test-user", agent_id="caller-agent")
+
+        with (
+            patch.object(AgentRequestHandler, "_get_allowed_agents_for_key", return_value=UnrestrictedAgentAccess()),
+            patch.object(AgentRequestHandler, "_get_allowed_agents_for_team", return_value=UnrestrictedAgentAccess()),
+            patch(
+                "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
+                new=AsyncMock(return_value=self._ceiling(frozenset())),
+            ),
+        ):
+            assert await AgentRequestHandler.resolve_agent_access(agent_key) == RestrictedAgentAccess(frozenset())
+            assert await AgentRequestHandler.is_agent_allowed("agent-alpha", agent_key) is False
+
+    async def test_key_without_agent_never_consults_agent_access_groups(self):
+        plain_key: Final = UserAPIKeyAuth(api_key="test-key", user_id="test-user")
+
+        with (
+            patch.object(AgentRequestHandler, "_get_allowed_agents_for_key", return_value=UnrestrictedAgentAccess()),
+            patch.object(AgentRequestHandler, "_get_allowed_agents_for_team", return_value=UnrestrictedAgentAccess()),
+            patch(
+                "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
+                new=AsyncMock(return_value=self._ceiling(frozenset())),
+            ) as mock_ceiling,
+        ):
+            assert await AgentRequestHandler.resolve_agent_access(plain_key) == UnrestrictedAgentAccess()
+            mock_ceiling.assert_not_called()
+
     async def test_empty_access_group_denies_every_agent(self):
         """LIT-5143: a key restricted to an access group that resolves to no agents is
         restricted to nothing, not unrestricted. A failed group lookup still fails open."""
diff --git a/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py b/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py
index 231626c7eb5..d15a3adadbd 100644
--- a/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py
+++ b/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py
@@ -990,3 +990,136 @@ async def test_patch_agent_in_db_preserves_secret_when_echoed_back_redacted():
     stored_params: Final = json.loads(mock_update.call_args.kwargs["data"]["litellm_params"])
     assert stored_params["aws_secret_access_key"] == SENTINEL_AWS_SECRET_ACCESS_KEY
     assert stored_params["is_public"] is True
+
+
+def _agent_row_mock(access_group_ids: list[str]) -> MagicMock:
+    row: Final = MagicMock()
+    row.model_dump.return_value = {
+        "agent_id": "agent-123",
+        "agent_name": "Test Agent",
+        "agent_card_params": _sample_agent_card_params(),
+        "litellm_params": {},
+        "object_permission": None,
+        "access_group_ids": access_group_ids,
+    }
+    row.object_permission = None
+    return row
+
+
+@pytest.mark.asyncio
+async def test_add_agent_to_db_persists_deduplicated_access_group_ids():
+    registry: Final = AgentRegistry()
+    mock_prisma: Final = MagicMock()
+    mock_create = AsyncMock(return_value=_agent_row_mock(["ag-1", "ag-2"]))
+    mock_prisma.db.litellm_agentstable.create = mock_create
+
+    result: Final = await registry.add_agent_to_db(
+        agent={
+            "agent_name": "Test Agent",
+            "agent_card_params": _sample_agent_card_params(),
+            "access_group_ids": ["ag-1", "ag-2", "ag-1"],
+        },
+        prisma_client=mock_prisma,
+        created_by="test-user",
+    )
+
+    assert tuple(mock_create.call_args.kwargs["data"]["access_group_ids"]) == ("ag-1", "ag-2")
+    assert result.access_group_ids == ["ag-1", "ag-2"]
+
+
+@pytest.mark.asyncio
+async def test_add_agent_to_db_without_access_group_ids_leaves_column_to_its_default():
+    registry: Final = AgentRegistry()
+    mock_prisma: Final = MagicMock()
+    mock_create = AsyncMock(return_value=_agent_row_mock([]))
+    mock_prisma.db.litellm_agentstable.create = mock_create
+
+    await registry.add_agent_to_db(
+        agent={"agent_name": "Test Agent", "agent_card_params": _sample_agent_card_params()},
+        prisma_client=mock_prisma,
+        created_by="test-user",
+    )
+
+    assert "access_group_ids" not in mock_create.call_args.kwargs["data"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("patch_body", "expected"),
+    [
+        ({"access_group_ids": ["ag-2", "ag-3"]}, ["ag-2", "ag-3"]),
+        ({"access_group_ids": []}, []),
+        ({"access_group_ids": None}, []),
+    ],
+)
+async def test_patch_agent_in_db_replaces_access_group_ids_when_provided(patch_body: dict, expected: list[str]):
+    registry: Final = AgentRegistry()
+    mock_prisma: Final = MagicMock()
+    mock_prisma.db.litellm_agentstable.find_unique = AsyncMock(
+        return_value={
+            "agent_id": "agent-123",
+            "agent_name": "Test Agent",
+            "litellm_params": {},
+            "object_permission_id": None,
+            "access_group_ids": ["ag-1"],
+        }
+    )
+    mock_update = AsyncMock(return_value=_agent_row_mock(expected))
+    mock_prisma.db.litellm_agentstable.update = mock_update
+
+    await registry.patch_agent_in_db(
+        agent_id="agent-123", agent=patch_body, prisma_client=mock_prisma, updated_by="test-user"
+    )
+
+    assert tuple(mock_update.call_args.kwargs["data"]["access_group_ids"]) == tuple(expected)
+
+
+@pytest.mark.asyncio
+async def test_patch_agent_in_db_keeps_access_group_ids_when_omitted():
+    registry: Final = AgentRegistry()
+    mock_prisma: Final = MagicMock()
+    mock_prisma.db.litellm_agentstable.find_unique = AsyncMock(
+        return_value={
+            "agent_id": "agent-123",
+            "agent_name": "Old Name",
+            "litellm_params": {},
+            "object_permission_id": None,
+            "access_group_ids": ["ag-1"],
+        }
+    )
+    mock_update = AsyncMock(return_value=_agent_row_mock(["ag-1"]))
+    mock_prisma.db.litellm_agentstable.update = mock_update
+
+    await registry.patch_agent_in_db(
+        agent_id="agent-123", agent={"agent_name": "New Name"}, prisma_client=mock_prisma, updated_by="test-user"
+    )
+
+    assert "access_group_ids" not in mock_update.call_args.kwargs["data"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("body_access_group_ids", "expected"),
+    [(["ag-9", "ag-9"], ["ag-9"]), (None, []), ("omitted", [])],
+)
+async def test_update_agent_in_db_always_writes_access_group_ids(body_access_group_ids, expected: list[str]):
+    """PUT is a full replacement: omitting the field clears any previously attached groups."""
+    registry: Final = AgentRegistry()
+    mock_prisma: Final = MagicMock()
+    mock_prisma.db.litellm_agentstable.find_unique = AsyncMock(
+        return_value=SimpleNamespace(litellm_params={}, object_permission_id=None, access_group_ids=["ag-1"])
+    )
+    mock_update = AsyncMock(return_value=_agent_row_mock(expected))
+    mock_prisma.db.litellm_agentstable.update = mock_update
+    body: Final = {
+        "agent_name": "Test Agent",
+        "agent_card_params": _sample_agent_card_params(),
+        "litellm_params": {"model": "bedrock/agentcore/my-agent"},
+        **({} if body_access_group_ids == "omitted" else {"access_group_ids": body_access_group_ids}),
+    }
+
+    await registry.update_agent_in_db(
+        agent_id="agent-123", agent=body, prisma_client=mock_prisma, updated_by="test-user"
+    )
+
+    assert tuple(mock_update.call_args.kwargs["data"]["access_group_ids"]) == tuple(expected)
diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py
index f480e096081..ad1742db4b7 100644
--- a/tests/test_litellm/proxy/auth/test_auth_checks.py
+++ b/tests/test_litellm/proxy/auth/test_auth_checks.py
@@ -8461,3 +8461,94 @@ def test_route_skips_budget_checks_marks_only_spend_free_routes() -> None:
 def test_request_skips_budget_checks_extends_route_rule_with_zero_cost_models() -> None:
     assert request_skips_budget_checks(route="/v1/models", model=None, llm_router=None) is True
     assert request_skips_budget_checks(route="/v1/chat/completions", model=None, llm_router=None) is False
+
+
+# Agent access group model ceiling
+
+
+def _agent_model_ceiling(models: frozenset[str]):
+    from litellm.proxy.agent_endpoints.auth.agent_access_groups import AgentAccessGroupCeiling
+
+    return AgentAccessGroupCeiling(
+        access_group_ids=("ag-1",), models=models, mcp_server_ids=frozenset(), agent_ids=frozenset()
+    )
+
+
+async def _run_common_checks_for_agent_key(model: str, valid_token: UserAPIKeyAuth):
+    from fastapi import Request
+
+    from litellm.proxy.auth.auth_checks import common_checks
+
+    return await common_checks(
+        request_body={"model": model, "messages": [{"role": "user", "content": "hi"}]},
+        team_object=None,
+        user_object=None,
+        end_user_object=None,
+        global_proxy_spend=None,
+        general_settings={},
+        route="/chat/completions",
+        llm_router=None,
+        proxy_logging_obj=MagicMock(),
+        valid_token=valid_token,
+        request=MagicMock(spec=Request),
+    )
+
+
+@pytest.mark.asyncio
+async def test_common_checks_agent_access_groups_cap_models_even_when_key_allows_them():
+    agent_key: Final = UserAPIKeyAuth(token="agent-token", agent_id="agent-1", models=["gpt-5", "claude-sonnet"])
+
+    with patch(
+        "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
+        new=AsyncMock(return_value=_agent_model_ceiling(frozenset({"gpt-5"}))),
+    ):
+        assert await _run_common_checks_for_agent_key("gpt-5", agent_key) is True
+
+        with pytest.raises(ProxyException) as exc_info:
+            await _run_common_checks_for_agent_key("claude-sonnet", agent_key)
+
+    assert exc_info.value.type == ProxyErrorTypes.agent_model_access_denied
+    assert exc_info.value.code == str(status.HTTP_403_FORBIDDEN)
+
+
+@pytest.mark.asyncio
+async def test_common_checks_agent_access_groups_naming_no_model_deny_every_model():
+    agent_key: Final = UserAPIKeyAuth(token="agent-token", agent_id="agent-1", models=[])
+
+    with (
+        patch(
+            "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
+            new=AsyncMock(return_value=_agent_model_ceiling(frozenset())),
+        ),
+        pytest.raises(ProxyException) as exc_info,
+    ):
+        await _run_common_checks_for_agent_key("gpt-5", agent_key)
+
+    assert exc_info.value.type == ProxyErrorTypes.agent_model_access_denied
+
+
+@pytest.mark.asyncio
+async def test_common_checks_agent_without_access_groups_adds_no_model_ceiling():
+    agent_key: Final = UserAPIKeyAuth(token="agent-token", agent_id="agent-1", models=["gpt-5", "claude-sonnet"])
+
+    with patch(
+        "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
+        new=AsyncMock(return_value=None),
+    ) as mock_ceiling:
+        assert await _run_common_checks_for_agent_key("gpt-5", agent_key) is True
+        assert await _run_common_checks_for_agent_key("claude-sonnet", agent_key) is True
+
+    mock_ceiling.assert_called_with("agent-1")
+
+
+@pytest.mark.asyncio
+async def test_common_checks_key_without_agent_never_consults_agent_access_groups():
+    plain_key: Final = UserAPIKeyAuth(token="plain-token", models=["gpt-5"])
+
+    with patch(
+        "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
+        new=AsyncMock(return_value=_agent_model_ceiling(frozenset())),
+    ) as mock_ceiling:
+        assert await _run_common_checks_for_agent_key("gpt-5", plain_key) is True
+
+    mock_ceiling.assert_not_called()
diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py
index d687f8d1c8d..13e57408bd0 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py
@@ -17,7 +17,9 @@ from litellm.proxy._types import (
     LitellmUserRoles,
     UserAPIKeyAuth,
 )
+from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
 from litellm.proxy.proxy_server import app
+from litellm.types.agents import AgentResponse
 
 
 def _make_access_group_record(
@@ -126,6 +128,7 @@ def client_and_mocks(monkeypatch):
 
     mock_agents_table = MagicMock()
     mock_agents_table.find_many = AsyncMock(return_value=[])
+    mock_agents_table.update = AsyncMock(return_value=None)
 
     @asynccontextmanager
     async def mock_tx():
@@ -133,6 +136,7 @@ def client_and_mocks(monkeypatch):
             litellm_accessgrouptable=mock_access_group_table,
             litellm_teamtable=mock_team_table,
             litellm_verificationtoken=mock_key_table,
+            litellm_agentstable=mock_agents_table,
         )
         yield tx
 
@@ -158,15 +162,9 @@ def client_and_mocks(monkeypatch):
     mock_proxy_logging = MagicMock()
     mock_proxy_logging.internal_usage_cache = MagicMock()
     mock_proxy_logging.internal_usage_cache.dual_cache = MagicMock()
-    mock_proxy_logging.internal_usage_cache.dual_cache.async_delete_cache = AsyncMock(
-        return_value=None
-    )
-    mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock(
-        return_value=None
-    )
-    mock_proxy_logging.internal_usage_cache.dual_cache.async_set_cache = AsyncMock(
-        return_value=None
-    )
+    mock_proxy_logging.internal_usage_cache.dual_cache.async_delete_cache = AsyncMock(return_value=None)
+    mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock(return_value=None)
+    mock_proxy_logging.internal_usage_cache.dual_cache.async_set_cache = AsyncMock(return_value=None)
     monkeypatch.setattr(ps, "proxy_logging_obj", mock_proxy_logging)
 
     admin_user = UserAPIKeyAuth(
@@ -239,9 +237,7 @@ def test_create_access_group_duplicate_name_conflict(client_and_mocks):
         "unique constraint violation",
     ],
 )
-def test_create_access_group_race_condition_returns_409(
-    client_and_mocks, error_message
-):
+def test_create_access_group_race_condition_returns_409(client_and_mocks, error_message):
     """Create race condition: Prisma unique constraint surfaces as 409, not 500."""
     client, _, mock_table, *_ = client_and_mocks
 
@@ -288,9 +284,7 @@ def test_create_access_group_500_on_non_constraint_prisma_error(client_and_mocks
 
     # Use raise_server_exceptions=False so unhandled exceptions become 500 responses
     test_client = TestClient(app, raise_server_exceptions=False)
-    resp = test_client.post(
-        "/v1/access_group", json={"access_group_name": "test-group"}
-    )
+    resp = test_client.post("/v1/access_group", json={"access_group_name": "test-group"})
     assert resp.status_code == 500
 
 
@@ -558,9 +552,7 @@ def test_update_access_group_empty_body(client_and_mocks):
     """Update with empty body succeeds; only updated_by is set."""
     client, _, mock_table, *_ = client_and_mocks
 
-    existing = _make_access_group_record(
-        access_group_id="ag-update", access_group_name="unchanged"
-    )
+    existing = _make_access_group_record(access_group_id="ag-update", access_group_name="unchanged")
     mock_table.find_unique = AsyncMock(return_value=existing)
 
     resp = client.put("/v1/access_group/ag-update", json={})
@@ -576,14 +568,10 @@ def test_update_access_group_name_success(client_and_mocks):
     """Update access_group_name succeeds when new name is unique."""
     client, _, mock_table, *_ = client_and_mocks
 
-    existing = _make_access_group_record(
-        access_group_id="ag-update", access_group_name="old-name"
-    )
+    existing = _make_access_group_record(access_group_id="ag-update", access_group_name="old-name")
     mock_table.find_unique = AsyncMock(return_value=existing)
 
-    resp = client.put(
-        "/v1/access_group/ag-update", json={"access_group_name": "new-name"}
-    )
+    resp = client.put("/v1/access_group/ag-update", json={"access_group_name": "new-name"})
     assert resp.status_code == 200
     mock_table.update.assert_awaited_once()
     call_kwargs = mock_table.update.call_args.kwargs
@@ -594,19 +582,13 @@ def test_update_access_group_name_duplicate_conflict(client_and_mocks):
     """Update access_group_name to existing name returns 409 (unique constraint)."""
     client, _, mock_table, *_ = client_and_mocks
 
-    existing = _make_access_group_record(
-        access_group_id="ag-update", access_group_name="old-name"
-    )
+    existing = _make_access_group_record(access_group_id="ag-update", access_group_name="old-name")
     mock_table.find_unique = AsyncMock(return_value=existing)
     mock_table.update = AsyncMock(
-        side_effect=Exception(
-            "Unique constraint failed on the fields: (`access_group_name`)"
-        )
+        side_effect=Exception("Unique constraint failed on the fields: (`access_group_name`)")
     )
 
-    resp = client.put(
-        "/v1/access_group/ag-update", json={"access_group_name": "taken-name"}
-    )
+    resp = client.put("/v1/access_group/ag-update", json={"access_group_name": "taken-name"})
     assert resp.status_code == 409
     assert "already exists" in resp.json()["detail"]
     mock_table.update.assert_awaited_once()
@@ -620,21 +602,15 @@ def test_update_access_group_name_duplicate_conflict(client_and_mocks):
         "unique constraint violation",
     ],
 )
-def test_update_access_group_name_unique_constraint_returns_409(
-    client_and_mocks, error_message
-):
+def test_update_access_group_name_unique_constraint_returns_409(client_and_mocks, error_message):
     """Update access_group_name: Prisma unique constraint surfaces as 409."""
     client, _, mock_table, *_ = client_and_mocks
 
-    existing = _make_access_group_record(
-        access_group_id="ag-update", access_group_name="old-name"
-    )
+    existing = _make_access_group_record(access_group_id="ag-update", access_group_name="old-name")
     mock_table.find_unique = AsyncMock(return_value=existing)
     mock_table.update = AsyncMock(side_effect=Exception(error_message))
 
-    resp = client.put(
-        "/v1/access_group/ag-update", json={"access_group_name": "race-name"}
-    )
+    resp = client.put("/v1/access_group/ag-update", json={"access_group_name": "race-name"})
     assert resp.status_code == 409
     assert "already exists" in resp.json()["detail"]
 
@@ -690,9 +666,7 @@ def test_delete_access_group_forbidden_non_admin(client_and_mocks, user_role):
 
 def test_delete_access_group_cleans_up_teams_and_keys(client_and_mocks):
     """Delete removes access_group_id from teams and keys before deleting the group."""
-    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = (
-        client_and_mocks
-    )
+    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks
     mock_team_table = mock_prisma.db.litellm_teamtable
     mock_key_table = mock_prisma.db.litellm_verificationtoken
 
@@ -722,10 +696,61 @@ def test_delete_access_group_cleans_up_teams_and_keys(client_and_mocks):
         where={"token": "key-token-1"},
         data={"access_group_ids": []},
     )
-    mock_access_group_table.delete.assert_awaited_once_with(
-        where={"access_group_id": "ag-to-delete"}
+    mock_access_group_table.delete.assert_awaited_once_with(where={"access_group_id": "ag-to-delete"})
+
+
+def test_delete_access_group_detaches_group_from_agents(client_and_mocks):
+    """Delete strips the group from every agent that had it attached, so agents are not left
+    pointing at a group that no longer exists (which would deny them every model, server and agent)."""
+    client, mock_prisma, mock_access_group_table, _mock_cache, _mock_proxy_logging = client_and_mocks
+    mock_agents_table = mock_prisma.db.litellm_agentstable
+
+    existing = _make_access_group_record(access_group_id="ag-to-delete")
+    mock_access_group_table.find_unique = AsyncMock(return_value=existing)
+
+    agent_with_group = MagicMock()
+    agent_with_group.agent_id = "agent-1"
+    agent_with_group.access_group_ids = ["ag-keep", "ag-to-delete"]
+    mock_agents_table.find_many = AsyncMock(return_value=[agent_with_group])
+    global_agent_registry.register_agent(
+        AgentResponse(
+            agent_id="agent-1",
+            agent_name="detach-test-agent",
+            agent_card_params={"name": "detach-test-agent", "url": "http://localhost:9", "version": "1"},
+            access_group_ids=["ag-keep", "ag-to-delete"],
+        )
     )
 
+    try:
+        resp = client.delete("/v1/access_group/ag-to-delete")
+        assert resp.status_code == 204
+
+        mock_agents_table.update.assert_awaited_once_with(
+            where={"agent_id": "agent-1"},
+            data={"access_group_ids": ("ag-keep",)},
+        )
+        mock_access_group_table.delete.assert_awaited_once_with(where={"access_group_id": "ag-to-delete"})
+        registered = global_agent_registry.get_agent_by_id("agent-1")
+        assert registered is not None
+        assert tuple(registered.access_group_ids or ()) == ("ag-keep",)
+    finally:
+        global_agent_registry.deregister_agent("detach-test-agent")
+
+
+def test_delete_access_group_without_attached_agents_leaves_agents_untouched(client_and_mocks):
+    client, mock_prisma, mock_access_group_table, _mock_cache, _mock_proxy_logging = client_and_mocks
+    mock_agents_table = mock_prisma.db.litellm_agentstable
+
+    mock_access_group_table.find_unique = AsyncMock(
+        return_value=_make_access_group_record(access_group_id="ag-to-delete")
+    )
+
+    resp = client.delete("/v1/access_group/ag-to-delete")
+    assert resp.status_code == 204
+
+    mock_agents_table.find_many.assert_awaited_once_with(where={"access_group_ids": {"hasSome": ("ag-to-delete",)}})
+    mock_agents_table.update.assert_not_awaited()
+
 
 @pytest.mark.parametrize(
     "team_cache_group_ids,key_cache_group_ids,expected_team_ids_after,expected_key_ids_after",
@@ -792,9 +817,7 @@ def test_delete_access_group_patches_cached_team_and_key(
     """Delete patches cached team/key objects to remove the deleted access_group_id."""
     from litellm.proxy._types import LiteLLM_TeamTableCachedObj
 
-    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = (
-        client_and_mocks
-    )
+    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks
     mock_team_table = mock_prisma.db.litellm_teamtable
     mock_key_table = mock_prisma.db.litellm_verificationtoken
 
@@ -820,13 +843,9 @@ def test_delete_access_group_patches_cached_team_and_key(
             team_id="team-1",
             access_group_ids=list(team_cache_group_ids),
         )
-        mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock(
-            return_value=cached_team
-        )
+        mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock(return_value=cached_team)
     else:
-        mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock(
-            return_value=None
-        )
+        mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock(return_value=None)
 
     # user_api_key_cache is queried both for teams (fallback after dual_cache) and
     # hashed keys — return the right stub per ``key``. A single AsyncMock(return_value=key)
@@ -834,9 +853,7 @@ def test_delete_access_group_patches_cached_team_and_key(
     # Use a synchronous side_effect (not async def): AsyncMock awaits coroutine side_effects
     # inconsistently across Python/unittest versions; sync returns are awaited as immediate results.
     def user_cache_get_side_effect(*args, **kwargs):
-        cache_key = (
-            kwargs.get("key") if "key" in kwargs else (args[0] if args else None)
-        )
+        cache_key = kwargs.get("key") if "key" in kwargs else (args[0] if args else None)
         if cache_key == "team_id:team-1":
             if team_cache_group_ids is None:
                 return None
@@ -868,14 +885,11 @@ def test_delete_access_group_patches_cached_team_and_key(
         team_set_calls = [
             c
             for c in mock_cache.async_set_cache.call_args_list
-            if c.kwargs.get("key", "") == "team_id:team-1"
-            or (len(c.args) >= 1 and c.args[0] == "team_id:team-1")
+            if c.kwargs.get("key", "") == "team_id:team-1" or (len(c.args) >= 1 and c.args[0] == "team_id:team-1")
         ]
         assert len(team_set_calls) >= 1, "Expected team cache to be patched"
         # The cached team object should have the updated access_group_ids
-        written_team = (
-            team_set_calls[0].kwargs.get("value") or team_set_calls[0].args[1]
-        )
+        written_team = team_set_calls[0].kwargs.get("value") or team_set_calls[0].args[1]
         if isinstance(written_team, LiteLLM_TeamTableCachedObj):
             assert written_team.access_group_ids == expected_team_ids_after
     else:
@@ -883,8 +897,7 @@ def test_delete_access_group_patches_cached_team_and_key(
         team_set_calls = [
             c
             for c in mock_cache.async_set_cache.call_args_list
-            if c.kwargs.get("key", "") == "team_id:team-1"
-            or (len(c.args) >= 1 and c.args[0] == "team_id:team-1")
+            if c.kwargs.get("key", "") == "team_id:team-1" or (len(c.args) >= 1 and c.args[0] == "team_id:team-1")
         ]
         assert len(team_set_calls) == 0, "Should not patch team cache when not cached"
 
@@ -892,8 +905,7 @@ def test_delete_access_group_patches_cached_team_and_key(
         key_set_calls = [
             c
             for c in mock_cache.async_set_cache.call_args_list
-            if c.kwargs.get("key", "") == "hashed-key-1"
-            or (len(c.args) >= 1 and c.args[0] == "hashed-key-1")
+            if c.kwargs.get("key", "") == "hashed-key-1" or (len(c.args) >= 1 and c.args[0] == "hashed-key-1")
         ]
         assert len(key_set_calls) >= 1, "Expected key cache to be patched"
         written_key = key_set_calls[0].kwargs.get("value") or key_set_calls[0].args[1]
@@ -903,17 +915,14 @@ def test_delete_access_group_patches_cached_team_and_key(
         key_set_calls = [
             c
             for c in mock_cache.async_set_cache.call_args_list
-            if c.kwargs.get("key", "") == "hashed-key-1"
-            or (len(c.args) >= 1 and c.args[0] == "hashed-key-1")
+            if c.kwargs.get("key", "") == "hashed-key-1" or (len(c.args) >= 1 and c.args[0] == "hashed-key-1")
         ]
         assert len(key_set_calls) == 0, "Should not patch key cache when not cached"
 
 
 def test_delete_access_group_patches_key_cached_as_dict(client_and_mocks):
     """Delete patches key cache — mock returns UserAPIKeyAuth (what UserApiKeyCache emits after deserialize)."""
-    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = (
-        client_and_mocks
-    )
+    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks
     mock_team_table = mock_prisma.db.litellm_teamtable
     mock_key_table = mock_prisma.db.litellm_verificationtoken
 
@@ -929,9 +938,7 @@ def test_delete_access_group_patches_key_cached_as_dict(client_and_mocks):
     mock_key_table.find_unique = AsyncMock(return_value=key_with_group)
 
     # No team in cache
-    mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock(
-        return_value=None
-    )
+    mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock(return_value=None)
 
     # Serialized shape from Redis dict; UserApiKeyCache.async_get_cache(model_type=...) yields a model — simulate that.
     cached_key_payload = {
@@ -940,18 +947,14 @@ def test_delete_access_group_patches_key_cached_as_dict(client_and_mocks):
     }
 
     def user_cache_get_dict_when_key_matches(*args, **kwargs):
-        cache_key = (
-            kwargs.get("key") if "key" in kwargs else (args[0] if args else None)
-        )
+        cache_key = kwargs.get("key") if "key" in kwargs else (args[0] if args else None)
         if cache_key == "team_id:team-1":
             return None
         if cache_key == "hashed-key-dict":
             return UserAPIKeyAuth.model_validate(cached_key_payload)
         return None
 
-    mock_cache.async_get_cache = AsyncMock(
-        side_effect=user_cache_get_dict_when_key_matches
-    )
+    mock_cache.async_get_cache = AsyncMock(side_effect=user_cache_get_dict_when_key_matches)
 
     resp = client.delete("/v1/access_group/ag-to-delete")
     assert resp.status_code == 204
@@ -960,8 +963,7 @@ def test_delete_access_group_patches_key_cached_as_dict(client_and_mocks):
     key_set_calls = [
         c
         for c in mock_cache.async_set_cache.call_args_list
-        if c.kwargs.get("key", "") == "hashed-key-dict"
-        or (len(c.args) >= 1 and c.args[0] == "hashed-key-dict")
+        if c.kwargs.get("key", "") == "hashed-key-dict" or (len(c.args) >= 1 and c.args[0] == "hashed-key-dict")
     ]
     assert len(key_set_calls) >= 1, "Expected key cache to be patched"
     written_key = key_set_calls[0].kwargs.get("value") or key_set_calls[0].args[1]
@@ -988,9 +990,7 @@ def test_delete_access_group_404_on_p2025_or_record_not_found(client_and_mocks):
 
     existing = _make_access_group_record(access_group_id="ag-to-delete")
     mock_table.find_unique = AsyncMock(return_value=existing)
-    mock_table.delete = AsyncMock(
-        side_effect=Exception("P2025: Record to delete does not exist")
-    )
+    mock_table.delete = AsyncMock(side_effect=Exception("P2025: Record to delete does not exist"))
 
     resp = client.delete("/v1/access_group/ag-to-delete")
     assert resp.status_code == 404
@@ -1039,9 +1039,7 @@ def test_delete_access_group_500_on_generic_exception(client_and_mocks):
         ("delete", "/v1/unified_access_group/ag-123", lambda: {}),
     ],
 )
-def test_access_group_endpoints_db_not_connected(
-    client_and_mocks, monkeypatch, method, url, factory
-):
+def test_access_group_endpoints_db_not_connected(client_and_mocks, monkeypatch, method, url, factory):
     """All endpoints return 500 when DB is not connected."""
     client, *_ = client_and_mocks
 
@@ -1049,9 +1047,7 @@ def test_access_group_endpoints_db_not_connected(
 
     resp = getattr(client, method)(url, **factory())
     assert resp.status_code == 500
-    assert (
-        resp.json()["detail"]["error"] == CommonProxyErrors.db_not_connected_error.value
-    )
+    assert resp.json()["detail"]["error"] == CommonProxyErrors.db_not_connected_error.value
 
 
 # ---------------------------------------------------------------------------
@@ -1107,9 +1103,7 @@ def test_attached_team_ids_by_group_keeps_column_order_then_appends_unmirrored_t
 
 def test_create_access_group_syncs_assigned_teams(client_and_mocks):
     """Create adds access_group_id to each assigned team's access_group_ids in DB."""
-    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = (
-        client_and_mocks
-    )
+    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks
     mock_team_table = mock_prisma.db.litellm_teamtable
 
     team_record = _make_team_record("team-1")
@@ -1132,9 +1126,7 @@ def test_create_access_group_syncs_assigned_teams(client_and_mocks):
 
 def test_create_access_group_syncs_assigned_keys(client_and_mocks):
     """Create adds access_group_id to each assigned key's access_group_ids in DB."""
-    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = (
-        client_and_mocks
-    )
+    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks
     mock_key_table = mock_prisma.db.litellm_verificationtoken
 
     key_record = MagicMock()
@@ -1148,9 +1140,7 @@ def test_create_access_group_syncs_assigned_keys(client_and_mocks):
     )
     assert resp.status_code == 201
 
-    mock_key_table.find_unique.assert_awaited_once_with(
-        where={"token": "hashed-token-1"}
-    )
+    mock_key_table.find_unique.assert_awaited_once_with(where={"token": "hashed-token-1"})
     mock_key_table.update.assert_awaited_once()
     call_kwargs = mock_key_table.update.call_args.kwargs
     assert call_kwargs["where"] == {"token": "hashed-token-1"}
@@ -1200,14 +1190,10 @@ def test_create_access_group_idempotent_team_sync(client_and_mocks):
 
 def test_update_access_group_syncs_added_teams(client_and_mocks):
     """Update adds access_group_id to newly assigned teams."""
-    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = (
-        client_and_mocks
-    )
+    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks
     mock_team_table = mock_prisma.db.litellm_teamtable
 
-    existing = _make_access_group_record(
-        access_group_id="ag-update", assigned_team_ids=["team-existing"]
-    )
+    existing = _make_access_group_record(access_group_id="ag-update", assigned_team_ids=["team-existing"])
     mock_access_group_table.find_unique = AsyncMock(return_value=existing)
 
     team_record = _make_team_record("team-new")
@@ -1248,14 +1234,10 @@ def test_update_access_group_rejects_nonexistent_team(client_and_mocks):
 
 def test_update_access_group_syncs_removed_teams(client_and_mocks):
     """Update removes access_group_id from de-assigned teams."""
-    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = (
-        client_and_mocks
-    )
+    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks
     mock_team_table = mock_prisma.db.litellm_teamtable
 
-    existing = _make_access_group_record(
-        access_group_id="ag-update", assigned_team_ids=["team-keep", "team-remove"]
-    )
+    existing = _make_access_group_record(access_group_id="ag-update", assigned_team_ids=["team-keep", "team-remove"])
     mock_access_group_table.find_unique = AsyncMock(return_value=existing)
 
     team_to_remove = _make_team_record("team-remove", ["ag-update"])
@@ -1268,9 +1250,7 @@ def test_update_access_group_syncs_removed_teams(client_and_mocks):
     )
     assert resp.status_code == 200
 
-    mock_team_table.find_unique.assert_awaited_once_with(
-        where={"team_id": "team-remove"}
-    )
+    mock_team_table.find_unique.assert_awaited_once_with(where={"team_id": "team-remove"})
     mock_team_table.update.assert_awaited_once()
     call_kwargs = mock_team_table.update.call_args.kwargs
     assert call_kwargs["where"] == {"team_id": "team-remove"}
@@ -1296,19 +1276,15 @@ def test_update_access_group_detaches_team_the_mirror_missed(client_and_mocks):
     mock_team_table.update.assert_awaited_once()
     call_kwargs = mock_team_table.update.call_args.kwargs
     assert call_kwargs["where"] == {"team_id": "team-unmirrored"}
-    assert call_kwargs["data"]["access_group_ids"] == ["ag-other"]
+    assert tuple(call_kwargs["data"]["access_group_ids"]) == ("ag-other",)
 
 
 def test_update_access_group_no_team_sync_when_ids_not_in_payload(client_and_mocks):
     """Update does not sync teams when assigned_team_ids is absent from the payload."""
-    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = (
-        client_and_mocks
-    )
+    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks
     mock_team_table = mock_prisma.db.litellm_teamtable
 
-    existing = _make_access_group_record(
-        access_group_id="ag-update", assigned_team_ids=["team-1"]
-    )
+    existing = _make_access_group_record(access_group_id="ag-update", assigned_team_ids=["team-1"])
     mock_access_group_table.find_unique = AsyncMock(return_value=existing)
 
     resp = client.put("/v1/access_group/ag-update", json={"description": "new desc"})
@@ -1320,14 +1296,10 @@ def test_update_access_group_no_team_sync_when_ids_not_in_payload(client_and_moc
 
 def test_update_access_group_syncs_added_keys(client_and_mocks):
     """Update adds access_group_id to newly assigned keys."""
-    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = (
-        client_and_mocks
-    )
+    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks
     mock_key_table = mock_prisma.db.litellm_verificationtoken
 
-    existing = _make_access_group_record(
-        access_group_id="ag-update", assigned_key_ids=["old-token"]
-    )
+    existing = _make_access_group_record(access_group_id="ag-update", assigned_key_ids=["old-token"])
     mock_access_group_table.find_unique = AsyncMock(return_value=existing)
 
     key_record = MagicMock()
@@ -1350,14 +1322,10 @@ def test_update_access_group_syncs_added_keys(client_and_mocks):
 
 def test_update_access_group_syncs_removed_keys(client_and_mocks):
     """Update removes access_group_id from de-assigned keys."""
-    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = (
-        client_and_mocks
-    )
+    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks
     mock_key_table = mock_prisma.db.litellm_verificationtoken
 
-    existing = _make_access_group_record(
-        access_group_id="ag-update", assigned_key_ids=["keep-token", "remove-token"]
-    )
+    existing = _make_access_group_record(access_group_id="ag-update", assigned_key_ids=["keep-token", "remove-token"])
     mock_access_group_table.find_unique = AsyncMock(return_value=existing)
 
     key_to_remove = MagicMock()
@@ -1385,9 +1353,7 @@ def test_update_access_group_syncs_removed_keys(client_and_mocks):
 
 def test_delete_access_group_handles_out_of_sync_assigned_teams(client_and_mocks):
     """Delete includes teams from assigned_team_ids even when not found by hasSome query."""
-    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = (
-        client_and_mocks
-    )
+    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks
     mock_team_table = mock_prisma.db.litellm_teamtable
 
     # Access group has assigned_team_ids but the team's access_group_ids is not synced
@@ -1409,18 +1375,14 @@ def test_delete_access_group_handles_out_of_sync_assigned_teams(client_and_mocks
     assert resp.status_code == 204
 
     # find_unique is called for the out-of-sync team (included via union with assigned_team_ids)
-    mock_team_table.find_unique.assert_awaited_once_with(
-        where={"team_id": "team-out-of-sync"}
-    )
+    mock_team_table.find_unique.assert_awaited_once_with(where={"team_id": "team-out-of-sync"})
     # No update needed since team's access_group_ids doesn't contain "ag-to-delete"
     mock_team_table.update.assert_not_awaited()
 
 
 def test_delete_access_group_handles_out_of_sync_assigned_keys(client_and_mocks):
     """Delete includes keys from assigned_key_ids even when not found by hasSome query."""
-    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = (
-        client_and_mocks
-    )
+    client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks
     mock_key_table = mock_prisma.db.litellm_verificationtoken
 
     existing = _make_access_group_record(
@@ -1439,9 +1401,7 @@ def test_delete_access_group_handles_out_of_sync_assigned_keys(client_and_mocks)
     resp = client.delete("/v1/access_group/ag-to-delete")
     assert resp.status_code == 204
 
-    mock_key_table.find_unique.assert_awaited_once_with(
-        where={"token": "token-out-of-sync"}
-    )
+    mock_key_table.find_unique.assert_awaited_once_with(where={"token": "token-out-of-sync"})
     mock_key_table.update.assert_not_awaited()
 
 
@@ -1536,10 +1496,16 @@ def test_list_access_groups_resolves_names_with_one_query_per_table(client_and_m
     mock_table.find_many = AsyncMock(
         return_value=[
             _make_access_group_record(
-                access_group_id="ag-1", access_mcp_server_ids=["mcp-a"], access_agent_ids=["agent-a"], assigned_key_ids=["key-a"]
+                access_group_id="ag-1",
+                access_mcp_server_ids=["mcp-a"],
+                access_agent_ids=["agent-a"],
+                assigned_key_ids=["key-a"],
             ),
             _make_access_group_record(
-                access_group_id="ag-2", access_mcp_server_ids=["mcp-b"], access_agent_ids=["agent-b"], assigned_key_ids=["key-b"]
+                access_group_id="ag-2",
+                access_mcp_server_ids=["mcp-b"],
+                access_agent_ids=["agent-b"],
+                assigned_key_ids=["key-b"],
             ),
         ]
     )
@@ -1573,7 +1539,10 @@ def test_list_access_groups_skips_lookups_when_nothing_to_resolve(client_and_moc
     """Groups with no MCP servers, agents or keys must not trigger an empty IN () query per table."""
     client, mock_prisma, mock_table, *_ = client_and_mocks
     mock_table.find_many = AsyncMock(
-        return_value=[_make_access_group_record(access_group_id="ag-1"), _make_access_group_record(access_group_id="ag-2")]
+        return_value=[
+            _make_access_group_record(access_group_id="ag-1"),
+            _make_access_group_record(access_group_id="ag-2"),
+        ]
     )
 
     resp = client.get("/v1/access_group")
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentFormKit.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentFormKit.tsx
index 15b85001cdc..8e100d0c3ed 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentFormKit.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentFormKit.tsx
@@ -90,6 +90,7 @@ export interface AgentFormValues {
   guardrails?: string[];
   entitlement_models?: string[];
   entitlement_agents?: string[];
+  access_group_ids?: string[];
   allowed_mcp_servers_and_groups?: McpServerSelection;
   mcp_tool_permissions?: Record<string, string[]>;
   defaultInputModes?: string[];
@@ -121,6 +122,7 @@ export interface AgentRequestPayload {
   agent_card_params?: Record<string, unknown>;
   litellm_params?: Record<string, unknown>;
   object_permission?: Record<string, unknown>;
+  access_group_ids?: string[];
 }
 
 interface AgentFormFieldProps {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.integration.test.tsx
index ebc97891744..457ee656415 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.integration.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.integration.test.tsx
@@ -21,6 +21,9 @@ vi.mock("./agent_card_discovery", () => ({ default: () => <div data-testid="agen
 vi.mock("@/components/mcp_server_management/MCPServerSelector", () => ({ default: () => <div /> }));
 vi.mock("@/components/mcp_server_management/MCPToolPermissions", () => ({ default: () => <div /> }));
 vi.mock("@/components/guardrails/GuardrailSelector", () => ({ default: () => <div /> }));
+vi.mock("@/app/(dashboard)/hooks/accessGroups/useAccessGroups", () => ({
+  useAccessGroups: () => ({ data: [], isLoading: false, isError: false }),
+}));
 vi.mock("@/components/common_components/team_dropdown", () => ({ default: () => <div /> }));
 
 const a2aInfo: AgentCreateInfo = {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.test.tsx
index ccac244f019..b5301d0e6a0 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.test.tsx
@@ -44,6 +44,14 @@ vi.mock("@/components/mcp_server_management/MCPToolPermissions", () => ({
   default: () => null,
 }));
 
+vi.mock("@/components/common_components/AccessGroupSelector", () => ({
+  default: ({ onChange }: { onChange: (value: string[]) => void }) => (
+    <button type="button" data-testid="select-access-group" onClick={() => onChange(["ag-1", "ag-2"])}>
+      Select access group
+    </button>
+  ),
+}));
+
 vi.mock("@/components/common_components/team_dropdown", () => ({
   default: () => null,
 }));
@@ -141,5 +149,27 @@ describe("AddAgentForm logos", () => {
     await vi.waitFor(() => expect(networking.createAgentCall).toHaveBeenCalled());
     const [, payload] = vi.mocked(networking.createAgentCall).mock.calls[0];
     expect(payload.object_permission).toEqual({ mcp_toolsets: ["ts-1"] });
+    expect(payload).not.toHaveProperty("access_group_ids");
+  });
+
+  it("includes selected access groups in the create payload", async () => {
+    const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never });
+    vi.mocked(networking.createAgentCall).mockReset().mockResolvedValue({
+      agent_id: "agent-1",
+      agent_name: "Test Agent",
+    } as never);
+    vi.mocked(networking.keyListCall).mockResolvedValue({ keys: [] });
+
+    renderForm();
+    await user.click(screen.getByRole("button", { name: "Next →" }));
+    await user.click(screen.getByTestId("select-access-group"));
+    await user.click(screen.getByRole("button", { name: "Next →" }));
+    await user.click(screen.getByRole("button", { name: "Next →" }));
+    await user.click(screen.getByText(/Skip for now/));
+    await user.click(screen.getByRole("button", { name: "Create Agent →" }));
+
+    await vi.waitFor(() => expect(networking.createAgentCall).toHaveBeenCalled());
+    const [, payload] = vi.mocked(networking.createAgentCall).mock.calls[0];
+    expect(payload.access_group_ids).toEqual(["ag-1", "ag-2"]);
   });
 });
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx
index e71fed40209..5bd6ea9b83a 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx
@@ -50,6 +50,7 @@ import {
 } from "./AgentFormKit";
 import MCPServerSelector from "@/components/mcp_server_management/MCPServerSelector";
 import MCPToolPermissions from "@/components/mcp_server_management/MCPToolPermissions";
+import AccessGroupSelector from "@/components/common_components/AccessGroupSelector";
 import GuardrailSelector from "@/components/guardrails/GuardrailSelector";
 import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog";
 
@@ -113,6 +114,7 @@ const SHARED_INITIAL_VALUES: AgentFormValues = {
   mcp_tool_permissions: {},
   entitlement_models: [],
   entitlement_agents: [],
+  access_group_ids: [],
   guardrails: [],
 };
 
@@ -374,6 +376,9 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
       if (Object.keys(objectPermission).length > 0) {
         agentData.object_permission = objectPermission;
       }
+      if (values.access_group_ids?.length) {
+        agentData.access_group_ids = values.access_group_ids;
+      }
 
       // Wire trace-id flags and budget controls into agent litellm_params (before create call)
       if (requireTraceIdInbound || requireTraceIdOutbound) {
@@ -494,6 +499,22 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
           )}
         </AgentFormField>
 
+        <AgentFormField
+          name="access_group_ids"
+          label={labelWithHint(
+            "Access Groups",
+            "Attach access groups to this agent. Attached groups cap which models, MCP servers, and agents the agent can reach, on top of its key and team permissions. Leave empty to apply no extra cap.",
+          )}
+        >
+          {({ value, onChange }) => (
+            <AccessGroupSelector
+              value={Array.isArray(value) ? (value as string[]) : []}
+              onChange={onChange}
+              placeholder="Select access groups (optional)"
+            />
+          )}
+        </AgentFormField>
+
         <Separator className="my-2" />
 
         <AgentFormField
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_config.ts b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_config.ts
index e0a5271366f..caa3b9ef472 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_config.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_config.ts
@@ -313,6 +313,10 @@ export const buildAgentDataFromForm = (values: any, existingAgent?: any) => {
   return agentData;
 };
 
+export const parseAccessGroupIdsForForm = (agent: { access_group_ids?: string[] | null }) => ({
+  access_group_ids: agent.access_group_ids ?? [],
+});
+
 export const parseMcpPermissionsForForm = (agent: any) => ({
   allowed_mcp_servers_and_groups: {
     servers: agent.object_permission?.mcp_servers ?? [],
@@ -377,5 +381,6 @@ export const parseAgentForForm = (agent: any) => {
     // extra_headers: already an array of strings
     extra_headers: agent.extra_headers ?? [],
     ...parseMcpPermissionsForForm(agent),
+    ...parseAccessGroupIdsForForm(agent),
   };
 };
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx
index 357f924cbe7..37e00766a75 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx
@@ -25,6 +25,10 @@ vi.mock("@/app/(dashboard)/hooks/keys/useKeys", () => ({
 
 vi.mock("./agent_card_discovery", () => ({ default: () => <div data-testid="agent-card-discovery" /> }));
 
+vi.mock("@/app/(dashboard)/hooks/accessGroups/useAccessGroups", () => ({
+  useAccessGroups: () => ({ data: [], isLoading: false, isError: false }),
+}));
+
 const A2A_AGENT = {
   agent_id: "agent-1",
   agent_name: "my-agent",
@@ -176,6 +180,7 @@ describe("AgentInfoView update payload", () => {
       session_tpm_limit: 333,
       session_rpm_limit: 444,
       object_permission: { mcp_servers: [], mcp_access_groups: [], mcp_toolsets: [], mcp_tool_permissions: {} },
+      access_group_ids: [],
     });
   });
 
@@ -217,6 +222,7 @@ describe("AgentInfoView update payload", () => {
       session_tpm_limit: 333,
       session_rpm_limit: 444,
       object_permission: { mcp_servers: [], mcp_access_groups: [], mcp_toolsets: [], mcp_tool_permissions: {} },
+      access_group_ids: [],
     });
   });
 
@@ -295,6 +301,7 @@ describe("AgentInfoView update payload", () => {
         model: "langgraph/asst_1",
       },
       object_permission: { mcp_servers: [], mcp_access_groups: [], mcp_toolsets: [], mcp_tool_permissions: {} },
+      access_group_ids: [],
     });
   });
 
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.test.tsx
index 29d97a20afe..7e6c7c0e05c 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.test.tsx
@@ -28,6 +28,28 @@ vi.mock("@/app/(dashboard)/hooks/mcpServers/useMCPServers", () => ({
   useMCPServers: () => ({ data: [{ server_id: "srv-1", server_name: "github" }] }),
 }));
 
+vi.mock("@/app/(dashboard)/hooks/accessGroups/useAccessGroups", () => ({
+  useAccessGroups: () => ({
+    data: [{ access_group_id: "ag-1", access_group_name: "support-tools" }],
+    isLoading: false,
+    isError: false,
+  }),
+}));
+
+vi.mock("@/components/common_components/AccessGroupSelector", () => ({
+  default: ({ value, onChange }: { value?: string[]; onChange: (value: string[]) => void }) => (
+    <div>
+      <span data-testid="selected-access-groups">{(value ?? []).join(",")}</span>
+      <button type="button" onClick={() => onChange(["ag-1"])}>
+        Attach ag-1
+      </button>
+      <button type="button" onClick={() => onChange([])}>
+        Detach all access groups
+      </button>
+    </div>
+  ),
+}));
+
 vi.mock("@/components/mcp_server_management/MCPServerSelector", () => ({
   default: () => <div data-testid="mcp-server-selector" />,
 }));
@@ -76,6 +98,38 @@ describe("AgentInfoView settings", () => {
     expect(payload.tpm_limit).toBe(42);
     const clearedMcpGrants = { mcp_servers: [], mcp_access_groups: [], mcp_toolsets: [], mcp_tool_permissions: {} };
     expect(payload.object_permission).toEqual(clearedMcpGrants);
+    expect(payload.access_group_ids).toEqual([]);
+  });
+
+  it("sends the newly attached access group in the update payload", async () => {
+    render(<AgentInfoView agentId="agent-1" onClose={vi.fn()} accessToken="sk-test" isAdmin={true} />);
+
+    fireEvent.click(await screen.findByRole("tab", { name: "Settings" }));
+    fireEvent.click(screen.getByRole("button", { name: "Edit Settings" }));
+    fireEvent.click(await screen.findByRole("button", { name: "Attach ag-1" }));
+    expect(screen.getByTestId("selected-access-groups")).toHaveTextContent("ag-1");
+
+    fireEvent.click(screen.getByRole("button", { name: /Save Changes/ }));
+
+    await waitFor(() => expect(networking.patchAgentCall).toHaveBeenCalledTimes(1));
+    const [, , payload] = vi.mocked(networking.patchAgentCall).mock.calls[0];
+    expect(payload.access_group_ids).toEqual(["ag-1"]);
+  });
+
+  it("loads the attached access groups into the editor and sends an empty list once detached", async () => {
+    vi.mocked(networking.getAgentInfo).mockResolvedValue({ ...agent, access_group_ids: ["ag-1"] });
+    render(<AgentInfoView agentId="agent-1" onClose={vi.fn()} accessToken="sk-test" isAdmin={true} />);
+
+    fireEvent.click(await screen.findByRole("tab", { name: "Settings" }));
+    fireEvent.click(screen.getByRole("button", { name: "Edit Settings" }));
+    expect(await screen.findByTestId("selected-access-groups")).toHaveTextContent("ag-1");
+
+    fireEvent.click(screen.getByRole("button", { name: "Detach all access groups" }));
+    fireEvent.click(screen.getByRole("button", { name: /Save Changes/ }));
+
+    await waitFor(() => expect(networking.patchAgentCall).toHaveBeenCalledTimes(1));
+    const [, , payload] = vi.mocked(networking.patchAgentCall).mock.calls[0];
+    expect(payload.access_group_ids).toEqual([]);
   });
 
   it("shows MCP grants with server names on the overview tab", async () => {
@@ -88,4 +142,20 @@ describe("AgentInfoView settings", () => {
 
     expect(await screen.findByText("github (srv-1)")).toBeInTheDocument();
   });
+
+  it("shows attached access groups with their names on the overview tab", async () => {
+    vi.mocked(networking.getAgentInfo).mockResolvedValue({ ...agent, access_group_ids: ["ag-1", "ag-unknown"] });
+
+    render(<AgentInfoView agentId="agent-1" onClose={vi.fn()} accessToken="sk-test" isAdmin={true} />);
+
+    expect(await screen.findByText("support-tools (ag-1)")).toBeInTheDocument();
+    expect(screen.getByText("ag-unknown")).toBeInTheDocument();
+  });
+
+  it("shows None when the agent has no access groups attached", async () => {
+    render(<AgentInfoView agentId="agent-1" onClose={vi.fn()} accessToken="sk-test" isAdmin={true} />);
+
+    expect(await screen.findByText("Access Groups")).toBeInTheDocument();
+    expect(screen.getByText("None")).toBeInTheDocument();
+  });
 });
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.tsx
index 6cb99e9692f..adac456f232 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.tsx
@@ -16,6 +16,8 @@ import { Agent } from "@/components/agents/types";
 import { KeyResponse } from "@/components/key_team_helpers/key_list";
 import { useKeys } from "@/app/(dashboard)/hooks/keys/useKeys";
 import { useMCPServers } from "@/app/(dashboard)/hooks/mcpServers/useMCPServers";
+import { useAccessGroups } from "@/app/(dashboard)/hooks/accessGroups/useAccessGroups";
+import AccessGroupSelector from "@/components/common_components/AccessGroupSelector";
 import KeyInfoView from "@/components/templates/key_info_view";
 import MCPServerSelector from "@/components/mcp_server_management/MCPServerSelector";
 import MCPToolPermissions from "@/components/mcp_server_management/MCPToolPermissions";
@@ -26,6 +28,7 @@ import {
   AGENT_FORM_CONFIG,
   buildAgentDataFromForm,
   buildMcpObjectPermission,
+  parseAccessGroupIdsForForm,
   parseAgentForForm,
   parseMcpPermissionsForForm,
 } from "./agent_config";
@@ -122,7 +125,11 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
       } else {
         const typeInfo = agentTypeMetadata.find((t) => t.agent_type === agentType);
         if (typeInfo) {
-          form.reset({ ...parseDynamicAgentForForm(data, typeInfo), ...parseMcpPermissionsForForm(data) });
+          form.reset({
+            ...parseDynamicAgentForForm(data, typeInfo),
+            ...parseMcpPermissionsForForm(data),
+            ...parseAccessGroupIdsForForm(data),
+          });
         } else {
           form.reset(parseAgentForForm(data));
         }
@@ -142,7 +149,11 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
       if (agentType !== "a2a") {
         const typeInfo = agentTypeMetadata.find((t) => t.agent_type === agentType);
         if (typeInfo) {
-          form.reset({ ...parseDynamicAgentForForm(agent, typeInfo), ...parseMcpPermissionsForForm(agent) });
+          form.reset({
+            ...parseDynamicAgentForForm(agent, typeInfo),
+            ...parseMcpPermissionsForForm(agent),
+            ...parseAccessGroupIdsForForm(agent),
+          });
         }
       }
     }
@@ -153,12 +164,18 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
   const mcpSelection = useWatch({ control: form.control, name: "allowed_mcp_servers_and_groups" });
   const mcpToolPermissions = useWatch({ control: form.control, name: "mcp_tool_permissions" });
   const { data: mcpServers = [] } = useMCPServers();
+  const { data: accessGroups = [] } = useAccessGroups();
 
   const mcpServerLabel = (serverId: string) => {
     const server = mcpServers.find((s) => s.server_id === serverId);
     return server?.server_name ? `${server.server_name} (${serverId})` : serverId;
   };
 
+  const accessGroupLabel = (accessGroupId: string) => {
+    const group = accessGroups.find((g) => g.access_group_id === accessGroupId);
+    return group ? `${group.access_group_name} (${accessGroupId})` : accessGroupId;
+  };
+
   const discoveryRequest = useMemo(
     () => buildDiscoveryRequest(detectedAgentType, watchedFormValues || {}, selectedAgentTypeInfo),
     [watchedFormValues, selectedAgentTypeInfo, detectedAgentType],
@@ -221,6 +238,7 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
       await patchAgentCall(accessToken, agentId, {
         ...updateData,
         object_permission: buildMcpObjectPermission(values),
+        access_group_ids: values.access_group_ids ?? [],
       });
       toast.success("Agent updated successfully");
       setIsEditing(false);
@@ -350,6 +368,17 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
               <DetailItem label="RPM Limit">{agent.rpm_limit ?? "Unlimited"}</DetailItem>
               <DetailItem label="Session TPM Limit">{agent.session_tpm_limit ?? "Unlimited"}</DetailItem>
               <DetailItem label="Session RPM Limit">{agent.session_rpm_limit ?? "Unlimited"}</DetailItem>
+              <DetailItem label="Access Groups">
+                {agent.access_group_ids?.length ? (
+                  <div className="space-y-1">
+                    {agent.access_group_ids.map((accessGroupId) => (
+                      <div key={accessGroupId}>{accessGroupLabel(accessGroupId)}</div>
+                    ))}
+                  </div>
+                ) : (
+                  "None"
+                )}
+              </DetailItem>
               <DetailItem label="Created At">{formatDate(agent.created_at)}</DetailItem>
               <DetailItem label="Updated At">{formatDate(agent.updated_at)}</DetailItem>
             </DetailList>
@@ -489,6 +518,26 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
                           {rateLimitField("session_rpm_limit", "Session RPM Limit")}
                         </div>
 
+                        <Separator className="my-6" />
+                        <h3 className="text-lg font-medium mb-4">Access Groups</h3>
+                        <FieldGroup>
+                          <AgentFormField
+                            name="access_group_ids"
+                            label={labelWithHint(
+                              "Access Groups",
+                              "Attached groups cap which models, MCP servers, and agents this agent can reach, on top of its key and team permissions. Leave empty to apply no extra cap.",
+                            )}
+                          >
+                            {({ value, onChange }) => (
+                              <AccessGroupSelector
+                                value={Array.isArray(value) ? (value as string[]) : []}
+                                onChange={onChange}
+                                placeholder="Select access groups (optional)"
+                              />
+                            )}
+                          </AgentFormField>
+                        </FieldGroup>
+
                         <Separator className="my-6" />
                         <h3 className="text-lg font-medium mb-4">MCP Servers</h3>
                         <FieldGroup>
diff --git a/ui/litellm-dashboard/src/components/agents/types.ts b/ui/litellm-dashboard/src/components/agents/types.ts
index 24ff0c0e12c..6adb3fa9dda 100644
--- a/ui/litellm-dashboard/src/components/agents/types.ts
+++ b/ui/litellm-dashboard/src/components/agents/types.ts
@@ -21,6 +21,7 @@ export interface Agent {
     [key: string]: any;
   };
   object_permission?: AgentObjectPermission;
+  access_group_ids?: string[] | null;
   keys?: AgentAttachedKey[] | null;
   spend?: number;
   tpm_limit?: number | null;
diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx
index e77c8ba7e41..7714f96804f 100644
--- a/ui/litellm-dashboard/src/components/networking.tsx
+++ b/ui/litellm-dashboard/src/components/networking.tsx
@@ -6267,6 +6267,7 @@ export const patchAgentCall = async (
     rpm_limit?: number | null;
     session_tpm_limit?: number | null;
     session_rpm_limit?: number | null;
+    access_group_ids?: string[];
   },
 ) => {
   try {
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 872875cc535..be193647dca 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -23195,6 +23195,8 @@ export interface components {
         };
         /** AgentConfig */
         AgentConfig: {
+            /** Access Group Ids */
+            access_group_ids?: string[] | null;
             agent_card_params: components["schemas"]["AgentCard"];
             /** Agent Name */
             agent_name: string;
@@ -23342,6 +23344,8 @@ export interface components {
         };
         /** AgentResponse */
         AgentResponse: {
+            /** Access Group Ids */
+            access_group_ids?: string[] | null;
             /** Agent Card Params */
             agent_card_params: {
                 [key: string]: unknown;
@@ -34111,6 +34115,8 @@ export interface components {
         };
         /** PatchAgentRequest */
         PatchAgentRequest: {
+            /** Access Group Ids */
+            access_group_ids?: string[] | null;
             agent_card_params?: components["schemas"]["AgentCard"];
             /** Agent Name */
             agent_name?: string;

From 433c804a6cfec9c6674872857f96b435f69f4d96 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Thu, 17 Sep 2026 19:32:39 +0000
Subject: [PATCH 005/160] refactor(agents): mark Callable loader aliases for
 the type discipline gate

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/agent_endpoints/auth/agent_access_groups.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/litellm/proxy/agent_endpoints/auth/agent_access_groups.py b/litellm/proxy/agent_endpoints/auth/agent_access_groups.py
index 4a579de679e..f5adc897e11 100644
--- a/litellm/proxy/agent_endpoints/auth/agent_access_groups.py
+++ b/litellm/proxy/agent_endpoints/auth/agent_access_groups.py
@@ -18,8 +18,9 @@ from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import LiteLLM_AccessGroupTable
 from litellm.types.agents import AgentResponse
 
-AgentLoader: TypeAlias = Callable[[str], Awaitable[AgentResponse | None]]
-AccessGroupLoader: TypeAlias = Callable[[str], Awaitable[LiteLLM_AccessGroupTable | None]]
+AgentLoader: TypeAlias = Callable[[str], Awaitable[AgentResponse | None]]  # mutable-ok: Callable parameter syntax
+LoadedAccessGroup: TypeAlias = LiteLLM_AccessGroupTable | None
+AccessGroupLoader: TypeAlias = Callable[[str], Awaitable[LoadedAccessGroup]]  # mutable-ok: Callable parameter syntax
 
 
 @dataclass(frozen=True, slots=True)
@@ -38,7 +39,7 @@ async def _load_agent(agent_id: str) -> AgentResponse | None:
     return await get_agent_with_read_through(agent_id)
 
 
-async def _load_access_group(access_group_id: str) -> LiteLLM_AccessGroupTable | None:
+async def _load_access_group(access_group_id: str) -> LoadedAccessGroup:
     from litellm.proxy.auth.auth_checks import get_access_object
     from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
 

From 322262db01f9ba69b1a77ab2ec26268c468a3099 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Thu, 17 Sep 2026 19:42:56 +0000
Subject: [PATCH 006/160] style(ui): format add_agent_form test with prettier

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../agents/_components/add_agent_form.test.tsx         | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.test.tsx
index b5301d0e6a0..fbf5cf8c1fb 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.test.tsx
@@ -154,10 +154,12 @@ describe("AddAgentForm logos", () => {
 
   it("includes selected access groups in the create payload", async () => {
     const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never });
-    vi.mocked(networking.createAgentCall).mockReset().mockResolvedValue({
-      agent_id: "agent-1",
-      agent_name: "Test Agent",
-    } as never);
+    vi.mocked(networking.createAgentCall)
+      .mockReset()
+      .mockResolvedValue({
+        agent_id: "agent-1",
+        agent_name: "Test Agent",
+      } as never);
     vi.mocked(networking.keyListCall).mockResolvedValue({ keys: [] });
 
     renderForm();

From d743e08432ee738355a144046b57321485401fea Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Thu, 17 Sep 2026 20:04:23 +0000
Subject: [PATCH 007/160] test(agents): inject the access group ceiling
 resolver instead of patching it

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../mcp_server/auth/user_api_key_auth_mcp.py  |  47 ++++---
 .../auth/agent_access_groups.py               |   3 +
 .../auth/agent_permission_handler.py          |  15 ++-
 litellm/proxy/auth/auth_checks.py             |   9 +-
 .../auth/test_user_api_key_auth_mcp.py        | 109 ++++++++--------
 .../auth/test_agent_permission_handler.py     | 118 ++++++++----------
 .../proxy/auth/test_auth_checks.py            |  93 ++++++--------
 7 files changed, 193 insertions(+), 201 deletions(-)

diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
index bbb3d30864f..05661584a6b 100644
--- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
+++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
@@ -44,6 +44,10 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
     user_api_key_has_admin_view,
 )
+from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
+    CeilingResolver,
+    resolve_agent_access_group_ceiling,
+)
 from litellm.proxy.auth.ip_address_utils import IPAddressUtils
 from litellm.proxy.auth.user_api_key_auth import (
     _get_bearer_token_or_received_api_key,  # pyright: ignore[reportPrivateUsage]  # shared x-litellm-api-key parser lives with user_api_key_auth
@@ -184,6 +188,24 @@ def _has_client_supplied_mcp_auth(
     return bool(mcp_auth_header) or bool(mcp_server_auth_headers)
 
 
+def _agent_capped_servers(
+    allowed_mcp_servers: Sequence[str],
+    agent_servers: Sequence[str],
+    agent_access_group_servers: frozenset[str] | None,
+) -> tuple[str, ...] | None:
+    """Servers left once the agent's object_permission and attached access groups both cap the
+    key/team result, or None when the agent restricts nothing. An attached group set naming no
+    server is an empty ceiling, not an absent one, so it denies every server."""
+    if not agent_servers and agent_access_group_servers is None:
+        return None
+    return tuple(
+        s
+        for s in allowed_mcp_servers
+        if (not agent_servers or s in agent_servers)
+        and (agent_access_group_servers is None or s in agent_access_group_servers)
+    )
+
+
 def _is_mcp_admitted_user_subject(user_api_key_auth: UserAPIKeyAuth | None) -> bool:
     """True when this auth is a keyless subject admitted by the gateway session / bridge user
     path, as opposed to a JWT or other keyless auth that merely lacks a ``team_id``.
@@ -1546,21 +1568,14 @@ class MCPRequestHandler:
             # Check agent permissions if agent_id is set on the key
             #########################################################
             if user_api_key_auth and user_api_key_auth.agent_id:
-                allowed_mcp_servers_for_agent: Final = await MCPRequestHandler._get_allowed_mcp_servers_for_agent(
-                    user_api_key_auth
+                agent_capped: Final = _agent_capped_servers(
+                    allowed_mcp_servers,
+                    await MCPRequestHandler._get_allowed_mcp_servers_for_agent(user_api_key_auth),
+                    await MCPRequestHandler._get_agent_access_group_server_ceiling(user_api_key_auth),
                 )
-                agent_access_group_servers: Final = await MCPRequestHandler._get_agent_access_group_server_ceiling(
-                    user_api_key_auth
-                )
-                if len(allowed_mcp_servers_for_agent) > 0 or agent_access_group_servers is not None:
+                if agent_capped is not None:
                     has_lower_level_mcp_restrictions = True
-                    # Intersect: agent can only use servers allowed by key/team AND agent config AND agent access groups
-                    allowed_mcp_servers = [
-                        s
-                        for s in allowed_mcp_servers
-                        if (len(allowed_mcp_servers_for_agent) == 0 or s in allowed_mcp_servers_for_agent)
-                        and (agent_access_group_servers is None or s in agent_access_group_servers)
-                    ]
+                    allowed_mcp_servers = list(agent_capped)
                     verbose_logger.debug(
                         "Applied agent intersection filter. Final allowed servers: %s", allowed_mcp_servers
                     )
@@ -3148,6 +3163,7 @@ class MCPRequestHandler:
     @staticmethod
     async def _get_agent_access_group_server_ceiling(
         user_api_key_auth: UserAPIKeyAuth,
+        resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling,
     ) -> frozenset[str] | None:
         """
         Server IDs the agent's attached unified access groups (``LiteLLM_AgentsTable.access_group_ids``)
@@ -3157,13 +3173,10 @@ class MCPRequestHandler:
         from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
             global_mcp_server_manager,
         )
-        from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
-            resolve_agent_access_group_ceiling,
-        )
 
         if not user_api_key_auth.agent_id:
             return None
-        ceiling: Final = await resolve_agent_access_group_ceiling(user_api_key_auth.agent_id)
+        ceiling: Final = await resolve_ceiling(user_api_key_auth.agent_id)
         if ceiling is None:
             return None
         return frozenset(global_mcp_server_manager.expand_permission_list(sorted(ceiling.mcp_server_ids)))
diff --git a/litellm/proxy/agent_endpoints/auth/agent_access_groups.py b/litellm/proxy/agent_endpoints/auth/agent_access_groups.py
index f5adc897e11..67bb43638e0 100644
--- a/litellm/proxy/agent_endpoints/auth/agent_access_groups.py
+++ b/litellm/proxy/agent_endpoints/auth/agent_access_groups.py
@@ -33,6 +33,9 @@ class AgentAccessGroupCeiling:
     agent_ids: frozenset[str]
 
 
+CeilingResolver: TypeAlias = Callable[[str], Awaitable[AgentAccessGroupCeiling | None]]  # mutable-ok: Callable params
+
+
 async def _load_agent(agent_id: str) -> AgentResponse | None:
     from litellm.proxy.common_utils.registry_read_through import get_agent_with_read_through
 
diff --git a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
index 11d2a68072c..1759090a29a 100644
--- a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
+++ b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
@@ -19,6 +19,10 @@ from litellm.proxy._types import (
     LitellmUserRoles,
     UserAPIKeyAuth,
 )
+from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
+    CeilingResolver,
+    resolve_agent_access_group_ceiling,
+)
 from litellm.repositories.table_repositories import AgentsRepository
 from litellm.types.agents import AgentResponse
 
@@ -61,6 +65,7 @@ class AgentRequestHandler:
     @staticmethod
     async def resolve_agent_access(
         user_api_key_auth: UserAPIKeyAuth | None = None,
+        resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling,
     ) -> AgentAccess:
         """
         Resolve the agents the given user/key may reach.
@@ -71,7 +76,7 @@ class AgentRequestHandler:
         never widen what it reaches.
         """
         key_team_access: Final = await AgentRequestHandler._resolve_key_team_agent_access(user_api_key_auth)
-        agent_ceiling: Final = await AgentRequestHandler._agent_access_group_ceiling(user_api_key_auth)
+        agent_ceiling: Final = await AgentRequestHandler._agent_access_group_ceiling(user_api_key_auth, resolve_ceiling)
         if agent_ceiling is None:
             return key_team_access
         match key_team_access:
@@ -104,13 +109,12 @@ class AgentRequestHandler:
     @staticmethod
     async def _agent_access_group_ceiling(
         user_api_key_auth: UserAPIKeyAuth | None,
+        resolve_ceiling: CeilingResolver,
     ) -> frozenset[str] | None:
         """Stable IDs of the agents the calling agent's attached access groups allow; None when none attached."""
-        from litellm.proxy.agent_endpoints.auth.agent_access_groups import resolve_agent_access_group_ceiling
-
         if user_api_key_auth is None or not user_api_key_auth.agent_id:
             return None
-        ceiling: Final = await resolve_agent_access_group_ceiling(user_api_key_auth.agent_id)
+        ceiling: Final = await resolve_ceiling(user_api_key_auth.agent_id)
         if ceiling is None:
             return None
         return _to_stable_ids(ceiling.agent_ids)
@@ -119,6 +123,7 @@ class AgentRequestHandler:
     async def is_agent_allowed(
         agent_id: str,
         user_api_key_auth: UserAPIKeyAuth | None = None,
+        resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling,
     ) -> bool:
         """
         Check if a specific agent is allowed for the given user/key.
@@ -132,7 +137,7 @@ class AgentRequestHandler:
         """
         from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
 
-        match await AgentRequestHandler.resolve_agent_access(user_api_key_auth):
+        match await AgentRequestHandler.resolve_agent_access(user_api_key_auth, resolve_ceiling):
             case UnrestrictedAgentAccess():
                 return True
             case RestrictedAgentAccess(allowed_agent_ids):
diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py
index 5c53ee49717..137cf849389 100644
--- a/litellm/proxy/auth/auth_checks.py
+++ b/litellm/proxy/auth/auth_checks.py
@@ -68,6 +68,10 @@ from litellm.proxy._types import (
     SpecialModelNames,
     UserAPIKeyAuth,
 )
+from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
+    CeilingResolver,
+    resolve_agent_access_group_ceiling,
+)
 from litellm.proxy.auth.budget_throttle import (
     budget_throttle_percentage,
     should_throttle_budget_exceeded,
@@ -4199,15 +4203,14 @@ async def _check_agent_access_group_model_access(
     model: str | list[str] | None,  # mutable-ok: _can_object_call_model and the client message helper take list[str]
     valid_token: UserAPIKeyAuth | None,
     llm_router: Router | None,
+    resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling,
 ) -> Literal[True]:
     """Raises when the key's agent has access groups attached and none of them names the model.
     Attached groups that name no model deny every model; ``_can_object_call_model`` would read
     an empty allowlist as unrestricted."""
-    from litellm.proxy.agent_endpoints.auth.agent_access_groups import resolve_agent_access_group_ceiling
-
     if not model or valid_token is None or not valid_token.agent_id:
         return True
-    ceiling: Final = await resolve_agent_access_group_ceiling(valid_token.agent_id)
+    ceiling: Final = await resolve_ceiling(valid_token.agent_id)
     if ceiling is None:
         return True
     if not ceiling.models:
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py
index 2f8e3d1cb82..f8e648fd383 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py
@@ -12,6 +12,7 @@ from starlette.datastructures import Headers
 from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
     MCPRequestHandler,
     UnloadableEntitlementError,
+    _agent_capped_servers,
     _is_mcp_admitted_user_subject,
 )
 from litellm.proxy._types import (
@@ -4169,6 +4170,27 @@ async def test_get_allowed_mcp_servers_for_key_prefers_in_memory_permission():
         global_mcp_server_manager.registry.pop("direct-server", None)
 
 
+@pytest.mark.parametrize(
+    ("agent_servers", "group_ceiling", "expected"),
+    [
+        ([], frozenset({"server_1"}), ("server_1",)),
+        ([], frozenset({"server_1", "server_2", "server_3"}), ("server_1", "server_2")),
+        ([], frozenset(), ()),
+        (["server_2"], frozenset({"server_1", "server_2"}), ("server_2",)),
+        (["server_1"], frozenset({"server_2"}), ()),
+        (["server_1"], None, ("server_1",)),
+    ],
+)
+def test_agent_capped_servers_intersects_agent_config_and_access_groups(agent_servers, group_ceiling, expected):
+    """The agent's attached access groups cap the key/team servers alongside its own
+    object_permission; groups naming no server deny all."""
+    assert _agent_capped_servers(["server_1", "server_2"], agent_servers, group_ceiling) == expected
+
+
+def test_agent_capped_servers_without_agent_restrictions_is_uncapped():
+    assert _agent_capped_servers(["server_1", "server_2"], [], None) is None
+
+
 @pytest.mark.asyncio
 class TestAgentMCPPermissions:
     """Test agent-level MCP server and tool permission intersection."""
@@ -4208,64 +4230,45 @@ class TestAgentMCPPermissions:
                     assert sorted(result) == ["server_1", "server_2"]
                     mock_agent.assert_called_once_with(user_api_key_auth)
 
-    @pytest.mark.parametrize(
-        ("group_ceiling", "expected"),
-        [
-            (frozenset({"server_1"}), ["server_1"]),
-            (frozenset({"server_1", "server_2", "server_3"}), ["server_1", "server_2"]),
-            (frozenset(), []),
-        ],
-    )
-    async def test_get_allowed_mcp_servers_agent_access_group_ceiling(self, group_ceiling, expected):
-        """The agent's attached access groups cap the key/team servers; groups naming no server deny all."""
-        user_api_key_auth = UserAPIKeyAuth(api_key="test-key", user_id="test-user", agent_id="agent-ag")
-        with (
-            patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_key", return_value=["server_1", "server_2"]),
-            patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_team", return_value=[]),
-            patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_agent", return_value=[]),
-            patch.object(MCPRequestHandler, "_get_agent_access_group_server_ceiling", return_value=group_ceiling),
-        ):
-            access = await MCPRequestHandler.get_mcp_server_access(user_api_key_auth=user_api_key_auth)
-            assert sorted(access.server_ids) == expected
-            assert access.scope == "scoped"
-
-    async def test_get_allowed_mcp_servers_agent_without_access_groups_is_uncapped(self):
-        user_api_key_auth = UserAPIKeyAuth(api_key="test-key", user_id="test-user", agent_id="agent-ag")
-        with (
-            patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_key", return_value=["server_1", "server_2"]),
-            patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_team", return_value=[]),
-            patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_agent", return_value=[]),
-            patch.object(MCPRequestHandler, "_get_agent_access_group_server_ceiling", return_value=None),
-        ):
-            result = await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth=user_api_key_auth)
-            assert sorted(result) == ["server_1", "server_2"]
-
     async def test_agent_access_group_server_ceiling_expands_group_servers(self):
+        from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager
         from litellm.proxy.agent_endpoints.auth.agent_access_groups import AgentAccessGroupCeiling
+        from litellm.types.mcp import MCPTransport
+        from litellm.types.mcp_server.mcp_server_manager import MCPServer
 
-        ceiling = AgentAccessGroupCeiling(
-            access_group_ids=("ag-1",),
-            models=frozenset(),
-            mcp_server_ids=frozenset({"server_1"}),
-            agent_ids=frozenset(),
-        )
-        with (
-            patch(
-                "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
-                new=AsyncMock(return_value=ceiling),
-            ),
-            patch(
-                "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager"
-            ) as mock_manager,
-        ):
-            mock_manager.expand_permission_list.return_value = ["server_1"]
-            result = await MCPRequestHandler._get_agent_access_group_server_ceiling(
-                UserAPIKeyAuth(api_key="test-key", agent_id="agent-ag")
+        asked: list[str] = []
+
+        async def resolve(agent_id: str) -> AgentAccessGroupCeiling | None:
+            asked.append(agent_id)
+            return AgentAccessGroupCeiling(
+                access_group_ids=("ag-1",),
+                models=frozenset(),
+                mcp_server_ids=frozenset({"aliased-server"}),
+                agent_ids=frozenset(),
             )
-            assert result == frozenset({"server_1"})
-            mock_manager.expand_permission_list.assert_called_once_with(["server_1"])
 
-        assert await MCPRequestHandler._get_agent_access_group_server_ceiling(UserAPIKeyAuth(api_key="k")) is None
+        global_mcp_server_manager.registry["ag-server-id"] = MCPServer(
+            server_id="ag-server-id",
+            name="ag-server",
+            server_name="ag-server",
+            alias="aliased-server",
+            url="https://ag-server.example.com",
+            transport=MCPTransport.http,
+        )
+        try:
+            result = await MCPRequestHandler._get_agent_access_group_server_ceiling(
+                UserAPIKeyAuth(api_key="test-key", agent_id="agent-ag"), resolve
+            )
+        finally:
+            global_mcp_server_manager.registry.pop("ag-server-id", None)
+
+        assert result == frozenset({"ag-server-id"})
+        assert asked == ["agent-ag"]
+        assert (
+            await MCPRequestHandler._get_agent_access_group_server_ceiling(UserAPIKeyAuth(api_key="k"), resolve)
+            is None
+        )
+        assert asked == ["agent-ag"]
 
     async def test_get_allowed_mcp_servers_key_team_agent_intersection(self):
         """Key allows [1, 2], agent allows [2, 3]. Result = [2]."""
diff --git a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py
index a8a55d332b6..2a98e6e4feb 100644
--- a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py
+++ b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py
@@ -11,9 +11,9 @@ import pytest
 
 
 from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
-from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
+from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LitellmUserRoles, UserAPIKeyAuth
 from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry
-from litellm.proxy.agent_endpoints.auth.agent_access_groups import AgentAccessGroupCeiling
+from litellm.proxy.agent_endpoints.auth.agent_access_groups import AgentAccessGroupCeiling, CeilingResolver
 from litellm.proxy.agent_endpoints.auth.agent_permission_handler import (
     AgentAccess,
     AgentRequestHandler,
@@ -159,86 +159,74 @@ class TestAgentRequestHandler:
                     ), agent_id
 
     @staticmethod
-    def _ceiling(agent_ids: frozenset[str]) -> AgentAccessGroupCeiling:
-        return AgentAccessGroupCeiling(
-            access_group_ids=("ag-1",),
-            models=frozenset(),
-            mcp_server_ids=frozenset(),
-            agent_ids=agent_ids,
+    def _ceiling_resolver(agent_ids: frozenset[str] | None) -> tuple[CeilingResolver, list[str]]:
+        """A resolver that records the agent ids it was asked about and answers with a fixed
+        ceiling, or None when the agent has no access groups attached."""
+        asked: Final[list[str]] = []
+
+        async def resolve(agent_id: str) -> AgentAccessGroupCeiling | None:
+            asked.append(agent_id)
+            if agent_ids is None:
+                return None
+            return AgentAccessGroupCeiling(
+                access_group_ids=("ag-1",), models=frozenset(), mcp_server_ids=frozenset(), agent_ids=agent_ids
+            )
+
+        return resolve, asked
+
+    @staticmethod
+    def _key_granting(agent_ids: list[str], agent_id: str | None) -> UserAPIKeyAuth:
+        return UserAPIKeyAuth(
+            api_key="test-key",
+            user_id="test-user",
+            agent_id=agent_id,
+            object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="obj-1", agents=agent_ids),
         )
 
     async def test_agent_access_groups_cap_an_otherwise_unrestricted_key(self):
         """A key with no agent grant of its own may still only reach the agents its
         agent's attached access groups name."""
         agent_key: Final = UserAPIKeyAuth(api_key="test-key", user_id="test-user", agent_id="caller-agent")
+        resolve, asked = self._ceiling_resolver(frozenset({"agent-beta"}))
 
-        with (
-            patch.object(AgentRequestHandler, "_get_allowed_agents_for_key", return_value=UnrestrictedAgentAccess()),
-            patch.object(AgentRequestHandler, "_get_allowed_agents_for_team", return_value=UnrestrictedAgentAccess()),
-            patch(
-                "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
-                new=AsyncMock(return_value=self._ceiling(frozenset({"agent-beta"}))),
-            ) as mock_ceiling,
-        ):
-            assert await AgentRequestHandler.resolve_agent_access(agent_key) == RestrictedAgentAccess(
-                frozenset({"agent-beta"})
-            )
-            assert await AgentRequestHandler.is_agent_allowed("agent-beta", agent_key) is True
-            assert await AgentRequestHandler.is_agent_allowed("agent-alpha", agent_key) is False
-            mock_ceiling.assert_called_with("caller-agent")
-
-    async def test_agent_access_groups_intersect_with_key_and_team_grants(self):
-        agent_key: Final = UserAPIKeyAuth(
-            api_key="test-key", user_id="test-user", team_id="test-team", agent_id="caller-agent"
+        assert await AgentRequestHandler.resolve_agent_access(agent_key, resolve) == RestrictedAgentAccess(
+            frozenset({"agent-beta"})
         )
+        assert await AgentRequestHandler.is_agent_allowed("agent-beta", agent_key, resolve) is True
+        assert await AgentRequestHandler.is_agent_allowed("agent-alpha", agent_key, resolve) is False
+        assert asked == ["caller-agent"] * 3
 
-        with (
-            patch.object(
-                AgentRequestHandler,
-                "_get_allowed_agents_for_key",
-                return_value=RestrictedAgentAccess(frozenset({"agent-alpha", "agent-beta"})),
-            ),
-            patch.object(
-                AgentRequestHandler,
-                "_get_allowed_agents_for_team",
-                return_value=RestrictedAgentAccess(frozenset({"agent-alpha", "agent-beta", "agent-gamma"})),
-            ),
-            patch(
-                "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
-                new=AsyncMock(return_value=self._ceiling(frozenset({"agent-beta", "agent-gamma"}))),
-            ),
-        ):
-            assert await AgentRequestHandler.resolve_agent_access(agent_key) == RestrictedAgentAccess(
-                frozenset({"agent-beta"})
-            )
+    async def test_agent_access_groups_intersect_with_key_grants(self):
+        agent_key: Final = self._key_granting(["agent-alpha", "agent-beta"], agent_id="caller-agent")
+        resolve, _ = self._ceiling_resolver(frozenset({"agent-beta", "agent-gamma"}))
+
+        assert await AgentRequestHandler.resolve_agent_access(agent_key, resolve) == RestrictedAgentAccess(
+            frozenset({"agent-beta"})
+        )
+        assert await AgentRequestHandler.is_agent_allowed("agent-gamma", agent_key, resolve) is False
 
     async def test_agent_access_groups_naming_no_agent_deny_every_agent(self):
         agent_key: Final = UserAPIKeyAuth(api_key="test-key", user_id="test-user", agent_id="caller-agent")
+        resolve, _ = self._ceiling_resolver(frozenset())
 
-        with (
-            patch.object(AgentRequestHandler, "_get_allowed_agents_for_key", return_value=UnrestrictedAgentAccess()),
-            patch.object(AgentRequestHandler, "_get_allowed_agents_for_team", return_value=UnrestrictedAgentAccess()),
-            patch(
-                "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
-                new=AsyncMock(return_value=self._ceiling(frozenset())),
-            ),
-        ):
-            assert await AgentRequestHandler.resolve_agent_access(agent_key) == RestrictedAgentAccess(frozenset())
-            assert await AgentRequestHandler.is_agent_allowed("agent-alpha", agent_key) is False
+        assert await AgentRequestHandler.resolve_agent_access(agent_key, resolve) == RestrictedAgentAccess(frozenset())
+        assert await AgentRequestHandler.is_agent_allowed("agent-alpha", agent_key, resolve) is False
+
+    async def test_agent_without_access_groups_keeps_key_grants(self):
+        agent_key: Final = self._key_granting(["agent-alpha"], agent_id="caller-agent")
+        resolve, asked = self._ceiling_resolver(None)
+
+        assert await AgentRequestHandler.resolve_agent_access(agent_key, resolve) == RestrictedAgentAccess(
+            frozenset({"agent-alpha"})
+        )
+        assert asked == ["caller-agent"]
 
     async def test_key_without_agent_never_consults_agent_access_groups(self):
         plain_key: Final = UserAPIKeyAuth(api_key="test-key", user_id="test-user")
+        resolve, asked = self._ceiling_resolver(frozenset())
 
-        with (
-            patch.object(AgentRequestHandler, "_get_allowed_agents_for_key", return_value=UnrestrictedAgentAccess()),
-            patch.object(AgentRequestHandler, "_get_allowed_agents_for_team", return_value=UnrestrictedAgentAccess()),
-            patch(
-                "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
-                new=AsyncMock(return_value=self._ceiling(frozenset())),
-            ) as mock_ceiling,
-        ):
-            assert await AgentRequestHandler.resolve_agent_access(plain_key) == UnrestrictedAgentAccess()
-            mock_ceiling.assert_not_called()
+        assert await AgentRequestHandler.resolve_agent_access(plain_key, resolve) == UnrestrictedAgentAccess()
+        assert asked == []
 
     async def test_empty_access_group_denies_every_agent(self):
         """LIT-5143: a key restricted to an access group that resolves to no agents is
diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py
index ad1742db4b7..b9b9a786d93 100644
--- a/tests/test_litellm/proxy/auth/test_auth_checks.py
+++ b/tests/test_litellm/proxy/auth/test_auth_checks.py
@@ -32,11 +32,13 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
     WebhookEvent,
 )
+from litellm.proxy.agent_endpoints.auth.agent_access_groups import AgentAccessGroupCeiling, CeilingResolver
 from litellm.proxy.auth.auth_checks import (
     ExperimentalUIJWTToken,
     _cache_management_object,
     _can_object_call_model,
     _can_object_call_vector_stores,
+    _check_agent_access_group_model_access,
     _check_end_user_budget,
     _check_team_member_budget,
     _fetch_key_object_from_db_with_reconnect,
@@ -8466,89 +8468,64 @@ def test_request_skips_budget_checks_extends_route_rule_with_zero_cost_models()
 # Agent access group model ceiling
 
 
-def _agent_model_ceiling(models: frozenset[str]):
-    from litellm.proxy.agent_endpoints.auth.agent_access_groups import AgentAccessGroupCeiling
+def _agent_model_ceiling_resolver(
+    models: frozenset[str] | None,
+) -> tuple[CeilingResolver, list[str]]:
+    """Resolver that records the agent ids it was asked about and answers with a fixed model
+    ceiling, or None when the agent has no access groups attached."""
+    asked: Final[list[str]] = []
 
-    return AgentAccessGroupCeiling(
-        access_group_ids=("ag-1",), models=models, mcp_server_ids=frozenset(), agent_ids=frozenset()
-    )
+    async def resolve(agent_id: str) -> AgentAccessGroupCeiling | None:
+        asked.append(agent_id)
+        if models is None:
+            return None
+        return AgentAccessGroupCeiling(
+            access_group_ids=("ag-1",), models=models, mcp_server_ids=frozenset(), agent_ids=frozenset()
+        )
 
-
-async def _run_common_checks_for_agent_key(model: str, valid_token: UserAPIKeyAuth):
-    from fastapi import Request
-
-    from litellm.proxy.auth.auth_checks import common_checks
-
-    return await common_checks(
-        request_body={"model": model, "messages": [{"role": "user", "content": "hi"}]},
-        team_object=None,
-        user_object=None,
-        end_user_object=None,
-        global_proxy_spend=None,
-        general_settings={},
-        route="/chat/completions",
-        llm_router=None,
-        proxy_logging_obj=MagicMock(),
-        valid_token=valid_token,
-        request=MagicMock(spec=Request),
-    )
+    return resolve, asked
 
 
 @pytest.mark.asyncio
-async def test_common_checks_agent_access_groups_cap_models_even_when_key_allows_them():
+async def test_agent_access_groups_cap_models_even_when_key_allows_them():
     agent_key: Final = UserAPIKeyAuth(token="agent-token", agent_id="agent-1", models=["gpt-5", "claude-sonnet"])
+    resolve, asked = _agent_model_ceiling_resolver(frozenset({"gpt-5"}))
 
-    with patch(
-        "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
-        new=AsyncMock(return_value=_agent_model_ceiling(frozenset({"gpt-5"}))),
-    ):
-        assert await _run_common_checks_for_agent_key("gpt-5", agent_key) is True
+    assert await _check_agent_access_group_model_access("gpt-5", agent_key, None, resolve) is True
 
-        with pytest.raises(ProxyException) as exc_info:
-            await _run_common_checks_for_agent_key("claude-sonnet", agent_key)
+    with pytest.raises(ProxyException) as exc_info:
+        await _check_agent_access_group_model_access("claude-sonnet", agent_key, None, resolve)
 
     assert exc_info.value.type == ProxyErrorTypes.agent_model_access_denied
     assert exc_info.value.code == str(status.HTTP_403_FORBIDDEN)
+    assert asked == ["agent-1", "agent-1"]
 
 
 @pytest.mark.asyncio
-async def test_common_checks_agent_access_groups_naming_no_model_deny_every_model():
+async def test_agent_access_groups_naming_no_model_deny_every_model():
     agent_key: Final = UserAPIKeyAuth(token="agent-token", agent_id="agent-1", models=[])
+    resolve, _ = _agent_model_ceiling_resolver(frozenset())
 
-    with (
-        patch(
-            "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
-            new=AsyncMock(return_value=_agent_model_ceiling(frozenset())),
-        ),
-        pytest.raises(ProxyException) as exc_info,
-    ):
-        await _run_common_checks_for_agent_key("gpt-5", agent_key)
+    with pytest.raises(ProxyException) as exc_info:
+        await _check_agent_access_group_model_access("gpt-5", agent_key, None, resolve)
 
     assert exc_info.value.type == ProxyErrorTypes.agent_model_access_denied
 
 
 @pytest.mark.asyncio
-async def test_common_checks_agent_without_access_groups_adds_no_model_ceiling():
+async def test_agent_without_access_groups_adds_no_model_ceiling():
     agent_key: Final = UserAPIKeyAuth(token="agent-token", agent_id="agent-1", models=["gpt-5", "claude-sonnet"])
+    resolve, asked = _agent_model_ceiling_resolver(None)
 
-    with patch(
-        "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
-        new=AsyncMock(return_value=None),
-    ) as mock_ceiling:
-        assert await _run_common_checks_for_agent_key("gpt-5", agent_key) is True
-        assert await _run_common_checks_for_agent_key("claude-sonnet", agent_key) is True
-
-    mock_ceiling.assert_called_with("agent-1")
+    assert await _check_agent_access_group_model_access("gpt-5", agent_key, None, resolve) is True
+    assert await _check_agent_access_group_model_access("claude-sonnet", agent_key, None, resolve) is True
+    assert asked == ["agent-1", "agent-1"]
 
 
 @pytest.mark.asyncio
-async def test_common_checks_key_without_agent_never_consults_agent_access_groups():
+async def test_key_without_agent_never_consults_agent_access_groups():
     plain_key: Final = UserAPIKeyAuth(token="plain-token", models=["gpt-5"])
+    resolve, asked = _agent_model_ceiling_resolver(frozenset())
 
-    with patch(
-        "litellm.proxy.agent_endpoints.auth.agent_access_groups.resolve_agent_access_group_ceiling",
-        new=AsyncMock(return_value=_agent_model_ceiling(frozenset())),
-    ) as mock_ceiling:
-        assert await _run_common_checks_for_agent_key("gpt-5", plain_key) is True
-
-    mock_ceiling.assert_not_called()
+    assert await _check_agent_access_group_model_access("gpt-5", plain_key, None, resolve) is True
+    assert asked == []

From 5b04560997e5838743d852dd6af97749e5979ff9 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Thu, 17 Sep 2026 20:30:37 +0000
Subject: [PATCH 008/160] refactor(agents): keep the model listing cap within
 the type-discipline budget

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../auth/agent_access_groups.py               | 104 ++++++++++++++----
 litellm/proxy/utils.py                        |  60 +++++++++-
 2 files changed, 144 insertions(+), 20 deletions(-)

diff --git a/litellm/proxy/agent_endpoints/auth/agent_access_groups.py b/litellm/proxy/agent_endpoints/auth/agent_access_groups.py
index 67bb43638e0..e0e5193d0a4 100644
--- a/litellm/proxy/agent_endpoints/auth/agent_access_groups.py
+++ b/litellm/proxy/agent_endpoints/auth/agent_access_groups.py
@@ -1,33 +1,41 @@
-"""
-Ceiling that an agent's attached access groups place on requests made with that agent's key.
-
-Keys and teams use access groups as grants. An agent uses them the way it already uses its
-``object_permission``: the union of the attached groups caps what the agent's key can reach,
-on top of whatever the key and team allow. A group that cannot be loaded contributes nothing,
-so a missing or unreadable group can only narrow the agent, never widen it.
-"""
-
 import asyncio
-from collections.abc import Awaitable, Callable
+from collections.abc import Awaitable, Callable, Sequence
 from dataclasses import dataclass
-from typing import Final, TypeAlias
+from typing import Final, Protocol, TypeAlias
 
 from fastapi import HTTPException
+from pydantic import TypeAdapter, ValidationError
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
+from litellm.caching.dual_cache import DualCache
 from litellm.proxy._types import LiteLLM_AccessGroupTable
-from litellm.types.agents import AgentResponse
+from litellm.proxy.common_utils.user_api_key_cache import get_management_object_ttl
 
-AgentLoader: TypeAlias = Callable[[str], Awaitable[AgentResponse | None]]  # mutable-ok: Callable parameter syntax
+
+class _AgentAccessGroupsRecord(Protocol):
+    @property
+    def access_group_ids(self) -> Sequence[str] | None: ...
+
+
+class _AgentIdWhere(TypedDict):
+    agent_id: ReadOnly[str]
+
+
+AccessGroupIds: TypeAlias = tuple[str, ...]
+AccessGroupIdsLoader: TypeAlias = Callable[[str], Awaitable[AccessGroupIds]]  # mutable-ok: Callable params
+AgentRecordFinder: TypeAlias = Callable[[str], Awaitable[_AgentAccessGroupsRecord | None]]  # mutable-ok: Callable
 LoadedAccessGroup: TypeAlias = LiteLLM_AccessGroupTable | None
 AccessGroupLoader: TypeAlias = Callable[[str], Awaitable[LoadedAccessGroup]]  # mutable-ok: Callable parameter syntax
 
+_CACHED_IDS: Final = TypeAdapter(list[str])
+
 
 @dataclass(frozen=True, slots=True)
 class AgentAccessGroupCeiling:
     """Everything the agent's attached access groups allow. An empty set denies that resource kind."""
 
-    access_group_ids: tuple[str, ...]
+    access_group_ids: AccessGroupIds
     models: frozenset[str]
     mcp_server_ids: frozenset[str]
     agent_ids: frozenset[str]
@@ -36,10 +44,69 @@ class AgentAccessGroupCeiling:
 CeilingResolver: TypeAlias = Callable[[str], Awaitable[AgentAccessGroupCeiling | None]]  # mutable-ok: Callable params
 
 
-async def _load_agent(agent_id: str) -> AgentResponse | None:
+def agent_access_group_ids_cache_key(agent_id: str) -> str:
+    return f"agent_access_group_ids:{agent_id}"
+
+
+def _cached_access_group_ids(cached: object) -> AccessGroupIds | None:
+    if cached is None:
+        return None
+    try:
+        return tuple(_CACHED_IDS.validate_python(cached))
+    except ValidationError:
+        return None
+
+
+async def _registry_access_group_ids(agent_id: str) -> AccessGroupIds:
     from litellm.proxy.common_utils.registry_read_through import get_agent_with_read_through
 
-    return await get_agent_with_read_through(agent_id)
+    agent: Final = await get_agent_with_read_through(agent_id)
+    return tuple(agent.access_group_ids or ()) if agent is not None else ()
+
+
+async def load_agent_access_group_ids(
+    agent_id: str,
+    cache: DualCache,
+    find_agent: AgentRecordFinder,
+    fallback: AccessGroupIdsLoader,
+) -> AccessGroupIds:
+    """The agent row's groups, cached for the management-object TTL and evicted on every agent write."""
+    cache_key: Final = agent_access_group_ids_cache_key(agent_id)
+    cached: Final = _cached_access_group_ids(await cache.async_get_cache(key=cache_key))
+    if cached is not None:
+        return cached
+    try:
+        record: Final = await find_agent(agent_id)
+    except Exception as e:  # noqa: BLE001  # prisma raises many error types; the registry snapshot answers instead
+        verbose_proxy_logger.warning("Failed to read access groups for agent %r, using registry: %s", agent_id, e)
+        return await fallback(agent_id)
+    access_group_ids: Final = tuple(record.access_group_ids or ()) if record is not None else ()
+    await cache.async_set_cache(key=cache_key, value=access_group_ids, ttl=get_management_object_ttl(cache))
+    return access_group_ids
+
+
+async def _load_agent_access_group_ids(agent_id: str) -> AccessGroupIds:
+    from litellm.proxy.agent_endpoints.agent_registry import agents_table
+    from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
+
+    if prisma_client is None:
+        return await _registry_access_group_ids(agent_id)
+    db: Final = prisma_client
+
+    async def find_agent(row_agent_id: str) -> _AgentAccessGroupsRecord | None:
+        return await agents_table(db).find_unique(where=_AgentIdWhere(agent_id=row_agent_id))
+
+    return await load_agent_access_group_ids(agent_id, user_api_key_cache, find_agent, _registry_access_group_ids)
+
+
+async def evict_agent_access_group_ids(agent_ids: Sequence[str]) -> None:
+    from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
+    from litellm.proxy.proxy_server import user_api_key_cache
+
+    await evict_and_broadcast(
+        cache_keys=tuple(agent_access_group_ids_cache_key(agent_id) for agent_id in agent_ids),
+        user_api_key_cache=user_api_key_cache,
+    )
 
 
 async def _load_access_group(access_group_id: str) -> LoadedAccessGroup:
@@ -65,12 +132,11 @@ async def _load_access_group(access_group_id: str) -> LoadedAccessGroup:
 
 async def resolve_agent_access_group_ceiling(
     agent_id: str,
-    load_agent: AgentLoader = _load_agent,
+    load_access_group_ids: AccessGroupIdsLoader = _load_agent_access_group_ids,
     load_access_group: AccessGroupLoader = _load_access_group,
 ) -> AgentAccessGroupCeiling | None:
     """``None`` when the agent has no access groups attached, so nothing is capped."""
-    agent: Final = await load_agent(agent_id)
-    access_group_ids: Final = tuple(agent.access_group_ids or ()) if agent is not None else ()
+    access_group_ids: Final = await load_access_group_ids(agent_id)
     if not access_group_ids:
         return None
 
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 950ac5e9906..95c221a8f20 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -122,6 +122,7 @@ from litellm.proxy._types import (
     Member,
     UserAPIKeyAuth,
 )
+from litellm.proxy.agent_endpoints.auth.agent_access_groups import CeilingResolver, resolve_agent_access_group_ceiling
 from litellm.proxy.auth.route_checks import RouteChecks
 from litellm.proxy.common_utils.config_sync_pubsub import publish_config_param_change
 from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
@@ -7980,6 +7981,51 @@ async def _get_access_group_models(
     return tuple(dict.fromkeys((*team_group_models, *key_group_models)))
 
 
+async def _agent_access_group_visible_models(
+    user_api_key_dict: "UserAPIKeyAuth",
+    llm_router: Optional["Router"],
+    include_model_access_groups: bool,
+    return_wildcard_routes: bool,
+    team_id: str | None,
+    resolve_agent_ceiling: CeilingResolver,
+) -> frozenset[str] | None:
+    """Models an agent key may still list once its attached access groups cap it, ``None`` when
+    nothing caps it, so ``/v1/models`` never advertises a model the same key would be denied on."""
+    from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models
+
+    if not user_api_key_dict.agent_id:
+        return None
+    ceiling: Final = await resolve_agent_ceiling(user_api_key_dict.agent_id)
+    if ceiling is None:
+        return None
+    if llm_router is None:
+        return ceiling.models
+    proxy_model_list: Final = llm_router.get_model_names()
+    model_access_groups: Final = llm_router.get_model_access_groups()
+    granted: Final = get_team_models(
+        team_models=sorted(ceiling.models),
+        proxy_model_list=proxy_model_list,
+        model_access_groups=model_access_groups,
+        include_model_access_groups=include_model_access_groups,
+    )
+    if not granted:
+        return frozenset()
+    return frozenset(
+        get_complete_model_list(
+            key_models=granted,
+            team_models=(),
+            proxy_model_list=proxy_model_list,
+            user_model=None,
+            infer_model_from_keys=False,
+            return_wildcard_routes=return_wildcard_routes,
+            llm_router=llm_router,
+            model_access_groups=model_access_groups,
+            include_model_access_groups=include_model_access_groups,
+            team_id=team_id,
+        )
+    )
+
+
 async def get_available_models_for_user(
     user_api_key_dict: "UserAPIKeyAuth",
     llm_router: Optional["Router"],
@@ -7992,6 +8038,7 @@ async def get_available_models_for_user(
     only_model_access_groups: bool = False,
     return_wildcard_routes: bool = False,
     user_api_key_cache: Optional["UserApiKeyCache"] = None,
+    resolve_agent_ceiling: CeilingResolver = resolve_agent_access_group_ceiling,
 ) -> list[str]:
     """
     Get the list of models available to a user based on their API key and team permissions.
@@ -8095,7 +8142,18 @@ async def get_available_models_for_user(
         team_id=effective_team_id,
     )
 
-    return all_models
+    agent_visible: Final = await _agent_access_group_visible_models(
+        user_api_key_dict=user_api_key_dict,
+        llm_router=llm_router,
+        include_model_access_groups=include_model_access_groups,
+        return_wildcard_routes=return_wildcard_routes,
+        team_id=effective_team_id,
+        resolve_agent_ceiling=resolve_agent_ceiling,
+    )
+    if agent_visible is None:
+        return all_models
+    capped: Final = [m for m in all_models if m in agent_visible]  # mutable-ok: callers expect the list all_models is
+    return capped
 
 
 def _safe_get_model_info(model: str, get_model_info: Callable[[str], ModelInfo]) -> ModelInfo | None:

From 3a86567c9d3ffcd12507407215ea14d9d89d2184 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Thu, 17 Sep 2026 20:31:03 +0000
Subject: [PATCH 009/160] fix(agents): evict the cached agent access groups on
 every agent write and cap the model listing

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../proxy/agent_endpoints/agent_registry.py   |  3 +
 litellm/proxy/agent_endpoints/endpoints.py    |  4 +
 .../access_group_endpoints.py                 |  2 +
 .../auth/test_agent_access_groups.py          | 91 ++++++++++++++++++-
 .../proxy/utils/helpers/test_model_access.py  | 87 ++++++++++++++++--
 5 files changed, 177 insertions(+), 10 deletions(-)

diff --git a/litellm/proxy/agent_endpoints/agent_registry.py b/litellm/proxy/agent_endpoints/agent_registry.py
index c8948d8d70e..d6b12e830e1 100644
--- a/litellm/proxy/agent_endpoints/agent_registry.py
+++ b/litellm/proxy/agent_endpoints/agent_registry.py
@@ -67,6 +67,9 @@ class AgentRecord(Protocol):
     @property
     def object_permission(self) -> AgentObjectPermissionRecord | None: ...
 
+    @property
+    def access_group_ids(self) -> Sequence[str] | None: ...
+
     @property
     def spend(self) -> float: ...
 
diff --git a/litellm/proxy/agent_endpoints/endpoints.py b/litellm/proxy/agent_endpoints/endpoints.py
index aa8979a73c6..62783fb412a 100644
--- a/litellm/proxy/agent_endpoints/endpoints.py
+++ b/litellm/proxy/agent_endpoints/endpoints.py
@@ -44,6 +44,7 @@ from litellm.proxy.agent_endpoints.agent_search import (
     global_agent_search_index,
     search_agents,
 )
+from litellm.proxy.agent_endpoints.auth.agent_access_groups import evict_agent_access_group_ids
 from litellm.proxy.agent_endpoints.auth.agent_permission_handler import accessible_agents
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils.rbac_utils import check_feature_access_for_user
@@ -696,6 +697,7 @@ async def update_agent(
             prisma_client=prisma_client,
             updated_by=updated_by,
         )
+        await evict_agent_access_group_ids((agent_id,))
 
         # deregister in memory
         AGENT_REGISTRY.deregister_agent(agent_name=existing_agent.get("agent_name"))
@@ -799,6 +801,7 @@ async def patch_agent(
             prisma_client=prisma_client,
             updated_by=updated_by,
         )
+        await evict_agent_access_group_ids((agent_id,))
 
         # deregister in memory
         AGENT_REGISTRY.deregister_agent(agent_name=existing_agent.get("agent_name"))
@@ -861,6 +864,7 @@ async def delete_agent(
             raise HTTPException(status_code=404, detail=f"Agent with ID {agent_id} not found in DB.")
 
         await AGENT_REGISTRY.delete_agent_from_db(agent_id=agent_id, prisma_client=prisma_client)
+        await evict_agent_access_group_ids((agent_id,))
 
         AGENT_REGISTRY.deregister_agent(agent_name=existing_agent.get("agent_name"))
 
diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py
index b4923b0a2dc..2694d00b17f 100644
--- a/litellm/proxy/management_endpoints/access_group_endpoints.py
+++ b/litellm/proxy/management_endpoints/access_group_endpoints.py
@@ -16,6 +16,7 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
 )
 from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
+from litellm.proxy.agent_endpoints.auth.agent_access_groups import evict_agent_access_group_ids
 from litellm.proxy.auth.auth_checks import (
     _cache_access_object,
     _cache_key_object,
@@ -782,6 +783,7 @@ async def delete_access_group(
 
         await invalidate_access_group_cache(access_group_id)
         _detach_access_group_from_agent_registry(detached_agent_ids, access_group_id)
+        await evict_agent_access_group_ids(detached_agent_ids)
         await _patch_team_caches_remove_access_group(
             affected_team_ids, access_group_id, user_api_key_cache, proxy_logging_obj
         )
diff --git a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_access_groups.py b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_access_groups.py
index 8c31c9428e7..266ce52e3a9 100644
--- a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_access_groups.py
+++ b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_access_groups.py
@@ -1,11 +1,16 @@
+from collections.abc import Sequence
+from dataclasses import dataclass
 from typing import Final
 
 import pytest
 from fastapi import HTTPException
 
+from litellm.caching.dual_cache import DualCache
 from litellm.models.access_group import LiteLLM_AccessGroupTable
 from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
     AgentAccessGroupCeiling,
+    agent_access_group_ids_cache_key,
+    load_agent_access_group_ids,
     resolve_agent_access_group_ceiling,
 )
 from litellm.types.agents import AgentResponse
@@ -35,8 +40,8 @@ def _group(
 
 
 def _loaders(agent: AgentResponse | None, groups: dict[str, LiteLLM_AccessGroupTable]):
-    async def load_agent(agent_id: str) -> AgentResponse | None:
-        return agent
+    async def load_agent(agent_id: str) -> tuple[str, ...]:
+        return tuple(agent.access_group_ids or ()) if agent is not None else ()
 
     async def load_group(group_id: str) -> LiteLLM_AccessGroupTable | None:
         return groups.get(group_id)
@@ -105,6 +110,88 @@ async def test_only_unloadable_groups_is_an_empty_ceiling_not_unrestricted():
     assert ceiling.agent_ids == frozenset()
 
 
+@dataclass(frozen=True, slots=True)
+class _AgentRow:
+    access_group_ids: Sequence[str] | None
+
+
+class _FakeAgentTable:
+    def __init__(self, rows: dict[str, _AgentRow], failing: bool = False) -> None:
+        self._rows: Final = rows
+        self._failing: Final = failing
+        self.reads = 0
+
+    async def find_agent(self, agent_id: str) -> _AgentRow | None:
+        self.reads += 1
+        if self._failing:
+            raise RuntimeError("db down")
+        return self._rows.get(agent_id)
+
+
+async def _registry_snapshot(agent_id: str) -> tuple[str, ...]:
+    return ("registry-group",)
+
+
+@pytest.mark.asyncio
+async def test_agent_row_is_read_once_then_served_from_cache():
+    cache: Final = DualCache()
+    table: Final = _FakeAgentTable({"agent-1": _AgentRow(["g1", "g2"])})
+
+    first: Final = await load_agent_access_group_ids("agent-1", cache, table.find_agent, _registry_snapshot)
+    second: Final = await load_agent_access_group_ids("agent-1", cache, table.find_agent, _registry_snapshot)
+
+    assert (first, second, table.reads) == (("g1", "g2"), ("g1", "g2"), 1)
+
+
+@pytest.mark.asyncio
+async def test_agent_with_no_row_or_no_groups_caches_an_empty_answer():
+    cache: Final = DualCache()
+    table: Final = _FakeAgentTable({"bare": _AgentRow(None)})
+
+    bare: Final = await load_agent_access_group_ids("bare", cache, table.find_agent, _registry_snapshot)
+    missing: Final = await load_agent_access_group_ids("missing", cache, table.find_agent, _registry_snapshot)
+    again: Final = await load_agent_access_group_ids("missing", cache, table.find_agent, _registry_snapshot)
+
+    assert (bare, missing, again, table.reads) == ((), (), (), 2)
+
+
+@pytest.mark.asyncio
+async def test_evicted_cache_entry_picks_up_the_patched_row():
+    cache: Final = DualCache()
+    rows: Final = {"agent-1": _AgentRow(["g1"])}
+    table: Final = _FakeAgentTable(rows)
+    await load_agent_access_group_ids("agent-1", cache, table.find_agent, _registry_snapshot)
+
+    rows["agent-1"] = _AgentRow(["g2"])
+    stale: Final = await load_agent_access_group_ids("agent-1", cache, table.find_agent, _registry_snapshot)
+    await cache.async_delete_cache(key=agent_access_group_ids_cache_key("agent-1"))
+    fresh: Final = await load_agent_access_group_ids("agent-1", cache, table.find_agent, _registry_snapshot)
+
+    assert (stale, fresh) == (("g1",), ("g2",))
+
+
+@pytest.mark.asyncio
+async def test_unreadable_row_falls_back_to_the_registry_without_caching():
+    cache: Final = DualCache()
+    table: Final = _FakeAgentTable({}, failing=True)
+
+    answer: Final = await load_agent_access_group_ids("agent-1", cache, table.find_agent, _registry_snapshot)
+
+    assert answer == ("registry-group",)
+    assert await cache.async_get_cache(key=agent_access_group_ids_cache_key("agent-1")) is None
+
+
+@pytest.mark.asyncio
+async def test_garbage_in_the_cache_is_treated_as_a_miss():
+    cache: Final = DualCache()
+    await cache.async_set_cache(key=agent_access_group_ids_cache_key("agent-1"), value={"not": "a list"})
+    table: Final = _FakeAgentTable({"agent-1": _AgentRow(["g1"])})
+
+    answer: Final = await load_agent_access_group_ids("agent-1", cache, table.find_agent, _registry_snapshot)
+
+    assert (answer, table.reads) == (("g1",), 1)
+
+
 @pytest.mark.asyncio
 async def test_default_loader_treats_a_missing_group_as_unreadable(monkeypatch: pytest.MonkeyPatch):
     from litellm.proxy import proxy_server
diff --git a/tests/test_litellm/proxy/utils/helpers/test_model_access.py b/tests/test_litellm/proxy/utils/helpers/test_model_access.py
index 5fb4392eec6..7f77f938323 100644
--- a/tests/test_litellm/proxy/utils/helpers/test_model_access.py
+++ b/tests/test_litellm/proxy/utils/helpers/test_model_access.py
@@ -110,8 +110,7 @@ def test_create_model_info_response_happy_path_no_metadata():
         "owned_by": result["owned_by"],
         "created_is_int": isinstance(result["created"], int),
         "metadata_absent": "metadata" not in result,
-        "max_input_tokens_positive_int": isinstance(result["max_input_tokens"], int)
-        and result["max_input_tokens"] > 0,
+        "max_input_tokens_positive_int": isinstance(result["max_input_tokens"], int) and result["max_input_tokens"] > 0,
         "max_output_tokens_positive_int": isinstance(result["max_output_tokens"], int)
         and result["max_output_tokens"] > 0,
     }
@@ -205,9 +204,7 @@ def test_validate_model_access_happy_path_single_model_in_list():
 
 def test_validate_model_access_happy_path_batch_all_accessible():
     summary = {
-        "result": validate_model_access(
-            "gpt-4o,claude-haiku", ["gpt-4o", "claude-haiku", "gemini"]
-        ),
+        "result": validate_model_access("gpt-4o,claude-haiku", ["gpt-4o", "claude-haiku", "gemini"]),
         "input": "gpt-4o,claude-haiku",
         "available": ["gpt-4o", "claude-haiku", "gemini"],
     }
@@ -389,9 +386,7 @@ async def test_get_available_models_for_user_error_path_complete_list_raises(
     def _boom(**_kwargs):
         raise RuntimeError("downstream failure")
 
-    monkeypatch.setattr(
-        "litellm.proxy.auth.model_checks.get_complete_model_list", _boom
-    )
+    monkeypatch.setattr("litellm.proxy.auth.model_checks.get_complete_model_list", _boom)
     user_api_key_dict = UserAPIKeyAuth(
         api_key="sk-test-key",
         user_id="user-1",
@@ -481,6 +476,7 @@ async def test_get_available_models_for_user_without_access_groups_grants_nothin
     )
     assert result == []
 
+
 @pytest.mark.asyncio
 async def test_get_available_models_for_user_resolves_key_access_group_models(
     monkeypatch,
@@ -521,3 +517,78 @@ async def test_get_available_models_for_user_resolves_key_access_group_models(
         user_api_key_cache=MagicMock(),
     )
     assert result == ["model-b"]
+
+
+def _agent_ceiling(models: frozenset[str] | None):
+    from litellm.proxy.agent_endpoints.auth.agent_access_groups import AgentAccessGroupCeiling
+
+    async def resolve(agent_id: str) -> AgentAccessGroupCeiling | None:
+        if models is None:
+            return None
+        return AgentAccessGroupCeiling(
+            access_group_ids=("ag-agent",), models=models, mcp_server_ids=frozenset(), agent_ids=frozenset()
+        )
+
+    return resolve
+
+
+def _agent_key(models: list[str]) -> UserAPIKeyAuth:
+    return UserAPIKeyAuth(api_key="sk-agent-key", user_id="user-1", agent_id="agent-1", models=models)
+
+
+@pytest.mark.asyncio
+async def test_agent_key_listing_is_capped_to_its_access_groups():
+    result = await get_available_models_for_user(
+        user_api_key_dict=_agent_key(["model-a", "model-b", "model-c"]),
+        llm_router=_router_with_models(["model-a", "model-b", "model-c"]),
+        general_settings={},
+        user_model=None,
+        resolve_agent_ceiling=_agent_ceiling(frozenset({"model-b", "model-d"})),
+    )
+    assert result == ["model-b"]
+
+
+@pytest.mark.asyncio
+async def test_agent_key_listing_is_empty_when_its_groups_grant_no_model():
+    result = await get_available_models_for_user(
+        user_api_key_dict=_agent_key(["model-a"]),
+        llm_router=_router_with_models(["model-a"]),
+        general_settings={},
+        user_model=None,
+        resolve_agent_ceiling=_agent_ceiling(frozenset()),
+    )
+    assert result == []
+
+
+@pytest.mark.asyncio
+async def test_agent_ceiling_expands_a_model_access_group_name_for_listing():
+    router = _router_with_models(["model-a", "model-b"])
+    router.get_model_access_groups.return_value = {"fast-models": ["model-b"]}
+    result = await get_available_models_for_user(
+        user_api_key_dict=_agent_key(["model-a", "model-b"]),
+        llm_router=router,
+        general_settings={},
+        user_model=None,
+        resolve_agent_ceiling=_agent_ceiling(frozenset({"fast-models"})),
+    )
+    assert result == ["model-b"]
+
+
+@pytest.mark.asyncio
+async def test_listing_is_unchanged_without_an_agent_or_without_attached_groups():
+    router = _router_with_models(["model-a", "model-b"])
+    plain_key = await get_available_models_for_user(
+        user_api_key_dict=UserAPIKeyAuth(api_key="sk-plain", user_id="user-1", models=["model-a", "model-b"]),
+        llm_router=router,
+        general_settings={},
+        user_model=None,
+        resolve_agent_ceiling=_agent_ceiling(frozenset({"model-a"})),
+    )
+    agent_without_groups = await get_available_models_for_user(
+        user_api_key_dict=_agent_key(["model-a", "model-b"]),
+        llm_router=router,
+        general_settings={},
+        user_model=None,
+        resolve_agent_ceiling=_agent_ceiling(None),
+    )
+    assert (plain_key, agent_without_groups) == (["model-a", "model-b"], ["model-a", "model-b"])

From 89330cdac6e3b103421114e1385efe719e417534 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Thu, 17 Sep 2026 20:34:06 +0000
Subject: [PATCH 010/160] refactor(agents): exhaust the agent access match and
 drop routine comments

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../mcp_server/auth/user_api_key_auth_mcp.py       |  3 ---
 .../auth/agent_permission_handler.py               | 14 ++++----------
 litellm/proxy/auth/auth_checks.py                  |  5 +----
 3 files changed, 5 insertions(+), 17 deletions(-)

diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
index 05661584a6b..4bca15190cf 100644
--- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
+++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
@@ -193,9 +193,6 @@ def _agent_capped_servers(
     agent_servers: Sequence[str],
     agent_access_group_servers: frozenset[str] | None,
 ) -> tuple[str, ...] | None:
-    """Servers left once the agent's object_permission and attached access groups both cap the
-    key/team result, or None when the agent restricts nothing. An attached group set naming no
-    server is an empty ceiling, not an absent one, so it denies every server."""
     if not agent_servers and agent_access_group_servers is None:
         return None
     return tuple(
diff --git a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
index 1759090a29a..ea73d4634e3 100644
--- a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
+++ b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
@@ -8,7 +8,7 @@ Follows the same pattern as MCP permission handling.
 import asyncio
 from collections.abc import Awaitable, Callable, Sequence
 from dataclasses import dataclass
-from typing import Final, TypeAlias
+from typing import Final, TypeAlias, assert_never
 
 from litellm._logging import verbose_logger
 from litellm.proxy._experimental.mcp_server.ui_session_utils import build_effective_auth_contexts
@@ -67,14 +67,7 @@ class AgentRequestHandler:
         user_api_key_auth: UserAPIKeyAuth | None = None,
         resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling,
     ) -> AgentAccess:
-        """
-        Resolve the agents the given user/key may reach.
-
-        ``UnrestrictedAgentAccess`` is only returned when neither the key nor its team
-        carries any grant and the agent behind the key has no access groups attached.
-        Grants that intersect to nothing stay restricted, so narrowing a caller can
-        never widen what it reaches.
-        """
+        """Agents the key may reach: key and team grants intersected with the agent's access group ceiling."""
         key_team_access: Final = await AgentRequestHandler._resolve_key_team_agent_access(user_api_key_auth)
         agent_ceiling: Final = await AgentRequestHandler._agent_access_group_ceiling(user_api_key_auth, resolve_ceiling)
         if agent_ceiling is None:
@@ -84,6 +77,8 @@ class AgentRequestHandler:
                 return RestrictedAgentAccess(agent_ceiling)
             case RestrictedAgentAccess(key_team_ids):
                 return RestrictedAgentAccess(key_team_ids & agent_ceiling)
+            case _:
+                assert_never(key_team_access)
 
     @staticmethod
     async def _resolve_key_team_agent_access(
@@ -111,7 +106,6 @@ class AgentRequestHandler:
         user_api_key_auth: UserAPIKeyAuth | None,
         resolve_ceiling: CeilingResolver,
     ) -> frozenset[str] | None:
-        """Stable IDs of the agents the calling agent's attached access groups allow; None when none attached."""
         if user_api_key_auth is None or not user_api_key_auth.agent_id:
             return None
         ceiling: Final = await resolve_ceiling(user_api_key_auth.agent_id)
diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py
index 137cf849389..df20358bcce 100644
--- a/litellm/proxy/auth/auth_checks.py
+++ b/litellm/proxy/auth/auth_checks.py
@@ -1007,7 +1007,6 @@ async def common_checks(
                         code=status.HTTP_400_BAD_REQUEST,
                     )
 
-    # 2.4 If the agent behind the key has access groups attached, they cap the models it can call
     await _check_agent_access_group_model_access(model=_model, valid_token=valid_token, llm_router=llm_router)
 
     ## 2.1 If user can call model (if personal key)
@@ -4205,9 +4204,7 @@ async def _check_agent_access_group_model_access(
     llm_router: Router | None,
     resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling,
 ) -> Literal[True]:
-    """Raises when the key's agent has access groups attached and none of them names the model.
-    Attached groups that name no model deny every model; ``_can_object_call_model`` would read
-    an empty allowlist as unrestricted."""
+    """Attached groups naming no model deny every model, unlike the empty allowlist ``_can_object_call_model`` allows."""
     if not model or valid_token is None or not valid_token.agent_id:
         return True
     ceiling: Final = await resolve_ceiling(valid_token.agent_id)

From 18c31e6fc6cb4f7a5ad5bb658b7f5d6356664922 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Thu, 17 Sep 2026 20:40:06 +0000
Subject: [PATCH 011/160] fix(agents): import assert_never from
 typing_extensions for Python 3.10

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../proxy/agent_endpoints/auth/agent_permission_handler.py    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
index ea73d4634e3..4e022e48bb4 100644
--- a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
+++ b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
@@ -8,7 +8,9 @@ Follows the same pattern as MCP permission handling.
 import asyncio
 from collections.abc import Awaitable, Callable, Sequence
 from dataclasses import dataclass
-from typing import Final, TypeAlias, assert_never
+from typing import Final, TypeAlias
+
+from typing_extensions import assert_never
 
 from litellm._logging import verbose_logger
 from litellm.proxy._experimental.mcp_server.ui_session_utils import build_effective_auth_contexts

From 1206fa802b851fb485cf494e234f6a628f86813c Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Thu, 17 Sep 2026 20:51:04 +0000
Subject: [PATCH 012/160] refactor(agents): resolve attached access groups from
 the agent registry instead of the DB on the request path

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../auth/agent_access_groups.py               | 81 +---------------
 litellm/proxy/agent_endpoints/endpoints.py    |  4 -
 .../access_group_endpoints.py                 |  2 -
 .../auth/test_agent_access_groups.py          | 93 +++----------------
 4 files changed, 14 insertions(+), 166 deletions(-)

diff --git a/litellm/proxy/agent_endpoints/auth/agent_access_groups.py b/litellm/proxy/agent_endpoints/auth/agent_access_groups.py
index e0e5193d0a4..49e5407ff88 100644
--- a/litellm/proxy/agent_endpoints/auth/agent_access_groups.py
+++ b/litellm/proxy/agent_endpoints/auth/agent_access_groups.py
@@ -1,35 +1,18 @@
 import asyncio
-from collections.abc import Awaitable, Callable, Sequence
+from collections.abc import Awaitable, Callable
 from dataclasses import dataclass
-from typing import Final, Protocol, TypeAlias
+from typing import Final, TypeAlias
 
 from fastapi import HTTPException
-from pydantic import TypeAdapter, ValidationError
-from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
-from litellm.caching.dual_cache import DualCache
 from litellm.proxy._types import LiteLLM_AccessGroupTable
-from litellm.proxy.common_utils.user_api_key_cache import get_management_object_ttl
-
-
-class _AgentAccessGroupsRecord(Protocol):
-    @property
-    def access_group_ids(self) -> Sequence[str] | None: ...
-
-
-class _AgentIdWhere(TypedDict):
-    agent_id: ReadOnly[str]
-
 
 AccessGroupIds: TypeAlias = tuple[str, ...]
 AccessGroupIdsLoader: TypeAlias = Callable[[str], Awaitable[AccessGroupIds]]  # mutable-ok: Callable params
-AgentRecordFinder: TypeAlias = Callable[[str], Awaitable[_AgentAccessGroupsRecord | None]]  # mutable-ok: Callable
 LoadedAccessGroup: TypeAlias = LiteLLM_AccessGroupTable | None
 AccessGroupLoader: TypeAlias = Callable[[str], Awaitable[LoadedAccessGroup]]  # mutable-ok: Callable parameter syntax
 
-_CACHED_IDS: Final = TypeAdapter(list[str])
-
 
 @dataclass(frozen=True, slots=True)
 class AgentAccessGroupCeiling:
@@ -44,19 +27,6 @@ class AgentAccessGroupCeiling:
 CeilingResolver: TypeAlias = Callable[[str], Awaitable[AgentAccessGroupCeiling | None]]  # mutable-ok: Callable params
 
 
-def agent_access_group_ids_cache_key(agent_id: str) -> str:
-    return f"agent_access_group_ids:{agent_id}"
-
-
-def _cached_access_group_ids(cached: object) -> AccessGroupIds | None:
-    if cached is None:
-        return None
-    try:
-        return tuple(_CACHED_IDS.validate_python(cached))
-    except ValidationError:
-        return None
-
-
 async def _registry_access_group_ids(agent_id: str) -> AccessGroupIds:
     from litellm.proxy.common_utils.registry_read_through import get_agent_with_read_through
 
@@ -64,51 +34,6 @@ async def _registry_access_group_ids(agent_id: str) -> AccessGroupIds:
     return tuple(agent.access_group_ids or ()) if agent is not None else ()
 
 
-async def load_agent_access_group_ids(
-    agent_id: str,
-    cache: DualCache,
-    find_agent: AgentRecordFinder,
-    fallback: AccessGroupIdsLoader,
-) -> AccessGroupIds:
-    """The agent row's groups, cached for the management-object TTL and evicted on every agent write."""
-    cache_key: Final = agent_access_group_ids_cache_key(agent_id)
-    cached: Final = _cached_access_group_ids(await cache.async_get_cache(key=cache_key))
-    if cached is not None:
-        return cached
-    try:
-        record: Final = await find_agent(agent_id)
-    except Exception as e:  # noqa: BLE001  # prisma raises many error types; the registry snapshot answers instead
-        verbose_proxy_logger.warning("Failed to read access groups for agent %r, using registry: %s", agent_id, e)
-        return await fallback(agent_id)
-    access_group_ids: Final = tuple(record.access_group_ids or ()) if record is not None else ()
-    await cache.async_set_cache(key=cache_key, value=access_group_ids, ttl=get_management_object_ttl(cache))
-    return access_group_ids
-
-
-async def _load_agent_access_group_ids(agent_id: str) -> AccessGroupIds:
-    from litellm.proxy.agent_endpoints.agent_registry import agents_table
-    from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
-
-    if prisma_client is None:
-        return await _registry_access_group_ids(agent_id)
-    db: Final = prisma_client
-
-    async def find_agent(row_agent_id: str) -> _AgentAccessGroupsRecord | None:
-        return await agents_table(db).find_unique(where=_AgentIdWhere(agent_id=row_agent_id))
-
-    return await load_agent_access_group_ids(agent_id, user_api_key_cache, find_agent, _registry_access_group_ids)
-
-
-async def evict_agent_access_group_ids(agent_ids: Sequence[str]) -> None:
-    from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
-    from litellm.proxy.proxy_server import user_api_key_cache
-
-    await evict_and_broadcast(
-        cache_keys=tuple(agent_access_group_ids_cache_key(agent_id) for agent_id in agent_ids),
-        user_api_key_cache=user_api_key_cache,
-    )
-
-
 async def _load_access_group(access_group_id: str) -> LoadedAccessGroup:
     from litellm.proxy.auth.auth_checks import get_access_object
     from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
@@ -132,7 +57,7 @@ async def _load_access_group(access_group_id: str) -> LoadedAccessGroup:
 
 async def resolve_agent_access_group_ceiling(
     agent_id: str,
-    load_access_group_ids: AccessGroupIdsLoader = _load_agent_access_group_ids,
+    load_access_group_ids: AccessGroupIdsLoader = _registry_access_group_ids,
     load_access_group: AccessGroupLoader = _load_access_group,
 ) -> AgentAccessGroupCeiling | None:
     """``None`` when the agent has no access groups attached, so nothing is capped."""
diff --git a/litellm/proxy/agent_endpoints/endpoints.py b/litellm/proxy/agent_endpoints/endpoints.py
index 62783fb412a..aa8979a73c6 100644
--- a/litellm/proxy/agent_endpoints/endpoints.py
+++ b/litellm/proxy/agent_endpoints/endpoints.py
@@ -44,7 +44,6 @@ from litellm.proxy.agent_endpoints.agent_search import (
     global_agent_search_index,
     search_agents,
 )
-from litellm.proxy.agent_endpoints.auth.agent_access_groups import evict_agent_access_group_ids
 from litellm.proxy.agent_endpoints.auth.agent_permission_handler import accessible_agents
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils.rbac_utils import check_feature_access_for_user
@@ -697,7 +696,6 @@ async def update_agent(
             prisma_client=prisma_client,
             updated_by=updated_by,
         )
-        await evict_agent_access_group_ids((agent_id,))
 
         # deregister in memory
         AGENT_REGISTRY.deregister_agent(agent_name=existing_agent.get("agent_name"))
@@ -801,7 +799,6 @@ async def patch_agent(
             prisma_client=prisma_client,
             updated_by=updated_by,
         )
-        await evict_agent_access_group_ids((agent_id,))
 
         # deregister in memory
         AGENT_REGISTRY.deregister_agent(agent_name=existing_agent.get("agent_name"))
@@ -864,7 +861,6 @@ async def delete_agent(
             raise HTTPException(status_code=404, detail=f"Agent with ID {agent_id} not found in DB.")
 
         await AGENT_REGISTRY.delete_agent_from_db(agent_id=agent_id, prisma_client=prisma_client)
-        await evict_agent_access_group_ids((agent_id,))
 
         AGENT_REGISTRY.deregister_agent(agent_name=existing_agent.get("agent_name"))
 
diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py
index 2694d00b17f..b4923b0a2dc 100644
--- a/litellm/proxy/management_endpoints/access_group_endpoints.py
+++ b/litellm/proxy/management_endpoints/access_group_endpoints.py
@@ -16,7 +16,6 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
 )
 from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
-from litellm.proxy.agent_endpoints.auth.agent_access_groups import evict_agent_access_group_ids
 from litellm.proxy.auth.auth_checks import (
     _cache_access_object,
     _cache_key_object,
@@ -783,7 +782,6 @@ async def delete_access_group(
 
         await invalidate_access_group_cache(access_group_id)
         _detach_access_group_from_agent_registry(detached_agent_ids, access_group_id)
-        await evict_agent_access_group_ids(detached_agent_ids)
         await _patch_team_caches_remove_access_group(
             affected_team_ids, access_group_id, user_api_key_cache, proxy_logging_obj
         )
diff --git a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_access_groups.py b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_access_groups.py
index 266ce52e3a9..e744e84d671 100644
--- a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_access_groups.py
+++ b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_access_groups.py
@@ -1,16 +1,11 @@
-from collections.abc import Sequence
-from dataclasses import dataclass
 from typing import Final
 
 import pytest
 from fastapi import HTTPException
 
-from litellm.caching.dual_cache import DualCache
 from litellm.models.access_group import LiteLLM_AccessGroupTable
 from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
     AgentAccessGroupCeiling,
-    agent_access_group_ids_cache_key,
-    load_agent_access_group_ids,
     resolve_agent_access_group_ceiling,
 )
 from litellm.types.agents import AgentResponse
@@ -110,86 +105,20 @@ async def test_only_unloadable_groups_is_an_empty_ceiling_not_unrestricted():
     assert ceiling.agent_ids == frozenset()
 
 
-@dataclass(frozen=True, slots=True)
-class _AgentRow:
-    access_group_ids: Sequence[str] | None
-
-
-class _FakeAgentTable:
-    def __init__(self, rows: dict[str, _AgentRow], failing: bool = False) -> None:
-        self._rows: Final = rows
-        self._failing: Final = failing
-        self.reads = 0
-
-    async def find_agent(self, agent_id: str) -> _AgentRow | None:
-        self.reads += 1
-        if self._failing:
-            raise RuntimeError("db down")
-        return self._rows.get(agent_id)
-
-
-async def _registry_snapshot(agent_id: str) -> tuple[str, ...]:
-    return ("registry-group",)
-
-
 @pytest.mark.asyncio
-async def test_agent_row_is_read_once_then_served_from_cache():
-    cache: Final = DualCache()
-    table: Final = _FakeAgentTable({"agent-1": _AgentRow(["g1", "g2"])})
+async def test_default_agent_loader_reads_the_attached_groups_from_the_registry():
+    from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
 
-    first: Final = await load_agent_access_group_ids("agent-1", cache, table.find_agent, _registry_snapshot)
-    second: Final = await load_agent_access_group_ids("agent-1", cache, table.find_agent, _registry_snapshot)
+    _, load_group = _loaders(None, {"g1": _group("g1", models=("gpt-5",))})
+    global_agent_registry.register_agent(_agent(["g1"]))
+    try:
+        ceiling: Final = await resolve_agent_access_group_ceiling("agent-1", load_access_group=load_group)
+    finally:
+        global_agent_registry.deregister_agent("agent")
 
-    assert (first, second, table.reads) == (("g1", "g2"), ("g1", "g2"), 1)
-
-
-@pytest.mark.asyncio
-async def test_agent_with_no_row_or_no_groups_caches_an_empty_answer():
-    cache: Final = DualCache()
-    table: Final = _FakeAgentTable({"bare": _AgentRow(None)})
-
-    bare: Final = await load_agent_access_group_ids("bare", cache, table.find_agent, _registry_snapshot)
-    missing: Final = await load_agent_access_group_ids("missing", cache, table.find_agent, _registry_snapshot)
-    again: Final = await load_agent_access_group_ids("missing", cache, table.find_agent, _registry_snapshot)
-
-    assert (bare, missing, again, table.reads) == ((), (), (), 2)
-
-
-@pytest.mark.asyncio
-async def test_evicted_cache_entry_picks_up_the_patched_row():
-    cache: Final = DualCache()
-    rows: Final = {"agent-1": _AgentRow(["g1"])}
-    table: Final = _FakeAgentTable(rows)
-    await load_agent_access_group_ids("agent-1", cache, table.find_agent, _registry_snapshot)
-
-    rows["agent-1"] = _AgentRow(["g2"])
-    stale: Final = await load_agent_access_group_ids("agent-1", cache, table.find_agent, _registry_snapshot)
-    await cache.async_delete_cache(key=agent_access_group_ids_cache_key("agent-1"))
-    fresh: Final = await load_agent_access_group_ids("agent-1", cache, table.find_agent, _registry_snapshot)
-
-    assert (stale, fresh) == (("g1",), ("g2",))
-
-
-@pytest.mark.asyncio
-async def test_unreadable_row_falls_back_to_the_registry_without_caching():
-    cache: Final = DualCache()
-    table: Final = _FakeAgentTable({}, failing=True)
-
-    answer: Final = await load_agent_access_group_ids("agent-1", cache, table.find_agent, _registry_snapshot)
-
-    assert answer == ("registry-group",)
-    assert await cache.async_get_cache(key=agent_access_group_ids_cache_key("agent-1")) is None
-
-
-@pytest.mark.asyncio
-async def test_garbage_in_the_cache_is_treated_as_a_miss():
-    cache: Final = DualCache()
-    await cache.async_set_cache(key=agent_access_group_ids_cache_key("agent-1"), value={"not": "a list"})
-    table: Final = _FakeAgentTable({"agent-1": _AgentRow(["g1"])})
-
-    answer: Final = await load_agent_access_group_ids("agent-1", cache, table.find_agent, _registry_snapshot)
-
-    assert (answer, table.reads) == (("g1",), 1)
+    assert ceiling == AgentAccessGroupCeiling(
+        access_group_ids=("g1",), models=frozenset({"gpt-5"}), mcp_server_ids=frozenset(), agent_ids=frozenset()
+    )
 
 
 @pytest.mark.asyncio

From 7229fc1952a83387e4126949114e9d253d5caec3 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Thu, 17 Sep 2026 21:00:56 +0000
Subject: [PATCH 013/160] fix(agents): return on every branch of the agent
 access ceiling so CodeQL sees no fall-through

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../agent_endpoints/auth/agent_permission_handler.py | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
index 4e022e48bb4..fe0a1b13b43 100644
--- a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
+++ b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
@@ -10,8 +10,6 @@ from collections.abc import Awaitable, Callable, Sequence
 from dataclasses import dataclass
 from typing import Final, TypeAlias
 
-from typing_extensions import assert_never
-
 from litellm._logging import verbose_logger
 from litellm.proxy._experimental.mcp_server.ui_session_utils import build_effective_auth_contexts
 from litellm.proxy._types import (
@@ -74,13 +72,9 @@ class AgentRequestHandler:
         agent_ceiling: Final = await AgentRequestHandler._agent_access_group_ceiling(user_api_key_auth, resolve_ceiling)
         if agent_ceiling is None:
             return key_team_access
-        match key_team_access:
-            case UnrestrictedAgentAccess():
-                return RestrictedAgentAccess(agent_ceiling)
-            case RestrictedAgentAccess(key_team_ids):
-                return RestrictedAgentAccess(key_team_ids & agent_ceiling)
-            case _:
-                assert_never(key_team_access)
+        if isinstance(key_team_access, UnrestrictedAgentAccess):
+            return RestrictedAgentAccess(agent_ceiling)
+        return RestrictedAgentAccess(key_team_access.agent_ids & agent_ceiling)
 
     @staticmethod
     async def _resolve_key_team_agent_access(

From 25729c521fc944b44f0d518f4b6bc45d87e5c68c Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Thu, 17 Sep 2026 21:22:39 +0000
Subject: [PATCH 014/160] refactor(agents): combine key and team agent grants
 without a fall-through match

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../auth/agent_permission_handler.py          | 27 ++++++++++++-------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
index fe0a1b13b43..b7b7638e478 100644
--- a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
+++ b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
@@ -48,6 +48,22 @@ def _to_stable_ids(agent_ids: frozenset[str]) -> frozenset[str]:
     return frozenset(global_agent_registry.stable_agent_id(agent_id) for agent_id in agent_ids)
 
 
+def _restricted_ids(access: AgentAccess) -> frozenset[str] | None:
+    if isinstance(access, UnrestrictedAgentAccess):
+        return None
+    return _to_stable_ids(access.agent_ids)
+
+
+def _intersect_agent_access(key_access: AgentAccess, team_access: AgentAccess) -> AgentAccess:
+    key_ids: Final = _restricted_ids(key_access)
+    team_ids: Final = _restricted_ids(team_access)
+    if key_ids is None:
+        return UnrestrictedAgentAccess() if team_ids is None else RestrictedAgentAccess(team_ids)
+    if team_ids is None:
+        return RestrictedAgentAccess(key_ids)
+    return RestrictedAgentAccess(key_ids & team_ids)
+
+
 class AgentRequestHandler:
     """
     Class to handle agent permission checking, including:
@@ -83,19 +99,10 @@ class AgentRequestHandler:
         try:
             key_access: Final = await AgentRequestHandler._get_allowed_agents_for_key(user_api_key_auth)
             team_access: Final = await AgentRequestHandler._get_allowed_agents_for_team(user_api_key_auth)
-
-            match (key_access, team_access):
-                case (UnrestrictedAgentAccess(), UnrestrictedAgentAccess()):
-                    return UnrestrictedAgentAccess()
-                case (UnrestrictedAgentAccess(), RestrictedAgentAccess(team_ids)):
-                    return RestrictedAgentAccess(_to_stable_ids(team_ids))
-                case (RestrictedAgentAccess(key_ids), UnrestrictedAgentAccess()):
-                    return RestrictedAgentAccess(_to_stable_ids(key_ids))
-                case (RestrictedAgentAccess(key_ids), RestrictedAgentAccess(team_ids)):
-                    return RestrictedAgentAccess(_to_stable_ids(key_ids) & _to_stable_ids(team_ids))
         except Exception as e:
             verbose_logger.warning("Failed to get allowed agents: %s", e)
             return UnrestrictedAgentAccess()
+        return _intersect_agent_access(key_access, team_access)
 
     @staticmethod
     async def _agent_access_group_ceiling(

From db17841b3d6ff41900276de6eac8c66fbc924801 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Thu, 17 Sep 2026 22:37:16 -0700
Subject: [PATCH 015/160] test(model_management): cover actor edges and
 wildcard models

---
 .../test_model_management_endpoints.py        | 340 +++++++++++++++++-
 .../handle_add_model_submit.test.tsx          |  19 +
 2 files changed, 356 insertions(+), 3 deletions(-)

diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py
index d1fe88df26c..302585e42c4 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py
@@ -1224,11 +1224,11 @@ class TestUpdateModel:
                 "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call",
                 new=AsyncMock(return_value=None),
             ),
-            patch(
+            patch(  # test-quality-ok: [TQ008] isolate persistence from encryption implementation
                 "litellm.proxy.management_endpoints.model_management_endpoints.encrypt_value_helper",
                 side_effect=lambda value: value,
             ),
-            patch(
+            patch(  # test-quality-ok: [TQ008] isolate persistence from router reload implementation
                 "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache",
                 new=AsyncMock(
                     return_value=ReconcileOutcome(still_desired=None, live_after=None)
@@ -4021,7 +4021,7 @@ class TestPatchModelBlockedAuthGate:
                 "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call",
                 new=AsyncMock(return_value=None),
             ),
-            patch(
+            patch(  # test-quality-ok: [TQ008] isolate persistence from router reload implementation
                 "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache",
                 new=AsyncMock(
                     return_value=ReconcileOutcome(still_desired=None, live_after=None)
@@ -6631,3 +6631,337 @@ class TestTeamMemberAutoRouterWrites:
         assert json.loads(written["model_info"])["member_auto_router"] is True
         assert appended.await_args.kwargs["data"].models == ["new-personal-router"]
         assert appended.await_args.kwargs["data"].team_id == "member-team"
+
+
+class TestModelManagementActorEdges:
+    @pytest.mark.asyncio
+    async def test_add_model_rejects_non_team_internal_user(self):
+        from litellm.proxy._types import ProxyException
+        from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model
+
+        actor: Final = UserAPIKeyAuth(user_id="internal-user", user_role=LitellmUserRoles.INTERNAL_USER)
+        prisma: Final = MagicMock()
+        deployment: Final = Deployment(
+            model_name="internal-model",
+            litellm_params=LiteLLM_Params(model="openai/test-model"),
+            model_info=ModelInfo(id="internal-model-id"),
+        )
+        with (
+            patch("litellm.proxy.proxy_server.prisma_client", prisma),  # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.store_model_in_db", True),  # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.premium_user", True),  # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.general_settings", {}),  # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam
+        ):
+            with pytest.raises(ProxyException) as exc_info:
+                await add_new_model(model_params=deployment, user_api_key_dict=actor)
+
+        assert str(exc_info.value.code) == "403"
+        assert "permission" in str(exc_info.value).lower()
+        prisma.db.litellm_proxymodeltable.create.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_add_model_rejects_proxy_admin_viewer(self):
+        from litellm.proxy._types import ProxyException
+        from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model
+
+        actor: Final = UserAPIKeyAuth(
+            user_id="view-only-user", user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY
+        )
+        prisma: Final = MagicMock()
+        deployment: Final = Deployment(
+            model_name="view-only-model",
+            litellm_params=LiteLLM_Params(model="openai/test-model"),
+            model_info=ModelInfo(id="view-only-model-id"),
+        )
+        with (
+            patch("litellm.proxy.proxy_server.prisma_client", prisma),  # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.store_model_in_db", True),  # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.premium_user", True),  # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.general_settings", {}),  # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam
+        ):
+            with pytest.raises(ProxyException) as exc_info:
+                await add_new_model(model_params=deployment, user_api_key_dict=actor)
+
+        assert str(exc_info.value.code) == "403"
+        assert "view-only" in str(exc_info.value).lower()
+        prisma.db.litellm_proxymodeltable.create.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_add_model_requires_database_storage(self):
+        from litellm.proxy._types import ProxyException
+        from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model
+
+        actor: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
+        prisma: Final = MagicMock()
+        deployment: Final = Deployment(
+            model_name="database-disabled-model",
+            litellm_params=LiteLLM_Params(model="openai/test-model"),
+            model_info=ModelInfo(id="database-disabled-model-id"),
+        )
+        with (
+            patch("litellm.proxy.proxy_server.prisma_client", prisma),  # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.store_model_in_db", False),  # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.premium_user", True),  # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.general_settings", {}),  # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam
+        ):
+            with pytest.raises(ProxyException) as exc_info:
+                await add_new_model(model_params=deployment, user_api_key_dict=actor)
+
+        assert str(exc_info.value.code) == "500"
+        assert "STORE_MODEL_IN_DB" in str(exc_info.value)
+        prisma.db.litellm_proxymodeltable.create.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_legacy_model_update_persists_changed_field(self):
+        from litellm.proxy.management_endpoints.model_management_endpoints import update_model
+
+        model_id: Final = "legacy-update-model-id"
+        existing_row: Final = MagicMock()
+        existing_row.litellm_params = {"model": "openai/test-model", "timeout": 30}
+        existing_row.model_dump.return_value = {
+            "model_name": "legacy-update-model",
+            "litellm_params": existing_row.litellm_params,
+            "model_info": {"id": model_id},
+        }
+        existing_row.model_dump_json.return_value = "{}"
+        updated_row: Final = MagicMock()
+        updated_row.model_dump_json.return_value = "{}"
+        prisma: Final = MagicMock()
+        prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=existing_row)
+        prisma.db.litellm_proxymodeltable.update = AsyncMock(return_value=updated_row)
+        router: Final = MagicMock()
+        router.get_model_ids.return_value = [model_id]
+        actor: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
+        with (
+            patch("litellm.proxy.proxy_server.prisma_client", prisma),  # test-quality-ok: [TQ008] update endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.llm_router", router),  # test-quality-ok: [TQ008] update endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.store_model_in_db", True),  # test-quality-ok: [TQ008] update endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.premium_user", True),  # test-quality-ok: [TQ008] update endpoint reads proxy-server state through its only test seam
+            patch(  # test-quality-ok: [TQ008] isolate persistence from encryption implementation
+                "litellm.proxy.management_endpoints.model_management_endpoints.encrypt_value_helper",
+                side_effect=lambda value: value,
+            ),
+            patch(  # test-quality-ok: [TQ008] isolate persistence from router reload implementation
+                "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache",
+                new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
+            ),
+        ):
+            await update_model(
+                model_params=updateDeployment(
+                    litellm_params=updateLiteLLMParams(timeout=42),
+                    model_info=ModelInfo(id=model_id),
+                ),
+                user_api_key_dict=actor,
+            )
+
+        written: Final = json.loads(
+            prisma.db.litellm_proxymodeltable.update.await_args.kwargs["data"]["litellm_params"]
+        )
+        assert written["timeout"] == 42
+        assert written["model"] == "openai/test-model"
+
+    @pytest.mark.asyncio
+    async def test_legacy_model_update_explicit_null_preserves_existing_field(self):
+        from litellm.proxy.management_endpoints.model_management_endpoints import update_model
+
+        model_id: Final = "legacy-null-model-id"
+        existing_row: Final = MagicMock()
+        existing_row.litellm_params = {"model": "openai/test-model", "timeout": 30}
+        existing_row.model_dump.return_value = {
+            "model_name": "legacy-null-model",
+            "litellm_params": existing_row.litellm_params,
+            "model_info": {"id": model_id},
+        }
+        existing_row.model_dump_json.return_value = "{}"
+        updated_row: Final = MagicMock()
+        updated_row.model_dump_json.return_value = "{}"
+        prisma: Final = MagicMock()
+        prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=existing_row)
+        prisma.db.litellm_proxymodeltable.update = AsyncMock(return_value=updated_row)
+        router: Final = MagicMock()
+        router.get_model_ids.return_value = [model_id]
+        actor: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
+        with (
+            patch("litellm.proxy.proxy_server.prisma_client", prisma),  # test-quality-ok: [TQ008] update endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.llm_router", router),  # test-quality-ok: [TQ008] update endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.store_model_in_db", True),  # test-quality-ok: [TQ008] update endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.premium_user", True),  # test-quality-ok: [TQ008] update endpoint reads proxy-server state through its only test seam
+            patch(  # test-quality-ok: [TQ008] isolate persistence from encryption implementation
+                "litellm.proxy.management_endpoints.model_management_endpoints.encrypt_value_helper",
+                side_effect=lambda value: value,
+            ),
+            patch(  # test-quality-ok: [TQ008] isolate persistence from router reload implementation
+                "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache",
+                new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
+            ),
+        ):
+            await update_model(
+                model_params=updateDeployment(
+                    litellm_params=updateLiteLLMParams(timeout=None),
+                    model_info=ModelInfo(id=model_id),
+                ),
+                user_api_key_dict=actor,
+            )
+
+        written: Final = json.loads(
+            prisma.db.litellm_proxymodeltable.update.await_args.kwargs["data"]["litellm_params"]
+        )
+        assert written["timeout"] == 30
+
+    @pytest.mark.asyncio
+    async def test_patch_model_rejects_config_file_model(self):
+        from litellm.proxy._types import ProxyException
+        from litellm.proxy.management_endpoints.model_management_endpoints import patch_model
+
+        model_id: Final = "config-model-id"
+        prisma: Final = MagicMock()
+        prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=None)
+        prisma.db.litellm_proxymodeltable.update = AsyncMock()
+        router: Final = MagicMock()
+        router.get_deployment.return_value = Deployment(
+            model_name="config-model",
+            litellm_params=LiteLLM_Params(model="openai/test-model"),
+            model_info=ModelInfo(id=model_id),
+        )
+        actor: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
+        with (
+            patch("litellm.proxy.proxy_server.prisma_client", prisma),  # test-quality-ok: [TQ008] patch endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.llm_router", router),  # test-quality-ok: [TQ008] patch endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.store_model_in_db", True),  # test-quality-ok: [TQ008] patch endpoint reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.premium_user", True),  # test-quality-ok: [TQ008] patch endpoint reads proxy-server state through its only test seam
+        ):
+            with pytest.raises(ProxyException) as exc_info:
+                await patch_model(
+                    model_id=model_id,
+                    patch_data=updateDeployment(
+                        litellm_params=updateLiteLLMParams(timeout=42),
+                        model_info=ModelInfo(id=model_id),
+                    ),
+                    user_api_key_dict=actor,
+                )
+
+        assert str(exc_info.value.code) == "400"
+        assert "Cannot edit config-based model" in str(exc_info.value)
+        prisma.db.litellm_proxymodeltable.update.assert_not_awaited()
+
+    @contextlib.contextmanager
+    def _client_for(self, actor: UserAPIKeyAuth) -> Iterator[TestClient]:
+        import litellm.proxy.proxy_server as proxy_server
+        from litellm.proxy.proxy_server import app
+
+        app.dependency_overrides[proxy_server.user_api_key_auth] = lambda: actor
+        try:
+            yield TestClient(app)
+        finally:
+            app.dependency_overrides.pop(proxy_server.user_api_key_auth, None)
+
+    def test_post_model_new_binds_to_actor_guard(self):
+        actor: Final = UserAPIKeyAuth(user_id="internal-user", user_role=LitellmUserRoles.INTERNAL_USER)
+        prisma: Final = MagicMock()
+        with (
+            patch("litellm.proxy.proxy_server.prisma_client", prisma),  # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.store_model_in_db", True),  # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.premium_user", True),  # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.general_settings", {}),  # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam
+            self._client_for(actor) as client,
+        ):
+            response: Final = client.post(
+                "/model/new",
+                json={
+                    "model_name": "internal-model",
+                    "litellm_params": {"model": "openai/test-model"},
+                    "model_info": {"id": "internal-model-id"},
+                },
+            )
+
+        assert response.status_code == 403
+        assert "permission" in response.text.lower()
+        prisma.db.litellm_proxymodeltable.create.assert_not_called()
+
+    def test_post_legacy_model_update_binds_to_persistence(self):
+        model_id: Final = "legacy-route-model-id"
+        existing_row: Final = LiteLLM_ProxyModelTable(
+            model_id=model_id,
+            model_name="legacy-route-model",
+            litellm_params={"model": "openai/test-model", "timeout": 30},
+            model_info={"id": model_id},
+            created_by="admin",
+            updated_by="admin",
+        )
+        updated_row: Final = LiteLLM_ProxyModelTable(
+            model_id=model_id,
+            model_name="legacy-route-model",
+            litellm_params={"model": "openai/test-model", "timeout": 42},
+            model_info={"id": model_id},
+            created_by="admin",
+            updated_by="admin",
+        )
+        prisma: Final = MagicMock()
+        prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=existing_row)
+        prisma.db.litellm_proxymodeltable.update = AsyncMock(return_value=updated_row)
+        router: Final = MagicMock()
+        router.get_model_ids.return_value = [model_id]
+        actor: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
+        with (
+            patch("litellm.proxy.proxy_server.prisma_client", prisma),  # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.llm_router", router),  # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.store_model_in_db", True),  # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.premium_user", True),  # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam
+            patch(  # test-quality-ok: [TQ008] isolate persistence from encryption implementation
+                "litellm.proxy.management_endpoints.model_management_endpoints.encrypt_value_helper",
+                side_effect=lambda value: value,
+            ),
+            patch(  # test-quality-ok: [TQ008] isolate persistence from router reload implementation
+                "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache",
+                new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
+            ),
+            patch(  # test-quality-ok: [TQ008] audit logging is outside the persistence contract
+                "litellm.proxy.management_endpoints.model_management_endpoints.create_object_audit_log",
+                new=AsyncMock(return_value=None),
+            ),
+            self._client_for(actor) as client,
+        ):
+            response: Final = client.post(
+                "/model/update",
+                json={
+                    "litellm_params": {"timeout": 42},
+                    "model_info": {"id": model_id},
+                },
+            )
+
+        assert response.status_code == 200, response.text
+        written: Final = json.loads(
+            prisma.db.litellm_proxymodeltable.update.await_args.kwargs["data"]["litellm_params"]
+        )
+        assert written["timeout"] == 42
+
+    def test_patch_config_model_binds_to_patch_route(self):
+        model_id: Final = "config-route-model-id"
+        prisma: Final = MagicMock()
+        prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=None)
+        prisma.db.litellm_proxymodeltable.update = AsyncMock()
+        router: Final = MagicMock()
+        router.get_deployment.return_value = Deployment(
+            model_name="config-route-model",
+            litellm_params=LiteLLM_Params(model="openai/test-model"),
+            model_info=ModelInfo(id=model_id),
+        )
+        actor: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
+        with (
+            patch("litellm.proxy.proxy_server.prisma_client", prisma),  # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.llm_router", router),  # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.store_model_in_db", True),  # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam
+            patch("litellm.proxy.proxy_server.premium_user", True),  # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam
+            self._client_for(actor) as client,
+        ):
+            response: Final = client.patch(
+                f"/model/{model_id}/update",
+                json={
+                    "litellm_params": {"timeout": 42},
+                    "model_info": {"id": model_id},
+                },
+            )
+
+        assert response.status_code == 400
+        assert "Cannot edit config-based model" in response.text
+        prisma.db.litellm_proxymodeltable.update.assert_not_awaited()
diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx
index 9d792480c9f..923cf2aa0e3 100644
--- a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx
@@ -101,4 +101,23 @@ describe("prepareModelAddRequest", () => {
     expect(deployment.litellmParamsObj.litellm_credential_name).toBe("from-json");
     expect(deployment.litellmParamsObj.timeout).toBe(5);
   });
+
+  it.each([
+    ["OpenAI", "openai/*"],
+    ["Azure_AI_Studio", "azure_ai/*"],
+    ["Petals", "petals/*"],
+  ])("composes wildcard names for the all-model selection", async (custom_llm_provider, wildcardModel) => {
+    const formValues = {
+      model_mappings: [],
+      model: "all-wildcard",
+      custom_llm_provider,
+    };
+
+    const deployments = await prepareModelAddRequest({ ...formValues }, "token", null);
+
+    expect(deployments).toHaveLength(1);
+    const [deployment] = deployments!;
+    expect(deployment.modelName).toBe(wildcardModel);
+    expect(deployment.litellmParamsObj.model).toBe(wildcardModel);
+  });
 });

From c34adb4ab2bd1b6579cc3eb9a3922cf9703aae58 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Thu, 17 Sep 2026 22:44:03 -0700
Subject: [PATCH 016/160] test(ui): cover narrowed dashboard form journeys

---
 tests/e2e/ui/tests/prompts/addPrompt.spec.ts  | 55 ++++++++++++++
 .../tests/tagManagement/tagManagement.spec.ts | 76 +++++++++++++++++++
 ...PaginatedSearchSelect.integration.test.tsx | 45 +++++++++++
 .../shared/SearchSelect.integration.test.tsx  | 33 ++++++++
 .../view_logs/RequestLogsFilters.test.tsx     | 11 +++
 5 files changed, 220 insertions(+)
 create mode 100644 tests/e2e/ui/tests/prompts/addPrompt.spec.ts
 create mode 100644 tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts

diff --git a/tests/e2e/ui/tests/prompts/addPrompt.spec.ts b/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
new file mode 100644
index 00000000000..cd87e4d3b56
--- /dev/null
+++ b/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
@@ -0,0 +1,55 @@
+import { test, expect } from "@playwright/test";
+
+import { ADMIN_STORAGE_PATH } from "../../constants";
+import { Page as DashboardPage } from "../../fixtures/pages";
+import { navigateToPage } from "../../helpers/navigation";
+import { readBack } from "../../helpers/roundTrip";
+import { masterKey, uniqueSuffix } from "../../helpers/traffic";
+
+test.use({ storageState: ADMIN_STORAGE_PATH });
+
+test.describe("Prompt upload form", () => {
+  test("uploads a prompt file and reads the created prompt back", async ({
+    page,
+  }) => {
+    const promptId = `e2e-prompt-${uniqueSuffix()}`;
+    await navigateToPage(page, DashboardPage.Prompts);
+    await page.getByRole("button", { name: "Upload .prompt File" }).click();
+
+    try {
+      await expect(
+        page.getByRole("dialog", { name: "Add New Prompt" }),
+      ).toBeVisible();
+      await page.getByLabel("Prompt ID").fill(promptId);
+      await page.locator('input[type="file"]').setInputFiles({
+        name: "e2e.prompt",
+        mimeType: "text/plain",
+        buffer: Buffer.from(
+          'model: fake-openai-gpt-4\ntemplate: "Hello {{name}}"\n',
+        ),
+      });
+      await expect(page.getByText("Selected: e2e.prompt")).toBeVisible();
+      await page.getByRole("button", { name: "Create Prompt" }).click();
+
+      await expect
+        .poll(async () => {
+          const response = await page.request.get(
+            `/prompts/${encodeURIComponent(promptId)}/info?environment=development`,
+            {
+              headers: { Authorization: `Bearer ${masterKey()}` },
+            },
+          );
+          return response.ok();
+        })
+        .toBe(true);
+      await expect(page.getByText(promptId, { exact: true })).toBeVisible();
+    } finally {
+      await page.request.delete(
+        `/prompts/${encodeURIComponent(promptId)}?environment=development`,
+        {
+          headers: { Authorization: `Bearer ${masterKey()}` },
+        },
+      );
+    }
+  });
+});
diff --git a/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts b/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
new file mode 100644
index 00000000000..785211463dd
--- /dev/null
+++ b/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
@@ -0,0 +1,76 @@
+import { test, expect } from "@playwright/test";
+
+import { ADMIN_STORAGE_PATH } from "../../constants";
+import { Page as DashboardPage } from "../../fixtures/pages";
+import { navigateToPage } from "../../helpers/navigation";
+import { captureRequestBody, readBack } from "../../helpers/roundTrip";
+import { masterKey, uniqueSuffix } from "../../helpers/traffic";
+
+test.use({ storageState: ADMIN_STORAGE_PATH });
+
+test.describe("Tag management", () => {
+  test("creates, edits, reopens, and reads back a tag", async ({ page }) => {
+    const tagName = `e2e-tag-${uniqueSuffix()}`;
+    const description = "synthetic tag description";
+    const updatedDescription = `${description} updated`;
+
+    await navigateToPage(page, DashboardPage.TagManagement);
+    await page.getByRole("button", { name: "+ Create New Tag" }).click();
+
+    try {
+      await expect(
+        page.getByRole("dialog", { name: "Create New Tag" }),
+      ).toBeVisible();
+      await page.getByLabel("Tag Name").fill(tagName);
+      await page.getByLabel("Description").fill(description);
+      await page.getByRole("button", { name: "Create Tag" }).click();
+
+      await expect
+        .poll(async () => {
+          const response = await readBack<
+            Record<string, Record<string, unknown>>
+          >(page, "/tag/list");
+          return Object.values(response).some((tag) => tag.name === tagName);
+        })
+        .toBe(true);
+      await expect(page.getByText(tagName, { exact: true })).toBeVisible();
+
+      await page.getByText(tagName, { exact: true }).click();
+      await expect(page.getByText("Tag Name:")).toBeVisible();
+      await page.getByRole("button", { name: "Edit Tag" }).click();
+      await page.getByLabel("Description").fill(updatedDescription);
+      const updateBody = await captureRequestBody(
+        page,
+        { method: "POST", urlIncludes: "/tag/update" },
+        () => page.getByRole("button", { name: "Save Changes" }).click(),
+      );
+      expect(updateBody).toMatchObject({
+        name: tagName,
+        description: updatedDescription,
+      });
+
+      await expect
+        .poll(async () => {
+          const infoResponse = await page.request.post("/tag/info", {
+            headers: { Authorization: `Bearer ${masterKey()}` },
+            data: { names: [tagName] },
+          });
+          expect(infoResponse.ok()).toBe(true);
+          const info = (await infoResponse.json()) as Record<
+            string,
+            { description?: string }
+          >;
+          return info[tagName]?.description;
+        })
+        .toBe(updatedDescription);
+    } finally {
+      await page.request.post("/tag/delete", {
+        headers: {
+          Authorization: `Bearer ${masterKey()}`,
+          "Content-Type": "application/json",
+        },
+        data: { name: tagName },
+      });
+    }
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.integration.test.tsx b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.integration.test.tsx
index 2b948ca8420..6d30f1513ef 100644
--- a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.integration.test.tsx
+++ b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.integration.test.tsx
@@ -1,5 +1,6 @@
 import { fireEvent, renderWithProviders as render, screen, waitFor } from "../../../tests/test-utils";
 import userEvent from "@testing-library/user-event";
+import { useQuery } from "@tanstack/react-query";
 import { useState } from "react";
 import { describe, expect, it, vi } from "vitest";
 
@@ -406,4 +407,48 @@ describe("PaginatedSearchSelect", () => {
     expect(input).toHaveValue("aliasalpha");
     await waitFor(() => expect(onSearchChange).toHaveBeenLastCalledWith("aliasalpha"));
   });
+
+  it("keeps the latest query results when an earlier response resolves last", async () => {
+    const pending = new Map<string, (options: SearchSelectOption[]) => void>();
+
+    function QueryBackedSelect() {
+      const [query, setQuery] = useState("");
+      const result = useQuery({
+        queryKey: ["paginated-select-race", query],
+        queryFn: () =>
+          new Promise<SearchSelectOption[]>((resolve) => {
+            pending.set(query, resolve);
+          }),
+        enabled: query.length > 0,
+      });
+      return (
+        <>
+          <button type="button" onClick={() => setQuery("A")}>
+            Search A
+          </button>
+          <button type="button" onClick={() => setQuery("B")}>
+            Search B
+          </button>
+          <PaginatedSearchSelect options={result.data ?? []} onValueChange={vi.fn()} onSearchChange={setQuery} />
+        </>
+      );
+    }
+
+    const user = userEvent.setup();
+    render(<QueryBackedSelect />);
+    await user.click(screen.getByRole("button", { name: "Search A" }));
+    await user.click(screen.getByRole("button", { name: "Search B" }));
+    await waitFor(() => {
+      expect(pending.has("A")).toBe(true);
+      expect(pending.has("B")).toBe(true);
+    });
+
+    pending.get("B")?.([{ label: "B result", value: "b" }]);
+    await user.click(screen.getByRole("combobox"));
+    expect(await screen.findByText("B result")).toBeInTheDocument();
+
+    pending.get("A")?.([{ label: "A result", value: "a" }]);
+    await waitFor(() => expect(screen.queryByText("A result")).not.toBeInTheDocument());
+    expect(screen.getByText("B result")).toBeInTheDocument();
+  });
 });
diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx
index c981010dff9..ed83acef14a 100644
--- a/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx
+++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx
@@ -111,4 +111,37 @@ describe("SearchSelect", () => {
     expect(screen.queryByText("Growth")).not.toBeInTheDocument();
     expect(onValueChange).not.toHaveBeenCalled();
   });
+
+  it("supports keyboard select, clear, escape, blur, and reopen", async () => {
+    const onValueChange = vi.fn();
+    const user = userEvent.setup();
+    function Controlled() {
+      const [value, setValue] = useState<string | null>(null);
+      return (
+        <SearchSelect
+          options={OPTIONS}
+          value={value}
+          onValueChange={(next) => {
+            setValue(next);
+            onValueChange(next);
+          }}
+        />
+      );
+    }
+
+    render(<Controlled />);
+    const input = screen.getByRole("combobox");
+    await user.tab();
+    await user.keyboard("{Enter}");
+    await user.keyboard("{ArrowDown}{Enter}");
+    expect(onValueChange).toHaveBeenLastCalledWith("team-1");
+    const clear = screen.getByRole("button", { name: "Clear" });
+    clear.focus();
+    await user.keyboard("{Enter}");
+    expect(onValueChange).toHaveBeenLastCalledWith(null);
+    await user.keyboard("{Escape}");
+    await user.tab();
+    await user.tab({ shift: true });
+    expect(input).toHaveFocus();
+  });
 });
diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx
index 5a35c7ae16b..8d1847e0121 100644
--- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx
@@ -344,4 +344,15 @@ describe("RequestLogsFilters", () => {
 
     expect(set).toHaveBeenCalledWith(LOG_FILTER_IDS.CACHE_STATUS, undefined);
   });
+
+  it("clears the raw Error Code combobox through the undefined filter contract", async () => {
+    const user = userEvent.setup();
+    const { set } = renderFilters({ [LOG_FILTER_IDS.ERROR_CODE]: "429" });
+    const input = await screen.findByPlaceholderText("Select or type an error code");
+
+    await user.click(input);
+    await user.click(screen.getByRole("button", { name: "Clear", hidden: true }));
+
+    expect(set).toHaveBeenCalledWith(LOG_FILTER_IDS.ERROR_CODE, undefined);
+  });
 });

From 8d1ca16652056512999a61c87e96ae3aaf9c236d Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Thu, 17 Sep 2026 23:06:17 -0700
Subject: [PATCH 017/160] test(ui): strengthen dashboard journey assertions

---
 tests/e2e/ui/tests/prompts/addPrompt.spec.ts  | 14 ++++++++---
 .../tests/tagManagement/tagManagement.spec.ts |  3 ++-
 ...PaginatedSearchSelect.integration.test.tsx | 25 ++++++-------------
 .../shared/SearchSelect.integration.test.tsx  |  9 +++----
 4 files changed, 24 insertions(+), 27 deletions(-)

diff --git a/tests/e2e/ui/tests/prompts/addPrompt.spec.ts b/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
index cd87e4d3b56..f9868f7b04b 100644
--- a/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
+++ b/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
@@ -13,6 +13,7 @@ test.describe("Prompt upload form", () => {
     page,
   }) => {
     const promptId = `e2e-prompt-${uniqueSuffix()}`;
+    const promptContent = "Hello {{name}}";
     await navigateToPage(page, DashboardPage.Prompts);
     await page.getByRole("button", { name: "Upload .prompt File" }).click();
 
@@ -25,7 +26,7 @@ test.describe("Prompt upload form", () => {
         name: "e2e.prompt",
         mimeType: "text/plain",
         buffer: Buffer.from(
-          'model: fake-openai-gpt-4\ntemplate: "Hello {{name}}"\n',
+          `model: fake-openai-gpt-4\ntemplate: "${promptContent}"\n`,
         ),
       });
       await expect(page.getByText("Selected: e2e.prompt")).toBeVisible();
@@ -39,17 +40,22 @@ test.describe("Prompt upload form", () => {
               headers: { Authorization: `Bearer ${masterKey()}` },
             },
           );
-          return response.ok();
+          if (!response.ok()) return undefined;
+          const promptInfo = (await response.json()) as {
+            raw_prompt_template?: { content?: string };
+          };
+          return promptInfo.raw_prompt_template?.content;
         })
-        .toBe(true);
+        .toContain(promptContent);
       await expect(page.getByText(promptId, { exact: true })).toBeVisible();
     } finally {
-      await page.request.delete(
+      const deleteResponse = await page.request.delete(
         `/prompts/${encodeURIComponent(promptId)}?environment=development`,
         {
           headers: { Authorization: `Bearer ${masterKey()}` },
         },
       );
+      expect(deleteResponse.ok()).toBe(true);
     }
   });
 });
diff --git a/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts b/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
index 785211463dd..e1d5138ea90 100644
--- a/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
+++ b/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
@@ -64,13 +64,14 @@ test.describe("Tag management", () => {
         })
         .toBe(updatedDescription);
     } finally {
-      await page.request.post("/tag/delete", {
+      const deleteResponse = await page.request.post("/tag/delete", {
         headers: {
           Authorization: `Bearer ${masterKey()}`,
           "Content-Type": "application/json",
         },
         data: { name: tagName },
       });
+      expect(deleteResponse.ok()).toBe(true);
     }
   });
 });
diff --git a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.integration.test.tsx b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.integration.test.tsx
index 6d30f1513ef..905610a77e6 100644
--- a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.integration.test.tsx
+++ b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.integration.test.tsx
@@ -421,27 +421,18 @@ describe("PaginatedSearchSelect", () => {
           }),
         enabled: query.length > 0,
       });
-      return (
-        <>
-          <button type="button" onClick={() => setQuery("A")}>
-            Search A
-          </button>
-          <button type="button" onClick={() => setQuery("B")}>
-            Search B
-          </button>
-          <PaginatedSearchSelect options={result.data ?? []} onValueChange={vi.fn()} onSearchChange={setQuery} />
-        </>
-      );
+      return <PaginatedSearchSelect options={result.data ?? []} onValueChange={vi.fn()} onSearchChange={setQuery} />;
     }
 
     const user = userEvent.setup();
     render(<QueryBackedSelect />);
-    await user.click(screen.getByRole("button", { name: "Search A" }));
-    await user.click(screen.getByRole("button", { name: "Search B" }));
-    await waitFor(() => {
-      expect(pending.has("A")).toBe(true);
-      expect(pending.has("B")).toBe(true);
-    });
+    const input = screen.getByRole("combobox");
+    await user.click(input);
+    await user.type(input, "A");
+    await waitFor(() => expect(pending.has("A")).toBe(true));
+    await user.clear(input);
+    await user.type(input, "B");
+    await waitFor(() => expect(pending.has("B")).toBe(true));
 
     pending.get("B")?.([{ label: "B result", value: "b" }]);
     await user.click(screen.getByRole("combobox"));
diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx
index ed83acef14a..9f320049b09 100644
--- a/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx
+++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx
@@ -112,7 +112,7 @@ describe("SearchSelect", () => {
     expect(onValueChange).not.toHaveBeenCalled();
   });
 
-  it("supports keyboard select, clear, escape, blur, and reopen", async () => {
+  it("supports keyboard select, clear, and reselect", async () => {
     const onValueChange = vi.fn();
     const user = userEvent.setup();
     function Controlled() {
@@ -139,9 +139,8 @@ describe("SearchSelect", () => {
     clear.focus();
     await user.keyboard("{Enter}");
     expect(onValueChange).toHaveBeenLastCalledWith(null);
-    await user.keyboard("{Escape}");
-    await user.tab();
-    await user.tab({ shift: true });
-    expect(input).toHaveFocus();
+    input.focus();
+    await user.keyboard("{Enter}{ArrowDown}{Enter}");
+    expect(onValueChange).toHaveBeenLastCalledWith("team-1");
   });
 });

From b4c3adc37d1be33550803bce9c88bc190c8f4ec6 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Thu, 17 Sep 2026 23:12:33 -0700
Subject: [PATCH 018/160] test(ui): assert dashboard form cleanup

---
 tests/e2e/ui/tests/prompts/addPrompt.spec.ts  | 39 ++++++++++++-------
 .../tests/tagManagement/tagManagement.spec.ts | 30 +++++++-------
 2 files changed, 40 insertions(+), 29 deletions(-)

diff --git a/tests/e2e/ui/tests/prompts/addPrompt.spec.ts b/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
index f9868f7b04b..a4a6cf62b7e 100644
--- a/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
+++ b/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
@@ -17,21 +17,32 @@ test.describe("Prompt upload form", () => {
     await navigateToPage(page, DashboardPage.Prompts);
     await page.getByRole("button", { name: "Upload .prompt File" }).click();
 
-    try {
-      await expect(
-        page.getByRole("dialog", { name: "Add New Prompt" }),
-      ).toBeVisible();
-      await page.getByLabel("Prompt ID").fill(promptId);
-      await page.locator('input[type="file"]').setInputFiles({
-        name: "e2e.prompt",
-        mimeType: "text/plain",
-        buffer: Buffer.from(
-          `model: fake-openai-gpt-4\ntemplate: "${promptContent}"\n`,
-        ),
-      });
-      await expect(page.getByText("Selected: e2e.prompt")).toBeVisible();
-      await page.getByRole("button", { name: "Create Prompt" }).click();
+    await expect(
+      page.getByRole("dialog", { name: "Add New Prompt" }),
+    ).toBeVisible();
+    await page.getByLabel("Prompt ID").fill(promptId);
+    await page.locator('input[type="file"]').setInputFiles({
+      name: "e2e.prompt",
+      mimeType: "text/plain",
+      buffer: Buffer.from(
+        `model: fake-openai-gpt-4\ntemplate: "${promptContent}"\n`,
+      ),
+    });
+    await expect(page.getByText("Selected: e2e.prompt")).toBeVisible();
+    await page.getByRole("button", { name: "Create Prompt" }).click();
 
+    await expect
+      .poll(async () => {
+        const response = await page.request.get(
+          `/prompts/${encodeURIComponent(promptId)}/info?environment=development`,
+          {
+            headers: { Authorization: `Bearer ${masterKey()}` },
+          },
+        );
+        return response.ok();
+      })
+      .toBe(true);
+    try {
       await expect
         .poll(async () => {
           const response = await page.request.get(
diff --git a/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts b/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
index e1d5138ea90..1104031263e 100644
--- a/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
+++ b/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
@@ -17,22 +17,22 @@ test.describe("Tag management", () => {
     await navigateToPage(page, DashboardPage.TagManagement);
     await page.getByRole("button", { name: "+ Create New Tag" }).click();
 
-    try {
-      await expect(
-        page.getByRole("dialog", { name: "Create New Tag" }),
-      ).toBeVisible();
-      await page.getByLabel("Tag Name").fill(tagName);
-      await page.getByLabel("Description").fill(description);
-      await page.getByRole("button", { name: "Create Tag" }).click();
+    await expect(
+      page.getByRole("dialog", { name: "Create New Tag" }),
+    ).toBeVisible();
+    await page.getByLabel("Tag Name").fill(tagName);
+    await page.getByLabel("Description").fill(description);
+    await page.getByRole("button", { name: "Create Tag" }).click();
 
-      await expect
-        .poll(async () => {
-          const response = await readBack<
-            Record<string, Record<string, unknown>>
-          >(page, "/tag/list");
-          return Object.values(response).some((tag) => tag.name === tagName);
-        })
-        .toBe(true);
+    await expect
+      .poll(async () => {
+        const response = await readBack<
+          Record<string, Record<string, unknown>>
+        >(page, "/tag/list");
+        return Object.values(response).some((tag) => tag.name === tagName);
+      })
+      .toBe(true);
+    try {
       await expect(page.getByText(tagName, { exact: true })).toBeVisible();
 
       await page.getByText(tagName, { exact: true }).click();

From 035271b510d5f4f4053685cf156410775d8e5d46 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Thu, 17 Sep 2026 23:15:31 -0700
Subject: [PATCH 019/160] test(ui): preserve cleanup on failed readback

---
 tests/e2e/ui/tests/prompts/addPrompt.spec.ts  | 53 +++++++++----------
 .../tests/tagManagement/tagManagement.spec.ts | 33 ++++++------
 2 files changed, 42 insertions(+), 44 deletions(-)

diff --git a/tests/e2e/ui/tests/prompts/addPrompt.spec.ts b/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
index a4a6cf62b7e..2b254c78b10 100644
--- a/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
+++ b/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
@@ -17,32 +17,32 @@ test.describe("Prompt upload form", () => {
     await navigateToPage(page, DashboardPage.Prompts);
     await page.getByRole("button", { name: "Upload .prompt File" }).click();
 
-    await expect(
-      page.getByRole("dialog", { name: "Add New Prompt" }),
-    ).toBeVisible();
-    await page.getByLabel("Prompt ID").fill(promptId);
-    await page.locator('input[type="file"]').setInputFiles({
-      name: "e2e.prompt",
-      mimeType: "text/plain",
-      buffer: Buffer.from(
-        `model: fake-openai-gpt-4\ntemplate: "${promptContent}"\n`,
-      ),
-    });
-    await expect(page.getByText("Selected: e2e.prompt")).toBeVisible();
-    await page.getByRole("button", { name: "Create Prompt" }).click();
-
-    await expect
-      .poll(async () => {
-        const response = await page.request.get(
-          `/prompts/${encodeURIComponent(promptId)}/info?environment=development`,
-          {
-            headers: { Authorization: `Bearer ${masterKey()}` },
-          },
-        );
-        return response.ok();
-      })
-      .toBe(true);
     try {
+      await expect(
+        page.getByRole("dialog", { name: "Add New Prompt" }),
+      ).toBeVisible();
+      await page.getByLabel("Prompt ID").fill(promptId);
+      await page.locator('input[type="file"]').setInputFiles({
+        name: "e2e.prompt",
+        mimeType: "text/plain",
+        buffer: Buffer.from(
+          `model: fake-openai-gpt-4\ntemplate: "${promptContent}"\n`,
+        ),
+      });
+      await expect(page.getByText("Selected: e2e.prompt")).toBeVisible();
+      await page.getByRole("button", { name: "Create Prompt" }).click();
+
+      await expect
+        .poll(async () => {
+          const response = await page.request.get(
+            `/prompts/${encodeURIComponent(promptId)}/info?environment=development`,
+            {
+              headers: { Authorization: `Bearer ${masterKey()}` },
+            },
+          );
+          return response.ok();
+        })
+        .toBe(true);
       await expect
         .poll(async () => {
           const response = await page.request.get(
@@ -60,13 +60,12 @@ test.describe("Prompt upload form", () => {
         .toContain(promptContent);
       await expect(page.getByText(promptId, { exact: true })).toBeVisible();
     } finally {
-      const deleteResponse = await page.request.delete(
+      await page.request.delete(
         `/prompts/${encodeURIComponent(promptId)}?environment=development`,
         {
           headers: { Authorization: `Bearer ${masterKey()}` },
         },
       );
-      expect(deleteResponse.ok()).toBe(true);
     }
   });
 });
diff --git a/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts b/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
index 1104031263e..785211463dd 100644
--- a/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
+++ b/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
@@ -17,22 +17,22 @@ test.describe("Tag management", () => {
     await navigateToPage(page, DashboardPage.TagManagement);
     await page.getByRole("button", { name: "+ Create New Tag" }).click();
 
-    await expect(
-      page.getByRole("dialog", { name: "Create New Tag" }),
-    ).toBeVisible();
-    await page.getByLabel("Tag Name").fill(tagName);
-    await page.getByLabel("Description").fill(description);
-    await page.getByRole("button", { name: "Create Tag" }).click();
-
-    await expect
-      .poll(async () => {
-        const response = await readBack<
-          Record<string, Record<string, unknown>>
-        >(page, "/tag/list");
-        return Object.values(response).some((tag) => tag.name === tagName);
-      })
-      .toBe(true);
     try {
+      await expect(
+        page.getByRole("dialog", { name: "Create New Tag" }),
+      ).toBeVisible();
+      await page.getByLabel("Tag Name").fill(tagName);
+      await page.getByLabel("Description").fill(description);
+      await page.getByRole("button", { name: "Create Tag" }).click();
+
+      await expect
+        .poll(async () => {
+          const response = await readBack<
+            Record<string, Record<string, unknown>>
+          >(page, "/tag/list");
+          return Object.values(response).some((tag) => tag.name === tagName);
+        })
+        .toBe(true);
       await expect(page.getByText(tagName, { exact: true })).toBeVisible();
 
       await page.getByText(tagName, { exact: true }).click();
@@ -64,14 +64,13 @@ test.describe("Tag management", () => {
         })
         .toBe(updatedDescription);
     } finally {
-      const deleteResponse = await page.request.post("/tag/delete", {
+      await page.request.post("/tag/delete", {
         headers: {
           Authorization: `Bearer ${masterKey()}`,
           "Content-Type": "application/json",
         },
         data: { name: tagName },
       });
-      expect(deleteResponse.ok()).toBe(true);
     }
   });
 });

From 43f096dde8ef6c0a0c20036f0a595a7608ea2ea1 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Thu, 17 Sep 2026 23:18:43 -0700
Subject: [PATCH 020/160] test(ui): preserve form failure evidence

---
 tests/e2e/ui/tests/prompts/addPrompt.spec.ts  | 116 +++++++++-------
 .../tests/tagManagement/tagManagement.spec.ts | 124 ++++++++++--------
 2 files changed, 136 insertions(+), 104 deletions(-)

diff --git a/tests/e2e/ui/tests/prompts/addPrompt.spec.ts b/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
index 2b254c78b10..b601b7e8c06 100644
--- a/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
+++ b/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
@@ -3,7 +3,6 @@ import { test, expect } from "@playwright/test";
 import { ADMIN_STORAGE_PATH } from "../../constants";
 import { Page as DashboardPage } from "../../fixtures/pages";
 import { navigateToPage } from "../../helpers/navigation";
-import { readBack } from "../../helpers/roundTrip";
 import { masterKey, uniqueSuffix } from "../../helpers/traffic";
 
 test.use({ storageState: ADMIN_STORAGE_PATH });
@@ -14,58 +13,75 @@ test.describe("Prompt upload form", () => {
   }) => {
     const promptId = `e2e-prompt-${uniqueSuffix()}`;
     const promptContent = "Hello {{name}}";
-    await navigateToPage(page, DashboardPage.Prompts);
-    await page.getByRole("button", { name: "Upload .prompt File" }).click();
+    const cleanup = async (): Promise<boolean> => {
+      try {
+        const response = await page.request.delete(
+          `/prompts/${encodeURIComponent(promptId)}?environment=development`,
+          {
+            headers: { Authorization: `Bearer ${masterKey()}` },
+          },
+        );
+        return response.ok();
+      } catch {
+        return false;
+      }
+    };
+    const testOutcome = await (async () => {
+      try {
+        await navigateToPage(page, DashboardPage.Prompts);
+        await page.getByRole("button", { name: "Upload .prompt File" }).click();
+        await expect(
+          page.getByRole("dialog", { name: "Add New Prompt" }),
+        ).toBeVisible();
+        await page.getByLabel("Prompt ID").fill(promptId);
+        await page.locator('input[type="file"]').setInputFiles({
+          name: "e2e.prompt",
+          mimeType: "text/plain",
+          buffer: Buffer.from(
+            `model: fake-openai-gpt-4\ntemplate: "${promptContent}"\n`,
+          ),
+        });
+        await expect(page.getByText("Selected: e2e.prompt")).toBeVisible();
+        await page.getByRole("button", { name: "Create Prompt" }).click();
+
+        await expect
+          .poll(async () => {
+            const response = await page.request.get(
+              `/prompts/${encodeURIComponent(promptId)}/info?environment=development`,
+              {
+                headers: { Authorization: `Bearer ${masterKey()}` },
+              },
+            );
+            return response.ok();
+          })
+          .toBe(true);
+        await expect
+          .poll(async () => {
+            const response = await page.request.get(
+              `/prompts/${encodeURIComponent(promptId)}/info?environment=development`,
+              {
+                headers: { Authorization: `Bearer ${masterKey()}` },
+              },
+            );
+            if (!response.ok()) return undefined;
+            const promptInfo = (await response.json()) as {
+              raw_prompt_template?: { content?: string };
+            };
+            return promptInfo.raw_prompt_template?.content;
+          })
+          .toContain(promptContent);
+        await expect(page.getByText(promptId, { exact: true })).toBeVisible();
+        return { passed: true as const };
+      } catch (error) {
+        return { passed: false as const, error };
+      }
+    })();
 
     try {
-      await expect(
-        page.getByRole("dialog", { name: "Add New Prompt" }),
-      ).toBeVisible();
-      await page.getByLabel("Prompt ID").fill(promptId);
-      await page.locator('input[type="file"]').setInputFiles({
-        name: "e2e.prompt",
-        mimeType: "text/plain",
-        buffer: Buffer.from(
-          `model: fake-openai-gpt-4\ntemplate: "${promptContent}"\n`,
-        ),
-      });
-      await expect(page.getByText("Selected: e2e.prompt")).toBeVisible();
-      await page.getByRole("button", { name: "Create Prompt" }).click();
-
-      await expect
-        .poll(async () => {
-          const response = await page.request.get(
-            `/prompts/${encodeURIComponent(promptId)}/info?environment=development`,
-            {
-              headers: { Authorization: `Bearer ${masterKey()}` },
-            },
-          );
-          return response.ok();
-        })
-        .toBe(true);
-      await expect
-        .poll(async () => {
-          const response = await page.request.get(
-            `/prompts/${encodeURIComponent(promptId)}/info?environment=development`,
-            {
-              headers: { Authorization: `Bearer ${masterKey()}` },
-            },
-          );
-          if (!response.ok()) return undefined;
-          const promptInfo = (await response.json()) as {
-            raw_prompt_template?: { content?: string };
-          };
-          return promptInfo.raw_prompt_template?.content;
-        })
-        .toContain(promptContent);
-      await expect(page.getByText(promptId, { exact: true })).toBeVisible();
+      if (!testOutcome.passed) throw testOutcome.error;
     } finally {
-      await page.request.delete(
-        `/prompts/${encodeURIComponent(promptId)}?environment=development`,
-        {
-          headers: { Authorization: `Bearer ${masterKey()}` },
-        },
-      );
+      const cleanupSucceeded = await cleanup();
+      if (testOutcome.passed) expect(cleanupSucceeded).toBe(true);
     }
   });
 });
diff --git a/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts b/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
index 785211463dd..4223324ff09 100644
--- a/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
+++ b/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
@@ -13,64 +13,80 @@ test.describe("Tag management", () => {
     const tagName = `e2e-tag-${uniqueSuffix()}`;
     const description = "synthetic tag description";
     const updatedDescription = `${description} updated`;
+    const cleanup = async (): Promise<boolean> => {
+      try {
+        const response = await page.request.post("/tag/delete", {
+          headers: {
+            Authorization: `Bearer ${masterKey()}`,
+            "Content-Type": "application/json",
+          },
+          data: { name: tagName },
+        });
+        return response.ok();
+      } catch {
+        return false;
+      }
+    };
+    const testOutcome = await (async () => {
+      try {
+        await navigateToPage(page, DashboardPage.TagManagement);
+        await page.getByRole("button", { name: "+ Create New Tag" }).click();
+        await expect(
+          page.getByRole("dialog", { name: "Create New Tag" }),
+        ).toBeVisible();
+        await page.getByLabel("Tag Name").fill(tagName);
+        await page.getByLabel("Description").fill(description);
+        await page.getByRole("button", { name: "Create Tag" }).click();
 
-    await navigateToPage(page, DashboardPage.TagManagement);
-    await page.getByRole("button", { name: "+ Create New Tag" }).click();
+        await expect
+          .poll(async () => {
+            const response = await readBack<
+              Record<string, Record<string, unknown>>
+            >(page, "/tag/list");
+            return Object.values(response).some((tag) => tag.name === tagName);
+          })
+          .toBe(true);
+        await expect(page.getByText(tagName, { exact: true })).toBeVisible();
+
+        await page.getByText(tagName, { exact: true }).click();
+        await expect(page.getByText("Tag Name:")).toBeVisible();
+        await page.getByRole("button", { name: "Edit Tag" }).click();
+        await page.getByLabel("Description").fill(updatedDescription);
+        const updateBody = await captureRequestBody(
+          page,
+          { method: "POST", urlIncludes: "/tag/update" },
+          () => page.getByRole("button", { name: "Save Changes" }).click(),
+        );
+        expect(updateBody).toMatchObject({
+          name: tagName,
+          description: updatedDescription,
+        });
+
+        await expect
+          .poll(async () => {
+            const infoResponse = await page.request.post("/tag/info", {
+              headers: { Authorization: `Bearer ${masterKey()}` },
+              data: { names: [tagName] },
+            });
+            expect(infoResponse.ok()).toBe(true);
+            const info = (await infoResponse.json()) as Record<
+              string,
+              { description?: string }
+            >;
+            return info[tagName]?.description;
+          })
+          .toBe(updatedDescription);
+        return { passed: true as const };
+      } catch (error) {
+        return { passed: false as const, error };
+      }
+    })();
 
     try {
-      await expect(
-        page.getByRole("dialog", { name: "Create New Tag" }),
-      ).toBeVisible();
-      await page.getByLabel("Tag Name").fill(tagName);
-      await page.getByLabel("Description").fill(description);
-      await page.getByRole("button", { name: "Create Tag" }).click();
-
-      await expect
-        .poll(async () => {
-          const response = await readBack<
-            Record<string, Record<string, unknown>>
-          >(page, "/tag/list");
-          return Object.values(response).some((tag) => tag.name === tagName);
-        })
-        .toBe(true);
-      await expect(page.getByText(tagName, { exact: true })).toBeVisible();
-
-      await page.getByText(tagName, { exact: true }).click();
-      await expect(page.getByText("Tag Name:")).toBeVisible();
-      await page.getByRole("button", { name: "Edit Tag" }).click();
-      await page.getByLabel("Description").fill(updatedDescription);
-      const updateBody = await captureRequestBody(
-        page,
-        { method: "POST", urlIncludes: "/tag/update" },
-        () => page.getByRole("button", { name: "Save Changes" }).click(),
-      );
-      expect(updateBody).toMatchObject({
-        name: tagName,
-        description: updatedDescription,
-      });
-
-      await expect
-        .poll(async () => {
-          const infoResponse = await page.request.post("/tag/info", {
-            headers: { Authorization: `Bearer ${masterKey()}` },
-            data: { names: [tagName] },
-          });
-          expect(infoResponse.ok()).toBe(true);
-          const info = (await infoResponse.json()) as Record<
-            string,
-            { description?: string }
-          >;
-          return info[tagName]?.description;
-        })
-        .toBe(updatedDescription);
+      if (!testOutcome.passed) throw testOutcome.error;
     } finally {
-      await page.request.post("/tag/delete", {
-        headers: {
-          Authorization: `Bearer ${masterKey()}`,
-          "Content-Type": "application/json",
-        },
-        data: { name: tagName },
-      });
+      const cleanupSucceeded = await cleanup();
+      if (testOutcome.passed) expect(cleanupSucceeded).toBe(true);
     }
   });
 });

From 0dc2f0b1c1ff86356d6e9ab9180f708402131df8 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Thu, 17 Sep 2026 23:20:54 -0700
Subject: [PATCH 021/160] test(ui): protect dashboard form cleanup

---
 tests/e2e/ui/helpers/roundTrip.ts             | 28 ++++++++++-
 tests/e2e/ui/tests/prompts/addPrompt.spec.ts  | 42 ++++++----------
 .../tests/tagManagement/tagManagement.spec.ts | 49 ++++++++-----------
 3 files changed, 61 insertions(+), 58 deletions(-)

diff --git a/tests/e2e/ui/helpers/roundTrip.ts b/tests/e2e/ui/helpers/roundTrip.ts
index 8d6e264e622..ee484b8d512 100644
--- a/tests/e2e/ui/helpers/roundTrip.ts
+++ b/tests/e2e/ui/helpers/roundTrip.ts
@@ -12,17 +12,41 @@ export async function captureRequestBody(
   match: { method: string; urlIncludes: string },
   action: () => Promise<void>,
 ): Promise<Record<string, any>> {
-  const pending = page.waitForRequest((req) => req.method() === match.method && req.url().includes(match.urlIncludes));
+  const pending = page.waitForRequest(
+    (req) =>
+      req.method() === match.method && req.url().includes(match.urlIncludes),
+  );
   await action();
   const request = await pending;
   return JSON.parse(request.postData() ?? "{}") as Record<string, any>;
 }
 
 /** Reads an endpoint as the master key, so a failure is bad data and not an expired UI token. */
-export async function readBack<T = any>(page: Page, endpoint: string): Promise<T> {
+export async function readBack<T = any>(
+  page: Page,
+  endpoint: string,
+): Promise<T> {
   const res = await page.request.get(endpoint, {
     headers: { Authorization: `Bearer ${masterKey()}` },
   });
   expect(res.ok(), `GET ${endpoint}`).toBe(true);
   return (await res.json()) as T;
 }
+
+export async function runWithCleanup(
+  action: () => Promise<void>,
+  cleanup: () => Promise<boolean>,
+): Promise<void> {
+  const outcome = await action().then(
+    () => ({ status: "success" as const }),
+    (error: unknown) => ({ status: "failure" as const, error }),
+  );
+  try {
+    if (outcome.status === "failure") throw outcome.error;
+  } finally {
+    const cleanupSucceeded = await cleanup().catch(() => false);
+    if (outcome.status === "success" && !cleanupSucceeded) {
+      throw new Error("Failed to clean up UI E2E resource");
+    }
+  }
+}
diff --git a/tests/e2e/ui/tests/prompts/addPrompt.spec.ts b/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
index b601b7e8c06..891739fda28 100644
--- a/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
+++ b/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
@@ -3,6 +3,7 @@ import { test, expect } from "@playwright/test";
 import { ADMIN_STORAGE_PATH } from "../../constants";
 import { Page as DashboardPage } from "../../fixtures/pages";
 import { navigateToPage } from "../../helpers/navigation";
+import { runWithCleanup } from "../../helpers/roundTrip";
 import { masterKey, uniqueSuffix } from "../../helpers/traffic";
 
 test.use({ storageState: ADMIN_STORAGE_PATH });
@@ -13,21 +14,9 @@ test.describe("Prompt upload form", () => {
   }) => {
     const promptId = `e2e-prompt-${uniqueSuffix()}`;
     const promptContent = "Hello {{name}}";
-    const cleanup = async (): Promise<boolean> => {
-      try {
-        const response = await page.request.delete(
-          `/prompts/${encodeURIComponent(promptId)}?environment=development`,
-          {
-            headers: { Authorization: `Bearer ${masterKey()}` },
-          },
-        );
-        return response.ok();
-      } catch {
-        return false;
-      }
-    };
-    const testOutcome = await (async () => {
-      try {
+
+    await runWithCleanup(
+      async () => {
         await navigateToPage(page, DashboardPage.Prompts);
         await page.getByRole("button", { name: "Upload .prompt File" }).click();
         await expect(
@@ -71,17 +60,16 @@ test.describe("Prompt upload form", () => {
           })
           .toContain(promptContent);
         await expect(page.getByText(promptId, { exact: true })).toBeVisible();
-        return { passed: true as const };
-      } catch (error) {
-        return { passed: false as const, error };
-      }
-    })();
-
-    try {
-      if (!testOutcome.passed) throw testOutcome.error;
-    } finally {
-      const cleanupSucceeded = await cleanup();
-      if (testOutcome.passed) expect(cleanupSucceeded).toBe(true);
-    }
+      },
+      async () => {
+        const response = await page.request.delete(
+          `/prompts/${encodeURIComponent(promptId)}?environment=development`,
+          {
+            headers: { Authorization: `Bearer ${masterKey()}` },
+          },
+        );
+        return response.ok();
+      },
+    );
   });
 });
diff --git a/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts b/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
index 4223324ff09..bf46b5ea161 100644
--- a/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
+++ b/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
@@ -3,7 +3,11 @@ import { test, expect } from "@playwright/test";
 import { ADMIN_STORAGE_PATH } from "../../constants";
 import { Page as DashboardPage } from "../../fixtures/pages";
 import { navigateToPage } from "../../helpers/navigation";
-import { captureRequestBody, readBack } from "../../helpers/roundTrip";
+import {
+  captureRequestBody,
+  readBack,
+  runWithCleanup,
+} from "../../helpers/roundTrip";
 import { masterKey, uniqueSuffix } from "../../helpers/traffic";
 
 test.use({ storageState: ADMIN_STORAGE_PATH });
@@ -13,22 +17,9 @@ test.describe("Tag management", () => {
     const tagName = `e2e-tag-${uniqueSuffix()}`;
     const description = "synthetic tag description";
     const updatedDescription = `${description} updated`;
-    const cleanup = async (): Promise<boolean> => {
-      try {
-        const response = await page.request.post("/tag/delete", {
-          headers: {
-            Authorization: `Bearer ${masterKey()}`,
-            "Content-Type": "application/json",
-          },
-          data: { name: tagName },
-        });
-        return response.ok();
-      } catch {
-        return false;
-      }
-    };
-    const testOutcome = await (async () => {
-      try {
+
+    await runWithCleanup(
+      async () => {
         await navigateToPage(page, DashboardPage.TagManagement);
         await page.getByRole("button", { name: "+ Create New Tag" }).click();
         await expect(
@@ -76,17 +67,17 @@ test.describe("Tag management", () => {
             return info[tagName]?.description;
           })
           .toBe(updatedDescription);
-        return { passed: true as const };
-      } catch (error) {
-        return { passed: false as const, error };
-      }
-    })();
-
-    try {
-      if (!testOutcome.passed) throw testOutcome.error;
-    } finally {
-      const cleanupSucceeded = await cleanup();
-      if (testOutcome.passed) expect(cleanupSucceeded).toBe(true);
-    }
+      },
+      async () => {
+        const response = await page.request.post("/tag/delete", {
+          headers: {
+            Authorization: `Bearer ${masterKey()}`,
+            "Content-Type": "application/json",
+          },
+          data: { name: tagName },
+        });
+        return response.ok();
+      },
+    );
   });
 });

From a8ab1187ca67f59432beed8b655e833f622a4055 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Thu, 17 Sep 2026 23:23:48 -0700
Subject: [PATCH 022/160] test(ui): clean up synchronous browser failures

---
 tests/e2e/ui/helpers/roundTrip.ts | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tests/e2e/ui/helpers/roundTrip.ts b/tests/e2e/ui/helpers/roundTrip.ts
index ee484b8d512..55eb5d6ad9c 100644
--- a/tests/e2e/ui/helpers/roundTrip.ts
+++ b/tests/e2e/ui/helpers/roundTrip.ts
@@ -37,10 +37,12 @@ export async function runWithCleanup(
   action: () => Promise<void>,
   cleanup: () => Promise<boolean>,
 ): Promise<void> {
-  const outcome = await action().then(
-    () => ({ status: "success" as const }),
-    (error: unknown) => ({ status: "failure" as const, error }),
-  );
+  const outcome = await Promise.resolve()
+    .then(action)
+    .then(
+      () => ({ status: "success" as const }),
+      (error: unknown) => ({ status: "failure" as const, error }),
+    );
   try {
     if (outcome.status === "failure") throw outcome.error;
   } finally {

From 1c08c78ad598f0fa1277305f5a32aa7ac45a2022 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Thu, 17 Sep 2026 23:27:54 -0700
Subject: [PATCH 023/160] test(ui): retain primary cleanup failures

---
 tests/e2e/ui/helpers/roundTrip.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/e2e/ui/helpers/roundTrip.ts b/tests/e2e/ui/helpers/roundTrip.ts
index 55eb5d6ad9c..4175ba6ec72 100644
--- a/tests/e2e/ui/helpers/roundTrip.ts
+++ b/tests/e2e/ui/helpers/roundTrip.ts
@@ -46,7 +46,9 @@ export async function runWithCleanup(
   try {
     if (outcome.status === "failure") throw outcome.error;
   } finally {
-    const cleanupSucceeded = await cleanup().catch(() => false);
+    const cleanupSucceeded = await Promise.resolve()
+      .then(cleanup)
+      .catch(() => false);
     if (outcome.status === "success" && !cleanupSucceeded) {
       throw new Error("Failed to clean up UI E2E resource");
     }

From eb831d956ccb328411ebb86a0161ac7a23b4aba8 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Thu, 17 Sep 2026 23:46:53 -0700
Subject: [PATCH 024/160] test(ui): address review feedback

---
 tests/e2e/ui/helpers/roundTrip.ts             | 23 +++++++++---
 tests/e2e/ui/tests/prompts/addPrompt.spec.ts  |  4 +--
 .../tests/tagManagement/tagManagement.spec.ts |  9 ++---
 ...PaginatedSearchSelect.integration.test.tsx | 36 -------------------
 4 files changed, 26 insertions(+), 46 deletions(-)

diff --git a/tests/e2e/ui/helpers/roundTrip.ts b/tests/e2e/ui/helpers/roundTrip.ts
index 4175ba6ec72..1fc2d0aec1a 100644
--- a/tests/e2e/ui/helpers/roundTrip.ts
+++ b/tests/e2e/ui/helpers/roundTrip.ts
@@ -46,11 +46,26 @@ export async function runWithCleanup(
   try {
     if (outcome.status === "failure") throw outcome.error;
   } finally {
-    const cleanupSucceeded = await Promise.resolve()
+    const cleanupOutcome = await Promise.resolve()
       .then(cleanup)
-      .catch(() => false);
-    if (outcome.status === "success" && !cleanupSucceeded) {
-      throw new Error("Failed to clean up UI E2E resource");
+      .then(
+        (succeeded) =>
+          succeeded
+            ? { status: "success" as const }
+            : {
+                status: "failure" as const,
+                error: new Error("Failed to clean up UI E2E resource"),
+              },
+        (error: unknown) => ({ status: "failure" as const, error }),
+      );
+    if (cleanupOutcome.status === "failure") {
+      if (outcome.status === "failure") {
+        throw new AggregateError(
+          [outcome.error, cleanupOutcome.error],
+          "Action and cleanup failed",
+        );
+      }
+      throw cleanupOutcome.error;
     }
   }
 }
diff --git a/tests/e2e/ui/tests/prompts/addPrompt.spec.ts b/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
index 891739fda28..9d85236c4a6 100644
--- a/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
+++ b/tests/e2e/ui/tests/prompts/addPrompt.spec.ts
@@ -27,7 +27,7 @@ test.describe("Prompt upload form", () => {
           name: "e2e.prompt",
           mimeType: "text/plain",
           buffer: Buffer.from(
-            `model: fake-openai-gpt-4\ntemplate: "${promptContent}"\n`,
+            `---\nmodel: fake-openai-gpt-4\n---\n${promptContent}\n`,
           ),
         });
         await expect(page.getByText("Selected: e2e.prompt")).toBeVisible();
@@ -58,7 +58,7 @@ test.describe("Prompt upload form", () => {
             };
             return promptInfo.raw_prompt_template?.content;
           })
-          .toContain(promptContent);
+          .toBe(promptContent);
         await expect(page.getByText(promptId, { exact: true })).toBeVisible();
       },
       async () => {
diff --git a/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts b/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
index bf46b5ea161..fe659080eab 100644
--- a/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
+++ b/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts
@@ -31,10 +31,11 @@ test.describe("Tag management", () => {
 
         await expect
           .poll(async () => {
-            const response = await readBack<
-              Record<string, Record<string, unknown>>
-            >(page, "/tag/list");
-            return Object.values(response).some((tag) => tag.name === tagName);
+            const response = await readBack<Array<{ name: string }>>(
+              page,
+              "/tag/list",
+            );
+            return response.some((tag) => tag.name === tagName);
           })
           .toBe(true);
         await expect(page.getByText(tagName, { exact: true })).toBeVisible();
diff --git a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.integration.test.tsx b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.integration.test.tsx
index 905610a77e6..2b948ca8420 100644
--- a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.integration.test.tsx
+++ b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.integration.test.tsx
@@ -1,6 +1,5 @@
 import { fireEvent, renderWithProviders as render, screen, waitFor } from "../../../tests/test-utils";
 import userEvent from "@testing-library/user-event";
-import { useQuery } from "@tanstack/react-query";
 import { useState } from "react";
 import { describe, expect, it, vi } from "vitest";
 
@@ -407,39 +406,4 @@ describe("PaginatedSearchSelect", () => {
     expect(input).toHaveValue("aliasalpha");
     await waitFor(() => expect(onSearchChange).toHaveBeenLastCalledWith("aliasalpha"));
   });
-
-  it("keeps the latest query results when an earlier response resolves last", async () => {
-    const pending = new Map<string, (options: SearchSelectOption[]) => void>();
-
-    function QueryBackedSelect() {
-      const [query, setQuery] = useState("");
-      const result = useQuery({
-        queryKey: ["paginated-select-race", query],
-        queryFn: () =>
-          new Promise<SearchSelectOption[]>((resolve) => {
-            pending.set(query, resolve);
-          }),
-        enabled: query.length > 0,
-      });
-      return <PaginatedSearchSelect options={result.data ?? []} onValueChange={vi.fn()} onSearchChange={setQuery} />;
-    }
-
-    const user = userEvent.setup();
-    render(<QueryBackedSelect />);
-    const input = screen.getByRole("combobox");
-    await user.click(input);
-    await user.type(input, "A");
-    await waitFor(() => expect(pending.has("A")).toBe(true));
-    await user.clear(input);
-    await user.type(input, "B");
-    await waitFor(() => expect(pending.has("B")).toBe(true));
-
-    pending.get("B")?.([{ label: "B result", value: "b" }]);
-    await user.click(screen.getByRole("combobox"));
-    expect(await screen.findByText("B result")).toBeInTheDocument();
-
-    pending.get("A")?.([{ label: "A result", value: "a" }]);
-    await waitFor(() => expect(screen.queryByText("A result")).not.toBeInTheDocument());
-    expect(screen.getByText("B result")).toBeInTheDocument();
-  });
 });

From 5a8d1f5ecaedac9348f93fd87873e7a16b9fba0f Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Fri, 18 Sep 2026 00:05:43 -0700
Subject: [PATCH 025/160] feat(proxy): report sources in config read endpoints

---
 litellm/proxy/_types.py                       |   3 +
 litellm/proxy/config_resolvers/__init__.py    |  11 +-
 .../proxy/config_resolvers/settings_store.py  |  12 +-
 .../router_settings_endpoints.py              |  12 +-
 litellm/proxy/proxy_server.py                 | 168 ++++++++++--------
 .../proxy_setting_endpoints.py                |  65 +++++--
 .../test_router_settings_endpoints.py         |  30 ++++
 .../proxy/proxy_server/test_routes_config.py  | 137 ++++++++++++++
 .../proxy_server/test_routes_model_metrics.py |  38 ++++
 .../test_proxy_setting_endpoints.py           |  39 ++++
 ui/litellm-dashboard/src/lib/http/schema.d.ts |  29 +++
 11 files changed, 450 insertions(+), 94 deletions(-)

diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index b0d31df92ce..8574f2d8bf4 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -2410,6 +2410,7 @@ class FieldDetail(BaseModel):
     field_description: str
     field_default_value: Any = None
     stored_in_db: bool | None
+    source: Literal["config", "db", "default", "unset"] = "unset"
 
 
 class ConfigList(LiteLLMPydanticObjectBase):
@@ -2418,6 +2419,7 @@ class ConfigList(LiteLLMPydanticObjectBase):
     field_description: str
     field_value: Any
     stored_in_db: bool | None
+    source: Literal["config", "db", "default", "unset"] = "unset"
     field_default_value: Any
     premium_field: bool = False
     nested_fields: list[FieldDetail] | None = None  # For nested dictionary or Pydantic fields
@@ -3693,6 +3695,7 @@ class InvitationClaim(LiteLLMPydanticObjectBase):
 class ConfigFieldInfo(LiteLLMPydanticObjectBase):
     field_name: str
     field_value: Any
+    source: Literal["config", "db", "default", "unset"] = "unset"
 
 
 class CallbackOnUI(LiteLLMPydanticObjectBase):
diff --git a/litellm/proxy/config_resolvers/__init__.py b/litellm/proxy/config_resolvers/__init__.py
index ebd339b34c3..77da2c413c2 100644
--- a/litellm/proxy/config_resolvers/__init__.py
+++ b/litellm/proxy/config_resolvers/__init__.py
@@ -5,6 +5,13 @@ from litellm.proxy.config_resolvers._descriptors import (
     FieldSource,
     resolve_fields,
 )
-from litellm.proxy.config_resolvers.settings_store import SettingsStore
+from litellm.proxy.config_resolvers.settings_store import SettingsSource, SettingsStore, source_for
 
-__all__ = ("FieldDescriptor", "FieldSource", "SettingsStore", "resolve_fields")
+__all__ = (
+    "FieldDescriptor",
+    "FieldSource",
+    "SettingsSource",
+    "SettingsStore",
+    "resolve_fields",
+    "source_for",
+)
diff --git a/litellm/proxy/config_resolvers/settings_store.py b/litellm/proxy/config_resolvers/settings_store.py
index 3fe869ee2ce..8f400853fa9 100644
--- a/litellm/proxy/config_resolvers/settings_store.py
+++ b/litellm/proxy/config_resolvers/settings_store.py
@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from collections.abc import Iterator, Mapping, MutableMapping
 from types import MappingProxyType
-from typing import Final
+from typing import Final, Literal, TypeAlias
 
 from litellm.proxy.config_resolvers._descriptors import FieldSource
 from litellm.proxy.config_resolvers.settings_rules import (
@@ -19,6 +19,7 @@ from litellm.proxy.config_resolvers.settings_rules import (
 
 _EMPTY_VALUES: Final[Mapping[str, JsonValue]] = MappingProxyType({})
 _EMPTY_ROWS: Final[Mapping[DbRow, Mapping[str, JsonValue]]] = MappingProxyType({})
+SettingsSource: TypeAlias = Literal["config", "db", "default", "unset"]
 
 
 class SettingsStore(MutableMapping[str, JsonValue]):
@@ -109,3 +110,12 @@ class SettingsStore(MutableMapping[str, JsonValue]):
         yaml_value: Final[SettingValue] = self._yaml_values.get(key, ABSENT)
         db_value: Final[SettingValue] = self._database_rows.get(rule.db_row, _EMPTY_VALUES).get(key, ABSENT)
         return resolve(rule, yaml_value, db_value)
+
+
+def source_for(settings: SettingsStore, key: str, default: object = None) -> SettingsSource:
+    source: Final = settings.source(key)
+    if source == "unset":
+        return "default" if default is not None else "unset"
+    if source in ("config", "db", "default"):
+        return source
+    return "unset"
diff --git a/litellm/proxy/management_endpoints/router_settings_endpoints.py b/litellm/proxy/management_endpoints/router_settings_endpoints.py
index fc000b1638b..5d3b6d40601 100644
--- a/litellm/proxy/management_endpoints/router_settings_endpoints.py
+++ b/litellm/proxy/management_endpoints/router_settings_endpoints.py
@@ -8,7 +8,7 @@ GET /router/fields - Get router settings field definitions without values (for U
 """
 
 import inspect
-from typing import Any, Final, get_args
+from typing import Any, Final, cast, get_args
 
 from fastapi import APIRouter, Depends
 from pydantic import BaseModel, Field
@@ -16,6 +16,7 @@ from pydantic import BaseModel, Field
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.config_resolvers import SettingsSource, source_for
 from litellm.router import Router
 from litellm.types.management_endpoints import (
     ROUTER_SETTINGS_FIELDS,
@@ -30,6 +31,7 @@ class RouterSettingsResponse(BaseModel):
     fields: list[RouterSettingsField] = Field(description="List of all configurable router settings with metadata")
     current_values: dict[str, Any] = Field(description="Current values of router settings")
     routing_strategy_descriptions: dict[str, str] = Field(description="Descriptions for each routing strategy option")
+    source: dict[str, SettingsSource] = Field(description="Source of each current router setting")
 
 
 class RouterFieldsResponse(BaseModel):
@@ -109,15 +111,21 @@ async def get_router_settings(
         # Merge with config values (config takes precedence)
         current_values.update(router_settings_from_config)
 
-        # Update field values with current values
         for field in router_fields:
             if field.field_name in current_values:
                 field.field_value = current_values[field.field_name]
 
+        field_defaults: Final[dict[str, object]] = {
+            field.field_name: cast(object, field.field_default) for field in router_fields
+        }
+        source: Final[dict[str, SettingsSource]] = {
+            key: source_for(proxy_config.router_settings, key, field_defaults.get(key)) for key in current_values
+        }
         return RouterSettingsResponse(
             fields=router_fields,
             current_values=current_values,
             routing_strategy_descriptions=ROUTING_STRATEGY_DESCRIPTIONS,
+            source=source,
         )
     except Exception as e:
         verbose_proxy_logger.error("Error fetching router settings: %s", e)
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 6a720a066b4..1131d7bea86 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -50,6 +50,7 @@ import anyio
 import websockets
 import websockets.exceptions
 from pydantic import BaseModel, Json, JsonValue, TypeAdapter, ValidationError
+from pydantic.fields import FieldInfo, PydanticUndefined
 from typing_extensions import NotRequired, ReadOnly, assert_never
 
 from litellm._uuid import uuid
@@ -431,7 +432,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
     model_access_group_spend_counter_key,
     tag_cache_key,
 )
-from litellm.proxy.config_resolvers import SettingsStore, resolve_fields
+from litellm.proxy.config_resolvers import SettingsStore, resolve_fields, source_for
 from litellm.proxy.config_resolvers.alerting import (
     EMAIL_DESCRIPTORS,
     MS_TEAMS_DESCRIPTORS,
@@ -4816,6 +4817,12 @@ def _as_settings_mapping(value: object) -> Mapping[str, SettingsJsonValue]:
     return _SETTINGS_MAPPING.validate_python(value)
 
 
+def _get_field_default(field_info: FieldInfo) -> JsonValue:
+    if field_info.default is PydanticUndefined:
+        return None
+    return cast(JsonValue, field_info.default)
+
+
 def _bind_general_settings_store(settings: SettingsStore) -> None:
     global general_settings
     general_settings = settings  # pyright: ignore[reportAssignmentType]  # legacy global accepts mappings
@@ -15635,17 +15642,16 @@ async def alerting_settings(
         where={"param_name": "general_settings"}
     )
 
-    if db_general_settings is not None and db_general_settings.param_value is not None:
-        db_general_settings_dict: Final = dict(db_general_settings.param_value)
-        alerting_args_dict: dict = cast(  # cast-ok: ConfigGeneralSettings validates alerting_args as a dict on write
-            dict[str, JsonValue], db_general_settings_dict.get("alerting_args", {})
-        )
-        alerting_values: list | None = cast(  # cast-ok: ConfigGeneralSettings validates alerting as a list on write
-            list[JsonValue] | None, db_general_settings_dict.get("alerting")
-        )
-    else:
-        alerting_args_dict = {}
-        alerting_values = None
+    db_general_settings_dict: Final[Mapping[str, JsonValue]] = (
+        dict(db_general_settings.param_value)
+        if db_general_settings is not None and db_general_settings.param_value is not None
+        else {}
+    )
+    alerting_args_dict: Final = cast(dict[str, JsonValue], db_general_settings_dict.get("alerting_args", {}))
+    alerting_values: Final = cast(list[JsonValue] | None, db_general_settings_dict.get("alerting"))
+
+    settings: Final = proxy_config.settings
+    settings.apply_db_row("general_settings", db_general_settings_dict)
 
     allowed_args: Final = MappingProxyType(
         {
@@ -15674,9 +15680,9 @@ async def alerting_settings(
 
     is_slack_enabled = False
 
-    if general_settings.get("alerting") and isinstance(general_settings["alerting"], list):
-        if "slack" in general_settings["alerting"]:
-            is_slack_enabled = True
+    alerting: Final = settings.get("alerting")
+    if isinstance(alerting, list) and "slack" in alerting:
+        is_slack_enabled = True
 
     _response_obj = ConfigList(
         field_name="slack_alerting",
@@ -15684,6 +15690,7 @@ async def alerting_settings(
         field_description="Enable slack alerting for monitoring proxy in production: llm outages, budgets, spend tracking failures.",
         field_value=is_slack_enabled,
         stored_in_db=True if alerting_values is not None else False,
+        source=source_for(settings, "alerting"),
         field_default_value=None,
         premium_field=False,
     )
@@ -15691,6 +15698,7 @@ async def alerting_settings(
 
     for field_name, field_info in SlackAlertingArgs.model_fields.items():
         if field_name in allowed_args:
+            field_default: JsonValue = _get_field_default(field_info)
             _stored_in_db: bool | None = None
             if field_name in alerting_args_dict:
                 _stored_in_db = True
@@ -15701,9 +15709,10 @@ async def alerting_settings(
                 field_name=field_name,
                 field_type=allowed_args[field_name],
                 field_description=field_info.description or "",
-                field_value=_slack_alerting_args_dict.get(field_name, None),
+                field_value=_slack_alerting_args_dict.get(field_name, field_default),
                 stored_in_db=_stored_in_db,
-                field_default_value=field_info.default,
+                source=source_for(settings, "alerting_args", field_default),
+                field_default_value=field_default,
                 premium_field=(True if field_name == "region_outage_alert_ttl" else False),
             )
             return_val.append(_response_obj)
@@ -17390,20 +17399,6 @@ async def get_config_general_settings(
     field_name: str,
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
 ):
-    global prisma_client
-
-    ## VALIDATION ##
-    """
-    - Check if prisma_client is None
-    - Check if user allowed to call this endpoint (admin-only)
-    - Check if param in general settings
-    """
-    if prisma_client is None:
-        raise HTTPException(
-            status_code=400,
-            detail={"error": CommonProxyErrors.db_not_connected_error.value},
-        )
-
     if not _user_has_admin_view(user_api_key_dict):
         raise HTTPException(
             status_code=400,
@@ -17416,37 +17411,47 @@ async def get_config_general_settings(
             detail={"error": f"Invalid field={field_name} passed in."},
         )
 
-    ## get general settings from db
-    db_general_settings: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
-        where={"param_name": "general_settings"}
-    )
-    ### pop the value
+    field_info: Final = ConfigGeneralSettings.model_fields[field_name]
+    field_default: JsonValue = _get_field_default(field_info)
+    settings: Final = proxy_config.settings
+    db_values: Mapping[str, JsonValue]
+    if prisma_client is None:
+        db_values = {}
+    else:
+        db_general_settings: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
+            where={"param_name": "general_settings"}
+        )
+        db_values = (
+            dict(db_general_settings.param_value)
+            if db_general_settings is not None and db_general_settings.param_value is not None
+            else {}
+        )
+        settings.apply_db_row("general_settings", db_values)
 
-    if db_general_settings is None or db_general_settings.param_value is None:
+    if field_name not in settings and field_default is None:
         raise HTTPException(
             status_code=400,
             detail={"error": f"Field name={field_name} not in DB"},
         )
-    else:
-        general_settings = dict(db_general_settings.param_value)
 
-        if field_name in general_settings:
-            field_value = _redact_general_setting_value(
-                field_name,
-                general_settings[field_name],
-                user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN,
-            )
-            if field_name == "plugins" and isinstance(field_value, list):
-                field_value = [
-                    ({k: ("***" if k == "plugin_key" else v) for k, v in p.items()} if isinstance(p, dict) else p)
-                    for p in field_value
-                ]
-            return ConfigFieldInfo(field_name=field_name, field_value=field_value)
-        else:
-            raise HTTPException(
-                status_code=400,
-                detail={"error": f"Field name={field_name} not in DB"},
-            )
+    redacted_field_value: Final = _redact_general_setting_value(
+        field_name,
+        settings.get(field_name, field_default),
+        user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN,
+    )
+    field_value: Final = (
+        [
+            ({k: ("***" if k == "plugin_key" else v) for k, v in p.items()} if isinstance(p, dict) else p)
+            for p in redacted_field_value
+        ]
+        if field_name == "plugins" and isinstance(redacted_field_value, list)
+        else redacted_field_value
+    )
+    return ConfigFieldInfo(
+        field_name=field_name,
+        field_value=field_value,
+        source=source_for(settings, field_name, field_default),
+    )
 
 
 GeneralSettingsUILiteLLMValue = float | bool | str | None
@@ -17600,7 +17605,7 @@ async def get_config_list(
     """
     List the available fields + current values for a given type of setting (currently just 'general_settings'user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),)
     """
-    global prisma_client, general_settings
+    global prisma_client
 
     ## VALIDATION ##
     """
@@ -17627,10 +17632,16 @@ async def get_config_list(
         where={"param_name": "general_settings"}
     )
 
-    if db_general_settings is not None and db_general_settings.param_value is not None:
-        db_general_settings_dict: Mapping[str, JsonValue] = dict(db_general_settings.param_value)
-    else:
-        db_general_settings_dict = {}
+    db_general_settings_dict: Final[Mapping[str, JsonValue]] = (
+        dict(db_general_settings.param_value)
+        if db_general_settings is not None and db_general_settings.param_value is not None
+        else {}
+    )
+    settings: Final = proxy_config.settings
+    settings.apply_db_row("general_settings", db_general_settings_dict)
+    runtime_settings: Final[Mapping[str, JsonValue]] = (
+        cast(Mapping[str, JsonValue], general_settings) if not isinstance(general_settings, SettingsStore) else settings
+    )
 
     allowed_args: Final = _GENERAL_SETTINGS_CONFIG_LIST_FIELD_TYPES
 
@@ -17638,6 +17649,7 @@ async def get_config_list(
 
     for field_name, field_info in ConfigGeneralSettings.model_fields.items():
         if field_name in allowed_args:
+            field_default: JsonValue = _get_field_default(field_info)
             ## HANDLE TYPED DICT
 
             typed_dict_type = allowed_args[field_name]
@@ -17657,10 +17669,11 @@ async def get_config_list(
                             field_description="",  # Add custom logic if descriptions are available
                             field_default_value=_redact_general_setting_value(
                                 sub_field,
-                                general_settings.get(sub_field, None),
+                                runtime_settings.get(sub_field, None),
                                 is_full_admin,
                             ),
                             stored_in_db=None,
+                            source=source_for(settings, field_name),
                         )
                         for sub_field, sub_field_type in pydantic_class.__annotations__.items()
                     ]
@@ -17677,7 +17690,7 @@ async def get_config_list(
                     _stored_in_db = None
                     if field_name in db_general_settings_dict:
                         _stored_in_db = True
-                    elif field_name in general_settings:
+                    elif field_name in runtime_settings:
                         _stored_in_db = False
 
                     _response_obj = ConfigList(
@@ -17686,11 +17699,12 @@ async def get_config_list(
                         field_description=field_info.description or "",
                         field_value=_redact_general_setting_value(
                             field_name,
-                            general_settings.get(field_name, None),
+                            runtime_settings.get(field_name, field_default),
                             is_full_admin,
                         ),
                         stored_in_db=_stored_in_db,
-                        field_default_value=field_info.default,
+                        source=source_for(settings, field_name, field_default),
+                        field_default_value=field_default,
                         nested_fields=nested_fields,
                     )
                     return_val.append(_response_obj)
@@ -17701,12 +17715,10 @@ async def get_config_list(
                 _stored_in_db = None
                 if field_name in db_general_settings_dict:
                     _stored_in_db = True
-                elif field_name in general_settings:
+                elif field_name in runtime_settings:
                     _stored_in_db = False
 
-                _field_value = general_settings.get(field_name, None)
-                if _field_value is None and field_name in db_general_settings_dict:
-                    _field_value = db_general_settings_dict[field_name]
+                _field_value: JsonValue = runtime_settings.get(field_name, field_default)
 
                 _response_obj = ConfigList(
                     field_name=field_name,
@@ -17714,7 +17726,8 @@ async def get_config_list(
                     field_description=field_info.description or "",
                     field_value=_redact_general_setting_value(field_name, _field_value, is_full_admin),
                     stored_in_db=_stored_in_db,
-                    field_default_value=field_info.default,
+                    source=source_for(settings, field_name, field_default),
+                    field_default_value=field_default,
                     nested_fields=nested_fields,
                 )
                 return_val.append(_response_obj)
@@ -17722,18 +17735,24 @@ async def get_config_list(
     db_litellm_settings_row: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
         where={"param_name": "litellm_settings"}
     )
-    db_litellm_settings: Final[dict] = (
+    db_litellm_settings: Final[Mapping[str, JsonValue]] = (
         dict(db_litellm_settings_row.param_value)
         if db_litellm_settings_row is not None and db_litellm_settings_row.param_value is not None
         else {}
     )
+    litellm_settings_store: Final = proxy_config.litellm_settings
+    litellm_settings_store.apply_db_row("litellm_settings", db_litellm_settings)
     for litellm_field_name, spec in _GENERAL_SETTINGS_UI_LITELLM_FIELDS.items():
-        current_value: GeneralSettingsUILiteLLMValue = getattr(litellm, litellm_field_name, None)
-        default_value = _general_settings_ui_litellm_default(spec)
+        default_value: GeneralSettingsUILiteLLMValue = _general_settings_ui_litellm_default(spec)
+        current_value: GeneralSettingsUILiteLLMValue = cast(
+            GeneralSettingsUILiteLLMValue,
+            litellm_settings_store.get(litellm_field_name, default_value),
+        )
+        source = source_for(litellm_settings_store, litellm_field_name, default_value)
         stored_in_db_litellm: bool | None
         if litellm_field_name in db_litellm_settings:
             stored_in_db_litellm = True
-        elif current_value != default_value:
+        elif source == "config":
             stored_in_db_litellm = False
         else:
             stored_in_db_litellm = None
@@ -17744,6 +17763,7 @@ async def get_config_list(
                 field_description=spec["description"],
                 field_value=current_value,
                 stored_in_db=stored_in_db_litellm,
+                source=source,
                 field_default_value=default_value,
                 field_options=list(spec.get("options", ())) or None,
                 field_tab=spec.get("tab"),
diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
index fd160636d46..4cd031b8780 100644
--- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
+++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
@@ -14,8 +14,8 @@ from typing import (
 from urllib.parse import urlparse
 
 from fastapi import APIRouter, Body, Depends, File, HTTPException, UploadFile
-from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError, create_model
-from pydantic.fields import FieldInfo
+from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError, create_model
+from pydantic.fields import FieldInfo, PydanticUndefined
 from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 import litellm
@@ -24,6 +24,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_keys
 from litellm.proxy._experimental.mcp_server.tool_search import MCP_TOOL_SEARCH_SETTINGS_KEY
 from litellm.proxy._types import *
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.config_resolvers import SettingsSource, source_for
 from litellm.proxy.config_resolvers.sso import (
     SSO_FIELD_ENV_VARS,
     SSO_SECRET_FIELDS,
@@ -197,6 +198,11 @@ class SettingsResponse(BaseModel):
     """Schema information including descriptions and property types for UI display"""
 
 
+class _SettingsWithSchema(BaseModel):
+    values: dict[str, object]
+    field_schema: dict[str, object]
+
+
 class SSOSettingsResponse(SettingsResponse):
     """Response model for SSO settings"""
 
@@ -327,6 +333,8 @@ class UISettings(BaseModel):
 class UISettingsResponse(SettingsResponse):
     """Response model for UI settings"""
 
+    source: dict[str, SettingsSource]
+
 
 # Allowlist of UI settings that can be stored
 ALLOWED_UI_SETTINGS_FIELDS: Final = {
@@ -658,6 +666,13 @@ def _root_schema(settings_class: type[BaseModel]) -> _RootSchema:
     )
 
 
+def _model_field_default(settings_class: type[BaseModel], field_name: str) -> object:
+    field_info: Final = settings_class.model_fields.get(field_name)
+    if field_info is None or field_info.default is PydanticUndefined:
+        return None
+    return cast(object, field_info.default)
+
+
 async def _get_settings_with_schema(
     settings_key: str,
     settings_class: type[BaseModel],
@@ -1527,7 +1542,7 @@ async def get_ui_settings():
     Get UI-specific configuration flags.
     All authenticated users can fetch these settings for client-side behavior.
     """
-    from litellm.proxy.proxy_server import prisma_client
+    from litellm.proxy.proxy_server import prisma_client, proxy_config
 
     if prisma_client is None:
         raise HTTPException(
@@ -1546,26 +1561,46 @@ async def get_ui_settings():
     ui_settings: Final = {k: v for k, v in parsed.items() if k in ALLOWED_UI_SETTINGS_FIELDS}
 
     apply_runtime_general_settings_flags(ui_settings)
+    proxy_config.settings.apply_db_row("ui_settings", ui_settings)
 
     # Refresh DualCache so other code paths (e.g. /user/filter/ui) see fresh values
     from litellm.proxy.proxy_server import user_api_key_cache
 
     await user_api_key_cache.async_set_cache(key=UI_SETTINGS_CACHE_KEY, value=ui_settings, ttl=UI_SETTINGS_CACHE_TTL)
 
-    # Build config-like object for schema helper
-    config: Final[dict[str, object]] = {"litellm_settings": {"ui_settings": ui_settings}}
-
-    settings: Final = await _get_settings_with_schema(
-        settings_key="ui_settings",
-        settings_class=_get_effective_ui_settings_class(),
-        config=config,
+    effective_ui_settings: Final = {
+        **{key: proxy_config.settings[key] for key in ALLOWED_UI_SETTINGS_FIELDS if key in proxy_config.settings},
+        **ui_settings,
+    }
+    config: Final[dict[str, object]] = {"litellm_settings": {"ui_settings": effective_ui_settings}}
+    settings_class: Final = _get_effective_ui_settings_class()
+    resolved_settings: Final = _SettingsWithSchema.model_validate(
+        await _get_settings_with_schema(
+            settings_key="ui_settings",
+            settings_class=settings_class,
+            config=config,
+        )
     )
+    values: Final = {
+        **resolved_settings.values,
+        ENABLE_PTU_COST_ATTRIBUTION_UI_SETTING: is_ptu_cost_attribution_enabled(),
+    }
+    source: Final[dict[str, SettingsSource]] = {
+        key: (
+            "db"
+            if key in ui_settings
+            else source_for(
+                proxy_config.settings,
+                key,
+                _model_field_default(settings_class, key),
+            )
+        )
+        for key in values
+    }
     return UISettingsResponse(
-        values={
-            **settings["values"],
-            ENABLE_PTU_COST_ATTRIBUTION_UI_SETTING: is_ptu_cost_attribution_enabled(),
-        },
-        field_schema=settings["field_schema"],
+        values=values,
+        field_schema=resolved_settings.field_schema,
+        source=source,
     )
 
 
diff --git a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py
index 308f4d88f02..148f30f517b 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py
@@ -75,6 +75,36 @@ class TestRouterSettingsEndpoints:
         assert isinstance(routing_strategy_field["options"], list)
         assert len(routing_strategy_field["options"]) > 0
 
+    @pytest.mark.asyncio
+    async def test_get_router_settings_reports_sources(self, monkeypatch):
+        from litellm.proxy.config_resolvers import SettingsStore
+
+        store = SettingsStore("router_settings")
+        store.load_yaml({"routing_strategy": "simple-shuffle"})
+        store.apply_db_row("router_settings", {"num_retries": 3})
+        monkeypatch.setattr(proxy_server.proxy_config, "router_settings", store)
+        monkeypatch.setattr(proxy_server, "llm_router", None)
+
+        async def fake_get_config(self, config_file_path=None):
+            return {
+                "router_settings": {
+                    "routing_strategy": "simple-shuffle",
+                    "num_retries": 3,
+                }
+            }
+
+        monkeypatch.setattr(
+            proxy_server.ProxyConfig, "get_config", fake_get_config, raising=True
+        )
+
+        admin_user = UserAPIKeyAuth(
+            user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-x"
+        )
+        response = await get_router_settings(user_api_key_dict=admin_user)
+
+        assert response.source["routing_strategy"] == "config"
+        assert response.source["num_retries"] == "db"
+
     @pytest.mark.asyncio
     async def test_get_router_settings_includes_routing_groups_from_live_router(
         self, monkeypatch
diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py
index dd3914e3ad5..d6c63ec9a78 100644
--- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py
+++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py
@@ -15,10 +15,15 @@ from __future__ import annotations
 
 import asyncio
 import json
+from collections.abc import Mapping
+from typing import Final
 from unittest.mock import AsyncMock, MagicMock
 
 import pytest
 
+from litellm.proxy.config_resolvers import SettingsStore
+from litellm.proxy.config_resolvers.settings_rules import JsonValue
+
 from .conftest import VOLATILE_KEYS, normalize
 
 
@@ -37,6 +42,21 @@ def _install_litellm_config(mock_prisma: MagicMock) -> MagicMock:
     return table
 
 
+def _install_settings_store(
+    monkeypatch: pytest.MonkeyPatch,
+    config_values: Mapping[str, JsonValue],
+    db_values: Mapping[str, JsonValue],
+) -> SettingsStore:
+    from litellm.proxy import proxy_server
+
+    store: Final = SettingsStore("general_settings")
+    store.load_yaml(config_values)
+    store.apply_db_row("general_settings", db_values)
+    monkeypatch.setattr(proxy_server.proxy_config, "settings", store)
+    monkeypatch.setattr(proxy_server, "general_settings", store)
+    return store
+
+
 # ---------------------------------------------------------------------------
 # POST /config/update
 # ---------------------------------------------------------------------------
@@ -338,6 +358,7 @@ def test_config_field_info_happy_admin(client, auth_as, mock_prisma, monkeypatch
     assert normalize(response.json()) == {
         "field_name": "max_parallel_requests",
         "field_value": 7,
+        "source": "db",
     }
 
 
@@ -566,6 +587,122 @@ def test_config_list_happy_admin(client, auth_as, mock_prisma, monkeypatch):
     }
 
 
+def test_config_read_routes_report_effective_values_and_sources(client, auth_as, mock_prisma, monkeypatch):
+    from litellm.proxy import proxy_server as ps
+    from litellm.proxy._types import LitellmUserRoles
+
+    table = _install_litellm_config(mock_prisma)
+    row = MagicMock()
+    row.param_value = {"max_parallel_requests": 7, "max_file_size_mb": 222}
+    table.find_first = AsyncMock(return_value=row)
+    monkeypatch.setattr(ps, "prisma_client", mock_prisma)
+    _install_settings_store(
+        monkeypatch,
+        {
+            "max_parallel_requests": 5,
+            "max_file_size_mb": 111,
+            "pass_through_endpoints": [{"path": "/synthetic"}],
+        },
+        {"max_parallel_requests": 7, "max_file_size_mb": 222},
+    )
+
+    with auth_as(LitellmUserRoles.PROXY_ADMIN):
+        list_response = client.get("/config/list", params={"config_type": "general_settings"})
+        config_only_response = client.get(
+            "/config/field/info", params={"field_name": "max_file_size_mb"}
+        )
+        db_wins_response = client.get(
+            "/config/field/info", params={"field_name": "max_parallel_requests"}
+        )
+
+    assert list_response.status_code == 200
+    by_name: Final = {entry["field_name"]: entry for entry in list_response.json()}
+    assert by_name["max_file_size_mb"]["field_value"] == 111
+    assert by_name["max_file_size_mb"]["source"] == "config"
+    assert by_name["pass_through_endpoints"]["source"] == "config"
+    assert by_name["pass_through_endpoints"]["nested_fields"][0]["source"] == "config"
+    assert by_name["max_parallel_requests"]["field_value"] == 7
+    assert by_name["max_parallel_requests"]["source"] == "db"
+
+    assert config_only_response.status_code == 200
+    assert config_only_response.json() == {
+        "field_name": "max_file_size_mb",
+        "field_value": 111,
+        "source": "config",
+    }
+    assert db_wins_response.status_code == 200
+    assert db_wins_response.json() == {
+        "field_name": "max_parallel_requests",
+        "field_value": 7,
+        "source": "db",
+    }
+
+
+def test_config_read_routes_report_default_source(client, auth_as, mock_prisma, monkeypatch):
+    from litellm.proxy import proxy_server as ps
+    from litellm.proxy._types import LitellmUserRoles
+
+    table = _install_litellm_config(mock_prisma)
+    row = MagicMock()
+    row.param_value = {}
+    table.find_first = AsyncMock(return_value=row)
+    monkeypatch.setattr(ps, "prisma_client", mock_prisma)
+    _install_settings_store(monkeypatch, {}, {})
+
+    with auth_as(LitellmUserRoles.PROXY_ADMIN):
+        list_response = client.get("/config/list", params={"config_type": "general_settings"})
+        field_response = client.get(
+            "/config/field/info", params={"field_name": "proxy_config_reload_interval_seconds"}
+        )
+
+    assert list_response.status_code == 200
+    by_name: Final = {entry["field_name"]: entry for entry in list_response.json()}
+    assert by_name["proxy_config_reload_interval_seconds"]["field_value"] == 30
+    assert by_name["proxy_config_reload_interval_seconds"]["source"] == "default"
+    assert field_response.status_code == 200
+    assert field_response.json() == {
+        "field_name": "proxy_config_reload_interval_seconds",
+        "field_value": 30,
+        "source": "default",
+    }
+
+
+def test_config_field_info_uses_store_without_db(client, auth_as, monkeypatch):
+    from litellm.proxy import proxy_server as ps
+    from litellm.proxy._types import LitellmUserRoles
+
+    monkeypatch.setattr(ps, "prisma_client", None)
+    _install_settings_store(monkeypatch, {"max_file_size_mb": 111}, {})
+
+    with auth_as(LitellmUserRoles.PROXY_ADMIN):
+        response = client.get("/config/field/info", params={"field_name": "max_file_size_mb"})
+
+    assert response.status_code == 200
+    assert response.json() == {
+        "field_name": "max_file_size_mb",
+        "field_value": 111,
+        "source": "config",
+    }
+
+
+def test_config_field_info_unset_source_remains_an_error(client, auth_as, mock_prisma, monkeypatch):
+    from litellm.proxy import proxy_server as ps
+    from litellm.proxy._types import LitellmUserRoles
+
+    table = _install_litellm_config(mock_prisma)
+    row = MagicMock()
+    row.param_value = {}
+    table.find_first = AsyncMock(return_value=row)
+    monkeypatch.setattr(ps, "prisma_client", mock_prisma)
+    _install_settings_store(monkeypatch, {}, {})
+
+    with auth_as(LitellmUserRoles.PROXY_ADMIN):
+        response = client.get("/config/field/info", params={"field_name": "max_parallel_requests"})
+
+    assert response.status_code == 400
+    assert "not in" in response.json()["detail"]["error"]
+
+
 def test_config_list_exposes_config_reload_interval(client, auth_as, mock_prisma, monkeypatch):
     """proxy_config_reload_interval_seconds must surface in the admin UI general-settings
     list as an Integer field defaulting to 30, so operators can tune multi-pod convergence
diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py
index 246e2cbba54..6a57ad1636d 100644
--- a/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py
+++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py
@@ -179,6 +179,44 @@ def test_model_settings_method_not_allowed(client, auth_as):
 # ---------------------------------------------------------------------------
 
 
+def test_alerting_settings_reports_sources(client, auth_as, monkeypatch):
+    from litellm.proxy.config_resolvers import SettingsStore
+
+    pc = MagicMock()
+    row = MagicMock()
+    row.param_value = {"alerting_args": {"daily_report_frequency": 7}}
+    pc.db.litellm_config.find_first = AsyncMock(return_value=row)
+    monkeypatch.setattr(proxy_server, "prisma_client", pc)
+
+    logging_obj = MagicMock()
+    args_model = MagicMock()
+    args_model.model_dump = MagicMock(return_value={"daily_report_frequency": 7})
+    logging_obj.slack_alerting_instance.alerting_args = args_model
+    monkeypatch.setattr(proxy_server, "proxy_logging_obj", logging_obj)
+
+    store = SettingsStore("general_settings")
+    store.load_yaml(
+        {
+            "alerting": ["slack"],
+            "alerting_args": {"daily_report_frequency": 3},
+        }
+    )
+    store.apply_db_row(
+        "general_settings",
+        {"alerting_args": {"daily_report_frequency": 7}},
+    )
+    monkeypatch.setattr(proxy_server.proxy_config, "settings", store)
+    monkeypatch.setattr(proxy_server, "general_settings", store)
+
+    with auth_as(LitellmUserRoles.PROXY_ADMIN):
+        response = client.get("/alerting/settings")
+
+    assert response.status_code == 200
+    by_name = {entry["field_name"]: entry for entry in response.json()}
+    assert by_name["slack_alerting"]["source"] == "config"
+    assert by_name["daily_report_frequency"]["source"] == "db"
+
+
 def test_alerting_settings_no_db_error(client, auth_as, no_prisma):
     """Pins ``GET /alerting/settings`` (error: db not connected)."""
     with auth_as(LitellmUserRoles.PROXY_ADMIN):
diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
index 93fb54f84eb..faf5b336410 100644
--- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
+++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
@@ -1342,6 +1342,45 @@ class TestProxySettingEndpoints:
             where={"id": "ui_settings"}
         )
 
+    def test_get_ui_settings_reports_sources(self, monkeypatch):
+        from unittest.mock import AsyncMock, MagicMock
+
+        from litellm.proxy import proxy_server
+        from litellm.proxy.config_resolvers import SettingsStore
+
+        mock_prisma = MagicMock()
+        mock_db_record = MagicMock()
+        mock_db_record.ui_settings = {
+            "disable_model_add_for_internal_users": True,
+        }
+        mock_prisma.db.litellm_uisettings.find_unique = AsyncMock(
+            return_value=mock_db_record
+        )
+        monkeypatch.setattr(proxy_server, "prisma_client", mock_prisma)
+
+        store = SettingsStore("general_settings")
+        store.load_yaml(
+            {
+                "disable_model_add_for_internal_users": False,
+                "forward_client_headers_to_llm_api": True,
+            }
+        )
+        store.apply_db_row(
+            "ui_settings",
+            {"disable_model_add_for_internal_users": True},
+        )
+        monkeypatch.setattr(proxy_server.proxy_config, "settings", store)
+        monkeypatch.setattr(proxy_server, "general_settings", store)
+
+        response = client.get("/get/ui_settings")
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["values"]["disable_model_add_for_internal_users"] is True
+        assert data["values"]["forward_client_headers_to_llm_api"] is True
+        assert data["source"]["disable_model_add_for_internal_users"] == "db"
+        assert data["source"]["forward_client_headers_to_llm_api"] == "config"
+
     def test_get_ui_settings_schema_description_preserved_with_extensions(
         self, mock_auth, monkeypatch
     ):
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index fd882937e79..790ddb93546 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -26393,6 +26393,12 @@ export interface components {
             field_name: string;
             /** Field Value */
             field_value: unknown;
+            /**
+             * Source
+             * @default unset
+             * @enum {string}
+             */
+            source: "config" | "db" | "default" | "unset";
         };
         /** ConfigFieldUpdate */
         ConfigFieldUpdate: {
@@ -26895,6 +26901,12 @@ export interface components {
              * @default false
              */
             premium_field: boolean;
+            /**
+             * Source
+             * @default unset
+             * @enum {string}
+             */
+            source: "config" | "db" | "default" | "unset";
             /** Stored In Db */
             stored_in_db: boolean | null;
         };
@@ -28286,6 +28298,12 @@ export interface components {
             field_name: string;
             /** Field Type */
             field_type: string;
+            /**
+             * Source
+             * @default unset
+             * @enum {string}
+             */
+            source: "config" | "db" | "default" | "unset";
             /** Stored In Db */
             stored_in_db: boolean | null;
         };
@@ -36439,6 +36457,13 @@ export interface components {
             routing_strategy_descriptions: {
                 [key: string]: string;
             };
+            /**
+             * Source
+             * @description Source of each current router setting
+             */
+            source: {
+                [key: string]: "config" | "db" | "default" | "unset";
+            };
         };
         /**
          * RoutingGroup
@@ -38856,6 +38881,10 @@ export interface components {
             field_schema: {
                 [key: string]: unknown;
             };
+            /** Source */
+            source: {
+                [key: string]: "config" | "db" | "default" | "unset";
+            };
             /** Values */
             values: {
                 [key: string]: unknown;

From 8dcf9e8b78509e0392eaef34abb43d2754e2a43f Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Fri, 18 Sep 2026 01:26:10 -0700
Subject: [PATCH 026/160] fix(proxy): correct settings source provenance

---
 litellm/proxy/_lazy_openapi_snapshot.json     |  2 +-
 .../proxy/config_resolvers/settings_store.py  | 20 ++++++
 .../router_settings_endpoints.py              | 25 ++++++-
 litellm/proxy/proxy_server.py                 | 65 +++++++++++++-----
 .../proxy_setting_endpoints.py                | 30 ++++++---
 .../config_resolvers/test_settings_store.py   | 33 +++++++++
 .../test_router_settings_endpoints.py         |  2 +
 .../proxy/proxy_server/test_routes_config.py  | 44 +++++++++---
 .../proxy_server/test_routes_model_metrics.py | 67 ++++++++++++++++---
 .../test_proxy_setting_endpoints.py           | 42 ++++++++++++
 10 files changed, 280 insertions(+), 50 deletions(-)

diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json
index b244678e201..fa046ef0a72 100644
--- a/litellm/proxy/_lazy_openapi_snapshot.json
+++ b/litellm/proxy/_lazy_openapi_snapshot.json
@@ -19346,7 +19346,7 @@
                   }
                 }
               },
-              "description": "\n    Unified rate-limit error.\n\n    Every rate-limit condition surfaced by litellm \u2014 whether it originated from\n    an upstream LLM provider, a vendor batch endpoint, or one of litellm's own\n    proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\n    max-iterations, etc.) \u2014 is raised as an instance of this class.\n\n    The :attr:`category` attribute lets callers distinguish the source. See\n    :class:`RateLimitErrorCategory` for the available values.\n    "
+              "description": "\nUnified rate-limit error.\n\nEvery rate-limit condition surfaced by litellm \u2014 whether it originated from\nan upstream LLM provider, a vendor batch endpoint, or one of litellm's own\nproxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\nmax-iterations, etc.) \u2014 is raised as an instance of this class.\n\nThe :attr:`category` attribute lets callers distinguish the source. See\n:class:`RateLimitErrorCategory` for the available values.\n"
             },
             "500": {
               "content": {
diff --git a/litellm/proxy/config_resolvers/settings_store.py b/litellm/proxy/config_resolvers/settings_store.py
index 8f400853fa9..73bca3222ea 100644
--- a/litellm/proxy/config_resolvers/settings_store.py
+++ b/litellm/proxy/config_resolvers/settings_store.py
@@ -28,6 +28,7 @@ class SettingsStore(MutableMapping[str, JsonValue]):
         self._yaml_values: Mapping[str, JsonValue] = _EMPTY_VALUES
         self._database_rows: Mapping[DbRow, Mapping[str, JsonValue]] = _EMPTY_ROWS
         self._runtime_values: Mapping[str, JsonValue] = _EMPTY_VALUES
+        self._runtime_sources: Mapping[str, FieldSource] = MappingProxyType({})
         self._deleted_runtime_keys: frozenset[str] = frozenset()
 
     def load_yaml(self, mapping: Mapping[str, JsonValue]) -> None:
@@ -39,11 +40,22 @@ class SettingsStore(MutableMapping[str, JsonValue]):
         self._database_rows = MappingProxyType({**self._database_rows, row: MappingProxyType(dict(db_row))})
         self._clear_runtime_keys(frozenset((*previous_row, *db_row)))
 
+    def without_db(self) -> SettingsStore:
+        copy: Final = SettingsStore(self._section)
+        copy.load_yaml(self._yaml_values)
+        runtime_values: Final = {
+            key: value for key, value in self._runtime_values.items() if self._runtime_sources.get(key) != "db"
+        }
+        copy.apply_runtime_values(runtime_values)
+        copy._deleted_runtime_keys = self._deleted_runtime_keys
+        return copy
+
     def resolved(self) -> Mapping[str, JsonValue]:
         return MappingProxyType(dict(self))
 
     def apply_runtime_values(self, values: Mapping[str, JsonValue]) -> None:
         self._runtime_values = MappingProxyType(dict(values))
+        self._runtime_sources = MappingProxyType({key: self.source(key) for key in values})
         self._deleted_runtime_keys = frozenset()
 
     def source(self, key: str) -> FieldSource:
@@ -61,6 +73,7 @@ class SettingsStore(MutableMapping[str, JsonValue]):
 
     def __setitem__(self, key: str, value: JsonValue) -> None:
         self._runtime_values = MappingProxyType({**self._runtime_values, key: value})
+        self._runtime_sources = MappingProxyType({**self._runtime_sources, key: self.source(key)})
         self._deleted_runtime_keys = self._deleted_runtime_keys - frozenset((key,))
 
     def __delitem__(self, key: str) -> None:
@@ -69,6 +82,9 @@ class SettingsStore(MutableMapping[str, JsonValue]):
         self._runtime_values = MappingProxyType(
             {key_: value for key_, value in self._runtime_values.items() if key_ != key}
         )
+        self._runtime_sources = MappingProxyType(
+            {key_: source for key_, source in self._runtime_sources.items() if key_ != key}
+        )
         self._deleted_runtime_keys = self._deleted_runtime_keys | frozenset((key,))
 
     def __iter__(self) -> Iterator[str]:
@@ -84,6 +100,7 @@ class SettingsStore(MutableMapping[str, JsonValue]):
 
     def _clear_runtime(self) -> None:
         self._runtime_values = _EMPTY_VALUES
+        self._runtime_sources = MappingProxyType({})
         self._deleted_runtime_keys = frozenset()
 
     def _clear_runtime_keys(self, keys: frozenset[str]) -> None:
@@ -92,6 +109,9 @@ class SettingsStore(MutableMapping[str, JsonValue]):
         self._runtime_values = MappingProxyType(
             {key: value for key, value in self._runtime_values.items() if key not in keys}
         )
+        self._runtime_sources = MappingProxyType(
+            {key: source for key, source in self._runtime_sources.items() if key not in keys}
+        )
         self._deleted_runtime_keys = self._deleted_runtime_keys - keys
 
     def _keys(self) -> tuple[str, ...]:
diff --git a/litellm/proxy/management_endpoints/router_settings_endpoints.py b/litellm/proxy/management_endpoints/router_settings_endpoints.py
index 5d3b6d40601..1869be88d1b 100644
--- a/litellm/proxy/management_endpoints/router_settings_endpoints.py
+++ b/litellm/proxy/management_endpoints/router_settings_endpoints.py
@@ -16,7 +16,7 @@ from pydantic import BaseModel, Field
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
-from litellm.proxy.config_resolvers import SettingsSource, source_for
+from litellm.proxy.config_resolvers import SettingsSource, SettingsStore, source_for
 from litellm.router import Router
 from litellm.types.management_endpoints import (
     ROUTER_SETTINGS_FIELDS,
@@ -41,6 +41,18 @@ class RouterFieldsResponse(BaseModel):
     routing_strategy_descriptions: dict[str, str] = Field(description="Descriptions for each routing strategy option")
 
 
+def _router_setting_source(
+    settings: SettingsStore,
+    key: str,
+    current_value: object,
+    field_default: object,
+) -> SettingsSource:
+    source: Final = source_for(settings, key, field_default)
+    if source != "unset":
+        return source
+    return "default" if current_value is not None else "unset"
+
+
 def _get_routing_strategies_from_router_class() -> list[str]:
     """
     Dynamically extract routing strategies from the Router class __init__ method.
@@ -116,10 +128,17 @@ async def get_router_settings(
                 field.field_value = current_values[field.field_name]
 
         field_defaults: Final[dict[str, object]] = {
-            field.field_name: cast(object, field.field_default) for field in router_fields
+            field.field_name: cast(object, field.field_default)  # cast-ok: Pydantic field defaults are untyped
+            for field in router_fields
         }
         source: Final[dict[str, SettingsSource]] = {
-            key: source_for(proxy_config.router_settings, key, field_defaults.get(key)) for key in current_values
+            key: _router_setting_source(
+                proxy_config.router_settings,
+                key,
+                cast(object, current_values[key]),  # cast-ok: current values are stored in a typed response map
+                field_defaults.get(key),
+            )
+            for key in current_values
         }
         return RouterSettingsResponse(
             fields=router_fields,
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 1131d7bea86..3898f522feb 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -432,7 +432,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
     model_access_group_spend_counter_key,
     tag_cache_key,
 )
-from litellm.proxy.config_resolvers import SettingsStore, resolve_fields, source_for
+from litellm.proxy.config_resolvers import SettingsSource, SettingsStore, resolve_fields, source_for
 from litellm.proxy.config_resolvers.alerting import (
     EMAIL_DESCRIPTORS,
     MS_TEAMS_DESCRIPTORS,
@@ -4820,7 +4820,7 @@ def _as_settings_mapping(value: object) -> Mapping[str, SettingsJsonValue]:
 def _get_field_default(field_info: FieldInfo) -> JsonValue:
     if field_info.default is PydanticUndefined:
         return None
-    return cast(JsonValue, field_info.default)
+    return cast(JsonValue, field_info.default)  # cast-ok: Pydantic field defaults are JSON values at runtime
 
 
 def _bind_general_settings_store(settings: SettingsStore) -> None:
@@ -15605,6 +15605,22 @@ async def model_settings():
 #### ALERTING MANAGEMENT ENDPOINTS ####
 
 
+def _nested_setting_source(
+    settings: SettingsStore,
+    db_values: Mapping[str, JsonValue],
+    parent_key: str,
+    field_name: str,
+    field_default: JsonValue,
+) -> SettingsSource:
+    db_value: Final = db_values.get(field_name)
+    if db_value is not None and db_value != []:
+        return "db"
+    parent_value: Final = settings.without_db().get(parent_key)
+    if isinstance(parent_value, Mapping) and field_name in parent_value:
+        return "config"
+    return "default" if field_default is not None else "unset"
+
+
 @router.get(
     "/alerting/settings",
     description="Return the configurable alerting param, description, and current value",
@@ -15647,8 +15663,13 @@ async def alerting_settings(
         if db_general_settings is not None and db_general_settings.param_value is not None
         else {}
     )
-    alerting_args_dict: Final = cast(dict[str, JsonValue], db_general_settings_dict.get("alerting_args", {}))
-    alerting_values: Final = cast(list[JsonValue] | None, db_general_settings_dict.get("alerting"))
+    alerting_args_value: Final = db_general_settings_dict.get("alerting_args")
+    alerting_args_dict: Final[Mapping[str, JsonValue]] = (
+        alerting_args_value if isinstance(alerting_args_value, dict) else {}
+    )
+    alerting_values: Final = cast(  # cast-ok: alerting is stored as a JSON list when present
+        list[JsonValue] | None, db_general_settings_dict.get("alerting")
+    )
 
     settings: Final = proxy_config.settings
     settings.apply_db_row("general_settings", db_general_settings_dict)
@@ -15711,7 +15732,13 @@ async def alerting_settings(
                 field_description=field_info.description or "",
                 field_value=_slack_alerting_args_dict.get(field_name, field_default),
                 stored_in_db=_stored_in_db,
-                source=source_for(settings, "alerting_args", field_default),
+                source=_nested_setting_source(
+                    settings,
+                    alerting_args_dict,
+                    "alerting_args",
+                    field_name,
+                    field_default,
+                ),
                 field_default_value=field_default,
                 premium_field=(True if field_name == "region_outage_alert_ttl" else False),
             )
@@ -17414,21 +17441,19 @@ async def get_config_general_settings(
     field_info: Final = ConfigGeneralSettings.model_fields[field_name]
     field_default: JsonValue = _get_field_default(field_info)
     settings: Final = proxy_config.settings
-    db_values: Mapping[str, JsonValue]
-    if prisma_client is None:
-        db_values = {}
-    else:
+    if prisma_client is not None:
         db_general_settings: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
             where={"param_name": "general_settings"}
         )
-        db_values = (
+        db_values: Final[Mapping[str, JsonValue]] = (
             dict(db_general_settings.param_value)
             if db_general_settings is not None and db_general_settings.param_value is not None
             else {}
         )
         settings.apply_db_row("general_settings", db_values)
+    effective_settings: Final = settings.without_db() if prisma_client is None else settings
 
-    if field_name not in settings and field_default is None:
+    if field_name not in effective_settings and field_default is None:
         raise HTTPException(
             status_code=400,
             detail={"error": f"Field name={field_name} not in DB"},
@@ -17436,7 +17461,7 @@ async def get_config_general_settings(
 
     redacted_field_value: Final = _redact_general_setting_value(
         field_name,
-        settings.get(field_name, field_default),
+        effective_settings.get(field_name, field_default),
         user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN,
     )
     field_value: Final = (
@@ -17450,7 +17475,7 @@ async def get_config_general_settings(
     return ConfigFieldInfo(
         field_name=field_name,
         field_value=field_value,
-        source=source_for(settings, field_name, field_default),
+        source=source_for(effective_settings, field_name, field_default),
     )
 
 
@@ -17640,7 +17665,11 @@ async def get_config_list(
     settings: Final = proxy_config.settings
     settings.apply_db_row("general_settings", db_general_settings_dict)
     runtime_settings: Final[Mapping[str, JsonValue]] = (
-        cast(Mapping[str, JsonValue], general_settings) if not isinstance(general_settings, SettingsStore) else settings
+        cast(  # cast-ok: legacy general_settings remains a mapping at this route boundary
+            Mapping[str, JsonValue], general_settings
+        )
+        if not isinstance(general_settings, SettingsStore)
+        else settings
     )
 
     allowed_args: Final = _GENERAL_SETTINGS_CONFIG_LIST_FIELD_TYPES
@@ -17744,9 +17773,11 @@ async def get_config_list(
     litellm_settings_store.apply_db_row("litellm_settings", db_litellm_settings)
     for litellm_field_name, spec in _GENERAL_SETTINGS_UI_LITELLM_FIELDS.items():
         default_value: GeneralSettingsUILiteLLMValue = _general_settings_ui_litellm_default(spec)
-        current_value: GeneralSettingsUILiteLLMValue = cast(
-            GeneralSettingsUILiteLLMValue,
-            litellm_settings_store.get(litellm_field_name, default_value),
+        current_value: GeneralSettingsUILiteLLMValue = (
+            cast(  # cast-ok: UI field defaults are validated by the field spec
+                GeneralSettingsUILiteLLMValue,
+                litellm_settings_store.get(litellm_field_name, default_value),
+            )
         )
         source = source_for(litellm_settings_store, litellm_field_name, default_value)
         stored_in_db_litellm: bool | None
diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
index 4cd031b8780..aba65719f9b 100644
--- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
+++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
@@ -24,7 +24,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_keys
 from litellm.proxy._experimental.mcp_server.tool_search import MCP_TOOL_SEARCH_SETTINGS_KEY
 from litellm.proxy._types import *
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
-from litellm.proxy.config_resolvers import SettingsSource, source_for
+from litellm.proxy.config_resolvers import SettingsSource, SettingsStore, source_for
 from litellm.proxy.config_resolvers.sso import (
     SSO_FIELD_ENV_VARS,
     SSO_SECRET_FIELDS,
@@ -34,7 +34,10 @@ from litellm.proxy.management_endpoints.team_admin_field_permissions import (
     SUPPORTED_TEAM_ADMIN_EDITABLE_TEAM_FIELDS,
     TEAM_ADMIN_EDITABLE_TEAM_FIELDS_SETTING,
 )
-from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attribution_enabled
+from litellm.proxy.spend_tracking.ptu_feature_flag import (
+    PTU_COST_ATTRIBUTION_ENV_VAR,
+    is_ptu_cost_attribution_enabled,
+)
 from litellm.proxy.utils import invalidate_config_param
 from litellm.repositories.config_repository import ConfigRepository
 from litellm.repositories.organization_repository import OrganizationRepository
@@ -44,6 +47,7 @@ from litellm.repositories.table_repositories import (
     UISettingsRepository,
 )
 from litellm.repositories.team_repository import TeamRepository
+from litellm.secret_managers.main import get_secret
 from litellm.types.mcp import MCPToolSearchSettings
 from litellm.types.proxy.management_endpoints.ui_sso import (
     DefaultTeamSSOParams,
@@ -670,7 +674,19 @@ def _model_field_default(settings_class: type[BaseModel], field_name: str) -> ob
     field_info: Final = settings_class.model_fields.get(field_name)
     if field_info is None or field_info.default is PydanticUndefined:
         return None
-    return cast(object, field_info.default)
+    return cast(object, field_info.default)  # cast-ok: Pydantic field defaults are untyped
+
+
+def _ui_setting_source(
+    key: str,
+    value: object,
+    settings: SettingsStore,
+    settings_class: type[BaseModel],
+) -> SettingsSource:
+    if key == ENABLE_PTU_COST_ATTRIBUTION_UI_SETTING:
+        configured_value: Final = get_secret(PTU_COST_ATTRIBUTION_ENV_VAR, None)
+        return "config" if configured_value is not None or value is True else "default"
+    return source_for(settings, key, _model_field_default(settings_class, key))
 
 
 async def _get_settings_with_schema(
@@ -1587,13 +1603,7 @@ async def get_ui_settings():
     }
     source: Final[dict[str, SettingsSource]] = {
         key: (
-            "db"
-            if key in ui_settings
-            else source_for(
-                proxy_config.settings,
-                key,
-                _model_field_default(settings_class, key),
-            )
+            "db" if key in ui_settings else _ui_setting_source(key, values[key], proxy_config.settings, settings_class)
         )
         for key in values
     }
diff --git a/tests/test_litellm/proxy/config_resolvers/test_settings_store.py b/tests/test_litellm/proxy/config_resolvers/test_settings_store.py
index e41c852eacb..efff9ad24c8 100644
--- a/tests/test_litellm/proxy/config_resolvers/test_settings_store.py
+++ b/tests/test_litellm/proxy/config_resolvers/test_settings_store.py
@@ -82,6 +82,39 @@ def test_settings_store_keeps_unaffected_runtime_values_on_a_db_row_refresh() ->
     assert store.source("changed") == "db"
 
 
+def test_settings_store_without_db_uses_yaml_without_mutating_runtime_values() -> None:
+    store: Final = SettingsStore("general_settings")
+    store.load_yaml({"max_parallel_requests": 5})
+    store.apply_db_row("general_settings", {"max_parallel_requests": 7})
+    store.apply_runtime_values({"max_parallel_requests": 7})
+
+    without_db: Final = store.without_db()
+
+    assert without_db["max_parallel_requests"] == 5
+    assert without_db.source("max_parallel_requests") == "config"
+    assert store["max_parallel_requests"] == 7
+    assert store.source("max_parallel_requests") == "db"
+
+
+def test_settings_store_without_db_preserves_non_db_runtime_values() -> None:
+    store: Final = SettingsStore("general_settings")
+    store.load_yaml({"max_parallel_requests": "os.environ/MAX_PARALLEL_REQUESTS"})
+    store.apply_runtime_values({"max_parallel_requests": 7})
+
+    without_db: Final = store.without_db()
+
+    assert without_db["max_parallel_requests"] == 7
+    assert without_db.source("max_parallel_requests") == "config"
+
+
+def test_settings_store_without_db_preserves_runtime_deletions() -> None:
+    store: Final = SettingsStore("general_settings")
+    store.load_yaml({"deleted": 1})
+    del store["deleted"]
+
+    assert "deleted" not in store.without_db()
+
+
 def test_settings_store_removes_only_runtime_values_affected_by_a_cleared_db_row() -> None:
     store: Final = SettingsStore("general_settings")
     store.load_yaml({"template": "os.environ/SETTING"})
diff --git a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py
index 148f30f517b..3af7de62abe 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py
@@ -146,6 +146,8 @@ class TestRouterSettingsEndpoints:
         response = await get_router_settings(user_api_key_dict=admin_user)
 
         assert response.current_values.get("routing_groups") == groups
+        assert response.current_values["timeout"] is not None
+        assert response.source["timeout"] == "default"
 
         rg_field = next(f for f in response.fields if f.field_name == "routing_groups")
         assert rg_field.field_value == groups
diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py
index d6c63ec9a78..fdaa2476219 100644
--- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py
+++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py
@@ -15,8 +15,11 @@ from __future__ import annotations
 
 import asyncio
 import json
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping
+from contextlib import AbstractContextManager
 from typing import Final
+
+from fastapi.testclient import TestClient
 from unittest.mock import AsyncMock, MagicMock
 
 import pytest
@@ -362,6 +365,33 @@ def test_config_field_info_happy_admin(client, auth_as, mock_prisma, monkeypatch
     }
 
 
+def test_config_field_info_clears_stale_db_source_without_connection(
+    client: TestClient,
+    auth_as: Callable[..., AbstractContextManager[None]],
+    monkeypatch: pytest.MonkeyPatch,
+):
+    from litellm.proxy import proxy_server as ps
+    from litellm.proxy._types import LitellmUserRoles
+
+    store = SettingsStore("general_settings")
+    store.load_yaml({"max_parallel_requests": 5})
+    store.apply_db_row("general_settings", {"max_parallel_requests": 7})
+    store.apply_runtime_values({"max_parallel_requests": 7})
+    monkeypatch.setattr(ps.proxy_config, "settings", store)
+    monkeypatch.setattr(ps, "prisma_client", None)
+
+    with auth_as(LitellmUserRoles.PROXY_ADMIN):
+        response = client.get("/config/field/info", params={"field_name": "max_parallel_requests"})
+
+    assert response.status_code == 200
+    assert normalize(response.json()) == {
+        "field_name": "max_parallel_requests",
+        "field_value": 5,
+        "source": "config",
+    }
+    assert store["max_parallel_requests"] == 7
+
+
 def test_config_field_info_non_admin_rejected(client, auth_as, mock_prisma, monkeypatch):
     """Non-admin (INTERNAL_USER) is denied — admin-view gate fires."""
     from litellm.proxy import proxy_server as ps
@@ -608,12 +638,8 @@ def test_config_read_routes_report_effective_values_and_sources(client, auth_as,
 
     with auth_as(LitellmUserRoles.PROXY_ADMIN):
         list_response = client.get("/config/list", params={"config_type": "general_settings"})
-        config_only_response = client.get(
-            "/config/field/info", params={"field_name": "max_file_size_mb"}
-        )
-        db_wins_response = client.get(
-            "/config/field/info", params={"field_name": "max_parallel_requests"}
-        )
+        config_only_response = client.get("/config/field/info", params={"field_name": "max_file_size_mb"})
+        db_wins_response = client.get("/config/field/info", params={"field_name": "max_parallel_requests"})
 
     assert list_response.status_code == 200
     by_name: Final = {entry["field_name"]: entry for entry in list_response.json()}
@@ -651,9 +677,7 @@ def test_config_read_routes_report_default_source(client, auth_as, mock_prisma,
 
     with auth_as(LitellmUserRoles.PROXY_ADMIN):
         list_response = client.get("/config/list", params={"config_type": "general_settings"})
-        field_response = client.get(
-            "/config/field/info", params={"field_name": "proxy_config_reload_interval_seconds"}
-        )
+        field_response = client.get("/config/field/info", params={"field_name": "proxy_config_reload_interval_seconds"})
 
     assert list_response.status_code == 200
     by_name: Final = {entry["field_name"]: entry for entry in list_response.json()}
diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py
index 6a57ad1636d..97ca3b3dbbe 100644
--- a/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py
+++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py
@@ -11,13 +11,17 @@ Pins (PR2):
 
 from __future__ import annotations
 
+from collections.abc import Callable
+from contextlib import AbstractContextManager
 from unittest.mock import AsyncMock, MagicMock
 
 import pytest
+from fastapi.testclient import TestClient
 
 import litellm
 from litellm.proxy import proxy_server
 from litellm.proxy._types import LitellmUserRoles
+from litellm.proxy.config_resolvers.settings_rules import JsonValue
 
 from .conftest import normalize  # type: ignore[import-not-found]
 
@@ -53,9 +57,7 @@ def test_model_streaming_metrics_happy(client, auth_as, prisma_with_query_raw):
     pin can rely on the exact response shape.
     """
     with auth_as():
-        response = client.get(
-            "/model/streaming_metrics", params={"_selected_model_group": "gpt-4"}
-        )
+        response = client.get("/model/streaming_metrics", params={"_selected_model_group": "gpt-4"})
     assert response.status_code == 200
     assert normalize(response.json()) == {"data": [], "all_api_bases": []}
 
@@ -94,9 +96,7 @@ def test_model_metrics_no_prisma_error(client, auth_as, no_prisma):
 # ---------------------------------------------------------------------------
 
 
-def test_model_metrics_slow_responses_happy(
-    client, auth_as, prisma_with_query_raw, monkeypatch
-):
+def test_model_metrics_slow_responses_happy(client, auth_as, prisma_with_query_raw, monkeypatch):
     """Pins ``GET /model/metrics/slow_responses`` (happy: empty list)."""
     logging_obj = MagicMock()
     logging_obj.slack_alerting_instance.alerting_threshold = 30
@@ -184,7 +184,12 @@ def test_alerting_settings_reports_sources(client, auth_as, monkeypatch):
 
     pc = MagicMock()
     row = MagicMock()
-    row.param_value = {"alerting_args": {"daily_report_frequency": 7}}
+    row.param_value = {
+        "alerting_args": {
+            "daily_report_frequency": 7,
+            "report_check_interval": None,
+        }
+    }
     pc.db.litellm_config.find_first = AsyncMock(return_value=row)
     monkeypatch.setattr(proxy_server, "prisma_client", pc)
 
@@ -198,12 +203,20 @@ def test_alerting_settings_reports_sources(client, auth_as, monkeypatch):
     store.load_yaml(
         {
             "alerting": ["slack"],
-            "alerting_args": {"daily_report_frequency": 3},
+            "alerting_args": {
+                "daily_report_frequency": 3,
+                "report_check_interval": 300,
+            },
         }
     )
     store.apply_db_row(
         "general_settings",
-        {"alerting_args": {"daily_report_frequency": 7}},
+        {
+            "alerting_args": {
+                "daily_report_frequency": 7,
+                "report_check_interval": None,
+            }
+        },
     )
     monkeypatch.setattr(proxy_server.proxy_config, "settings", store)
     monkeypatch.setattr(proxy_server, "general_settings", store)
@@ -215,6 +228,42 @@ def test_alerting_settings_reports_sources(client, auth_as, monkeypatch):
     by_name = {entry["field_name"]: entry for entry in response.json()}
     assert by_name["slack_alerting"]["source"] == "config"
     assert by_name["daily_report_frequency"]["source"] == "db"
+    assert by_name["report_check_interval"]["source"] == "config"
+    assert by_name["budget_alert_ttl"]["source"] == "default"
+
+
+@pytest.mark.parametrize("db_alerting_args", [None, []])
+def test_alerting_settings_handles_empty_db_args(
+    client: TestClient,
+    auth_as: Callable[..., AbstractContextManager[None]],
+    monkeypatch: pytest.MonkeyPatch,
+    db_alerting_args: JsonValue,
+):
+    from litellm.proxy.config_resolvers import SettingsStore
+
+    pc = MagicMock()
+    row = MagicMock()
+    row.param_value = {"alerting_args": db_alerting_args}
+    pc.db.litellm_config.find_first = AsyncMock(return_value=row)
+    monkeypatch.setattr(proxy_server, "prisma_client", pc)
+
+    logging_obj = MagicMock()
+    args_model = MagicMock()
+    args_model.model_dump = MagicMock(return_value={})
+    logging_obj.slack_alerting_instance.alerting_args = args_model
+    monkeypatch.setattr(proxy_server, "proxy_logging_obj", logging_obj)
+
+    store = SettingsStore("general_settings")
+    store.load_yaml({"alerting_args": {"report_check_interval": 300}})
+    monkeypatch.setattr(proxy_server.proxy_config, "settings", store)
+    monkeypatch.setattr(proxy_server, "general_settings", store)
+
+    with auth_as(LitellmUserRoles.PROXY_ADMIN):
+        response = client.get("/alerting/settings")
+
+    assert response.status_code == 200
+    by_name = {entry["field_name"]: entry for entry in response.json()}
+    assert by_name["report_check_interval"]["source"] == "config"
 
 
 def test_alerting_settings_no_db_error(client, auth_as, no_prisma):
diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
index faf5b336410..d8308d7831f 100644
--- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
+++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
@@ -3198,6 +3198,7 @@ class TestPtuCostAttributionUISetting:
 
         assert response.status_code == 200
         assert response.json()["values"]["enable_ptu_cost_attribution"] is False
+        assert response.json()["source"]["enable_ptu_cost_attribution"] == "default"
 
     def test_reported_true_once_the_env_var_is_set(self, mock_auth, monkeypatch):
         from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_ENV_VAR
@@ -3209,6 +3210,47 @@ class TestPtuCostAttributionUISetting:
 
         assert response.status_code == 200
         assert response.json()["values"]["enable_ptu_cost_attribution"] is True
+        assert response.json()["source"]["enable_ptu_cost_attribution"] == "config"
+
+    def test_reported_config_when_secret_manager_enables_the_flag(
+        self, mock_auth: None, monkeypatch: pytest.MonkeyPatch
+    ):
+        from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_ENV_VAR
+
+        monkeypatch.delenv(PTU_COST_ATTRIBUTION_ENV_VAR, raising=False)
+        monkeypatch.setattr(
+            "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.is_ptu_cost_attribution_enabled",
+            lambda: True,
+        )
+        self._mock_prisma(monkeypatch)
+
+        response = client.get("/get/ui_settings")
+
+        assert response.status_code == 200
+        assert response.json()["values"]["enable_ptu_cost_attribution"] is True
+        assert response.json()["source"]["enable_ptu_cost_attribution"] == "config"
+
+    def test_reported_config_when_secret_manager_disables_the_flag(
+        self, mock_auth: None, monkeypatch: pytest.MonkeyPatch
+    ):
+        from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_ENV_VAR
+
+        monkeypatch.delenv(PTU_COST_ATTRIBUTION_ENV_VAR, raising=False)
+        monkeypatch.setattr(
+            "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.is_ptu_cost_attribution_enabled",
+            lambda: False,
+        )
+        monkeypatch.setattr(
+            "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.get_secret",
+            lambda *_args: False,
+        )
+        self._mock_prisma(monkeypatch)
+
+        response = client.get("/get/ui_settings")
+
+        assert response.status_code == 200
+        assert response.json()["values"]["enable_ptu_cost_attribution"] is False
+        assert response.json()["source"]["enable_ptu_cost_attribution"] == "config"
 
     def test_a_persisted_true_cannot_forge_the_derived_value(self, mock_auth, monkeypatch):
         """A row written before the allowlist existed must not be able to turn the feature on."""

From 066cc1883afe3a69a49638df4c8d7cfbc1fd0f2d Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Fri, 18 Sep 2026 02:39:58 -0700
Subject: [PATCH 027/160] test(router): cover legacy lowest TPM selection

---
 .../router_strategy/test_lowest_tpm_rpm.py    | 54 +++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 tests/test_litellm/router_strategy/test_lowest_tpm_rpm.py

diff --git a/tests/test_litellm/router_strategy/test_lowest_tpm_rpm.py b/tests/test_litellm/router_strategy/test_lowest_tpm_rpm.py
new file mode 100644
index 00000000000..7b13b196d5b
--- /dev/null
+++ b/tests/test_litellm/router_strategy/test_lowest_tpm_rpm.py
@@ -0,0 +1,54 @@
+from datetime import datetime, timedelta
+from typing import Final
+
+from litellm import Router
+from litellm.types.router import DeploymentTypedDict, LiteLLMParamsTypedDict
+
+MODEL_GROUP: Final = "lowest-tpm-router"
+HIGH_USAGE_DEPLOYMENT_ID: Final = "highest-usage"
+LOW_USAGE_DEPLOYMENT_ID: Final = "lowest-usage"
+
+
+def _deployment(deployment_id: str) -> DeploymentTypedDict:
+    params: LiteLLMParamsTypedDict = {
+        "model": "gpt-4o",
+        "api_key": "key",
+        "mock_response": f"from {deployment_id}",
+    }
+    return {
+        "model_name": MODEL_GROUP,
+        "litellm_params": params,
+        "model_info": {"id": deployment_id},
+    }
+
+
+def test_usage_based_routing_v1_selects_the_lowest_recorded_tpm() -> None:
+    router: Final = Router(
+        model_list=[
+            _deployment(HIGH_USAGE_DEPLOYMENT_ID),
+            _deployment(LOW_USAGE_DEPLOYMENT_ID),
+        ],
+        routing_strategy="usage-based-routing",
+        num_retries=0,
+    )
+    usage_by_deployment: Final = {
+        HIGH_USAGE_DEPLOYMENT_ID: 100,
+        LOW_USAGE_DEPLOYMENT_ID: 1,
+    }
+    now: Final = datetime.now()
+    cache_keys: Final = tuple(
+        f"{MODEL_GROUP}:tpm:{(now + timedelta(minutes=offset)).strftime('%H-%M')}"
+        for offset in range(60)
+    )
+
+    for cache_key in cache_keys:
+        router.cache.set_cache(
+            key=cache_key, value=usage_by_deployment, ttl=float("inf")
+        )
+
+    deployment: Final = router.get_available_deployment(
+        model=MODEL_GROUP,
+        messages=[{"role": "user", "content": "test"}],
+    )
+
+    assert deployment["model_info"]["id"] == LOW_USAGE_DEPLOYMENT_ID

From da3bd9e31c72e29a2a75ef82107d05e7f91cc4db Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Fri, 18 Sep 2026 04:08:34 -0700
Subject: [PATCH 028/160] test(ui): model E2E cleanup failures as values

---
 tests/e2e/ui/helpers/roundTrip.ts | 111 ++++++++++++++++++++++--------
 1 file changed, 81 insertions(+), 30 deletions(-)

diff --git a/tests/e2e/ui/helpers/roundTrip.ts b/tests/e2e/ui/helpers/roundTrip.ts
index 1fc2d0aec1a..91e48b7087c 100644
--- a/tests/e2e/ui/helpers/roundTrip.ts
+++ b/tests/e2e/ui/helpers/roundTrip.ts
@@ -33,39 +33,90 @@ export async function readBack<T = any>(
   return (await res.json()) as T;
 }
 
-export async function runWithCleanup(
-  action: () => Promise<void>,
-  cleanup: () => Promise<boolean>,
-): Promise<void> {
-  const outcome = await Promise.resolve()
+type OperationOutcome =
+  | { readonly status: "success" }
+  | { readonly status: "failure"; readonly error: unknown };
+
+type RunFailure =
+  | { readonly status: "action_failure"; readonly error: unknown }
+  | { readonly status: "cleanup_failure"; readonly error: unknown }
+  | {
+      readonly status: "action_and_cleanup_failure";
+      readonly actionError: unknown;
+      readonly cleanupError: unknown;
+    };
+
+function toRunFailure(
+  actionOutcome: OperationOutcome,
+  cleanupOutcome: OperationOutcome,
+): RunFailure | null {
+  if (
+    actionOutcome.status === "failure" &&
+    cleanupOutcome.status === "failure"
+  ) {
+    return {
+      status: "action_and_cleanup_failure",
+      actionError: actionOutcome.error,
+      cleanupError: cleanupOutcome.error,
+    };
+  }
+  if (actionOutcome.status === "failure") {
+    return { status: "action_failure", error: actionOutcome.error };
+  }
+  if (cleanupOutcome.status === "failure") {
+    return { status: "cleanup_failure", error: cleanupOutcome.error };
+  }
+  return null;
+}
+
+function raiseRunFailure(failure: RunFailure): never {
+  switch (failure.status) {
+    case "action_failure":
+      throw failure.error;
+    case "cleanup_failure":
+      throw failure.error;
+    case "action_and_cleanup_failure":
+      throw new AggregateError(
+        [failure.actionError, failure.cleanupError],
+        "Action and cleanup failed",
+      );
+  }
+}
+
+async function runAction(
+  action: () => void | Promise<void>,
+): Promise<OperationOutcome> {
+  return Promise.resolve()
     .then(action)
     .then(
       () => ({ status: "success" as const }),
       (error: unknown) => ({ status: "failure" as const, error }),
     );
-  try {
-    if (outcome.status === "failure") throw outcome.error;
-  } finally {
-    const cleanupOutcome = await Promise.resolve()
-      .then(cleanup)
-      .then(
-        (succeeded) =>
-          succeeded
-            ? { status: "success" as const }
-            : {
-                status: "failure" as const,
-                error: new Error("Failed to clean up UI E2E resource"),
-              },
-        (error: unknown) => ({ status: "failure" as const, error }),
-      );
-    if (cleanupOutcome.status === "failure") {
-      if (outcome.status === "failure") {
-        throw new AggregateError(
-          [outcome.error, cleanupOutcome.error],
-          "Action and cleanup failed",
-        );
-      }
-      throw cleanupOutcome.error;
-    }
-  }
+}
+
+async function runCleanup(
+  cleanup: () => boolean | Promise<boolean>,
+): Promise<OperationOutcome> {
+  return Promise.resolve()
+    .then(cleanup)
+    .then(
+      (succeeded) =>
+        succeeded
+          ? { status: "success" as const }
+          : {
+              status: "failure" as const,
+              error: new Error("Failed to clean up UI E2E resource"),
+            },
+      (error: unknown) => ({ status: "failure" as const, error }),
+    );
+}
+
+export async function runWithCleanup(
+  action: () => void | Promise<void>,
+  cleanup: () => boolean | Promise<boolean>,
+): Promise<void> {
+  const actionOutcome = await runAction(action);
+  const cleanupOutcome = await runCleanup(cleanup);
+  const failure = toRunFailure(actionOutcome, cleanupOutcome);
+  if (failure !== null) raiseRunFailure(failure);
 }

From c1c566db875f31e28e07f662be39207dca9d8344 Mon Sep 17 00:00:00 2001
From: Yucheng He <yucheng@berri.ai>
Date: Tue, 15 Sep 2026 01:18:18 -0700
Subject: [PATCH 029/160] fix(proxy): record aborted outcome when spend-log
 cleanup is cancelled at shutdown

cleanup_old_spend_logs only caught Exception, so a run cut short by
CancelledError recorded no outcome and logged nothing. Under uvicorn the
job was never cancelled at all: uvicorn re-raises the captured SIGTERM as
soon as the lifespan shutdown returns, before asyncio cancels outstanding
tasks, so an in-flight scheduler job simply died with the process.

The cleanup now handles CancelledError by logging elapsed time, rows
deleted and batch count at error level, recording outcome="aborted", and
re-raising. The lifespan shutdown stops the scheduler and awaits the jobs
it cancels while the database is still connected, so that handler runs
under uvicorn too, and the pod lock is released instead of orphaned.

Resolves LIT-6990
---
 .../db_transaction_queue/spend_log_cleanup.py |  16 +++
 litellm/proxy/proxy_server.py                 |  18 ++-
 litellm/proxy/shutdown/scheduled_jobs.py      |  70 ++++++++++
 .../proxy/shutdown/test_scheduled_jobs.py     | 125 ++++++++++++++++++
 .../proxy/test_spend_log_cleanup.py           |  92 +++++++++++++
 5 files changed, 318 insertions(+), 3 deletions(-)
 create mode 100644 litellm/proxy/shutdown/scheduled_jobs.py
 create mode 100644 tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py

diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
index e97e9f6e683..1a14210dbec 100644
--- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
+++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
@@ -96,6 +96,8 @@ class SpendLogCleanup:
 
         self.general_settings = general_settings or default_settings
         self._refresh_bounds()
+        self._run_rows_deleted: int = 0
+        self._run_batches: int = 0
         from litellm.proxy.proxy_server import proxy_logging_obj
 
         pod_lock_manager: Final = proxy_logging_obj.db_spend_update_writer.pod_lock_manager
@@ -422,6 +424,8 @@ class SpendLogCleanup:
 
             total_deleted += deleted_count
             run_count += 1
+            self._run_rows_deleted += deleted_count
+            self._run_batches += 1
 
             # Add a small sleep to prevent overwhelming the database
             await asyncio.sleep(0.1)
@@ -590,6 +594,9 @@ class SpendLogCleanup:
         If no pod_lock_manager, runs cleanup without distributed locking.
         """
         lock_acquired = False
+        run_started_at: Final = time.monotonic()
+        self._run_rows_deleted = 0
+        self._run_batches = 0
         try:
             verbose_proxy_logger.info("Cleanup job triggered at %s", datetime.now())
             self._refresh_bounds()
@@ -670,6 +677,15 @@ class SpendLogCleanup:
                 self._run_outcome(spend_log_results + session_results + health_check_results)
             )
 
+        except asyncio.CancelledError:
+            verbose_proxy_logger.error(
+                "Spend log cleanup cancelled after %.2fs (rows_deleted=%d, batches=%d); the next run resumes from here",
+                time.monotonic() - run_started_at,
+                self._run_rows_deleted,
+                self._run_batches,
+            )
+            SpendLogCleanupMetrics.record_run("aborted")
+            raise
         except Exception as e:
             # .exception() captures the traceback; str(e) alone on a Prisma/DB
             # timeout is often empty and gives operators no signal to diagnose.
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 3b5f4236d22..cab0f4d0733 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -680,6 +680,10 @@ from litellm.proxy.route_llm_request import route_request
 from litellm.proxy.route_priority import hot_routes_first
 from litellm.proxy.search_endpoints.endpoints import router as search_router
 from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
+from litellm.proxy.shutdown.scheduled_jobs import (
+    AwaitableAsyncIOExecutor,
+    cancel_in_flight_scheduler_jobs,
+)
 from litellm.proxy.spend_tracking.budget_reservation import get_budget_window_start
 from litellm.proxy.spend_tracking.spend_counter_batch import (
     PendingSpendIncrement,
@@ -1456,6 +1460,13 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
 
     await proxy_config.stop_auth_cache_invalidation_subscriber()
 
+    # Shutdown event - cancel and await in-flight scheduled jobs while the DB is still connected
+    if scheduler is not None and scheduler_executor is not None:
+        try:
+            await cancel_in_flight_scheduler_jobs(scheduler, scheduler_executor)
+        except Exception as e:
+            verbose_proxy_logger.error("Error cancelling in-flight scheduled jobs: %s", e)
+
     await proxy_shutdown_event(worker_heartbeat=worker_heartbeat)
 
     if prometheus_multiproc_dir:
@@ -2451,6 +2462,7 @@ celery_app_conn: Final = None
 celery_fn: Final = None  # Redis Queue for handling requests
 
 scheduler = None
+scheduler_executor: AwaitableAsyncIOExecutor | None = None  # rebind-ok: bound once the scheduler is built at startup
 
 # Global variable for anthropic beta headers reload scheduling
 last_anthropic_beta_headers_reload = None
@@ -9763,7 +9775,7 @@ class ProxyStartupEvent:
         proxy_logging_obj: ProxyLogging,
     ) -> ProxyWorkerHeartbeat:
         """Initializes scheduled background jobs"""
-        global heuristic_v1_tuning_baselines, store_model_in_db, scheduler  # rebind-ok: startup publishes the one read-only baseline snapshot
+        global heuristic_v1_tuning_baselines, store_model_in_db, scheduler, scheduler_executor  # rebind-ok: startup publishes the one read-only baseline snapshot
 
         # MEMORY LEAK FIX: Configure scheduler with optimized settings
         # Memray analysis showed APScheduler's normalize() and _apply_jitter() causing
@@ -9772,9 +9784,9 @@ class ProxyStartupEvent:
         # 1. Remove/minimize jitter to avoid normalize() memory explosion
         # 2. Use larger misfire_grace_time to prevent backlog calculations
         # 3. Set replace_existing=True to avoid duplicate jobs
-        from apscheduler.executors.asyncio import AsyncIOExecutor
         from apscheduler.jobstores.memory import MemoryJobStore
 
+        scheduler_executor = AwaitableAsyncIOExecutor()  # rebind-ok: shutdown awaits the jobs this executor runs
         scheduler = AsyncIOScheduler(
             job_defaults={
                 "coalesce": APSCHEDULER_COALESCE,
@@ -9787,7 +9799,7 @@ class ProxyStartupEvent:
             jobstores={"default": MemoryJobStore()},  # explicitly use memory job store
             # Use simple executor to minimize overhead
             executors={
-                "default": AsyncIOExecutor(),
+                "default": scheduler_executor,
             },
             # Disable timezone awareness to reduce computation
             timezone=None,
diff --git a/litellm/proxy/shutdown/scheduled_jobs.py b/litellm/proxy/shutdown/scheduled_jobs.py
new file mode 100644
index 00000000000..3c9e791f51c
--- /dev/null
+++ b/litellm/proxy/shutdown/scheduled_jobs.py
@@ -0,0 +1,70 @@
+"""
+Cancel the proxy's in-flight scheduled jobs at shutdown so they can record how they ended.
+
+APScheduler's ``AsyncIOExecutor.shutdown`` cancels the job tasks it has in flight but cannot
+wait for them, because it is not a coroutine, and under uvicorn nothing else ever will: uvicorn
+re-raises the SIGTERM it captured as soon as the ASGI lifespan shutdown returns, so the process
+dies before ``asyncio.run`` reaches its cancel-all-tasks step. A job mid-run at that point is
+killed without ever observing cancellation, which is how a spend-log cleanup interrupted by a
+rolling restart left no outcome metric and no log line behind. Cancelling here and awaiting the
+cancelled tasks while the database is still connected is what lets a job's own
+``CancelledError`` handler run.
+
+The wait is bounded by ``JOB_CANCEL_TIMEOUT_SECONDS``. A job that has just been cancelled has only
+its own cleanup left to do, so the bound is there for a job that swallows cancellation, not one
+that honours it, and it keeps shutdown well inside a Kubernetes termination grace period.
+"""
+
+# pyright: reportMissingTypeStubs=false  # apscheduler ships no type information
+
+import asyncio
+from collections.abc import Collection
+from typing import Final, Protocol
+
+from apscheduler.executors.asyncio import AsyncIOExecutor
+
+from litellm._logging import verbose_proxy_logger
+
+JOB_CANCEL_TIMEOUT_SECONDS: Final = 5.0
+
+
+class StoppableScheduler(Protocol):
+    """The slice of ``AsyncIOScheduler`` shutdown uses, which ships no type information"""
+
+    @property
+    def running(self) -> bool: ...
+
+    def shutdown(self, wait: bool = ...) -> None: ...
+
+
+class AwaitableAsyncIOExecutor(AsyncIOExecutor):  # pyright: ignore[reportUntypedBaseClass]  # apscheduler ships no type information and is absent from the type-check env
+    """``AsyncIOExecutor`` whose in-flight job tasks can be awaited after ``shutdown`` cancels them"""
+
+    _pending_futures: Collection["asyncio.Future[object]"]
+
+    def in_flight_jobs(self) -> tuple["asyncio.Future[object]", ...]:
+        """The job tasks that are running right now, as a snapshot"""
+        return tuple(future for future in self._pending_futures if not future.done())
+
+
+async def cancel_in_flight_scheduler_jobs(scheduler: StoppableScheduler, executor: AwaitableAsyncIOExecutor) -> None:
+    """
+    Stop the scheduler and wait, bounded by JOB_CANCEL_TIMEOUT_SECONDS, for the jobs it cancels.
+
+    Must run before the database is disconnected: a job's cancellation handler is what records
+    the run's outcome, and it needs the connection the job was using.
+    """
+    if not scheduler.running:
+        return
+    in_flight: Final = executor.in_flight_jobs()
+    scheduler.shutdown(wait=False)
+    if not in_flight:
+        return
+    verbose_proxy_logger.info("Cancelling %d in-flight scheduled job(s) for shutdown", len(in_flight))
+    _done, pending = await asyncio.wait(in_flight, timeout=JOB_CANCEL_TIMEOUT_SECONDS)
+    if pending:
+        verbose_proxy_logger.warning(
+            "%d scheduled job(s) did not finish within %ss of cancellation; giving up on them",
+            len(pending),
+            JOB_CANCEL_TIMEOUT_SECONDS,
+        )
diff --git a/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py b/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
new file mode 100644
index 00000000000..b77d7c4ae50
--- /dev/null
+++ b/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
@@ -0,0 +1,125 @@
+"""
+Tests for cancelling in-flight scheduled jobs at proxy shutdown.
+
+These drive a real AsyncIOScheduler: the point of the helper is the hand-off
+between APScheduler's fire-and-forget cancellation and the lifespan shutdown
+that has to outlive it, and a mocked scheduler would not exercise that.
+"""
+
+import asyncio
+import logging
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from datetime import datetime
+
+import pytest
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+
+import litellm.proxy.shutdown.scheduled_jobs as scheduled_jobs
+from litellm.proxy.shutdown.scheduled_jobs import (
+    AwaitableAsyncIOExecutor,
+    cancel_in_flight_scheduler_jobs,
+)
+
+
+class _Job:
+    """A scheduled job that blocks until cancelled and records what it observed."""
+
+    def __init__(self, swallow_cancellation: bool = False) -> None:
+        self.started = asyncio.Event()
+        self.events: list[str] = []
+        self.swallow_cancellation = swallow_cancellation
+
+    async def run(self) -> None:
+        self.started.set()
+        try:
+            await asyncio.Event().wait()
+        except asyncio.CancelledError:
+            self.events.append("cancelled")
+            if self.swallow_cancellation:
+                await asyncio.Event().wait()
+            raise
+        finally:
+            self.events.append("finished")
+
+
+@asynccontextmanager
+async def _running_scheduler(*jobs: _Job) -> AsyncIterator[tuple[AsyncIOScheduler, AwaitableAsyncIOExecutor]]:
+    """A started scheduler with every job in flight; stopped on the way out whatever the test did."""
+    executor = AwaitableAsyncIOExecutor()
+    scheduler = AsyncIOScheduler(executors={"default": executor})
+    for index, job in enumerate(jobs):
+        scheduler.add_job(job.run, id=f"job-{index}", next_run_time=datetime.now())
+    scheduler.start()
+    try:
+        for job in jobs:
+            await asyncio.wait_for(job.started.wait(), timeout=5)
+        yield scheduler, executor
+    finally:
+        if scheduler.running:
+            scheduler.shutdown(wait=False)
+        stragglers = executor.in_flight_jobs()
+        for straggler in stragglers:
+            straggler.cancel()
+        await asyncio.gather(*stragglers, return_exceptions=True)
+
+
+@pytest.mark.asyncio
+async def test_in_flight_jobs_observe_cancellation_before_shutdown_returns():
+    """
+    The job's own CancelledError handler is what records how a run ended, so
+    shutdown must not return until that handler has run.
+    """
+    job = _Job()
+    async with _running_scheduler(job) as (scheduler, executor):
+        await cancel_in_flight_scheduler_jobs(scheduler, executor)
+
+        assert job.events == ["cancelled", "finished"]
+        assert scheduler.running is False
+        assert executor.in_flight_jobs() == ()
+
+
+@pytest.mark.asyncio
+async def test_every_in_flight_job_is_cancelled_not_only_the_first():
+    first, second = _Job(), _Job()
+    async with _running_scheduler(first, second) as (scheduler, executor):
+        await cancel_in_flight_scheduler_jobs(scheduler, executor)
+
+    assert first.events == ["cancelled", "finished"]
+    assert second.events == ["cancelled", "finished"]
+
+
+@pytest.mark.asyncio
+async def test_a_job_that_ignores_cancellation_is_abandoned_after_the_timeout(monkeypatch, caplog):
+    """
+    A job that swallows CancelledError must not hold the pod past its
+    termination grace period, so shutdown gives up on it and says so.
+    """
+    monkeypatch.setattr(scheduled_jobs, "JOB_CANCEL_TIMEOUT_SECONDS", 0.05)
+    job = _Job(swallow_cancellation=True)
+    async with _running_scheduler(job) as (scheduler, executor):
+        with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+            await cancel_in_flight_scheduler_jobs(scheduler, executor)
+
+        assert job.events == ["cancelled"]
+        assert "1 scheduled job(s) did not finish within 0.05s of cancellation" in caplog.text
+
+
+@pytest.mark.asyncio
+async def test_shutdown_with_nothing_in_flight_still_stops_the_scheduler():
+    async with _running_scheduler() as (scheduler, executor):
+        await cancel_in_flight_scheduler_jobs(scheduler, executor)
+        await asyncio.sleep(0)
+
+        assert scheduler.running is False
+
+
+@pytest.mark.asyncio
+async def test_a_scheduler_that_never_started_is_left_alone():
+    """The proxy runs without a scheduler when it has no database; shutdown must not trip on that."""
+    executor = AwaitableAsyncIOExecutor()
+    scheduler = AsyncIOScheduler(executors={"default": executor})
+
+    await cancel_in_flight_scheduler_jobs(scheduler, executor)
+
+    assert scheduler.running is False
diff --git a/tests/test_litellm/proxy/test_spend_log_cleanup.py b/tests/test_litellm/proxy/test_spend_log_cleanup.py
index bf1538183ab..ed35af7ee38 100644
--- a/tests/test_litellm/proxy/test_spend_log_cleanup.py
+++ b/tests/test_litellm/proxy/test_spend_log_cleanup.py
@@ -7,6 +7,7 @@ import math
 import time
 from contextlib import asynccontextmanager
 from datetime import datetime, timedelta, timezone
+from typing import Final
 from unittest.mock import AsyncMock, MagicMock
 
 import pytest
@@ -1417,3 +1418,94 @@ def test_the_reported_run_outcome_is_the_most_significant_reason_in_any_order(st
     """
     results = tuple(TableCleanupResult(rows_deleted=0, stop_reason=reason) for reason in stop_reasons)
     assert SpendLogCleanup._run_outcome(results) == expected
+
+
+_OTHER_OUTCOMES: Final = ("completed", "budget_exhausted", "batch_cap_reached", "skipped_locked", "skipped_disabled")
+
+
+def _runs_recorded(outcome: str) -> float:
+    """The real ``litellm_spend_log_cleanup_runs_total`` sample for one outcome, 0 when unset"""
+    from prometheus_client import REGISTRY
+
+    return REGISTRY.get_sample_value("litellm_spend_log_cleanup_runs_total", {"outcome": outcome}) or 0.0
+
+
+@pytest.mark.asyncio
+async def test_a_cancelled_run_records_aborted_and_logs_its_progress_before_re_raising(monkeypatch):
+    """
+    Shutdown cancels a run by throwing CancelledError into whichever batch is in
+    flight. That is a BaseException, so the Exception handler never saw it and
+    an interrupted run left no outcome metric and no log line; operators could
+    not tell that cleanup stopped early, let alone how far it got.
+    """
+    import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module
+
+    mock_logger = MagicMock()
+    monkeypatch.setattr(cleanup_module, "verbose_proxy_logger", mock_logger)
+    aborted_runs_before = _runs_recorded("aborted")
+    other_runs_before = {outcome: _runs_recorded(outcome) for outcome in _OTHER_OUTCOMES}
+
+    third_batch_reached = asyncio.Event()
+
+    async def _execute_raw(sql, *args):
+        if third_batch_reached.is_set():
+            raise AssertionError("no batch may be issued after the cancelled one")
+        if _execute_raw.calls < 2:
+            _execute_raw.calls += 1
+            return 150
+        third_batch_reached.set()
+        await asyncio.Event().wait()
+
+    _execute_raw.calls = 0
+    mock_prisma_client = MagicMock()
+    _wire_tx(mock_prisma_client.db)
+    mock_prisma_client.db.execute_raw = _execute_raw
+
+    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
+    cleaner.pod_lock_manager = MagicMock()
+    cleaner.pod_lock_manager.redis_cache = MagicMock()
+    cleaner.pod_lock_manager.acquire_lock = AsyncMock(return_value=True)
+    cleaner.pod_lock_manager.release_lock = AsyncMock()
+
+    run = asyncio.ensure_future(cleaner.cleanup_old_spend_logs(mock_prisma_client))
+    await asyncio.wait_for(third_batch_reached.wait(), timeout=5)
+    run.cancel()
+    with pytest.raises(asyncio.CancelledError):
+        await run
+
+    assert _runs_recorded("aborted") == aborted_runs_before + 1
+    assert {outcome: _runs_recorded(outcome) for outcome in _OTHER_OUTCOMES} == other_runs_before
+    cleaner.pod_lock_manager.release_lock.assert_awaited_once()
+    mock_logger.exception.assert_not_called()
+    (error_call,) = mock_logger.error.call_args_list
+    rendered = error_call[0][0] % error_call[0][1:]
+    assert rendered.startswith("Spend log cleanup cancelled after ")
+    assert "s (rows_deleted=300, batches=2)" in rendered
+
+
+@pytest.mark.asyncio
+async def test_progress_reported_for_a_cancelled_run_is_that_run_only(monkeypatch):
+    """
+    The scheduler holds one cleaner for the life of the process, so the
+    progress counters must start from zero on every run rather than carrying
+    an earlier run's totals into the cancellation line.
+    """
+    import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module
+
+    mock_logger = MagicMock()
+    monkeypatch.setattr(cleanup_module, "verbose_proxy_logger", mock_logger)
+
+    mock_prisma_client = MagicMock()
+    _wire_tx(mock_prisma_client.db)
+    mock_prisma_client.db.execute_raw = AsyncMock(side_effect=[150, 0, 0])
+    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
+    cleaner.pod_lock_manager = None
+    await cleaner.cleanup_old_spend_logs(mock_prisma_client)
+
+    mock_prisma_client.db.execute_raw = AsyncMock(side_effect=[150, asyncio.CancelledError()])
+    with pytest.raises(asyncio.CancelledError):
+        await cleaner.cleanup_old_spend_logs(mock_prisma_client)
+
+    (error_call,) = mock_logger.error.call_args_list
+    rendered = error_call[0][0] % error_call[0][1:]
+    assert "(rows_deleted=150, batches=1)" in rendered

From 8ce8dd9b3b184a8e8e8884cde29f07e07063f6f7 Mon Sep 17 00:00:00 2001
From: Yucheng He <yucheng@berri.ai>
Date: Tue, 15 Sep 2026 02:25:36 -0700
Subject: [PATCH 030/160] fix(proxy): pause the scheduler at shutdown start and
 keep cleanup progress per run

Review follow-ups on #41213:

- Pause the scheduler as the first shutdown step so a job whose fire time
  falls inside the shutdown window does not start only to be cancelled.
  Jobs already running keep the whole window and are cancelled and
  awaited before the database disconnects, as before.
- Keep the cleanup run's progress in a task-scoped ContextVar rather than
  on the cleaner instance, so two runs overlapping on one cleaner
  (APSCHEDULER_MAX_INSTANCES above 1 without a Redis lock) each report
  their own rows and batches on cancellation.
- Drop the module docstrings the repository comment policy does not
  allow; the rationale lives in the PR description.
---
 .../db_transaction_queue/spend_log_cleanup.py | 37 +++++++++---
 litellm/proxy/proxy_server.py                 |  5 ++
 litellm/proxy/shutdown/scheduled_jobs.py      | 25 +++-----
 .../proxy/shutdown/test_scheduled_jobs.py     | 57 ++++++++++++-------
 .../proxy/test_spend_log_cleanup.py           | 53 +++++++++++++----
 5 files changed, 121 insertions(+), 56 deletions(-)

diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
index 1a14210dbec..34213c0d2ce 100644
--- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
+++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
@@ -1,5 +1,6 @@
 import asyncio
 import time
+from contextvars import ContextVar
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from typing import Final, Literal, TypeAlias
@@ -40,6 +41,28 @@ class TableCleanupResult:
     stop_reason: StopReason
 
 
+class _RunProgress:
+    """How far one cleanup run has got, reported if that run is cancelled"""
+
+    def __init__(self) -> None:
+        self.rows_deleted: int = 0
+        self.batches: int = 0
+
+    def record_batch(self, rows_deleted: int) -> None:
+        self.rows_deleted += rows_deleted
+        self.batches += 1
+
+
+_run_progress: ContextVar[_RunProgress] = ContextVar("spend_log_cleanup_run_progress")
+
+
+def _record_run_batch(rows_deleted: int) -> None:
+    """Count a batch towards the run in progress, if a run is what issued it"""
+    progress: Final = _run_progress.get(None)
+    if progress is not None:
+        progress.record_batch(rows_deleted)
+
+
 class _RemainingRow(BaseModel):
     """One row of the capped outstanding-rows probe, validated out of prisma's untyped result."""
 
@@ -96,8 +119,6 @@ class SpendLogCleanup:
 
         self.general_settings = general_settings or default_settings
         self._refresh_bounds()
-        self._run_rows_deleted: int = 0
-        self._run_batches: int = 0
         from litellm.proxy.proxy_server import proxy_logging_obj
 
         pod_lock_manager: Final = proxy_logging_obj.db_spend_update_writer.pod_lock_manager
@@ -424,8 +445,7 @@ class SpendLogCleanup:
 
             total_deleted += deleted_count
             run_count += 1
-            self._run_rows_deleted += deleted_count
-            self._run_batches += 1
+            _record_run_batch(deleted_count)
 
             # Add a small sleep to prevent overwhelming the database
             await asyncio.sleep(0.1)
@@ -595,8 +615,8 @@ class SpendLogCleanup:
         """
         lock_acquired = False
         run_started_at: Final = time.monotonic()
-        self._run_rows_deleted = 0
-        self._run_batches = 0
+        progress: Final = _RunProgress()
+        progress_token: Final = _run_progress.set(progress)
         try:
             verbose_proxy_logger.info("Cleanup job triggered at %s", datetime.now())
             self._refresh_bounds()
@@ -681,8 +701,8 @@ class SpendLogCleanup:
             verbose_proxy_logger.error(
                 "Spend log cleanup cancelled after %.2fs (rows_deleted=%d, batches=%d); the next run resumes from here",
                 time.monotonic() - run_started_at,
-                self._run_rows_deleted,
-                self._run_batches,
+                progress.rows_deleted,
+                progress.batches,
             )
             SpendLogCleanupMetrics.record_run("aborted")
             raise
@@ -697,6 +717,7 @@ class SpendLogCleanup:
             SpendLogCleanupMetrics.record_run("aborted")
             return  # Return after error handling
         finally:
+            _run_progress.reset(progress_token)
             # Only release the lock if it was actually acquired
             if lock_acquired and self.pod_lock_manager and self.pod_lock_manager.redis_cache:
                 await self.pod_lock_manager.release_lock(cronjob_id=SPEND_LOG_CLEANUP_JOB_NAME)
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index cab0f4d0733..1a7b3a6ccfb 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -683,6 +683,7 @@ from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownMan
 from litellm.proxy.shutdown.scheduled_jobs import (
     AwaitableAsyncIOExecutor,
     cancel_in_flight_scheduler_jobs,
+    pause_scheduled_jobs,
 )
 from litellm.proxy.spend_tracking.budget_reservation import get_budget_window_start
 from litellm.proxy.spend_tracking.spend_counter_batch import (
@@ -1419,6 +1420,10 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
         if model_info_scheduler is not scheduler:
             model_info_scheduler.shutdown(wait=False)
 
+    # Shutdown event - stop starting scheduled jobs; the ones already running keep the drain window
+    if scheduler is not None:
+        pause_scheduled_jobs(scheduler)
+
     # Shutdown event - drain in-flight requests before tearing down dependencies
     # so SIGTERM (rolling update, scale-down, liveness kill) doesn't drop them.
     GracefulShutdownManager.start_shutdown()
diff --git a/litellm/proxy/shutdown/scheduled_jobs.py b/litellm/proxy/shutdown/scheduled_jobs.py
index 3c9e791f51c..46c57e1a608 100644
--- a/litellm/proxy/shutdown/scheduled_jobs.py
+++ b/litellm/proxy/shutdown/scheduled_jobs.py
@@ -1,20 +1,3 @@
-"""
-Cancel the proxy's in-flight scheduled jobs at shutdown so they can record how they ended.
-
-APScheduler's ``AsyncIOExecutor.shutdown`` cancels the job tasks it has in flight but cannot
-wait for them, because it is not a coroutine, and under uvicorn nothing else ever will: uvicorn
-re-raises the SIGTERM it captured as soon as the ASGI lifespan shutdown returns, so the process
-dies before ``asyncio.run`` reaches its cancel-all-tasks step. A job mid-run at that point is
-killed without ever observing cancellation, which is how a spend-log cleanup interrupted by a
-rolling restart left no outcome metric and no log line behind. Cancelling here and awaiting the
-cancelled tasks while the database is still connected is what lets a job's own
-``CancelledError`` handler run.
-
-The wait is bounded by ``JOB_CANCEL_TIMEOUT_SECONDS``. A job that has just been cancelled has only
-its own cleanup left to do, so the bound is there for a job that swallows cancellation, not one
-that honours it, and it keeps shutdown well inside a Kubernetes termination grace period.
-"""
-
 # pyright: reportMissingTypeStubs=false  # apscheduler ships no type information
 
 import asyncio
@@ -34,6 +17,8 @@ class StoppableScheduler(Protocol):
     @property
     def running(self) -> bool: ...
 
+    def pause(self) -> None: ...
+
     def shutdown(self, wait: bool = ...) -> None: ...
 
 
@@ -47,6 +32,12 @@ class AwaitableAsyncIOExecutor(AsyncIOExecutor):  # pyright: ignore[reportUntype
         return tuple(future for future in self._pending_futures if not future.done())
 
 
+def pause_scheduled_jobs(scheduler: StoppableScheduler) -> None:
+    """Stop the scheduler from starting jobs that shutdown would only cancel; running jobs continue"""
+    if scheduler.running:
+        scheduler.pause()
+
+
 async def cancel_in_flight_scheduler_jobs(scheduler: StoppableScheduler, executor: AwaitableAsyncIOExecutor) -> None:
     """
     Stop the scheduler and wait, bounded by JOB_CANCEL_TIMEOUT_SECONDS, for the jobs it cancels.
diff --git a/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py b/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
index b77d7c4ae50..3301ce34cd6 100644
--- a/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
+++ b/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
@@ -1,16 +1,8 @@
-"""
-Tests for cancelling in-flight scheduled jobs at proxy shutdown.
-
-These drive a real AsyncIOScheduler: the point of the helper is the hand-off
-between APScheduler's fire-and-forget cancellation and the lifespan shutdown
-that has to outlive it, and a mocked scheduler would not exercise that.
-"""
-
 import asyncio
 import logging
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import pytest
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
@@ -19,11 +11,12 @@ import litellm.proxy.shutdown.scheduled_jobs as scheduled_jobs
 from litellm.proxy.shutdown.scheduled_jobs import (
     AwaitableAsyncIOExecutor,
     cancel_in_flight_scheduler_jobs,
+    pause_scheduled_jobs,
 )
 
 
 class _Job:
-    """A scheduled job that blocks until cancelled and records what it observed."""
+    """A scheduled job that blocks until cancelled and records what it observed"""
 
     def __init__(self, swallow_cancellation: bool = False) -> None:
         self.started = asyncio.Event()
@@ -45,7 +38,7 @@ class _Job:
 
 @asynccontextmanager
 async def _running_scheduler(*jobs: _Job) -> AsyncIterator[tuple[AsyncIOScheduler, AwaitableAsyncIOExecutor]]:
-    """A started scheduler with every job in flight; stopped on the way out whatever the test did."""
+    """A started scheduler with every job in flight, stopped on the way out whatever the test did"""
     executor = AwaitableAsyncIOExecutor()
     scheduler = AsyncIOScheduler(executors={"default": executor})
     for index, job in enumerate(jobs):
@@ -66,10 +59,7 @@ async def _running_scheduler(*jobs: _Job) -> AsyncIterator[tuple[AsyncIOSchedule
 
 @pytest.mark.asyncio
 async def test_in_flight_jobs_observe_cancellation_before_shutdown_returns():
-    """
-    The job's own CancelledError handler is what records how a run ended, so
-    shutdown must not return until that handler has run.
-    """
+    """The job's own CancelledError handler records how a run ended, so shutdown must wait for it"""
     job = _Job()
     async with _running_scheduler(job) as (scheduler, executor):
         await cancel_in_flight_scheduler_jobs(scheduler, executor)
@@ -91,10 +81,7 @@ async def test_every_in_flight_job_is_cancelled_not_only_the_first():
 
 @pytest.mark.asyncio
 async def test_a_job_that_ignores_cancellation_is_abandoned_after_the_timeout(monkeypatch, caplog):
-    """
-    A job that swallows CancelledError must not hold the pod past its
-    termination grace period, so shutdown gives up on it and says so.
-    """
+    """A job that swallows CancelledError must not hold the pod past its termination grace period"""
     monkeypatch.setattr(scheduled_jobs, "JOB_CANCEL_TIMEOUT_SECONDS", 0.05)
     job = _Job(swallow_cancellation=True)
     async with _running_scheduler(job) as (scheduler, executor):
@@ -116,10 +103,40 @@ async def test_shutdown_with_nothing_in_flight_still_stops_the_scheduler():
 
 @pytest.mark.asyncio
 async def test_a_scheduler_that_never_started_is_left_alone():
-    """The proxy runs without a scheduler when it has no database; shutdown must not trip on that."""
+    """The proxy runs without a scheduler when it has no database"""
     executor = AwaitableAsyncIOExecutor()
     scheduler = AsyncIOScheduler(executors={"default": executor})
 
     await cancel_in_flight_scheduler_jobs(scheduler, executor)
 
     assert scheduler.running is False
+
+
+@pytest.mark.asyncio
+async def test_pausing_stops_new_jobs_from_starting_but_leaves_running_ones_alone():
+    """A job due during the shutdown drain would only be cancelled, so it must not start at all"""
+    running = _Job()
+    async with _running_scheduler(running) as (scheduler, executor):
+        late = _Job()
+        scheduler.add_job(late.run, id="late", next_run_time=datetime.now() + timedelta(seconds=0.1))
+
+        pause_scheduled_jobs(scheduler)
+        await asyncio.sleep(0.3)
+
+        assert late.started.is_set() is False
+        assert running.events == []
+        assert scheduler.running is True
+
+        await cancel_in_flight_scheduler_jobs(scheduler, executor)
+
+    assert running.events == ["cancelled", "finished"]
+    assert late.started.is_set() is False
+
+
+@pytest.mark.asyncio
+async def test_pausing_a_scheduler_that_never_started_is_a_no_op():
+    scheduler = AsyncIOScheduler(executors={"default": AwaitableAsyncIOExecutor()})
+
+    pause_scheduled_jobs(scheduler)
+
+    assert scheduler.running is False
diff --git a/tests/test_litellm/proxy/test_spend_log_cleanup.py b/tests/test_litellm/proxy/test_spend_log_cleanup.py
index ed35af7ee38..1691b2d174a 100644
--- a/tests/test_litellm/proxy/test_spend_log_cleanup.py
+++ b/tests/test_litellm/proxy/test_spend_log_cleanup.py
@@ -1432,12 +1432,7 @@ def _runs_recorded(outcome: str) -> float:
 
 @pytest.mark.asyncio
 async def test_a_cancelled_run_records_aborted_and_logs_its_progress_before_re_raising(monkeypatch):
-    """
-    Shutdown cancels a run by throwing CancelledError into whichever batch is in
-    flight. That is a BaseException, so the Exception handler never saw it and
-    an interrupted run left no outcome metric and no log line; operators could
-    not tell that cleanup stopped early, let alone how far it got.
-    """
+    """A run cut short by shutdown must leave its outcome and how far it got behind"""
     import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module
 
     mock_logger = MagicMock()
@@ -1485,11 +1480,7 @@ async def test_a_cancelled_run_records_aborted_and_logs_its_progress_before_re_r
 
 @pytest.mark.asyncio
 async def test_progress_reported_for_a_cancelled_run_is_that_run_only(monkeypatch):
-    """
-    The scheduler holds one cleaner for the life of the process, so the
-    progress counters must start from zero on every run rather than carrying
-    an earlier run's totals into the cancellation line.
-    """
+    """The scheduler holds one cleaner for the life of the process, so progress must not carry over"""
     import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module
 
     mock_logger = MagicMock()
@@ -1509,3 +1500,43 @@ async def test_progress_reported_for_a_cancelled_run_is_that_run_only(monkeypatc
     (error_call,) = mock_logger.error.call_args_list
     rendered = error_call[0][0] % error_call[0][1:]
     assert "(rows_deleted=150, batches=1)" in rendered
+
+
+@pytest.mark.asyncio
+async def test_progress_reported_by_an_overlapping_run_is_its_own(monkeypatch):
+    """With APSCHEDULER_MAX_INSTANCES above one, two runs share the cleaner but not their progress"""
+    import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module
+
+    mock_logger = MagicMock()
+    monkeypatch.setattr(cleanup_module, "verbose_proxy_logger", mock_logger)
+
+    first_batch_done = asyncio.Event()
+    second_run_done = asyncio.Event()
+
+    async def _slow_execute_raw(sql, *args):
+        first_batch_done.set()
+        await second_run_done.wait()
+        return 100
+
+    slow_client = MagicMock()
+    _wire_tx(slow_client.db)
+    slow_client.db.execute_raw = _slow_execute_raw
+    fast_client = MagicMock()
+    _wire_tx(fast_client.db)
+    fast_client.db.execute_raw = AsyncMock(side_effect=[150, 150, 0, 0])
+
+    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
+    cleaner.pod_lock_manager = None
+
+    slow_run = asyncio.ensure_future(cleaner.cleanup_old_spend_logs(slow_client))
+    await asyncio.wait_for(first_batch_done.wait(), timeout=5)
+    await cleaner.cleanup_old_spend_logs(fast_client)
+    second_run_done.set()
+    await asyncio.sleep(0)
+    slow_run.cancel()
+    with pytest.raises(asyncio.CancelledError):
+        await slow_run
+
+    (error_call,) = mock_logger.error.call_args_list
+    rendered = error_call[0][0] % error_call[0][1:]
+    assert "(rows_deleted=100, batches=1)" in rendered

From 39a199d9a2285a0647fd0d70b3f2a7e2d72120d1 Mon Sep 17 00:00:00 2001
From: Yucheng He <yucheng@berri.ai>
Date: Tue, 15 Sep 2026 03:13:35 -0700
Subject: [PATCH 031/160] fix(proxy): let in-flight scheduled jobs finish
 before cancelling them at shutdown

Cancelling every in-flight job the moment shutdown reached the scheduler
dropped the rows a write job had already popped: flush_gateway_requests
drains its accumulator before committing and does not restore it on
CancelledError, and update_spend requeues its batch only after the
shutdown drain had already run.

Shutdown now waits up to JOB_FINISH_TIMEOUT_SECONDS for in-flight jobs
to finish on their own, cancels the ones still running, and does both
before the shutdown flushes so a requeued batch is still written. The
cleanup run never finishes inside the grace, so it is still cancelled
and still records outcome="aborted".

Resolves LIT-6990
---
 litellm/proxy/proxy_server.py                 | 16 ++++----
 litellm/proxy/shutdown/scheduled_jobs.py      | 22 +++++++----
 .../proxy/shutdown/test_scheduled_jobs.py     | 39 ++++++++++++++-----
 3 files changed, 52 insertions(+), 25 deletions(-)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 1a7b3a6ccfb..7505714b418 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -682,8 +682,8 @@ from litellm.proxy.search_endpoints.endpoints import router as search_router
 from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
 from litellm.proxy.shutdown.scheduled_jobs import (
     AwaitableAsyncIOExecutor,
-    cancel_in_flight_scheduler_jobs,
     pause_scheduled_jobs,
+    stop_in_flight_scheduler_jobs,
 )
 from litellm.proxy.spend_tracking.budget_reservation import get_budget_window_start
 from litellm.proxy.spend_tracking.spend_counter_batch import (
@@ -1457,6 +1457,13 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
 
     await _drain_spend_event_producer_on_shutdown()
 
+    # Shutdown event - finish or cancel in-flight scheduled jobs before the shutdown flushes and the DB disconnect
+    if scheduler is not None and scheduler_executor is not None:
+        try:
+            await stop_in_flight_scheduler_jobs(scheduler, scheduler_executor)
+        except Exception as e:
+            verbose_proxy_logger.error("Error stopping in-flight scheduled jobs: %s", e)
+
     await flush_spend_counters_on_shutdown()
 
     await _flush_spend_logs_queue_on_shutdown()
@@ -1465,13 +1472,6 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
 
     await proxy_config.stop_auth_cache_invalidation_subscriber()
 
-    # Shutdown event - cancel and await in-flight scheduled jobs while the DB is still connected
-    if scheduler is not None and scheduler_executor is not None:
-        try:
-            await cancel_in_flight_scheduler_jobs(scheduler, scheduler_executor)
-        except Exception as e:
-            verbose_proxy_logger.error("Error cancelling in-flight scheduled jobs: %s", e)
-
     await proxy_shutdown_event(worker_heartbeat=worker_heartbeat)
 
     if prometheus_multiproc_dir:
diff --git a/litellm/proxy/shutdown/scheduled_jobs.py b/litellm/proxy/shutdown/scheduled_jobs.py
index 46c57e1a608..e7625a73b47 100644
--- a/litellm/proxy/shutdown/scheduled_jobs.py
+++ b/litellm/proxy/shutdown/scheduled_jobs.py
@@ -8,6 +8,7 @@ from apscheduler.executors.asyncio import AsyncIOExecutor
 
 from litellm._logging import verbose_proxy_logger
 
+JOB_FINISH_TIMEOUT_SECONDS: Final = 5.0
 JOB_CANCEL_TIMEOUT_SECONDS: Final = 5.0
 
 
@@ -38,21 +39,28 @@ def pause_scheduled_jobs(scheduler: StoppableScheduler) -> None:
         scheduler.pause()
 
 
-async def cancel_in_flight_scheduler_jobs(scheduler: StoppableScheduler, executor: AwaitableAsyncIOExecutor) -> None:
+async def stop_in_flight_scheduler_jobs(scheduler: StoppableScheduler, executor: AwaitableAsyncIOExecutor) -> None:
     """
-    Stop the scheduler and wait, bounded by JOB_CANCEL_TIMEOUT_SECONDS, for the jobs it cancels.
+    Let in-flight jobs finish for up to JOB_FINISH_TIMEOUT_SECONDS, then stop the scheduler and
+    wait, bounded by JOB_CANCEL_TIMEOUT_SECONDS, for the jobs it cancels.
 
-    Must run before the database is disconnected: a job's cancellation handler is what records
-    the run's outcome, and it needs the connection the job was using.
+    Must run before the database is disconnected: a write job that finishes needs its connection,
+    and a job's cancellation handler is what records the run's outcome.
     """
     if not scheduler.running:
         return
     in_flight: Final = executor.in_flight_jobs()
+    still_running: set[asyncio.Future[object]] = set()
+    if in_flight:
+        verbose_proxy_logger.info(
+            "Waiting up to %ss for %d in-flight scheduled job(s) to finish", JOB_FINISH_TIMEOUT_SECONDS, len(in_flight)
+        )
+        _done, still_running = await asyncio.wait(in_flight, timeout=JOB_FINISH_TIMEOUT_SECONDS)
     scheduler.shutdown(wait=False)
-    if not in_flight:
+    if not still_running:
         return
-    verbose_proxy_logger.info("Cancelling %d in-flight scheduled job(s) for shutdown", len(in_flight))
-    _done, pending = await asyncio.wait(in_flight, timeout=JOB_CANCEL_TIMEOUT_SECONDS)
+    verbose_proxy_logger.info("Cancelling %d in-flight scheduled job(s) for shutdown", len(still_running))
+    _done, pending = await asyncio.wait(still_running, timeout=JOB_CANCEL_TIMEOUT_SECONDS)
     if pending:
         verbose_proxy_logger.warning(
             "%d scheduled job(s) did not finish within %ss of cancellation; giving up on them",
diff --git a/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py b/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
index 3301ce34cd6..7defd6cef6c 100644
--- a/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
+++ b/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
@@ -10,23 +10,28 @@ from apscheduler.schedulers.asyncio import AsyncIOScheduler
 import litellm.proxy.shutdown.scheduled_jobs as scheduled_jobs
 from litellm.proxy.shutdown.scheduled_jobs import (
     AwaitableAsyncIOExecutor,
-    cancel_in_flight_scheduler_jobs,
+    stop_in_flight_scheduler_jobs,
     pause_scheduled_jobs,
 )
 
 
 class _Job:
-    """A scheduled job that blocks until cancelled and records what it observed"""
+    """A scheduled job that blocks until cancelled, or for ``work_seconds``, and records what it observed"""
 
-    def __init__(self, swallow_cancellation: bool = False) -> None:
+    def __init__(self, swallow_cancellation: bool = False, work_seconds: float | None = None) -> None:
         self.started = asyncio.Event()
         self.events: list[str] = []
         self.swallow_cancellation = swallow_cancellation
+        self.work_seconds = work_seconds
 
     async def run(self) -> None:
         self.started.set()
         try:
-            await asyncio.Event().wait()
+            if self.work_seconds is None:
+                await asyncio.Event().wait()
+            else:
+                await asyncio.sleep(self.work_seconds)
+                self.events.append("committed")
         except asyncio.CancelledError:
             self.events.append("cancelled")
             if self.swallow_cancellation:
@@ -62,18 +67,32 @@ async def test_in_flight_jobs_observe_cancellation_before_shutdown_returns():
     """The job's own CancelledError handler records how a run ended, so shutdown must wait for it"""
     job = _Job()
     async with _running_scheduler(job) as (scheduler, executor):
-        await cancel_in_flight_scheduler_jobs(scheduler, executor)
+        await stop_in_flight_scheduler_jobs(scheduler, executor)
 
         assert job.events == ["cancelled", "finished"]
         assert scheduler.running is False
         assert executor.in_flight_jobs() == ()
 
 
+@pytest.mark.asyncio
+async def test_a_job_that_is_finishing_is_allowed_to_finish_rather_than_cancelled(monkeypatch):
+    """A spend write cancelled mid-commit drops the rows it popped, so short jobs get to finish first"""
+    monkeypatch.setattr(scheduled_jobs, "JOB_FINISH_TIMEOUT_SECONDS", 2.0)
+    write = _Job(work_seconds=0.2)
+    stuck = _Job()
+    async with _running_scheduler(write, stuck) as (scheduler, executor):
+        await stop_in_flight_scheduler_jobs(scheduler, executor)
+
+        assert write.events == ["committed", "finished"]
+        assert stuck.events == ["cancelled", "finished"]
+        assert scheduler.running is False
+
+
 @pytest.mark.asyncio
 async def test_every_in_flight_job_is_cancelled_not_only_the_first():
     first, second = _Job(), _Job()
     async with _running_scheduler(first, second) as (scheduler, executor):
-        await cancel_in_flight_scheduler_jobs(scheduler, executor)
+        await stop_in_flight_scheduler_jobs(scheduler, executor)
 
     assert first.events == ["cancelled", "finished"]
     assert second.events == ["cancelled", "finished"]
@@ -86,7 +105,7 @@ async def test_a_job_that_ignores_cancellation_is_abandoned_after_the_timeout(mo
     job = _Job(swallow_cancellation=True)
     async with _running_scheduler(job) as (scheduler, executor):
         with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
-            await cancel_in_flight_scheduler_jobs(scheduler, executor)
+            await stop_in_flight_scheduler_jobs(scheduler, executor)
 
         assert job.events == ["cancelled"]
         assert "1 scheduled job(s) did not finish within 0.05s of cancellation" in caplog.text
@@ -95,7 +114,7 @@ async def test_a_job_that_ignores_cancellation_is_abandoned_after_the_timeout(mo
 @pytest.mark.asyncio
 async def test_shutdown_with_nothing_in_flight_still_stops_the_scheduler():
     async with _running_scheduler() as (scheduler, executor):
-        await cancel_in_flight_scheduler_jobs(scheduler, executor)
+        await stop_in_flight_scheduler_jobs(scheduler, executor)
         await asyncio.sleep(0)
 
         assert scheduler.running is False
@@ -107,7 +126,7 @@ async def test_a_scheduler_that_never_started_is_left_alone():
     executor = AwaitableAsyncIOExecutor()
     scheduler = AsyncIOScheduler(executors={"default": executor})
 
-    await cancel_in_flight_scheduler_jobs(scheduler, executor)
+    await stop_in_flight_scheduler_jobs(scheduler, executor)
 
     assert scheduler.running is False
 
@@ -127,7 +146,7 @@ async def test_pausing_stops_new_jobs_from_starting_but_leaves_running_ones_alon
         assert running.events == []
         assert scheduler.running is True
 
-        await cancel_in_flight_scheduler_jobs(scheduler, executor)
+        await stop_in_flight_scheduler_jobs(scheduler, executor)
 
     assert running.events == ["cancelled", "finished"]
     assert late.started.is_set() is False

From e9109ddf4a563c7d72b30db9bbcc1d334111fec8 Mon Sep 17 00:00:00 2001
From: Tin Chi Lo <tin@berri.ai>
Date: Fri, 18 Sep 2026 15:11:12 -0500
Subject: [PATCH 032/160] fix(router): make context-window escalation opt-in

---
 .../complexity_router/README.md               | 13 ++++
 .../complexity_router/config.py               |  5 +-
 .../router_strategy/test_complexity_router.py | 70 ++++++++++++++-----
 .../ContextWindowEscalationConfig.tsx         |  5 +-
 .../add_model/add_auto_router_tab.test.tsx    | 11 +--
 .../build_complexity_router_config.test.ts    | 19 +++--
 .../build_complexity_router_config.ts         |  6 +-
 ...d_updated_complexity_router_config.test.ts | 23 ++++--
 .../src/lib/autorouter_presets.test.ts        |  6 +-
 ui/litellm-dashboard/src/lib/http/schema.d.ts |  4 +-
 10 files changed, 111 insertions(+), 51 deletions(-)

diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md
index 6505746bca1..2c2aea333f9 100644
--- a/litellm/router_strategy/complexity_router/README.md
+++ b/litellm/router_strategy/complexity_router/README.md
@@ -68,6 +68,19 @@ still resolve to a deployment in `model_list`; this configuration does not creat
             - abc
 ```
 
+### Context-window escalation
+
+Context-window escalation is opt-in. Omit `enable_context_window_escalation` or set it to
+`false` to keep the complexity-selected model without context-window replacement or filtering
+
+Set `enable_context_window_escalation: true` inside `complexity_router_config` to restrict the
+selected tier to models whose declared windows fit the prompt, or move to the lowest configured
+tier with a fitting model when none in the selected tier fit. Unknown windows do not justify
+moving a request. `context_window_escalation_buffer` defaults to `0.95`
+
+Existing saved configurations with explicit `true` keep escalation enabled. Configurations that
+omit the setting now default to disabled; set it to `true` to retain their previous behavior
+
 ### Capability forecasting
 
 Set `classifier_type: capability` to use
diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py
index aa39dff8c53..213d3864dc0 100644
--- a/litellm/router_strategy/complexity_router/config.py
+++ b/litellm/router_strategy/complexity_router/config.py
@@ -1305,7 +1305,7 @@ class ComplexityRouterConfig(BaseModel):
     )
 
     enable_context_window_escalation: bool = Field(
-        default=True,
+        default=False,
         description=(
             "Escalate a request off a tier whose models provably cannot hold its prompt, before "
             "dispatch. The classifier scores complexity and never prompt size, so a long agentic "
@@ -1315,7 +1315,8 @@ class ComplexityRouterConfig(BaseModel):
             "moves to the lowest configured tier with a model whose declared window fits; when "
             "only some of the tier's models fit, the pick is restricted to those and the tier "
             "keeps the request. Models with no resolvable window are never escalated away from "
-            "and never escalated onto. Set false to dispatch on complexity alone, as before."
+            "and never escalated onto. Disabled by default: omit or set false to dispatch on "
+            "complexity alone; set true to enable context-window escalation."
         ),
     )
     context_window_escalation_buffer: float = Field(
diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py
index 9b25c869f1c..10d674f1ab8 100644
--- a/tests/test_litellm/router_strategy/test_complexity_router.py
+++ b/tests/test_litellm/router_strategy/test_complexity_router.py
@@ -13,6 +13,7 @@ import time
 from collections.abc import AsyncIterator, Mapping, Sequence
 from copy import deepcopy
 from functools import partial
+from types import MappingProxyType
 from typing import Dict, Final, List, Literal
 from unittest.mock import AsyncMock, MagicMock, patch
 
@@ -90,6 +91,7 @@ from litellm.types.router import (
     TaggedPreRoutingStrategy,
 )
 from litellm.types.llms.openai import ResponsesAPIResponse
+from litellm.types.management_endpoints.auto_router_endpoints import RequestComplexityRouterConfig
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 
 
@@ -6558,6 +6560,7 @@ class TestTierModelAffinity:
             litellm_router_instance=_windowed_router(_SMALL, _BIG),
             complexity_router_config={
                 "tiers": {"SIMPLE": ["small-model", "big-model"]},
+                "enable_context_window_escalation": True,
                 "adaptive": adaptive,
                 "deployment_affinity": True,
                 "session_affinity": False,
@@ -13723,8 +13726,12 @@ _CJK_TURNS = [
 ]
 
 
-def _tier_config(**overrides) -> Dict:
-    return {"tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"}, **overrides}
+def _tier_config(**overrides: object) -> dict[str, object]:
+    return {
+        "tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"},
+        "enable_context_window_escalation": True,
+        **overrides,
+    }
 
 
 class TestContextWindowEscalation:
@@ -13783,7 +13790,7 @@ class TestContextWindowEscalation:
         router = ComplexityRouter(
             model_name="test-router",
             litellm_router_instance=_windowed_router(_SMALL, ("mid-model", "openai/gpt-4o-mini", 200000), _BIG),
-            complexity_router_config={"tiers": {"SIMPLE": ["small-model", "mid-model"], "COMPLEX": "big-model"}},
+            complexity_router_config=_tier_config(tiers={"SIMPLE": ["small-model", "mid-model"], "COMPLEX": "big-model"}),
         )
 
         result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS)
@@ -13820,7 +13827,7 @@ class TestContextWindowEscalation:
                     },
                 ]
             ),
-            complexity_router_config={"tiers": {"SIMPLE": "mixed-pool", "COMPLEX": "big-model"}},
+            complexity_router_config=_tier_config(tiers={"SIMPLE": "mixed-pool", "COMPLEX": "big-model"}),
         )
 
         result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS)
@@ -13871,7 +13878,7 @@ class TestContextWindowEscalation:
         router = ComplexityRouter(
             model_name="test-router",
             litellm_router_instance=_windowed_router(*deployments),
-            complexity_router_config={"tiers": tiers},
+            complexity_router_config=_tier_config(tiers=tiers),
         )
 
         result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS)
@@ -13880,19 +13887,37 @@ class TestContextWindowEscalation:
         assert result.model == expected_model
 
     @pytest.mark.asyncio
-    async def test_the_disabled_gate_dispatches_on_complexity_alone(self):
-        """The escape hatch: enable_context_window_escalation false restores today's behavior."""
-        router = ComplexityRouter(
+    @pytest.mark.parametrize("enabled", (None, False, True), ids=("omitted", "disabled", "enabled"))
+    @pytest.mark.parametrize("serialized", (False, True), ids=("config", "http-json"))
+    async def test_context_window_escalation_requires_opt_in(self, enabled: bool | None, serialized: bool) -> None:
+        setting: Final = (
+            MappingProxyType({"enable_context_window_escalation": enabled})
+            if enabled is not None
+            else MappingProxyType({})
+        )
+        raw_config: Final = RequestComplexityRouterConfig.model_validate(
+            MappingProxyType(
+                {"tiers": MappingProxyType({"SIMPLE": "small-model", "COMPLEX": "big-model"}), **setting}
+            )
+        )
+        config: Final = (
+            RequestComplexityRouterConfig.model_validate_json(raw_config.model_dump_json())
+            if serialized
+            else raw_config
+        )
+        router: Final = ComplexityRouter(
             model_name="test-router",
             litellm_router_instance=_windowed_router(_SMALL, _BIG),
-            complexity_router_config=_tier_config(enable_context_window_escalation=False),
+            complexity_router_config=config.model_dump(exclude_unset=not serialized, exclude_none=True),
         )
 
-        result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS)
+        result: Final = await router.async_pre_routing_hook(
+            model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS
+        )
 
         assert result is not None
-        assert result.model == "small-model"
-        assert "context_escalated" not in result.routing_decision
+        assert result.model == ("big-model" if enabled else "small-model")
+        assert result.routing_decision.get("context_escalated", False) is (enabled is True)
 
     @pytest.mark.asyncio
     async def test_out_of_band_system_and_tools_count_against_the_window(self):
@@ -14001,7 +14026,7 @@ class TestContextWindowEscalation:
                     },
                 ]
             ),
-            complexity_router_config={"adaptive": True, "tiers": {"SIMPLE": ["small-model", "mid-model"]}},
+            complexity_router_config=_tier_config(adaptive=True, tiers={"SIMPLE": ["small-model", "mid-model"]}),
         )
 
         result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS)
@@ -14031,7 +14056,7 @@ class TestContextWindowEscalation:
                     },
                 ]
             ),
-            complexity_router_config={"tiers": {"SIMPLE": "cop-pool", "COMPLEX": "big-model"}},
+            complexity_router_config=_tier_config(tiers={"SIMPLE": "cop-pool", "COMPLEX": "big-model"}),
         )
         real_get_llm_provider = litellm.get_llm_provider
         copilot_resolutions: List = []
@@ -14064,7 +14089,7 @@ class TestContextWindowEscalation:
                     "model_name": "smart-router",
                     "litellm_params": {
                         "model": "auto_router/complexity_router",
-                        "complexity_router_config": {"tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"}},
+                        "complexity_router_config": _tier_config(),
                     },
                 },
                 {
@@ -14894,7 +14919,12 @@ class TestHealthFallbackDispatch:
     ) -> None:
         from litellm.types.router import RouterRateLimitError
 
-        router: Final = self._router(config={"tiers": {"SIMPLE": "primary", "MEDIUM": "peer", "COMPLEX": "large"}})
+        router: Final = self._router(
+            config={
+                "tiers": {"SIMPLE": "primary", "MEDIUM": "peer", "COMPLEX": "large"},
+                "enable_context_window_escalation": True,
+            }
+        )
         router.add_deployment(
             Deployment(
                 model_name="large",
@@ -14971,7 +15001,13 @@ class TestHealthFallbackDispatch:
     @pytest.mark.asyncio
     @pytest.mark.parametrize("default_fits", [True, False])
     async def test_modality_default_must_also_fit_context(self, default_fits: bool) -> None:
-        router: Final = self._router(config={"modality_routing": True, "tiers": {"SIMPLE": "primary"}})
+        router: Final = self._router(
+            config={
+                "modality_routing": True,
+                "tiers": {"SIMPLE": "primary"},
+                "enable_context_window_escalation": True,
+            }
+        )
         for deployment in router.model_list:
             deployment["model_info"]["supports_vision"] = deployment["model_name"] == "fallback"
             deployment["model_info"]["max_input_tokens"] = 10000 if default_fits else 10
diff --git a/ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx
index c0a65076d20..ad09efd8059 100644
--- a/ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx
@@ -7,7 +7,7 @@ const ContextWindowEscalationConfig: React.FC<{
   value: ComplexityRouterConfigValue;
   onChange: (value: ComplexityRouterConfigValue) => void;
 }> = ({ value, onChange }) => {
-  const enabled = value.enable_context_window_escalation ?? true;
+  const enabled = value.enable_context_window_escalation ?? false;
   // A number input renders Number("0.") as "0", so a decimal cannot be typed without a local draft.
   const [bufferDraft, setBufferDraft] = React.useState<string | null>(null);
   const commitBuffer = (raw: string) => {
@@ -32,7 +32,8 @@ const ContextWindowEscalationConfig: React.FC<{
       </div>
       <span className="block text-xs mb-3 text-muted-foreground">
         When a prompt provably cannot fit the decided tier&apos;s context windows, route it to the lowest tier whose
-        window holds it instead of letting the provider reject it. Off means requests dispatch on complexity alone.
+        window holds it instead of letting the provider reject it. Disabled by default. Off means requests dispatch on
+        complexity alone.
       </span>
       {enabled && (
         <div style={{ maxWidth: 320 }}>
diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx
index 48903d585ff..578b455d2b8 100644
--- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx
@@ -669,7 +669,7 @@ describe("AddAutoRouterTab", () => {
     });
   });
 
-  it("carries a context-window escalation opt-out through to the create payload", async () => {
+  it("starts context-window escalation disabled and carries an explicit opt-in to the create payload", async () => {
     const user = userEvent.setup();
     vi.mocked(getMissingTiersError).mockReturnValue(null);
 
@@ -679,14 +679,15 @@ describe("AddAutoRouterTab", () => {
     expandDetailedConfiguration();
     await user.click(screen.getByText("Advanced: Context Window Escalation"));
     const toggle = await screen.findByRole("switch", { name: "Escalate oversized prompts to a tier that fits" });
-    expect(toggle).toBeChecked();
+    expect(toggle).not.toBeChecked();
+    expect(screen.queryByLabelText("Window fit buffer")).not.toBeInTheDocument();
     await user.click(toggle);
 
     await user.click(screen.getByRole("button", { name: /add auto router/i }));
 
     await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
     expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({
-      enable_context_window_escalation: false,
+      enable_context_window_escalation: true,
     });
   });
 
@@ -699,6 +700,7 @@ describe("AddAutoRouterTab", () => {
     await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-buffer-router");
     expandDetailedConfiguration();
     await user.click(screen.getByText("Advanced: Context Window Escalation"));
+    await user.click(screen.getByRole("switch", { name: "Escalate oversized prompts to a tier that fits" }));
     const buffer = await screen.findByLabelText("Window fit buffer");
     fireEvent.change(buffer, { target: { value: "1.5" } });
     fireEvent.blur(buffer, { target: { value: "1.5" } });
@@ -708,7 +710,7 @@ describe("AddAutoRouterTab", () => {
     await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
     const config = vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config;
     expect(config).toMatchObject({ context_window_escalation_buffer: 1 });
-    expect(config).not.toHaveProperty("enable_context_window_escalation");
+    expect(config).toHaveProperty("enable_context_window_escalation", true);
   });
 
   it("clearing the buffer removes it from the payload so the router tracks the backend default", async () => {
@@ -720,6 +722,7 @@ describe("AddAutoRouterTab", () => {
     await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-clear-router");
     expandDetailedConfiguration();
     await user.click(screen.getByText("Advanced: Context Window Escalation"));
+    await user.click(screen.getByRole("switch", { name: "Escalate oversized prompts to a tier that fits" }));
     const buffer = await screen.findByLabelText("Window fit buffer");
     fireEvent.change(buffer, { target: { value: "0.8" } });
     fireEvent.blur(buffer, { target: { value: "0.8" } });
diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
index 6e6e7a3c6cd..0d70b18cd94 100644
--- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
+++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
@@ -49,7 +49,7 @@ const baseParams: BuildComplexityRouterConfigParams = {
 
 describe("buildComplexityRouterConfig", () => {
   it.each(["capability", "llm_v2", "heuristic"] as const)(
-    "disables the removed overrides only for forecast creates: %s",
+    "preserves explicit context-window opt-in beside forecast restrictions: %s",
     (classifierType) => {
       const forecast = classifierType !== "heuristic";
       const params = {
@@ -61,14 +61,10 @@ describe("buildComplexityRouterConfig", () => {
       };
       const config = buildComplexityRouterConfig(params);
       expect(config.adaptive).toBe(!forecast);
-      expect(config.enable_context_window_escalation).toBe(!forecast);
+      expect(config.enable_context_window_escalation).toBe(true);
+      expect(config.context_window_escalation_buffer).toBe(0.9);
       expect(config.escalation_keywords).toEqual(forecast ? [] : ["LITELLM ESCALATE"]);
-      for (const key of [
-        "adaptive_weights",
-        "adaptive_eligible",
-        "tier_distance_penalty",
-        "context_window_escalation_buffer",
-      ]) {
+      for (const key of ["adaptive_weights", "adaptive_eligible", "tier_distance_penalty"]) {
         expect(Object.hasOwn(config, key)).toBe(!forecast);
       }
       if (forecast) {
@@ -107,13 +103,14 @@ describe("buildComplexityRouterConfig", () => {
     expect(config).toEqual(expected);
   });
 
-  it("carries an explicit context-window escalation opt-out and buffer, false included", () => {
+  it.each([undefined, false, true])("preserves the context-window escalation setting: %s", (enabled) => {
     const config = buildComplexityRouterConfig({
       ...baseParams,
-      enableContextWindowEscalation: false,
+      enableContextWindowEscalation: enabled,
       contextWindowEscalationBuffer: 0.9,
     });
-    expect(config.enable_context_window_escalation).toBe(false);
+    expect(config.enable_context_window_escalation).toBe(enabled);
+    expect(Object.hasOwn(config, "enable_context_window_escalation")).toBe(enabled !== undefined);
     expect(config.context_window_escalation_buffer).toBe(0.9);
   });
 
diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
index 8a377c17ad7..d1cea6d48a9 100644
--- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
+++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
@@ -629,6 +629,7 @@ export const buildComplexityRouterConfig = ({
   // the form never rewrote. The UI gates the same controls on this, not on the raw value.
   const effectiveType: ClassifierType = customTierSet ? "llm" : classifierType;
   const forecast = isForecastClassifier(effectiveType);
+  const preserveContextWindowBuffer = !forecast || enableContextWindowEscalation === true;
 
   const supportsOpeningPrompt = !customTierSet && !forecast && usesLlmClassifier(effectiveType);
   const payload: ComplexityRouterConfigPayload = {
@@ -682,11 +683,10 @@ export const buildComplexityRouterConfig = ({
         adaptive_eligible: adaptiveEligible,
       }),
     ...(returnRawModelName && { return_raw_model_name: true }),
-    // Omission enables the backend default, so hidden forecast controls need an explicit opt-out.
     ...((forecast || enableContextWindowEscalation !== undefined) && {
-      enable_context_window_escalation: forecast ? false : enableContextWindowEscalation,
+      enable_context_window_escalation: enableContextWindowEscalation ?? false,
     }),
-    ...(!forecast &&
+    ...(preserveContextWindowBuffer &&
       contextWindowEscalationBuffer !== undefined && {
         context_window_escalation_buffer: contextWindowEscalationBuffer,
       }),
diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts
index 4ae6efbb12d..31f2b8ef68a 100644
--- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts
+++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts
@@ -64,14 +64,10 @@ describe("buildUpdatedComplexityRouterConfig keyword matching", () => {
         const saved = buildUpdatedComplexityRouterConfig(stored, value, undefined, keywordState);
         const forecast = classifier_type !== "heuristic";
         expect(saved.adaptive).toBe(!forecast);
-        expect(saved.enable_context_window_escalation).toBe(!forecast);
+        expect(saved.enable_context_window_escalation).toBe(true);
+        expect(saved.context_window_escalation_buffer).toBe(0.9);
         expect(saved.escalation_keywords).toEqual(forecast ? [] : stored.escalation_keywords);
-        for (const key of [
-          "adaptive_weights",
-          "adaptive_eligible",
-          "tier_distance_penalty",
-          "context_window_escalation_buffer",
-        ]) {
+        for (const key of ["adaptive_weights", "adaptive_eligible", "tier_distance_penalty"]) {
           expect(Object.hasOwn(saved, key)).toBe(!forecast);
         }
         expect(saved.keyword_tier_rules).toEqual(STORED.keyword_tier_rules);
@@ -83,6 +79,19 @@ describe("buildUpdatedComplexityRouterConfig keyword matching", () => {
     },
   );
 
+  it.each([undefined, false, true])("preserves stored context-window escalation on save: %s", (enabled) => {
+    const stored = {
+      ...STORED,
+      ...(enabled !== undefined && { enable_context_window_escalation: enabled }),
+    };
+    const value = hydrateComplexityRouterConfig(stored, undefined);
+    const saved = buildUpdatedComplexityRouterConfig(stored, value, undefined, hydratedState);
+    const serialized: typeof saved = JSON.parse(JSON.stringify(saved));
+    expect(value.enable_context_window_escalation).toBe(enabled);
+    expect(serialized.enable_context_window_escalation).toBe(enabled);
+    expect(Object.hasOwn(serialized, "enable_context_window_escalation")).toBe(enabled !== undefined);
+  });
+
   it("round-trips an untouched edit without changing any keyword-matching value", () => {
     // Opening the modal hydrates state from STORED; saving with nothing changed must be a
     // no-op. These keys are now MANAGED, so a hydration bug silently wipes them.
diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts
index fed11454c23..cda9ba104e5 100644
--- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts
+++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts
@@ -708,7 +708,7 @@ describe("autorouter_presets", () => {
       expect(prefill.escalationKeywords).toEqual([]);
     });
 
-    it("carries a preset's context-window escalation opt-out and buffer through the prefill", () => {
+    it.each([undefined, false, true])("preserves a preset's context-window escalation setting: %s", (enabled) => {
       const prefill = buildPresetPrefill(
         {
           tiers: { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] },
@@ -716,12 +716,12 @@ describe("autorouter_presets", () => {
           classification_mode: "every_request",
           session_affinity: false,
           deployment_affinity: true,
-          enable_context_window_escalation: false,
+          enable_context_window_escalation: enabled,
           context_window_escalation_buffer: 0.9,
         },
         groupsOnly(["gpt-5-nano"]),
       );
-      expect(prefill.complexityRouterConfig.enable_context_window_escalation).toBe(false);
+      expect(prefill.complexityRouterConfig.enable_context_window_escalation).toBe(enabled);
       expect(prefill.complexityRouterConfig.context_window_escalation_buffer).toBe(0.9);
     });
 
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index d43adfe1ae4..a18b02646ee 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -36502,8 +36502,8 @@ export interface components {
             embedding_model?: string | null;
             /**
              * Enable Context Window Escalation
-             * @description Escalate a request off a tier whose models provably cannot hold its prompt, before dispatch. The classifier scores complexity and never prompt size, so a long agentic session whose newest ask is trivial lands on a small-window tier and the provider rejects it with a context-window 400 that nothing retries. When every model of the decided tier has a declared window smaller than the estimated prompt, the request moves to the lowest configured tier with a model whose declared window fits; when only some of the tier's models fit, the pick is restricted to those and the tier keeps the request. Models with no resolvable window are never escalated away from and never escalated onto. Set false to dispatch on complexity alone, as before.
-             * @default true
+             * @description Escalate a request off a tier whose models provably cannot hold its prompt, before dispatch. The classifier scores complexity and never prompt size, so a long agentic session whose newest ask is trivial lands on a small-window tier and the provider rejects it with a context-window 400 that nothing retries. When every model of the decided tier has a declared window smaller than the estimated prompt, the request moves to the lowest configured tier with a model whose declared window fits; when only some of the tier's models fit, the pick is restricted to those and the tier keeps the request. Models with no resolvable window are never escalated away from and never escalated onto. Disabled by default: omit or set false to dispatch on complexity alone; set true to enable context-window escalation.
+             * @default false
              */
             enable_context_window_escalation: boolean;
             /**

From 5f722bc19559a82df160b748f26746dac7b55488 Mon Sep 17 00:00:00 2001
From: Tin <tin@berri.ai>
Date: Sat, 19 Sep 2026 10:26:42 -0700
Subject: [PATCH 033/160] chore(router): remove in-repo escalation docs

---
 litellm/router_strategy/complexity_router/README.md | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md
index 2c2aea333f9..6505746bca1 100644
--- a/litellm/router_strategy/complexity_router/README.md
+++ b/litellm/router_strategy/complexity_router/README.md
@@ -68,19 +68,6 @@ still resolve to a deployment in `model_list`; this configuration does not creat
             - abc
 ```
 
-### Context-window escalation
-
-Context-window escalation is opt-in. Omit `enable_context_window_escalation` or set it to
-`false` to keep the complexity-selected model without context-window replacement or filtering
-
-Set `enable_context_window_escalation: true` inside `complexity_router_config` to restrict the
-selected tier to models whose declared windows fit the prompt, or move to the lowest configured
-tier with a fitting model when none in the selected tier fit. Unknown windows do not justify
-moving a request. `context_window_escalation_buffer` defaults to `0.95`
-
-Existing saved configurations with explicit `true` keep escalation enabled. Configurations that
-omit the setting now default to disabled; set it to `true` to retain their previous behavior
-
 ### Capability forecasting
 
 Set `classifier_type: capability` to use

From 0068df5a8beac2b137fc9412586048c2f42a0af3 Mon Sep 17 00:00:00 2001
From: Tin Chi Lo <tin@berri.ai>
Date: Sat, 19 Sep 2026 14:46:58 -0700
Subject: [PATCH 034/160] feat(ui): add internal-user savings and auto-router
 usage

---
 .../migration.sql                             |  42 +++
 .../litellm_proxy_extras/schema.prisma        |  41 ++
 litellm/proxy/db/autorouter_session_rollup.py | 170 ++++++---
 litellm/proxy/db/baseline_accounting.py       |  40 +-
 .../db_transaction_queue/spend_log_cleanup.py |  24 +-
 .../auto_router_endpoints.py                  |  11 +-
 litellm/proxy/schema.prisma                   |  41 ++
 schema.prisma                                 |  41 ++
 .../spend/test_autorouter_session_rollup.py   | 171 ++++++++-
 .../spend/test_baseline_accounting.py         |  67 +++-
 .../db/test_autorouter_session_rollup.py      | 115 +++++-
 .../test_auto_router_endpoints.py             |  42 ++-
 .../proxy/test_spend_log_cleanup.py           |  13 +-
 .../AutoRouterBenchmarksTab.test.tsx          |   4 +-
 .../_components/AutoRouterBenchmarksTab.tsx   |  19 +-
 .../_components/useAutoRouterBenchmarks.ts    |   9 +-
 .../useDailyActivityRange.test.tsx            |   2 +
 .../_components/useDailyActivityRange.ts      |   6 +-
 .../user_info_view.integration.test.tsx       | 357 +++++++++++++++++-
 .../_components/view_users/user_info_view.tsx |  60 ++-
 .../components/shared/ScopedSavingsTab.tsx    | 133 +++++++
 .../components/templates/KeySavingsTab.tsx    | 133 +------
 ui/litellm-dashboard/src/lib/http/schema.d.ts |   9 +-
 23 files changed, 1321 insertions(+), 229 deletions(-)
 create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260919000000_add_autorouter_user_session_rollup/migration.sql
 create mode 100644 ui/litellm-dashboard/src/components/shared/ScopedSavingsTab.tsx

diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260919000000_add_autorouter_user_session_rollup/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260919000000_add_autorouter_user_session_rollup/migration.sql
new file mode 100644
index 00000000000..2b864131ab2
--- /dev/null
+++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260919000000_add_autorouter_user_session_rollup/migration.sql
@@ -0,0 +1,42 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_AutoRouterUserSession" (
+    "user_id" TEXT NOT NULL,
+    "api_key" TEXT NOT NULL,
+    "session_id" TEXT NOT NULL,
+    "router_name" TEXT NOT NULL,
+    "router_type" TEXT NOT NULL,
+    "first_turn_at" TIMESTAMP(3) NOT NULL,
+    "last_turn_at" TIMESTAMP(3) NOT NULL,
+    "last_model" TEXT NOT NULL,
+    "models" JSONB NOT NULL DEFAULT '{}',
+    "turns" INTEGER NOT NULL DEFAULT 0,
+    "unordered_turns" INTEGER NOT NULL DEFAULT 0,
+    "covered_turns" INTEGER NOT NULL DEFAULT 0,
+    "cache_hits" INTEGER NOT NULL DEFAULT 0,
+    "same_model_turns" INTEGER NOT NULL DEFAULT 0,
+    "same_model_hits" INTEGER NOT NULL DEFAULT 0,
+    "first_visit_turns" INTEGER NOT NULL DEFAULT 0,
+    "first_visit_hits" INTEGER NOT NULL DEFAULT 0,
+    "return_turns" INTEGER NOT NULL DEFAULT 0,
+    "return_hits" INTEGER NOT NULL DEFAULT 0,
+    "return_expired_misses" INTEGER NOT NULL DEFAULT 0,
+    "return_within_ttl_misses" INTEGER NOT NULL DEFAULT 0,
+    "ttl_5m_turns" INTEGER NOT NULL DEFAULT 0,
+    "ttl_1h_turns" INTEGER NOT NULL DEFAULT 0,
+    "total_tokens" BIGINT NOT NULL DEFAULT 0,
+    "spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "saved_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "savings_estimated_turns" INTEGER NOT NULL DEFAULT 0,
+    "savings_estimated_actual_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "savings_estimated_saved_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "savings_estimated_baseline_models" JSONB NOT NULL DEFAULT '{}',
+    "classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "classifier_cost_recorded_turns" INTEGER NOT NULL DEFAULT 0,
+    "tier_turns" JSONB NOT NULL DEFAULT '{}',
+    "baseline_models" JSONB NOT NULL DEFAULT '{}',
+
+    CONSTRAINT "LiteLLM_AutoRouterUserSession_pkey" PRIMARY KEY ("user_id", "api_key", "session_id", "router_name")
+);
+
+CREATE INDEX IF NOT EXISTS "idx_autorouter_user_session_last_turn" ON "LiteLLM_AutoRouterUserSession"("last_turn_at");
+
+CREATE INDEX IF NOT EXISTS "idx_autorouter_user_session_user_last_turn" ON "LiteLLM_AutoRouterUserSession"("user_id", "last_turn_at");
diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
index d2032cec0d0..f4015ed9277 100644
--- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
@@ -1620,6 +1620,47 @@ model LiteLLM_AutoRouterSession {
   @@index([last_turn_at], map: "idx_autorouter_session_last_turn")
 }
 
+model LiteLLM_AutoRouterUserSession {
+  user_id               String
+  api_key               String
+  session_id            String
+  router_name           String
+  router_type           String
+  first_turn_at         DateTime
+  last_turn_at          DateTime
+  last_model            String
+  models                Json     @default("{}")
+  turns                 Int      @default(0)
+  unordered_turns       Int      @default(0)
+  covered_turns         Int      @default(0)
+  cache_hits            Int      @default(0)
+  same_model_turns      Int      @default(0)
+  same_model_hits       Int      @default(0)
+  first_visit_turns     Int      @default(0)
+  first_visit_hits      Int      @default(0)
+  return_turns          Int      @default(0)
+  return_hits           Int      @default(0)
+  return_expired_misses Int      @default(0)
+  return_within_ttl_misses  Int      @default(0)
+  ttl_5m_turns          Int      @default(0)
+  ttl_1h_turns          Int      @default(0)
+  total_tokens          BigInt   @default(0)
+  spend                 Float    @default(0)
+  saved_spend           Float    @default(0)
+  savings_estimated_turns Int    @default(0)
+  savings_estimated_actual_spend Float @default(0)
+  savings_estimated_saved_spend Float @default(0)
+  savings_estimated_baseline_models Json @default("{}")
+  classifier_cost       Float    @default(0)
+  classifier_cost_recorded_turns Int @default(0)
+  tier_turns            Json     @default("{}")
+  baseline_models       Json     @default("{}")
+
+  @@id([user_id, api_key, session_id, router_name])
+  @@index([last_turn_at], map: "idx_autorouter_user_session_last_turn")
+  @@index([user_id, last_turn_at], map: "idx_autorouter_user_session_user_last_turn")
+}
+
 // Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in
 // either direction. forward duplicates the requests the keys did not route through the
 // router through it, answering whether they should adopt it; reverse duplicates the
diff --git a/litellm/proxy/db/autorouter_session_rollup.py b/litellm/proxy/db/autorouter_session_rollup.py
index 0d812ee812a..dd08cfd1bef 100644
--- a/litellm/proxy/db/autorouter_session_rollup.py
+++ b/litellm/proxy/db/autorouter_session_rollup.py
@@ -4,7 +4,7 @@ Per-session auto-router benchmarks rollup.
 At request time the spend writer builds one AutoRouterTurnTransaction per successful
 auto-routed request (a request whose metadata carries a routing_decision) and queues it
 on the prisma client. The spend-log flush job drains the queue into
-LiteLLM_AutoRouterSession with one conditional upsert per turn: the statement classifies
+key and user session rollups with one atomic statement per turn: each upsert classifies
 the turn (same model, first visit, return to a model the session already used, out of
 order) against the row's own columns, so nothing is read before the write and concurrent
 pods compose. The benchmarks endpoint aggregates these rows and never touches
@@ -35,10 +35,27 @@ if TYPE_CHECKING:
 CACHE_TTL_5M_SECONDS: Final = 300
 CACHE_TTL_1H_SECONDS: Final = 3600
 
-AUTOROUTER_BENCHMARKS_SQL: Final = """
+_SESSION_COLUMNS: Final = """
+    api_key, session_id, router_name, router_type, first_turn_at, last_turn_at,
+    last_model, models, turns, unordered_turns, covered_turns, cache_hits,
+    same_model_turns, same_model_hits, first_visit_turns, first_visit_hits,
+    return_turns, return_hits, return_expired_misses, return_within_ttl_misses,
+    ttl_5m_turns, ttl_1h_turns, total_tokens, spend, saved_spend, classifier_cost, classifier_cost_recorded_turns, tier_turns,
+    baseline_models, savings_estimated_turns, savings_estimated_actual_spend, savings_estimated_saved_spend,
+    savings_estimated_baseline_models
+"""
+
+AUTOROUTER_BENCHMARKS_SQL: Final = f"""
 WITH windowed AS (
-    SELECT * FROM "LiteLLM_AutoRouterSession"
-    WHERE last_turn_at >= $1::timestamp
+    SELECT {_SESSION_COLUMNS} FROM "LiteLLM_AutoRouterSession"
+    WHERE $4::text IS NULL
+      AND last_turn_at >= $1::timestamp
+      AND first_turn_at < $2::timestamp
+      AND ($3::text IS NULL OR api_key = $3::text)
+    UNION ALL
+    SELECT {_SESSION_COLUMNS} FROM "LiteLLM_AutoRouterUserSession"
+    WHERE (($4::text IS NOT NULL AND user_id = $4::text) OR ($4::text IS NULL AND api_key = ''))
+      AND last_turn_at >= $1::timestamp
       AND first_turn_at < $2::timestamp
       AND ($3::text IS NULL OR api_key = $3::text)
 ),
@@ -53,7 +70,7 @@ tier_maps AS (
 )
 SELECT
     agg.*,
-    COALESCE(tier_maps.tier_turns, '{}'::jsonb) AS tier_turns
+    COALESCE(tier_maps.tier_turns, '{{}}'::jsonb) AS tier_turns
 FROM (
 SELECT
     router_name,
@@ -111,6 +128,7 @@ class AutoRouterTurnTransaction:
     savings_estimated_turns: int = 0
     savings_estimated_actual_spend: float = 0.0
     savings_estimated_saved_spend: float = 0.0
+    user_id: str = ""
 
 
 class TurnCacheFacts(NamedTuple):
@@ -214,10 +232,11 @@ def build_autorouter_turn_transaction(
     if not isinstance(routing_decision, Mapping) or not routing_decision:
         return None
     router_name: Final = routing_decision.get("router_model_name") or payload.get("model_group")
-    api_key: Final = payload.get("api_key")
+    api_key: Final = payload.get("api_key") or ""
+    user_id: Final = payload.get("user") or ""
     session_id: Final = payload.get("session_id")
     model: Final = payload.get("model")
-    if not (isinstance(router_name, str) and router_name and api_key and session_id and model):
+    if not (isinstance(router_name, str) and router_name and (api_key or user_id) and session_id and model):
         return None
     turn_at: Final = _turn_time_utc(str(payload.get("startTime") or ""))
     if turn_at is None:
@@ -236,6 +255,7 @@ def build_autorouter_turn_transaction(
     estimated_savings: Final = recorded_estimated_autorouter_savings(metadata)
     return AutoRouterTurnTransaction(
         api_key=api_key,
+        user_id=user_id,
         session_id=bounded_session_id(session_id),
         router_name=router_name,
         router_type=str(routing_decision.get("router_type") or "unknown"),
@@ -293,18 +313,18 @@ _RETURN_MISS: Final = (
 _IDLE_SECONDS: Final = f"EXTRACT(EPOCH FROM {_TURN_AT}::timestamp) - (t.models -> {_MODEL} ->> 'at')::float8"
 _CACHE_TOUCHED: Final = f"{_TOUCHED}::int = 1"
 
-UPSERT_AUTOROUTER_SESSION_SQL: Final = f"""
-INSERT INTO "LiteLLM_AutoRouterSession" AS t (
-    api_key, session_id, router_name, router_type, first_turn_at, last_turn_at,
-    last_model, models, turns, unordered_turns, covered_turns, cache_hits,
-    same_model_turns, same_model_hits, first_visit_turns, first_visit_hits,
-    return_turns, return_hits, return_expired_misses, return_within_ttl_misses,
-    ttl_5m_turns, ttl_1h_turns, total_tokens, spend, saved_spend, classifier_cost, classifier_cost_recorded_turns, tier_turns,
-    baseline_models, savings_estimated_turns, savings_estimated_actual_spend, savings_estimated_saved_spend,
-    savings_estimated_baseline_models
+
+def _session_upsert_sql(*, user_scoped: bool) -> str:
+    table_name: Final = "LiteLLM_AutoRouterUserSession" if user_scoped else "LiteLLM_AutoRouterSession"
+    user_column: Final = "user_id, " if user_scoped else ""
+    user_value: Final = f"{_p('user_id')}::text, " if user_scoped else ""
+    required_identity: Final = _p("user_id" if user_scoped else "api_key")
+    return f"""
+INSERT INTO "{table_name}" AS t (
+    {user_column}{_SESSION_COLUMNS}
 )
-VALUES (
-    {_p("api_key")}, {_p("session_id")}, {_p("router_name")}, {_p("router_type")}, {_TURN_AT}::timestamp, {_TURN_AT}::timestamp,
+SELECT
+    {user_value}{_p("api_key")}, {_p("session_id")}, {_p("router_name")}, {_p("router_type")}, {_TURN_AT}::timestamp, {_TURN_AT}::timestamp,
     {_MODEL}, jsonb_build_object({_MODEL}, jsonb_build_object('at', EXTRACT(EPOCH FROM {_TURN_AT}::timestamp), 'ttl', {_CACHE_TTL}::int)),
     1, 0, {_COVERED}::int, {_CACHE_HIT}::int,
     0, 0, 1, {_CACHE_HIT}::int,
@@ -315,8 +335,8 @@ VALUES (
     {_p("classifier_cost")}::float8, 1, {_TIER_DELTA}, {_BASELINE_DELTA},
     {_p("savings_estimated_turns")}::int, {_p("savings_estimated_actual_spend")}::float8,
     {_p("savings_estimated_saved_spend")}::float8, {_ESTIMATED_BASELINE_DELTA}
-)
-ON CONFLICT (api_key, session_id, router_name) DO UPDATE SET
+WHERE {required_identity}::text <> ''
+ON CONFLICT ({user_column}api_key, session_id, router_name) DO UPDATE SET
     turns = t.turns + 1,
     total_tokens = t.total_tokens + EXCLUDED.total_tokens,
     spend = t.spend + EXCLUDED.spend,
@@ -365,6 +385,17 @@ ON CONFLICT (api_key, session_id, router_name) DO UPDATE SET
 """
 
 
+UPSERT_AUTOROUTER_SESSION_SQL: Final = f"""
+WITH key_rollup AS (
+    {_session_upsert_sql(user_scoped=False)}
+    RETURNING 1
+)
+{_session_upsert_sql(user_scoped=True)}
+"""
+
+UPSERT_AUTOROUTER_USER_SESSION_SQL: Final = _session_upsert_sql(user_scoped=True)
+
+
 def _as_sql_param(value: str | float | bool | datetime | None) -> str | float | None:
     if isinstance(value, bool):
         return int(value)
@@ -377,18 +408,23 @@ def _upsert_params(transaction: AutoRouterTurnTransaction) -> tuple[str | float
     return tuple(_as_sql_param(getattr(transaction, name)) for name in _UPSERT_PARAM_FIELDS)
 
 
-async def write_autorouter_turn(db: SupportsExecuteRaw, transaction: AutoRouterTurnTransaction) -> None:
-    await db.execute_raw(UPSERT_AUTOROUTER_SESSION_SQL, *_upsert_params(transaction))
+async def write_autorouter_turn(
+    db: SupportsExecuteRaw,
+    transaction: AutoRouterTurnTransaction,
+    statement: str = UPSERT_AUTOROUTER_SESSION_SQL,
+) -> None:
+    await db.execute_raw(statement, *_upsert_params(transaction))
 
 
 async def _upsert_turn_with_retry(
     prisma_client: PrismaClient,
     transaction: AutoRouterTurnTransaction,
     n_retry_times: int,
+    statement: str,
 ) -> None:
     for attempt in range(n_retry_times + 1):
         try:
-            await write_autorouter_turn(prisma_client.db, transaction)
+            await write_autorouter_turn(prisma_client.db, transaction, statement)
         except DB_RETRY_SAFE_ERROR_TYPES:
             if attempt >= n_retry_times:
                 raise
@@ -397,6 +433,58 @@ async def _upsert_turn_with_retry(
             return
 
 
+def _session_partition(transaction: AutoRouterTurnTransaction) -> tuple[str, str, str, str]:
+    identity: Final = ("key", transaction.api_key) if transaction.api_key else ("user", transaction.user_id)
+    return (*identity, transaction.session_id, transaction.router_name)
+
+
+async def _drain_session_partition(
+    prisma_client: PrismaClient,
+    transactions: tuple[AutoRouterTurnTransaction, ...],
+    n_retry_times: int,
+    statement: str,
+) -> tuple[AutoRouterTurnTransaction, ...]:
+    for position, transaction in enumerate(transactions):
+        try:
+            await _upsert_turn_with_retry(prisma_client, transaction, n_retry_times, statement)
+        except Exception as flush_err:  # noqa: BLE001  # stop dependent turns without retrying an ambiguous write
+            verbose_proxy_logger.error(
+                "Spend tracking - auto-router session rollup flush failed for router %s; "
+                "%s of %s turn writes stopped in this partition: %s",
+                transaction.router_name,
+                len(transactions) - position,
+                len(transactions),
+                flush_err,
+            )
+            return transactions[position:]
+    return ()
+
+
+async def _flush_session_partition(
+    prisma_client: PrismaClient,
+    transactions: tuple[AutoRouterTurnTransaction, ...],
+    n_retry_times: int,
+) -> None:
+    failed_suffix: Final = await _drain_session_partition(
+        prisma_client, transactions, n_retry_times, UPSERT_AUTOROUTER_SESSION_SQL
+    )
+    if not failed_suffix or not failed_suffix[0].api_key:
+        return
+    failed_user: Final = failed_suffix[0].user_id
+    other_users: Final = sorted(
+        (
+            transaction
+            for transaction in failed_suffix[1:]
+            if transaction.user_id and transaction.user_id != failed_user
+        ),
+        key=lambda transaction: transaction.user_id,
+    )
+    for _, user_turns in groupby(other_users, key=lambda transaction: transaction.user_id):
+        await _drain_session_partition(
+            prisma_client, tuple(user_turns), n_retry_times, UPSERT_AUTOROUTER_USER_SESSION_SQL
+        )
+
+
 async def flush_autorouter_turn_transactions(
     prisma_client: PrismaClient,
     transactions: Sequence[AutoRouterTurnTransaction],
@@ -407,38 +495,20 @@ async def flush_autorouter_turn_transactions(
     Statements run sequentially in per-session event order: a turn's classification
     depends on the turns before it, and Postgres rejects one multi-row INSERT touching
     the same key twice. Only ConnectError is retried, per statement, because it proves
-    that statement never reached the database. Any other failure drops the remaining
-    turns of THAT session only, with an error log, and the flush continues with the
-    next session: sessions are independent state machines, so one poisoned statement
-    must not discard unrelated sessions, and a repeated increment is worse than an
-    undercount. Callers must not add their own retry around this function.
+    that statement never reached the database. A failed write stops its key and user
+    histories for this batch. Other users sharing that key can still advance their
+    independent user histories, with the key projection disabled and the real key
+    identity preserved. The failed turn is never replayed. Callers must not add their
+    own retry around this function.
     """
     if not transactions:
         return
     ordered: Final = sorted(
         transactions,
-        key=lambda transaction: (
-            transaction.api_key,
-            transaction.session_id,
-            transaction.router_name,
-            transaction.turn_at,
-        ),
+        key=lambda transaction: (*_session_partition(transaction), transaction.turn_at),
     )
-    for session_key, session_group in groupby(
+    for _, session_group in groupby(
         ordered,
-        key=lambda transaction: (transaction.api_key, transaction.session_id, transaction.router_name),
+        key=_session_partition,
     ):
-        session_turns = tuple(session_group)
-        for position, transaction in enumerate(session_turns):
-            try:
-                await _upsert_turn_with_retry(prisma_client, transaction, n_retry_times)
-            except Exception as flush_err:  # noqa: BLE001  # a statement failure drops only its session's remainder by design
-                verbose_proxy_logger.error(
-                    "Spend tracking - auto-router session rollup flush failed for router %s; "
-                    "%s of %s turn transactions dropped for one session: %s",
-                    session_key[2],
-                    len(session_turns) - position,
-                    len(session_turns),
-                    flush_err,
-                )
-                break
+        await _flush_session_partition(prisma_client, tuple(session_group), n_retry_times)
diff --git a/litellm/proxy/db/baseline_accounting.py b/litellm/proxy/db/baseline_accounting.py
index 8622cb9e481..4219102d9aa 100644
--- a/litellm/proxy/db/baseline_accounting.py
+++ b/litellm/proxy/db/baseline_accounting.py
@@ -171,6 +171,7 @@ class _Change(BaseModel):
     request_id: str
     publication: BaselinePublication
     api_key: str
+    user_id: str = ""
     session_id: str
     router_name: str
     baseline_model: str
@@ -256,42 +257,54 @@ SET publication = x.publication::text
 FROM jsonb_to_recordset($1::jsonb) AS x(request_id text, publication jsonb)
 WHERE observations.request_id = x.request_id
 """
-_UPDATE_SESSIONS: Final = """
+
+
+def _session_correction_sql(*, user_scoped: bool) -> str:
+    table_name: Final = "LiteLLM_AutoRouterUserSession" if user_scoped else "LiteLLM_AutoRouterSession"
+    identity_columns: Final = ("user_id, " if user_scoped else "") + "api_key, session_id, router_name"
+    user_filter: Final = "WHERE user_id <> ''" if user_scoped else ""
+    user_match: Final = "session.user_id = totals.user_id AND " if user_scoped else ""
+    return f"""
 WITH changes AS (
     SELECT * FROM jsonb_to_recordset($1::jsonb) AS x(
-        api_key text, session_id text, router_name text, baseline_model text,
+        user_id text, api_key text, session_id text, router_name text, baseline_model text,
         covered_delta int, actual_delta float8, savings_delta float8
     )
+    {user_filter}
 ), totals AS (
-    SELECT api_key, session_id, router_name, SUM(covered_delta)::int AS covered_delta,
+    SELECT {identity_columns}, SUM(covered_delta)::int AS covered_delta,
         SUM(actual_delta) AS actual_delta, SUM(savings_delta) AS savings_delta
-    FROM changes GROUP BY api_key, session_id, router_name
+    FROM changes GROUP BY {identity_columns}
 ), models AS (
-    SELECT api_key, session_id, router_name, jsonb_object_agg(baseline_model, delta) AS deltas
+    SELECT {identity_columns}, jsonb_object_agg(baseline_model, delta) AS deltas
     FROM (
-        SELECT api_key, session_id, router_name, baseline_model, SUM(covered_delta)::int AS delta
-        FROM changes GROUP BY api_key, session_id, router_name, baseline_model
-    ) grouped GROUP BY api_key, session_id, router_name
+        SELECT {identity_columns}, baseline_model, SUM(covered_delta)::int AS delta
+        FROM changes GROUP BY {identity_columns}, baseline_model
+    ) grouped GROUP BY {identity_columns}
 )
-UPDATE "LiteLLM_AutoRouterSession" AS session
+UPDATE "{table_name}" AS session
 SET saved_spend = session.saved_spend + totals.savings_delta,
     savings_estimated_turns = session.savings_estimated_turns + totals.covered_delta,
     savings_estimated_actual_spend = session.savings_estimated_actual_spend + totals.actual_delta,
     savings_estimated_saved_spend = session.savings_estimated_saved_spend + totals.savings_delta,
     savings_estimated_baseline_models = (
-        SELECT COALESCE(jsonb_object_agg(key, value), '{}'::jsonb) FROM (
+        SELECT COALESCE(jsonb_object_agg(key, value), '{{}}'::jsonb) FROM (
             SELECT key, SUM(value::int)::int AS value FROM (
                 SELECT * FROM jsonb_each_text(session.savings_estimated_baseline_models)
                 UNION ALL SELECT * FROM jsonb_each_text(models.deltas)
             ) combined GROUP BY key HAVING SUM(value::int) > 0
         ) counts
     )
-FROM totals JOIN models USING (api_key, session_id, router_name)
-WHERE session.api_key = totals.api_key AND session.session_id = totals.session_id
+FROM totals JOIN models USING ({identity_columns})
+WHERE {user_match}session.api_key = totals.api_key AND session.session_id = totals.session_id
     AND session.router_name = totals.router_name
 """
 
 
+_UPDATE_SESSIONS: Final = _session_correction_sql(user_scoped=False)
+_UPDATE_USER_SESSIONS: Final = _session_correction_sql(user_scoped=True)
+
+
 def _primary_transaction(client: PrismaClient) -> _TransactionManager:
     primary: Final = cast(_TransactionalDatabase, writer_wrapper(client.db))
     return primary.tx(timeout=_TRANSACTION_TIMEOUT)
@@ -308,6 +321,7 @@ def _change(record: BaselineAccountingRecord, old: BaselinePublication | None, n
         request_id=record.observation.request_id,
         publication=new,
         api_key=record.api_key,
+        user_id=record.turn.user_id if record.turn is not None else "",
         session_id=record.session_id,
         router_name=record.router_name,
         baseline_model=record.baseline_model,
@@ -357,6 +371,8 @@ async def _publish(db: SupportsRawQueries, changes: Sequence[_Change]) -> None:
     serialized: Final = json.dumps(tuple(change.model_dump(mode="json") for change in changes), separators=(",", ":"))
     await db.execute_raw(_UPDATE_LOGS, serialized)
     await db.execute_raw(_UPDATE_SESSIONS, serialized)
+    if any(change.user_id for change in changes):
+        await db.execute_raw(_UPDATE_USER_SESSIONS, serialized)
     for entity, table in DAILY_SPEND_TABLES.items():
         if adjustments := tuple(
             change.daily.adjustment(target, change.savings_delta, change.request_id)
diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
index b28a653c9aa..db6045071a8 100644
--- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
+++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
@@ -492,6 +492,18 @@ class SpendLogCleanup:
             deadline=deadline,
         )
 
+    async def _delete_old_autorouter_user_session_rows(
+        self, prisma_client: PrismaClient, cutoff_date: datetime, deadline: float
+    ) -> TableCleanupResult:
+        return await self._delete_old_rows_batched(
+            prisma_client,
+            cutoff_date,
+            table_name="LiteLLM_AutoRouterUserSession",
+            key_columns=("user_id", "api_key", "session_id", "router_name"),
+            time_column="last_turn_at",
+            deadline=deadline,
+        )
+
     async def _delete_old_health_check_rows(
         self, prisma_client: PrismaClient, cutoff_date: datetime, deadline: float
     ) -> TableCleanupResult:
@@ -560,9 +572,17 @@ class SpendLogCleanup:
                 )
             except Exception:  # noqa: BLE001  # retained observations are retried by the next cleanup job
                 verbose_proxy_logger.warning("Auto-router baseline retention remains pending")
-        sessions_result: Final = await self._delete_old_autorouter_session_rows(prisma_client, session_cutoff, deadline)
+        sessions_result: Final = await self._delete_old_autorouter_session_rows(
+            prisma_client, session_cutoff, self._group_deadline(deadline, 2)
+        )
         verbose_proxy_logger.info("Deleted %s expired auto-router session rollup rows", sessions_result.rows_deleted)
-        return (sessions_result,)
+        user_sessions_result: Final = await self._delete_old_autorouter_user_session_rows(
+            prisma_client, session_cutoff, deadline
+        )
+        verbose_proxy_logger.info(
+            "Deleted %s expired auto-router user session rollup rows", user_sessions_result.rows_deleted
+        )
+        return (sessions_result, user_sessions_result)
 
     async def _clean_health_checks(
         self, prisma_client: PrismaClient, retention_seconds: int, deadline: float
diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py
index a6d5a17d73e..32f3bf9accb 100644
--- a/litellm/proxy/management_endpoints/auto_router_endpoints.py
+++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py
@@ -746,14 +746,18 @@ async def get_auto_router_benchmarks(
     ] = None,
     end_date: Annotated[str | None, Query(description="YYYY-MM-DD UTC, inclusive (defaults to today)")] = None,
     api_key: Annotated[str | None, Query(description="Filter to one virtual key token hash")] = None,
+    user_id: Annotated[
+        str | None, Query(min_length=1, description="Filter to one canonical internal user recorded on each turn")
+    ] = None,
 ) -> AutoRouterBenchmarksResponse:
     """
     Benchmarks for the auto-router dashboard: session shape, savings against the configured
     baseline, and prompt-caching behaviour bucketed by what the router did.
 
-    Reads the LiteLLM_AutoRouterSession rollup, folded once per request at spend-write time,
-    so this endpoint never scans LiteLLM_SpendLogs. A session is in the window when it
-    overlaps it: its last turn is on or after start_date and its first turn is on or before
+    Reads session rollups folded once per request at spend-write time, so this endpoint
+    never scans LiteLLM_SpendLogs. A user filter selects only turns attributed to that
+    internal user when written; older key-only history remains outside user views. A session
+    is in the window when it overlaps it: its last turn is on or after start_date and its first turn is on or before
     end_date. Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is
     over that bucket's turns.
 
@@ -783,6 +787,7 @@ async def get_auto_router_benchmarks(
         start_day.isoformat(),
         (end_day + timedelta(days=1)).isoformat(),
         api_key,
+        user_id,
     )
     rows: Final = _SESSION_AGG_ROWS.validate_python(raw_rows or ())
     groups: Final = (
diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma
index d2032cec0d0..f4015ed9277 100644
--- a/litellm/proxy/schema.prisma
+++ b/litellm/proxy/schema.prisma
@@ -1620,6 +1620,47 @@ model LiteLLM_AutoRouterSession {
   @@index([last_turn_at], map: "idx_autorouter_session_last_turn")
 }
 
+model LiteLLM_AutoRouterUserSession {
+  user_id               String
+  api_key               String
+  session_id            String
+  router_name           String
+  router_type           String
+  first_turn_at         DateTime
+  last_turn_at          DateTime
+  last_model            String
+  models                Json     @default("{}")
+  turns                 Int      @default(0)
+  unordered_turns       Int      @default(0)
+  covered_turns         Int      @default(0)
+  cache_hits            Int      @default(0)
+  same_model_turns      Int      @default(0)
+  same_model_hits       Int      @default(0)
+  first_visit_turns     Int      @default(0)
+  first_visit_hits      Int      @default(0)
+  return_turns          Int      @default(0)
+  return_hits           Int      @default(0)
+  return_expired_misses Int      @default(0)
+  return_within_ttl_misses  Int      @default(0)
+  ttl_5m_turns          Int      @default(0)
+  ttl_1h_turns          Int      @default(0)
+  total_tokens          BigInt   @default(0)
+  spend                 Float    @default(0)
+  saved_spend           Float    @default(0)
+  savings_estimated_turns Int    @default(0)
+  savings_estimated_actual_spend Float @default(0)
+  savings_estimated_saved_spend Float @default(0)
+  savings_estimated_baseline_models Json @default("{}")
+  classifier_cost       Float    @default(0)
+  classifier_cost_recorded_turns Int @default(0)
+  tier_turns            Json     @default("{}")
+  baseline_models       Json     @default("{}")
+
+  @@id([user_id, api_key, session_id, router_name])
+  @@index([last_turn_at], map: "idx_autorouter_user_session_last_turn")
+  @@index([user_id, last_turn_at], map: "idx_autorouter_user_session_user_last_turn")
+}
+
 // Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in
 // either direction. forward duplicates the requests the keys did not route through the
 // router through it, answering whether they should adopt it; reverse duplicates the
diff --git a/schema.prisma b/schema.prisma
index d2032cec0d0..f4015ed9277 100644
--- a/schema.prisma
+++ b/schema.prisma
@@ -1620,6 +1620,47 @@ model LiteLLM_AutoRouterSession {
   @@index([last_turn_at], map: "idx_autorouter_session_last_turn")
 }
 
+model LiteLLM_AutoRouterUserSession {
+  user_id               String
+  api_key               String
+  session_id            String
+  router_name           String
+  router_type           String
+  first_turn_at         DateTime
+  last_turn_at          DateTime
+  last_model            String
+  models                Json     @default("{}")
+  turns                 Int      @default(0)
+  unordered_turns       Int      @default(0)
+  covered_turns         Int      @default(0)
+  cache_hits            Int      @default(0)
+  same_model_turns      Int      @default(0)
+  same_model_hits       Int      @default(0)
+  first_visit_turns     Int      @default(0)
+  first_visit_hits      Int      @default(0)
+  return_turns          Int      @default(0)
+  return_hits           Int      @default(0)
+  return_expired_misses Int      @default(0)
+  return_within_ttl_misses  Int      @default(0)
+  ttl_5m_turns          Int      @default(0)
+  ttl_1h_turns          Int      @default(0)
+  total_tokens          BigInt   @default(0)
+  spend                 Float    @default(0)
+  saved_spend           Float    @default(0)
+  savings_estimated_turns Int    @default(0)
+  savings_estimated_actual_spend Float @default(0)
+  savings_estimated_saved_spend Float @default(0)
+  savings_estimated_baseline_models Json @default("{}")
+  classifier_cost       Float    @default(0)
+  classifier_cost_recorded_turns Int @default(0)
+  tier_turns            Json     @default("{}")
+  baseline_models       Json     @default("{}")
+
+  @@id([user_id, api_key, session_id, router_name])
+  @@index([last_turn_at], map: "idx_autorouter_user_session_last_turn")
+  @@index([user_id, last_turn_at], map: "idx_autorouter_user_session_user_last_turn")
+}
+
 // Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in
 // either direction. forward duplicates the requests the keys did not route through the
 // router through it, answering whether they should adopt it; reverse duplicates the
diff --git a/tests/proxy_behavior/spend/test_autorouter_session_rollup.py b/tests/proxy_behavior/spend/test_autorouter_session_rollup.py
index f3c68b489a5..77549b527d8 100644
--- a/tests/proxy_behavior/spend/test_autorouter_session_rollup.py
+++ b/tests/proxy_behavior/spend/test_autorouter_session_rollup.py
@@ -6,17 +6,24 @@ tests/test_litellm/proxy/db/test_autorouter_session_rollup.py.
 """
 
 import asyncio
+import time
 import uuid
 from datetime import datetime, timedelta, timezone
-from typing import Final
+from types import SimpleNamespace
+from typing import Final, TypedDict, cast
 
 import pytest
 from prisma import Prisma
+from prisma.errors import RawQueryError
+from typing_extensions import ReadOnly
 
 from litellm.proxy.db.autorouter_session_rollup import (
     AUTOROUTER_BENCHMARKS_SQL,
     UPSERT_AUTOROUTER_SESSION_SQL,
+    AutoRouterTurnTransaction,
+    flush_autorouter_turn_transactions,
 )
+from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import SpendLogCleanup
 
 pytestmark = pytest.mark.asyncio(loop_scope="session")
 
@@ -45,6 +52,7 @@ async def _turn(
     tier: "str | None" = None,
     baseline: "str | None" = None,
     estimated: bool = True,
+    user_id: str = "",
 ) -> None:
     touched: Final = 1 if (hit or ttl is not None or not covered) else 0
     await db.execute_raw(
@@ -68,6 +76,7 @@ async def _turn(
         int(estimated),
         spend if estimated else 0.0,
         saved if estimated else 0.0,
+        user_id,
     )
 
 
@@ -217,7 +226,7 @@ async def test_subtotal_coverage_survives_legacy_and_rolling_writers(db, writers
     assert row["savings_estimated_actual_spend"] == pytest.approx(0.01 * sum(writers))
     assert row["savings_estimated_saved_spend"] == pytest.approx(0.02 * sum(writers))
     groups: Final = await db.query_raw(
-        AUTOROUTER_BENCHMARKS_SQL, T0.isoformat(), (T0 + timedelta(days=1)).isoformat(), key
+        AUTOROUTER_BENCHMARKS_SQL, T0.isoformat(), (T0 + timedelta(days=1)).isoformat(), key, None
     )
     assert len(groups) == 1
     assert groups[0]["classifier_cost"] == row["classifier_cost"]
@@ -242,7 +251,7 @@ async def test_unknown_and_legacy_turns_preserve_actual_spend_without_entering_t
     assert row["saved_spend"] == pytest.approx(-0.03)
     assert row["savings_estimated_baseline_models"] == {"opus": 1}
     groups: Final = await db.query_raw(
-        AUTOROUTER_BENCHMARKS_SQL, T0.isoformat(), (T0 + timedelta(days=1)).isoformat(), key
+        AUTOROUTER_BENCHMARKS_SQL, T0.isoformat(), (T0 + timedelta(days=1)).isoformat(), key, None
     )
     assert len(groups) == 1
     for actual in (row, groups[0]):
@@ -277,6 +286,7 @@ async def test_the_benchmarks_aggregate_reads_only_overlapping_sessions(db):
         (T0 - timedelta(days=1)).isoformat(),
         (T0 + timedelta(days=1)).isoformat(),
         None,
+        None,
     )
     matching = [row for row in rows if row["router_name"] == router]
     assert len(matching) == 1
@@ -304,6 +314,7 @@ async def test_the_benchmarks_aggregate_can_filter_to_one_key(db):
         (T0 - timedelta(days=1)).isoformat(),
         (T0 + timedelta(days=1)).isoformat(),
         first_key,
+        None,
     )
     matching = [row for row in rows if row["router_name"] == router]
     assert len(matching) == 1
@@ -317,10 +328,160 @@ async def test_the_benchmarks_aggregate_can_filter_to_one_key(db):
         (T0 - timedelta(days=1)).isoformat(),
         (T0 + timedelta(days=1)).isoformat(),
         f"k-{uuid.uuid4()}",
+        None,
     )
     assert [row for row in unknown_key_rows if row["router_name"] == router] == []
 
 
+class _BenchmarkRow(TypedDict):
+    sessions: ReadOnly[int]
+    turns: ReadOnly[int]
+    same_model_turns: ReadOnly[int]
+    first_visit_turns: ReadOnly[int]
+    spend: ReadOnly[float]
+    saved_spend: ReadOnly[float]
+    tier_turns: ReadOnly[dict[str, int]]
+    cache_hits: ReadOnly[int]
+    savings_estimated_turns: ReadOnly[int]
+    savings_estimated_actual_spend: ReadOnly[float]
+    savings_estimated_saved_spend: ReadOnly[float]
+
+
+async def _scoped_benchmarks(
+    db: Prisma, router: str, user_id: str | None = None, key: str | None = None
+) -> tuple[_BenchmarkRow, ...]:
+    rows: Final = await db.query_raw(
+        AUTOROUTER_BENCHMARKS_SQL,
+        (T0 - timedelta(days=1)).isoformat(),
+        (T0 + timedelta(days=1)).isoformat(),
+        key,
+        user_id,
+    )
+    return tuple(cast(_BenchmarkRow, row) for row in rows if row["router_name"] == router)
+
+
+async def test_users_keep_written_identity_across_shared_keys_and_keyless_sessions(db: Prisma) -> None:
+    router: Final = f"r-{uuid.uuid4()}"
+    alice: Final = f"u-{uuid.uuid4()}"
+    bob: Final = f"u-{uuid.uuid4()}"
+    first_key: Final = f"k-{uuid.uuid4()}"
+    second_key: Final = f"k-{uuid.uuid4()}"
+    await _legacy_turn(db, first_key, T0, router=router)
+    await _turn(db, first_key, "A", T0 + timedelta(seconds=10), router=router, user_id=alice, tier="simple")
+    await _turn(
+        db, first_key, "B", T0 + timedelta(seconds=20), router=router, user_id=bob, spend=0.03, saved=0.06, tier="complex"
+    )
+    await _turn(db, second_key, "C", T0, router=router, user_id=alice, spend=0.02, saved=0.04)
+    await _turn(db, "", "A", T0, router=router, user_id=alice, ttl=300)
+    await _turn(db, "", "A", T0 + timedelta(seconds=1), router=router, user_id=alice, hit=1)
+    await _turn(db, "", "B", T0, router=router, user_id=bob, spend=0.04, saved=0.08)
+    await _turn(db, second_key, "C", T0 - timedelta(days=40), router=router, user_id=alice, session_id="expired")
+
+    alice_rows: Final = await _scoped_benchmarks(db, router, user_id=alice)
+    bob_rows: Final = await _scoped_benchmarks(db, router, user_id=bob)
+    global_rows: Final = await _scoped_benchmarks(db, router)
+    key_rows: Final = await _scoped_benchmarks(db, router, key=first_key)
+    intersection: Final = await _scoped_benchmarks(db, router, user_id=alice, key=first_key)
+    assert len(alice_rows) == len(bob_rows) == len(global_rows) == len(key_rows) == len(intersection) == 1
+    assert (alice_rows[0]["sessions"], alice_rows[0]["turns"], alice_rows[0]["same_model_turns"]) == (3, 4, 1)
+    assert (bob_rows[0]["sessions"], bob_rows[0]["turns"], bob_rows[0]["first_visit_turns"]) == (2, 2, 2)
+    assert alice_rows[0]["spend"] == pytest.approx(0.05)
+    assert bob_rows[0]["spend"] == pytest.approx(0.07)
+    assert alice_rows[0]["tier_turns"] == {"simple": 1}
+    assert bob_rows[0]["tier_turns"] == {"complex": 1}
+    assert (alice_rows[0]["cache_hits"], bob_rows[0]["cache_hits"]) == (1, 0)
+    assert (global_rows[0]["sessions"], global_rows[0]["turns"]) == (4, 7)
+    assert (alice_rows[0]["savings_estimated_turns"], bob_rows[0]["savings_estimated_turns"]) == (4, 2)
+    assert global_rows[0]["savings_estimated_turns"] == 6
+    for scoped in (alice_rows[0], bob_rows[0]):
+        assert scoped["savings_estimated_actual_spend"] == pytest.approx(scoped["spend"])
+        assert scoped["savings_estimated_saved_spend"] == pytest.approx(scoped["saved_spend"])
+    assert global_rows[0]["spend"] == pytest.approx(alice_rows[0]["spend"] + bob_rows[0]["spend"] + 0.01)
+    assert global_rows[0]["saved_spend"] == pytest.approx(alice_rows[0]["saved_spend"] + bob_rows[0]["saved_spend"] + 0.02)
+    assert global_rows[0]["tier_turns"] == {"simple": 1, "complex": 1}
+    assert (key_rows[0]["sessions"], key_rows[0]["turns"]) == (1, 3)
+    assert key_rows[0]["spend"] == pytest.approx(0.05)
+    assert (intersection[0]["sessions"], intersection[0]["turns"]) == (1, 1)
+    assert intersection[0]["spend"] == pytest.approx(0.01)
+    assert await _scoped_benchmarks(db, router, user_id=bob, key=second_key) == ()
+    assert await _scoped_benchmarks(db, router, user_id=f"u-{uuid.uuid4()}") == ()
+    assert await _scoped_benchmarks(db, router, user_id="") == ()
+
+
+async def test_a_failed_user_projection_rolls_back_the_keys_increment(db: Prisma) -> None:
+    key: Final = f"k-{uuid.uuid4()}"
+    user_id: Final = "".join(str(uuid.uuid4()) for _ in range(200))
+    await _turn(db, key, "A", T0)
+    before: Final = await _row(db, key)
+
+    with pytest.raises(RawQueryError, match=r"index row (requires|size)"):
+        await _turn(db, key, "B", T0 + timedelta(seconds=1), user_id=user_id)
+
+    assert await _row(db, key) == before
+    assert await db.query_raw('SELECT user_id FROM "LiteLLM_AutoRouterUserSession" WHERE user_id = $1', user_id) == []
+
+    first_user: Final = f"u-{uuid.uuid4()}"
+    second_user: Final = f"u-{uuid.uuid4()}"
+    turns: Final = tuple(
+        AutoRouterTurnTransaction(
+            api_key=key,
+            user_id=user,
+            session_id="s1",
+            router_name="auto-1",
+            router_type="complexity",
+            model=model,
+            turn_at=T0 + timedelta(seconds=second),
+            total_tokens=100,
+            spend=0.01,
+            saved_spend=0.02,
+            classifier_cost=0.0,
+            covered=True,
+            cache_hit=False,
+            cache_ttl_seconds=None,
+            cache_touched=False,
+        )
+        for user, model, second in (
+            (first_user, "A", 1),
+            (user_id, "B", 2),
+            (first_user, "B", 3),
+            (second_user, "C", 4),
+            (first_user, "B", 5),
+            (second_user, "C", 6),
+            (user_id, "A", 7),
+        )
+    )
+    await flush_autorouter_turn_transactions(SimpleNamespace(db=db), tuple(reversed(turns)), n_retry_times=0)
+
+    key_row: Final = await _row(db, key)
+    assert (key_row["turns"], key_row["last_model"], key_row["unordered_turns"]) == (2, "A", 0)
+    assert key_row["spend"] == pytest.approx(0.02)
+    user_rows: Final = await db.query_raw('SELECT * FROM "LiteLLM_AutoRouterUserSession" WHERE api_key = $1', key)
+    by_user: Final = {row["user_id"]: row for row in user_rows}
+    assert set(by_user) == {first_user, second_user}
+    for user, count, model in ((first_user, 3, "B"), (second_user, 2, "C")):
+        row: Final = by_user[user]
+        assert (row["turns"], row["same_model_turns"], row["unordered_turns"], row["last_model"]) == (count, 1, 0, model)
+        assert row["spend"] == pytest.approx(count * 0.01)
+        assert row["saved_spend"] == pytest.approx(count * 0.02)
+
+
+async def test_user_session_cleanup_keeps_another_users_recent_keyless_session(db: Prisma) -> None:
+    router: Final = f"r-{uuid.uuid4()}"
+    expired_user: Final = f"u-{uuid.uuid4()}"
+    recent_user: Final = f"u-{uuid.uuid4()}"
+    await _turn(db, "", "A", T0 - timedelta(days=1), router=router, user_id=expired_user)
+    await _turn(db, "", "A", T0 + timedelta(days=1), router=router, user_id=recent_user)
+    cleaner: Final = SpendLogCleanup(general_settings={})
+
+    await cleaner._delete_old_autorouter_user_session_rows(
+        SimpleNamespace(db=db), T0.replace(tzinfo=timezone.utc), time.monotonic() + 60
+    )
+
+    assert await db.query_raw(
+        'SELECT user_id, turns FROM "LiteLLM_AutoRouterUserSession" WHERE router_name = $1', router
+    ) == [{"user_id": recent_user, "turns": 1}]
+
+
 async def test_a_reconfigured_alias_reports_each_router_type_as_its_own_group(db):
     key = f"k-{uuid.uuid4()}"
     router = f"r-{uuid.uuid4()}"
@@ -334,6 +495,7 @@ async def test_a_reconfigured_alias_reports_each_router_type_as_its_own_group(db
         (T0 - timedelta(days=1)).isoformat(),
         (T0 + timedelta(days=1)).isoformat(),
         None,
+        None,
     )
     matching = sorted(
         (row for row in rows if row["router_name"] == router),
@@ -418,6 +580,7 @@ async def test_the_benchmarks_aggregate_sums_tier_turns_across_sessions(db):
         (T0 - timedelta(days=1)).isoformat(),
         (T0 + timedelta(days=1)).isoformat(),
         None,
+        None,
     )
     grouped = next(row for row in rows if row["router_name"] == router)
     assert grouped["tier_turns"] == {"simple": 2, "complex": 1}
@@ -446,6 +609,7 @@ async def test_tier_maps_stay_separate_per_router_type_on_a_reconfigured_alias(d
         (T0 - timedelta(days=1)).isoformat(),
         (T0 + timedelta(days=1)).isoformat(),
         None,
+        None,
     )
     by_type = {row["router_type"]: row["tier_turns"] for row in rows if row["router_name"] == router}
     assert by_type == {"complexity": {"medium": 1}, "quality": {"2": 1}}
@@ -461,6 +625,7 @@ async def test_a_window_with_no_tiered_turns_aggregates_to_an_empty_map(db):
         (T0 - timedelta(days=1)).isoformat(),
         (T0 + timedelta(days=1)).isoformat(),
         None,
+        None,
     )
     grouped = next(row for row in rows if row["router_name"] == router)
     assert grouped["tier_turns"] == {}
diff --git a/tests/proxy_behavior/spend/test_baseline_accounting.py b/tests/proxy_behavior/spend/test_baseline_accounting.py
index e187a44c29d..3504751d132 100644
--- a/tests/proxy_behavior/spend/test_baseline_accounting.py
+++ b/tests/proxy_behavior/spend/test_baseline_accounting.py
@@ -56,7 +56,9 @@ def record() -> Callable[..., BaselineAccountingRecord]:
         },
     )
 
-    def create(label: str = "first", started: float = 10000.0, identical: bool = True) -> BaselineAccountingRecord:
+    def create(
+        label: str = "first", started: float = 10000.0, identical: bool = True, user_id: str = ""
+    ) -> BaselineAccountingRecord:
         return BaselineAccountingRecord(
             scope="autorouter-baseline:v3:" + run * 2, api_key=run, session_id=run,
             router_name="test-router", baseline_model="anthropic/claude-opus-5",
@@ -76,6 +78,7 @@ def record() -> Callable[..., BaselineAccountingRecord]:
                 total_tokens=6230, spend=0.17, saved_spend=0.0, classifier_cost=0.0,
                 covered=True, cache_hit=False, cache_ttl_seconds=3600, cache_touched=True,
                 baseline_model="anthropic/claude-opus-5",
+                user_id=user_id,
             ),
             daily=DailyBaselineAttribution(
                 date="2026-09-15", api_key=run, model="claude-opus-5", custom_llm_provider="anthropic",
@@ -99,21 +102,36 @@ async def _session(db: Prisma, record: BaselineAccountingRecord):
     return rows[0]
 
 
+async def _user_sessions(db: Prisma, record: BaselineAccountingRecord) -> dict[str, dict[str, object]]:
+    rows: Final = await db.query_raw('SELECT * FROM "LiteLLM_AutoRouterUserSession" WHERE api_key=$1', record.api_key)
+    return {str(row["user_id"]): row for row in rows}
+
+
 async def test_late_replay_updates_all_projections_without_rebilling(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
     store: Final = _store(db)
-    late: Final = record("late", 10001.0)
-    early: Final = record("early", identical=False)
+    late: Final = record("late", 10001.0, user_id="late-user")
+    early: Final = record("early", identical=False, user_id="early-user")
     await _log(db, late)
     assert await store.append(late) == "recorded"
     assert await store.project(late.scope) == "published"
     before: Final = await _session(db, late)
     assert before["savings_estimated_actual_spend"] == before["spend"] == 0.17
     assert before["saved_spend"] == 0.0
+    before_users: Final = await _user_sessions(db, late)
+    assert set(before_users) == {"late-user"}
+    assert before_users["late-user"]["savings_estimated_turns"] == 1
+    assert before_users["late-user"]["savings_estimated_baseline_models"] == {late.baseline_model: 1}
     await _log(db, early)
     assert await store.append(early) == "recorded"
     pending: Final = await _session(db, late)
     assert pending["spend"] == 0.34 and pending["savings_estimated_turns"] == 0
     assert pending["saved_spend"] == pending["savings_estimated_actual_spend"] == 0.0
+    pending_users: Final = await _user_sessions(db, late)
+    assert set(pending_users) == {"late-user", "early-user"}
+    for user in pending_users.values():
+        assert user["turns"] == 1 and user["spend"] == 0.17
+        assert user["savings_estimated_turns"] == user["savings_estimated_actual_spend"] == user["saved_spend"] == 0
+        assert user["savings_estimated_baseline_models"] == {}
     waiting: Final = await db.query_raw('SELECT metadata FROM "LiteLLM_SpendLogs" WHERE request_id=$1', late.observation.request_id)
     assert waiting[0]["metadata"]["autorouter_savings"] is None
     assert waiting[0]["metadata"]["autorouter_savings_estimate"]["reason"] == "pending_projection"
@@ -125,37 +143,69 @@ async def test_late_replay_updates_all_projections_without_rebilling(db: Prisma,
     assert logs[0]["spend"] == 0.17
     assert logs[0]["metadata"]["autorouter_savings_estimate"]["provenance"] == "modeled"
     assert after["saved_spend"] == pytest.approx(logs[0]["metadata"]["autorouter_savings"])
+    after_users: Final = await _user_sessions(db, late)
+    assert after_users["early-user"] == pending_users["early-user"]
+    for field in (
+        "saved_spend", "savings_estimated_turns", "savings_estimated_actual_spend",
+        "savings_estimated_saved_spend", "savings_estimated_baseline_models",
+    ):
+        assert after_users["late-user"][field] == after[field]
+    assert after_users["late-user"]["turns"] == 1 and after_users["late-user"]["spend"] == 0.17
     for table in ("DailyUserSpend", "DailyTeamSpend", "DailyOrganizationSpend", "DailyEndUserSpend", "DailyAgentSpend", "DailyTagSpend"):
         rows: Final = await db.query_raw(f'SELECT spend,api_requests,autorouter_savings_spend FROM "LiteLLM_{table}" WHERE api_key=$1', late.api_key)
         assert rows[0]["spend"] == rows[0]["api_requests"] == 0
         assert rows[0]["autorouter_savings_spend"] == pytest.approx(after["saved_spend"])
 
 
-async def test_commit_ack_loss_and_concurrent_duplicate_delivery_are_idempotent(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
-    event: Final = record()
+@pytest.mark.parametrize("attributed", [True, False])
+async def test_commit_ack_loss_and_concurrent_duplicate_delivery_are_idempotent(
+    db: Prisma, record: Callable[..., BaselineAccountingRecord], attributed: bool
+) -> None:
+    event: Final = record(user_id="first-user" if attributed else "")
+    other: Final = record("other", 10001.0, user_id="second-user" if attributed else "")
     await _log(db, event)
     assert await _store(db, after_commit=True).append(event) == "unavailable"
     store: Final = _store(db)
     assert set(await asyncio.gather(*(store.append(event) for _ in range(4)))) == {"recorded"}
+    await _log(db, other)
+    assert await store.append(other) == "recorded"
+    if not attributed:
+        await db.execute_raw(
+            'UPDATE "LiteLLM_AutoRouterBaselineObservation" SET data=(data::jsonb #- \'{turn,user_id}\')::text WHERE scope=$1',
+            event.scope,
+        )
     assert await store.project(event.scope) == "published"
     assert await store.project(event.scope) == "unchanged"
     session: Final = await _session(db, event)
-    assert session["turns"] == session["savings_estimated_turns"] == 1
-    assert session["spend"] == session["savings_estimated_actual_spend"] == 0.17
+    assert session["turns"] == session["savings_estimated_turns"] == 2
+    assert session["spend"] == session["savings_estimated_actual_spend"] == 0.34
+    users: Final = await _user_sessions(db, event)
+    assert set(users) == ({"first-user", "second-user"} if attributed else set())
+    for user in users.values():
+        assert user["turns"] == user["savings_estimated_turns"] == 1
+        assert user["spend"] == user["savings_estimated_actual_spend"] == 0.17
+        assert user["savings_estimated_baseline_models"] == {event.baseline_model: 1}
 
 
 async def test_publication_rollback_keeps_dirty_revision_for_retry(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
-    event: Final = record()
+    event: Final = record(user_id="rollback-user")
     await _log(db, event)
     store: Final = _store(db)
     assert await store.append(event) == "recorded"
     assert await _store(db, before_commit=True).project(event.scope) == "unavailable"
     session: Final = await _session(db, event)
     assert session["spend"] == 0.17 and session["savings_estimated_turns"] == 0
+    before_users: Final = await _user_sessions(db, event)
+    assert before_users["rollback-user"]["spend"] == 0.17
+    assert before_users["rollback-user"]["savings_estimated_turns"] == 0
+    assert before_users["rollback-user"]["savings_estimated_baseline_models"] == {}
     revisions: Final = await db.query_raw('SELECT revision,published_revision FROM "LiteLLM_AutoRouterBaselineComparison" WHERE scope=$1', event.scope)
     assert revisions[0]["revision"] > revisions[0]["published_revision"]
     assert await store.project(event.scope) == "published"
     assert (await _session(db, event))["savings_estimated_turns"] == 1
+    after_users: Final = await _user_sessions(db, event)
+    assert after_users["rollback-user"]["turns"] == after_users["rollback-user"]["savings_estimated_turns"] == 1
+    assert after_users["rollback-user"]["spend"] == after_users["rollback-user"]["savings_estimated_actual_spend"] == 0.17
 
 
 async def test_conflicting_duplicate_cannot_restore_an_observed_estimate(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
@@ -196,6 +246,7 @@ async def test_native_observation_enters_spend_pipeline_once_with_shared_daily_a
     db: Prisma, record: Callable[..., BaselineAccountingRecord], monkeypatch: pytest.MonkeyPatch,
 ) -> None:
     import os
+
     from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
     from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
     from litellm.proxy.hooks.autorouter_baseline_cache import CapturedBaselineObservation
diff --git a/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py b/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py
index acd3dc18b54..c61a489f894 100644
--- a/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py
+++ b/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py
@@ -56,6 +56,31 @@ def _build(payload: dict | None = None, metadata: dict | None = None):
 
 
 class TestBuildTransaction:
+    @pytest.mark.parametrize(
+        "api_key, user_id, included",
+        [
+            ("hashed-key", "canonical-user", True),
+            ("hashed-key", None, True),
+            ("hashed-key", "", True),
+            ("", "canonical-user", True),
+            ("", None, False),
+            ("", "", False),
+        ],
+    )
+    def test_attribution_uses_the_canonical_user_even_without_a_key(
+        self, api_key: str, user_id: str | None, included: bool
+    ) -> None:
+        transaction: Final = _build(
+            payload=_payload(api_key=api_key, user=user_id),
+            metadata=_metadata(user="client-user", user_api_key_user_id="metadata-user"),
+        )
+        if not included:
+            assert transaction is None
+            return
+        assert transaction is not None
+        assert transaction.api_key == api_key
+        assert transaction.user_id == (user_id or "")
+
     def test_successful_auto_routed_turn_builds_every_field(self):
         transaction = _build(
             metadata=_metadata(
@@ -205,23 +230,43 @@ class TestBuildTransaction:
 
 
 class _FakeDB:
-    def __init__(self, failures: "list[Exception] | None" = None, poison_session: str | None = None):
+    def __init__(
+        self,
+        failures: "list[Exception] | None" = None,
+        poison_session: str | None = None,
+        poison_user: str | None = None,
+        commit_then_error_users: frozenset[str] = frozenset(),
+    ):
         self.calls: list[tuple] = []
+        self.attempts: list[tuple[str, tuple[object, ...]]] = []
         self._failures = list(failures or [])
         self._poison_session = poison_session
+        self._poison_user = poison_user
+        self._commit_then_error_users = commit_then_error_users
 
     async def execute_raw(self, sql: str, *params: object) -> int:
+        self.attempts.append((sql, params))
         if self._poison_session is not None and params[1] == self._poison_session:
             raise RuntimeError("index row size exceeds btree maximum")
+        if self._poison_user is not None and params[19] == self._poison_user:
+            raise RuntimeError("index row size exceeds btree maximum")
         if self._failures:
             raise self._failures.pop(0)
         self.calls.append((sql, params))
+        if params[19] in self._commit_then_error_users:
+            raise RuntimeError("commit succeeded but acknowledgement was lost")
         return 1
 
 
 class _FakeClient:
-    def __init__(self, failures: "list[Exception] | None" = None, poison_session: str | None = None):
-        self.db = _FakeDB(failures, poison_session)
+    def __init__(
+        self,
+        failures: "list[Exception] | None" = None,
+        poison_session: str | None = None,
+        poison_user: str | None = None,
+        commit_then_error_users: frozenset[str] = frozenset(),
+    ):
+        self.db = _FakeDB(failures, poison_session, poison_user, commit_then_error_users)
 
 
 def _transaction(
@@ -229,9 +274,11 @@ def _transaction(
     at: datetime = datetime(2026, 8, 1, 12, 0, 0),
     tier: str | None = "medium",
     baseline_model: str | None = "anthropic/claude-opus-5",
+    api_key: str = "k1",
+    user_id: str = "",
 ) -> AutoRouterTurnTransaction:
     return AutoRouterTurnTransaction(
-        api_key="k1",
+        api_key=api_key,
         session_id=session_id,
         router_name="live-auto",
         router_type="complexity",
@@ -247,6 +294,7 @@ def _transaction(
         cache_touched=False,
         tier=tier,
         baseline_model=baseline_model,
+        user_id=user_id,
     )
 
 
@@ -261,7 +309,7 @@ class TestFlush:
 
     def test_params_marshal_in_statement_order(self):
         client = _FakeClient()
-        asyncio.run(flush_autorouter_turn_transactions(client, [_transaction()]))
+        asyncio.run(flush_autorouter_turn_transactions(client, [_transaction(user_id="canonical-user")]))
         sql, params = client.db.calls[0]
         assert sql == UPSERT_AUTOROUTER_SESSION_SQL
         assert params == (
@@ -284,8 +332,65 @@ class TestFlush:
             0,
             0.0,
             0.0,
+            "canonical-user",
         )
 
+    def test_a_keys_turns_stay_chronological_when_its_canonical_user_changes(self) -> None:
+        client: Final = _FakeClient()
+        earlier: Final = _transaction(user_id="z-user", at=datetime(2026, 8, 1, 12, 0, 0))
+        later: Final = _transaction(user_id="a-user", at=datetime(2026, 8, 1, 12, 0, 10))
+        asyncio.run(flush_autorouter_turn_transactions(client, [later, earlier]))
+        assert [(params[5], params[19]) for _, params in client.db.calls] == [
+            ("2026-08-01T12:00:00", "z-user"),
+            ("2026-08-01T12:00:10", "a-user"),
+        ]
+
+    def test_one_keyless_users_failed_session_does_not_drop_another_users_turn(self) -> None:
+        client: Final = _FakeClient(poison_user="a-user")
+        failed: Final = _transaction(api_key="", user_id="a-user")
+        other: Final = _transaction(api_key="", user_id="b-user", at=datetime(2026, 8, 1, 12, 0, 10))
+        asyncio.run(flush_autorouter_turn_transactions(client, [other, failed]))
+        assert [(params[0], params[1], params[19]) for _, params in client.db.calls] == [("", "s1", "b-user")]
+
+    def test_uncertain_commits_quarantine_only_the_key_and_each_failed_user(self) -> None:
+        client: Final = _FakeClient(commit_then_error_users=frozenset({"a-failed", "c-failed"}))
+        turns: Final = tuple(
+            _transaction(user_id=user, at=datetime(2026, 8, 1, 12, 0, second), api_key=key)
+            for user, second, key in (
+                ("b-healthy", 0, "k1"),
+                ("a-failed", 1, "k1"),
+                ("b-healthy", 2, "k1"),
+                ("c-failed", 3, "k1"),
+                ("b-healthy", 4, "k1"),
+                ("d-healthy", 5, "k1"),
+                ("c-failed", 6, "k1"),
+                ("d-healthy", 7, "k1"),
+                ("a-failed", 8, "k1"),
+                ("", 9, "k1"),
+                ("z-other", 10, "k2"),
+            )
+        )
+        asyncio.run(flush_autorouter_turn_transactions(client, tuple(reversed(turns))))
+
+        assert client.db.attempts == client.db.calls
+        assert [
+            (params[0], params[19], params[5])
+            for sql, params in client.db.calls
+            if sql == UPSERT_AUTOROUTER_SESSION_SQL
+        ] == [
+            ("k1", "b-healthy", "2026-08-01T12:00:00"),
+            ("k1", "a-failed", "2026-08-01T12:00:01"),
+            ("k2", "z-other", "2026-08-01T12:00:10"),
+        ]
+        assert [params[19] for _, params in client.db.attempts].count("a-failed") == 1
+        assert [params[19] for _, params in client.db.attempts].count("c-failed") == 1
+        for user, seconds in (("b-healthy", (2, 4)), ("c-failed", (3,)), ("d-healthy", (5, 7))):
+            assert [
+                (params[0], params[5])
+                for sql, params in client.db.calls
+                if sql != UPSERT_AUTOROUTER_SESSION_SQL and params[19] == user
+            ] == [("k1", f"2026-08-01T12:00:{second:02d}") for second in seconds]
+
     def test_a_connect_error_retries_the_same_statement(self):
         client = _FakeClient(failures=[httpx.ConnectError("boom")])
         asyncio.run(flush_autorouter_turn_transactions(client, [_transaction()]))
diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py
index 6ac053f4e15..d5ddd5a78c3 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py
@@ -4,6 +4,7 @@ Unit tests for auto router management endpoints
 
 from collections.abc import Mapping, Sequence
 from pathlib import Path
+from types import SimpleNamespace
 from typing import Final
 
 import pytest
@@ -654,17 +655,43 @@ class TestAutoRouterBenchmarks:
         assert _summed_agg_row([complexity, quality]).tier_turns == {}
 
     @pytest.mark.asyncio
-    async def test_non_admin_roles_cannot_read_benchmarks(self):
+    @pytest.mark.parametrize("user_id", [None, "own-user", "other-user"])
+    async def test_non_admin_roles_cannot_read_benchmarks(self, user_id: str | None):
         from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_benchmarks
 
         with pytest.raises(HTTPException) as err:
             await get_auto_router_benchmarks(
-                user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-x"),
+                user_api_key_dict=UserAPIKeyAuth(
+                    user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-x", user_id="own-user"
+                ),
                 start_date="2026-08-01",
                 end_date="2026-08-02",
+                user_id=user_id,
             )
         assert err.value.status_code == 403
 
+    @pytest.mark.asyncio
+    async def test_an_empty_user_filter_is_rejected_before_querying_deployment_data(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        import httpx
+        from fastapi import FastAPI
+
+        from litellm.proxy import proxy_server
+        from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+        from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_benchmarks
+
+        query: Final = AsyncMock(return_value=[])
+        monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=SimpleNamespace(query_raw=query)))
+        app: Final = FastAPI()
+        app.get("/auto_router/benchmarks")(get_auto_router_benchmarks)
+        app.dependency_overrides[user_api_key_auth] = lambda: ADMIN
+        async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client:
+            response: Final = await client.get("/auto_router/benchmarks", params={"user_id": ""})
+
+        assert response.status_code == 422
+        query.assert_not_awaited()
+
     @pytest.mark.asyncio
     async def test_a_reversed_window_is_rejected(self, monkeypatch: pytest.MonkeyPatch):
         from litellm.proxy import proxy_server
@@ -680,7 +707,11 @@ class TestAutoRouterBenchmarks:
         assert err.value.status_code == 400
 
     @pytest.mark.asyncio
-    async def test_endpoint_returns_groups_and_totals_from_the_rollup(self, monkeypatch: pytest.MonkeyPatch):
+    @pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY])
+    @pytest.mark.parametrize("user_id", [None, "selected-user"])
+    async def test_endpoint_returns_groups_and_totals_from_the_rollup(
+        self, monkeypatch: pytest.MonkeyPatch, role: LitellmUserRoles, user_id: str | None
+    ):
         from litellm.proxy import proxy_server
         from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_benchmarks
 
@@ -695,12 +726,13 @@ class TestAutoRouterBenchmarks:
         monkeypatch.setattr(proxy_server, "prisma_client", type("P", (), {"db": _DB()})())
 
         response = await get_auto_router_benchmarks(
-            user_api_key_dict=ADMIN,
+            user_api_key_dict=UserAPIKeyAuth(user_role=role, api_key="sk-admin", user_id="viewer"),
             start_date="2026-07-01",
             end_date="2026-08-01",
             api_key="key-hash",
+            user_id=user_id,
         )
-        assert captured["params"] == ("2026-07-01T00:00:00", "2026-08-02T00:00:00", "key-hash")
+        assert captured["params"] == ("2026-07-01T00:00:00", "2026-08-02T00:00:00", "key-hash", user_id)
         assert response.routers_in_scope == 1
         assert response.groups[0].router_name == "live-auto"
         assert response.groups[0].saved_pct == response.totals.saved_pct == 75.0
diff --git a/tests/test_litellm/proxy/test_spend_log_cleanup.py b/tests/test_litellm/proxy/test_spend_log_cleanup.py
index bf1538183ab..f395c146cf0 100644
--- a/tests/test_litellm/proxy/test_spend_log_cleanup.py
+++ b/tests/test_litellm/proxy/test_spend_log_cleanup.py
@@ -793,18 +793,20 @@ async def test_spend_logs_retention_alone_does_not_touch_the_session_rollup():
     tables = [call[0][0] for call in client.db.execute_raw.call_args_list]
     assert any('"LiteLLM_SpendLogs"' in sql for sql in tables)
     assert not any('"LiteLLM_AutoRouterSession"' in sql for sql in tables)
+    assert not any('"LiteLLM_AutoRouterUserSession"' in sql for sql in tables)
     assert not any('"LiteLLM_HealthCheckTable"' in sql for sql in tables)
 
 
 @pytest.mark.asyncio
-async def test_session_retention_alone_cleans_only_the_session_rollup():
-    client = _mock_prisma_for_retention([0])
+async def test_session_retention_alone_cleans_both_session_rollups():
+    client = _mock_prisma_for_retention([0, 0])
     cleaner = SpendLogCleanup(general_settings={"maximum_autorouter_session_retention_period": "365d"})
     cleaner.pod_lock_manager = None
     await cleaner.cleanup_old_spend_logs(client)
     tables = [call[0][0] for call in client.db.execute_raw.call_args_list]
-    assert len(tables) == 1
+    assert len(tables) == 2
     assert '"LiteLLM_AutoRouterSession"' in tables[0]
+    assert '"LiteLLM_AutoRouterUserSession"' in tables[1]
 
 
 @pytest.mark.asyncio
@@ -825,7 +827,7 @@ async def test_health_check_retention_alone_cleans_only_the_health_check_table()
 
 @pytest.mark.asyncio
 async def test_each_retention_key_cuts_off_at_its_own_horizon():
-    client = _mock_prisma_for_retention([0, 0, 0, 0])
+    client = _mock_prisma_for_retention([0, 0, 0, 0, 0])
     cleaner = SpendLogCleanup(
         general_settings={
             "maximum_spend_logs_retention_period": "7d",
@@ -839,6 +841,8 @@ async def test_each_retention_key_cuts_off_at_its_own_horizon():
         (
             "LiteLLM_AutoRouterSession"
             if '"LiteLLM_AutoRouterSession"' in call[0][0]
+            else "LiteLLM_AutoRouterUserSession"
+            if '"LiteLLM_AutoRouterUserSession"' in call[0][0]
             else "LiteLLM_HealthCheckTable"
             if '"LiteLLM_HealthCheckTable"' in call[0][0]
             else "logs"
@@ -848,6 +852,7 @@ async def test_each_retention_key_cuts_off_at_its_own_horizon():
     now = datetime.now(timezone.utc)
     assert (now - cutoffs["logs"]).days == 7
     assert (now - cutoffs["LiteLLM_AutoRouterSession"]).days == 365
+    assert cutoffs["LiteLLM_AutoRouterUserSession"] == cutoffs["LiteLLM_AutoRouterSession"]
     assert (now - cutoffs["LiteLLM_HealthCheckTable"]).days == 30
 
 
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx
index 5c7453c1394..a144630cdd0 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx
@@ -434,7 +434,7 @@ describe("AutoRouterBenchmarksTab", () => {
     mockHook({ data: response([group()]) });
     const { dateValue, onDateChange } = renderTab();
 
-    expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", dateValue, undefined);
+    expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", dateValue, undefined, undefined);
     expect(screen.getByText("Jul 6 – Aug 5 (UTC)")).toBeInTheDocument();
 
     fireEvent.click(screen.getByTestId("date-picker"));
@@ -460,7 +460,7 @@ describe("AutoRouterBenchmarksTab", () => {
       </QueryClientProvider>,
     );
 
-    expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", dateValue, "key-hash-1");
+    expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", dateValue, "key-hash-1", undefined);
     expect(screen.getByText("Total estimated savings")).toBeInTheDocument();
     expect(screen.queryByRole("tab", { name: "Shadow Evals" })).not.toBeInTheDocument();
   });
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx
index ce55b633b60..063598bd46e 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx
@@ -312,8 +312,7 @@ const BenchmarksBody: React.FC<BenchmarksBodyProps> = ({ isPending, error, data,
         length. Total actual spend includes every turn; savings and baseline spend include only turns with a current
         estimate, including turns with zero savings. Savings are net of recorded LLM classification cost. Classification
         cost per 1K turns is averaged over all auto-router turns, including those that skip classification. The range
-        counts whole sessions that overlap it, so totals can differ slightly from the Overall tab, which buckets savings
-        by UTC day.
+        counts whole sessions that overlap it, so totals can differ from savings views that group usage by UTC day.
       </p>
 
       <div className="space-y-4">
@@ -333,11 +332,17 @@ interface AutoRouterBenchmarksTabProps {
   accessToken: string | null;
   activity: Pick<DailyActivityRange, "dateValue" | "onDateChange">;
   apiKey?: string;
+  userId?: string;
 }
 
-export const AutoRouterUsageView: React.FC<AutoRouterBenchmarksTabProps> = ({ accessToken, activity, apiKey }) => {
+export const AutoRouterUsageView: React.FC<AutoRouterBenchmarksTabProps> = ({
+  accessToken,
+  activity,
+  apiKey,
+  userId,
+}) => {
   const { dateValue, onDateChange } = activity;
-  const { data, isPending, error } = useAutoRouterBenchmarks(accessToken, dateValue, apiKey);
+  const { data, isPending, error } = useAutoRouterBenchmarks(accessToken, dateValue, apiKey, userId);
   const [selectedKey, setSelectedKey] = useState<string>(ALL_ROUTERS);
   const { data: autoRouters } = useAutoRouters();
 
@@ -372,6 +377,12 @@ export const AutoRouterUsageView: React.FC<AutoRouterBenchmarksTabProps> = ({ ac
         </div>
       </div>
 
+      {userId && (
+        <p className="text-sm text-muted-foreground">
+          Usage for this user across API keys and JWT-authenticated requests. Older sessions recorded without a user ID
+          are not included.
+        </p>
+      )}
       <BenchmarksBody
         isPending={isPending}
         error={error}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterBenchmarks.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterBenchmarks.ts
index 284342f0aec..a7e6dffa2c3 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterBenchmarks.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterBenchmarks.ts
@@ -23,10 +23,15 @@ export const benchmarksWindow = (
   };
 };
 
-export const useAutoRouterBenchmarks = (accessToken: string | null, range: DateRange, apiKey?: string) =>
+export const useAutoRouterBenchmarks = (
+  accessToken: string | null,
+  range: DateRange,
+  apiKey?: string,
+  userId?: string,
+) =>
   $api.useQuery(
     "get",
     "/auto_router/benchmarks",
-    { params: { query: { ...benchmarksWindow(range, new Date()), api_key: apiKey } } },
+    { params: { query: { ...benchmarksWindow(range, new Date()), api_key: apiKey, user_id: userId } } },
     { enabled: Boolean(accessToken && range.from && range.to), retry: false },
   );
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.test.tsx
index 4059303d5a5..e501cf00b90 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.test.tsx
@@ -15,6 +15,8 @@ vi.mock("@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity", (
       isFetchingMore: false,
       progress: { currentPage: 4, totalPages: 9 },
       cancelled: false,
+      failed: false,
+      coversRange: true,
       cancel: mockCancel,
     };
   },
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts
index 92dd24b8d6d..4eb9f257d30 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts
@@ -67,14 +67,16 @@ export const useScopedDailyActivityRange = (
     args: [accessToken, startTime, endTime, userId, true, apiKey],
     enabled: !!accessToken && !!startTime && !!endTime,
   };
-  const { data, loading, isFetchingMore, progress, cancelled, failed, cancel } =
+  const { data, loading, isFetchingMore, progress, cancelled, failed, coversRange, cancel } =
     usePaginatedDailyActivity(activityQueryOptions);
+  const readUnavailable = failed || cancelled;
+  const waitingForRange = activityQueryOptions.enabled && !coversRange && !readUnavailable;
 
   return {
     dateValue,
     onDateChange,
     results: data.results as DailyData[],
-    loading,
+    loading: loading || waitingForRange,
     isFetchingMore,
     progress,
     cancelled,
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx
index 0f1a44851c7..6a0e55a6dda 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx
@@ -1,7 +1,17 @@
-import { fireEvent, renderWithProviders as render, screen, waitFor } from "../../../../../../tests/test-utils";
+import {
+  act,
+  fireEvent,
+  renderWithProviders as render,
+  screen,
+  testQueryClient,
+  waitFor,
+} from "../../../../../../tests/test-utils";
 import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event";
-import { describe, expect, it, vi, beforeEach } from "vitest";
+import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
+import { Profiler } from "react";
 import UserInfoView from "./user_info_view";
+import type { DailyData, SpendMetrics } from "@/components/UsagePage/types";
+import type { AutoRouterBenchmarksResponse } from "@/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks";
 
 const mockTeamMemberAddCall = vi.fn();
 const mockTeamMemberDeleteCall = vi.fn();
@@ -11,6 +21,8 @@ const mockTeamInfoCall = vi.fn();
 const mockUserUpdateUserCall = vi.fn();
 const mockFetchMCPServers = vi.fn();
 const mockListMCPTools = vi.fn();
+const mockUserDailyActivityCall = vi.fn();
+const mockUserDailyActivityAggregatedCall = vi.fn();
 
 const MCP_SERVER = { server_id: "srv-1", server_name: "GitHub MCP", alias: "GitHub MCP" };
 
@@ -47,13 +59,18 @@ vi.mock("next/navigation", () => ({
   useSearchParams: () => new URLSearchParams(window.location.search),
 }));
 
-vi.mock("@/components/networking", () => {
+vi.mock("@/components/networking", async (importOriginal) => {
+  const original = await importOriginal<typeof import("@/components/networking")>();
   return {
+    formatDate: original.formatDate,
     serverRootPath: "/",
     userGetInfoV2: (...args: unknown[]) => mockUserGetInfoV2(...args),
+    userDailyActivityCall: (...args: unknown[]) => mockUserDailyActivityCall(...args),
+    userDailyActivityAggregatedCall: (...args: unknown[]) => mockUserDailyActivityAggregatedCall(...args),
     userDeleteCall: vi.fn(),
     userUpdateUserCall: (...args: unknown[]) => mockUserUpdateUserCall(...args),
     modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }),
+    modelInfoCall: vi.fn().mockResolvedValue({ data: [], total_pages: 1 }),
     invitationCreateCall: vi.fn(),
     teamInfoCall: (...args: unknown[]) => mockTeamInfoCall(...args),
     teamListCall: (...args: unknown[]) => mockTeamListCall(...args),
@@ -291,3 +308,337 @@ describe("UserInfoView add-to-team form", () => {
     expect(screen.getByText("Add User to Team")).toBeInTheDocument();
   });
 });
+
+const savingsDay = (date: string, metrics: Partial<SpendMetrics>): DailyData => ({
+  date,
+  metrics: {
+    spend: 0,
+    prompt_tokens: 0,
+    completion_tokens: 0,
+    total_tokens: 0,
+    api_requests: 1,
+    successful_requests: 1,
+    failed_requests: 0,
+    cache_read_input_tokens: 0,
+    cache_creation_input_tokens: 0,
+    ...metrics,
+  },
+  breakdown: { models: {}, model_groups: {}, mcp_servers: {}, providers: {}, api_keys: {}, entities: {} },
+});
+
+const savingsResponse = (results: DailyData[]) => ({
+  results,
+  metadata: { total_pages: 1, has_more: false, page: 1 },
+});
+
+const routerUsageResponse = (saved: number): AutoRouterBenchmarksResponse => ({
+  start_date: "2026-09-01",
+  end_date: "2026-09-19",
+  routers_in_scope: 0,
+  groups: [],
+  totals: {
+    sessions: 2,
+    turns: 2,
+    avg_turns_per_session: 1,
+    avg_session_seconds: 0,
+    avg_tokens_per_session: 100,
+    spend: 10,
+    savings_estimated_turns: 2,
+    savings_estimated_actual_spend: 10,
+    classifier_cost: 0,
+    saved_spend: saved,
+    baseline_spend: 10 + saved,
+    saved_pct: (100 * saved) / (10 + saved),
+    saved_per_session: saved / 2,
+    cache: {
+      coverage_pct: 100,
+      hit_rate_pct: 0,
+      same_model: { turns: 0, hits: 0, hit_rate_pct: 0 },
+      first_visit: { turns: 2, hits: 0, hit_rate_pct: 0 },
+      return_to_tier: { turns: 0, hits: 0, hit_rate_pct: 0 },
+      unordered_turns: 0,
+      return_misses_expired: 0,
+      return_misses_within_ttl: 0,
+      return_misses_unknown: 0,
+      ttl_5m_turns: 0,
+      ttl_1h_turns: 0,
+    },
+  },
+});
+
+describe("UserInfoView auto-router usage", () => {
+  const props = {
+    userId: "user-123",
+    onClose: vi.fn(),
+    accessToken: "admin-token",
+    userRole: "proxy_admin",
+    possibleUIRoles: null,
+  };
+  const mockFetch = vi.fn<typeof fetch>();
+
+  beforeEach(() => {
+    testQueryClient.clear();
+    vi.clearAllMocks();
+    mockUserGetInfoV2.mockImplementation((_token: string, userId: string) =>
+      Promise.resolve({ ...MOCK_USER_DATA_NO_TEAMS, user_id: userId }),
+    );
+    mockFetch.mockReset().mockResolvedValue(Response.json(routerUsageResponse(42)));
+    vi.stubGlobal("fetch", mockFetch);
+  });
+
+  afterEach(() => {
+    testQueryClient.clear();
+    vi.unstubAllGlobals();
+  });
+
+  it.each(["proxy_admin", "proxy_admin_viewer"])(
+    "loads selected-user usage lazily for %s without a key filter",
+    async (userRole) => {
+      const user = userEvent.setup();
+      render(<UserInfoView {...props} userRole={userRole} />);
+      const tab = await screen.findByRole("tab", { name: "Auto-router usage" });
+      expect(mockFetch).not.toHaveBeenCalled();
+      await user.click(tab);
+
+      expect(await screen.findByText("$42.00")).toBeInTheDocument();
+      const request = mockFetch.mock.calls[0][0] as Request;
+      const params = new URL(request.url).searchParams;
+      expect(params.get("user_id")).toBe("user-123");
+      expect(params.has("api_key")).toBe(false);
+      expect(screen.getByText(/Older sessions recorded without a user ID are not included/)).toBeInTheDocument();
+    },
+  );
+
+  it("switches query scope without displaying the previous user's usage", async () => {
+    const nextUser = Promise.withResolvers<Response>();
+    mockFetch.mockResolvedValueOnce(Response.json(routerUsageResponse(42))).mockReturnValue(nextUser.promise);
+    const user = userEvent.setup();
+    const { rerender } = render(<UserInfoView {...props} />);
+    await user.click(await screen.findByRole("tab", { name: "Auto-router usage" }));
+    expect(await screen.findByText("$42.00")).toBeInTheDocument();
+
+    rerender(<UserInfoView {...props} userId="user-456" />);
+    expect(screen.getByText("Loading auto-router usage...")).toBeInTheDocument();
+    expect(screen.queryByText("$42.00")).not.toBeInTheDocument();
+    await act(async () => nextUser.resolve(Response.json(routerUsageResponse(-7))));
+    expect(await screen.findByText("-$7.00")).toBeInTheDocument();
+    expect(
+      mockFetch.mock.calls.map(([request]) => new URL((request as Request).url).searchParams.get("user_id")),
+    ).toEqual(["user-123", "user-456"]);
+  });
+
+  it.each(["internal_user", "org_admin", null])("keeps the admin-only tab unavailable to %s", async (userRole) => {
+    render(<UserInfoView {...props} userRole={userRole} />);
+    await screen.findByRole("tab", { name: "Overview" });
+    expect(screen.queryByRole("tab", { name: "Auto-router usage" })).not.toBeInTheDocument();
+    expect(mockFetch).not.toHaveBeenCalled();
+  });
+
+  it("never turns an absent user ID into a deployment-wide request", async () => {
+    const user = userEvent.setup();
+    render(<UserInfoView {...props} userId="" />);
+    await user.click(await screen.findByRole("tab", { name: "Auto-router usage" }));
+    expect(screen.getByRole("alert")).toHaveTextContent("this user has no ID");
+    expect(mockFetch).not.toHaveBeenCalled();
+  });
+});
+
+describe("UserInfoView savings", () => {
+  const props = {
+    userId: "user-123",
+    onClose: vi.fn(),
+    accessToken: "admin-token",
+    userRole: "proxy_admin",
+    possibleUIRoles: null,
+  };
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockUserGetInfoV2.mockImplementation((_token: string, userId: string) =>
+      Promise.resolve({ ...MOCK_USER_DATA_NO_TEAMS, user_id: userId }),
+    );
+    mockUserDailyActivityAggregatedCall.mockReset().mockResolvedValue(savingsResponse([]));
+    mockUserDailyActivityCall.mockReset().mockResolvedValue(savingsResponse([]));
+  });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+  });
+
+  it.each(["internal_user", "org_admin", "team_admin"])(
+    "only offers self savings to %s and stops querying after switching to another user",
+    async (userRole) => {
+      const user = userEvent.setup();
+      const { rerender } = render(<UserInfoView {...props} userId="user-1" userRole={userRole} />);
+      await user.click(await screen.findByRole("tab", { name: "Savings" }));
+      expect(await screen.findByText("No usage recorded for this user in this range.")).toBeInTheDocument();
+      expect(mockUserDailyActivityAggregatedCall.mock.calls[0][3]).toBe("user-1");
+
+      mockUserDailyActivityAggregatedCall.mockClear();
+      mockUserDailyActivityCall.mockClear();
+      rerender(<UserInfoView {...props} userId="another-user" userRole={userRole} />);
+      await screen.findAllByText("another-user");
+      expect(screen.getByRole("tab", { name: "Overview" })).toHaveAttribute("aria-selected", "true");
+      expect(screen.queryByRole("tab", { name: "Savings" })).not.toBeInTheDocument();
+      expect(screen.queryByText("No usage recorded for this user in this range.")).not.toBeInTheDocument();
+      expect(mockUserDailyActivityAggregatedCall).not.toHaveBeenCalled();
+      expect(mockUserDailyActivityCall).not.toHaveBeenCalled();
+    },
+  );
+
+  it("loads selected user savings without a key filter, including losses", async () => {
+    const firstDay: Partial<SpendMetrics> = {
+      compression_savings_spend: 1.5,
+      gateway_injected_caching_savings_spend: 0.1,
+      prompt_caching_savings_spend: 0.25,
+      autorouter_savings_spend: -1,
+    };
+    const secondDay: Partial<SpendMetrics> = {
+      compression_savings_spend: 0.5,
+      gateway_injected_caching_savings_spend: 0.3,
+      prompt_caching_savings_spend: 0.75,
+      autorouter_savings_spend: -2,
+    };
+    mockUserDailyActivityAggregatedCall.mockResolvedValue(
+      savingsResponse([savingsDay("2026-09-18", firstDay), savingsDay("2026-09-19", secondDay)]),
+    );
+    const user = userEvent.setup();
+    render(<UserInfoView {...props} />);
+    const savingsTab = await screen.findByRole("tab", { name: "Savings" });
+    expect(mockUserDailyActivityAggregatedCall).not.toHaveBeenCalled();
+    expect(mockUserDailyActivityCall).not.toHaveBeenCalled();
+
+    await user.click(savingsTab);
+
+    expect(await screen.findByTestId("summary-card-total-recorded-savings")).toHaveTextContent("-$0.6000");
+    expect(screen.getByTestId("summary-card-compression-savings")).toHaveTextContent("$2.00");
+    expect(screen.getByTestId("summary-card-prompt-caching-savings")).toHaveTextContent("$0.4000");
+    expect(screen.getByTestId("summary-card-prompt-caching-savings")).toHaveTextContent("$1.00Total");
+    expect(screen.getByTestId("summary-card-auto-router-savings")).toHaveTextContent("-$3.00");
+    expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalledExactlyOnceWith(
+      "admin-token",
+      expect.any(Date),
+      expect.any(Date),
+      "user-123",
+      true,
+      null,
+    );
+    expect(screen.getByTestId("user-savings-scope-note")).toHaveTextContent("JWT-authenticated requests");
+    await user.click(screen.getByRole("tab", { name: "Per day" }));
+    expect(screen.getByRole("tab", { name: "Per day" })).toHaveAttribute("aria-selected", "true");
+    expect(screen.getByTestId("summary-card-total-recorded-savings")).toHaveTextContent("-$0.6000");
+  });
+
+  it("removes the prior user's savings while the newly selected user's results are loading", async () => {
+    const nextUser = Promise.withResolvers<ReturnType<typeof savingsResponse>>();
+    mockUserDailyActivityAggregatedCall
+      .mockResolvedValueOnce(savingsResponse([savingsDay("2026-09-19", { compression_savings_spend: 42 })]))
+      .mockReturnValueOnce(nextUser.promise);
+    const user = userEvent.setup();
+    const { rerender } = render(<UserInfoView {...props} />);
+    await user.click(await screen.findByRole("tab", { name: "Savings" }));
+    expect(await screen.findByTestId("summary-card-total-recorded-savings")).toHaveTextContent("$42.00");
+
+    rerender(<UserInfoView {...props} userId="user-456" />);
+
+    expect(await screen.findByTestId("user-savings-empty")).toHaveTextContent("Loading savings");
+    expect(screen.queryByTestId("summary-card-total-recorded-savings")).not.toBeInTheDocument();
+    expect(mockUserDailyActivityAggregatedCall).toHaveBeenLastCalledWith(
+      "admin-token",
+      expect.any(Date),
+      expect.any(Date),
+      "user-456",
+      true,
+      null,
+    );
+    await act(async () => {
+      nextUser.resolve(savingsResponse([savingsDay("2026-09-19", { autorouter_savings_spend: -7 })]));
+    });
+    expect(await screen.findByTestId("summary-card-total-recorded-savings")).toHaveTextContent("-$7.00");
+    expect(screen.queryByText("$42.00")).not.toBeInTheDocument();
+  });
+
+  it("never commits the previous range's savings under the newly selected dates", async () => {
+    vi.stubGlobal("requestIdleCallback", (callback: IdleRequestCallback) =>
+      window.setTimeout(() => callback({ didTimeout: false, timeRemaining: () => 0 }), 0),
+    );
+    const nextRange = Promise.withResolvers<ReturnType<typeof savingsResponse>>();
+    mockUserDailyActivityAggregatedCall
+      .mockResolvedValueOnce(savingsResponse([savingsDay("2026-09-19", { compression_savings_spend: 42 })]))
+      .mockReturnValue(nextRange.promise);
+    const committedTotals: Array<string | null> = [];
+    const captureNewRange = () => {
+      if (screen.queryByText("Running total saved · Sep 1 – Sep 2 (UTC)")) {
+        committedTotals.push(screen.queryByTestId("summary-card-total-recorded-savings")?.textContent ?? null);
+      }
+    };
+    const user = userEvent.setup();
+    render(
+      <Profiler id="user-savings" onRender={captureNewRange}>
+        <UserInfoView {...props} />
+      </Profiler>,
+    );
+    await user.click(await screen.findByRole("tab", { name: "Savings" }));
+    expect(await screen.findByTestId("summary-card-total-recorded-savings")).toHaveTextContent("$42.00");
+
+    await user.click(screen.getByRole("button", { name: / - / }));
+    const [startDateInput, endDateInput] = screen.getAllByDisplayValue(/^\d{4}-\d{2}-\d{2}$/);
+    fireEvent.change(startDateInput, { target: { value: "2026-09-01" } });
+    fireEvent.change(endDateInput, { target: { value: "2026-09-02" } });
+    await user.click(screen.getByRole("button", { name: "Apply" }));
+
+    expect(committedTotals.length).toBeGreaterThan(0);
+    expect(committedTotals.every((total) => total === null)).toBe(true);
+    expect(screen.getByTestId("user-savings-empty")).toHaveTextContent("Loading savings");
+    await act(async () => {
+      nextRange.resolve(savingsResponse([savingsDay("2026-09-02", { autorouter_savings_spend: -7 })]));
+    });
+    expect(await screen.findByTestId("summary-card-total-recorded-savings")).toHaveTextContent("-$7.00");
+  });
+
+  it("reports an incomplete paginated read as unavailable instead of displaying a partial savings total", async () => {
+    mockUserDailyActivityAggregatedCall.mockRejectedValue(new Error("aggregated unavailable"));
+    mockUserDailyActivityCall
+      .mockResolvedValueOnce({
+        results: [savingsDay("2026-09-19", { compression_savings_spend: 42 })],
+        metadata: { total_pages: 2, has_more: true, page: 1 },
+      })
+      .mockRejectedValueOnce(new Error("next page unavailable"));
+    const user = userEvent.setup();
+    render(<UserInfoView {...props} />);
+    await user.click(await screen.findByRole("tab", { name: "Savings" }));
+
+    expect(await screen.findByRole("alert")).toHaveTextContent("Savings are unavailable for this range");
+    expect(mockUserDailyActivityCall).toHaveBeenLastCalledWith(
+      "admin-token",
+      expect.any(Date),
+      expect.any(Date),
+      2,
+      "user-123",
+      true,
+      null,
+    );
+    expect(screen.queryByTestId("summary-card-total-recorded-savings")).not.toBeInTheDocument();
+    expect(screen.queryByText(/No usage recorded/)).not.toBeInTheDocument();
+  });
+
+  it("distinguishes a user with no usage from an unavailable read", async () => {
+    const user = userEvent.setup();
+    render(<UserInfoView {...props} />);
+    await user.click(await screen.findByRole("tab", { name: "Savings" }));
+
+    expect(await screen.findByTestId("user-savings-empty")).toHaveTextContent("No usage recorded for this user");
+    expect(screen.getByTestId("summary-card-total-recorded-savings")).toHaveTextContent("$0.00");
+    expect(screen.queryByRole("alert")).not.toBeInTheDocument();
+  });
+
+  it.each(["", "   "])("never queries an absent selected user ID (%j)", async (userId) => {
+    const user = userEvent.setup();
+    render(<UserInfoView {...props} userId={userId} />);
+    await user.click(await screen.findByRole("tab", { name: "Savings" }));
+
+    expect(screen.getByRole("alert")).toHaveTextContent("this user has no ID");
+    expect(mockUserDailyActivityAggregatedCall).not.toHaveBeenCalled();
+    expect(mockUserDailyActivityCall).not.toHaveBeenCalled();
+  });
+});
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx
index e083e549552..c95badc587a 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx
@@ -28,7 +28,7 @@ import {
   ComboboxList,
 } from "@/components/ui/combobox";
 import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
-import { rolesWithWriteAccess } from "@/utils/roles";
+import { hasProxyWideSpendView, rolesWithWriteAccess } from "@/utils/roles";
 import { teamDetailHref } from "@/utils/entityLinks";
 import { BadgeLink } from "@/components/shared/BadgeLink";
 import { UserEditView } from "../user_edit_view";
@@ -44,6 +44,9 @@ import { useMCPServers } from "@/app/(dashboard)/hooks/mcpServers/useMCPServers"
 import { useMCPToolsets } from "@/app/(dashboard)/hooks/mcpServers/useMCPToolsets";
 import { extractMcpEntitlement } from "@/components/mcp_server_management/mcpEntitlement";
 import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog";
+import ScopedSavingsTab from "@/components/shared/ScopedSavingsTab";
+import { AutoRouterUsageView } from "@/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab";
+import { useActivityDateRange } from "@/app/(dashboard)/cost-optimization/_components/useDailyActivityRange";
 
 interface UserInfoViewProps {
   userId: string;
@@ -85,7 +88,10 @@ export default function UserInfoView({
   initialTab = 0,
   startInEditMode = false,
 }: UserInfoViewProps) {
-  const { premiumUser } = useAuthorized();
+  const { premiumUser, userId: signedInUserId } = useAuthorized();
+  const canViewAutoRouterUsage = hasProxyWideSpendView(userRole);
+  const canViewSavings = canViewAutoRouterUsage || (Boolean(userId.trim()) && userId === signedInUserId);
+  const activityDateRange = useActivityDateRange();
   const [userData, setUserData] = useState<UserInfoV2Response | null>(null);
   const [teamDetails, setTeamDetails] = useState<TeamDisplayInfo[]>([]);
   const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false);
@@ -97,6 +103,8 @@ export default function UserInfoView({
   const [invitationLinkData, setInvitationLinkData] = useState<InvitationLink | null>(null);
   const [baseUrl, setBaseUrl] = useState<string | null>(null);
   const [activeTab, setActiveTab] = useState<string>(initialTab === 1 ? "details" : "overview");
+  const hiddenSavingsTab = activeTab === "savings" && !canViewSavings;
+  const hiddenRouterTab = activeTab === "auto-router-usage" && !canViewAutoRouterUsage;
   const [copiedStates, setCopiedStates] = useState<Record<string, boolean>>({});
   const [isTeamsExpanded, setIsTeamsExpanded] = useState(false);
   const [isAddTeamModalOpen, setIsAddTeamModalOpen] = useState(false);
@@ -467,7 +475,11 @@ export default function UserInfoView({
         confirmLoading={isDeletingUser}
       />
 
-      <Tabs value={activeTab} onValueChange={(v: unknown) => setActiveTab(String(v))} className="gap-0">
+      <Tabs
+        value={hiddenSavingsTab || hiddenRouterTab ? "overview" : activeTab}
+        onValueChange={(v: unknown) => setActiveTab(String(v))}
+        className="gap-0"
+      >
         <TabsList variant="line" className="mb-4">
           <TabsTrigger value="overview" className="flex-none data-active:text-primary after:bg-primary">
             Overview
@@ -475,6 +487,16 @@ export default function UserInfoView({
           <TabsTrigger value="details" className="flex-none data-active:text-primary after:bg-primary">
             Details
           </TabsTrigger>
+          {canViewSavings && (
+            <TabsTrigger value="savings" className="flex-none data-active:text-primary after:bg-primary">
+              Savings
+            </TabsTrigger>
+          )}
+          {canViewAutoRouterUsage && (
+            <TabsTrigger value="auto-router-usage" className="flex-none data-active:text-primary after:bg-primary">
+              Auto-router usage
+            </TabsTrigger>
+          )}
         </TabsList>
 
         {/* Overview Panel */}
@@ -685,6 +707,38 @@ export default function UserInfoView({
             )}
           </Card>
         </TabsContent>
+        {canViewSavings && (
+          <TabsContent value="savings">
+            {activeTab === "savings" &&
+              (userId.trim() ? (
+                <ScopedSavingsTab
+                  key={userId}
+                  accessToken={accessToken}
+                  scope={{ userId }}
+                  activity={activityDateRange}
+                  entityType="user"
+                  scopeNote="Savings for this user across API keys and JWT-authenticated requests."
+                />
+              ) : (
+                <p role="alert">Savings are unavailable because this user has no ID.</p>
+              ))}
+          </TabsContent>
+        )}
+        {canViewAutoRouterUsage && (
+          <TabsContent value="auto-router-usage">
+            {activeTab === "auto-router-usage" &&
+              (userId.trim() ? (
+                <AutoRouterUsageView
+                  key={userId}
+                  accessToken={accessToken}
+                  userId={userId}
+                  activity={activityDateRange}
+                />
+              ) : (
+                <p role="alert">Auto-router usage is unavailable because this user has no ID.</p>
+              ))}
+          </TabsContent>
+        )}
       </Tabs>
       <OnboardingModal
         isInvitationLinkModalVisible={isInvitationLinkModalVisible}
diff --git a/ui/litellm-dashboard/src/components/shared/ScopedSavingsTab.tsx b/ui/litellm-dashboard/src/components/shared/ScopedSavingsTab.tsx
new file mode 100644
index 00000000000..7a0455f9d7e
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/shared/ScopedSavingsTab.tsx
@@ -0,0 +1,133 @@
+"use client";
+
+import { useMemo, useState } from "react";
+
+import { AreaChart, BarChart, CustomLegend } from "@/components/shared/charts";
+import AdvancedDatePicker from "@/components/shared/advanced_date_picker";
+import SavingsTiles from "@/components/shared/SavingsTiles";
+import { Card, CardAction, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
+import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
+import {
+  formatRangeLabel,
+  localIsoDay,
+  MAX_POINTS_WITH_DOTS,
+  SAVINGS_COLORS,
+  SAVINGS_SERIES,
+  SavingsAccumulation,
+  SavingsPoint,
+  savingsSeriesOf,
+  shortDate,
+  toCumulative,
+  usd,
+  withStartAnchor,
+} from "@/app/(dashboard)/cost-optimization/_components/costOptimizationUtils";
+import {
+  useScopedDailyActivityRange,
+  type ActivityDateRange,
+  type DailyActivityScope,
+} from "@/app/(dashboard)/cost-optimization/_components/useDailyActivityRange";
+
+interface ScopedSavingsTabProps {
+  accessToken: string | null;
+  scope: DailyActivityScope;
+  activity: ActivityDateRange;
+  entityType: "key" | "user";
+  scopeNote?: string;
+}
+
+const ScopedSavingsTab = ({ accessToken, scope, activity, entityType, scopeNote }: ScopedSavingsTabProps) => {
+  const { dateValue, onDateChange, results, loading, isFetchingMore, failed, cancelled } = useScopedDailyActivityRange(
+    accessToken,
+    scope,
+    activity,
+  );
+  const startTime = dateValue.from;
+  const endTime = dateValue.to;
+
+  const [accumulation, setAccumulation] = useState<SavingsAccumulation>("cumulative");
+
+  const perInterval = useMemo<SavingsPoint[]>(() => savingsSeriesOf(results), [results]);
+
+  const overTime = useMemo(() => {
+    if (accumulation !== "cumulative") return perInterval;
+    const startLabel = startTime ? shortDate(localIsoDay(startTime)) : "";
+    return withStartAnchor(toCumulative(perInterval), startLabel);
+  }, [accumulation, perInterval, startTime]);
+
+  const intervalLabel = "Per day";
+  const rangeLabel = formatRangeLabel(startTime, endTime);
+  const savingsSubtitle = [
+    accumulation === "cumulative" ? "Running total saved" : `Saved ${intervalLabel.toLowerCase()}`,
+    rangeLabel && `${rangeLabel} (UTC)`,
+  ]
+    .filter(Boolean)
+    .join(" · ");
+
+  const isLoading = loading || isFetchingMore;
+  const unavailable = failed || cancelled;
+  const showResults = !isLoading && !unavailable;
+  const hasRows = results.length > 0;
+  const showEmpty = !unavailable && (isLoading || !hasRows);
+  const showChart = showResults && hasRows;
+  const chartProps = {
+    data: overTime,
+    index: "date",
+    categories: SAVINGS_SERIES,
+    colors: SAVINGS_COLORS,
+    valueFormatter: usd,
+    showLegend: false,
+  };
+
+  return (
+    <div className="w-full space-y-6">
+      <div className="flex flex-wrap items-center justify-end gap-4">
+        <span className="text-sm text-muted-foreground">Spend is bucketed by UTC day</span>
+        <AdvancedDatePicker value={dateValue} onValueChange={onDateChange} />
+      </div>
+
+      {scopeNote && (
+        <p className="text-sm text-muted-foreground" data-testid={`${entityType}-savings-scope-note`}>
+          {scopeNote}
+        </p>
+      )}
+
+      {unavailable && (
+        <p role="alert" className="text-sm text-muted-foreground">
+          Savings are unavailable for this range. Try another date range or reopen this tab.
+        </p>
+      )}
+      {showResults && <SavingsTiles results={results} isLoading={false} />}
+
+      <Card>
+        <CardHeader>
+          <CardTitle>Savings</CardTitle>
+          <CardDescription>{savingsSubtitle}</CardDescription>
+          <CardAction className="flex flex-wrap items-center justify-end gap-x-4 gap-y-2">
+            <CustomLegend categories={SAVINGS_SERIES} colors={SAVINGS_COLORS} />
+            <Tabs value={accumulation} onValueChange={(value) => setAccumulation(value as SavingsAccumulation)}>
+              <TabsList>
+                <TabsTrigger value="cumulative">Cumulative</TabsTrigger>
+                <TabsTrigger value="per-interval">{intervalLabel}</TabsTrigger>
+              </TabsList>
+            </Tabs>
+          </CardAction>
+        </CardHeader>
+        <CardContent>
+          {showEmpty && (
+            <p className="py-12 text-center text-sm text-muted-foreground" data-testid={`${entityType}-savings-empty`}>
+              {isLoading ? "Loading savings..." : `No usage recorded for this ${entityType} in this range.`}
+            </p>
+          )}
+          {showChart &&
+            (accumulation === "cumulative" ? (
+              <AreaChart {...chartProps} showDots={overTime.length <= MAX_POINTS_WITH_DOTS} />
+            ) : (
+              <BarChart {...chartProps} />
+            ))}
+        </CardContent>
+      </Card>
+    </div>
+  );
+};
+
+export default ScopedSavingsTab;
diff --git a/ui/litellm-dashboard/src/components/templates/KeySavingsTab.tsx b/ui/litellm-dashboard/src/components/templates/KeySavingsTab.tsx
index c33529eb042..d1395e0be51 100644
--- a/ui/litellm-dashboard/src/components/templates/KeySavingsTab.tsx
+++ b/ui/litellm-dashboard/src/components/templates/KeySavingsTab.tsx
@@ -1,132 +1,29 @@
 "use client";
 
-import React, { useMemo, useState } from "react";
-
-import { AreaChart, BarChart, CustomLegend } from "@/components/shared/charts";
-import AdvancedDatePicker from "@/components/shared/advanced_date_picker";
-import SavingsTiles from "@/components/shared/SavingsTiles";
-import { Card, CardAction, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
-import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
+import ScopedSavingsTab from "@/components/shared/ScopedSavingsTab";
 import { hasProxyWideSpendView, spendScopeUserId } from "@/utils/roles";
-import {
-  formatRangeLabel,
-  localIsoDay,
-  MAX_POINTS_WITH_DOTS,
-  SAVINGS_COLORS,
-  SAVINGS_SERIES,
-  SavingsAccumulation,
-  SavingsPoint,
-  savingsSeriesOf,
-  shortDate,
-  toCumulative,
-  usd,
-  withStartAnchor,
-} from "@/app/(dashboard)/cost-optimization/_components/costOptimizationUtils";
-import {
-  useScopedDailyActivityRange,
-  type ActivityDateRange,
-} from "@/app/(dashboard)/cost-optimization/_components/useDailyActivityRange";
+import type { ActivityDateRange } from "@/app/(dashboard)/cost-optimization/_components/useDailyActivityRange";
 
 interface KeySavingsTabProps {
   accessToken: string | null;
-  /** The key's token hash — what spend rows are keyed by, not the one-time plaintext secret. */
   keyToken: string;
   userId: string | null;
   userRole: string;
   activity: ActivityDateRange;
 }
 
-const KeySavingsTab: React.FC<KeySavingsTabProps> = ({ accessToken, keyToken, userId, userRole, activity }) => {
-  // Proxy admins read the whole key. For anyone else the endpoint applies the caller's own user_id
-  // alongside the key filter, so the figures cover only that viewer's requests on this key -- said
-  // plainly in the scope note below rather than left to be misread as the key's total.
-  const readsWholeKey = hasProxyWideSpendView(userRole);
-  const { dateValue, onDateChange, results, loading, isFetchingMore } = useScopedDailyActivityRange(
-    accessToken,
-    { userId: spendScopeUserId(userRole, userId), apiKey: keyToken },
-    activity,
-  );
-  const startTime = dateValue.from ?? null;
-  const endTime = dateValue.to ?? null;
-
-  const [accumulation, setAccumulation] = useState<SavingsAccumulation>("cumulative");
-
-  const perInterval = useMemo<SavingsPoint[]>(() => savingsSeriesOf(results), [results]);
-
-  const overTime = useMemo(() => {
-    if (accumulation !== "cumulative") return perInterval;
-    const startLabel = startTime ? shortDate(localIsoDay(startTime)) : "";
-    return withStartAnchor(toCumulative(perInterval), startLabel);
-  }, [accumulation, perInterval, startTime]);
-
-  const intervalLabel = "Per day";
-  const rangeLabel = formatRangeLabel(startTime ?? undefined, endTime ?? undefined);
-  const savingsSubtitle = [
-    accumulation === "cumulative" ? "Running total saved" : `Saved ${intervalLabel.toLowerCase()}`,
-    rangeLabel && `${rangeLabel} (UTC)`,
-  ]
-    .filter(Boolean)
-    .join(" · ");
-
-  const isLoading = loading || isFetchingMore;
-  const hasRows = results.length > 0;
-  const chartProps = {
-    data: overTime,
-    index: "date",
-    categories: SAVINGS_SERIES,
-    colors: SAVINGS_COLORS,
-    valueFormatter: usd,
-    showLegend: false,
-  };
-
-  return (
-    <div className="w-full space-y-6">
-      <div className="flex flex-wrap items-center justify-end gap-4">
-        <span className="text-sm text-muted-foreground">Spend is bucketed by UTC day</span>
-        <AdvancedDatePicker value={dateValue} onValueChange={onDateChange} />
-      </div>
-
-      {!readsWholeKey && (
-        <p className="text-sm text-muted-foreground" data-testid="key-savings-scope-note">
-          Showing your own requests on this key. A key shared across a team will have spend from other members that is
-          not counted here.
-        </p>
-      )}
-
-      <SavingsTiles results={results} isLoading={isLoading} />
-
-      <Card>
-        <CardHeader>
-          <CardTitle>Savings</CardTitle>
-          <CardDescription>{savingsSubtitle}</CardDescription>
-          <CardAction className="flex flex-wrap items-center justify-end gap-x-4 gap-y-2">
-            <CustomLegend categories={SAVINGS_SERIES} colors={SAVINGS_COLORS} />
-            <Tabs value={accumulation} onValueChange={(value) => setAccumulation(value as SavingsAccumulation)}>
-              <TabsList>
-                <TabsTrigger value="cumulative">Cumulative</TabsTrigger>
-                <TabsTrigger value="per-interval">{intervalLabel}</TabsTrigger>
-              </TabsList>
-            </Tabs>
-          </CardAction>
-        </CardHeader>
-        <CardContent>
-          {/* Distinguishes "still fetching" from "this key genuinely had no traffic": an empty
-              chart alone reads as a broken panel, and a $0.00 tile reads as a real zero. */}
-          {!hasRows && (
-            <p className="py-12 text-center text-sm text-muted-foreground" data-testid="key-savings-empty">
-              {isLoading ? "Loading savings..." : "No usage recorded for this key in this range."}
-            </p>
-          )}
-          {hasRows && accumulation === "cumulative" && (
-            <AreaChart {...chartProps} showDots={overTime.length <= MAX_POINTS_WITH_DOTS} />
-          )}
-          {/* Not stacked: auto-router can go negative on a cold-cache write, and stacking would
-              draw that below the axis while the rest of the bar still read as the total. */}
-          {hasRows && accumulation !== "cumulative" && <BarChart {...chartProps} />}
-        </CardContent>
-      </Card>
-    </div>
-  );
-};
+const KeySavingsTab = ({ accessToken, keyToken, userId, userRole, activity }: KeySavingsTabProps) => (
+  <ScopedSavingsTab
+    accessToken={accessToken}
+    scope={{ userId: spendScopeUserId(userRole, userId), apiKey: keyToken }}
+    activity={activity}
+    entityType="key"
+    scopeNote={
+      hasProxyWideSpendView(userRole)
+        ? undefined
+        : "Showing your own requests on this key. A key shared across a team will have spend from other members that is not counted here."
+    }
+  />
+);
 
 export default KeySavingsTab;
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 81580c8bfb1..0055ef4337c 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -1246,9 +1246,10 @@ export interface paths {
          * @description Benchmarks for the auto-router dashboard: session shape, savings against the configured
          *     baseline, and prompt-caching behaviour bucketed by what the router did.
          *
-         *     Reads the LiteLLM_AutoRouterSession rollup, folded once per request at spend-write time,
-         *     so this endpoint never scans LiteLLM_SpendLogs. A session is in the window when it
-         *     overlaps it: its last turn is on or after start_date and its first turn is on or before
+         *     Reads session rollups folded once per request at spend-write time, so this endpoint
+         *     never scans LiteLLM_SpendLogs. A user filter selects only turns attributed to that
+         *     internal user when written; older key-only history remains outside user views. A session
+         *     is in the window when it overlaps it: its last turn is on or after start_date and its first turn is on or before
          *     end_date. Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is
          *     over that bucket's turns.
          *
@@ -43601,6 +43602,8 @@ export interface operations {
                 end_date?: string | null;
                 /** @description Filter to one virtual key token hash */
                 api_key?: string | null;
+                /** @description Filter to one canonical internal user recorded on each turn */
+                user_id?: string | null;
             };
             header?: never;
             path?: never;

From 39a14f39e5961605ba70deacc0475892f50d3cb7 Mon Sep 17 00:00:00 2001
From: yucheng <yucheng@berri.ai>
Date: Sun, 20 Sep 2026 08:20:47 +0000
Subject: [PATCH 035/160] style(proxy): format cleanup shutdown tests

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../proxy/shutdown/test_scheduled_jobs.py     |   4 +-
 .../proxy/test_spend_log_cleanup.py           | 127 +++++-------------
 2 files changed, 37 insertions(+), 94 deletions(-)

diff --git a/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py b/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
index 7defd6cef6c..87adc464608 100644
--- a/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
+++ b/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
@@ -7,11 +7,11 @@ from datetime import datetime, timedelta
 import pytest
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 
-import litellm.proxy.shutdown.scheduled_jobs as scheduled_jobs
+from litellm.proxy.shutdown import scheduled_jobs
 from litellm.proxy.shutdown.scheduled_jobs import (
     AwaitableAsyncIOExecutor,
-    stop_in_flight_scheduler_jobs,
     pause_scheduled_jobs,
+    stop_in_flight_scheduler_jobs,
 )
 
 
diff --git a/tests/test_litellm/proxy/test_spend_log_cleanup.py b/tests/test_litellm/proxy/test_spend_log_cleanup.py
index 1691b2d174a..b8f28d0c780 100644
--- a/tests/test_litellm/proxy/test_spend_log_cleanup.py
+++ b/tests/test_litellm/proxy/test_spend_log_cleanup.py
@@ -83,10 +83,10 @@ def test_spend_log_cleanup_cron_scheduling():
     assert trigger_weekly is not None
 
     # Invalid cron expression should raise ValueError
-    with pytest.raises(ValueError, match='Wrong number of fields; got'):
+    with pytest.raises(ValueError, match="Wrong number of fields; got"):
         CronTrigger.from_crontab("invalid cron")
 
-    with pytest.raises(ValueError, match='is higher than the maximum value'):
+    with pytest.raises(ValueError, match="is higher than the maximum value"):
         CronTrigger.from_crontab("60 25 * * *")  # Invalid minute and hour
 
 
@@ -99,6 +99,7 @@ def test_spend_log_cleanup_cron_scheduler_integration():
     a real database connection.
     """
     from unittest.mock import MagicMock
+
     from apscheduler.triggers.cron import CronTrigger
 
     # Mock scheduler
@@ -145,15 +146,11 @@ def test_spend_log_cleanup_cron_scheduler_integration():
         # No cron, so it should fall back to interval
     }
 
-    cleanup_cron_fallback = general_settings_interval.get(
-        "maximum_spend_logs_cleanup_cron"
-    )
+    cleanup_cron_fallback = general_settings_interval.get("maximum_spend_logs_cleanup_cron")
     assert cleanup_cron_fallback is None  # No cron configured
 
     # Simulate interval-based scheduling fallback
-    retention_interval = general_settings_interval.get(
-        "maximum_spend_logs_retention_interval", "1d"
-    )
+    retention_interval = general_settings_interval.get("maximum_spend_logs_retention_interval", "1d")
     from litellm.litellm_core_utils.duration_parser import duration_in_seconds
 
     interval_seconds = duration_in_seconds(retention_interval)
@@ -181,27 +178,19 @@ async def test_should_delete_spend_logs():
     assert cleaner._should_delete_spend_logs() is False
 
     # Test case 2: Valid seconds string
-    cleaner = SpendLogCleanup(
-        general_settings={"maximum_spend_logs_retention_period": "3600s"}
-    )
+    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "3600s"})
     assert cleaner._should_delete_spend_logs() is True
 
     # Test case 3: Valid days string
-    cleaner = SpendLogCleanup(
-        general_settings={"maximum_spend_logs_retention_period": "30d"}
-    )
+    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "30d"})
     assert cleaner._should_delete_spend_logs() is True
 
     # Test case 4: Valid hours string
-    cleaner = SpendLogCleanup(
-        general_settings={"maximum_spend_logs_retention_period": "24h"}
-    )
+    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "24h"})
     assert cleaner._should_delete_spend_logs() is True
 
     # Test case 5: Invalid format
-    cleaner = SpendLogCleanup(
-        general_settings={"maximum_spend_logs_retention_period": "invalid"}
-    )
+    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "invalid"})
     assert cleaner._should_delete_spend_logs() is False
 
 
@@ -288,9 +277,7 @@ async def test_cleanup_old_spend_logs_retention_period_cutoff():
     # Verify the cutoff date is correct
     cutoff_date = mock_db.execute_raw.call_args[0][1]
     expected_cutoff = datetime.now(timezone.utc) - timedelta(seconds=86400)
-    assert (
-        abs((cutoff_date - expected_cutoff).total_seconds()) < 1
-    )  # Allow 1 second difference for test execution time
+    assert abs((cutoff_date - expected_cutoff).total_seconds()) < 1  # Allow 1 second difference for test execution time
 
 
 @pytest.mark.asyncio
@@ -310,9 +297,7 @@ async def test_cleanup_drops_partitions_when_enabled_and_partitioned():
     partition_manager = MagicMock()
     partition_manager.is_partitioned = AsyncMock(return_value=True)
     partition_manager.ensure_partitions = AsyncMock(return_value=["p1"])
-    partition_manager.drop_partitions_older_than = AsyncMock(
-        return_value=["LiteLLM_SpendLogs_p20260601"]
-    )
+    partition_manager.drop_partitions_older_than = AsyncMock(return_value=["LiteLLM_SpendLogs_p20260601"])
 
     cleaner = SpendLogCleanup(
         general_settings={
@@ -450,9 +435,7 @@ async def test_integer_retention_treated_as_days():
     An integer value for maximum_spend_logs_retention_period should be treated
     as days (e.g., 3 → '3d' → 259200 seconds).
     """
-    cleaner = SpendLogCleanup(
-        general_settings={"maximum_spend_logs_retention_period": 3}
-    )
+    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": 3})
     result = cleaner._should_delete_spend_logs()
     assert result is True
     assert cleaner.retention_seconds == 3 * 86400  # 3 days in seconds
@@ -469,13 +452,11 @@ def test_string_retention_still_works():
         ("2w", 2 * 604800),
     ]
     for setting, expected_seconds in cases:
-        cleaner = SpendLogCleanup(
-            general_settings={"maximum_spend_logs_retention_period": setting}
-        )
+        cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": setting})
         assert cleaner._should_delete_spend_logs() is True, f"Failed for {setting}"
-        assert (
-            cleaner.retention_seconds == expected_seconds
-        ), f"Expected {expected_seconds} for {setting}, got {cleaner.retention_seconds}"
+        assert cleaner.retention_seconds == expected_seconds, (
+            f"Expected {expected_seconds} for {setting}, got {cleaner.retention_seconds}"
+        )
 
 
 @pytest.mark.asyncio
@@ -489,9 +470,7 @@ async def test_delete_old_logs_aborts_on_non_int_execute_raw_return():
     mock_db.execute_raw = AsyncMock(return_value=None)
     mock_prisma_client.db = mock_db
 
-    cleaner = SpendLogCleanup(
-        general_settings={"maximum_spend_logs_retention_period": "7d"}
-    )
+    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
 
     cutoff_date = datetime.now(timezone.utc) - timedelta(days=7)
     result = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date, _far_deadline())
@@ -510,9 +489,7 @@ async def test_delete_old_logs_continues_on_valid_int_return():
     mock_db.execute_raw = AsyncMock(side_effect=[500, 300, 0])
     mock_prisma_client.db = mock_db
 
-    cleaner = SpendLogCleanup(
-        general_settings={"maximum_spend_logs_retention_period": "7d"}
-    )
+    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
 
     cutoff_date = datetime.now(timezone.utc) - timedelta(days=7)
     result = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date, _far_deadline())
@@ -559,9 +536,7 @@ async def test_delete_old_tool_index_rows_deletes_on_composite_key():
     mock_db.execute_raw = AsyncMock(side_effect=[5, 0])
     mock_prisma_client.db = mock_db
 
-    cleaner = SpendLogCleanup(
-        general_settings={"maximum_spend_logs_retention_period": "7d"}
-    )
+    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
 
     cutoff_date = datetime.now(timezone.utc) - timedelta(days=7)
     result = await cleaner._delete_old_tool_index_rows(mock_prisma_client, cutoff_date, _far_deadline())
@@ -581,9 +556,7 @@ async def test_delete_old_logs_continues_after_single_batch_failure(monkeypatch)
     import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module
 
     # Zero out the failure backoff so the test doesn't take ~0.5s of real sleep.
-    monkeypatch.setattr(
-        cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0
-    )
+    monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0)
 
     mock_prisma_client = MagicMock()
     _wire_tx(mock_prisma_client.db)
@@ -591,14 +564,10 @@ async def test_delete_old_logs_continues_after_single_batch_failure(monkeypatch)
     _wire_tx(mock_db)
     # batch 1 succeeds, batch 2 raises (one-off DB timeout), batches 3-4 succeed,
     # batch 5 returns 0 → loop exits naturally.
-    mock_db.execute_raw = AsyncMock(
-        side_effect=[100, TimeoutError("simulated DB timeout"), 200, 50, 0]
-    )
+    mock_db.execute_raw = AsyncMock(side_effect=[100, TimeoutError("simulated DB timeout"), 200, 50, 0])
     mock_prisma_client.db = mock_db
 
-    cleaner = cleanup_module.SpendLogCleanup(
-        general_settings={"maximum_spend_logs_retention_period": "7d"}
-    )
+    cleaner = cleanup_module.SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
 
     cutoff_date = datetime.now(timezone.utc) - timedelta(days=7)
     result = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date, _far_deadline())
@@ -615,26 +584,18 @@ async def test_delete_old_logs_aborts_after_consecutive_failures(monkeypatch):
     import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module
 
     # Lower the threshold so the test is fast and deterministic.
-    monkeypatch.setattr(
-        cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3
-    )
-    monkeypatch.setattr(
-        cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0
-    )
+    monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3)
+    monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0)
 
     mock_prisma_client = MagicMock()
     _wire_tx(mock_prisma_client.db)
     mock_db = MagicMock()
     _wire_tx(mock_db)
     # Every batch raises — must abort after exactly 3 attempts, not loop forever.
-    mock_db.execute_raw = AsyncMock(
-        side_effect=ConnectionError("simulated persistent DB outage")
-    )
+    mock_db.execute_raw = AsyncMock(side_effect=ConnectionError("simulated persistent DB outage"))
     mock_prisma_client.db = mock_db
 
-    cleaner = cleanup_module.SpendLogCleanup(
-        general_settings={"maximum_spend_logs_retention_period": "7d"}
-    )
+    cleaner = cleanup_module.SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
 
     cutoff_date = datetime.now(timezone.utc) - timedelta(days=7)
     result = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date, _far_deadline())
@@ -649,12 +610,8 @@ async def test_delete_old_logs_resets_consecutive_failures_on_success(monkeypatc
     intermittent timeouts don't trip the abort threshold."""
     import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module
 
-    monkeypatch.setattr(
-        cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3
-    )
-    monkeypatch.setattr(
-        cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0
-    )
+    monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3)
+    monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0)
 
     mock_prisma_client = MagicMock()
     _wire_tx(mock_prisma_client.db)
@@ -675,9 +632,7 @@ async def test_delete_old_logs_resets_consecutive_failures_on_success(monkeypatc
     )
     mock_prisma_client.db = mock_db
 
-    cleaner = cleanup_module.SpendLogCleanup(
-        general_settings={"maximum_spend_logs_retention_period": "7d"}
-    )
+    cleaner = cleanup_module.SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
 
     cutoff_date = datetime.now(timezone.utc) - timedelta(days=7)
     result = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date, _far_deadline())
@@ -698,9 +653,7 @@ async def test_cleanup_uses_logger_exception_for_full_traceback(monkeypatch):
     mock_prisma_client = MagicMock()
     _wire_tx(mock_prisma_client.db)
     # Force the outer try/except to fire by making _should_delete_spend_logs raise.
-    cleaner = cleanup_module.SpendLogCleanup(
-        general_settings={"maximum_spend_logs_retention_period": "7d"}
-    )
+    cleaner = cleanup_module.SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
     cleaner.pod_lock_manager = None
 
     def boom():
@@ -725,12 +678,8 @@ async def test_cleanup_releases_lock_after_persistent_batch_failures(monkeypatch
     must still be released so the next scheduled run isn't permanently blocked."""
     import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module
 
-    monkeypatch.setattr(
-        cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 2
-    )
-    monkeypatch.setattr(
-        cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0
-    )
+    monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 2)
+    monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0)
 
     mock_prisma_client = MagicMock()
     _wire_tx(mock_prisma_client.db)
@@ -744,9 +693,7 @@ async def test_cleanup_releases_lock_after_persistent_batch_failures(monkeypatch
     mock_pod_lock_manager.acquire_lock = AsyncMock(return_value=True)
     mock_pod_lock_manager.release_lock = AsyncMock()
 
-    cleaner = cleanup_module.SpendLogCleanup(
-        general_settings={"maximum_spend_logs_retention_period": "7d"}
-    )
+    cleaner = cleanup_module.SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
     cleaner.pod_lock_manager = mock_pod_lock_manager
 
     await cleaner.cleanup_old_spend_logs(mock_prisma_client)
@@ -996,9 +943,7 @@ async def test_each_batch_carries_a_statement_and_lock_timeout():
         }
     )
 
-    await cleaner._delete_old_logs(
-        mock_prisma_client, datetime.now(timezone.utc) - timedelta(days=7), _far_deadline()
-    )
+    await cleaner._delete_old_logs(mock_prisma_client, datetime.now(timezone.utc) - timedelta(days=7), _far_deadline())
 
     assert "SET LOCAL statement_timeout = 12000" in recorded
     assert "SET LOCAL lock_timeout = 12000" in recorded
@@ -1134,9 +1079,7 @@ async def test_remaining_rows_probe_is_capped_so_it_cannot_scan_the_table():
 
     cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
 
-    await cleaner._delete_old_logs(
-        mock_prisma_client, datetime.now(timezone.utc) - timedelta(days=7), _far_deadline()
-    )
+    await cleaner._delete_old_logs(mock_prisma_client, datetime.now(timezone.utc) - timedelta(days=7), _far_deadline())
 
     count_sql = mock_db.query_raw.call_args[0][0]
     assert "count(*)" in count_sql

From 0b2d52edc29a7098e95314fbf2c45e58b508efc5 Mon Sep 17 00:00:00 2001
From: yucheng <yucheng@berri.ai>
Date: Sun, 20 Sep 2026 08:21:35 +0000
Subject: [PATCH 036/160] Revert "style(proxy): format cleanup shutdown tests"

This reverts commit 39a14f39e5961605ba70deacc0475892f50d3cb7.
---
 .../proxy/shutdown/test_scheduled_jobs.py     |   4 +-
 .../proxy/test_spend_log_cleanup.py           | 127 +++++++++++++-----
 2 files changed, 94 insertions(+), 37 deletions(-)

diff --git a/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py b/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
index 87adc464608..7defd6cef6c 100644
--- a/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
+++ b/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
@@ -7,11 +7,11 @@ from datetime import datetime, timedelta
 import pytest
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 
-from litellm.proxy.shutdown import scheduled_jobs
+import litellm.proxy.shutdown.scheduled_jobs as scheduled_jobs
 from litellm.proxy.shutdown.scheduled_jobs import (
     AwaitableAsyncIOExecutor,
-    pause_scheduled_jobs,
     stop_in_flight_scheduler_jobs,
+    pause_scheduled_jobs,
 )
 
 
diff --git a/tests/test_litellm/proxy/test_spend_log_cleanup.py b/tests/test_litellm/proxy/test_spend_log_cleanup.py
index b8f28d0c780..1691b2d174a 100644
--- a/tests/test_litellm/proxy/test_spend_log_cleanup.py
+++ b/tests/test_litellm/proxy/test_spend_log_cleanup.py
@@ -83,10 +83,10 @@ def test_spend_log_cleanup_cron_scheduling():
     assert trigger_weekly is not None
 
     # Invalid cron expression should raise ValueError
-    with pytest.raises(ValueError, match="Wrong number of fields; got"):
+    with pytest.raises(ValueError, match='Wrong number of fields; got'):
         CronTrigger.from_crontab("invalid cron")
 
-    with pytest.raises(ValueError, match="is higher than the maximum value"):
+    with pytest.raises(ValueError, match='is higher than the maximum value'):
         CronTrigger.from_crontab("60 25 * * *")  # Invalid minute and hour
 
 
@@ -99,7 +99,6 @@ def test_spend_log_cleanup_cron_scheduler_integration():
     a real database connection.
     """
     from unittest.mock import MagicMock
-
     from apscheduler.triggers.cron import CronTrigger
 
     # Mock scheduler
@@ -146,11 +145,15 @@ def test_spend_log_cleanup_cron_scheduler_integration():
         # No cron, so it should fall back to interval
     }
 
-    cleanup_cron_fallback = general_settings_interval.get("maximum_spend_logs_cleanup_cron")
+    cleanup_cron_fallback = general_settings_interval.get(
+        "maximum_spend_logs_cleanup_cron"
+    )
     assert cleanup_cron_fallback is None  # No cron configured
 
     # Simulate interval-based scheduling fallback
-    retention_interval = general_settings_interval.get("maximum_spend_logs_retention_interval", "1d")
+    retention_interval = general_settings_interval.get(
+        "maximum_spend_logs_retention_interval", "1d"
+    )
     from litellm.litellm_core_utils.duration_parser import duration_in_seconds
 
     interval_seconds = duration_in_seconds(retention_interval)
@@ -178,19 +181,27 @@ async def test_should_delete_spend_logs():
     assert cleaner._should_delete_spend_logs() is False
 
     # Test case 2: Valid seconds string
-    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "3600s"})
+    cleaner = SpendLogCleanup(
+        general_settings={"maximum_spend_logs_retention_period": "3600s"}
+    )
     assert cleaner._should_delete_spend_logs() is True
 
     # Test case 3: Valid days string
-    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "30d"})
+    cleaner = SpendLogCleanup(
+        general_settings={"maximum_spend_logs_retention_period": "30d"}
+    )
     assert cleaner._should_delete_spend_logs() is True
 
     # Test case 4: Valid hours string
-    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "24h"})
+    cleaner = SpendLogCleanup(
+        general_settings={"maximum_spend_logs_retention_period": "24h"}
+    )
     assert cleaner._should_delete_spend_logs() is True
 
     # Test case 5: Invalid format
-    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "invalid"})
+    cleaner = SpendLogCleanup(
+        general_settings={"maximum_spend_logs_retention_period": "invalid"}
+    )
     assert cleaner._should_delete_spend_logs() is False
 
 
@@ -277,7 +288,9 @@ async def test_cleanup_old_spend_logs_retention_period_cutoff():
     # Verify the cutoff date is correct
     cutoff_date = mock_db.execute_raw.call_args[0][1]
     expected_cutoff = datetime.now(timezone.utc) - timedelta(seconds=86400)
-    assert abs((cutoff_date - expected_cutoff).total_seconds()) < 1  # Allow 1 second difference for test execution time
+    assert (
+        abs((cutoff_date - expected_cutoff).total_seconds()) < 1
+    )  # Allow 1 second difference for test execution time
 
 
 @pytest.mark.asyncio
@@ -297,7 +310,9 @@ async def test_cleanup_drops_partitions_when_enabled_and_partitioned():
     partition_manager = MagicMock()
     partition_manager.is_partitioned = AsyncMock(return_value=True)
     partition_manager.ensure_partitions = AsyncMock(return_value=["p1"])
-    partition_manager.drop_partitions_older_than = AsyncMock(return_value=["LiteLLM_SpendLogs_p20260601"])
+    partition_manager.drop_partitions_older_than = AsyncMock(
+        return_value=["LiteLLM_SpendLogs_p20260601"]
+    )
 
     cleaner = SpendLogCleanup(
         general_settings={
@@ -435,7 +450,9 @@ async def test_integer_retention_treated_as_days():
     An integer value for maximum_spend_logs_retention_period should be treated
     as days (e.g., 3 → '3d' → 259200 seconds).
     """
-    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": 3})
+    cleaner = SpendLogCleanup(
+        general_settings={"maximum_spend_logs_retention_period": 3}
+    )
     result = cleaner._should_delete_spend_logs()
     assert result is True
     assert cleaner.retention_seconds == 3 * 86400  # 3 days in seconds
@@ -452,11 +469,13 @@ def test_string_retention_still_works():
         ("2w", 2 * 604800),
     ]
     for setting, expected_seconds in cases:
-        cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": setting})
-        assert cleaner._should_delete_spend_logs() is True, f"Failed for {setting}"
-        assert cleaner.retention_seconds == expected_seconds, (
-            f"Expected {expected_seconds} for {setting}, got {cleaner.retention_seconds}"
+        cleaner = SpendLogCleanup(
+            general_settings={"maximum_spend_logs_retention_period": setting}
         )
+        assert cleaner._should_delete_spend_logs() is True, f"Failed for {setting}"
+        assert (
+            cleaner.retention_seconds == expected_seconds
+        ), f"Expected {expected_seconds} for {setting}, got {cleaner.retention_seconds}"
 
 
 @pytest.mark.asyncio
@@ -470,7 +489,9 @@ async def test_delete_old_logs_aborts_on_non_int_execute_raw_return():
     mock_db.execute_raw = AsyncMock(return_value=None)
     mock_prisma_client.db = mock_db
 
-    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
+    cleaner = SpendLogCleanup(
+        general_settings={"maximum_spend_logs_retention_period": "7d"}
+    )
 
     cutoff_date = datetime.now(timezone.utc) - timedelta(days=7)
     result = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date, _far_deadline())
@@ -489,7 +510,9 @@ async def test_delete_old_logs_continues_on_valid_int_return():
     mock_db.execute_raw = AsyncMock(side_effect=[500, 300, 0])
     mock_prisma_client.db = mock_db
 
-    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
+    cleaner = SpendLogCleanup(
+        general_settings={"maximum_spend_logs_retention_period": "7d"}
+    )
 
     cutoff_date = datetime.now(timezone.utc) - timedelta(days=7)
     result = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date, _far_deadline())
@@ -536,7 +559,9 @@ async def test_delete_old_tool_index_rows_deletes_on_composite_key():
     mock_db.execute_raw = AsyncMock(side_effect=[5, 0])
     mock_prisma_client.db = mock_db
 
-    cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
+    cleaner = SpendLogCleanup(
+        general_settings={"maximum_spend_logs_retention_period": "7d"}
+    )
 
     cutoff_date = datetime.now(timezone.utc) - timedelta(days=7)
     result = await cleaner._delete_old_tool_index_rows(mock_prisma_client, cutoff_date, _far_deadline())
@@ -556,7 +581,9 @@ async def test_delete_old_logs_continues_after_single_batch_failure(monkeypatch)
     import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module
 
     # Zero out the failure backoff so the test doesn't take ~0.5s of real sleep.
-    monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0)
+    monkeypatch.setattr(
+        cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0
+    )
 
     mock_prisma_client = MagicMock()
     _wire_tx(mock_prisma_client.db)
@@ -564,10 +591,14 @@ async def test_delete_old_logs_continues_after_single_batch_failure(monkeypatch)
     _wire_tx(mock_db)
     # batch 1 succeeds, batch 2 raises (one-off DB timeout), batches 3-4 succeed,
     # batch 5 returns 0 → loop exits naturally.
-    mock_db.execute_raw = AsyncMock(side_effect=[100, TimeoutError("simulated DB timeout"), 200, 50, 0])
+    mock_db.execute_raw = AsyncMock(
+        side_effect=[100, TimeoutError("simulated DB timeout"), 200, 50, 0]
+    )
     mock_prisma_client.db = mock_db
 
-    cleaner = cleanup_module.SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
+    cleaner = cleanup_module.SpendLogCleanup(
+        general_settings={"maximum_spend_logs_retention_period": "7d"}
+    )
 
     cutoff_date = datetime.now(timezone.utc) - timedelta(days=7)
     result = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date, _far_deadline())
@@ -584,18 +615,26 @@ async def test_delete_old_logs_aborts_after_consecutive_failures(monkeypatch):
     import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module
 
     # Lower the threshold so the test is fast and deterministic.
-    monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3)
-    monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0)
+    monkeypatch.setattr(
+        cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3
+    )
+    monkeypatch.setattr(
+        cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0
+    )
 
     mock_prisma_client = MagicMock()
     _wire_tx(mock_prisma_client.db)
     mock_db = MagicMock()
     _wire_tx(mock_db)
     # Every batch raises — must abort after exactly 3 attempts, not loop forever.
-    mock_db.execute_raw = AsyncMock(side_effect=ConnectionError("simulated persistent DB outage"))
+    mock_db.execute_raw = AsyncMock(
+        side_effect=ConnectionError("simulated persistent DB outage")
+    )
     mock_prisma_client.db = mock_db
 
-    cleaner = cleanup_module.SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
+    cleaner = cleanup_module.SpendLogCleanup(
+        general_settings={"maximum_spend_logs_retention_period": "7d"}
+    )
 
     cutoff_date = datetime.now(timezone.utc) - timedelta(days=7)
     result = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date, _far_deadline())
@@ -610,8 +649,12 @@ async def test_delete_old_logs_resets_consecutive_failures_on_success(monkeypatc
     intermittent timeouts don't trip the abort threshold."""
     import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module
 
-    monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3)
-    monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0)
+    monkeypatch.setattr(
+        cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3
+    )
+    monkeypatch.setattr(
+        cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0
+    )
 
     mock_prisma_client = MagicMock()
     _wire_tx(mock_prisma_client.db)
@@ -632,7 +675,9 @@ async def test_delete_old_logs_resets_consecutive_failures_on_success(monkeypatc
     )
     mock_prisma_client.db = mock_db
 
-    cleaner = cleanup_module.SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
+    cleaner = cleanup_module.SpendLogCleanup(
+        general_settings={"maximum_spend_logs_retention_period": "7d"}
+    )
 
     cutoff_date = datetime.now(timezone.utc) - timedelta(days=7)
     result = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date, _far_deadline())
@@ -653,7 +698,9 @@ async def test_cleanup_uses_logger_exception_for_full_traceback(monkeypatch):
     mock_prisma_client = MagicMock()
     _wire_tx(mock_prisma_client.db)
     # Force the outer try/except to fire by making _should_delete_spend_logs raise.
-    cleaner = cleanup_module.SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
+    cleaner = cleanup_module.SpendLogCleanup(
+        general_settings={"maximum_spend_logs_retention_period": "7d"}
+    )
     cleaner.pod_lock_manager = None
 
     def boom():
@@ -678,8 +725,12 @@ async def test_cleanup_releases_lock_after_persistent_batch_failures(monkeypatch
     must still be released so the next scheduled run isn't permanently blocked."""
     import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module
 
-    monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 2)
-    monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0)
+    monkeypatch.setattr(
+        cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 2
+    )
+    monkeypatch.setattr(
+        cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0
+    )
 
     mock_prisma_client = MagicMock()
     _wire_tx(mock_prisma_client.db)
@@ -693,7 +744,9 @@ async def test_cleanup_releases_lock_after_persistent_batch_failures(monkeypatch
     mock_pod_lock_manager.acquire_lock = AsyncMock(return_value=True)
     mock_pod_lock_manager.release_lock = AsyncMock()
 
-    cleaner = cleanup_module.SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
+    cleaner = cleanup_module.SpendLogCleanup(
+        general_settings={"maximum_spend_logs_retention_period": "7d"}
+    )
     cleaner.pod_lock_manager = mock_pod_lock_manager
 
     await cleaner.cleanup_old_spend_logs(mock_prisma_client)
@@ -943,7 +996,9 @@ async def test_each_batch_carries_a_statement_and_lock_timeout():
         }
     )
 
-    await cleaner._delete_old_logs(mock_prisma_client, datetime.now(timezone.utc) - timedelta(days=7), _far_deadline())
+    await cleaner._delete_old_logs(
+        mock_prisma_client, datetime.now(timezone.utc) - timedelta(days=7), _far_deadline()
+    )
 
     assert "SET LOCAL statement_timeout = 12000" in recorded
     assert "SET LOCAL lock_timeout = 12000" in recorded
@@ -1079,7 +1134,9 @@ async def test_remaining_rows_probe_is_capped_so_it_cannot_scan_the_table():
 
     cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
 
-    await cleaner._delete_old_logs(mock_prisma_client, datetime.now(timezone.utc) - timedelta(days=7), _far_deadline())
+    await cleaner._delete_old_logs(
+        mock_prisma_client, datetime.now(timezone.utc) - timedelta(days=7), _far_deadline()
+    )
 
     count_sql = mock_db.query_raw.call_args[0][0]
     assert "count(*)" in count_sql

From 29bd2ceb2b5c8f2a172d6b08e6c54cb20c41daaa Mon Sep 17 00:00:00 2001
From: yucheng <yucheng@berri.ai>
Date: Sun, 20 Sep 2026 08:27:06 +0000
Subject: [PATCH 037/160] fix(proxy): avoid mutable shutdown wait set

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/shutdown/scheduled_jobs.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/litellm/proxy/shutdown/scheduled_jobs.py b/litellm/proxy/shutdown/scheduled_jobs.py
index e7625a73b47..5345d380112 100644
--- a/litellm/proxy/shutdown/scheduled_jobs.py
+++ b/litellm/proxy/shutdown/scheduled_jobs.py
@@ -50,12 +50,13 @@ async def stop_in_flight_scheduler_jobs(scheduler: StoppableScheduler, executor:
     if not scheduler.running:
         return
     in_flight: Final = executor.in_flight_jobs()
-    still_running: set[asyncio.Future[object]] = set()
     if in_flight:
         verbose_proxy_logger.info(
             "Waiting up to %ss for %d in-flight scheduled job(s) to finish", JOB_FINISH_TIMEOUT_SECONDS, len(in_flight)
         )
-        _done, still_running = await asyncio.wait(in_flight, timeout=JOB_FINISH_TIMEOUT_SECONDS)
+    still_running: Final = (
+        (await asyncio.wait(in_flight, timeout=JOB_FINISH_TIMEOUT_SECONDS))[1] if in_flight else frozenset()
+    )
     scheduler.shutdown(wait=False)
     if not still_running:
         return

From 6c13e0912b43e54ba912f8257ce436d28e6e8e01 Mon Sep 17 00:00:00 2001
From: ryan <ryan@berri.ai>
Date: Sun, 20 Sep 2026 08:59:30 +0000
Subject: [PATCH 038/160] fix(proxy): apply DB-stored callback redaction
 settings before logger init

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/constants.py                          |  6 ++++
 tests/test_litellm/proxy/test_proxy_server.py | 36 +++++++++++++++++++
 2 files changed, 42 insertions(+)

diff --git a/litellm/constants.py b/litellm/constants.py
index bbeb4846e27..4bc2e4e0d26 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -1853,6 +1853,12 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
     "max_ui_session_budget",
     "budget_rollover",
     "mcp_tool_search",
+    "turn_off_message_logging",
+    "datadog_params",
+    "datadog_llm_observability_params",
+    "newrelic_params",
+    "pointfive_params",
+    "aws_sqs_callback_params",
 ]
 SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"]
 DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60))
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index f71f9c20f3b..75741b125cb 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -11766,6 +11766,42 @@ def test_prompt_caching_settings_propagate_on_config_reload(monkeypatch, field_n
     assert getattr(litellm, field_name) == db_value
 
 
+@pytest.mark.asyncio
+async def test_db_stored_datadog_redaction_settings_apply_before_logger_init(monkeypatch):
+    """A DB-only litellm_settings row that pairs success_callback: ["datadog"] with
+    datadog_params.turn_off_message_logging: true must build the DataDogLogger redacted, the
+    same as the identical block in YAML. Regression for the redaction keys being absent from
+    the safe-override allowlist while the callback half of the row was honoured."""
+    import litellm.proxy.proxy_server as ps
+    from litellm.integrations.datadog.datadog import DataDogLogger
+    from litellm.litellm_core_utils import litellm_logging
+
+    monkeypatch.setenv("DD_API_KEY", "test-key")
+    monkeypatch.setenv("DD_SITE", "us5.datadoghq.com")
+    monkeypatch.setattr(litellm, "datadog_params", None)
+    monkeypatch.setattr(litellm, "turn_off_message_logging", False)
+    monkeypatch.setattr(litellm, "success_callback", [])
+    monkeypatch.setattr(litellm, "_async_success_callback", [])
+    monkeypatch.setattr(litellm, "failure_callback", [])
+    monkeypatch.setattr(litellm, "_async_failure_callback", [])
+    monkeypatch.setattr(litellm, "callbacks", [])
+    monkeypatch.setattr(litellm_logging, "_in_memory_loggers", [])
+
+    db_row = {
+        "success_callback": ["datadog"],
+        "datadog_params": {"turn_off_message_logging": True},
+        "turn_off_message_logging": True,
+    }
+    pc = ps.ProxyConfig()
+    pc._apply_litellm_settings_db_values(pc._prepared_db_settings_values("litellm_settings", db_row))
+    pc._add_callbacks_from_db_config({"litellm_settings": db_row})
+
+    datadog_loggers = [cb for cb in litellm.success_callback if isinstance(cb, DataDogLogger)]
+    assert len(datadog_loggers) == 1
+    assert datadog_loggers[0].turn_off_message_logging is True
+    assert litellm.turn_off_message_logging is True
+
+
 def test_get_config_list_marks_untouched_prompt_caching_flag_as_not_set(monkeypatch):
     """The flag defaults to False rather than None, so a plain 'is not None' check would
     report the default as 'In Config' and imply an admin had set it."""

From a90d852d378b46e3541fbb76ec519845e5caa785 Mon Sep 17 00:00:00 2001
From: ryan <ryan@berri.ai>
Date: Sun, 20 Sep 2026 09:26:30 +0000
Subject: [PATCH 039/160] test(proxy): type monkeypatch and cover every
 DB-overridable callback params key

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 tests/test_litellm/proxy/test_proxy_server.py | 26 ++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index 75741b125cb..c56645313c3 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -11767,7 +11767,7 @@ def test_prompt_caching_settings_propagate_on_config_reload(monkeypatch, field_n
 
 
 @pytest.mark.asyncio
-async def test_db_stored_datadog_redaction_settings_apply_before_logger_init(monkeypatch):
+async def test_db_stored_datadog_redaction_settings_apply_before_logger_init(monkeypatch: pytest.MonkeyPatch):
     """A DB-only litellm_settings row that pairs success_callback: ["datadog"] with
     datadog_params.turn_off_message_logging: true must build the DataDogLogger redacted, the
     same as the identical block in YAML. Regression for the redaction keys being absent from
@@ -11802,6 +11802,30 @@ async def test_db_stored_datadog_redaction_settings_apply_before_logger_init(mon
     assert litellm.turn_off_message_logging is True
 
 
+@pytest.mark.parametrize(
+    "field_name",
+    [
+        "datadog_params",
+        "datadog_llm_observability_params",
+        "newrelic_params",
+        "pointfive_params",
+        "aws_sqs_callback_params",
+    ],
+)
+def test_db_stored_callback_params_propagate_to_litellm_module(monkeypatch: pytest.MonkeyPatch, field_name: str):
+    """Every callback init params block stored in the DB litellm_settings row must land on the
+    litellm module before the matching logger is built, so the DB row behaves like YAML."""
+    import litellm.proxy.proxy_server as ps
+
+    monkeypatch.setattr(litellm, field_name, None)
+    db_value = {"turn_off_message_logging": True}
+
+    pc = ps.ProxyConfig()
+    pc._apply_litellm_settings_db_values(pc._prepared_db_settings_values("litellm_settings", {field_name: db_value}))
+
+    assert getattr(litellm, field_name) == db_value
+
+
 def test_get_config_list_marks_untouched_prompt_caching_flag_as_not_set(monkeypatch):
     """The flag defaults to False rather than None, so a plain 'is not None' check would
     report the default as 'In Config' and imply an admin had set it."""

From d338d3f2d2f6529de70a273b6f0d622708209c9c Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Sun, 20 Sep 2026 09:40:46 +0000
Subject: [PATCH 040/160] style(agents): tidy typing and docstring in access
 group ceiling helpers

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/auth/auth_checks.py                            | 2 +-
 litellm/proxy/utils.py                                       | 2 +-
 .../proxy/agent_endpoints/test_agent_registry.py             | 5 ++++-
 tests/test_litellm/proxy/auth/test_auth_checks.py            | 3 ---
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py
index 5412a9d9f6a..b5e7ef73d36 100644
--- a/litellm/proxy/auth/auth_checks.py
+++ b/litellm/proxy/auth/auth_checks.py
@@ -4332,7 +4332,7 @@ async def _check_agent_access_group_model_access(
     llm_router: Router | None,
     resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling,
 ) -> Literal[True]:
-    """Attached groups naming no model deny every model, unlike the empty allowlist ``_can_object_call_model`` allows."""
+    """Attached groups naming no model deny every model; the empty allowlist in ``_can_object_call_model`` allows."""
     if not model or valid_token is None or not valid_token.agent_id:
         return True
     ceiling: Final = await resolve_ceiling(valid_token.agent_id)
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index e736f2fa1c4..a8e7d3232eb 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -8188,7 +8188,7 @@ async def _get_access_group_models(
 
 async def _agent_access_group_visible_models(
     user_api_key_dict: "UserAPIKeyAuth",
-    llm_router: Optional["Router"],
+    llm_router: "Router | None",
     include_model_access_groups: bool,
     return_wildcard_routes: bool,
     team_id: str | None,
diff --git a/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py b/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py
index d15a3adadbd..b036e0dac4d 100644
--- a/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py
+++ b/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py
@@ -15,6 +15,7 @@ from litellm.proxy.agent_endpoints.agent_registry import (
     _restore_redacted_litellm_params,
     redact_sensitive_agent_litellm_params,
 )
+from litellm.types.agents import PatchAgentRequest
 
 # Obviously-fake stand-ins for a real AWS credential pair (LIT-6736 regression
 # fixtures) -- never a real key shape, and must never appear in any response.
@@ -1052,7 +1053,9 @@ async def test_add_agent_to_db_without_access_group_ids_leaves_column_to_its_def
         ({"access_group_ids": None}, []),
     ],
 )
-async def test_patch_agent_in_db_replaces_access_group_ids_when_provided(patch_body: dict, expected: list[str]):
+async def test_patch_agent_in_db_replaces_access_group_ids_when_provided(
+    patch_body: PatchAgentRequest, expected: list[str]
+):
     registry: Final = AgentRegistry()
     mock_prisma: Final = MagicMock()
     mock_prisma.db.litellm_agentstable.find_unique = AsyncMock(
diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py
index d8cd578d265..a1179617718 100644
--- a/tests/test_litellm/proxy/auth/test_auth_checks.py
+++ b/tests/test_litellm/proxy/auth/test_auth_checks.py
@@ -8900,9 +8900,6 @@ def test_request_skips_budget_checks_extends_route_rule_with_zero_cost_models()
     assert request_skips_budget_checks(route="/v1/chat/completions", model=None, llm_router=None) is False
 
 
-# Agent access group model ceiling
-
-
 def _agent_model_ceiling_resolver(
     models: frozenset[str] | None,
 ) -> tuple[CeilingResolver, list[str]]:

From 3fe405a5ccdf568ce16d82bd08c4b98e936909bf Mon Sep 17 00:00:00 2001
From: ryan <ryan@berri.ai>
Date: Mon, 21 Sep 2026 18:48:35 +0000
Subject: [PATCH 041/160] feat(auth): breached password detection, self-service
 change-password and forced password reset

Cherry-pick of merge commit b3882d8e43 (PRs #39321, #39562, #40107), which landed on litellm_internal_staging instead of main.

Co-authored-by: ojensen-berri <ojensen@berri.ai>
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../migration.sql                             |   3 +
 .../litellm_proxy_extras/schema.prisma        |   2 +
 litellm/constants.py                          |   3 +
 litellm/models/user.py                        |   2 +
 litellm/proxy/_types.py                       |  31 +-
 litellm/proxy/auth/login_utils.py             |  87 ++-
 litellm/proxy/auth/password_policy.py         | 128 +++-
 litellm/proxy/auth/route_checks.py            |  27 +-
 .../internal_user_endpoints.py                |  62 +-
 .../password_endpoints.py                     | 126 ++++
 litellm/proxy/management_endpoints/ui_sso.py  |   1 +
 litellm/proxy/proxy_server.py                 |  16 +-
 litellm/proxy/schema.prisma                   |   2 +
 litellm/types/llms/custom_http.py             |   1 +
 litellm/types/proxy/ui_sso.py                 |   3 +-
 schema.prisma                                 |   2 +
 .../endpointaudit/coverage_allowlist.txt      |   1 +
 .../proxy/auth/test_login_utils.py            | 272 ++++++++
 .../proxy/auth/test_onboarding.py             | 144 +++-
 .../proxy/auth/test_password_policy.py        | 209 ++++++
 .../proxy/auth/test_route_checks.py           | 274 +++++---
 .../test_internal_user_endpoints.py           | 623 ++++++++----------
 .../test_password_endpoints.py                | 331 ++++++++++
 tests/test_litellm/proxy/test__types.py       |  42 ++
 tests/unit/models/test_models.py              |   6 +-
 .../ChangePasswordForm.integration.test.tsx   | 110 ++++
 .../change-password/ChangePasswordForm.tsx    | 120 ++++
 .../app/(dashboard)/change-password/page.tsx  |   7 +
 .../app/(dashboard)/hooks/useAuthorized.ts    |   2 +
 .../src/app/(dashboard)/layout.test.tsx       |  58 +-
 .../src/app/(dashboard)/layout.tsx            |  13 +-
 .../Navbar/UserDropdown/UserDropdown.test.tsx |  52 +-
 .../Navbar/UserDropdown/UserDropdown.tsx      |  23 +-
 .../SidebarAccountMenu.test.tsx               |  43 ++
 .../SidebarAccountMenu/SidebarAccountMenu.tsx |  24 +-
 .../src/components/leftnav.test.tsx           |   1 +
 .../src/components/networking.tsx             |  14 +
 .../src/contexts/AuthContext.tsx              |   4 +
 ui/litellm-dashboard/src/lib/http/schema.d.ts |  90 ++-
 39 files changed, 2456 insertions(+), 503 deletions(-)
 create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260921000000_add_password_reset_columns/migration.sql
 create mode 100644 litellm/proxy/management_endpoints/password_endpoints.py
 create mode 100644 tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py
 create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.integration.test.tsx
 create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx
 create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/change-password/page.tsx

diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260921000000_add_password_reset_columns/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260921000000_add_password_reset_columns/migration.sql
new file mode 100644
index 00000000000..960b0d4d7eb
--- /dev/null
+++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260921000000_add_password_reset_columns/migration.sql
@@ -0,0 +1,3 @@
+ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "password_reset_required" BOOLEAN;
+
+ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "last_breach_check_at" TIMESTAMP(3);
diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
index d2032cec0d0..bcafa6dbd0e 100644
--- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
@@ -246,6 +246,8 @@ model LiteLLM_UserTable {
     organization_id String?
     object_permission_id String?
     password  String?
+    password_reset_required Boolean?
+    last_breach_check_at DateTime?
     teams    String[] @default([])
     user_role  String?
 		max_budget Float?
diff --git a/litellm/constants.py b/litellm/constants.py
index bbeb4846e27..b5b3647e525 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -2114,3 +2114,6 @@ BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY: Final = "batch_enqueued_token_limit"
 # Shared read-only empty mapping, for defaulting optional Mapping parameters without
 # constructing a fresh mutable dict at each call site.
 EMPTY_MAPPING: Final = MappingProxyType({})
+
+# API endpoint for breached password k-anonymity search
+HIBP_RANGE_API_BASE: Final = "https://api.pwnedpasswords.com/range"
diff --git a/litellm/models/user.py b/litellm/models/user.py
index 82f78c28078..92aca87d303 100644
--- a/litellm/models/user.py
+++ b/litellm/models/user.py
@@ -24,6 +24,8 @@ class LiteLLM_UserTable(LiteLLMPydanticObjectBase):
     organization_id: str | None = None
     object_permission_id: str | None = None
     password: str | None = Field(default=None, exclude=True)
+    password_reset_required: bool | None = None
+    last_breach_check_at: datetime | None = None
     teams: list[str] = []
     user_role: str | None = None
     max_budget: float | None = None
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 3b34440d0bf..16b405d8e02 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -904,6 +904,7 @@ class LiteLLMRoutes(enum.Enum):
         "/claude_code_gateway/v1/traces",
         "/user/list",  # org admins checked in endpoint; non-admins get 403
         "/management/v1/users/bulk_delete",  # proxy admins delete anyone, org admins only their orgs' users; others 403
+        "/user/password/change",  # endpoint only ever writes the caller's own row
         "/model/{model_id}/update",
         "/prompt/list",
         "/prompt/info",
@@ -1864,6 +1865,17 @@ class NewUserRequest(GenerateRequestBase):
     send_invite_email: bool | None = None
     sso_user_id: str | None = None
     organizations: list[str] | None = None
+    password: str | None = None
+
+    @field_validator("password")
+    @classmethod
+    def password_not_supported(cls, value: str | None) -> str | None:
+        if value is not None:
+            raise ValueError(
+                "password cannot be set via /user/new. Users set their own password through an "
+                "invitation link (POST /invitation/new)."
+            )
+        return value
 
 
 class NewUserResponse(GenerateKeyResponse):
@@ -1886,7 +1898,8 @@ class NewUserResponse(GenerateKeyResponse):
 
 
 class UpdateUserRequestNoUserIDorEmail(GenerateRequestBase):  # shared with BulkUpdateUserRequest
-    password: str | None = None
+    # repr=False keeps the plaintext out of management-endpoint alerts, which str() the request model
+    password: str | None = Field(default=None, repr=False)
     spend: float | None = None
     metadata: dict | None = None
     user_alias: str | None = None
@@ -1916,6 +1929,16 @@ class UpdateUserRequest(UpdateUserRequestNoUserIDorEmail):
         return values
 
 
+class ChangePasswordRequest(LiteLLMPydanticObjectBase):
+    current_password: str = Field(repr=False)
+    new_password: str = Field(repr=False)
+
+
+class ChangePasswordResponse(LiteLLMPydanticObjectBase):
+    user_id: str
+    message: str
+
+
 class DeleteUserRequest(LiteLLMPydanticObjectBase):
     user_ids: list[str]  # required
 
@@ -3937,6 +3960,12 @@ class AllCallbacks(LiteLLMPydanticObjectBase):
     )
 
 
+class HTTPExceptionErrorDetail(TypedDict):
+    """The `{"error": <message>}` shape most proxy endpoints raise as `HTTPException.detail`."""
+
+    error: ReadOnly[str]
+
+
 class SpendLogsRouterMetadata(TypedDict):
     """
     Router provenance stamped on spend logs for deployments flagged with
diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py
index e0d599b0017..0dddb16b531 100644
--- a/litellm/proxy/auth/login_utils.py
+++ b/litellm/proxy/auth/login_utils.py
@@ -10,14 +10,16 @@ import secrets
 from collections.abc import Mapping
 from datetime import datetime, timedelta, timezone
 from types import MappingProxyType
-from typing import Final, Literal, cast
+from typing import TYPE_CHECKING, Final, Literal, cast
 
 import jwt
 from fastapi import HTTPException
 
 import litellm
+from litellm._logging import verbose_proxy_logger
 from litellm.constants import LITELLM_PROXY_ADMIN_NAME, LITELLM_UI_SESSION_DURATION
 from litellm.litellm_core_utils.duration_parser import duration_in_seconds
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 from litellm.proxy._types import (
     LiteLLM_UserTable,
     LitellmUserRoles,
@@ -28,6 +30,7 @@ from litellm.proxy._types import (
 )
 from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured
 from litellm.proxy.auth.login_throttle import LoginAttempt, LoginThrottle
+from litellm.proxy.auth.password_policy import is_breach_check_enabled, is_password_breached
 from litellm.proxy.management_endpoints.internal_user_endpoints import user_update
 from litellm.proxy.management_endpoints.key_management_endpoints import (
     generate_key_helper_fn,
@@ -50,6 +53,56 @@ INVALID_UI_CREDENTIALS_MESSAGE: Final = (
 )
 INVALID_USER_PASSWORD_MESSAGE: Final = "Invalid credentials used to access UI. Check the password set for your user"
 
+if TYPE_CHECKING:
+    from prisma import types as prisma_types
+
+BREACH_RECHECK_INTERVAL: Final = timedelta(hours=24)
+PASSWORD_RESET_ALLOWED_ROUTES: Final = ("/user/password/change",)
+
+
+def _breach_recheck_due(last_breach_check_at: datetime | None) -> bool:
+    if last_breach_check_at is None:
+        return True
+    last_checked_utc: Final = (
+        last_breach_check_at
+        if last_breach_check_at.tzinfo is not None
+        else last_breach_check_at.replace(tzinfo=timezone.utc)
+    )
+    return datetime.now(timezone.utc) - last_checked_utc >= BREACH_RECHECK_INTERVAL
+
+
+async def screen_login_password_for_breach(
+    user_id: str,
+    password: str,
+    last_breach_check_at: datetime | None,
+    general_settings: Mapping[str, object],
+    prisma_client: PrismaClient,
+    client: AsyncHTTPHandler | None = None,
+) -> bool:
+    """Screens a successfully verified login password against HIBP, stamps
+    ``password_reset_required`` when breached, and returns whether a breach was
+    found so the login it runs in can restrict the session it is about to mint.
+    Fails open (HIBP or DB trouble never fails the login) and rechecks a given
+    user at most once per ``BREACH_RECHECK_INTERVAL``."""
+    if not is_breach_check_enabled(general_settings):
+        return False
+    if not _breach_recheck_due(last_breach_check_at):
+        return False
+    breached: Final = await is_password_breached(password, general_settings, client)
+    checked_at: Final = datetime.now(timezone.utc)
+    breached_update: Final[prisma_types.LiteLLM_UserTableUpdateInput] = {
+        "last_breach_check_at": checked_at,
+        "password_reset_required": True,
+    }
+    recheck_update: Final[prisma_types.LiteLLM_UserTableUpdateInput] = {"last_breach_check_at": checked_at}
+    update_data: Final = breached_update if breached else recheck_update
+    find_user: Final[prisma_types.LiteLLM_UserTableWhereInput] = {"user_id": user_id}
+    try:
+        await UserRepository(prisma_client).table.update(where=find_user, data=update_data)
+    except Exception as e:  # noqa: BLE001  # a failed stamp must never surface into the login
+        verbose_proxy_logger.warning("Login-time breach screening could not update user %s: %s", user_id, e)
+    return breached
+
 
 async def _rehash_password_if_needed(user_id: str, password: str, stored: str) -> None:
     """Rehash legacy password (SHA256) to scrypt on successful login."""
@@ -137,6 +190,7 @@ class LoginResult:
     user_email: str | None
     user_role: str
     login_method: Literal["sso", "username_password"]
+    password_reset_required: bool
 
     def __init__(
         self,
@@ -145,12 +199,14 @@ class LoginResult:
         user_email: str | None,
         user_role: str,
         login_method: Literal["sso", "username_password"] = "username_password",
+        password_reset_required: bool = False,
     ):
         self.user_id = user_id
         self.key = key
         self.user_email = user_email
         self.user_role = user_role
         self.login_method = login_method
+        self.password_reset_required = password_reset_required
 
 
 async def authenticate_user(
@@ -356,21 +412,26 @@ async def _sign_in(
 
         if verify_password(password, _password):
             await _rehash_password_if_needed(_user_row.user_id, password, _password)
+            breached_now: Final = prisma_client is not None and await screen_login_password_for_breach(
+                user_id=_user_row.user_id,
+                password=password,
+                last_breach_check_at=getattr(_user_row, "last_breach_check_at", None),
+                general_settings=general_settings,
+                prisma_client=prisma_client,
+            )
+            password_reset_required: Final = breached_now or getattr(_user_row, "password_reset_required", None) is True
             if os.getenv("DATABASE_URL") is not None:
                 response = await generate_key_helper_fn(
                     llm_router=None,
                     request_type="key",
-                    **{
-                        "user_role": user_role,
-                        "duration": LITELLM_UI_SESSION_DURATION,
-                        "key_max_budget": litellm.max_ui_session_budget,
-                        "models": [],
-                        "aliases": {},
-                        "config": {},
-                        "spend": 0,
-                        "user_id": user_id,
-                        "team_id": "litellm-dashboard",
-                    },
+                    user_role=user_role,
+                    duration=LITELLM_UI_SESSION_DURATION,
+                    key_max_budget=litellm.max_ui_session_budget,
+                    spend=0,
+                    user_id=user_id,
+                    team_id="litellm-dashboard",
+                    allowed_routes=list(PASSWORD_RESET_ALLOWED_ROUTES) if password_reset_required else None,
+                    metadata={"password_reset_required": True} if password_reset_required else {},
                 )
             else:
                 raise ProxyException(
@@ -390,6 +451,7 @@ async def _sign_in(
                 user_email=user_email,
                 user_role=cast(str, user_role),
                 login_method="username_password",
+                password_reset_required=password_reset_required,
             )
         else:
             await attempt.failed()
@@ -460,4 +522,5 @@ def create_ui_token_object(
         auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"),
         disabled_non_admin_personal_key_creation=disabled_non_admin_personal_key_creation,
         server_root_path=get_server_root_path(),
+        password_reset_required=login_result.password_reset_required,
     )
diff --git a/litellm/proxy/auth/password_policy.py b/litellm/proxy/auth/password_policy.py
index ab7a565894a..7f06a0993d3 100644
--- a/litellm/proxy/auth/password_policy.py
+++ b/litellm/proxy/auth/password_policy.py
@@ -4,13 +4,28 @@ Applied at every path that persists a new or changed password for a DB-backed
 user (``/user/update``, ``/user/bulk_update``, and the invitation onboarding
 claim flow), so the strength bar is configured in one place instead of
 per-endpoint.
+
+Also screens new passwords against known data breaches via the
+haveibeenpwned.com (HIBP) k-anonymity range API: only the first 5 characters
+of the password's SHA-1 hash ever leave the proxy, and the check fails open
+(allows the password) when HIBP is unreachable.
 """
 
-from collections.abc import Mapping
+import asyncio
+import hashlib
+from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
+from types import MappingProxyType
 from typing import Final
 
+from litellm._logging import verbose_proxy_logger
+from litellm._version import version
+from litellm.constants import HIBP_RANGE_API_BASE
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
 from litellm.proxy._types import ProxyErrorTypes, ProxyException
+from litellm.types.llms.custom_http import httpxSpecialProvider
+
+HIBP_TIMEOUT_SECONDS: Final = 5.0
 
 DEFAULT_MIN_LENGTH: Final = 12
 MIN_ALLOWED_LENGTH: Final = 8
@@ -90,3 +105,114 @@ def validate_password_policy(password: str, general_settings: Mapping[str, objec
         param="password",
         code=400,
     )
+
+
+def _hibp_client() -> AsyncHTTPHandler:
+    return get_async_httpx_client(
+        llm_provider=httpxSpecialProvider.PasswordBreachCheck,
+        params={"timeout": HIBP_TIMEOUT_SECONDS},  # mutable-ok: callee takes a bare dict (PEP 589)
+    )
+
+
+def _is_suffix_in_range_response(response_body: str, hash_suffix: str) -> bool:
+    for line in response_body.upper().splitlines():
+        entry_suffix, _, count = line.strip().partition(":")
+        if entry_suffix == hash_suffix:
+            return int(count.strip() or "0") > 0
+    return False
+
+
+async def _is_password_breached(password: str, client: AsyncHTTPHandler) -> bool:
+    # usedforsecurity=False: SHA-1 is only a lookup key into the HIBP dataset, so no security property rests on it
+    sha1_hex: Final = hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper()
+    headers: Final = {  # mutable-ok: callee takes a bare dict (PEP 589)
+        "Add-Padding": "true",
+        "User-Agent": f"litellm-proxy/{version}",
+    }
+    try:
+        response: Final = await client.get(
+            f"{HIBP_RANGE_API_BASE}/{sha1_hex[:5]}",
+            headers=headers,
+        )
+        response.raise_for_status()
+        breached: Final = _is_suffix_in_range_response(response.text, sha1_hex[5:])
+    except Exception as e:  # noqa: BLE001  # fail-open: any HIBP failure skips the check, never breaks the caller
+        verbose_proxy_logger.warning("Breached-password check skipped, HIBP lookup failed: %s", e)
+        return False
+    return breached
+
+
+def is_breach_check_enabled(general_settings: Mapping[str, object]) -> bool:
+    return general_settings.get("password_policy_check_breached_passwords", True) is not False
+
+
+async def is_password_breached(
+    password: str,
+    general_settings: Mapping[str, object],
+    client: AsyncHTTPHandler | None = None,
+) -> bool:
+    """False when the check is disabled, the password is absent from the HIBP
+    corpus, or HIBP is unreachable (fail open)."""
+    if not is_breach_check_enabled(general_settings):
+        return False
+    return await _is_password_breached(password, client if client is not None else _hibp_client())
+
+
+def breached_password_error() -> ProxyException:
+    return ProxyException(
+        message=(
+            "This password appears in known data breaches and cannot be used. Please choose a different password."
+        ),
+        type=ProxyErrorTypes.validation_error,
+        param="password",
+        code=400,
+    )
+
+
+async def validate_password_not_breached(
+    password: str,
+    general_settings: Mapping[str, object],
+    client: AsyncHTTPHandler | None = None,
+) -> None:
+    """Raise ``ProxyException`` (400) if ``password`` appears in a known data breach.
+
+    Fails open: an unreachable or misbehaving HIBP allows the password."""
+    if not await is_password_breached(password, general_settings, client):
+        return
+    raise breached_password_error()
+
+
+def _strength_verdict(password: str, general_settings: Mapping[str, object]) -> ProxyException | None:
+    try:
+        validate_password_policy(password, general_settings)
+    except ProxyException as e:
+        return e
+    return None
+
+
+async def validate_passwords_bulk(
+    passwords: Sequence[str],
+    general_settings: Mapping[str, object],
+    client: AsyncHTTPHandler | None = None,
+) -> Mapping[str, ProxyException | None]:
+    """Per-unique-password policy verdicts for a batch: the ProxyException to
+    surface, or None when the password is acceptable.
+
+    Deduplicates first, then issues every needed HIBP lookup concurrently, so a
+    batch caller pays one HIBP timeout window in the worst case instead of one
+    per password (each lookup still fails open independently)."""
+    unique_passwords: Final = tuple(dict.fromkeys(passwords))
+    strength_verdicts: Final[Mapping[str, ProxyException | None]] = MappingProxyType(
+        {password: _strength_verdict(password, general_settings) for password in unique_passwords}
+    )
+    to_screen: Final = tuple(password for password in unique_passwords if strength_verdicts[password] is None)
+    breached_flags: Final = await asyncio.gather(
+        *(is_password_breached(password, general_settings, client) for password in to_screen)
+    )
+    breached_passwords: Final = frozenset(password for password, breached in zip(to_screen, breached_flags) if breached)
+    return MappingProxyType(
+        {
+            password: breached_password_error() if password in breached_passwords else strength_verdicts[password]
+            for password in unique_passwords
+        }
+    )
diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py
index 1b9fd7c42bf..afddb4866a9 100644
--- a/litellm/proxy/auth/route_checks.py
+++ b/litellm/proxy/auth/route_checks.py
@@ -194,6 +194,16 @@ class RouteChecks:
         if denied_auth_enforced_pass_through_route:
             raise RouteChecks._auth_pass_through_denied_exception(route=route)
 
+        if valid_token.metadata.get("password_reset_required") is True:
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN,
+                detail=(
+                    "This account's password must be changed before the session can be used: "
+                    "it was either found in a known data breach or set by an admin. "
+                    "Change it via POST /user/password/change (UI: /ui/change-password), then log in again."
+                ),
+            )
+
         raise HTTPException(
             status_code=status.HTTP_403_FORBIDDEN,
             detail=f"Virtual key is not allowed to call this route. Only allowed to call routes: {valid_token.allowed_routes}. Tried to call route: {route}",
@@ -812,7 +822,8 @@ class RouteChecks:
              in the codebase is automatically readable by Admin Viewer
              without needing to remember to add it to an allowlist.
           3. Unsafe HTTP method (POST/PUT/PATCH/DELETE):
-             - Allow `/user/update` only when restricted to user_email/password.
+             - Allow `/user/update` only when restricted to user_email.
+             - Allow `/user/password/change` (endpoint only writes the caller's own row).
              - Block all explicit writes in `_ADMIN_VIEWER_BLOCKED_WRITE_ROUTES`.
              - Otherwise allow only if the route is in admin_viewer_routes /
                global_spend_tracking_routes (legacy explicit-allow set).
@@ -832,10 +843,10 @@ class RouteChecks:
                 if request_data is not None and isinstance(request_data, dict):
                     _params_updated: Final = request_data.keys()
                     for param in _params_updated:
-                        if param not in ["user_email", "password"]:
+                        if param != "user_email":
                             raise HTTPException(
                                 status_code=status.HTTP_403_FORBIDDEN,
-                                detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route} and updating invalid param: {param}. only user_email and password can be updated",
+                                detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route} and updating invalid param: {param}. only user_email can be updated",
                             )
             elif RouteChecks.check_route_access(route=route, allowed_routes=_PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES) or (
                 route.startswith("/key/") and route.endswith(_PROXY_ADMIN_VIEW_ONLY_BLOCKED_KEY_SUFFIXES)
@@ -854,21 +865,25 @@ class RouteChecks:
             return
 
         # ── Unsafe HTTP method: explicit checks ──────────────────────────
-        # Allow `/user/update` for self-service email / password change.
+        # Allow `/user/update` for self-service email change.
         if route == "/user/update":
             if request_data is not None and isinstance(request_data, dict):
                 for param in request_data:
-                    if param not in ["user_email", "password"]:
+                    if param != "user_email":
                         raise HTTPException(
                             status_code=status.HTTP_403_FORBIDDEN,
                             detail=(
                                 f"user not allowed to access this route, role= {_user_role}. "
                                 f"Trying to access: {route} and updating invalid param: {param}. "
-                                "only user_email and password can be updated"
+                                "only user_email can be updated"
                             ),
                         )
             return
 
+        # Self-service password change; the endpoint only writes the caller's own row.
+        if route == "/user/password/change":
+            return
+
         # Hard-block known write routes regardless of HTTP method (defensive
         # — these are POSTs in practice, but pinning them here protects
         # against future GET-shaped writes).
diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py
index 1c986305c21..285be23bdf6 100644
--- a/litellm/proxy/management_endpoints/internal_user_endpoints.py
+++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py
@@ -28,6 +28,7 @@ from typing_extensions import ReadOnly, TypedDict
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 from litellm.proxy._types import *
 from litellm.proxy.auth.auth_checks import (
     delete_cache_key_objects,
@@ -35,7 +36,11 @@ from litellm.proxy.auth.auth_checks import (
     get_team_object,
     get_user_object,
 )
-from litellm.proxy.auth.password_policy import validate_password_policy
+from litellm.proxy.auth.password_policy import (
+    validate_password_not_breached,
+    validate_password_policy,
+    validate_passwords_bulk,
+)
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
 from litellm.proxy.common_utils.user_api_key_cache import (
@@ -173,11 +178,23 @@ def _team_membership_table(
     return team_membership_table
 
 
-def _hash_password_in_dict(data: dict, general_settings: Mapping[str, object]) -> None:
-    """Validate and hash password field in-place if present."""
+async def _hash_password_in_dict(
+    data: dict, general_settings: Mapping[str, object], password_prevalidated: bool = False
+) -> None:
+    """Validate and hash password field in-place if present.
+
+    ``password_prevalidated`` skips the policy checks for callers that already
+    validated the password (the bulk path screens its whole batch upfront).
+
+    An admin-set password is known to whoever set it, so the user is also
+    flagged for a forced password change at next login."""
     if "password" in data and data["password"] is not None:
-        validate_password_policy(data["password"], general_settings)
+        if not password_prevalidated:
+            validate_password_policy(data["password"], general_settings)
+            await validate_password_not_breached(data["password"], general_settings)
         data["password"] = hash_password(data["password"])
+        data["password_reset_required"] = True
+        data["last_breach_check_at"] = None
 
 
 def _strip_password_from_response(response) -> None:
@@ -505,6 +522,7 @@ async def new_user(
     - prompts: Optional[List[str]] - List of allowed prompts for the user. If specified, the user will only be able to use these specific prompts.
     - organizations: List[str] - List of organization id's the user is a member of
     - budget_limits: Optional[list] - List of concurrent budget windows for the user. Each window specifies a budget_limit, time_period, and optional budget_duration. Example - [{"budget_limit": 10.0, "time_period": "1d"}, {"budget_limit": 50.0, "time_period": "7d"}].
+    - password: Optional[str] - Not supported; any value is rejected with a 422. Users set their own password through an invitation link (POST /invitation/new).
     Returns:
     - key: (str) The generated api key for the user
     - expires: (datetime) Datetime object for when key expires.
@@ -524,7 +542,7 @@ async def new_user(
     ```
     """
     try:
-        from litellm.proxy.proxy_server import _license_check, general_settings, prisma_client
+        from litellm.proxy.proxy_server import _license_check, prisma_client
 
         if prisma_client is None:
             raise HTTPException(status_code=400, detail=CommonProxyErrors.db_not_connected_error.value)
@@ -572,7 +590,7 @@ async def new_user(
         # generate_key_helper_fn only forwards object_permission_id, so without this the entitlement
         # the caller sent would be dropped on the floor.
         data_json = await _set_object_permission(data_json=data_json, prisma_client=prisma_client)
-        _hash_password_in_dict(data_json, general_settings)
+        data_json.pop("password", None)
         teams = data.teams
         if teams is None:
             teams = check_if_default_team_set()
@@ -1438,6 +1456,7 @@ async def _update_single_user_helper(
     user_request: UpdateUserRequest,
     user_api_key_dict: UserAPIKeyAuth,
     litellm_changed_by: str | None = None,
+    password_prevalidated: bool = False,
 ) -> dict[str, Any]:
     """
     Helper function to update a single user.
@@ -1460,7 +1479,7 @@ async def _update_single_user_helper(
 
     data_json: Final[dict] = user_request.model_dump(exclude_unset=True)
     non_default_values = _update_internal_user_params(data_json=data_json, data=user_request)
-    _hash_password_in_dict(non_default_values, general_settings)
+    await _hash_password_in_dict(non_default_values, general_settings, password_prevalidated=password_prevalidated)
 
     existing_user_row: BaseModel | None = None
     if user_request.user_id:
@@ -1641,7 +1660,7 @@ async def user_update(
     Parameters:
         - user_id: Optional[str] - Specify a user id. If not set, a unique id will be generated.
         - user_email: Optional[str] - Specify a user email.
-        - password: Optional[str] - Specify a user password.
+        - password: Optional[str] - Set the user's password (admin only). Must satisfy the configured password policy. The user is required to change it at their next login. Users change their own password with POST /user/password/change.
         - user_alias: Optional[str] - A descriptive name for you to know who this user id refers to.
         - teams: Optional[list] - specify a list of team id's a user belongs to.
         - send_invite_email: Optional[bool] - Specify if an invite email should be sent.
@@ -1709,19 +1728,38 @@ async def bulk_update_processed_users(
     users_to_update: list[UpdateUserRequest],
     user_api_key_dict: UserAPIKeyAuth,
     litellm_changed_by: str | None = None,
+    hibp_client: AsyncHTTPHandler | None = None,
 ) -> BulkUpdateUserResponse:
+    from litellm.proxy.proxy_server import general_settings
+
     results: Final[list[UserUpdateResult]] = []
     successful_updates = 0
     failed_updates = 0
 
+    # Screen the batch's passwords upfront and concurrently: done per-user
+    # inside the loop below, each HIBP lookup would be awaited serially and a
+    # degraded-slow HIBP could stretch a full batch to minutes, timing out the
+    # request after some updates already persisted.
+    password_verdicts: Final = await validate_passwords_bulk(
+        tuple(u.password for u in users_to_update if u.password is not None),
+        general_settings,
+        client=hibp_client,
+    )
+
     # Process each user update independently
     try:
         for user_request in users_to_update:
             try:
+                if (
+                    user_request.password is not None
+                    and (password_error := password_verdicts.get(user_request.password)) is not None
+                ):
+                    raise password_error
                 response = await _update_single_user_helper(
                     user_request=user_request,
                     user_api_key_dict=user_api_key_dict,
                     litellm_changed_by=litellm_changed_by,
+                    password_prevalidated=True,
                 )
                 # Record success
                 results.append(
@@ -1859,6 +1897,14 @@ async def bulk_user_update(
                 status_code=403,
                 detail="Only proxy admins can update all users at once.",
             )
+        if data.user_updates.password is not None:
+            bulk_password_error: Final[HTTPExceptionErrorDetail] = {
+                "error": (
+                    "Setting one password for all users is not supported. "
+                    "Use per-user updates via the 'users' list instead."
+                )
+            }
+            raise HTTPException(status_code=400, detail=bulk_password_error)
         # Optimized path for updating all users directly in database
         all_users_in_db: Final = await _user_table(prisma_client).find_many(order={"created_at": "desc"})
 
diff --git a/litellm/proxy/management_endpoints/password_endpoints.py b/litellm/proxy/management_endpoints/password_endpoints.py
new file mode 100644
index 00000000000..bd3c9722d4d
--- /dev/null
+++ b/litellm/proxy/management_endpoints/password_endpoints.py
@@ -0,0 +1,126 @@
+"""
+Self-service password management.
+
+/user/password/change
+
+Deliberately NOT wrapped in `management_endpoint_wrapper`: the wrapper emits
+request kwargs to OTEL spans, which would log plaintext passwords. The audit
+signal is emitted by hand below, with field names only, never values.
+"""
+
+from typing import TYPE_CHECKING, Final
+
+from fastapi import APIRouter, Depends, HTTPException
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy._types import (
+    ChangePasswordRequest,
+    ChangePasswordResponse,
+    CommonProxyErrors,
+    HTTPExceptionErrorDetail,
+    LitellmTableNames,
+    UserAPIKeyAuth,
+)
+from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy
+from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.management_helpers.audit_logs import create_object_audit_log
+from litellm.proxy.utils import hash_password, verify_password
+from litellm.repositories.prisma_protocols import TableActions
+from litellm.repositories.user_repository import UserRepository
+
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
+    from prisma import types as prisma_types
+
+    from litellm.proxy.utils import PrismaClient
+
+router: Final = APIRouter()
+
+_PASSWORD_CHANGED_AUDIT_VALUES: Final = '{"fields_changed": ["password"]}'
+
+
+def _error_detail(message: str) -> HTTPExceptionErrorDetail:
+    detail: Final[HTTPExceptionErrorDetail] = {"error": message}
+    return detail
+
+
+def _user_table(
+    prisma_client: "PrismaClient | None",
+) -> "TableActions[prisma_models.LiteLLM_UserTable]":
+    user_table: Final[TableActions[prisma_models.LiteLLM_UserTable]] = UserRepository(prisma_client).table
+    return user_table
+
+
+@router.post(
+    "/user/password/change",
+    tags=("Internal User management",),
+    dependencies=(Depends(user_api_key_auth),),
+)
+async def change_password(
+    data: ChangePasswordRequest,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+) -> ChangePasswordResponse:
+    """
+    Change the calling user's own password.
+
+    Requires the current password. The new password must satisfy the
+    configured password policy (`general_settings.password_policy_*`: minimum
+    length, character classes, and, when enabled, breached-password screening
+    via haveibeenpwned.com). A successful change lifts any pending forced
+    password reset (`password_reset_required`) on the account.
+
+    Parameters:
+    - current_password: str - The user's current password.
+    - new_password: str - The password to change to.
+    """
+    from litellm.proxy.proxy_server import general_settings, litellm_proxy_admin_name, prisma_client
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=500,
+            detail=_error_detail(CommonProxyErrors.db_not_connected_error.value),
+        )
+
+    user_id: Final = user_api_key_dict.user_id
+    if user_id is None:
+        raise HTTPException(
+            status_code=400,
+            detail=_error_detail("No user is associated with this session, so there is no password to change."),
+        )
+
+    find_user: Final[prisma_types.LiteLLM_UserTableWhereInput] = {"user_id": user_id}
+    user_row: Final = await _user_table(prisma_client).find_first(where=find_user)
+    stored_password: Final = user_row.password if user_row is not None else None
+    if stored_password is None:
+        raise HTTPException(
+            status_code=400,
+            detail=_error_detail(
+                "This account has no password set, so there is no password to change. "
+                "Passwords are set through an invitation link (POST /invitation/new)."
+            ),
+        )
+
+    if not verify_password(data.current_password, stored_password):
+        raise HTTPException(status_code=400, detail=_error_detail("Current password is incorrect."))
+
+    validate_password_policy(data.new_password, general_settings)
+    await validate_password_not_breached(data.new_password, general_settings)
+
+    password_update: Final[prisma_types.LiteLLM_UserTableUpdateInput] = {
+        "password": hash_password(data.new_password),
+        "password_reset_required": False,
+        "last_breach_check_at": None,
+    }
+    await _user_table(prisma_client).update(where=find_user, data=password_update)
+
+    verbose_proxy_logger.info("Password changed via /user/password/change for user_id=%s", user_id)
+    await create_object_audit_log(
+        object_id=user_id,
+        action="updated",
+        litellm_changed_by=None,
+        user_api_key_dict=user_api_key_dict,
+        litellm_proxy_admin_name=litellm_proxy_admin_name,
+        table_name=LitellmTableNames.USER_TABLE_NAME,
+        after_value=_PASSWORD_CHANGED_AUDIT_VALUES,
+    )
+    return ChangePasswordResponse(user_id=user_id, message="Password updated successfully.")
diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py
index 00cf357d89d..7859c678c07 100644
--- a/litellm/proxy/management_endpoints/ui_sso.py
+++ b/litellm/proxy/management_endpoints/ui_sso.py
@@ -3665,6 +3665,7 @@ class SSOAuthenticationHandler:
             auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"),
             disabled_non_admin_personal_key_creation=disabled_non_admin_personal_key_creation,
             server_root_path=get_server_root_path(),
+            password_reset_required=False,
         )
 
         from litellm.proxy.auth.login_utils import encode_ui_session_jwt
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 3a06753834a..638d685d3e0 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -359,7 +359,7 @@ from litellm.proxy.auth.model_checks import (
     get_mcp_server_ids,
     get_team_models,
 )
-from litellm.proxy.auth.password_policy import validate_password_policy
+from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy
 from litellm.proxy.auth.user_api_key_auth import (
     _fetch_global_spend_with_event_coordination,
     user_api_key_auth,
@@ -601,6 +601,9 @@ from litellm.proxy.management_endpoints.model_management_endpoints import (
 from litellm.proxy.management_endpoints.organization_endpoints import (
     router as organization_router,
 )
+from litellm.proxy.management_endpoints.password_endpoints import (
+    router as password_management_router,
+)
 from litellm.proxy.management_endpoints.router_settings_endpoints import (
     router as router_settings_router,
 )
@@ -16614,6 +16617,7 @@ async def onboarding(invite_link: str, request: Request):
         auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"),
         disabled_non_admin_personal_key_creation=disabled_non_admin_personal_key_creation,
         server_root_path=get_server_root_path(),
+        password_reset_required=False,
     )
     jwt_token: Final = jwt.encode(
         cast(dict, returned_ui_token_object),
@@ -16724,6 +16728,7 @@ async def _generate_onboarding_ui_session_token(user_obj: _UserTableRow) -> str:
         auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"),
         disabled_non_admin_personal_key_creation=disabled_non_admin_personal_key_creation,
         server_root_path=get_server_root_path(),
+        password_reset_required=False,
     )
     assert master_key is not None
     return jwt.encode(
@@ -16794,6 +16799,7 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request):
         )
 
     validate_password_policy(data.password, general_settings)
+    await validate_password_not_breached(data.password, general_settings)
     hashed_pw: Final = hash_password(data.password)
     current_time = litellm.utils.get_utc_datetime()
     async with prisma_client.db.tx() as tx:
@@ -16813,7 +16819,12 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request):
 
         ### UPDATE USER OBJECT ###
         user_obj: Final[_UserTableRow | None] = await tx.litellm_usertable.update(
-            where={"user_id": invite_obj.user_id}, data={"password": hashed_pw}
+            where={"user_id": invite_obj.user_id},
+            data={
+                "password": hashed_pw,
+                "password_reset_required": False,
+                "last_breach_check_at": None,
+            },
         )
 
         if user_obj is None:
@@ -19251,6 +19262,7 @@ app.include_router(pass_through_router)
 app.include_router(health_router)
 app.include_router(key_management_router)
 app.include_router(internal_user_router)
+app.include_router(password_management_router)
 app.include_router(team_router)
 app.include_router(ui_sso_router)
 app.include_router(organization_router)
diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma
index d2032cec0d0..bcafa6dbd0e 100644
--- a/litellm/proxy/schema.prisma
+++ b/litellm/proxy/schema.prisma
@@ -246,6 +246,8 @@ model LiteLLM_UserTable {
     organization_id String?
     object_permission_id String?
     password  String?
+    password_reset_required Boolean?
+    last_breach_check_at DateTime?
     teams    String[] @default([])
     user_role  String?
 		max_budget Float?
diff --git a/litellm/types/llms/custom_http.py b/litellm/types/llms/custom_http.py
index d80d7410aae..793893451df 100644
--- a/litellm/types/llms/custom_http.py
+++ b/litellm/types/llms/custom_http.py
@@ -31,6 +31,7 @@ class httpxSpecialProvider(str, Enum):
     UI = "ui"
     Sandbox = "sandbox"
     ModelCostMap = "model_cost_map"
+    PasswordBreachCheck = "password_breach_check"
 
 
 VerifyTypes = str | bool | ssl.SSLContext
diff --git a/litellm/types/proxy/ui_sso.py b/litellm/types/proxy/ui_sso.py
index 0d7e0b99cf0..03b0b92a4d1 100644
--- a/litellm/types/proxy/ui_sso.py
+++ b/litellm/types/proxy/ui_sso.py
@@ -1,6 +1,6 @@
 from typing import Literal
 
-from typing_extensions import TypedDict
+from typing_extensions import ReadOnly, TypedDict
 
 
 class ReturnedUITokenObject(TypedDict):
@@ -17,6 +17,7 @@ class ReturnedUITokenObject(TypedDict):
     auth_header_name: str
     disabled_non_admin_personal_key_creation: bool
     server_root_path: str  # e.g. `/litellm`
+    password_reset_required: ReadOnly[bool]
 
 
 class ParsedOpenIDResult(TypedDict, total=False):
diff --git a/schema.prisma b/schema.prisma
index d2032cec0d0..bcafa6dbd0e 100644
--- a/schema.prisma
+++ b/schema.prisma
@@ -246,6 +246,8 @@ model LiteLLM_UserTable {
     organization_id String?
     object_permission_id String?
     password  String?
+    password_reset_required Boolean?
+    last_breach_check_at DateTime?
     teams    String[] @default([])
     user_role  String?
 		max_budget Float?
diff --git a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt
index 4ea64b152f1..f8277c83a64 100644
--- a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt
+++ b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt
@@ -86,6 +86,7 @@ POST /team/key/bulk_update
 POST /team/permissions_bulk_update
 POST /team/{team_id}/disable_logging
 POST /user/bulk_update
+POST /user/password/change
 
 # Alternate method or path for functionality the provider already manages elsewhere
 GET /credentials/by_model/{model_id}
diff --git a/tests/test_litellm/proxy/auth/test_login_utils.py b/tests/test_litellm/proxy/auth/test_login_utils.py
index 55ece36252d..d41b90fe566 100644
--- a/tests/test_litellm/proxy/auth/test_login_utils.py
+++ b/tests/test_litellm/proxy/auth/test_login_utils.py
@@ -5,12 +5,15 @@ This module tests the refactored login logic that was moved from proxy_server.py
 to login_utils.py for better reusability.
 """
 
+import hashlib
 import os
 from collections.abc import Mapping
 from contextlib import ExitStack
 from typing import TYPE_CHECKING, Final
+from datetime import datetime, timedelta, timezone
 from unittest.mock import AsyncMock, MagicMock, patch
 
+import httpx
 import pytest
 
 if TYPE_CHECKING:
@@ -34,6 +37,7 @@ def _unlimited_throttle():
 
 
 from litellm.constants import LITELLM_PROXY_ADMIN_NAME
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 from litellm.proxy._types import (
     LiteLLM_UserTable,
     LitellmUserRoles,
@@ -46,8 +50,13 @@ from litellm.proxy.auth.login_utils import (
     authenticate_user,
     get_ui_credentials,
     is_env_credential_login_enabled,
+    screen_login_password_for_breach,
 )
 
+# Successful DB-user logins schedule the background HIBP screen; disable it so
+# no test ever does live network I/O to haveibeenpwned.com from CI.
+_POLICY_NO_BREACH_CHECK = {"password_policy_check_breached_passwords": False}
+
 
 def test_get_ui_credentials_prefers_explicit_password():
     """The configured UI password should be returned when available."""
@@ -326,6 +335,7 @@ async def test_authenticate_user_email_case_insensitive_login():
                 master_key=master_key,
                 prisma_client=mock_prisma_client,
                 throttle=_unlimited_throttle(),
+                general_settings=_POLICY_NO_BREACH_CHECK,
             )
             result_lower = await authenticate_user(
                 username=stored_email,
@@ -333,6 +343,7 @@ async def test_authenticate_user_email_case_insensitive_login():
                 master_key=master_key,
                 prisma_client=mock_prisma_client,
                 throttle=_unlimited_throttle(),
+                general_settings=_POLICY_NO_BREACH_CHECK,
             )
 
     assert result_mixed.user_id == result_lower.user_id == "test-user-123"
@@ -576,6 +587,7 @@ async def test_authenticate_user_database_login_with_non_ascii_password():
                 master_key=master_key,
                 prisma_client=mock_prisma_client,
                 throttle=_unlimited_throttle(),
+                general_settings=_POLICY_NO_BREACH_CHECK,
             )
 
             assert isinstance(result, LoginResult)
@@ -2064,3 +2076,263 @@ class TestIsEnvCredentialLoginEnabled:
         with ExitStack() as stack:
             _patch_sso_configured(stack, configured=False)
             assert is_env_credential_login_enabled({"disable_password_login_when_sso_enabled": True}) is True
+
+
+def _db_user_row(*, password: str, password_reset_required: bool | None = None, last_breach_check_at=None):
+    hashed = hash_token(token=password)
+    row = MagicMock()
+    row.user_id = "reset-user-1"
+    row.user_email = "reset@example.com"
+    row.password = hashed
+    row.user_role = LitellmUserRoles.INTERNAL_USER
+    row.password_reset_required = password_reset_required
+    row.last_breach_check_at = last_breach_check_at
+    return row
+
+
+def _prisma_with_user(row) -> MagicMock:
+    mock_prisma_client = MagicMock()
+    mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=row)
+    mock_prisma_client.db.litellm_usertable.update = AsyncMock(return_value=row)
+    return mock_prisma_client
+
+
+_DB_LOGIN_ENV = {
+    "DATABASE_URL": "postgresql://test:test@localhost/test",
+    "UI_USERNAME": "admin",
+    "UI_PASSWORD": "admin-password",
+}
+
+
+class TestPasswordResetRequiredSessionMinting:
+    """A user flagged `password_reset_required` must receive a UI session key
+    restricted to the change-password endpoint (server-side enforcement, so a
+    script driving the management API with the session key is blocked too);
+    an unflagged user must keep getting an unrestricted key."""
+
+    async def _login(self, mock_prisma_client) -> tuple[LoginResult, dict]:
+        with patch.dict(os.environ, _DB_LOGIN_ENV):
+            with patch(  # test-quality-ok: asserting the minted key's restriction requires seeing its kwargs
+                "litellm.proxy.auth.login_utils.generate_key_helper_fn",
+                new_callable=AsyncMock,
+                return_value={"token": "session-token"},
+            ) as mock_generate_key:
+                result = await authenticate_user(
+                    username="reset@example.com",
+                    password="Str0ng!Passw0rd",
+                    master_key="sk-1234",
+                    prisma_client=mock_prisma_client,
+                    general_settings=_POLICY_NO_BREACH_CHECK,
+                )
+        return result, mock_generate_key.call_args.kwargs
+
+    @pytest.mark.asyncio
+    async def test_flagged_user_gets_key_restricted_to_change_password(self):
+        row = _db_user_row(password="Str0ng!Passw0rd", password_reset_required=True)
+        result, key_kwargs = await self._login(_prisma_with_user(row))
+
+        assert key_kwargs["allowed_routes"] == ["/user/password/change"]
+        assert key_kwargs["metadata"] == {"password_reset_required": True}
+        assert result.password_reset_required is True
+
+    @pytest.mark.asyncio
+    async def test_unflagged_user_gets_unrestricted_key(self):
+        row = _db_user_row(password="Str0ng!Passw0rd", password_reset_required=None)
+        result, key_kwargs = await self._login(_prisma_with_user(row))
+
+        assert key_kwargs["allowed_routes"] is None
+        assert not key_kwargs["metadata"]
+        assert result.password_reset_required is False
+
+    async def _login_with_screen_result(self, mock_prisma_client, breached: bool) -> tuple[LoginResult, dict, dict]:
+        with patch.dict(os.environ, _DB_LOGIN_ENV):
+            with patch(  # test-quality-ok: asserting the minted key's restriction requires seeing its kwargs
+                "litellm.proxy.auth.login_utils.generate_key_helper_fn",
+                new_callable=AsyncMock,
+                return_value={"token": "session-token"},
+            ) as mock_generate_key:
+                with (
+                    patch(  # test-quality-ok: authenticate_user has no HIBP client seam; the screen itself is tested against MockTransport below
+                        "litellm.proxy.auth.login_utils.screen_login_password_for_breach",
+                        new_callable=AsyncMock,
+                        return_value=breached,
+                    ) as mock_screen
+                ):
+                    result = await authenticate_user(
+                        username="reset@example.com",
+                        password="Str0ng!Passw0rd",
+                        master_key="sk-1234",
+                        prisma_client=mock_prisma_client,
+                        general_settings=_POLICY_NO_BREACH_CHECK,
+                    )
+        return result, mock_generate_key.call_args.kwargs, mock_screen.call_args.kwargs
+
+    @pytest.mark.asyncio
+    async def test_login_screens_with_row_state_before_minting(self):
+        """The login must hand the screen the row's recheck timestamp, or the
+        24h throttle can never work."""
+        checked_at = datetime.now(timezone.utc) - timedelta(hours=1)
+        row = _db_user_row(password="Str0ng!Passw0rd", last_breach_check_at=checked_at)
+        mock_prisma_client = _prisma_with_user(row)
+
+        _, _, screen_kwargs = await self._login_with_screen_result(mock_prisma_client, breached=False)
+
+        assert screen_kwargs["user_id"] == "reset-user-1"
+        assert screen_kwargs["password"] == "Str0ng!Passw0rd"
+        assert screen_kwargs["last_breach_check_at"] == checked_at
+        assert screen_kwargs["prisma_client"] is mock_prisma_client
+
+    @pytest.mark.asyncio
+    async def test_fresh_breach_hit_restricts_the_current_session(self):
+        """A breach found during THIS login must restrict THIS session, not
+        just the next one."""
+        row = _db_user_row(password="Str0ng!Passw0rd", password_reset_required=None)
+        mock_prisma_client = _prisma_with_user(row)
+
+        result, key_kwargs, _ = await self._login_with_screen_result(mock_prisma_client, breached=True)
+
+        assert key_kwargs["allowed_routes"] == ["/user/password/change"]
+        assert key_kwargs["metadata"] == {"password_reset_required": True}
+        assert result.password_reset_required is True
+
+
+def _sha1_upper(password: str) -> str:
+    return hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper()
+
+
+def _client_with_transport(handler) -> AsyncHTTPHandler:
+    http_handler = AsyncHTTPHandler()
+    http_handler.client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    return http_handler
+
+
+def _client_returning_breach_hit(password: str) -> AsyncHTTPHandler:
+    body = f"{_sha1_upper(password)[5:]}:42"
+    return _client_with_transport(lambda request: httpx.Response(200, text=body))
+
+
+def _client_returning_no_hit() -> AsyncHTTPHandler:
+    return _client_with_transport(lambda request: httpx.Response(200, text="0000000000000000000000000000000000A:3"))
+
+
+def _client_never_called() -> AsyncHTTPHandler:
+    def handler(request: httpx.Request) -> httpx.Response:
+        raise AssertionError(f"unexpected HTTP call to {request.url}")
+
+    return _client_with_transport(handler)
+
+
+class TestScreenLoginPasswordForBreach:
+    """The awaited login-time screen: flags a breached password for a forced
+    reset, stamps the recheck timestamp, rechecks at most every 24h, returns
+    the breach verdict so the login can restrict the session it is minting,
+    and never raises into the login."""
+
+    @pytest.mark.asyncio
+    async def test_breached_password_sets_reset_flag_and_timestamp(self):
+        password = "Password123!"
+        mock_prisma_client = _prisma_with_user(None)
+
+        breached = await screen_login_password_for_breach(
+            user_id="reset-user-1",
+            password=password,
+            last_breach_check_at=None,
+            general_settings={},
+            prisma_client=mock_prisma_client,
+            client=_client_returning_breach_hit(password),
+        )
+
+        assert breached is True
+        update_kwargs = mock_prisma_client.db.litellm_usertable.update.call_args.kwargs
+        assert update_kwargs["where"] == {"user_id": "reset-user-1"}
+        assert update_kwargs["data"]["password_reset_required"] is True
+        assert isinstance(update_kwargs["data"]["last_breach_check_at"], datetime)
+
+    @pytest.mark.asyncio
+    async def test_clean_password_stamps_timestamp_without_flag(self):
+        mock_prisma_client = _prisma_with_user(None)
+
+        breached = await screen_login_password_for_breach(
+            user_id="reset-user-1",
+            password="Str0ng!Passw0rd",
+            last_breach_check_at=None,
+            general_settings={},
+            prisma_client=mock_prisma_client,
+            client=_client_returning_no_hit(),
+        )
+
+        assert breached is False
+        update_kwargs = mock_prisma_client.db.litellm_usertable.update.call_args.kwargs
+        assert "password_reset_required" not in update_kwargs["data"]
+        assert isinstance(update_kwargs["data"]["last_breach_check_at"], datetime)
+
+    @pytest.mark.asyncio
+    async def test_skips_hibp_when_checked_within_24_hours(self):
+        mock_prisma_client = _prisma_with_user(None)
+
+        breached = await screen_login_password_for_breach(
+            user_id="reset-user-1",
+            password="Password123!",
+            last_breach_check_at=datetime.now(timezone.utc) - timedelta(hours=23),
+            general_settings={},
+            prisma_client=mock_prisma_client,
+            client=_client_never_called(),
+        )
+
+        assert breached is False
+        mock_prisma_client.db.litellm_usertable.update.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_rechecks_when_last_check_is_older_than_24_hours(self):
+        password = "Password123!"
+        mock_prisma_client = _prisma_with_user(None)
+
+        breached = await screen_login_password_for_breach(
+            user_id="reset-user-1",
+            password=password,
+            last_breach_check_at=datetime.now(timezone.utc) - timedelta(hours=25),
+            general_settings={},
+            prisma_client=mock_prisma_client,
+            client=_client_returning_breach_hit(password),
+        )
+
+        assert breached is True
+        assert (
+            mock_prisma_client.db.litellm_usertable.update.call_args.kwargs["data"]["password_reset_required"] is True
+        )
+
+    @pytest.mark.asyncio
+    async def test_skips_hibp_when_check_disabled(self):
+        mock_prisma_client = _prisma_with_user(None)
+
+        breached = await screen_login_password_for_breach(
+            user_id="reset-user-1",
+            password="Password123!",
+            last_breach_check_at=None,
+            general_settings=_POLICY_NO_BREACH_CHECK,
+            prisma_client=mock_prisma_client,
+            client=_client_never_called(),
+        )
+
+        assert breached is False
+        mock_prisma_client.db.litellm_usertable.update.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_db_failure_never_raises_but_still_reports_the_breach(self):
+        """A failed flag write must not fail the login, but the breach verdict
+        still has to restrict the session being minted right now."""
+        password = "Password123!"
+        mock_prisma_client = _prisma_with_user(None)
+        mock_prisma_client.db.litellm_usertable.update = AsyncMock(side_effect=RuntimeError("db down"))
+
+        assert (
+            await screen_login_password_for_breach(
+                user_id="reset-user-1",
+                password=password,
+                last_breach_check_at=None,
+                general_settings={},
+                prisma_client=mock_prisma_client,
+                client=_client_returning_breach_hit(password),
+            )
+            is True
+        )
diff --git a/tests/test_litellm/proxy/auth/test_onboarding.py b/tests/test_litellm/proxy/auth/test_onboarding.py
index 524b655b465..0454aea1239 100644
--- a/tests/test_litellm/proxy/auth/test_onboarding.py
+++ b/tests/test_litellm/proxy/auth/test_onboarding.py
@@ -8,15 +8,20 @@ Covers the security behavior of:
                                   session key only after the password is written
 """
 
+import hashlib
 from datetime import timedelta
 from unittest.mock import AsyncMock, MagicMock, patch
 
+import httpx
 import jwt
 import pytest
+import respx
 from fastapi import HTTPException
 
 import litellm
-from litellm.proxy._types import InvitationClaim
+from litellm.proxy._types import InvitationClaim, ProxyException
+
+_POLICY_NO_BREACH_CHECK = {"password_policy_check_breached_passwords": False}
 
 # ---------------------------------------------------------------------------
 # Helpers
@@ -386,7 +391,9 @@ async def test_claim_token_rejects_concurrent_reuse_before_password_write():
     with (
         patch("litellm.proxy.proxy_server.prisma_client", prisma),
         patch("litellm.proxy.proxy_server.master_key", "sk-test"),
-        patch("litellm.proxy.proxy_server.general_settings", {}),
+        patch(  # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
+        ),
         patch(
             "litellm.proxy.proxy_server.generate_key_helper_fn",
             new_callable=AsyncMock,
@@ -426,7 +433,9 @@ async def test_claim_token_sets_accepted_at_after_password_written():
     with (
         patch("litellm.proxy.proxy_server.prisma_client", prisma),
         patch("litellm.proxy.proxy_server.master_key", "sk-test"),
-        patch("litellm.proxy.proxy_server.general_settings", {}),
+        patch(  # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
+        ),
         patch("litellm.proxy.proxy_server.premium_user", False),
         patch(
             "litellm.proxy.proxy_server.generate_key_helper_fn",
@@ -454,6 +463,10 @@ async def test_claim_token_sets_accepted_at_after_password_written():
     call_kwargs = prisma.db.litellm_usertable.update.call_args
     assert call_kwargs.kwargs["where"] == {"user_id": "user-123"}
     assert "password" in call_kwargs.kwargs["data"]
+    # A freshly claimed, policy-screened password lifts any pending forced
+    # reset and re-arms the login-time breach screen.
+    assert call_kwargs.kwargs["data"]["password_reset_required"] is False
+    assert call_kwargs.kwargs["data"]["last_breach_check_at"] is None
 
     # is_accepted was flipped to True on the invitation link
     prisma.db.litellm_invitationlink.update.assert_called_once()
@@ -483,7 +496,9 @@ async def test_claim_token_rolls_back_invite_when_session_key_mint_fails():
     with (
         patch("litellm.proxy.proxy_server.prisma_client", prisma),
         patch("litellm.proxy.proxy_server.master_key", "sk-test"),
-        patch("litellm.proxy.proxy_server.general_settings", {}),
+        patch(  # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
+        ),
         patch(
             "litellm.proxy.proxy_server.generate_key_helper_fn",
             new_callable=AsyncMock,
@@ -505,3 +520,124 @@ async def test_claim_token_rolls_back_invite_when_session_key_mint_fails():
     }
     assert rollback_kwargs["data"]["accepted_at"] is None
     assert rollback_kwargs["data"]["is_accepted"] is False
+
+
+# ---------------------------------------------------------------------------
+# POST /onboarding/claim_token - password policy
+# ---------------------------------------------------------------------------
+
+
+def _hibp_url_for(password: str) -> str:
+    sha1 = hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper()
+    return f"https://api.pwnedpasswords.com/range/{sha1[:5]}"
+
+
+def _hibp_suffix_for(password: str) -> str:
+    return hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper()[5:]
+
+
+@pytest.mark.asyncio
+async def test_claim_token_rejects_short_password_before_consuming_invite():
+    """Default policy requires 12 characters; the invite must stay claimable."""
+    from litellm.proxy.proxy_server import claim_onboarding_link
+
+    invite = _make_invite(is_accepted=False)
+    prisma = _make_prisma(invite, _make_user())
+    request = _make_claim_request(_make_onboarding_token())
+    data = InvitationClaim(
+        invitation_link="invite-abc",
+        user_id="user-123",
+        password="Sh0rt!pw",
+    )
+
+    with (
+        patch("litellm.proxy.proxy_server.prisma_client", prisma),  # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam
+        patch("litellm.proxy.proxy_server.master_key", "sk-test"),  # test-quality-ok: same as above
+        patch("litellm.proxy.proxy_server.general_settings", {}),  # test-quality-ok: same as above
+    ):
+        with pytest.raises(ProxyException) as exc_info:
+            await claim_onboarding_link(data=data, request=request)
+
+    assert exc_info.value.code == "400"
+    assert "at least 12 characters" in exc_info.value.message
+    prisma.db.litellm_invitationlink.update_many.assert_not_called()
+    prisma.db.litellm_usertable.update.assert_not_called()
+
+
+@pytest.mark.asyncio
+@respx.mock
+async def test_claim_token_rejects_breached_password_before_consuming_invite():
+    """A password found in the HIBP corpus must be rejected and never stored."""
+    from litellm.proxy.proxy_server import claim_onboarding_link
+
+    password = "P@ssword123456"
+    respx.get(_hibp_url_for(password)).mock(
+        return_value=httpx.Response(200, text=f"{_hibp_suffix_for(password)}:1387")
+    )
+
+    invite = _make_invite(is_accepted=False)
+    prisma = _make_prisma(invite, _make_user())
+    request = _make_claim_request(_make_onboarding_token())
+    data = InvitationClaim(
+        invitation_link="invite-abc",
+        user_id="user-123",
+        password=password,
+    )
+
+    with (
+        patch("litellm.proxy.proxy_server.prisma_client", prisma),  # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam
+        patch("litellm.proxy.proxy_server.master_key", "sk-test"),  # test-quality-ok: same as above
+        patch("litellm.proxy.proxy_server.general_settings", {}),  # test-quality-ok: same as above
+    ):
+        with pytest.raises(ProxyException) as exc_info:
+            await claim_onboarding_link(data=data, request=request)
+
+    assert exc_info.value.code == "400"
+    assert "data breaches" in exc_info.value.message
+    prisma.db.litellm_invitationlink.update_many.assert_not_called()
+    prisma.db.litellm_usertable.update.assert_not_called()
+
+
+@pytest.mark.asyncio
+@respx.mock
+async def test_claim_token_fails_open_when_hibp_unreachable():
+    """An HIBP outage must never block onboarding: the claim proceeds."""
+    from litellm.proxy.proxy_server import claim_onboarding_link
+
+    password = "NewP@ssw0rd-2026"
+    respx.get(_hibp_url_for(password)).mock(side_effect=httpx.ConnectError("no route to host"))
+
+    invite = _make_invite(is_accepted=False)
+    user = _make_user()
+    prisma = _make_prisma(invite, user)
+    request = _make_claim_request(_make_onboarding_token())
+    data = InvitationClaim(
+        invitation_link="invite-abc",
+        user_id="user-123",
+        password=password,
+    )
+
+    with (
+        patch("litellm.proxy.proxy_server.prisma_client", prisma),  # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam
+        patch("litellm.proxy.proxy_server.master_key", "sk-test"),  # test-quality-ok: same as above
+        patch("litellm.proxy.proxy_server.general_settings", {}),  # test-quality-ok: same as above
+        patch("litellm.proxy.proxy_server.premium_user", False),  # test-quality-ok: same as above
+        patch(  # test-quality-ok: same as above
+            "litellm.proxy.proxy_server.generate_key_helper_fn",
+            new_callable=AsyncMock,
+            return_value={"token": "sk-generated-key", "user_id": "user-123"},
+        ),
+        patch(  # test-quality-ok: same as above
+            "litellm.proxy.proxy_server.get_custom_url",
+            return_value="http://localhost:4000/",
+        ),
+        patch(  # test-quality-ok: same as above
+            "litellm.proxy.proxy_server.get_disabled_non_admin_personal_key_creation",
+            return_value=False,
+        ),
+        patch("litellm.proxy.proxy_server.get_server_root_path", return_value=""),  # test-quality-ok: same as above
+    ):
+        result = await claim_onboarding_link(data=data, request=request)
+
+    assert "token" in result
+    prisma.db.litellm_usertable.update.assert_called_once()
diff --git a/tests/test_litellm/proxy/auth/test_password_policy.py b/tests/test_litellm/proxy/auth/test_password_policy.py
index f6e7d443907..f9f5025b57f 100644
--- a/tests/test_litellm/proxy/auth/test_password_policy.py
+++ b/tests/test_litellm/proxy/auth/test_password_policy.py
@@ -2,22 +2,56 @@
 Tests for the configurable password-strength policy in
 `litellm.proxy.auth.password_policy`, enforced on every path that persists a
 new or changed password for a locally-managed user.
+
+The breach-check (HIBP) tests inject a real AsyncHTTPHandler wrapping an
+httpx.MockTransport, so no network is touched and nothing is monkeypatched.
 """
 
+import asyncio
+import hashlib
+
+import httpx
 import pytest
 
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 from litellm.proxy._types import ProxyErrorTypes, ProxyException
 from litellm.proxy.auth.password_policy import (
     DEFAULT_MIN_LENGTH,
     MIN_ALLOWED_LENGTH,
     PasswordPolicy,
     get_password_policy,
+    validate_password_not_breached,
     validate_password_policy,
+    validate_passwords_bulk,
 )
 
 STRONG_PASSWORD = "Str0ng!Passw0rd"
 
 
+def _sha1_upper(password: str) -> str:
+    return hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper()
+
+
+def _client_with_transport(handler) -> AsyncHTTPHandler:
+    http_handler = AsyncHTTPHandler()
+    http_handler.client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    return http_handler
+
+
+def _client_never_called() -> AsyncHTTPHandler:
+    def handler(request: httpx.Request) -> httpx.Response:
+        raise AssertionError(f"unexpected HTTP call to {request.url}")
+
+    return _client_with_transport(handler)
+
+
+def _client_returning(body: str, status_code: int = 200) -> AsyncHTTPHandler:
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(status_code, text=body)
+
+    return _client_with_transport(handler)
+
+
 def test_get_password_policy_defaults_to_pif_baseline():
     policy = get_password_policy({})
     assert policy == PasswordPolicy(
@@ -134,3 +168,178 @@ def test_validate_password_policy_rejects_unicode_letter_as_special_character():
 def test_validate_password_policy_accepts_real_special_character_with_unicode_letters():
     """Same base password as the rejection test above, plus an actual symbol."""
     assert validate_password_policy("Passwörd1234!", {}) is None
+
+
+@pytest.mark.asyncio
+async def test_breach_check_skipped_when_disabled():
+    result = await validate_password_not_breached(
+        password="password12345",  # breached in reality, but the check is off
+        general_settings={"password_policy_check_breached_passwords": False},
+        client=_client_never_called(),
+    )
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_rejects_breached_password():
+    password = "correct horse battery staple"
+    sha1 = _sha1_upper(password)
+    body = f"AAAA000000000000000000000000000000A:0\r\n{sha1[5:]}:42\r\nBBBB000000000000000000000000000000B:7"
+
+    with pytest.raises(ProxyException) as exc_info:
+        await validate_password_not_breached(password=password, general_settings={}, client=_client_returning(body))
+    assert exc_info.value.code == "400"
+    assert exc_info.value.type == ProxyErrorTypes.validation_error
+    assert exc_info.value.param == "password"
+    assert "data breaches" in exc_info.value.message
+
+
+@pytest.mark.asyncio
+async def test_only_sha1_prefix_leaves_the_proxy():
+    password = "a very secret password"
+    sha1 = _sha1_upper(password)
+    captured_requests: list[httpx.Request] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        captured_requests.append(request)
+        return httpx.Response(200, text="0000000000000000000000000000000000A:1")
+
+    result = await validate_password_not_breached(
+        password=password, general_settings={}, client=_client_with_transport(handler)
+    )
+    assert result is None
+
+    (request,) = captured_requests
+    assert request.url.path == f"/range/{sha1[:5]}"
+    assert sha1[5:] not in str(request.url)
+    assert request.headers["Add-Padding"] == "true"
+    assert "litellm" in request.headers["User-Agent"]
+
+
+@pytest.mark.asyncio
+async def test_ignores_padding_entries_with_zero_count():
+    """HIBP padding entries (requested via Add-Padding) carry count 0 and must
+    not be treated as breaches when they collide with the password's suffix."""
+    password = "a padded-away password"
+    sha1 = _sha1_upper(password)
+
+    result = await validate_password_not_breached(
+        password=password, general_settings={}, client=_client_returning(f"{sha1[5:]}:0")
+    )
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_accepts_password_absent_from_breach_corpus():
+    result = await validate_password_not_breached(
+        password="a genuinely novel password",
+        general_settings={},
+        client=_client_returning("0018A45C4D1DEF81644B54AB7F969B88D65:1\r\n00D4F6E8FA6EECAD2A3AA415EEC418D38EC:2"),
+    )
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_breach_check_fails_open_on_network_error():
+    def handler(request: httpx.Request) -> httpx.Response:
+        raise httpx.ConnectError("no route to host")
+
+    result = await validate_password_not_breached(
+        password="password12345",  # breached, but HIBP is unreachable
+        general_settings={},
+        client=_client_with_transport(handler),
+    )
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_breach_check_fails_open_on_http_error_status():
+    result = await validate_password_not_breached(
+        password="password12345",
+        general_settings={},
+        client=_client_returning("service unavailable", status_code=503),
+    )
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_breach_check_fails_open_on_malformed_response_body():
+    result = await validate_password_not_breached(
+        password="password12345",
+        general_settings={},
+        client=_client_returning(f"{_sha1_upper('password12345')[5:]}:not-a-number"),
+    )
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_validate_passwords_bulk_screens_concurrently():
+    """All HIBP lookups for a batch must be in flight at once: each handler
+    call stalls until every expected request has arrived, and a handler that
+    gives up waiting reports the password as breached. Serial awaiting (the
+    old per-user behavior) leaves each earlier request waiting forever for the
+    later ones, so every verdict comes back as a breach and the test fails."""
+    passwords = ("Uniqu3!Passw0rd-a", "Uniqu3!Passw0rd-b", "Uniqu3!Passw0rd-c")
+    suffix_by_prefix = {_sha1_upper(p)[:5]: _sha1_upper(p)[5:] for p in passwords}
+    all_arrived = asyncio.Event()
+    arrivals: list[str] = []
+
+    async def handler(request: httpx.Request) -> httpx.Response:
+        arrivals.append(request.url.path)
+        if len(arrivals) == len(passwords):
+            all_arrived.set()
+        try:
+            await asyncio.wait_for(all_arrived.wait(), timeout=5)
+        except TimeoutError:
+            return httpx.Response(200, text=f"{suffix_by_prefix[request.url.path.rsplit('/', 1)[-1]]}:1")
+        return httpx.Response(200, text="0000000000000000000000000000000000A:1")
+
+    verdicts = await validate_passwords_bulk(passwords, {}, client=_client_with_transport(handler))
+    assert set(arrivals) == {f"/range/{prefix}" for prefix in suffix_by_prefix}
+    assert all(verdicts[p] is None for p in passwords)
+
+
+@pytest.mark.asyncio
+async def test_validate_passwords_bulk_deduplicates_lookups():
+    """500 users sharing one password must cost exactly one HIBP lookup."""
+    password = "Sh@red-Passw0rd!"
+    request_count = 0
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        nonlocal request_count
+        request_count += 1
+        return httpx.Response(200, text="0000000000000000000000000000000000A:1")
+
+    verdicts = await validate_passwords_bulk((password,) * 500, {}, client=_client_with_transport(handler))
+    assert request_count == 1
+    assert verdicts == {password: None}
+
+
+@pytest.mark.asyncio
+async def test_validate_passwords_bulk_mixed_verdicts():
+    """Weak passwords are rejected without an HIBP lookup; breached ones get
+    the breach error; acceptable ones map to None."""
+    breached = "Br3ached!Passw0rd"
+    clean = "Cl3an!!Passw0rd42"
+    weak = "short1!"
+    breached_sha1 = _sha1_upper(breached)
+    looked_up_prefixes: list[str] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        looked_up_prefixes.append(request.url.path.rsplit("/", 1)[-1])
+        if request.url.path == f"/range/{breached_sha1[:5]}":
+            return httpx.Response(200, text=f"{breached_sha1[5:]}:99")
+        return httpx.Response(200, text="0000000000000000000000000000000000A:1")
+
+    verdicts = await validate_passwords_bulk((breached, clean, weak), {}, client=_client_with_transport(handler))
+    assert _sha1_upper(weak)[:5] not in looked_up_prefixes
+    assert verdicts[clean] is None
+    assert "data breaches" in verdicts[breached].message
+    assert verdicts[breached].code == "400"
+    assert "12 characters" in verdicts[weak].message
+
+
+@pytest.mark.asyncio
+async def test_validate_passwords_bulk_empty_batch_makes_no_lookups():
+    verdicts = await validate_passwords_bulk((), {}, client=_client_never_called())
+    assert verdicts == {}
diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py
index 603a8686692..8382c7842b0 100644
--- a/tests/test_litellm/proxy/auth/test_route_checks.py
+++ b/tests/test_litellm/proxy/auth/test_route_checks.py
@@ -3,7 +3,6 @@ from datetime import datetime
 from typing import Final
 from unittest.mock import MagicMock, patch
 
-
 import pytest
 from fastapi import HTTPException, Request
 
@@ -39,7 +38,7 @@ def test_non_admin_config_update_route_rejected():
     request.query_params = {}
 
     # Test that calling /config/update route raises HTTPException with 403 status
-    with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info:
+    with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info:
         RouteChecks.non_proxy_admin_allowed_routes_check(
             user_obj=user_obj,
             _user_role=LitellmUserRoles.INTERNAL_USER.value,
@@ -50,9 +49,8 @@ def test_non_admin_config_update_route_rejected():
         )
 
     # Verify the exception is raised with the correct message
-    assert (
-        "Only proxy admin can be used to generate, delete, update info for new keys/users/teams"
-        in str(exc_info.value)
+    assert "Only proxy admin can be used to generate, delete, update info for new keys/users/teams" in str(
+        exc_info.value
     )
     assert "Route=/config/update" in str(exc_info.value)
     assert "Your role=internal_user" in str(exc_info.value)
@@ -131,7 +129,7 @@ def test_user_banner_update_rejected_for_non_admin():
     request = MagicMock(spec=Request)
     request.query_params = {}
 
-    with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info:
+    with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info:
         RouteChecks.non_proxy_admin_allowed_routes_check(
             user_obj=user_obj,
             _user_role=LitellmUserRoles.INTERNAL_USER.value,
@@ -706,9 +704,7 @@ def test_virtual_key_llm_api_route_includes_passthrough_prefix(route):
 
     valid_token = UserAPIKeyAuth(user_id="test_user", allowed_routes=["llm_api_routes"])
 
-    result = RouteChecks.is_virtual_key_allowed_to_call_route(
-        route=route, valid_token=valid_token
-    )
+    result = RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token)
 
     assert result is True
 
@@ -733,9 +729,7 @@ def test_virtual_key_llm_api_routes_allows_google_routes(route):
 
     valid_token = UserAPIKeyAuth(user_id="test_user", allowed_routes=["llm_api_routes"])
 
-    result = RouteChecks.is_virtual_key_allowed_to_call_route(
-        route=route, valid_token=valid_token
-    )
+    result = RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token)
 
     assert result is True
 
@@ -805,18 +799,14 @@ def test_google_routes_with_dynamic_model_names_accessible_to_internal_users():
         )
         # If no exception is raised, the test passes
     except Exception as e:
-        pytest.fail(
-            f"Internal user should be able to access Google generateContent route. Got error: {str(e)}"
-        )
+        pytest.fail(f"Internal user should be able to access Google generateContent route. Got error: {e!s}")
 
 
 def test_virtual_key_allowed_routes_with_multiple_litellm_routes_member_names():
     """Test that virtual key works with multiple LiteLLMRoutes member names in allowed_routes"""
 
     # Create a UserAPIKeyAuth with multiple LiteLLMRoutes member names
-    valid_token = UserAPIKeyAuth(
-        user_id="test_user", allowed_routes=["openai_routes", "info_routes"]
-    )
+    valid_token = UserAPIKeyAuth(user_id="test_user", allowed_routes=["openai_routes", "info_routes"])
 
     # Test that routes from both groups are allowed
     result1 = RouteChecks.is_virtual_key_allowed_to_call_route(
@@ -870,13 +860,9 @@ def test_virtual_key_allowed_routes_with_no_member_names_only_explicit():
     )
 
     # Test that explicit routes are allowed
-    result1 = RouteChecks.is_virtual_key_allowed_to_call_route(
-        route="/chat/completions", valid_token=valid_token
-    )
+    result1 = RouteChecks.is_virtual_key_allowed_to_call_route(route="/chat/completions", valid_token=valid_token)
 
-    result2 = RouteChecks.is_virtual_key_allowed_to_call_route(
-        route="/custom/route", valid_token=valid_token
-    )
+    result2 = RouteChecks.is_virtual_key_allowed_to_call_route(route="/custom/route", valid_token=valid_token)
 
     assert result1 is True
     assert result2 is True
@@ -1274,9 +1260,7 @@ def test_virtual_key_without_llm_api_routes_cannot_access_pass_through():
             )
 
         assert exc_info.value.status_code == 403
-        assert "Virtual key is not allowed to call this route" in str(
-            exc_info.value.detail
-        )
+        assert "Virtual key is not allowed to call this route" in str(exc_info.value.detail)
 
 
 def test_check_passthrough_route_access_key_metadata_exact_match():
@@ -1735,9 +1719,7 @@ def test_videos_route_accessible_to_internal_users():
         )
         # If no exception is raised, the test passes
     except Exception as e:
-        pytest.fail(
-            f"Internal user should be able to access /v1/videos route. Got error: {str(e)}"
-        )
+        pytest.fail(f"Internal user should be able to access /v1/videos route. Got error: {e!s}")
 
 
 def test_videos_route_with_virtual_key_llm_api_routes():
@@ -1759,12 +1741,8 @@ def test_videos_route_with_virtual_key_llm_api_routes():
     ]
 
     for route in test_routes:
-        result = RouteChecks.is_virtual_key_allowed_to_call_route(
-            route=route, valid_token=valid_token
-        )
-        assert (
-            result is True
-        ), f"Virtual key with llm_api_routes should be able to access {route}"
+        result = RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token)
+        assert result is True, f"Virtual key with llm_api_routes should be able to access {route}"
 
 
 def test_non_proxy_admin_wildcard_allowed_routes():
@@ -1835,9 +1813,7 @@ def test_proxy_admin_viewer_can_access_global_spend_tags():
         )
         # If no exception is raised, the test passes
     except Exception as e:
-        pytest.fail(
-            f"proxy_admin_viewer should be able to access /global/spend/tags route. Got error: {str(e)}"
-        )
+        pytest.fail(f"proxy_admin_viewer should be able to access /global/spend/tags route. Got error: {e!s}")
 
 
 # Routes returning proxy-wide spend across every team / customer / api_key.
@@ -1865,7 +1841,7 @@ def test_internal_user_blocked_from_global_spend_routes(route):
     request = MagicMock(spec=Request)
     request.query_params = {}
 
-    with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info:
+    with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info:
         RouteChecks.non_proxy_admin_allowed_routes_check(
             user_obj=user_obj,
             _user_role=LitellmUserRoles.INTERNAL_USER.value,
@@ -1894,7 +1870,7 @@ def test_internal_user_view_only_blocked_from_global_spend_routes(route):
     request = MagicMock(spec=Request)
     request.query_params = {}
 
-    with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info:
+    with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info:
         RouteChecks.non_proxy_admin_allowed_routes_check(
             user_obj=user_obj,
             _user_role=LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value,
@@ -1996,9 +1972,7 @@ def test_proxy_admin_viewer_can_access_audit_logs(route):
             request_data={},
         )
     except Exception as e:
-        pytest.fail(
-            f"proxy_admin_viewer should be able to access {route} route. Got error: {str(e)}"
-        )
+        pytest.fail(f"proxy_admin_viewer should be able to access {route} route. Got error: {e!s}")
 
 
 # ── Admin Viewer parity: Logs page endpoints ──────────────────────────────────
@@ -2061,9 +2035,7 @@ def test_proxy_admin_viewer_can_access_logs_page_endpoints(route):
             request_data={},
         )
     except Exception as e:
-        pytest.fail(
-            f"proxy_admin_viewer should be able to access {route}. Got error: {str(e)}"
-        )
+        pytest.fail(f"proxy_admin_viewer should be able to access {route}. Got error: {e!s}")
 
 
 @pytest.mark.parametrize(
@@ -2173,7 +2145,7 @@ def test_internal_user_blocked_from_admin_viewer_logs_routes(route):
     if route not in INTERNAL_USER_BLOCKED_SUBSET:
         return
 
-    with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info:
+    with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info:
         RouteChecks.non_proxy_admin_allowed_routes_check(
             user_obj=user_obj,
             _user_role=LitellmUserRoles.INTERNAL_USER.value,
@@ -2249,9 +2221,7 @@ def test_proxy_admin_viewer_can_access_settings_read_endpoints(route):
             request_data={},
         )
     except Exception as e:
-        pytest.fail(
-            f"proxy_admin_viewer should be able to access {route}. Got error: {str(e)}"
-        )
+        pytest.fail(f"proxy_admin_viewer should be able to access {route}. Got error: {e!s}")
 
 
 # ── Admin Viewer parity: default-allow GET semantics ─────────────────────────
@@ -2450,9 +2420,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints:
         )
         local_file = os.path.abspath(local_file)
 
-        spec = importlib.util.spec_from_file_location(
-            "local_enterprise_route_checks", local_file
-        )
+        spec = importlib.util.spec_from_file_location("local_enterprise_route_checks", local_file)
         mod = importlib.util.module_from_spec(spec)
         spec.loader.exec_module(mod)
         return mod.EnterpriseRouteChecks
@@ -2463,9 +2431,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints:
         EnterpriseRouteChecks = self._get_enterprise_route_checks()
 
         with (
-            patch.object(
-                EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True
-            ),
+            patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True),
             patch.object(
                 EnterpriseRouteChecks,
                 "is_management_routes_disabled",
@@ -2481,9 +2447,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints:
         EnterpriseRouteChecks = self._get_enterprise_route_checks()
 
         with (
-            patch.object(
-                EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True
-            ),
+            patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True),
             patch.object(
                 EnterpriseRouteChecks,
                 "is_management_routes_disabled",
@@ -2499,9 +2463,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints:
         EnterpriseRouteChecks = self._get_enterprise_route_checks()
 
         with (
-            patch.object(
-                EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True
-            ),
+            patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True),
             patch.object(
                 EnterpriseRouteChecks,
                 "is_management_routes_disabled",
@@ -2512,9 +2474,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints:
                 EnterpriseRouteChecks.should_call_route("/v1/chat/completions")
 
             assert exc_info.value.status_code == 403
-            assert "LLM API routes are disabled for this instance." in str(
-                exc_info.value.detail
-            )
+            assert "LLM API routes are disabled for this instance." in str(exc_info.value.detail)
 
     @patch("litellm.proxy.proxy_server.premium_user", True)
     def test_should_embeddings_still_blocked_when_llm_api_disabled(self):
@@ -2522,9 +2482,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints:
         EnterpriseRouteChecks = self._get_enterprise_route_checks()
 
         with (
-            patch.object(
-                EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True
-            ),
+            patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True),
             patch.object(
                 EnterpriseRouteChecks,
                 "is_management_routes_disabled",
@@ -2542,9 +2500,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints:
         EnterpriseRouteChecks = self._get_enterprise_route_checks()
 
         with (
-            patch.object(
-                EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=False
-            ),
+            patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=False),
             patch.object(
                 EnterpriseRouteChecks,
                 "is_management_routes_disabled",
@@ -2563,9 +2519,7 @@ def test_route_in_additional_public_routes_wildcard_match():
     from litellm.proxy.auth.auth_utils import route_in_additonal_public_routes
 
     with (
-        patch(
-            "litellm.proxy.proxy_server.general_settings", {"public_routes": ["/api/*"]}
-        ),
+        patch("litellm.proxy.proxy_server.general_settings", {"public_routes": ["/api/*"]}),
         patch("litellm.proxy.proxy_server.premium_user", True),
     ):
         # Wildcard should match subpaths
@@ -2657,7 +2611,7 @@ def test_non_admin_non_team_admin_cannot_access_config_update_but_can_attempt_re
     )
 
     # /config/update is still blocked
-    with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info:
+    with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info:
         RouteChecks.non_proxy_admin_allowed_routes_check(
             user_obj=user_obj,
             _user_role=LitellmUserRoles.INTERNAL_USER.value,
@@ -2745,8 +2699,6 @@ def test_available_roles_accessible_to_non_admin_users(user_role):
 # ── _user_is_org_admin tests ──────────────────────────────────────────────────
 
 
-
-
 def _make_org_admin_user(org_id: str) -> LiteLLM_UserTable:
     membership = LiteLLM_OrganizationMembershipTable(
         user_id="org-admin-user",
@@ -2869,9 +2821,7 @@ async def test_add_team_org_context_noop_when_org_id_already_present():
         raise AssertionError("must not resolve when organization_id is present")
 
     body = {"team_id": "team-1", "organization_id": "org-explicit"}
-    out = await add_team_org_context_to_request_body(
-        route="/team/update", request_body=body, fetch_team_org_id=fetch
-    )
+    out = await add_team_org_context_to_request_body(route="/team/update", request_body=body, fetch_team_org_id=fetch)
     assert out == body
 
 
@@ -2883,9 +2833,7 @@ async def test_add_team_org_context_noop_for_other_routes():
         raise AssertionError("must not resolve for a non-opted-in route")
 
     body = {"team_id": "team-1"}
-    out = await add_team_org_context_to_request_body(
-        route="/team/delete", request_body=body, fetch_team_org_id=fetch
-    )
+    out = await add_team_org_context_to_request_body(route="/team/delete", request_body=body, fetch_team_org_id=fetch)
     assert out == body
 
 
@@ -2898,9 +2846,7 @@ async def test_add_team_org_context_noop_when_team_has_no_org():
         return None
 
     body = {"team_id": "team-1"}
-    out = await add_team_org_context_to_request_body(
-        route="/team/update", request_body=body, fetch_team_org_id=fetch
-    )
+    out = await add_team_org_context_to_request_body(route="/team/update", request_body=body, fetch_team_org_id=fetch)
     assert out == body
 
 
@@ -3171,9 +3117,7 @@ async def test_initialize_pass_through_registers_wildcard_for_auth_subpath():
             # Removing the endpoint should clean up openai_routes
             # remove_endpoint_routes takes endpoint_id (UUID portion of
             # the route key "{id}:exact:{path}:{methods}")
-            registered = (
-                InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes()
-            )
+            registered = InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes()
             endpoint_ids = {k.split(":")[0] for k in registered}
             for eid in endpoint_ids:
                 InitPassThroughEndpointHelpers.remove_endpoint_routes(eid)
@@ -3183,9 +3127,7 @@ async def test_initialize_pass_through_registers_wildcard_for_auth_subpath():
         LiteLLMRoutes.openai_routes.value[:] = original_routes
         # Clean up any routes registered during this test to avoid
         # polluting the module-level _registered_pass_through_routes
-        registered = (
-            InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes()
-        )
+        registered = InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes()
         for k in registered:
             InitPassThroughEndpointHelpers.remove_endpoint_routes(k.split(":")[0])
 
@@ -3216,8 +3158,7 @@ def test_provider_name_substring_not_classified_as_llm_route(route):
     from litellm.proxy.auth.route_checks import RouteChecks
 
     assert RouteChecks.is_llm_api_route(route=route) is False, (
-        f"{route!r} should NOT be classified as an LLM API route — "
-        "provider-name substring match bypass"
+        f"{route!r} should NOT be classified as an LLM API route — provider-name substring match bypass"
     )
 
 
@@ -3239,9 +3180,7 @@ def test_legitimate_passthrough_routes_still_classified_as_llm_route(route):
     """Legitimate passthrough routes must still pass is_llm_api_route."""
     from litellm.proxy.auth.route_checks import RouteChecks
 
-    assert (
-        RouteChecks.is_llm_api_route(route=route) is True
-    ), f"{route!r} should be classified as an LLM API route"
+    assert RouteChecks.is_llm_api_route(route=route) is True, f"{route!r} should be classified as an LLM API route"
 
 
 @pytest.mark.parametrize(
@@ -3299,7 +3238,7 @@ def test_internal_user_blocked_from_search_tool_writes(route):
     request = MagicMock(spec=Request)
     request.query_params = {}
 
-    with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info:
+    with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info:
         RouteChecks.non_proxy_admin_allowed_routes_check(
             user_obj=user_obj,
             _user_role=LitellmUserRoles.INTERNAL_USER.value,
@@ -3675,12 +3614,7 @@ def test_agent_inference_routes_stay_llm_api(route):
 def test_agent_routes_union_still_covers_both_halves(route):
     """Keys configured with allowed_routes=["agent_routes"] must keep both halves."""
 
-    assert (
-        RouteChecks.check_route_access(
-            route=route, allowed_routes=LiteLLMRoutes.agent_routes.value
-        )
-        is True
-    )
+    assert RouteChecks.check_route_access(route=route, allowed_routes=LiteLLMRoutes.agent_routes.value) is True
 
 
 @pytest.mark.parametrize("route", AGENT_MANAGEMENT_ROUTES)
@@ -3734,6 +3668,136 @@ def test_agent_registry_route_gate_open_to_non_admin_roles(user_role, method, ro
         valid_token=valid_token,
         request_data={},
     )
+
+
+def test_proxy_admin_viewer_user_update_password_param_rejected():
+    """The self-service /user/update password carve-out is closed: non-admins
+    change their own password through /user/password/change, which verifies
+    the current password. Admin password sets don't pass through this check."""
+    with pytest.raises(HTTPException) as exc_info:
+        RouteChecks._check_proxy_admin_viewer_access(
+            route="/user/update",
+            _user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value,
+            request_data={"password": "hunter2hunter2"},
+        )
+    assert exc_info.value.status_code == 403
+    assert "password" in str(exc_info.value.detail)
+
+
+def test_proxy_admin_viewer_user_update_user_email_still_allowed():
+    request = MagicMock(spec=Request)
+    request.method = "POST"
+
+    allowed = RouteChecks._check_proxy_admin_viewer_access(
+        route="/user/update",
+        _user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value,
+        request_data={"user_email": "viewer@example.com"},
+        request=request,
+    )
+
+    assert allowed is None
+
+
+def test_proxy_admin_viewer_can_change_own_password():
+    request = MagicMock(spec=Request)
+    request.method = "POST"
+
+    allowed = RouteChecks._check_proxy_admin_viewer_access(
+        route="/user/password/change",
+        _user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value,
+        request_data={"current_password": "a", "new_password": "b"},
+        request=request,
+    )
+
+    assert allowed is None
+
+
+@pytest.mark.parametrize(
+    "user_role",
+    [
+        LitellmUserRoles.INTERNAL_USER.value,
+        LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value,
+    ],
+)
+def test_non_admin_roles_can_change_own_password(user_role):
+    valid_token = UserAPIKeyAuth(user_id="test_user", user_role=user_role)
+    request = MagicMock(spec=Request)
+    request.method = "POST"
+    request.query_params = {}
+
+    allowed = RouteChecks.non_proxy_admin_allowed_routes_check(
+        user_obj=LiteLLM_UserTable(user_id="test_user", user_role=user_role),
+        _user_role=user_role,
+        route="/user/password/change",
+        request=request,
+        valid_token=valid_token,
+        request_data={"current_password": "a", "new_password": "b"},
+    )
+
+    assert allowed is None
+
+
+def _password_reset_session_token() -> UserAPIKeyAuth:
+    """The UI session key `authenticate_user` mints for a user flagged
+    `password_reset_required`."""
+    return UserAPIKeyAuth(
+        user_id="flagged_user",
+        allowed_routes=["/user/password/change"],
+        metadata={"password_reset_required": True},
+    )
+
+
+def test_password_reset_session_can_reach_change_password():
+    result = RouteChecks.is_virtual_key_allowed_to_call_route(
+        route="/user/password/change",
+        valid_token=_password_reset_session_token(),
+    )
+
+    assert result is True
+
+
+@pytest.mark.parametrize(
+    "route",
+    [
+        "/user/info",
+        "/key/generate",
+        "/user/update",
+        "/chat/completions",
+    ],
+)
+def test_password_reset_session_is_blocked_everywhere_else_with_reset_message(route):
+    """Server-side enforcement of the forced reset: a script that logs in via
+    /v2/login and drives the management API with the session key must get a 403
+    naming the remediation endpoint, on every route but the change-password one."""
+    with pytest.raises(HTTPException) as exc_info:
+        RouteChecks.is_virtual_key_allowed_to_call_route(
+            route=route,
+            valid_token=_password_reset_session_token(),
+        )
+
+    assert exc_info.value.status_code == 403
+    assert "password must be changed" in str(exc_info.value.detail)
+    assert "/user/password/change" in str(exc_info.value.detail)
+
+
+def test_restricted_key_without_reset_marker_keeps_generic_message():
+    """The reset-specific 403 must not leak onto ordinary allowed_routes keys."""
+    valid_token = UserAPIKeyAuth(
+        user_id="test_user",
+        allowed_routes=["/chat/completions"],
+    )
+
+    with pytest.raises(HTTPException) as exc_info:
+        RouteChecks.is_virtual_key_allowed_to_call_route(
+            route="/user/info",
+            valid_token=valid_token,
+        )
+
+    assert exc_info.value.status_code == 403
+    assert "password must be changed" not in str(exc_info.value.detail)
+    assert "not allowed to call this route" in str(exc_info.value.detail)
+
+
 TEAM_CALLBACK_ROUTES = (
     "/team/06bda574-5ca9-43d3-beb8-3b23c2f17112/callback",
     "/team/06bda574-5ca9-43d3-beb8-3b23c2f17112/callback/langfuse",
diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py
index b8f1aa0330b..6e11dbb6bae 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py
@@ -1,13 +1,18 @@
+import hashlib
 import json
 from datetime import datetime, timezone
 from types import SimpleNamespace
 from typing import Final
 
+import httpx
 import pytest
+import respx
 from fastapi import HTTPException
 from fastapi.testclient import TestClient
 from pytest_mock import MockerFixture
 
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+
 from litellm.proxy._types import (
     LiteLLM_UserTableFiltered,
     LitellmUserRoles,
@@ -67,9 +72,7 @@ async def test_ui_view_users_with_null_email(mocker, caplog):
 
     # Proxy admin: no org filter, no get_user_object call
     response = await ui_view_users(
-        user_api_key_dict=UserAPIKeyAuth(
-            user_id="test_user", user_role=LitellmUserRoles.PROXY_ADMIN
-        ),
+        user_api_key_dict=UserAPIKeyAuth(user_id="test_user", user_role=LitellmUserRoles.PROXY_ADMIN),
         user_id="test_user",
         user_email=None,
         team_id=None,
@@ -77,9 +80,7 @@ async def test_ui_view_users_with_null_email(mocker, caplog):
         page_size=50,
     )
 
-    assert response == [
-        LiteLLM_UserTableFiltered(user_id="test-user-null-email", user_email=None)
-    ]
+    assert response == [LiteLLM_UserTableFiltered(user_id="test-user-null-email", user_email=None)]
 
 
 @pytest.mark.asyncio
@@ -103,9 +104,7 @@ async def test_ui_view_users_proxy_admin_no_org_filter(mocker):
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     await ui_view_users(
-        user_api_key_dict=UserAPIKeyAuth(
-            user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN
-        ),
+        user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
         user_id=None,
         user_email="foo",
         team_id=None,
@@ -128,9 +127,7 @@ async def test_ui_view_users_org_admin_filtered_by_org(mocker):
     async def mock_find_many(*args, **kwargs):
         where = kwargs.get("where") or {}
         assert "organization_memberships" in where
-        assert where["organization_memberships"] == {
-            "some": {"organization_id": {"in": [org_id]}}
-        }
+        assert where["organization_memberships"] == {"some": {"organization_id": {"in": [org_id]}}}
         return []
 
     mock_prisma_client.db.litellm_usertable.find_many = mock_find_many
@@ -268,9 +265,7 @@ async def test_ui_view_users_flag_on_team_admin_org_team(mocker):
     async def mock_find_many(*args, **kwargs):
         where = kwargs.get("where") or {}
         assert "organization_memberships" in where
-        assert where["organization_memberships"] == {
-            "some": {"organization_id": {"in": [org_id]}}
-        }
+        assert where["organization_memberships"] == {"some": {"organization_id": {"in": [org_id]}}}
         return []
 
     mock_prisma_client.db.litellm_usertable.find_many = mock_find_many
@@ -401,9 +396,7 @@ async def test_ui_view_users_flag_on_team_admin_org_member_no_team_id(mocker):
     async def mock_find_many(*args, **kwargs):
         where = kwargs.get("where") or {}
         assert "organization_memberships" in where
-        assert where["organization_memberships"] == {
-            "some": {"organization_id": {"in": [org_id]}}
-        }
+        assert where["organization_memberships"] == {"some": {"organization_id": {"in": [org_id]}}}
         return []
 
     mock_prisma_client.db.litellm_usertable.find_many = mock_find_many
@@ -462,9 +455,7 @@ async def test_ui_view_users_flag_on_team_admin_not_in_org_resolves_via_key_team
     async def mock_find_many(*args, **kwargs):
         where = kwargs.get("where") or {}
         assert "organization_memberships" in where
-        assert where["organization_memberships"] == {
-            "some": {"organization_id": {"in": [org_id]}}
-        }
+        assert where["organization_memberships"] == {"some": {"organization_id": {"in": [org_id]}}}
         return []
 
     mock_prisma_client.db.litellm_usertable.find_many = mock_find_many
@@ -507,9 +498,7 @@ async def test_ui_view_users_flag_on_team_admin_not_in_org_resolves_via_key_team
 
     # No team_id query param, but team_id on the API key
     response = await ui_view_users(
-        user_api_key_dict=UserAPIKeyAuth(
-            user_id="team-admin-no-org", user_role=None, team_id=tid
-        ),
+        user_api_key_dict=UserAPIKeyAuth(user_id="team-admin-no-org", user_role=None, team_id=tid),
         user_id=None,
         user_email="u",
         team_id=None,
@@ -538,13 +527,9 @@ def test_user_daily_activity_types():
     # Assert all fields in SpendMetrics are reported in DailySpendMetadata as "total_"
     for field in spend_metrics.__dict__:
         if field.startswith("total_"):
-            assert hasattr(
-                daily_spend_metadata, field
-            ), f"Field {field} is not reported in DailySpendMetadata"
+            assert hasattr(daily_spend_metadata, field), f"Field {field} is not reported in DailySpendMetadata"
         else:
-            assert not hasattr(
-                daily_spend_metadata, field
-            ), f"Field {field} is reported in DailySpendMetadata"
+            assert not hasattr(daily_spend_metadata, field), f"Field {field} is reported in DailySpendMetadata"
 
 
 @pytest.mark.asyncio
@@ -591,9 +576,7 @@ async def test_get_users_includes_timestamps(mocker):
 
     # Call get_users function directly with proxy admin auth
     admin_key = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
-    response = await get_users(
-        page=1, page_size=1, user_api_key_dict=admin_key, organization_ids=None
-    )
+    response = await get_users(page=1, page_size=1, user_api_key_dict=admin_key, organization_ids=None)
 
     print("user /list response: ", response)
 
@@ -654,14 +637,10 @@ async def test_get_users_redacts_scim_enterprise_metadata(mocker):
     )
 
     admin_key = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
-    response = await get_users(
-        page=1, page_size=1, user_api_key_dict=admin_key, organization_ids=None
-    )
+    response = await get_users(page=1, page_size=1, user_api_key_dict=admin_key, organization_ids=None)
 
     listed = response["users"][0]
-    assert listed.metadata == {
-        "scim_metadata": {"givenName": "Jane", "familyName": "Doe"}
-    }
+    assert listed.metadata == {"scim_metadata": {"givenName": "Jane", "familyName": "Doe"}}
     assert "scim_enterprise" not in (listed.metadata or {})
 
 
@@ -853,9 +832,7 @@ async def test_new_user_license_over_limit(mocker):
     mocker.patch("litellm.proxy.proxy_server._license_check", mock_license_check)
 
     # Create test request data
-    user_request = NewUserRequest(
-        user_email="test@example.com", user_role="internal_user"
-    )
+    user_request = NewUserRequest(user_email="test@example.com", user_role="internal_user")
 
     # Mock user_api_key_dict
     mock_user_api_key_dict = UserAPIKeyAuth(user_id="test_admin")
@@ -916,9 +893,7 @@ async def test_new_user_license_gate_counts_only_billable_users(mocker):
     request = NewUserRequest(user_role="internal_user")
 
     # 2 active + 3 deactivated -> billable 2, not over max_users 2: gate passes
-    mocker.patch(
-        "litellm.proxy.proxy_server.prisma_client", _prisma(total=5, deactivated=3)
-    )
+    mocker.patch("litellm.proxy.proxy_server.prisma_client", _prisma(total=5, deactivated=3))
     with pytest.raises(ProxyException) as passed:
         await new_user(data=request, user_api_key_dict=admin)
     assert key_gen.call_count == 1
@@ -926,9 +901,7 @@ async def test_new_user_license_gate_counts_only_billable_users(mocker):
 
     # 3 active, 0 deactivated -> billable 3, over max_users 2: gate blocks
     key_gen.reset_mock()
-    mocker.patch(
-        "litellm.proxy.proxy_server.prisma_client", _prisma(total=3, deactivated=0)
-    )
+    mocker.patch("litellm.proxy.proxy_server.prisma_client", _prisma(total=3, deactivated=0))
     with pytest.raises(ProxyException) as blocked:
         await new_user(data=request, user_api_key_dict=admin)
     assert blocked.value.code == 403 or blocked.value.code == "403"
@@ -978,14 +951,10 @@ async def test_new_user_non_admin_cannot_create_admin(mocker):
     mocker.patch("litellm.proxy.proxy_server._license_check", mock_license_check)
 
     # Test Case 1: INTERNAL_USER trying to create PROXY_ADMIN
-    user_request = NewUserRequest(
-        user_email="admin@example.com", user_role=LitellmUserRoles.PROXY_ADMIN
-    )
+    user_request = NewUserRequest(user_email="admin@example.com", user_role=LitellmUserRoles.PROXY_ADMIN)
 
     # Mock user_api_key_dict with non-admin role
-    mock_user_api_key_dict = UserAPIKeyAuth(
-        user_id="test_internal_user", user_role=LitellmUserRoles.INTERNAL_USER
-    )
+    mock_user_api_key_dict = UserAPIKeyAuth(user_id="test_internal_user", user_role=LitellmUserRoles.INTERNAL_USER)
 
     # Call new_user function and expect ProxyException
     with pytest.raises(ProxyException) as exc_info:
@@ -993,9 +962,7 @@ async def test_new_user_non_admin_cannot_create_admin(mocker):
 
     # Verify the exception details
     assert exc_info.value.code == 403 or exc_info.value.code == "403"
-    assert "Only proxy admins can create administrative users" in str(
-        exc_info.value.message
-    )
+    assert "Only proxy admins can create administrative users" in str(exc_info.value.message)
     assert "proxy_admin" in str(exc_info.value.message)
     assert "proxy_admin_viewer" in str(exc_info.value.message)
     assert str(LitellmUserRoles.PROXY_ADMIN) in str(exc_info.value.message)
@@ -1008,15 +975,11 @@ async def test_new_user_non_admin_cannot_create_admin(mocker):
     )
 
     with pytest.raises(ProxyException) as exc_info2:
-        await new_user(
-            data=user_request_viewer, user_api_key_dict=mock_user_api_key_dict
-        )
+        await new_user(data=user_request_viewer, user_api_key_dict=mock_user_api_key_dict)
 
     # Verify the exception details
     assert exc_info2.value.code == 403 or exc_info2.value.code == "403"
-    assert "Only proxy admins can create administrative users" in str(
-        exc_info2.value.message
-    )
+    assert "Only proxy admins can create administrative users" in str(exc_info2.value.message)
     assert str(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) in str(exc_info2.value.message)
 
 
@@ -1055,9 +1018,7 @@ async def test_new_user_non_admin_permissions_non_empty_rejected(mocker):
         user_role=LitellmUserRoles.INTERNAL_USER,
         permissions={"get_spend_routes": True},
     )
-    caller = UserAPIKeyAuth(
-        user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN
-    )
+    caller = UserAPIKeyAuth(user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN)
 
     with pytest.raises(ProxyException) as exc_info:
         await new_user(data=data, user_api_key_dict=caller)
@@ -1101,9 +1062,7 @@ async def test_new_user_non_admin_permissions_explicit_empty_rejected(mocker):
         permissions={},
     )
     assert "permissions" in data.model_fields_set
-    caller = UserAPIKeyAuth(
-        user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN
-    )
+    caller = UserAPIKeyAuth(user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN)
 
     with pytest.raises(ProxyException) as exc_info:
         await new_user(data=data, user_api_key_dict=caller)
@@ -1156,9 +1115,7 @@ async def test_new_user_non_admin_omits_permissions_succeeds(mocker):
         user_role=LitellmUserRoles.INTERNAL_USER,
     )
     assert "permissions" not in data.model_fields_set
-    caller = UserAPIKeyAuth(
-        user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN
-    )
+    caller = UserAPIKeyAuth(user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN)
 
     result = await new_user(data=data, user_api_key_dict=caller)
     assert result is not None
@@ -1232,14 +1189,10 @@ async def test_update_single_user_non_admin_permissions_rejected(mocker):
         user_id="alice",
         permissions={"get_spend_routes": True},
     )
-    caller = UserAPIKeyAuth(
-        user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN
-    )
+    caller = UserAPIKeyAuth(user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN)
 
     with pytest.raises(HTTPException) as exc_info:
-        await _update_single_user_helper(
-            user_request=data, user_api_key_dict=caller
-        )
+        await _update_single_user_helper(user_request=data, user_api_key_dict=caller)
     assert exc_info.value.status_code == 403
     assert "permissions" in str(exc_info.value.detail)
 
@@ -1261,14 +1214,10 @@ async def test_update_single_user_non_admin_permissions_explicit_empty_rejected(
 
     data = UpdateUserRequest(user_id="alice", permissions={})
     assert "permissions" in data.model_fields_set
-    caller = UserAPIKeyAuth(
-        user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN
-    )
+    caller = UserAPIKeyAuth(user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN)
 
     with pytest.raises(HTTPException) as exc_info:
-        await _update_single_user_helper(
-            user_request=data, user_api_key_dict=caller
-        )
+        await _update_single_user_helper(user_request=data, user_api_key_dict=caller)
     assert exc_info.value.status_code == 403
     assert "permissions" in str(exc_info.value.detail)
 
@@ -1324,15 +1273,11 @@ async def test_user_info_url_encoding_plus_character(mocker):
     mock_request.url.query = "user_id=machine-user+alp-air-admin-b58-b@tempus.com"
 
     # Mock user_api_key_dict
-    mock_user_api_key_dict = UserAPIKeyAuth(
-        user_id="test_admin", user_role="proxy_admin"
-    )
+    mock_user_api_key_dict = UserAPIKeyAuth(user_id="test_admin", user_role="proxy_admin")
 
     # Call user_info function with the URL-decoded user_id (as FastAPI would pass it)
     # FastAPI would normally convert + to space, but our fix should handle this
-    decoded_user_id = (
-        "machine-user alp-air-admin-b58-b@tempus.com"  # What FastAPI gives us
-    )
+    decoded_user_id = "machine-user alp-air-admin-b58-b@tempus.com"  # What FastAPI gives us
     expected_user_id = "machine-user+alp-air-admin-b58-b@tempus.com"
 
     response = await user_info(
@@ -1383,9 +1328,7 @@ async def test_user_info_nonexistent_user(mocker):
     mock_request = mocker.MagicMock(spec=Request)
 
     # Mock user_api_key_dict
-    mock_user_api_key_dict = UserAPIKeyAuth(
-        user_id="test_admin", user_role="proxy_admin"
-    )
+    mock_user_api_key_dict = UserAPIKeyAuth(user_id="test_admin", user_role="proxy_admin")
 
     # Call user_info function with a non-existent user_id
     nonexistent_user_id = "nonexistent-user@example.com"
@@ -1423,14 +1366,10 @@ async def test_user_info_no_user_id_view_only_admin_gets_proxy_admin_payload(moc
         mock_get_user_info_for_proxy_admin,
     )
 
-    viewer = UserAPIKeyAuth(
-        user_id="viewer", user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value
-    )
+    viewer = UserAPIKeyAuth(user_id="viewer", user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value)
     mock_request = mocker.MagicMock(spec=Request)
 
-    response = await user_info(
-        user_id=None, user_api_key_dict=viewer, request=mock_request
-    )
+    response = await user_info(user_id=None, user_api_key_dict=viewer, request=mock_request)
 
     mock_get_user_info_for_proxy_admin.assert_awaited_once_with(user_api_key_dict=viewer)
     assert response is admin_payload
@@ -1457,9 +1396,7 @@ async def test_new_user_default_teams_flow(mocker):
     mock_prisma_client.db.litellm_usertable.count = mock_count
     persisted_user_row = mocker.MagicMock()
     persisted_user_row.teams = ["96fed65b-0182-4ff4-8429-2721cd7d42af"]
-    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(
-        return_value=persisted_user_row
-    )
+    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(return_value=persisted_user_row)
 
     # Mock duplicate checks to pass
     async def mock_check_duplicate_user_email(*args, **kwargs):
@@ -1527,26 +1464,20 @@ async def test_new_user_default_teams_flow(mocker):
         )
 
         # Create test request data WITHOUT teams (teams should come from defaults)
-        user_request = NewUserRequest(
-            user_email="test@example.com", user_role="internal_user"
-        )
+        user_request = NewUserRequest(user_email="test@example.com", user_role="internal_user")
 
         # Mock user_api_key_dict
         mock_user_api_key_dict = UserAPIKeyAuth(user_id="test_admin")
 
         # Call new_user function
-        response = await new_user(
-            data=user_request, user_api_key_dict=mock_user_api_key_dict
-        )
+        response = await new_user(data=user_request, user_api_key_dict=mock_user_api_key_dict)
 
         # Verify generate_key_helper_fn was called WITHOUT teams
         mock_generate_key_helper_fn.assert_called_once()
         call_kwargs = mock_generate_key_helper_fn.call_args.kwargs
 
         # Teams should be removed from the data passed to generate_key_helper_fn
-        assert (
-            "teams" not in call_kwargs
-        ), "Teams should not be passed to generate_key_helper_fn"
+        assert "teams" not in call_kwargs, "Teams should not be passed to generate_key_helper_fn"
         assert call_kwargs["request_type"] == "user"
         assert call_kwargs["user_email"] == "test@example.com"
         assert call_kwargs["user_role"] == "internal_user"
@@ -1591,24 +1522,16 @@ def test_update_internal_new_user_params_proxy_admin_role():
 
     try:
         # Create test data with PROXY_ADMIN role
-        data = NewUserRequest(
-            user_email="admin@example.com", user_role=LitellmUserRoles.PROXY_ADMIN.value
-        )
+        data = NewUserRequest(user_email="admin@example.com", user_role=LitellmUserRoles.PROXY_ADMIN.value)
         data_json = data.model_dump(exclude_unset=True)
 
         # Call the function
         result = _update_internal_new_user_params(data_json=data_json, data=data)
 
         # Assertions - default params should NOT be applied for PROXY_ADMIN
-        assert (
-            "max_budget" not in result
-        ), "Default max_budget should NOT be applied to PROXY_ADMIN"
-        assert (
-            "models" not in result
-        ), "Default models should NOT be applied to PROXY_ADMIN"
-        assert (
-            "tpm_limit" not in result
-        ), "Default tpm_limit should NOT be applied to PROXY_ADMIN"
+        assert "max_budget" not in result, "Default max_budget should NOT be applied to PROXY_ADMIN"
+        assert "models" not in result, "Default models should NOT be applied to PROXY_ADMIN"
+        assert "tpm_limit" not in result, "Default tpm_limit should NOT be applied to PROXY_ADMIN"
 
         # These should still work
         assert result["user_email"] == "admin@example.com"
@@ -1722,15 +1645,9 @@ async def test_check_duplicate_user_email_case_insensitive(mocker):
         user_email_clause = where_clause.get("user_email", {})
 
         # Check that the query structure is correct for case insensitive search
-        assert (
-            "equals" in user_email_clause
-        ), "Query should use 'equals' for case insensitive search"
-        assert (
-            user_email_clause.get("mode") == "insensitive"
-        ), "Query should use 'insensitive' mode"
-        assert (
-            user_email_clause.get("equals") == "user@example.com"
-        ), "Query should search for the provided email"
+        assert "equals" in user_email_clause, "Query should use 'equals' for case insensitive search"
+        assert user_email_clause.get("mode") == "insensitive", "Query should use 'insensitive' mode"
+        assert user_email_clause.get("equals") == "user@example.com", "Query should search for the provided email"
 
         return mock_existing_user  # Return existing user to simulate duplicate
 
@@ -1741,9 +1658,7 @@ async def test_check_duplicate_user_email_case_insensitive(mocker):
         await _check_duplicate_user_email("user@example.com", mock_prisma_client)
 
     assert exc_info.value.status_code == 409
-    assert "User with email User@Example.com already exists" in str(
-        exc_info.value.detail
-    )
+    assert "User with email User@Example.com already exists" in str(exc_info.value.detail)
 
     # Test Case 2: No duplicate found
     async def mock_find_first_no_duplicate(*args, **kwargs):
@@ -1768,9 +1683,7 @@ async def test_check_duplicate_user_email_case_insensitive(mocker):
         pytest.fail(f"Should not raise exception when no duplicate found, but got: {e}")
 
     # Test Case 3: None email should not cause issues
-    await _check_duplicate_user_email(
-        None, mock_prisma_client
-    )  # Should not raise exception
+    await _check_duplicate_user_email(None, mock_prisma_client)  # Should not raise exception
 
 
 @pytest.mark.asyncio
@@ -1880,9 +1793,7 @@ def test_process_keys_for_user_info_filters_dashboard_keys(monkeypatch):
 
     # Verify dashboard key is not in results
     result_team_ids = [key.get("team_id") for key in result]
-    assert (
-        UI_SESSION_TOKEN_TEAM_ID not in result_team_ids
-    ), "Dashboard key should be filtered out"
+    assert UI_SESSION_TOKEN_TEAM_ID not in result_team_ids, "Dashboard key should be filtered out"
 
     # Verify regular keys are included
     assert "regular-team" in result_team_ids, "Regular team key should be included"
@@ -1892,9 +1803,7 @@ def test_process_keys_for_user_info_filters_dashboard_keys(monkeypatch):
     result_tokens = [key.get("token") for key in result]
     assert "sk-regular-token" in result_tokens, "Regular key should be included"
     assert "sk-no-team-token" in result_tokens, "No-team key should be included"
-    assert (
-        "sk-dashboard-token" not in result_tokens
-    ), "Dashboard key should not be included"
+    assert "sk-dashboard-token" not in result_tokens, "Dashboard key should not be included"
 
 
 def test_process_keys_for_user_info_handles_none_keys(monkeypatch):
@@ -2419,9 +2328,7 @@ async def test_get_user_daily_activity_non_admin_cannot_view_other_users(monkeyp
         )
 
     assert exc_info.value.status_code == 403
-    assert "Non-admin users can only view their own spend data" in str(
-        exc_info.value.detail
-    )
+    assert "Non-admin users can only view their own spend data" in str(exc_info.value.detail)
 
     # Case 2: Non-admin omits user_id — should default to their own user_id
     mock_response = MagicMock()
@@ -2708,14 +2615,10 @@ async def test_delete_user_cleans_up_created_by_invitation_links(mocker):
     async def mock_find_unique(*args, **kwargs):
         return mock_user_row
 
-    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(
-        side_effect=mock_find_unique
-    )
+    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique)
 
     # Mock find_many for teams (no teams)
-    mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock(
-        return_value=[]
-    )
+    mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock(return_value=[])
 
     # Mock all delete_many calls
     mock_prisma_client.db.litellm_verificationtoken.find_many = mocker.AsyncMock(
@@ -2741,9 +2644,7 @@ async def test_delete_user_cleans_up_created_by_invitation_links(mocker):
 
     # Call delete_user
     data = DeleteUserRequest(user_ids=["admin-creator"])
-    user_api_key_dict = UserAPIKeyAuth(
-        user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN
-    )
+    user_api_key_dict = UserAPIKeyAuth(user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN)
 
     await delete_user(data=data, user_api_key_dict=user_api_key_dict)
 
@@ -2752,9 +2653,7 @@ async def test_delete_user_cleans_up_created_by_invitation_links(mocker):
     call_kwargs = mock_prisma_client.db.litellm_invitationlink.delete_many.call_args
     where_clause = call_kwargs.kwargs.get("where") or call_kwargs[1].get("where")
 
-    assert (
-        "OR" in where_clause
-    ), "Should use OR to match user_id, created_by, and updated_by"
+    assert "OR" in where_clause, "Should use OR to match user_id, created_by, and updated_by"
     or_conditions = where_clause["OR"]
     assert len(or_conditions) == 3, "Should have 3 OR conditions"
 
@@ -2875,9 +2774,7 @@ async def test_delete_user_rejects_org_admin_deleting_outside_scope(mocker):
     async def mock_find_unique(*args, **kwargs):
         return mock_target_user
 
-    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(
-        side_effect=mock_find_unique
-    )
+    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique)
 
     # Caller (org_admin_user) administers org-A.
     caller_membership = mocker.MagicMock()
@@ -2903,16 +2800,12 @@ async def test_delete_user_rejects_org_admin_deleting_outside_scope(mocker):
             return [caller_membership]
         return []
 
-    mock_prisma_client.db.litellm_organizationmembership.find_many = mocker.AsyncMock(
-        side_effect=mock_find_memberships
-    )
+    mock_prisma_client.db.litellm_organizationmembership.find_many = mocker.AsyncMock(side_effect=mock_find_memberships)
 
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     data = DeleteUserRequest(user_ids=["victim"])
-    user_api_key_dict = UserAPIKeyAuth(
-        user_id="org_admin_user", user_role=LitellmUserRoles.ORG_ADMIN
-    )
+    user_api_key_dict = UserAPIKeyAuth(user_id="org_admin_user", user_role=LitellmUserRoles.ORG_ADMIN)
 
     with pytest.raises(HTTPException) as exc:
         await delete_user(data=data, user_api_key_dict=user_api_key_dict)
@@ -2920,11 +2813,8 @@ async def test_delete_user_rejects_org_admin_deleting_outside_scope(mocker):
 
     # Critical: no delete_many calls should have executed.
     assert (
-        not hasattr(
-            mock_prisma_client.db.litellm_verificationtoken.delete_many, "mock_calls"
-        )
-        or len(mock_prisma_client.db.litellm_verificationtoken.delete_many.mock_calls)
-        == 0
+        not hasattr(mock_prisma_client.db.litellm_verificationtoken.delete_many, "mock_calls")
+        or len(mock_prisma_client.db.litellm_verificationtoken.delete_many.mock_calls) == 0
     )
 
 
@@ -2943,9 +2833,7 @@ async def test_user_update_rejects_silent_create_for_non_proxy_admin(mocker):
 
     mock_prisma_client = mocker.MagicMock()
     # user_email lookup yields None → would silently create pre-fix.
-    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(
-        return_value=None
-    )
+    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=None)
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     user_request = UpdateUserRequest(
@@ -2959,9 +2847,7 @@ async def test_user_update_rejects_silent_create_for_non_proxy_admin(mocker):
     )
 
     with pytest.raises(HTTPException) as exc:
-        await _update_single_user_helper(
-            user_request=user_request, user_api_key_dict=org_admin
-        )
+        await _update_single_user_helper(user_request=user_request, user_api_key_dict=org_admin)
     assert exc.value.status_code == 404
 
 
@@ -3005,17 +2891,13 @@ async def test_user_info_v2_proxy_admin_can_query_any_user(mocker):
             return mock_user_row
         return None
 
-    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(
-        side_effect=mock_find_unique
-    )
+    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique)
 
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     mock_request = mocker.MagicMock(spec=Request)
 
-    admin_key = UserAPIKeyAuth(
-        user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN
-    )
+    admin_key = UserAPIKeyAuth(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN)
 
     response = await user_info_v2(
         request=mock_request,
@@ -3069,17 +2951,13 @@ async def test_user_info_v2_redacts_scim_enterprise_metadata(mocker):
             return mock_user_row
         return None
 
-    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(
-        side_effect=mock_find_unique
-    )
+    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique)
 
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     mock_request = mocker.MagicMock(spec=Request)
 
-    admin_key = UserAPIKeyAuth(
-        user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN
-    )
+    admin_key = UserAPIKeyAuth(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN)
 
     response = await user_info_v2(
         request=mock_request,
@@ -3088,9 +2966,7 @@ async def test_user_info_v2_redacts_scim_enterprise_metadata(mocker):
     )
 
     assert isinstance(response, UserInfoV2Response)
-    assert response.metadata == {
-        "scim_metadata": {"givenName": "Jane", "familyName": "Doe"}
-    }
+    assert response.metadata == {"scim_metadata": {"givenName": "Jane", "familyName": "Doe"}}
     assert "scim_enterprise" not in (response.metadata or {})
 
 
@@ -3159,17 +3035,13 @@ async def test_user_info_v2_internal_user_can_query_self(mocker):
             return mock_user_row
         return None
 
-    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(
-        side_effect=mock_find_unique
-    )
+    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique)
 
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     mock_request = mocker.MagicMock(spec=Request)
 
-    user_key = UserAPIKeyAuth(
-        user_id="self-user", user_role=LitellmUserRoles.INTERNAL_USER
-    )
+    user_key = UserAPIKeyAuth(user_id="self-user", user_role=LitellmUserRoles.INTERNAL_USER)
 
     response = await user_info_v2(
         request=mock_request,
@@ -3204,17 +3076,13 @@ async def test_user_info_v2_internal_user_cannot_query_other(mocker):
             return mock_caller_row
         return None
 
-    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(
-        side_effect=mock_find_unique
-    )
+    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique)
 
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     mock_request = mocker.MagicMock(spec=Request)
 
-    user_key = UserAPIKeyAuth(
-        user_id="caller-user", user_role=LitellmUserRoles.INTERNAL_USER
-    )
+    user_key = UserAPIKeyAuth(user_id="caller-user", user_role=LitellmUserRoles.INTERNAL_USER)
 
     with pytest.raises(ProxyException) as exc_info:
         await user_info_v2(
@@ -3261,17 +3129,13 @@ async def test_user_info_v2_no_user_id_defaults_to_self(mocker):
             return mock_user_row
         return None
 
-    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(
-        side_effect=mock_find_unique
-    )
+    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique)
 
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     mock_request = mocker.MagicMock(spec=Request)
 
-    user_key = UserAPIKeyAuth(
-        user_id="my-user-id", user_role=LitellmUserRoles.INTERNAL_USER
-    )
+    user_key = UserAPIKeyAuth(user_id="my-user-id", user_role=LitellmUserRoles.INTERNAL_USER)
 
     # Call without user_id
     response = await user_info_v2(
@@ -3299,17 +3163,13 @@ async def test_user_info_v2_nonexistent_user_returns_404(mocker):
     async def mock_find_unique(*args, **kwargs):
         return None
 
-    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(
-        side_effect=mock_find_unique
-    )
+    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique)
 
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     mock_request = mocker.MagicMock(spec=Request)
 
-    admin_key = UserAPIKeyAuth(
-        user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN
-    )
+    admin_key = UserAPIKeyAuth(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN)
 
     with pytest.raises(ProxyException) as exc_info:
         await user_info_v2(
@@ -3357,17 +3217,13 @@ async def test_user_info_v2_response_shape(mocker):
     async def mock_find_unique(*args, **kwargs):
         return mock_user_row
 
-    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(
-        side_effect=mock_find_unique
-    )
+    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique)
 
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     mock_request = mocker.MagicMock(spec=Request)
 
-    admin_key = UserAPIKeyAuth(
-        user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN
-    )
+    admin_key = UserAPIKeyAuth(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN)
 
     response = await user_info_v2(
         request=mock_request,
@@ -3402,9 +3258,7 @@ async def test_user_info_v2_response_shape(mocker):
 
     # The dashboard's user edit form hydrates its per-model budget rows from
     # these two, so dropping them makes a save replace the user's budgets.
-    assert response_dict["model_max_budget"] == {
-        "gpt-3.5-turbo": {"budget_limit": 5.0, "time_period": "30d"}
-    }
+    assert response_dict["model_max_budget"] == {"gpt-3.5-turbo": {"budget_limit": 5.0, "time_period": "30d"}}
     assert response_dict["model_max_budget_usage"] == {
         "gpt-3.5-turbo": {"current_spend": 0.0, "budget_limit": 5.0, "time_period": "30d"}
     }
@@ -3463,9 +3317,7 @@ async def test_user_info_v2_team_admin_can_query_team_member(mocker):
             return mock_target
         return None
 
-    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(
-        side_effect=mock_find_unique
-    )
+    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique)
 
     # Mock team with caller as admin
     mock_team = mocker.MagicMock()
@@ -3482,17 +3334,13 @@ async def test_user_info_v2_team_admin_can_query_team_member(mocker):
     async def mock_find_many_teams(*args, **kwargs):
         return [mock_team]
 
-    mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock(
-        side_effect=mock_find_many_teams
-    )
+    mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock(side_effect=mock_find_many_teams)
 
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     mock_request = mocker.MagicMock(spec=Request)
 
-    team_admin_key = UserAPIKeyAuth(
-        user_id="team-admin-user", user_role=LitellmUserRoles.INTERNAL_USER
-    )
+    team_admin_key = UserAPIKeyAuth(user_id="team-admin-user", user_role=LitellmUserRoles.INTERNAL_USER)
 
     response = await user_info_v2(
         request=mock_request,
@@ -3532,9 +3380,7 @@ async def test_user_info_v2_team_admin_cannot_query_non_team_member(mocker):
             return mock_target
         return None
 
-    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(
-        side_effect=mock_find_unique
-    )
+    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique)
 
     # Mock team where caller is admin
     mock_team = mocker.MagicMock()
@@ -3550,17 +3396,13 @@ async def test_user_info_v2_team_admin_cannot_query_non_team_member(mocker):
     async def mock_find_many_teams(*args, **kwargs):
         return [mock_team]
 
-    mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock(
-        side_effect=mock_find_many_teams
-    )
+    mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock(side_effect=mock_find_many_teams)
 
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     mock_request = mocker.MagicMock(spec=Request)
 
-    team_admin_key = UserAPIKeyAuth(
-        user_id="team-admin-user", user_role=LitellmUserRoles.INTERNAL_USER
-    )
+    team_admin_key = UserAPIKeyAuth(user_id="team-admin-user", user_role=LitellmUserRoles.INTERNAL_USER)
 
     with pytest.raises(ProxyException) as exc_info:
         await user_info_v2(
@@ -3610,18 +3452,14 @@ async def test_user_info_v2_url_encoding_plus_character(mocker):
             return mock_user_row
         return None
 
-    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(
-        side_effect=mock_find_unique
-    )
+    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique)
 
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     mock_request = mocker.MagicMock(spec=Request)
     mock_request.url.query = f"user_id={expected_user_id}"
 
-    admin_key = UserAPIKeyAuth(
-        user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN
-    )
+    admin_key = UserAPIKeyAuth(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN)
 
     # Simulate FastAPI converting + to space
     decoded_user_id = "machine-user admin@example.com"
@@ -3718,9 +3556,7 @@ def test_enforce_user_info_access_admin_bypass():
         _enforce_user_info_access,
     )
 
-    admin = UserAPIKeyAuth(
-        user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN.value
-    )
+    admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN.value)
     # Should not raise even when querying a different user
     _enforce_user_info_access(user_id="someone_else", user_api_key_dict=admin)
 
@@ -3759,9 +3595,7 @@ def test_enforce_user_info_access_owner_allowed():
         _enforce_user_info_access,
     )
 
-    user = UserAPIKeyAuth(
-        user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value
-    )
+    user = UserAPIKeyAuth(user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value)
     _enforce_user_info_access(user_id="alice", user_api_key_dict=user)
 
 
@@ -3773,9 +3607,7 @@ def test_enforce_user_info_access_no_user_id_allowed():
         _enforce_user_info_access,
     )
 
-    user = UserAPIKeyAuth(
-        user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value
-    )
+    user = UserAPIKeyAuth(user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value)
     _enforce_user_info_access(user_id=None, user_api_key_dict=user)
 
 
@@ -3832,9 +3664,7 @@ async def test_ghsa_wvg4_non_admin_cannot_self_escalate_max_budget(mocker, budge
         "max_budget": 100,
     }
     existing_user.user_id = "user-1"
-    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(
-        return_value=existing_user
-    )
+    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user)
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     user_request = UpdateUserRequest.model_validate({"user_id": "user-1", budget_field: budget_value})
@@ -3844,9 +3674,7 @@ async def test_ghsa_wvg4_non_admin_cannot_self_escalate_max_budget(mocker, budge
     )
 
     with pytest.raises(HTTPException) as exc:
-        await _update_single_user_helper(
-            user_request=user_request, user_api_key_dict=caller
-        )
+        await _update_single_user_helper(user_request=user_request, user_api_key_dict=caller)
     assert exc.value.status_code == 403
     assert budget_field in str(exc.value.detail)
     mock_prisma_client.update_data.assert_not_called()
@@ -3868,9 +3696,7 @@ async def test_ghsa_wvg4_non_admin_cannot_self_escalate_spend(mocker):
         "spend": 50.0,
     }
     existing_user.user_id = "user-1"
-    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(
-        return_value=existing_user
-    )
+    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user)
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     user_request = UpdateUserRequest(
@@ -3883,9 +3709,7 @@ async def test_ghsa_wvg4_non_admin_cannot_self_escalate_spend(mocker):
     )
 
     with pytest.raises(HTTPException) as exc:
-        await _update_single_user_helper(
-            user_request=user_request, user_api_key_dict=caller
-        )
+        await _update_single_user_helper(user_request=user_request, user_api_key_dict=caller)
     assert exc.value.status_code == 403
     assert "spend" in str(exc.value.detail)
 
@@ -3904,12 +3728,8 @@ async def test_ghsa_wvg4_proxy_admin_can_update_user_budget(mocker):
         "max_budget": 100,
     }
     existing_user.user_id = "target-user"
-    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(
-        return_value=existing_user
-    )
-    mock_prisma_client.update_data = mocker.AsyncMock(
-        return_value={"user_id": "target-user", "max_budget": 500}
-    )
+    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user)
+    mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "target-user", "max_budget": 500})
     mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x)
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
     mocker.patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin")
@@ -3923,9 +3743,7 @@ async def test_ghsa_wvg4_proxy_admin_can_update_user_budget(mocker):
         user_role=LitellmUserRoles.PROXY_ADMIN,
     )
 
-    result = await _update_single_user_helper(
-        user_request=user_request, user_api_key_dict=admin_caller
-    )
+    result = await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller)
     assert result is not None
 
 
@@ -3941,12 +3759,8 @@ async def test_admin_user_update_spend_invalidates_counter(mocker):
     existing_user = mocker.MagicMock()
     existing_user.model_dump.return_value = {"user_id": "target-user", "spend": 50.0}
     existing_user.user_id = "target-user"
-    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(
-        return_value=existing_user
-    )
-    mock_prisma_client.update_data = mocker.AsyncMock(
-        return_value={"user_id": "target-user", "spend": -25.0}
-    )
+    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user)
+    mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "target-user", "spend": -25.0})
     mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x)
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
     mocker.patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin")
@@ -3961,13 +3775,9 @@ async def test_admin_user_update_spend_invalidates_counter(mocker):
     # without raising the recurring budget ceiling. Future changes should
     # continue allowing negative spend counters.
     user_request = UpdateUserRequest(user_id="target-user", spend=-25)
-    admin_caller = UserAPIKeyAuth(
-        user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN
-    )
+    admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN)
 
-    await _update_single_user_helper(
-        user_request=user_request, user_api_key_dict=admin_caller
-    )
+    await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller)
     mock_invalidate.assert_awaited_once_with(counter_key="spend:user:target-user")
 
 
@@ -3984,9 +3794,7 @@ async def test_user_update_rejects_non_finite_spend(mocker):
     existing_user = mocker.MagicMock()
     existing_user.model_dump.return_value = {"user_id": "target-user", "spend": 50.0}
     existing_user.user_id = "target-user"
-    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(
-        return_value=existing_user
-    )
+    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user)
     mock_prisma_client.update_data = mocker.AsyncMock()
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
     mocker.patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin")
@@ -3996,14 +3804,10 @@ async def test_user_update_rejects_non_finite_spend(mocker):
     )
 
     user_request = UpdateUserRequest(user_id="target-user", spend=float("nan"))
-    admin_caller = UserAPIKeyAuth(
-        user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN
-    )
+    admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN)
 
     with pytest.raises(HTTPException) as exc:
-        await _update_single_user_helper(
-            user_request=user_request, user_api_key_dict=admin_caller
-        )
+        await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller)
     assert exc.value.status_code == 400
     mock_prisma_client.update_data.assert_not_called()
     mock_invalidate.assert_not_awaited()
@@ -4023,9 +3827,7 @@ async def test_resolve_user_email_metadata_maps_page_user_ids_to_email(mocker):
     mock_prisma_client = mocker.MagicMock()
     find_many = mocker.AsyncMock(
         return_value=[
-            SimpleNamespace(
-                user_id="u1", user_email="alice@example.com", user_alias="Alice"
-            ),
+            SimpleNamespace(user_id="u1", user_email="alice@example.com", user_alias="Alice"),
             SimpleNamespace(user_id="u2", user_email=None, user_alias="bob-alias"),
         ]
     )
@@ -4224,19 +4026,13 @@ def _object_permission_mocks(mocker, existing_object_permission_id=None):
     }
     existing_user.user_id = "target-user"
     existing_user.object_permission_id = existing_object_permission_id
-    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(
-        return_value=existing_user
-    )
-    mock_prisma_client.db.litellm_objectpermissiontable.find_unique = mocker.AsyncMock(
-        return_value=None
-    )
+    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user)
+    mock_prisma_client.db.litellm_objectpermissiontable.find_unique = mocker.AsyncMock(return_value=None)
     mock_prisma_client.db.litellm_objectpermissiontable.upsert = mocker.AsyncMock(
         return_value=SimpleNamespace(object_permission_id="perm-new")
     )
     mock_prisma_client.db.litellm_mcpservertable.find_many = mocker.AsyncMock(return_value=[])
-    mock_prisma_client.update_data = mocker.AsyncMock(
-        return_value={"user_id": "target-user"}
-    )
+    mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "target-user"})
     mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x)
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
     mocker.patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin")
@@ -4272,9 +4068,7 @@ async def test_user_update_persists_mcp_entitlement_and_links_it(mocker):
                 "mcp_tool_permissions": {"github": ["list_issues"]},
             },
         ),
-        user_api_key_dict=UserAPIKeyAuth(
-            user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN
-        ),
+        user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
     )
 
     upsert_kwargs = mock_prisma_client.db.litellm_objectpermissiontable.upsert.call_args.kwargs
@@ -4308,9 +4102,7 @@ async def test_user_update_invalidates_the_cached_entitlement(mocker):
             user_id="target-user",
             object_permission={"mcp_tool_permissions": {"github": []}},
         ),
-        user_api_key_dict=UserAPIKeyAuth(
-            user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN
-        ),
+        user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
     )
 
     deleted = {call.kwargs["key"] for call in cache.async_delete_cache.call_args_list}
@@ -4343,9 +4135,7 @@ async def test_admin_can_clear_a_users_mcp_entitlement(mocker):
 
     await _update_single_user_helper(
         user_request=UpdateUserRequest(user_id="target-user", object_permission={}),
-        user_api_key_dict=UserAPIKeyAuth(
-            user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN
-        ),
+        user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
     )
 
     written = mock_prisma_client.update_data.call_args.kwargs["data"]
@@ -4382,9 +4172,7 @@ async def test_user_update_invalidates_both_the_old_and_new_permission_rows(mock
             user_id="target-user",
             object_permission={"mcp_tool_permissions": {"github": ["list_issues"]}},
         ),
-        user_api_key_dict=UserAPIKeyAuth(
-            user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN
-        ),
+        user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
     )
 
     deleted = {call.kwargs["key"] for call in cache.async_delete_cache.call_args_list}
@@ -4415,9 +4203,7 @@ async def test_non_admin_cannot_clear_their_own_mcp_entitlement(mocker):
     with pytest.raises(HTTPException) as exc:
         await _update_single_user_helper(
             user_request=UpdateUserRequest(user_id="target-user", object_permission={}),
-            user_api_key_dict=UserAPIKeyAuth(
-                user_id="target-user", user_role=LitellmUserRoles.INTERNAL_USER
-            ),
+            user_api_key_dict=UserAPIKeyAuth(user_id="target-user", user_role=LitellmUserRoles.INTERNAL_USER),
         )
 
     assert exc.value.status_code == 403
@@ -4445,9 +4231,7 @@ async def test_non_admin_cannot_rewrite_their_own_mcp_entitlement(mocker):
                 user_id="target-user",
                 object_permission={"mcp_servers": [], "mcp_tool_permissions": {}},
             ),
-            user_api_key_dict=UserAPIKeyAuth(
-                user_id="target-user", user_role=LitellmUserRoles.INTERNAL_USER
-            ),
+            user_api_key_dict=UserAPIKeyAuth(user_id="target-user", user_role=LitellmUserRoles.INTERNAL_USER),
         )
 
     assert exc.value.status_code == 403
@@ -4464,9 +4248,7 @@ async def test_new_user_persists_the_requested_mcp_entitlement(mocker):
         return_value=SimpleNamespace(object_permission_id="perm-created")
     )
     mock_prisma_client.db.litellm_mcpservertable.find_many = mocker.AsyncMock(return_value=[])
-    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(
-        return_value=None
-    )
+    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=None)
     mock_prisma_client.db.litellm_usertable.count = mocker.AsyncMock(return_value=0)
     mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
     mocker.patch(
@@ -4475,9 +4257,7 @@ async def test_new_user_persists_the_requested_mcp_entitlement(mocker):
     )
     mock_generate = mocker.patch(
         "litellm.proxy.management_endpoints.internal_user_endpoints.generate_key_helper_fn",
-        new=mocker.AsyncMock(
-            return_value={"user_id": "new-human", "token": "sk-x", "expires": None}
-        ),
+        new=mocker.AsyncMock(return_value={"user_id": "new-human", "token": "sk-x", "expires": None}),
     )
     mocker.patch(
         "litellm.proxy.hooks.user_management_event_hooks.UserManagementEventHooks.async_user_created_hook",
@@ -4489,9 +4269,7 @@ async def test_new_user_persists_the_requested_mcp_entitlement(mocker):
             user_id="new-human",
             object_permission={"mcp_tool_permissions": {"github": ["list_issues"]}},
         ),
-        user_api_key_dict=UserAPIKeyAuth(
-            user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN
-        ),
+        user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
     )
 
     created = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"]
@@ -4533,16 +4311,12 @@ async def test_user_info_v2_returns_the_mcp_entitlement(mocker):
     response = await user_info_v2(
         request=SimpleNamespace(query_params={}),
         user_id="human-1",
-        user_api_key_dict=UserAPIKeyAuth(
-            user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN
-        ),
+        user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
     )
 
     assert response.object_permission is not None
     assert response.object_permission.mcp_servers == ["github"]
-    assert response.object_permission.mcp_tool_permissions == {
-        "github": ["list_issues"]
-    }
+    assert response.object_permission.mcp_tool_permissions == {"github": ["list_issues"]}
 
 
 @pytest.mark.asyncio
@@ -4558,9 +4332,7 @@ async def test_user_info_v2_returns_the_mcp_entitlement(mocker):
     ],
     ids=["supplied", "omitted", "empty"],
 )
-async def test_user_new_persists_model_max_budget(
-    monkeypatch, model_max_budget, expected_written
-):
+async def test_user_new_persists_model_max_budget(monkeypatch, model_max_budget, expected_written):
     """
     /user/new used to echo model_max_budget back while writing {} to the user row,
     so a per-model budget looked configured and was read by nothing.
@@ -4674,6 +4446,11 @@ async def test_user_update_hashes_and_persists_strong_password(_admin_prisma, mo
         _update_single_user_helper,
     )
 
+    mocker.patch(  # test-quality-ok: same module-global mocking every test in this file already uses
+        "litellm.proxy.proxy_server.general_settings",
+        {"password_policy_check_breached_passwords": False},
+    )
+
     mock_prisma_client = _admin_prisma
     existing_user = mocker.MagicMock()
     existing_user.model_dump.return_value = {"user_id": "target-user"}
@@ -4691,3 +4468,149 @@ async def test_user_update_hashes_and_persists_strong_password(_admin_prisma, mo
     written_data = mock_prisma_client.update_data.call_args.kwargs["data"]
     assert written_data.get("password") is not None
     assert written_data["password"] != strong_password
+    # An admin-set password is known to the admin, so the user must be forced
+    # to change it at next login and the breach screen re-armed.
+    assert written_data["password_reset_required"] is True
+    assert written_data["last_breach_check_at"] is None
+
+
+@pytest.mark.asyncio
+@respx.mock
+async def test_user_update_rejects_breached_password(_admin_prisma):
+    """A strength-passing password found in the HIBP corpus must be rejected
+    before it ever reaches the DB write."""
+    from litellm.proxy.management_endpoints.internal_user_endpoints import (
+        _update_single_user_helper,
+    )
+
+    password = "Str0ng!Passw0rd"
+    sha1 = hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper()
+    respx.get(f"https://api.pwnedpasswords.com/range/{sha1[:5]}").mock(
+        return_value=httpx.Response(200, text=f"{sha1[5:]}:1387")
+    )
+
+    user_request = UpdateUserRequest(user_id="target-user", password=password)
+    admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN)
+
+    with pytest.raises(ProxyException) as exc_info:
+        await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller)
+
+    assert exc_info.value.code == "400"
+    assert "data breaches" in exc_info.value.message
+    _admin_prisma.db.litellm_usertable.find_first.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_bulk_update_all_users_rejects_a_password(_admin_prisma):
+    """The all_users fast path writes user_updates straight to update_many,
+    bypassing _update_single_user_helper. A password riding along would be
+    stored as unvalidated plaintext on every row, so it must be rejected
+    before any DB access."""
+    from fastapi import HTTPException
+
+    from litellm.proxy._types import UpdateUserRequestNoUserIDorEmail
+    from litellm.proxy.management_endpoints.internal_user_endpoints import bulk_user_update
+    from litellm.types.proxy.management_endpoints.internal_user_endpoints import (
+        BulkUpdateUserRequest,
+    )
+
+    data = BulkUpdateUserRequest(
+        all_users=True,
+        user_updates=UpdateUserRequestNoUserIDorEmail(password="Str0ng!Passw0rd"),
+    )
+    admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN)
+
+    with pytest.raises(HTTPException) as exc_info:
+        await bulk_user_update(data=data, user_api_key_dict=admin_caller)
+
+    assert exc_info.value.status_code == 400
+    assert "not supported" in str(exc_info.value.detail)
+    _admin_prisma.db.litellm_usertable.find_many.assert_not_called()
+    _admin_prisma.db.litellm_usertable.update_many.assert_not_called()
+
+
+def _hibp_client_with_handler(handler) -> AsyncHTTPHandler:
+    """A real AsyncHTTPHandler over httpx.MockTransport (the DI seam used
+    throughout test_password_policy.py), so no network is touched."""
+    http_handler = AsyncHTTPHandler()
+    http_handler.client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    return http_handler
+
+
+@pytest.mark.asyncio
+async def test_bulk_update_breached_password_fails_only_that_user(_admin_prisma, mocker):
+    """In a bulk batch, a breached password fails only its own entry, before
+    any DB write for it; sibling entries with acceptable passwords persist."""
+    from litellm.proxy.management_endpoints.internal_user_endpoints import (
+        bulk_update_processed_users,
+    )
+
+    breached = "Br3ached!Passw0rd"
+    clean = "NewP@ssw0rd123"
+    breached_sha1 = hashlib.sha1(breached.encode("utf-8"), usedforsecurity=False).hexdigest().upper()
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        if request.url.path == f"/range/{breached_sha1[:5]}":
+            return httpx.Response(200, text=f"{breached_sha1[5:]}:1387")
+        return httpx.Response(200, text="0000000000000000000000000000000000A:1")
+
+    mock_prisma_client = _admin_prisma
+    existing_user = mocker.MagicMock()
+    existing_user.model_dump.return_value = {"user_id": "user-clean"}
+    existing_user.user_id = "user-clean"
+    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user)
+    mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "user-clean"})
+    mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x)
+
+    admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN)
+    response = await bulk_update_processed_users(
+        users_to_update=[
+            UpdateUserRequest(user_id="user-breached", password=breached),
+            UpdateUserRequest(user_id="user-clean", password=clean),
+        ],
+        user_api_key_dict=admin_caller,
+        hibp_client=_hibp_client_with_handler(handler),
+    )
+
+    assert response.successful_updates == 1
+    assert response.failed_updates == 1
+    by_user = {r.user_id: r for r in response.results}
+    assert by_user["user-breached"].success is False
+    assert "data breaches" in by_user["user-breached"].error
+    assert by_user["user-clean"].success is True
+    (write_call,) = mock_prisma_client.update_data.call_args_list
+    assert write_call.kwargs["user_id"] == "user-clean"
+
+
+@pytest.mark.asyncio
+async def test_bulk_update_screens_shared_password_with_single_lookup(_admin_prisma, mocker):
+    """A batch where every user gets the same password costs one HIBP lookup,
+    not one per user (the serial per-user checks this regresses against)."""
+    from litellm.proxy.management_endpoints.internal_user_endpoints import (
+        bulk_update_processed_users,
+    )
+
+    lookup_count = 0
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        nonlocal lookup_count
+        lookup_count += 1
+        return httpx.Response(200, text="0000000000000000000000000000000000A:1")
+
+    mock_prisma_client = _admin_prisma
+    existing_user = mocker.MagicMock()
+    existing_user.model_dump.return_value = {"user_id": "user-0"}
+    existing_user.user_id = "user-0"
+    mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user)
+    mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "user-0"})
+    mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x)
+
+    admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN)
+    response = await bulk_update_processed_users(
+        users_to_update=[UpdateUserRequest(user_id=f"user-{i}", password="NewP@ssw0rd123") for i in range(5)],
+        user_api_key_dict=admin_caller,
+        hibp_client=_hibp_client_with_handler(handler),
+    )
+
+    assert response.successful_updates == 5
+    assert lookup_count == 1
diff --git a/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py
new file mode 100644
index 00000000000..bd154ebab41
--- /dev/null
+++ b/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py
@@ -0,0 +1,331 @@
+"""
+Tests for POST /user/password/change (litellm/proxy/management_endpoints/password_endpoints.py).
+
+HIBP traffic is intercepted with respx; no test here touches the network.
+"""
+
+import hashlib
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import httpx
+import pytest
+import respx
+from fastapi import HTTPException
+
+from litellm.proxy._types import LitellmTableNames, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
+from litellm.proxy.management_endpoints.password_endpoints import change_password
+from litellm.proxy.utils import hash_password, verify_password
+
+CURRENT_PASSWORD = "OldP@ssw0rd-2026"
+NEW_PASSWORD = "NewP@ssw0rd-2026"
+
+_POLICY_NO_BREACH_CHECK = {"password_policy_check_breached_passwords": False}
+
+
+def _make_user_row(password: str | None) -> MagicMock:
+    user = MagicMock()
+    user.user_id = "user-123"
+    user.password = password
+    return user
+
+
+def _make_prisma(user: MagicMock | None) -> MagicMock:
+    prisma = MagicMock()
+    prisma.db.litellm_usertable.find_first = AsyncMock(return_value=user)
+    prisma.db.litellm_usertable.update = AsyncMock(return_value=user)
+    return prisma
+
+
+def _caller(user_id: str | None = "user-123") -> UserAPIKeyAuth:
+    return UserAPIKeyAuth(user_id=user_id)
+
+
+def _hibp_url_for(password: str) -> str:
+    sha1 = hashlib.sha1(password.encode(), usedforsecurity=False).hexdigest().upper()
+    return f"https://api.pwnedpasswords.com/range/{sha1[:5]}"
+
+
+def _hibp_suffix_for(password: str) -> str:
+    return hashlib.sha1(password.encode(), usedforsecurity=False).hexdigest().upper()[5:]
+
+
+@pytest.mark.asyncio
+async def test_change_password_success_writes_new_scrypt_hash():
+    from litellm.proxy._types import ChangePasswordRequest
+
+    prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
+
+    with (
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", prisma
+        ),
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
+        ),
+    ):
+        response = await change_password(
+            data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD),
+            user_api_key_dict=_caller(),
+        )
+
+    assert response.user_id == "user-123"
+    update_kwargs = prisma.db.litellm_usertable.update.call_args.kwargs
+    assert update_kwargs["where"] == {"user_id": "user-123"}
+    stored = update_kwargs["data"]["password"]
+    assert stored != NEW_PASSWORD
+    assert verify_password(NEW_PASSWORD, stored)
+    # A successful change lifts any pending forced reset and re-arms the
+    # login-time breach screen for the new password.
+    assert update_kwargs["data"]["password_reset_required"] is False
+    assert update_kwargs["data"]["last_breach_check_at"] is None
+
+
+@pytest.mark.asyncio
+async def test_change_password_rejects_wrong_current_password():
+    from litellm.proxy._types import ChangePasswordRequest
+
+    prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
+
+    with (
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", prisma
+        ),
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
+        ),
+    ):
+        with pytest.raises(HTTPException) as exc_info:
+            await change_password(
+                data=ChangePasswordRequest(current_password="not-the-password", new_password=NEW_PASSWORD),
+                user_api_key_dict=_caller(),
+            )
+
+    assert exc_info.value.status_code == 400
+    assert "Current password is incorrect" in exc_info.value.detail["error"]
+    prisma.db.litellm_usertable.update.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_change_password_rejects_session_without_user():
+    from litellm.proxy._types import ChangePasswordRequest
+
+    prisma = _make_prisma(user=None)
+
+    with (
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", prisma
+        ),
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
+        ),
+    ):
+        with pytest.raises(HTTPException) as exc_info:
+            await change_password(
+                data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD),
+                user_api_key_dict=_caller(user_id=None),
+            )
+
+    assert exc_info.value.status_code == 400
+    prisma.db.litellm_usertable.find_first.assert_not_called()
+    prisma.db.litellm_usertable.update.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_change_password_rejects_account_without_password():
+    """SSO users and the env-credential admin have no DB password row to change."""
+    from litellm.proxy._types import ChangePasswordRequest
+
+    prisma = _make_prisma(_make_user_row(password=None))
+
+    with (
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", prisma
+        ),
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
+        ),
+    ):
+        with pytest.raises(HTTPException) as exc_info:
+            await change_password(
+                data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD),
+                user_api_key_dict=_caller(),
+            )
+
+    assert exc_info.value.status_code == 400
+    assert "no password set" in exc_info.value.detail["error"]
+    prisma.db.litellm_usertable.update.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_change_password_enforces_min_length():
+    from litellm.proxy._types import ChangePasswordRequest
+
+    prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
+
+    with (
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", prisma
+        ),
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
+        ),
+    ):
+        with pytest.raises(ProxyException) as exc_info:
+            await change_password(
+                data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password="Short1!"),
+                user_api_key_dict=_caller(),
+            )
+
+    assert exc_info.value.code == "400"
+    assert exc_info.value.type == ProxyErrorTypes.validation_error
+    assert exc_info.value.param == "password"
+    assert "at least 12 characters" in exc_info.value.message
+    prisma.db.litellm_usertable.update.assert_not_called()
+
+
+@pytest.mark.asyncio
+@respx.mock
+async def test_change_password_rejects_breached_password():
+    """With the default policy, the new password is screened against HIBP."""
+    from litellm.proxy._types import ChangePasswordRequest
+
+    breached_password = "Password123!"
+    respx.get(_hibp_url_for(breached_password)).mock(
+        return_value=httpx.Response(200, text=f"{_hibp_suffix_for(breached_password)}:1")
+    )
+    prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
+
+    with (
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", prisma
+        ),
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.general_settings", {}
+        ),
+    ):
+        with pytest.raises(ProxyException) as exc_info:
+            await change_password(
+                data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=breached_password),
+                user_api_key_dict=_caller(),
+            )
+
+    assert exc_info.value.code == "400"
+    assert exc_info.value.type == ProxyErrorTypes.validation_error
+    assert exc_info.value.param == "password"
+    assert "data breaches" in exc_info.value.message
+    prisma.db.litellm_usertable.update.assert_not_called()
+
+
+@pytest.mark.asyncio
+@respx.mock
+async def test_change_password_verifies_current_password_before_hibp_lookup():
+    """A caller who fails current-password verification must not trigger any
+    HIBP traffic. The HIBP check fails open on errors, so an unmocked lookup
+    could not prove ordering; instead the route is registered and asserted
+    uncalled."""
+    from litellm.proxy._types import ChangePasswordRequest
+
+    hibp_route = respx.get(_hibp_url_for(NEW_PASSWORD)).mock(return_value=httpx.Response(200, text=""))
+    prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
+
+    with (
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", prisma
+        ),
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.general_settings", {}
+        ),
+    ):
+        with pytest.raises(HTTPException) as exc_info:
+            await change_password(
+                data=ChangePasswordRequest(current_password="not-the-password", new_password=NEW_PASSWORD),
+                user_api_key_dict=_caller(),
+            )
+
+    assert exc_info.value.status_code == 400
+    assert "Current password is incorrect" in exc_info.value.detail["error"]
+    assert not hibp_route.called
+    prisma.db.litellm_usertable.update.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_change_password_success_emits_redacted_audit_log():
+    """A successful change must land in the audit trail as field names only;
+    the plaintext passwords must never reach the audit call."""
+    from litellm.proxy._types import ChangePasswordRequest
+
+    prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
+    audit_mock = AsyncMock()
+
+    with (
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", prisma
+        ),
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
+        ),
+        patch(  # test-quality-ok: audit sink is a module-level import; no injection seam
+            "litellm.proxy.management_endpoints.password_endpoints.create_object_audit_log", audit_mock
+        ),
+    ):
+        await change_password(
+            data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD),
+            user_api_key_dict=_caller(),
+        )
+
+    audit_mock.assert_awaited_once()
+    audit_kwargs = audit_mock.await_args.kwargs
+    assert audit_kwargs["object_id"] == "user-123"
+    assert audit_kwargs["action"] == "updated"
+    assert audit_kwargs["table_name"] == LitellmTableNames.USER_TABLE_NAME
+    assert audit_kwargs["after_value"] == '{"fields_changed": ["password"]}'
+    assert CURRENT_PASSWORD not in str(audit_kwargs)
+    assert NEW_PASSWORD not in str(audit_kwargs)
+
+
+@pytest.mark.asyncio
+async def test_change_password_failure_emits_no_audit_log():
+    from litellm.proxy._types import ChangePasswordRequest
+
+    prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
+    audit_mock = AsyncMock()
+
+    with (
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", prisma
+        ),
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
+        ),
+        patch(  # test-quality-ok: audit sink is a module-level import; no injection seam
+            "litellm.proxy.management_endpoints.password_endpoints.create_object_audit_log", audit_mock
+        ),
+    ):
+        with pytest.raises(HTTPException):
+            await change_password(
+                data=ChangePasswordRequest(current_password="not-the-password", new_password=NEW_PASSWORD),
+                user_api_key_dict=_caller(),
+            )
+
+    audit_mock.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_change_password_requires_db():
+    from litellm.proxy._types import ChangePasswordRequest
+
+    with (
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", None
+        ),
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
+        ),
+    ):
+        with pytest.raises(HTTPException) as exc_info:
+            await change_password(
+                data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD),
+                user_api_key_dict=_caller(),
+            )
+
+    assert exc_info.value.status_code == 500
diff --git a/tests/test_litellm/proxy/test__types.py b/tests/test_litellm/proxy/test__types.py
index 9e1486ce90f..f5abe0561db 100644
--- a/tests/test_litellm/proxy/test__types.py
+++ b/tests/test_litellm/proxy/test__types.py
@@ -5,11 +5,13 @@ from pydantic import ValidationError
 
 from litellm.proxy._types import (
     ROLES_WITHIN_ORG,
+    ChangePasswordRequest,
     GenerateKeyRequest,
     KeyRequest,
     LiteLLM_AuditLogs,
     LiteLLM_TeamMembership,
     LitellmUserRoles,
+    NewUserRequest,
     OrganizationMemberUpdateRequest,
     ResetSpendRequest,
     UpdateKeyRequest,
@@ -335,3 +337,43 @@ def test_virtual_key_mapping_counts_as_configured_when_any_issuer_sets_the_claim
     )
 
     assert jwt_auth.is_virtual_key_mapping_configured() is is_configured
+
+
+def test_new_user_request_loudly_rejects_a_password():
+    """
+    /user/new has never persisted a password (the field used to be silently
+    dropped). Sending one must now fail visibly so the dead path cannot be
+    revived without going through the password policy.
+    """
+    with pytest.raises(ValidationError, match="invitation link"):
+        NewUserRequest(user_email="alice@example.com", password="hunter2hunter2")
+
+
+def test_new_user_request_without_password_still_works():
+    request = NewUserRequest(user_email="alice@example.com")
+    assert request.password is None
+
+
+def test_update_user_request_accepts_a_password():
+    """Admins set user passwords through /user/update; the value must survive
+    model validation so the endpoint can policy-check and hash it."""
+    request = UpdateUserRequest(user_id="user-123", password="hunter2hunter2")
+    assert request.password == "hunter2hunter2"
+
+
+def test_update_user_request_password_hidden_from_repr():
+    """management_endpoint_wrapper string-formats endpoint kwargs into Slack
+    alerts, so the model's repr/str must never contain the plaintext password."""
+    request = UpdateUserRequest(user_id="user-123", password="hunter2hunter2")
+    assert "hunter2hunter2" not in repr(request)
+    assert "hunter2hunter2" not in str(request)
+
+
+def test_change_password_request_passwords_hidden_from_repr():
+    """Any accidental str()/repr() of the request model (debug logs, exception
+    handlers, a future management_endpoint_wrapper) must never contain either
+    plaintext password."""
+    request = ChangePasswordRequest(current_password="hunter2hunter2", new_password="NewP@ssw0rd-2026")
+    for rendered in (repr(request), str(request)):
+        assert "hunter2hunter2" not in rendered
+        assert "NewP@ssw0rd-2026" not in rendered
diff --git a/tests/unit/models/test_models.py b/tests/unit/models/test_models.py
index b8bf55f1b4a..ab456bb1624 100644
--- a/tests/unit/models/test_models.py
+++ b/tests/unit/models/test_models.py
@@ -324,6 +324,8 @@ class TestUser:
         assert user_no_models.has_model_access("any-model")
 
     def test_password_hash_excluded_from_serialization(self):
+        import json
+
         from litellm.proxy._types import LiteLLM_UserTableWithKeyCount
 
         secret = "$2b$12$abcdefghijklmnopqrstuv"
@@ -331,12 +333,12 @@ class TestUser:
 
         assert user.password == secret
         assert "password" not in user.model_dump()
-        assert "password" not in user.model_dump_json()
+        assert "password" not in json.loads(user.model_dump_json())
 
         with_keys = LiteLLM_UserTableWithKeyCount(user_id="u1", user_email="a@b.c", password=secret, key_count=2)
         assert with_keys.password == secret
         assert "password" not in with_keys.model_dump()
-        assert "password" not in with_keys.model_dump_json()
+        assert "password" not in json.loads(with_keys.model_dump_json())
 
 
 class TestVerificationToken:
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.integration.test.tsx
new file mode 100644
index 00000000000..c4cf8ebcf9d
--- /dev/null
+++ b/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.integration.test.tsx
@@ -0,0 +1,110 @@
+import { fireEvent, render, screen, waitFor } from "@testing-library/react";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import ChangePasswordForm from "./ChangePasswordForm";
+
+const mockChangePasswordCall = vi.fn();
+const mockToastSuccess = vi.fn();
+const mockClearTokenCookies = vi.fn();
+let mockPasswordResetRequired = false;
+
+vi.mock("@/components/networking", () => ({
+  changePasswordCall: (...args: unknown[]) => mockChangePasswordCall(...args),
+  getProxyBaseUrl: () => "",
+}));
+
+vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({
+  default: () => ({ accessToken: "sk-session-token", passwordResetRequired: mockPasswordResetRequired }),
+}));
+
+vi.mock("@/lib/toast", () => ({
+  toast: {
+    success: (...args: unknown[]) => mockToastSuccess(...args),
+    fromError: vi.fn(),
+  },
+}));
+
+vi.mock("@/utils/cookieUtils", () => ({
+  clearTokenCookies: (...args: unknown[]) => mockClearTokenCookies(...args),
+}));
+
+const fillForm = (values: { current: string; next: string; confirm: string }) => {
+  fireEvent.change(screen.getByLabelText("Current Password"), { target: { value: values.current } });
+  fireEvent.change(screen.getByLabelText("New Password"), { target: { value: values.next } });
+  fireEvent.change(screen.getByLabelText("Confirm New Password"), { target: { value: values.confirm } });
+};
+
+const submit = () => fireEvent.click(screen.getByRole("button", { name: "Change Password" }));
+
+describe("ChangePasswordForm", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockPasswordResetRequired = false;
+  });
+
+  it("sends the current and new password to the change endpoint and resets on success", async () => {
+    mockChangePasswordCall.mockResolvedValue({ user_id: "user-123", message: "Password updated successfully." });
+    render(<ChangePasswordForm />);
+
+    fillForm({ current: "OldP@ssw0rd-2026", next: "NewP@ssw0rd-2026", confirm: "NewP@ssw0rd-2026" });
+    submit();
+
+    expect(await screen.findByLabelText("Current Password")).toHaveValue("");
+    expect(mockChangePasswordCall).toHaveBeenCalledWith("sk-session-token", "OldP@ssw0rd-2026", "NewP@ssw0rd-2026");
+    expect(mockToastSuccess).toHaveBeenCalled();
+  });
+
+  it("blocks submission when the confirmation does not match", async () => {
+    render(<ChangePasswordForm />);
+
+    fillForm({ current: "OldP@ssw0rd-2026", next: "NewP@ssw0rd-2026", confirm: "Different-2026" });
+    submit();
+
+    expect(await screen.findByText("New passwords do not match")).toBeInTheDocument();
+    expect(mockChangePasswordCall).not.toHaveBeenCalled();
+  });
+
+  it("shows the proxy's rejection message unwrapped", async () => {
+    mockChangePasswordCall.mockRejectedValue(new Error("{'error': 'Current password is incorrect.'}"));
+    render(<ChangePasswordForm />);
+
+    fillForm({ current: "wrong-password", next: "NewP@ssw0rd-2026", confirm: "NewP@ssw0rd-2026" });
+    submit();
+
+    expect(await screen.findByText("Current password is incorrect.")).toBeInTheDocument();
+    expect(mockToastSuccess).not.toHaveBeenCalled();
+  });
+
+  describe("forced password reset", () => {
+    it("shows the forced-reset warning only when the session is flagged", () => {
+      mockPasswordResetRequired = true;
+      render(<ChangePasswordForm />);
+
+      expect(screen.getByText(/must be changed before you can use the dashboard/)).toBeInTheDocument();
+    });
+
+    it("hides the forced-reset warning for a normal session", () => {
+      render(<ChangePasswordForm />);
+
+      expect(screen.queryByText(/must be changed before you can use the dashboard/)).not.toBeInTheDocument();
+    });
+
+    it("signs the user out to re-login after a successful forced change", async () => {
+      mockPasswordResetRequired = true;
+      mockChangePasswordCall.mockResolvedValue({ user_id: "user-123", message: "Password updated successfully." });
+      const replaceMock = vi.fn();
+      const realLocation = window.location;
+      Object.defineProperty(window, "location", { configurable: true, value: { replace: replaceMock } });
+
+      try {
+        render(<ChangePasswordForm />);
+        fillForm({ current: "OldP@ssw0rd-2026", next: "NewP@ssw0rd-2026", confirm: "NewP@ssw0rd-2026" });
+        submit();
+
+        await waitFor(() => expect(replaceMock).toHaveBeenCalledWith("/ui/login/"));
+        expect(mockClearTokenCookies).toHaveBeenCalled();
+      } finally {
+        Object.defineProperty(window, "location", { configurable: true, value: realLocation });
+      }
+    });
+  });
+});
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx
new file mode 100644
index 00000000000..05a6bf3ae94
--- /dev/null
+++ b/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx
@@ -0,0 +1,120 @@
+"use client";
+
+import React, { useState } from "react";
+import { CircleAlert } from "lucide-react";
+import { z } from "zod/v4";
+import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
+import { Alert, AlertTitle } from "@/components/shared/Alert";
+import { PasswordInput } from "@/components/shared/PasswordInput";
+import { FormField } from "@/components/shared/form/FormField";
+import { Button } from "@/components/ui/button";
+import { Card, CardContent } from "@/components/ui/card";
+import { FieldGroup } from "@/components/ui/field";
+import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
+import { changePasswordCall, getProxyBaseUrl } from "@/components/networking";
+import { extractProxyErrorMessage } from "@/lib/http/client";
+import { useZodForm } from "@/lib/forms/useZodForm";
+import { toast } from "@/lib/toast";
+import { clearTokenCookies } from "@/utils/cookieUtils";
+import { getLoginUrl } from "@/utils/returnUrlUtils";
+
+const changePasswordSchema = z
+  .object({
+    currentPassword: z.string().min(1, "Current password is required"),
+    newPassword: z.string().min(1, "New password is required"),
+    confirmNewPassword: z.string().min(1, "Confirm your new password"),
+  })
+  .refine((values) => values.newPassword === values.confirmNewPassword, {
+    message: "New passwords do not match",
+    path: ["confirmNewPassword"],
+  });
+
+type ChangePasswordValues = z.infer<typeof changePasswordSchema>;
+
+export function ChangePasswordForm() {
+  const { accessToken, passwordResetRequired } = useAuthorized();
+  const form = useZodForm(changePasswordSchema, {
+    defaultValues: { currentPassword: "", newPassword: "", confirmNewPassword: "" },
+  });
+  const [isPending, setIsPending] = useState(false);
+  const [submitError, setSubmitError] = useState<string | null>(null);
+
+  const handleSubmit = async (values: ChangePasswordValues) => {
+    if (!accessToken) return;
+    setSubmitError(null);
+    setIsPending(true);
+    try {
+      await changePasswordCall(accessToken, values.currentPassword, values.newPassword);
+      if (passwordResetRequired) {
+        // The session key was minted restricted; only a fresh login lifts it.
+        toast.success("Password updated. Please log in with your new password.");
+        clearTokenCookies();
+        window.location.replace(getLoginUrl(getProxyBaseUrl()));
+        return;
+      }
+      toast.success("Password updated");
+      form.reset();
+    } catch (error) {
+      setSubmitError(extractProxyErrorMessage(error));
+    } finally {
+      setIsPending(false);
+    }
+  };
+
+  return (
+    <div className="mx-auto mt-10 w-full max-w-md">
+      <Card>
+        <CardContent>
+          <h3 className="text-2xl font-semibold text-foreground">Change Password</h3>
+          <p className="text-sm text-muted-foreground">
+            Enter your current password and choose a new one. The new password must meet this proxy&apos;s password
+            policy.
+          </p>
+
+          {passwordResetRequired && (
+            <Alert variant="warning" className="mt-4">
+              <CircleAlert />
+              <AlertTitle>
+                Your password must be changed before you can use the dashboard: it was either found in a known data
+                breach or set by an administrator as a temporary password. After updating it, you will be signed out to
+                log in again.
+              </AlertTitle>
+            </Alert>
+          )}
+
+          <form className="mb-2 mt-8" onSubmit={form.handleSubmit(handleSubmit)}>
+            <FieldGroup>
+              <FormField control={form.control} name="currentPassword" label="Current Password">
+                {({ ref, ...field }) => <PasswordInput {...field} ref={ref} autoComplete="current-password" />}
+              </FormField>
+
+              <FormField control={form.control} name="newPassword" label="New Password">
+                {({ ref, ...field }) => <PasswordInput {...field} ref={ref} autoComplete="new-password" />}
+              </FormField>
+
+              <FormField control={form.control} name="confirmNewPassword" label="Confirm New Password">
+                {({ ref, ...field }) => <PasswordInput {...field} ref={ref} autoComplete="new-password" />}
+              </FormField>
+            </FieldGroup>
+
+            {submitError && (
+              <Alert variant="error" className="mt-6">
+                <CircleAlert />
+                <AlertTitle>{submitError}</AlertTitle>
+              </Alert>
+            )}
+
+            <div className="mt-8">
+              <Button type="submit" disabled={isPending}>
+                {isPending && <UiLoadingSpinner className="size-4" role="img" aria-label="loading" />}
+                Change Password
+              </Button>
+            </div>
+          </form>
+        </CardContent>
+      </Card>
+    </div>
+  );
+}
+
+export default ChangePasswordForm;
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/change-password/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/change-password/page.tsx
new file mode 100644
index 00000000000..0a6ae926ceb
--- /dev/null
+++ b/ui/litellm-dashboard/src/app/(dashboard)/change-password/page.tsx
@@ -0,0 +1,7 @@
+"use client";
+
+import ChangePasswordForm from "./ChangePasswordForm";
+
+export default function ChangePasswordPage() {
+  return <ChangePasswordForm />;
+}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts
index 40d1ec09d1f..581ee8b2580 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts
@@ -50,7 +50,9 @@ const useAuthorized = () => {
     isViewOnly: isViewOnlySessionRole(decoded?.user_role),
     premiumUser: decoded?.premium_user ?? null,
     disabledPersonalKeyCreation: decoded?.disabled_non_admin_personal_key_creation ?? null,
+    loginMethod: decoded?.login_method ?? null,
     showSSOBanner: decoded?.login_method === "username_password",
+    passwordResetRequired: decoded?.password_reset_required === true,
   };
 };
 
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx
index 3fe34610260..ae6575e2349 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx
@@ -1,4 +1,4 @@
-import { describe, it, expect, vi, beforeEach } from "vitest";
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
 import { render, screen, waitFor } from "@testing-library/react";
 import { AuthProvider } from "@/contexts/AuthContext";
 import Layout from "./layout";
@@ -117,4 +117,60 @@ describe("(dashboard) Layout", () => {
     expect(screen.queryByTestId("dashboard-header")).not.toBeInTheDocument();
     expect(screen.queryByTestId("sidebar")).not.toBeInTheDocument();
   });
+
+  describe("forced password reset routing", () => {
+    const sessionCookie = (claims: Record<string, unknown>) => {
+      const encode = (part: Record<string, unknown>) =>
+        btoa(JSON.stringify(part)).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
+      const exp = Math.floor(Date.now() / 1000) + 3600;
+      return `${encode({ alg: "HS256", typ: "JWT" })}.${encode({ ...claims, exp })}.sig`;
+    };
+
+    afterEach(() => {
+      document.cookie = "token=; Max-Age=0; Path=/";
+    });
+
+    it("routes a session flagged password_reset_required to the change-password page", async () => {
+      const flaggedClaims = {
+        user_id: "flagged-user",
+        key: "sk-session",
+        login_method: "username_password",
+        password_reset_required: true,
+      };
+      document.cookie = `token=${sessionCookie(flaggedClaims)}; Path=/`;
+
+      render(
+        <AuthProvider>
+          <Layout>
+            <div data-testid="page-content" />
+          </Layout>
+        </AuthProvider>,
+      );
+
+      pendingUiConfig.resolve();
+
+      await waitFor(() => expect(replaceMock).toHaveBeenCalledWith(expect.stringContaining("/change-password")));
+    });
+
+    it("does not reroute an unflagged session", async () => {
+      document.cookie = `token=${sessionCookie({
+        user_id: "normal-user",
+        key: "sk-session",
+        login_method: "username_password",
+      })}; Path=/`;
+
+      render(
+        <AuthProvider>
+          <Layout>
+            <div data-testid="page-content" />
+          </Layout>
+        </AuthProvider>,
+      );
+
+      pendingUiConfig.resolve();
+
+      expect(await screen.findByTestId("page-content")).toBeInTheDocument();
+      expect(replaceMock).not.toHaveBeenCalled();
+    });
+  });
 });
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx
index fa6df7f176a..309d7dd1ac2 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx
@@ -7,7 +7,7 @@ import LoadingScreen from "@/components/common_components/LoadingScreen";
 import { ThemeProvider } from "@/contexts/ThemeContext";
 import { useAuth } from "@/contexts/AuthContext";
 import SidebarProvider from "@/app/(dashboard)/components/SidebarProvider";
-import { useRouter, useSearchParams } from "next/navigation";
+import { usePathname, useRouter, useSearchParams } from "next/navigation";
 import { DebugWarningBanner } from "@/components/DebugWarningBanner";
 import { NoRedisWarningBanner } from "@/components/NoRedisWarningBanner";
 import { EnvCredentialLoginWarningBanner } from "@/components/EnvCredentialLoginWarningBanner";
@@ -146,7 +146,8 @@ function DashboardShell({ children }: { children: React.ReactNode }) {
 function LayoutContent({ children }: { children: React.ReactNode }) {
   const router = useRouter();
   const searchParams = useSearchParams();
-  const { accessToken, authLoading } = useAuth();
+  const pathname = usePathname();
+  const { accessToken, authLoading, passwordResetRequired } = useAuth();
   const isInvitationFlow = Boolean(searchParams.get("invitation_id"));
 
   // Legacy invitation links point at /ui/?invitation_id=; the onboarding form now lives at its own
@@ -157,6 +158,14 @@ function LayoutContent({ children }: { children: React.ReactNode }) {
     }
   }, [authLoading, isInvitationFlow, router, searchParams]);
 
+  // A session flagged for a forced password reset can only reach the change-password
+  // endpoint server-side; keep the UI on the matching page.
+  useEffect(() => {
+    if (!authLoading && passwordResetRequired && !pathname?.endsWith("/change-password")) {
+      router.replace(uiHref("change-password"));
+    }
+  }, [authLoading, passwordResetRequired, pathname, router]);
+
   if (authLoading || isInvitationFlow) {
     return <LoadingScreen />;
   }
diff --git a/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.test.tsx b/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.test.tsx
index cad5ced340e..4bdf0da3407 100644
--- a/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.test.tsx
+++ b/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.test.tsx
@@ -3,13 +3,25 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
 import { renderWithProviders, screen, waitFor } from "../../../../tests/test-utils";
 import UserDropdown from "./UserDropdown";
 
-let mockUseAuthorizedImpl = () => ({
+let mockUseAuthorizedImpl: () => {
+  userId: string | null;
+  userEmail: string | null;
+  userRoleLabel: string;
+  premiumUser: boolean;
+  loginMethod?: string | null;
+} = () => ({
   userId: "test-user-id",
   userEmail: "test@example.com",
   userRoleLabel: "Admin",
   premiumUser: false,
 });
 
+const mockRouterPush = vi.fn();
+
+vi.mock("next/navigation", () => ({
+  useRouter: () => ({ push: mockRouterPush }),
+}));
+
 let mockUseDisableShowPromptsImpl = () => false;
 
 let mockGetLocalStorageItemImpl = (key: string): string | null => {
@@ -143,6 +155,44 @@ describe("UserDropdown", () => {
     expect(mockOnLogout).toHaveBeenCalledTimes(1);
   });
 
+  it("should navigate to the change-password page for username/password sessions", async () => {
+    mockUseAuthorizedImpl = () => ({
+      userId: "test-user-id",
+      userEmail: "test@example.com",
+      userRoleLabel: "Admin",
+      premiumUser: false,
+      loginMethod: "username_password",
+    });
+    const user = userEvent.setup();
+    renderWithProviders(<UserDropdown onLogout={mockOnLogout} />);
+
+    await user.click(getAccountTrigger());
+
+    await user.click(await screen.findByText("Change Password"));
+
+    expect(mockRouterPush).toHaveBeenCalledWith(expect.stringContaining("change-password"));
+  });
+
+  it("should hide the change-password entry for SSO sessions", async () => {
+    mockUseAuthorizedImpl = () => ({
+      userId: "test-user-id",
+      userEmail: "test@example.com",
+      userRoleLabel: "Admin",
+      premiumUser: false,
+      loginMethod: "sso",
+    });
+    const user = userEvent.setup();
+    renderWithProviders(<UserDropdown onLogout={mockOnLogout} />);
+
+    await user.click(getAccountTrigger());
+
+    await waitFor(() => {
+      expect(screen.getAllByText("test@example.com").length).toBeGreaterThan(0);
+    });
+
+    expect(screen.queryByText("Change Password")).not.toBeInTheDocument();
+  });
+
   it("should toggle hide new feature indicators switch", async () => {
     const user = userEvent.setup();
     renderWithProviders(<UserDropdown onLogout={mockOnLogout} />);
diff --git a/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx b/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx
index 95f76dbb2cc..9c02defc778 100644
--- a/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx
+++ b/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx
@@ -9,7 +9,9 @@ import {
   setLocalStorageItem,
 } from "@/utils/localStorageUtils";
 import { navAccountDisplayName } from "@/components/Navbar/navDisplayName";
-import { ChevronDown, ChevronsUpDown, Crown, LogOut, Mail, ShieldCheck, User } from "lucide-react";
+import { uiHref } from "@/utils/uiHref";
+import { ChevronDown, ChevronsUpDown, Crown, KeyRound, LogOut, Mail, ShieldCheck, User } from "lucide-react";
+import { useRouter } from "next/navigation";
 import { Avatar, AvatarFallback } from "@/components/ui/avatar";
 import { Badge } from "@/components/ui/badge";
 import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
@@ -63,7 +65,9 @@ interface UserDropdownProps {
 }
 
 const UserDropdown: React.FC<UserDropdownProps> = ({ onLogout, variant = "navbar", collapsed = false }) => {
-  const { userId, userEmail, userRoleLabel: userRole, premiumUser } = useAuthorized();
+  const { userId, userEmail, userRoleLabel: userRole, premiumUser, loginMethod } = useAuthorized();
+  const router = useRouter();
+  const [open, setOpen] = useState(false);
   const disableShowPrompts = useDisableShowPrompts();
   const disableBlogPosts = useDisableBlogPosts();
   const disableBouncingIcon = useDisableBouncingIcon();
@@ -197,7 +201,7 @@ const UserDropdown: React.FC<UserDropdownProps> = ({ onLogout, variant = "navbar
   const displayName = navAccountDisplayName(userEmail, userId);
 
   return (
-    <Popover>
+    <Popover open={open} onOpenChange={setOpen}>
       {variant === "sidebar" ? (
         <PopoverTrigger
           render={
@@ -258,6 +262,19 @@ const UserDropdown: React.FC<UserDropdownProps> = ({ onLogout, variant = "navbar
       >
         {renderUserInfoSection()}
         <Separator />
+        {loginMethod === "username_password" && (
+          <button
+            type="button"
+            onClick={() => {
+              setOpen(false);
+              router.push(uiHref("change-password"));
+            }}
+            className="flex w-full items-center gap-2 rounded-sm px-2 py-1.5 text-sm hover:bg-accent"
+          >
+            <KeyRound className="size-4" />
+            Change Password
+          </button>
+        )}
         <button
           type="button"
           onClick={onLogout}
diff --git a/ui/litellm-dashboard/src/components/SidebarAccountMenu/SidebarAccountMenu.test.tsx b/ui/litellm-dashboard/src/components/SidebarAccountMenu/SidebarAccountMenu.test.tsx
index 9d56a889ed4..d9d6605dff7 100644
--- a/ui/litellm-dashboard/src/components/SidebarAccountMenu/SidebarAccountMenu.test.tsx
+++ b/ui/litellm-dashboard/src/components/SidebarAccountMenu/SidebarAccountMenu.test.tsx
@@ -9,6 +9,7 @@ interface AuthMock {
   userRoleLabel: string;
   premiumUser: boolean;
   accessToken: string;
+  loginMethod?: string | null;
 }
 
 let mockUseAuthorizedImpl: () => AuthMock = () => ({
@@ -19,6 +20,12 @@ let mockUseAuthorizedImpl: () => AuthMock = () => ({
   accessToken: "test-token",
 });
 
+const mockRouterPush = vi.fn();
+
+vi.mock("next/navigation", () => ({
+  useRouter: () => ({ push: mockRouterPush }),
+}));
+
 let mockUseDisableShowPromptsImpl = () => false;
 let mockUseDisableBouncingIconImpl = () => false;
 let mockHealthDataImpl = (): { litellm_version?: string } | undefined => ({ litellm_version: "1.99.0" });
@@ -201,6 +208,42 @@ describe("SidebarAccountMenu", () => {
     expect(mockOnLogout).toHaveBeenCalledTimes(1);
   });
 
+  it("should navigate to the change-password page for username/password sessions", async () => {
+    mockUseAuthorizedImpl = () => ({
+      userId: "test-user-id",
+      userEmail: "test@example.com",
+      userRoleLabel: "Admin",
+      premiumUser: false,
+      accessToken: "test-token",
+      loginMethod: "username_password",
+    });
+    const user = userEvent.setup();
+    renderWithProviders(<SidebarAccountMenu onLogout={mockOnLogout} />);
+
+    await openMenu(user);
+
+    await user.click(screen.getByRole("button", { name: /change password/i }));
+
+    expect(mockRouterPush).toHaveBeenCalledWith(expect.stringContaining("change-password"));
+  });
+
+  it("should hide the change-password entry for SSO sessions", async () => {
+    mockUseAuthorizedImpl = () => ({
+      userId: "test-user-id",
+      userEmail: "test@example.com",
+      userRoleLabel: "Admin",
+      premiumUser: false,
+      accessToken: "test-token",
+      loginMethod: "sso",
+    });
+    const user = userEvent.setup();
+    renderWithProviders(<SidebarAccountMenu onLogout={mockOnLogout} />);
+
+    await openMenu(user);
+
+    expect(screen.queryByRole("button", { name: /change password/i })).not.toBeInTheDocument();
+  });
+
   it("should toggle hide new feature indicators on", async () => {
     const user = userEvent.setup();
     renderWithProviders(<SidebarAccountMenu onLogout={mockOnLogout} />);
diff --git a/ui/litellm-dashboard/src/components/SidebarAccountMenu/SidebarAccountMenu.tsx b/ui/litellm-dashboard/src/components/SidebarAccountMenu/SidebarAccountMenu.tsx
index ea0b82869cd..e697cc6f34a 100644
--- a/ui/litellm-dashboard/src/components/SidebarAccountMenu/SidebarAccountMenu.tsx
+++ b/ui/litellm-dashboard/src/components/SidebarAccountMenu/SidebarAccountMenu.tsx
@@ -14,7 +14,9 @@ import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover
 import { Separator } from "@/components/ui/separator";
 import { Switch } from "@/components/ui/switch";
 import { cn } from "@/lib/cva.config";
-import { ChevronsUpDown, Crown, IdCard, LogOut, Mail, ShieldCheck } from "lucide-react";
+import { uiHref } from "@/utils/uiHref";
+import { ChevronsUpDown, Crown, IdCard, KeyRound, LogOut, Mail, ShieldCheck } from "lucide-react";
+import { useRouter } from "next/navigation";
 import React from "react";
 
 const RELEASE_NOTES_URL = "https://docs.litellm.ai/release_notes";
@@ -81,7 +83,9 @@ interface SidebarAccountMenuProps {
 }
 
 const SidebarAccountMenu: React.FC<SidebarAccountMenuProps> = ({ onLogout, collapsed = false }) => {
-  const { userId, userEmail, userRoleLabel: userRole, premiumUser, accessToken } = useAuthorized();
+  const { userId, userEmail, userRoleLabel: userRole, premiumUser, accessToken, loginMethod } = useAuthorized();
+  const router = useRouter();
+  const [open, setOpen] = React.useState(false);
   const { data: healthData } = useHealthReadinessDetails(accessToken);
   const version = healthData?.litellm_version;
   const disableShowPrompts = useDisableShowPrompts();
@@ -136,7 +140,7 @@ const SidebarAccountMenu: React.FC<SidebarAccountMenuProps> = ({ onLogout, colla
   const triggerLabel = `Account menu — ${userRole ?? "Unknown role"} — signed in as ${userEmail || userId || "unknown"}`;
 
   return (
-    <Popover>
+    <Popover open={open} onOpenChange={setOpen}>
       <PopoverTrigger
         className={cn(
           "flex w-full items-center rounded-lg border border-transparent transition-colors hover:bg-sidebar-accent",
@@ -235,6 +239,20 @@ const SidebarAccountMenu: React.FC<SidebarAccountMenuProps> = ({ onLogout, colla
 
         <Separator />
 
+        {loginMethod === "username_password" && (
+          <Button
+            variant="ghost"
+            onClick={() => {
+              setOpen(false);
+              router.push(uiHref("change-password"));
+            }}
+            className="h-[42px] w-full justify-start gap-2.5 rounded-none px-3 text-sm font-medium text-foreground"
+          >
+            <KeyRound className="size-[19px] text-muted-foreground" />
+            Change Password
+          </Button>
+        )}
+
         <Button
           variant="ghost"
           onClick={onLogout}
diff --git a/ui/litellm-dashboard/src/components/leftnav.test.tsx b/ui/litellm-dashboard/src/components/leftnav.test.tsx
index 6eb0218c41d..5fa6e9728cf 100644
--- a/ui/litellm-dashboard/src/components/leftnav.test.tsx
+++ b/ui/litellm-dashboard/src/components/leftnav.test.tsx
@@ -21,6 +21,7 @@ const navState = vi.hoisted(() => ({ pathname: "/ui/api-keys" }));
 
 vi.mock("next/navigation", () => ({
   usePathname: () => navState.pathname,
+  useRouter: () => ({ push: vi.fn() }),
 }));
 
 const { mockUseAuthorized, mockUseOrganizations } = vi.hoisted(() => {
diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx
index ab1203cf440..178bbeb947f 100644
--- a/ui/litellm-dashboard/src/components/networking.tsx
+++ b/ui/litellm-dashboard/src/components/networking.tsx
@@ -1595,6 +1595,20 @@ export const claimOnboardingToken = async (
   }
 };
 
+export const changePasswordCall = async (
+  accessToken: string,
+  currentPassword: string,
+  newPassword: string,
+): Promise<{ user_id: string; message: string }> => {
+  return await apiClient.post(`/user/password/change`, {
+    accessToken,
+    body: {
+      current_password: currentPassword,
+      new_password: newPassword,
+    },
+  });
+};
+
 export const regenerateKeyCall = async (accessToken: string, keyToRegenerate: string, formData: any) => {
   try {
     const url = proxyBaseUrl
diff --git a/ui/litellm-dashboard/src/contexts/AuthContext.tsx b/ui/litellm-dashboard/src/contexts/AuthContext.tsx
index 123feb18a6c..c68c8b9d81a 100644
--- a/ui/litellm-dashboard/src/contexts/AuthContext.tsx
+++ b/ui/litellm-dashboard/src/contexts/AuthContext.tsx
@@ -24,6 +24,7 @@ type AuthContextValue = {
   premiumUser: boolean;
   disabledPersonalKeyCreation: boolean;
   showSSOBanner: boolean;
+  passwordResetRequired: boolean;
 
   setToken: React.Dispatch<React.SetStateAction<string | null>>;
   setUserID: React.Dispatch<React.SetStateAction<string | null>>;
@@ -46,6 +47,7 @@ export function AuthProvider({ children }: { children: React.ReactNode }) {
   const [premiumUser, setPremiumUser] = useState(false);
   const [disabledPersonalKeyCreation, setDisabledPersonalKeyCreation] = useState(false);
   const [showSSOBanner, setShowSSOBanner] = useState(true);
+  const [passwordResetRequired, setPasswordResetRequired] = useState(false);
 
   // Load runtime UI config (populates proxyBaseUrl etc.) before clearing
   // authLoading, so any consumer that builds proxy-rooted URLs from authLoading=false
@@ -124,6 +126,7 @@ export function AuthProvider({ children }: { children: React.ReactNode }) {
     if (decoded.user_id) {
       setUserID(decoded.user_id);
     }
+    setPasswordResetRequired(decoded.password_reset_required === true);
   }, [token]);
 
   const value: AuthContextValue = {
@@ -136,6 +139,7 @@ export function AuthProvider({ children }: { children: React.ReactNode }) {
     premiumUser,
     disabledPersonalKeyCreation,
     showSSOBanner,
+    passwordResetRequired,
     setToken,
     setUserID,
     setUserRole,
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 29b7cd206eb..b12e5b68c49 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -17312,6 +17312,7 @@ export interface paths {
          *     - prompts: Optional[List[str]] - List of allowed prompts for the user. If specified, the user will only be able to use these specific prompts.
          *     - organizations: List[str] - List of organization id's the user is a member of
          *     - budget_limits: Optional[list] - List of concurrent budget windows for the user. Each window specifies a budget_limit, time_period, and optional budget_duration. Example - [{"budget_limit": 10.0, "time_period": "1d"}, {"budget_limit": 50.0, "time_period": "7d"}].
+         *     - password: Optional[str] - Not supported; any value is rejected with a 422. Users set their own password through an invitation link (POST /invitation/new).
          *     Returns:
          *     - key: (str) The generated api key for the user
          *     - expires: (datetime) Datetime object for when key expires.
@@ -17334,6 +17335,36 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
+    "/user/password/change": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        get?: never;
+        put?: never;
+        /**
+         * Change Password
+         * @description Change the calling user's own password.
+         *
+         *     Requires the current password. The new password must satisfy the
+         *     configured password policy (`general_settings.password_policy_*`: minimum
+         *     length, character classes, and, when enabled, breached-password screening
+         *     via haveibeenpwned.com). A successful change lifts any pending forced
+         *     password reset (`password_reset_required`) on the account.
+         *
+         *     Parameters:
+         *     - current_password: str - The user's current password.
+         *     - new_password: str - The password to change to.
+         */
+        post: operations["change_password_user_password_change_post"];
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/user/spend/report": {
         parameters: {
             query?: never;
@@ -17381,7 +17412,7 @@ export interface paths {
          *     Parameters:
          *         - user_id: Optional[str] - Specify a user id. If not set, a unique id will be generated.
          *         - user_email: Optional[str] - Specify a user email.
-         *         - password: Optional[str] - Specify a user password.
+         *         - password: Optional[str] - Set the user's password (admin only). Must satisfy the configured password policy. The user is required to change it at their next login. Users change their own password with POST /user/password/change.
          *         - user_alias: Optional[str] - A descriptive name for you to know who this user id refers to.
          *         - teams: Optional[list] - specify a list of team id's a user belongs to.
          *         - send_invite_email: Optional[bool] - Specify if an invite email should be sent.
@@ -25901,6 +25932,20 @@ export interface components {
              */
             threshold_step: number;
         };
+        /** ChangePasswordRequest */
+        ChangePasswordRequest: {
+            /** Current Password */
+            current_password: string;
+            /** New Password */
+            new_password: string;
+        };
+        /** ChangePasswordResponse */
+        ChangePasswordResponse: {
+            /** Message */
+            message: string;
+            /** User Id */
+            user_id: string;
+        };
         /** ChatCompletionAnnotation */
         ChatCompletionAnnotation: {
             /**
@@ -31604,6 +31649,8 @@ export interface components {
             budget_reset_at?: string | null;
             /** Created At */
             created_at?: string | null;
+            /** Last Breach Check At */
+            last_breach_check_at?: string | null;
             /** Max Budget */
             max_budget?: number | null;
             /** Max Parallel Requests */
@@ -31638,6 +31685,8 @@ export interface components {
             organization_id?: string | null;
             /** Organization Memberships */
             organization_memberships?: components["schemas"]["LiteLLM_OrganizationMembershipTable"][] | null;
+            /** Password Reset Required */
+            password_reset_required?: boolean | null;
             /**
              * Policies
              * @default []
@@ -31697,6 +31746,8 @@ export interface components {
              * @default 0
              */
             key_count: number;
+            /** Last Breach Check At */
+            last_breach_check_at?: string | null;
             /** Max Budget */
             max_budget?: number | null;
             /** Max Parallel Requests */
@@ -31731,6 +31782,8 @@ export interface components {
             organization_id?: string | null;
             /** Organization Memberships */
             organization_memberships?: components["schemas"]["LiteLLM_OrganizationMembershipTable"][] | null;
+            /** Password Reset Required */
+            password_reset_required?: boolean | null;
             /**
              * Policies
              * @default []
@@ -34327,6 +34380,8 @@ export interface components {
             object_permission?: components["schemas"]["LiteLLM_ObjectPermissionBase"] | null;
             /** Organizations */
             organizations?: string[] | null;
+            /** Password */
+            password?: string | null;
             /**
              * Permissions
              * @default {}
@@ -63569,6 +63624,39 @@ export interface operations {
             };
         };
     };
+    change_password_user_password_change_post: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody: {
+            content: {
+                "application/json": components["schemas"]["ChangePasswordRequest"];
+            };
+        };
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["ChangePasswordResponse"];
+                };
+            };
+            /** @description Validation Error */
+            422: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["HTTPValidationError"];
+                };
+            };
+        };
+    };
     get_user_spend_report_user_spend_report_get: {
         parameters: {
             query?: {

From 07c9355f4ea1650cad904932fa90fe42eb48ebb9 Mon Sep 17 00:00:00 2001
From: ryan <ryan@berri.ai>
Date: Mon, 21 Sep 2026 18:49:31 +0000
Subject: [PATCH 042/160] fix(ui): regenerate schema.d.ts against main

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index b12e5b68c49..e2165bdc021 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -25283,6 +25283,8 @@ export interface components {
             object_permission?: components["schemas"]["LiteLLM_ObjectPermissionBase"] | null;
             /** Organizations */
             organizations?: string[] | null;
+            /** Password */
+            password?: string | null;
             /**
              * Permissions
              * @default {}

From 6bfbc7dba4cbf70417ba98d7792bc18005d44486 Mon Sep 17 00:00:00 2001
From: ryan <ryan@berri.ai>
Date: Mon, 21 Sep 2026 18:53:39 +0000
Subject: [PATCH 043/160] test(auth): pass throttle to authenticate_user in
 reset-required tests

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 tests/test_litellm/proxy/auth/test_login_utils.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/test_litellm/proxy/auth/test_login_utils.py b/tests/test_litellm/proxy/auth/test_login_utils.py
index d41b90fe566..ada231a3e95 100644
--- a/tests/test_litellm/proxy/auth/test_login_utils.py
+++ b/tests/test_litellm/proxy/auth/test_login_utils.py
@@ -733,7 +733,12 @@ async def _db_login(throttle, username: str, password: str, *, correct: bool):
         ),
     ):
         return await authenticate_user(
-            username=username, password=password, master_key="sk-master", prisma_client=MagicMock(), throttle=throttle
+            username=username,
+            password=password,
+            master_key="sk-master",
+            prisma_client=MagicMock(),
+            throttle=throttle,
+            general_settings=_POLICY_NO_BREACH_CHECK,
         )
 
 
@@ -2122,6 +2127,7 @@ class TestPasswordResetRequiredSessionMinting:
                     password="Str0ng!Passw0rd",
                     master_key="sk-1234",
                     prisma_client=mock_prisma_client,
+                    throttle=_unlimited_throttle(),
                     general_settings=_POLICY_NO_BREACH_CHECK,
                 )
         return result, mock_generate_key.call_args.kwargs
@@ -2163,6 +2169,7 @@ class TestPasswordResetRequiredSessionMinting:
                         password="Str0ng!Passw0rd",
                         master_key="sk-1234",
                         prisma_client=mock_prisma_client,
+                        throttle=_unlimited_throttle(),
                         general_settings=_POLICY_NO_BREACH_CHECK,
                     )
         return result, mock_generate_key.call_args.kwargs, mock_screen.call_args.kwargs

From ec55ab1ddb1ff656498314887510890c2320b812 Mon Sep 17 00:00:00 2001
From: ryan <ryan@berri.ai>
Date: Mon, 21 Sep 2026 18:57:48 +0000
Subject: [PATCH 044/160] refactor(auth): use Annotated dependency in
 change_password to satisfy strict gate

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/management_endpoints/password_endpoints.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/litellm/proxy/management_endpoints/password_endpoints.py b/litellm/proxy/management_endpoints/password_endpoints.py
index bd3c9722d4d..a2466c83ef9 100644
--- a/litellm/proxy/management_endpoints/password_endpoints.py
+++ b/litellm/proxy/management_endpoints/password_endpoints.py
@@ -8,7 +8,7 @@ request kwargs to OTEL spans, which would log plaintext passwords. The audit
 signal is emitted by hand below, with field names only, never values.
 """
 
-from typing import TYPE_CHECKING, Final
+from typing import TYPE_CHECKING, Annotated, Final
 
 from fastapi import APIRouter, Depends, HTTPException
 
@@ -58,7 +58,7 @@ def _user_table(
 )
 async def change_password(
     data: ChangePasswordRequest,
-    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
 ) -> ChangePasswordResponse:
     """
     Change the calling user's own password.

From 2074273faf21d58a681fb34c6a4039d86f3ffa72 Mon Sep 17 00:00:00 2001
From: ryan <ryan@berri.ai>
Date: Mon, 21 Sep 2026 19:06:00 +0000
Subject: [PATCH 045/160] test(proxy): give the faked login user the
 breach-check columns

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py b/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py
index 88f8be4e49a..8daea8d0ad2 100644
--- a/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py
+++ b/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py
@@ -10,6 +10,7 @@ Routes covered:
 
 from __future__ import annotations
 
+from datetime import datetime, timezone
 from unittest.mock import AsyncMock, MagicMock
 
 from .conftest import normalize
@@ -510,6 +511,8 @@ def _db_user(monkeypatch, email: str):
     user.user_email = email
     user.user_role = "internal_user"
     user.password = "scrypt:stored"
+    user.password_reset_required = None
+    user.last_breach_check_at = datetime.now(timezone.utc)
     repo = MagicMock()
     repo.return_value.table.find_first = AsyncMock(return_value=user)
     monkeypatch.setattr(ps, "prisma_client", MagicMock())

From 1098604ed661ee54b66ec7a378a6163f589b10df Mon Sep 17 00:00:00 2001
From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
Date: Mon, 21 Sep 2026 12:24:15 -0700
Subject: [PATCH 046/160] refactor(mcp): extract explicit operation context and
 dispatch

---
 .github/workflows/test-linting.yml            |    4 +
 Makefile                                      |    1 +
 .../_experimental/mcp_server/contracts.py     |   95 +
 .../mcp_server/legacy_callbacks.py            |   83 +
 .../mcp_server/mcp_server_manager.py          |  129 +-
 .../_experimental/mcp_server/operations.py    | 3102 +++++++++++++++++
 .../mcp_server/rest_endpoints.py              |   13 +-
 .../proxy/_experimental/mcp_server/server.py  | 3059 ++--------------
 .../_experimental/mcp_server/tool_search.py   |   11 +-
 litellm/proxy/_types.py                       |    2 +
 scripts/check_mcp_operation_boundary.py       |   65 +
 scripts/pre_commit_lint.sh                    |    3 +
 .../mcp_server/test_byok_oauth_endpoints.py   |   19 +-
 .../mcp_server/test_contracts.py              |   60 +
 .../mcp_server/test_mcp_block_recording.py    |    5 +-
 .../mcp_server/test_mcp_hook_extra_headers.py |    6 +-
 .../test_mcp_oauth_passthrough_tools.py       |   61 +-
 .../mcp_server/test_mcp_proxy_mode.py         |    8 +-
 .../mcp_server/test_mcp_server.py             |  385 +-
 .../mcp_server/test_mcp_server_manager.py     |  174 +-
 .../mcp_server/test_mcp_stale_session.py      |   40 +-
 .../mcp_server/test_mcp_tool_search.py        |   40 +-
 .../mcp_server/test_mcp_toolset_scope.py      |    2 +-
 .../mcp_server/test_openapi_tool_auth.py      |   75 +-
 .../mcp_server/test_operations.py             |  343 ++
 .../mcp_server/test_rest_endpoints.py         |   13 +-
 .../test_check_mcp_operation_boundary.py      |   52 +
 27 files changed, 4624 insertions(+), 3226 deletions(-)
 create mode 100644 litellm/proxy/_experimental/mcp_server/contracts.py
 create mode 100644 litellm/proxy/_experimental/mcp_server/legacy_callbacks.py
 create mode 100644 litellm/proxy/_experimental/mcp_server/operations.py
 create mode 100644 scripts/check_mcp_operation_boundary.py
 create mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/test_contracts.py
 create mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/test_operations.py
 create mode 100644 tests/test_litellm/test_check_mcp_operation_boundary.py

diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml
index 06d369eabcd..592d8edf6b8 100644
--- a/.github/workflows/test-linting.yml
+++ b/.github/workflows/test-linting.yml
@@ -130,6 +130,10 @@ jobs:
           echo "File content around line 43:"
           head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
 
+      - name: Check MCP operation boundary
+        if: steps.changes.outputs.decision != 'skip'
+        run: uv run --no-sync python scripts/check_mcp_operation_boundary.py
+
       - name: Run Ruff linting
         if: steps.changes.outputs.decision != 'skip'
         run: |
diff --git a/Makefile b/Makefile
index 0e9d2bbf82c..ab7fab6aa99 100644
--- a/Makefile
+++ b/Makefile
@@ -164,6 +164,7 @@ lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
 
 # Linting targets
 lint-ruff: $(LINT_DEP_INSTALL)
+	$(UV_RUN) python scripts/check_mcp_operation_boundary.py
 	cd litellm && $(UV_RUN) ruff check . && cd ..
 	$(UV_RUN) ruff check --config ruff-tests.toml tests
 
diff --git a/litellm/proxy/_experimental/mcp_server/contracts.py b/litellm/proxy/_experimental/mcp_server/contracts.py
new file mode 100644
index 00000000000..c3129d171ad
--- /dev/null
+++ b/litellm/proxy/_experimental/mcp_server/contracts.py
@@ -0,0 +1,95 @@
+from collections.abc import Mapping
+from copy import deepcopy
+from dataclasses import dataclass, field
+from datetime import datetime
+from types import MappingProxyType
+from typing import Final, Protocol
+
+from litellm.proxy._types import UserAPIKeyAuth
+from litellm.types.mcp_server.mcp_server_manager import MCPServer
+
+
+def copy_caller(auth: UserAPIKeyAuth | None) -> UserAPIKeyAuth | None:
+    if auth is None:
+        return None
+    span: Final = auth.parent_otel_span
+    return deepcopy(auth, {id(span): span} if span is not None else None)  # mutable-ok: deepcopy mutates its memo
+
+
+@dataclass(frozen=True, slots=True)
+class OperationContext:
+    _caller: UserAPIKeyAuth | None = field(repr=False)
+    mcp_auth_header: str | None = field(default=None, repr=False)
+    mcp_servers: tuple[str, ...] | None = None
+    mcp_server_auth_headers: Mapping[str, Mapping[str, str]] | None = field(default=None, repr=False)
+    oauth2_headers: Mapping[str, str] | None = field(default=None, repr=False)
+    raw_headers: Mapping[str, str] | None = field(default=None, repr=False)
+    client_ip: str | None = None
+    mcp_proxy_mode: bool = False
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "_caller", copy_caller(self._caller))
+        object.__setattr__(self, "mcp_servers", tuple(self.mcp_servers) if self.mcp_servers is not None else None)
+        object.__setattr__(
+            self,
+            "oauth2_headers",
+            MappingProxyType(dict(self.oauth2_headers)) if self.oauth2_headers is not None else None,
+        )
+        object.__setattr__(
+            self, "raw_headers", MappingProxyType(dict(self.raw_headers)) if self.raw_headers is not None else None
+        )
+        object.__setattr__(
+            self,
+            "mcp_server_auth_headers",
+            MappingProxyType(
+                {key: MappingProxyType(dict(value)) for key, value in self.mcp_server_auth_headers.items()}
+            )
+            if self.mcp_server_auth_headers is not None
+            else None,
+        )
+
+    @property
+    def user_api_key_auth(self) -> UserAPIKeyAuth | None:
+        return copy_caller(self._caller)
+
+    def legacy_auth(
+        self,
+    ) -> tuple[
+        UserAPIKeyAuth | None,
+        str | None,
+        list[str] | None,  # mutable-ok: detached legacy server-list payload
+        dict[str, dict[str, str]] | None,  # mutable-ok: legacy auth dispatch requires concrete dict headers
+        dict[str, str] | None,  # mutable-ok: detached legacy header payload
+        dict[str, str] | None,  # mutable-ok: detached legacy header payload
+        str | None,
+    ]:
+        return (
+            self.user_api_key_auth,
+            self.mcp_auth_header,
+            list(self.mcp_servers) if self.mcp_servers is not None else None,  # mutable-ok: legacy policy list input
+            {
+                key: dict(value) for key, value in self.mcp_server_auth_headers.items()
+            }  # mutable-ok: legacy auth dispatch checks concrete dict headers
+            if self.mcp_server_auth_headers is not None
+            else None,
+            dict(self.oauth2_headers)
+            if self.oauth2_headers is not None
+            else None,  # mutable-ok: legacy OAuth header input
+            dict(self.raw_headers) if self.raw_headers is not None else None,  # mutable-ok: legacy request header input
+            self.client_ip,
+        )
+
+
+class ProgressCallback(Protocol):
+    async def __call__(self, progress: float, total: float | None, /) -> None: ...
+
+
+@dataclass(frozen=True, slots=True)
+class AuthorizedToolCall:
+    name: str
+    arguments: Mapping[str, object]
+    allowed_mcp_servers: tuple[MCPServer, ...]
+    start_time: datetime
+    host_progress_callback: ProgressCallback | None
+    guardrail_context: Mapping[str, object] | None
+    logging_data: Mapping[str, object]
diff --git a/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py b/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py
new file mode 100644
index 00000000000..4424907c28a
--- /dev/null
+++ b/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py
@@ -0,0 +1,83 @@
+from collections.abc import Mapping
+from typing import Final, Protocol
+
+from mcp.client.session import ClientRequestContext
+from mcp.types import (
+    CreateMessageRequestParams,
+    CreateMessageResult,
+    CreateMessageResultWithTools,
+    ElicitRequestParams,
+    ElicitResult,
+    ErrorData,
+)
+
+from litellm.proxy._experimental.mcp_server.contracts import OperationContext
+from litellm.proxy._types import UserAPIKeyAuth
+
+
+class SamplingCallback(Protocol):
+    async def __call__(
+        self, context: ClientRequestContext, params: CreateMessageRequestParams, /
+    ) -> CreateMessageResult | CreateMessageResultWithTools | ErrorData: ...
+
+
+class ElicitationCallback(Protocol):
+    async def __call__(self, context: object, params: ElicitRequestParams, /) -> ElicitResult | ErrorData: ...
+
+
+def create_sampling_callback(
+    user_api_key_auth: UserAPIKeyAuth | None = None,
+    raw_headers: Mapping[str, str] | None = None,
+    client_ip: str | None = None,
+    operation_context: OperationContext | None = None,
+) -> SamplingCallback:
+    from litellm.proxy._experimental.mcp_server.server import get_active_auth_context
+
+    auth: Final = get_active_auth_context() if operation_context is None else None
+    captured: Final = (
+        operation_context
+        if operation_context is not None
+        else OperationContext(
+            _caller=user_api_key_auth if user_api_key_auth is not None else (auth.user_api_key_auth if auth else None),
+            raw_headers=raw_headers if raw_headers is not None else (auth.raw_headers if auth else None),
+            client_ip=client_ip if client_ip is not None else (auth.client_ip if auth else None),
+        )
+    )
+
+    async def callback(
+        context: ClientRequestContext, params: CreateMessageRequestParams
+    ) -> CreateMessageResult | CreateMessageResultWithTools | ErrorData:
+        import litellm
+        from litellm.proxy._experimental.mcp_server.sampling_handler import handle_sampling_create_message
+
+        return await handle_sampling_create_message(
+            context=context,
+            params=params,
+            default_model=getattr(litellm, "default_mcp_sampling_model", None),
+            user_api_key_auth=captured.user_api_key_auth,
+            raw_headers=dict(captured.raw_headers)
+            if captured.raw_headers is not None
+            else None,  # mutable-ok: handler consumes an owned request header dict
+            client_ip=captured.client_ip,
+        )
+
+    return callback
+
+
+def create_elicitation_callback() -> ElicitationCallback:
+    from litellm.proxy._experimental.mcp_server.server import get_active_mcp_session
+
+    downstream_session: Final = get_active_mcp_session()
+    downstream_capabilities: Final = getattr(downstream_session, "capabilities", None)
+
+    async def callback(context: object, params: ElicitRequestParams) -> ElicitResult | ErrorData:
+        from litellm.proxy._experimental.mcp_server.elicitation_handler import handle_elicitation_request
+
+        return await handle_elicitation_request(
+            context=context,
+            params=params,
+            downstream_session=downstream_session,
+            downstream_capabilities=downstream_capabilities,
+        )
+
+    return callback
diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
index b293ab5a206..264d7cf19aa 100644
--- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
+++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
@@ -73,6 +73,7 @@ from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
     MCPServerAccess,
     _is_mcp_admitted_user_subject,
 )
+from litellm.proxy._experimental.mcp_server.contracts import OperationContext
 from litellm.proxy._experimental.mcp_server.elicitation_handler import (
     MCP_ELICITATION_AVAILABLE,
 )
@@ -195,9 +196,6 @@ from litellm.types.mcp_server.mcp_server_manager import (
 from litellm.types.utils import CallTypes
 
 if TYPE_CHECKING:
-    from mcp.client.session import ClientRequestContext
-    from mcp.types import CreateMessageRequestParams
-
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
     from litellm.types.mcp_server.mcp_toolset import MCPToolset
 
@@ -1218,7 +1216,7 @@ async def _resolve_byok_mcp_auth_header(
     if not mcp_server.is_byok:
         return mcp_auth_header
 
-    from litellm.proxy._experimental.mcp_server.server import (
+    from litellm.proxy._experimental.mcp_server.operations import (
         _check_byok_credential,
         _get_byok_credential,
     )
@@ -1577,77 +1575,25 @@ def _normalize_mcp_server_cost_info(mcp_info: MCPInfo) -> None:
     mcp_info["mcp_server_cost_info"] = normalized
 
 
-def _create_sampling_callback(user_api_key_auth: UserAPIKeyAuth | None = None):
-    """
-    Create a sampling callback for MCP ClientSession.
-    Returns a callable that handles sampling/createMessage requests from
-    upstream MCP servers by routing them through litellm.acompletion().
-    """
+def _create_sampling_callback(
+    user_api_key_auth: UserAPIKeyAuth | None = None,
+    raw_headers: Mapping[str, str] | None = None,
+    client_ip: str | None = None,
+    operation_context: OperationContext | None = None,
+):
     if not MCP_SAMPLING_AVAILABLE:
         return None
+    from litellm.proxy._experimental.mcp_server.legacy_callbacks import create_sampling_callback
 
-    async def _sampling_callback(
-        context: "ClientRequestContext",
-        params: "CreateMessageRequestParams",
-    ):
-        import litellm
-        from litellm.proxy._experimental.mcp_server.sampling_handler import (
-            handle_sampling_create_message,
-        )
-        from litellm.proxy._experimental.mcp_server.server import (
-            get_active_auth_context,
-        )
-
-        auth_context: Final = get_active_auth_context()
-        resolved_auth: Final = user_api_key_auth or (auth_context.user_api_key_auth if auth_context else None)
-        # Forward original HTTP headers and client IP so that
-        # header-dependent guardrails, tag-based routing, trace
-        # correlation, and forward_llm_provider_auth_headers work
-        # correctly for sampling sub-calls.
-        _raw_headers: Final = getattr(auth_context, "raw_headers", None)
-        _client_ip: Final = getattr(auth_context, "client_ip", None)
-
-        return await handle_sampling_create_message(
-            context=context,
-            params=params,
-            default_model=getattr(litellm, "default_mcp_sampling_model", None),
-            user_api_key_auth=resolved_auth,
-            raw_headers=_raw_headers,
-            client_ip=_client_ip,
-        )
-
-    return _sampling_callback
+    return create_sampling_callback(user_api_key_auth, raw_headers, client_ip, operation_context)
 
 
 def _create_elicitation_callback():
-    """
-    Create an elicitation callback for MCP ClientSession.
-    Returns a callable that handles elicitation/create requests from
-    upstream MCP servers. In gateway mode, this relays to the downstream
-    client; in tool bridge mode, it returns a decline response.
-    """
     if not MCP_ELICITATION_AVAILABLE:
         return None
+    from litellm.proxy._experimental.mcp_server.legacy_callbacks import create_elicitation_callback
 
-    async def _elicitation_callback(context, params):
-        from litellm.proxy._experimental.mcp_server.elicitation_handler import (
-            handle_elicitation_request,
-        )
-        from litellm.proxy._experimental.mcp_server.server import get_active_mcp_session
-
-        # In Gateway mode, we relay the elicitation request to the downstream client
-        # that triggered the current operation.
-        downstream_session: Final = get_active_mcp_session()
-        downstream_capabilities = getattr(downstream_session, "capabilities", None) if downstream_session else None
-
-        return await handle_elicitation_request(
-            context=context,
-            params=params,
-            downstream_session=downstream_session,
-            downstream_capabilities=downstream_capabilities,
-        )
-
-    return _elicitation_callback
+    return create_elicitation_callback()
 
 
 def _record_mcp_guardrail_evaluations(
@@ -3373,17 +3319,13 @@ class MCPServerManager:
         listable but uninvokable.
 
         Empty inside a toolset scope: toolset_mcp_route / dynamic_mcp_route set
-        ``_mcp_active_toolset_id`` before calling the handler, pinning the request to the toolset's
+        the caller's server-only ``mcp_toolset_id`` before calling the handler, pinning the request to the toolset's
         own servers (checking op.mcp_toolsets==[] instead would false-positive on DB-default rows
         where Postgres initialises the column to ARRAY[]::TEXT[]).
 
         ``allow_all_server_ids`` / ``submitted_server_ids`` are injectable so the server union,
         which precomputes both for its fallback path, does not compute them twice."""
-        from litellm.proxy._experimental.mcp_server.mcp_context import (  # noqa: PLC0415
-            _mcp_active_toolset_id,
-        )
-
-        if _mcp_active_toolset_id.get() is not None:
+        if user_api_key_auth is not None and user_api_key_auth.mcp_toolset_id is not None:
             return set()
         if allow_all_server_ids is None:
             allow_all_server_ids = self.get_allow_all_keys_server_ids()
@@ -4151,6 +4093,8 @@ class MCPServerManager:
         subject_token: str | None = None,
         user_api_key_auth: UserAPIKeyAuth | None = None,
         cred_provider: UpstreamCredentialProvider | None = None,
+        raw_headers: Mapping[str, str] | None = None,
+        client_ip: str | None = None,
     ) -> MCPClient:
         """
         Create an MCPClient instance for the given server.
@@ -4199,7 +4143,13 @@ class MCPServerManager:
 
         # Create sampling and elicitation callbacks for this client
         sampling_cb = (
-            _create_sampling_callback(user_api_key_auth=user_api_key_auth) if resolved_server.allow_sampling else None
+            _create_sampling_callback(
+                operation_context=OperationContext(
+                    _caller=user_api_key_auth, raw_headers=raw_headers, client_ip=client_ip
+                )
+            )
+            if resolved_server.allow_sampling
+            else None
         )
         elicitation_cb: Final = _create_elicitation_callback() if resolved_server.allow_elicitation else None
 
@@ -4344,6 +4294,7 @@ class MCPServerManager:
         raw_headers: dict[str, str] | None = None,
         user_api_key_auth: UserAPIKeyAuth | None = None,
         oauth2_headers: dict[str, str] | None = None,
+        client_ip: str | None = None,
     ) -> list[MCPTool]:
         """
         Helper method to get tools from a single MCP server with prefixed names.
@@ -4433,6 +4384,8 @@ class MCPServerManager:
                 stdio_env=stdio_env,
                 subject_token=subject_token,
                 user_api_key_auth=user_api_key_auth,
+                raw_headers=raw_headers,
+                client_ip=client_ip,
             )
 
             ## HANDLE OPENAPI TOOLS
@@ -4543,6 +4496,7 @@ class MCPServerManager:
         extra_headers: dict[str, str] | None = None,
         add_prefix: bool = True,
         raw_headers: dict[str, str] | None = None,
+        client_ip: str | None = None,
     ) -> list[Prompt]:
         try:
             headers: Final = (
@@ -4563,6 +4517,8 @@ class MCPServerManager:
                 stdio_env=stdio_env,
                 subject_token=subject_token,
                 user_api_key_auth=user_api_key_auth,
+                raw_headers=raw_headers,
+                client_ip=client_ip,
             )
             credential_fingerprint: Final = await client.discovery_auth_fingerprint()
             key: Final = self._discovery_key(
@@ -4586,6 +4542,7 @@ class MCPServerManager:
         extra_headers: dict[str, str] | None = None,
         add_prefix: bool = True,
         raw_headers: dict[str, str] | None = None,
+        client_ip: str | None = None,
     ) -> list[Resource]:
         try:
             headers: Final = (
@@ -4606,6 +4563,8 @@ class MCPServerManager:
                 stdio_env=stdio_env,
                 subject_token=subject_token,
                 user_api_key_auth=user_api_key_auth,
+                raw_headers=raw_headers,
+                client_ip=client_ip,
             )
             credential_fingerprint: Final = await client.discovery_auth_fingerprint()
             key: Final = self._discovery_key(
@@ -4629,6 +4588,7 @@ class MCPServerManager:
         extra_headers: dict[str, str] | None = None,
         add_prefix: bool = True,
         raw_headers: dict[str, str] | None = None,
+        client_ip: str | None = None,
     ) -> list[ResourceTemplate]:
         try:
             headers: Final = (
@@ -4649,6 +4609,8 @@ class MCPServerManager:
                 stdio_env=stdio_env,
                 subject_token=subject_token,
                 user_api_key_auth=user_api_key_auth,
+                raw_headers=raw_headers,
+                client_ip=client_ip,
             )
             credential_fingerprint: Final = await client.discovery_auth_fingerprint()
             key: Final = self._discovery_key(
@@ -4672,6 +4634,7 @@ class MCPServerManager:
         mcp_auth_header: str | dict[str, str] | None = None,
         extra_headers: dict[str, str] | None = None,
         raw_headers: dict[str, str] | None = None,
+        client_ip: str | None = None,
     ) -> ReadResourceResult:
         """Read resource contents from a specific MCP server."""
 
@@ -4692,6 +4655,9 @@ class MCPServerManager:
             extra_headers=extra_headers,
             stdio_env=stdio_env,
             subject_token=subject_token,
+            raw_headers=raw_headers,
+            client_ip=client_ip,
+            user_api_key_auth=user_api_key_auth,
         )
 
         return await client.read_resource(url)
@@ -4705,6 +4671,7 @@ class MCPServerManager:
         mcp_auth_header: str | dict[str, str] | None = None,
         extra_headers: dict[str, str] | None = None,
         raw_headers: dict[str, str] | None = None,
+        client_ip: str | None = None,
     ) -> GetPromptResult:
         """Fetch a specific prompt definition from a single MCP server."""
 
@@ -4725,6 +4692,9 @@ class MCPServerManager:
             extra_headers=extra_headers,
             stdio_env=stdio_env,
             subject_token=subject_token,
+            raw_headers=raw_headers,
+            client_ip=client_ip,
+            user_api_key_auth=user_api_key_auth,
         )
 
         get_prompt_request_params: Final = GetPromptRequestParams(
@@ -5805,6 +5775,8 @@ class MCPServerManager:
         stdio_env: dict[str, str] | None,
         subject_token: str | None,
         user_api_key_auth: UserAPIKeyAuth | None,
+        raw_headers: Mapping[str, str] | None = None,
+        client_ip: str | None = None,
     ) -> CallToolResult:
         """Call a token_exchange (OBO) tool; on an upstream 401/403 re-mint the token once and retry.
 
@@ -5830,6 +5802,8 @@ class MCPServerManager:
                 stdio_env=stdio_env,
                 subject_token=subject_token,
                 user_api_key_auth=user_api_key_auth,
+                raw_headers=raw_headers,
+                client_ip=client_ip,
             )
             return await retry_client.call_tool(call_tool_params, host_progress_callback=host_progress_callback)
 
@@ -5847,6 +5821,7 @@ class MCPServerManager:
         host_progress_callback: Callable | None = None,
         hook_extra_headers: dict[str, str] | None = None,
         user_api_key_auth: UserAPIKeyAuth | None = None,
+        client_ip: str | None = None,
     ) -> CallToolResult:
         """
         Call a regular MCP tool using the MCP client.
@@ -5991,6 +5966,8 @@ class MCPServerManager:
             stdio_env=stdio_env,
             subject_token=subject_token,
             user_api_key_auth=user_api_key_auth,
+            raw_headers=raw_headers,
+            client_ip=client_ip,
         )
 
         call_tool_params: Final = MCPCallToolRequestParams(
@@ -6014,6 +5991,8 @@ class MCPServerManager:
                         stdio_env=stdio_env,
                         subject_token=subject_token,
                         user_api_key_auth=user_api_key_auth,
+                        raw_headers=raw_headers,
+                        client_ip=client_ip,
                     )
 
             tool_call_coro = _obo_call_tool_limited()
@@ -6189,7 +6168,7 @@ class MCPServerManager:
             return oauth2_headers
 
         try:
-            from litellm.proxy._experimental.mcp_server.server import (  # noqa: PLC0415
+            from litellm.proxy._experimental.mcp_server.operations import (  # noqa: PLC0415
                 _get_user_oauth_extra_headers_from_db,
             )
 
@@ -6295,6 +6274,7 @@ class MCPServerManager:
         host_progress_callback: Callable | None = None,
         litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
         guardrail_context: Mapping[str, object] | None = None,
+        client_ip: str | None = None,
     ) -> CallToolResult:
         """
         Call a tool with the given name and arguments
@@ -6421,6 +6401,7 @@ class MCPServerManager:
                 mcp_server_auth_headers=mcp_server_auth_headers,
                 oauth2_headers=oauth2_headers,
                 raw_headers=raw_headers,
+                client_ip=client_ip,
                 proxy_logging_obj=proxy_logging_obj,
                 host_progress_callback=host_progress_callback,
                 hook_extra_headers=hook_result.get("extra_headers"),
diff --git a/litellm/proxy/_experimental/mcp_server/operations.py b/litellm/proxy/_experimental/mcp_server/operations.py
new file mode 100644
index 00000000000..3ee4f37add7
--- /dev/null
+++ b/litellm/proxy/_experimental/mcp_server/operations.py
@@ -0,0 +1,3102 @@
+"""Shared MCP operation policy and dispatch."""
+
+import asyncio
+import traceback
+import types
+import uuid
+from collections.abc import Mapping, Sequence
+from datetime import datetime
+from typing import Any, Final, NoReturn, TypeAlias, assert_never, overload
+
+from fastapi import HTTPException
+from mcp import ReadResourceResult, Resource
+from mcp.types import (
+    CallToolRequest,
+    CallToolRequestParams,
+    CallToolResult,
+    GetPromptRequest,
+    GetPromptRequestParams,
+    GetPromptResult,
+    ListPromptsRequest,
+    ListPromptsResult,
+    ListResourcesRequest,
+    ListResourcesResult,
+    ListResourceTemplatesRequest,
+    ListResourceTemplatesResult,
+    ListToolsRequest,
+    ListToolsResult,
+    PaginatedRequestParams,
+    Prompt,
+    ReadResourceRequest,
+    ReadResourceRequestParams,
+    ResourceTemplate,
+    TextContent,
+)
+from mcp.types import Tool as MCPTool
+from pydantic import AnyUrl, ConfigDict, Field, TypeAdapter
+from typing_extensions import ReadOnly, TypedDict
+
+from litellm._logging import verbose_logger
+from litellm.constants import (
+    MAXIMUM_TRACEBACK_LINES_TO_LOG,
+)
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
+    MCPRequestHandler,
+)
+from litellm.proxy._experimental.mcp_server.byok_credential_cache import (
+    byok_credential_cache,
+    byok_credential_cache_key,
+    cache_byok_credential,
+    get_cached_byok_credential,
+)
+from litellm.proxy._experimental.mcp_server.contracts import (
+    AuthorizedToolCall,
+    OperationContext,
+    ProgressCallback,
+)
+from litellm.proxy._experimental.mcp_server.db import OAuthCredentialPayload
+from litellm.proxy._experimental.mcp_server.exceptions import (
+    MCPToolResultError,
+    MCPUpstreamAuthError,
+)
+from litellm.proxy._experimental.mcp_server.faults.list_outcomes import (
+    SERVER_OUTCOMES_META_KEY,
+    AggregateToolListing,
+    ServerListOk,
+    ServerOutcome,
+    classify_list_exception,
+    outcome_wire_value,
+)
+from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
+    MCPServerManager,
+    _caller_authorization_fans_out,
+    _client_forwarded_authorization_headers,
+    _resolve_openapi_tool_auth,
+    _should_strip_caller_authorization,
+    global_mcp_server_manager,
+)
+from litellm.proxy._experimental.mcp_server.oauth_utils import (
+    _redact_mcp_resource_url,
+    get_byok_www_authenticate,
+)
+from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
+    _request_auth_header,
+    _request_extra_headers,
+    _request_resolved_auth_headers,
+)
+from litellm.proxy._experimental.mcp_server.tool_registry import (
+    global_mcp_tool_registry,
+)
+from litellm.proxy._experimental.mcp_server.utils import (
+    MCP_TOOL_PREFIX_SEPARATOR,
+    MCPMissingUserEnvVarsError,
+    add_server_prefix_to_name,
+    build_synthetic_mcp_request,
+    extract_mcp_tool_result_error_message,
+    get_server_prefix,
+    is_tool_name_prefixed,
+    iter_known_server_prefixes,
+    logging_safe_mcp_headers,
+    match_known_tool_name,
+    normalize_server_name,
+    split_server_prefix_from_name,
+    strip_known_server_prefix,
+)
+from litellm.proxy._types import (
+    UserAPIKeyAuth,
+)
+from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
+    publish_auth_cache_invalidation,
+)
+from litellm.proxy.litellm_pre_call_utils import (
+    LiteLLMProxyRequestSetup,
+    get_chain_id_from_headers,
+)
+from litellm.types.mcp import (
+    DEFAULT_CREDENTIAL_HEADER,
+    MCPAuth,
+    without_header,
+)
+from litellm.types.mcp_server.mcp_server_manager import MCPInfo, MCPServer
+from litellm.types.utils import CallTypes, StandardLoggingMCPToolCall
+from litellm.utils import Rules, client, function_setup
+
+__all__ = (
+    "_MCP_CREDENTIAL_REQUEST_FIELDS",
+    "ListMCPToolsRestAPIResponseObject",
+    "MCPInfo",
+    "MCPServer",
+    "_McpDeniedDetail",
+    "_aggregate_server_key",
+    "_build_virtual_call_logging_obj",
+    "_check_byok_credential",
+    "_client_has_passthrough_authorization",
+    "_client_has_per_server_auth_header",
+    "_dispatch_virtual_mcp_tool",
+    "_fire_mcp_tool_call_logging",
+    "_get_allowed_mcp_servers",
+    "_get_allowed_mcp_servers_from_mcp_server_names",
+    "_get_byok_credential",
+    "_get_prompts_from_mcp_servers",
+    "_get_resource_templates_from_mcp_servers",
+    "_get_resources_from_mcp_servers",
+    "_get_standard_logging_mcp_tool_call",
+    "_get_tools_from_mcp_servers",
+    "_get_user_oauth_extra_headers_from_db",
+    "_handle_local_mcp_tool",
+    "_handle_managed_mcp_tool",
+    "_http_detail_message",
+    "_invalidate_byok_cred_cache",
+    "_list_mcp_prompts",
+    "_list_mcp_resource_templates",
+    "_list_mcp_resources",
+    "_list_mcp_tools",
+    "_list_tools_before_first_call",
+    "_mcp_session_id_from_headers",
+    "_merge_gateway_initialize_instructions",
+    "_prefetch_oauth_creds_for_user",
+    "_prepare_mcp_server_headers",
+    "_raise_if_initialize_grants_no_mcp_servers",
+    "_resolve_display_name_to_original",
+    "_run_post_mcp_call_guardrails",
+    "_server_answers_to",
+    "_tool_name_matches",
+    "apply_tool_overrides",
+    "call_mcp_tool",
+    "execute_mcp_tool",
+    "filter_tools_by_allowed_tools",
+    "filter_tools_by_key_team_permissions",
+    "fire_mcp_tool_call_failure_logging",
+    "mcp_get_prompt",
+    "mcp_read_resource",
+    "raise_denied_scoped_mcp_access",
+)
+
+
+async def _invalidate_byok_cred_cache(user_id: str, server_id: str) -> None:
+    """Drop a stored-or-deleted BYOK credential from this worker's cache and from every peer worker's."""
+    cache_key: Final = byok_credential_cache_key(user_id, server_id)
+    byok_credential_cache.delete_cache(cache_key)
+    await publish_auth_cache_invalidation(cache_key=cache_key)
+
+
+def _mcp_session_id_from_headers(
+    raw_headers: dict[str, str] | None,
+) -> str | None:
+    """The ``mcp-session-id`` of a stateful MCP session, read case-insensitively
+    from the request headers. ``None`` for stateless calls (no such header)."""
+    if not raw_headers:
+        return None
+    for key, value in raw_headers.items():
+        if isinstance(key, str) and key.lower() == "mcp-session-id":
+            return value or None
+    return None
+
+
+class ListMCPToolsRestAPIResponseObject(MCPTool):
+    """
+    Object returned by the /tools/list REST API route.
+    """
+
+    mcp_info: MCPInfo | None = Field(default=None, alias="mcp_info")
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+async def _build_virtual_call_logging_obj(
+    name: str,
+    arguments: dict[str, object],
+    user_api_key_auth: UserAPIKeyAuth,
+    raw_headers: Mapping[str, str] | None = None,
+    client_ip: str | None = None,
+) -> LiteLLMLoggingObj | None:
+    """Run the pre-call pipeline (guardrails + logging setup) for a virtual
+    mcp_tool_call so the SSE path spend-logs like the REST path."""
+    from litellm.proxy.common_request_processing import (
+        ProxyBaseLLMRequestProcessing,
+    )
+    from litellm.proxy.proxy_server import (
+        general_settings,
+        proxy_config,
+        proxy_logging_obj,
+    )
+
+    request: Final = build_synthetic_mcp_request(
+        path="/mcp/tools/call",
+        raw_headers=raw_headers,
+        client_ip=client_ip,
+    )
+    _, virtual_logging_obj = await ProxyBaseLLMRequestProcessing(
+        data={"name": name, "arguments": arguments}
+    ).common_processing_pre_call_logic(
+        request=request,
+        user_api_key_dict=user_api_key_auth,
+        proxy_config=proxy_config,
+        route_type=CallTypes.call_mcp_tool.value,
+        proxy_logging_obj=proxy_logging_obj,
+        general_settings=general_settings,
+    )
+    return virtual_logging_obj
+
+
+async def _dispatch_virtual_mcp_tool(
+    name: str,
+    arguments: dict[str, object] | None,
+    user_api_key_auth: UserAPIKeyAuth | None,
+    client_ip: str | None,
+    mcp_servers: list[str] | None = None,
+    mcp_auth_header: str | None = None,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
+    oauth2_headers: dict[str, str] | None = None,
+    raw_headers: dict[str, str] | None = None,
+    mcp_proxy_mode: bool = False,
+) -> CallToolResult | None:
+    """Handle the mcp_tool_search / mcp_tool_call virtual tools.
+
+    Returns a CallToolResult when ``name`` is a virtual tool, else ``None`` so
+    the caller falls through to normal tool routing.
+    """
+    from litellm.llms.litellm_proxy.skills.skill_search import DEFAULT_SKILL_SEARCH_TOP_K
+    from litellm.proxy._experimental.mcp_server.tool_search import (
+        AGENT_SEARCH_TOOL_NAME,
+        DEFAULT_AGENT_SEARCH_TOP_K,
+        MCP_PROXY_CALL_TOOL_NAME,
+        MCP_PROXY_TOOL_NAMES,
+        MCP_TOOL_SEARCH_TOOL_NAME,
+        SKILL_SEARCH_TOOL_NAME,
+        VIRTUAL_TOOL_NAMES,
+        coerce_top_k,
+        handle_agent_search,
+        handle_mcp_proxy_tool,
+        handle_mcp_tool_call,
+        handle_mcp_tool_search,
+        handle_skill_search,
+    )
+
+    if mcp_proxy_mode and name not in MCP_PROXY_TOOL_NAMES:
+        return CallToolResult(
+            content=[  # mutable-ok: MCP result content
+                TextContent(type="text", text=f"Tool {name} is unavailable on /mcp/proxy")
+            ],
+            is_error=True,
+        )
+
+    if mcp_proxy_mode and name in MCP_PROXY_TOOL_NAMES:
+        assert user_api_key_auth is not None
+        proxy_call_start: Final = datetime.now()  # noqa: DTZ005  # logging pipeline uses naive datetimes
+        proxy_logging_obj: Final = (
+            await _build_virtual_call_logging_obj(
+                name=name,
+                arguments=arguments or {},  # mutable-ok: logging pipeline payload
+                user_api_key_auth=user_api_key_auth,
+                raw_headers=raw_headers,
+                client_ip=client_ip,
+            )
+            if name == MCP_PROXY_CALL_TOOL_NAME
+            else None
+        )
+        try:
+            proxy_result: Final = await handle_mcp_proxy_tool(
+                name=name,
+                arguments=arguments or {},  # mutable-ok: proxy handler payload
+                user_api_key_dict=user_api_key_auth,
+                client_ip=client_ip,
+                mcp_servers=mcp_servers,
+                mcp_auth_header=mcp_auth_header,
+                mcp_server_auth_headers=mcp_server_auth_headers,
+                oauth2_headers=oauth2_headers,
+                raw_headers=raw_headers,
+                litellm_logging_obj=proxy_logging_obj,
+            )
+        except Exception as exc:
+            if proxy_logging_obj is not None:
+                from litellm.proxy.proxy_server import proxy_logging_obj as request_logging_obj
+
+                failure_end: Final = datetime.now()  # noqa: DTZ005  # matches the logging pipeline start time
+                failure_traceback: Final = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG)
+                try:
+                    proxy_logging_obj.failure_handler(exc, failure_traceback, proxy_call_start, failure_end)
+                    await proxy_logging_obj.async_failure_handler(exc, failure_traceback, proxy_call_start, failure_end)
+                    if not isinstance(exc, MCPUpstreamAuthError):
+                        await request_logging_obj.post_call_failure_hook(
+                            request_data={  # mutable-ok: failure hook mutates its request payload
+                                "name": name,
+                                "arguments": arguments,
+                                "litellm_logging_obj": proxy_logging_obj,
+                            },
+                            original_exception=exc,
+                            user_api_key_dict=user_api_key_auth,
+                            route="/mcp/call_tool",
+                            traceback_str=failure_traceback,
+                        )
+                except Exception:  # noqa: BLE001  # a failing failure hook must not mask the tool call's own error
+                    verbose_logger.exception("Error logging failed MCP proxy tool call")
+            raise
+        if proxy_logging_obj is not None:
+            return await _fire_mcp_tool_call_logging(
+                logging_obj=proxy_logging_obj,
+                result=proxy_result,
+                start_time=proxy_call_start,
+                end_time=datetime.now(),  # noqa: DTZ005  # matches the logging pipeline start time
+                user_api_key_auth=user_api_key_auth,
+                request_data=types.MappingProxyType({"name": name, "arguments": arguments}),
+            )
+        return proxy_result
+
+    if name not in VIRTUAL_TOOL_NAMES:
+        return None
+
+    if not getattr(
+        getattr(user_api_key_auth, "object_permission", None),
+        "mcp_tool_search_enabled",
+        False,
+    ):
+        return CallToolResult(
+            content=[
+                TextContent(
+                    type="text",
+                    text=f"Tool {name} requires mcp_tool_search_enabled on the key",
+                )
+            ],
+            is_error=True,
+        )
+
+    args: Final = arguments or {}
+    if name == MCP_TOOL_SEARCH_TOOL_NAME:
+        return await handle_mcp_tool_search(
+            query=TypeAdapter(str).validate_python(args.get("query", "")),
+            top_k=coerce_top_k(args.get("top_k", 5)),
+            user_api_key_dict=user_api_key_auth,
+            client_ip=client_ip,
+            mcp_servers=mcp_servers,
+            mcp_auth_header=mcp_auth_header,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+        )
+
+    assert user_api_key_auth is not None  # guaranteed by the flag check above
+    if name == AGENT_SEARCH_TOOL_NAME:
+        return await handle_agent_search(
+            query=str(args.get("query", "")),
+            top_k=coerce_top_k(args.get("top_k", DEFAULT_AGENT_SEARCH_TOP_K), default=DEFAULT_AGENT_SEARCH_TOP_K),
+            user_api_key_dict=user_api_key_auth,
+        )
+    if name == SKILL_SEARCH_TOOL_NAME:
+        return await handle_skill_search(
+            query=str(args.get("query", "")),
+            top_k=coerce_top_k(args.get("top_k", DEFAULT_SKILL_SEARCH_TOP_K), default=DEFAULT_SKILL_SEARCH_TOP_K),
+            user_api_key_dict=user_api_key_auth,
+        )
+    virtual_logging_obj: Final = await _build_virtual_call_logging_obj(
+        name=name,
+        arguments=args,
+        user_api_key_auth=user_api_key_auth,
+        raw_headers=raw_headers,
+        client_ip=client_ip,
+    )
+    tool_request: Final = CallToolRequestParams.model_validate(
+        types.MappingProxyType({"name": args.get("tool_name", ""), "arguments": args.get("arguments") or {}})
+    )
+    return await handle_mcp_tool_call(
+        tool_name=tool_request.name,
+        arguments=tool_request.arguments or {},
+        user_api_key_dict=user_api_key_auth,
+        client_ip=client_ip,
+        mcp_servers=mcp_servers,
+        mcp_auth_header=mcp_auth_header,
+        mcp_server_auth_headers=mcp_server_auth_headers,
+        oauth2_headers=oauth2_headers,
+        raw_headers=raw_headers,
+        litellm_logging_obj=virtual_logging_obj,
+    )
+
+
+async def _get_allowed_mcp_servers_from_mcp_server_names(
+    mcp_servers: Sequence[str] | None,
+    allowed_mcp_servers: list[MCPServer],
+) -> list[MCPServer]:
+    """
+    Get the filtered MCP servers from the MCP server names.
+
+    Fails closed when ``mcp_servers`` is explicitly provided (path- or
+    header-derived) but none of the names resolve to a server alias or
+    access group the caller can access. The previous behavior returned
+    the full ``allowed_mcp_servers`` set, which silently widened scope
+    when a client targeted ``/mcp/<unknown>/`` and made URL/header
+    namespacing appear to work when it did not.
+    """
+
+    filtered_server: Final[dict[str, MCPServer]] = {}
+    # Filter servers based on mcp_servers parameter if provided
+    if mcp_servers is not None:
+        for server_or_group in mcp_servers:
+            server_name_matched = False
+
+            for server in allowed_mcp_servers:
+                if server and _server_answers_to(server, server_or_group):
+                    filtered_server[server.server_id] = server
+                    server_name_matched = True
+                    break
+
+            if not server_name_matched:
+                try:
+                    access_group_server_ids = await MCPRequestHandler._get_mcp_servers_from_access_groups(
+                        [server_or_group]
+                    )
+                    # Only include servers that the user has access to
+                    for server_id in access_group_server_ids:
+                        for server in allowed_mcp_servers:
+                            if server_id == server.server_id:
+                                filtered_server[server.server_id] = server
+                except Exception as e:
+                    verbose_logger.debug("Could not resolve '%s' as access group: %s", server_or_group, e)
+
+    if filtered_server:
+        return list(filtered_server.values())
+
+    if mcp_servers is not None:
+        # Caller asked for a specific scope but nothing resolved. Fail
+        # closed so URL/header namespacing cannot silently fall back to
+        # the caller's full allowed-server set.
+        verbose_logger.debug(
+            "MCP scope filter resolved to no servers for requested names %s; returning empty list (fail-closed).",
+            mcp_servers,
+        )
+        return []
+
+    return allowed_mcp_servers
+
+
+def _http_detail_message(detail: object) -> str:
+    return str(detail.get("error")) if isinstance(detail, dict) and detail.get("error") else str(detail)
+
+
+def _server_answers_to(server: MCPServer, name: str) -> bool:
+    requested: Final = name.lower()
+    return any(requested == known.lower() for known in iter_known_server_prefixes(server) if known)
+
+
+async def raise_denied_scoped_mcp_access(
+    requested_names: Sequence[str],
+    user_api_key_auth: UserAPIKeyAuth | None,
+    client_ip: str | None = None,
+) -> None:
+    """A scoped request (``/mcp/<name>`` path or ``x-mcp-servers`` header) resolved to zero
+    allowed servers, so the denial must be loud: a silent 200 with no tools reads as a healthy
+    server with no tools. Unknown, unauthorized, and access-group names all share one generic
+    error so scoping cannot probe which servers exist; the agent variant fires only when the
+    same request resolves once the agent binding is stripped, proving the binding caused the veto."""
+    agent_id: Final = user_api_key_auth.agent_id if user_api_key_auth else None
+    if user_api_key_auth is not None and agent_id:
+        resolved_without_agent: Final = await _get_allowed_mcp_servers(
+            user_api_key_auth=user_api_key_auth.model_copy(update=types.MappingProxyType({"agent_id": None})),
+            mcp_servers=requested_names,
+            client_ip=client_ip,
+        )
+
+        def _resolved_to_server(name: str) -> bool:
+            return any(_server_answers_to(server, name) for server in resolved_without_agent)
+
+        vetoed_server: Final = next((name for name in requested_names if _resolved_to_server(name)), None)
+        if vetoed_server is not None:
+            agent_denial: Final[_McpDeniedDetail] = {
+                "error": (
+                    f"MCP server '{vetoed_server}' is not available to this key: the key is bound to "
+                    f"agent '{agent_id}', whose MCP grants do not include this server. Add the server "
+                    f"to the agent's object_permission.mcp_servers (edit the agent in the Admin UI or "
+                    f"PATCH /v1/agents/{agent_id}), or use a key that is not bound to the agent."
+                )
+            }
+            raise HTTPException(status_code=403, detail=agent_denial)
+        vetoed_group: Final = next(
+            (
+                name
+                for name in requested_names
+                if not _resolved_to_server(name)
+                and any(name in (server.access_groups or ()) for server in resolved_without_agent)
+            ),
+            None,
+        )
+        if vetoed_group is not None:
+            group_denial: Final[_McpDeniedDetail] = {
+                "error": (
+                    f"MCP access group '{vetoed_group}' is not available to this key: the key is bound to "
+                    f"agent '{agent_id}', whose MCP grants do not include it. Add the group to the "
+                    f"agent's object_permission.mcp_access_groups (edit the agent in the Admin UI or "
+                    f"PATCH /v1/agents/{agent_id}), or use a key that is not bound to the agent."
+                )
+            }
+            raise HTTPException(status_code=403, detail=group_denial)
+    generic_denial: Final[_McpDeniedDetail] = {
+        "error": f"The key is not allowed to access the requested MCP servers: {', '.join(requested_names)}"
+    }
+    raise HTTPException(status_code=403, detail=generic_denial)
+
+
+def _tool_name_matches(tool_name: str, filter_list: list[str], mcp_server: MCPServer) -> bool:
+    """
+    Check if a tool name matches any name in the filter list.
+
+    Reads the same owner the server-level permission checks use, so discovery hides
+    exactly what dispatch refuses. ``mcp_server`` is required: guessing the boundary
+    at the first separator mismatches every tool on a server whose prefix contains
+    the separator.
+    """
+    bare_name: Final = strip_known_server_prefix(tool_name, mcp_server)
+    return match_known_tool_name(bare_name, mcp_server, filter_list) is not None
+
+
+def filter_tools_by_allowed_tools(
+    tools: list[MCPTool],
+    mcp_server: MCPServer,
+) -> list[MCPTool]:
+    """
+    Filter tools by allowed/disallowed tools configuration.
+
+    If allowed_tools is set, only tools in that list are returned.
+    If disallowed_tools is set, tools in that list are excluded.
+    Tool names are matched with and without server prefixes for flexibility.
+
+    Args:
+        tools: List of tools to filter
+        mcp_server: Server configuration with allowed_tools/disallowed_tools
+
+    Returns:
+        Filtered list of tools
+    """
+    from litellm.proxy._experimental.mcp_server.utils import (
+        server_applies_tool_allowlist,
+    )
+
+    tools_to_return = tools
+
+    # Filter by allowed_tools (whitelist)
+    if server_applies_tool_allowlist(mcp_server):
+        if not mcp_server.allowed_tools:
+            return []
+        tools_to_return = [
+            tool for tool in tools if _tool_name_matches(tool.name, mcp_server.allowed_tools, mcp_server)
+        ]
+
+    # Filter by disallowed_tools (blacklist)
+    if mcp_server.disallowed_tools:
+        tools_to_return = [
+            tool
+            for tool in tools_to_return
+            if not _tool_name_matches(tool.name, mcp_server.disallowed_tools, mcp_server)
+        ]
+
+    return tools_to_return
+
+
+def apply_tool_overrides(
+    tools: list[MCPTool],
+    mcp_server: MCPServer,
+) -> list[MCPTool]:
+    """Apply admin-configured display name/description overrides to tools.
+
+    Overrides are keyed by the unprefixed tool name, same convention as
+    allowed_tools configuration.
+    """
+    display_name_map: Final = mcp_server.tool_name_to_display_name or {}
+    description_map: Final = mcp_server.tool_name_to_description or {}
+    if not display_name_map and not description_map:
+        return tools
+
+    for tool in tools:
+        unprefixed = strip_known_server_prefix(tool.name, mcp_server)
+        lookup_key = unprefixed or tool.name
+        if lookup_key in display_name_map:
+            tool.name = display_name_map[lookup_key]
+        if lookup_key in description_map:
+            tool.description = description_map[lookup_key]
+    return tools
+
+
+async def _get_allowed_mcp_servers(
+    user_api_key_auth: UserAPIKeyAuth | None,
+    mcp_servers: Sequence[str] | None,
+    client_ip: str | None = None,
+) -> list[MCPServer]:
+    """Return allowed MCP servers for a request after applying filters.
+
+    Args:
+        user_api_key_auth: The authenticated user's API key info.
+        mcp_servers: Optional list of server names to filter to.
+        client_ip: Client IP for IP-based access control. If None, falls back to
+                  auth context. Pass explicitly from request handlers for safety.
+    Note: If client_ip is None and auth context is not set, IP filtering is skipped.
+          This is intentional for internal callers but may indicate a bug if called
+          from a request handler without proper context setup.
+    """
+    allowed_mcp_server_ids = await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth)
+    (
+        allowed_mcp_server_ids,
+        _ip_blocked,
+    ) = global_mcp_server_manager.filter_server_ids_by_ip_with_info(allowed_mcp_server_ids, client_ip)
+    verbose_logger.debug(
+        "MCP IP filter: client_ip=%s, allowed_server_ids=%s",
+        client_ip,
+        allowed_mcp_server_ids,
+    )
+    if _ip_blocked > 0:
+        verbose_logger.debug(
+            "MCP IP filtering: %d server(s) are not accessible from client IP %s "
+            "because they are restricted to internal networks. "
+            "No tools from those servers will be returned. "
+            "To expose a server externally, set 'available_on_public_internet: true' "
+            "in its configuration.",
+            _ip_blocked,
+            client_ip,
+        )
+    allowed_mcp_servers: list[MCPServer] = []
+    for allowed_mcp_server_id in allowed_mcp_server_ids:
+        mcp_server = global_mcp_server_manager.get_mcp_server_by_id(allowed_mcp_server_id)
+        if mcp_server is not None:
+            # Apply the request-time oauth2_flow backstop for legacy null rows.
+            mcp_server = MCPServerManager.resolve_oauth2_flow_for_request(mcp_server)
+            allowed_mcp_servers.append(mcp_server)
+
+    if mcp_servers is not None:
+        allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names(
+            mcp_servers=mcp_servers,
+            allowed_mcp_servers=allowed_mcp_servers,
+        )
+
+    return allowed_mcp_servers
+
+
+def _client_has_per_server_auth_header(
+    server: MCPServer,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None,
+) -> bool:
+    """True if the request carries a per-server ``x-mcp-{alias}-authorization``
+    header for this server. This is the multi-server binding: it names one
+    upstream, so it is unambiguously the caller's upstream token regardless of
+    auth mode (never the LiteLLM admission credential).
+
+    Resolves through the same ``lookup_mcp_server_auth_in_headers`` egress uses, so
+    the connect gate and egress agree on which per-server header names match: a
+    dashboard client sends ``x-mcp-{sanitize_mcp_alias_for_header(alias)}-authorization``,
+    and matching only the raw alias here would 401 a token egress would forward.
+    """
+    if not mcp_server_auth_headers:
+        return False
+    from litellm.proxy._experimental.mcp_server.utils import (
+        lookup_mcp_server_auth_in_headers,
+    )
+
+    server_headers: Final = lookup_mcp_server_auth_in_headers(
+        mcp_server_auth_headers,
+        alias=server.alias,
+        server_name=server.server_name,
+        access_groups=server.access_groups,
+    )
+    if isinstance(server_headers, str):
+        return bool(server_headers.strip())
+    if isinstance(server_headers, dict):
+        return any(isinstance(hk, str) and hk.lower() == "authorization" for hk in server_headers)
+    return False
+
+
+def _client_has_passthrough_authorization(
+    server: MCPServer,
+    oauth2_headers: dict[str, str] | None,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None,
+) -> bool:
+    """True if the incoming request already carries an ``Authorization``
+    header the gateway will forward to this pass-through server.
+
+    The client may supply the bearer as either the top-level
+    ``Authorization`` header (surfaced via ``oauth2_headers``) or a
+    per-server ``x-mcp-auth-<alias>`` style header (surfaced via
+    ``mcp_server_auth_headers``). Either form skips the pre-emptive 401.
+    """
+    if oauth2_headers:
+        for k in oauth2_headers:
+            if k.lower() == "authorization":
+                return True
+    return _client_has_per_server_auth_header(server, mcp_server_auth_headers)
+
+
+async def _get_user_oauth_extra_headers_from_db(
+    server: MCPServer,
+    user_api_key_auth: UserAPIKeyAuth | None,
+    prefetched_creds: 'Mapping[str, "OAuthCredentialPayload"] | None' = None,
+) -> dict[str, str] | None:
+    """Stored OAuth2 token for (user, server) as an ``Authorization: Bearer`` header, or None.
+
+    Thin wrapper over ``resolve_user_oauth_access_token`` (Redis cache, else DB + refresh);
+    ``prefetched_creds`` skips the per-server Redis/DB lookups for the batch path.
+    """
+    if server.auth_type != MCPAuth.oauth2 or user_api_key_auth is None:
+        return None
+    from litellm.proxy._experimental.mcp_server.db import (  # noqa: PLC0415
+        resolve_user_oauth_access_token,
+    )
+
+    token: Final = await resolve_user_oauth_access_token(
+        getattr(user_api_key_auth, "user_id", None), server, prefetched_creds
+    )
+    return {"Authorization": f"Bearer {token}"} if token else None
+
+
+async def _prefetch_oauth_creds_for_user(
+    user_api_key_auth: UserAPIKeyAuth | None,
+) -> dict[str, "OAuthCredentialPayload"]:
+    """Fetch all OAuth2 credentials for the user in one DB query.
+
+    Returns a dict keyed by server_id to avoid N+1 queries in asyncio.gather loops.
+    """
+    user_id: Final[str | None] = getattr(user_api_key_auth, "user_id", None) if user_api_key_auth else None
+    if not user_id:
+        return {}
+    try:
+        from litellm.proxy._experimental.mcp_server.db import (  # noqa: PLC0415
+            list_user_oauth_credentials,
+        )
+        from litellm.proxy.utils import get_prisma_client_or_throw  # noqa: PLC0415
+
+        prisma_client: Final = get_prisma_client_or_throw(
+            "Database not connected. Connect a database to use OAuth2 MCP tools."
+        )
+        creds: Final = await list_user_oauth_credentials(prisma_client, user_id)
+        return {c["server_id"]: c for c in creds if "server_id" in c}
+    except Exception as e:
+        verbose_logger.warning("_prefetch_oauth_creds_for_user: failed to prefetch for user=%s: %s", user_id, e)
+        return {}
+
+
+def _prepare_mcp_server_headers(
+    server: MCPServer,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None,
+    mcp_auth_header: str | None,
+    oauth2_headers: dict[str, str] | None,
+    raw_headers: dict[str, str] | None,
+    user_api_key_auth: UserAPIKeyAuth | None = None,
+    scope_servers: list[MCPServer] | None = None,
+) -> tuple[dict[str, str] | str | None, dict[str, str] | None]:
+    """Build auth and extra headers for a server.
+
+    ``scope_servers`` is the full server list a fan-out handler iterates. Passing it lets the
+    client-forwarded token modes withhold the caller's request-wide ``Authorization`` when
+    another server in the scope would also receive it (``_caller_authorization_fans_out``);
+    explicitly-addressed operations leave it None. Per-server ``x-mcp-{alias}-authorization``
+    headers are unaffected — they bind one token to one server and are the multi-server shape.
+    """
+    server_auth_header: dict[str, str] | str | None = None
+    if mcp_server_auth_headers:
+        from litellm.proxy._experimental.mcp_server.utils import (
+            lookup_mcp_server_auth_in_headers,
+        )
+
+        server_auth_header = lookup_mcp_server_auth_in_headers(
+            mcp_server_auth_headers,
+            alias=server.alias,
+            server_name=server.server_name,
+            access_groups=server.access_groups,
+        )
+
+    extra_headers: dict[str, str] | None = None
+    is_client_forwarded_mode: Final = server.is_client_forwarded_token
+    # In a multi-server listing scope the request-wide Authorization can only carry one token,
+    # so it is withheld from a client-forwarded server when another server in scope also consumes
+    # it (RFC 9700 cross-resource replay); such scopes must bind per-server via
+    # x-mcp-{alias}-authorization. The decision is computed once so BOTH the forwarding branch and
+    # the extra_headers copy loop below honor it — otherwise a server that lists Authorization in
+    # extra_headers would re-copy the withheld bearer from raw_headers and replay it anyway.
+    withhold_forwarded_authorization: Final = is_client_forwarded_mode and _caller_authorization_fans_out(
+        server, scope_servers
+    )
+    if server.auth_type == MCPAuth.oauth2:
+        # For OAuth2 M2M servers, upstream Authorization must come from
+        # client_credentials token fetch, never from caller headers.
+        if server.has_client_credentials:
+            extra_headers = None
+        else:
+            # Copy to avoid mutating the original dict (important for parallel fetching)
+            extra_headers = oauth2_headers.copy() if oauth2_headers else None
+            # Migrated authorization_code: the v2 resolver injects the stored per-user
+            # token, so drop the caller-forwarded Authorization (apply-if-absent would
+            # otherwise let it shadow the resolved token). Delegate keeps it. Centralized
+            # via _should_strip_caller_authorization to match _call_regular_mcp_tool.
+            if extra_headers and _should_strip_caller_authorization(
+                mcp_server=server,
+                raw_headers=raw_headers,
+                user_api_key_auth=user_api_key_auth,
+            ):
+                extra_headers = without_header(extra_headers, DEFAULT_CREDENTIAL_HEADER)
+    elif is_client_forwarded_mode:
+        if not withhold_forwarded_authorization:
+            extra_headers = _client_forwarded_authorization_headers(
+                mcp_server=server,
+                oauth2_headers=oauth2_headers,
+                raw_headers=raw_headers,
+                user_api_key_auth=user_api_key_auth,
+            )
+
+    if server.extra_headers and raw_headers:
+        if extra_headers is None:
+            extra_headers = {}
+
+        normalized_raw_headers: Final = {str(k).lower(): v for k, v in raw_headers.items() if isinstance(k, str)}
+
+        # Centralized strip decision shared with
+        # ``MCPServerManager._call_regular_mcp_tool`` so the two
+        # code paths cannot drift on this security-sensitive choice.
+        # See ``_should_strip_caller_authorization`` for the rules.
+        strip_caller_authorization: Final = _should_strip_caller_authorization(
+            mcp_server=server,
+            raw_headers=raw_headers,
+            user_api_key_auth=user_api_key_auth,
+        )
+
+        for header in server.extra_headers:
+            if not isinstance(header, str):
+                continue
+            if header.lower() == "authorization" and (strip_caller_authorization or withhold_forwarded_authorization):
+                continue
+            header_value = normalized_raw_headers.get(header.lower())
+            if header_value is None:
+                continue
+            extra_headers[header] = header_value
+
+    # Reset to None if no headers were actually added
+    if extra_headers is not None and len(extra_headers) == 0:
+        extra_headers = None
+
+    if server_auth_header is None:
+        server_auth_header = mcp_auth_header
+
+    return server_auth_header, extra_headers
+
+
+def _merge_gateway_initialize_instructions(
+    allowed_mcp_servers: list[MCPServer],
+) -> str | None:
+    """YAML/DB override, else upstream text (prefetch on init, or list_tools / health_check / call_tool cache)."""
+    if not allowed_mcp_servers:
+        return None
+
+    texts: Final[list[tuple[str, str]]] = []
+    for server in allowed_mcp_servers:
+        label = server.alias or server.server_name or server.name or server.server_id or "mcp"
+        if server.instructions and server.instructions.strip():
+            texts.append((label, server.instructions.strip()))
+            continue
+        if server.spec_path:
+            continue
+        cached = global_mcp_server_manager._upstream_initialize_instructions_by_server_id.get(server.server_id)
+        if cached and cached.strip():
+            texts.append((label, cached.strip()))
+
+    if not texts:
+        return None
+    if len(texts) == 1:
+        return texts[0][1]
+    return "\n\n---\n\n".join(f"[{lbl}]\n{txt}" for lbl, txt in texts)
+
+
+async def _raise_if_initialize_grants_no_mcp_servers(
+    allowed: Sequence[MCPServer],
+    user_api_key_auth: UserAPIKeyAuth | None,
+    mcp_servers: Sequence[str] | None,
+    client_ip: str | None,
+) -> None:
+    if allowed or user_api_key_auth is None or not user_api_key_auth.api_key:
+        return
+    if mcp_servers:
+        await raise_denied_scoped_mcp_access(
+            requested_names=mcp_servers,
+            user_api_key_auth=user_api_key_auth,
+            client_ip=client_ip,
+        )
+    no_servers_denial: Final[_McpDeniedDetail] = {
+        "error": (
+            "The key has no MCP servers granted, or none of its granted servers is loaded and allowed for "
+            "this client IP. Grant servers or access groups to the key, its team, or its organization "
+            "(object_permission.mcp_servers), check the server's allowed IPs, and reconnect."
+        )
+    }
+    raise HTTPException(status_code=403, detail=no_servers_denial)
+
+
+def _aggregate_server_key(server: MCPServer) -> str:
+    """The client-visible key for a server in listing outcomes and spend metadata: the same
+    display prefix (alias, or the short prefix when that mode is enabled) the caller already
+    sees on the tool names. Canonical internal server names never key a caller-readable
+    surface; when the display naming deliberately hides them, the outcome keys must too."""
+    return get_server_prefix(server) or "unknown"
+
+
+async def _get_tools_from_mcp_servers(
+    user_api_key_auth: UserAPIKeyAuth | None,
+    mcp_auth_header: str | None,
+    mcp_servers: list[str] | None,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
+    oauth2_headers: dict[str, str] | None = None,
+    raw_headers: dict[str, str] | None = None,
+    log_list_tools_to_spendlogs: bool = False,
+    list_tools_log_source: str | None = None,
+    litellm_trace_id: str | None = None,
+    request_tags: list[str] | None = None,
+    client_ip: str | None = None,
+    mcp_proxy_mode: bool = False,
+) -> AggregateToolListing:
+    """
+    Helper method to fetch tools from MCP servers based on server filtering criteria.
+
+    Args:
+        user_api_key_auth: User authentication info for access control
+        mcp_auth_header: Optional auth header for MCP server (deprecated)
+        mcp_servers: Optional list of server names/aliases to filter by
+        mcp_server_auth_headers: Optional dict of server-specific auth headers
+        oauth2_headers: Optional dict of oauth2 headers
+
+    Returns:
+        AggregateToolListing: Combined tools from filtered servers plus each server's
+        classified listing outcome
+    """
+
+    list_tools_start_time: Final = datetime.now()
+    litellm_logging_obj: LiteLLMLoggingObj | None = None
+    list_tools_request_data: dict[str, object] = {}
+
+    if log_list_tools_to_spendlogs:
+        # This is intentionally minimal: only async_success_handler / post_call_failure_hook
+        rules_obj: Final = Rules()
+        list_tools_call_id: Final = str(uuid.uuid4())
+        # Derive trace_id from raw_headers when not explicitly passed (same as A2A / MCP call_tool)
+        effective_litellm_trace_id: Final = litellm_trace_id or get_chain_id_from_headers(raw_headers)
+        spend_logs_metadata: Final[dict[str, object]] = {
+            "mcp_operation": "list_tools",
+        }
+        if isinstance(list_tools_log_source, str):
+            spend_logs_metadata["source"] = list_tools_log_source
+        if isinstance(mcp_servers, list):
+            spend_logs_metadata["requested_mcp_servers"] = mcp_servers
+
+        list_tools_request_data = {
+            "model": "MCP: list_tools",
+            "call_type": CallTypes.list_mcp_tools.value,
+            "litellm_call_id": list_tools_call_id,
+            "litellm_trace_id": effective_litellm_trace_id,
+            "metadata": {
+                "spend_logs_metadata": spend_logs_metadata,
+                "headers": logging_safe_mcp_headers(raw_headers),
+                **({"tags": request_tags} if request_tags else {}),
+            },
+            # Provide a small input payload for standard logging
+            "input": [
+                {
+                    "role": "system",
+                    "content": {
+                        "mcp_operation": "list_tools",
+                        "requested_mcp_servers": mcp_servers,
+                    },
+                }
+            ],
+        }
+
+        # Attach user identifiers using the standard helper
+        if user_api_key_auth is not None:
+            LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata(
+                data=list_tools_request_data,
+                user_api_key_dict=user_api_key_auth,
+                _metadata_variable_name="metadata",
+            )
+
+            user_identifier: Final = getattr(user_api_key_auth, "end_user_id", None) or getattr(
+                user_api_key_auth, "user_id", None
+            )
+            if user_identifier:
+                list_tools_request_data["user"] = user_identifier
+
+        try:
+            litellm_logging_obj, _ = function_setup(
+                original_function="list_mcp_tools",
+                is_async_call=False,
+                rules_obj=rules_obj,
+                start_time=list_tools_start_time,
+                **list_tools_request_data,
+            )
+            if litellm_logging_obj:
+                litellm_logging_obj.call_type = CallTypes.list_mcp_tools.value
+                litellm_logging_obj.model = "MCP: list_tools"
+        except Exception as logging_error:
+            verbose_logger.debug("Failed to initialize logging for MCP list_tools: %s", logging_error)
+            litellm_logging_obj = None
+
+    try:
+        allowed_mcp_servers: Final = await _get_allowed_mcp_servers(
+            user_api_key_auth=user_api_key_auth,
+            mcp_servers=mcp_servers,
+            client_ip=client_ip,
+        )
+        if mcp_servers and not allowed_mcp_servers:
+            await raise_denied_scoped_mcp_access(
+                requested_names=mcp_servers,
+                user_api_key_auth=user_api_key_auth,
+                client_ip=client_ip,
+            )
+
+        # Pre-fetch OAuth credentials only when at least one server uses OAuth2,
+        # to avoid an unnecessary DB round-trip on requests with no OAuth2 MCP servers.
+        _has_oauth2_server = any(getattr(s, "auth_type", None) == MCPAuth.oauth2 for s in allowed_mcp_servers)
+        _prefetched_oauth_creds: Final = (
+            await _prefetch_oauth_creds_for_user(user_api_key_auth) if _has_oauth2_server else {}
+        )
+
+        async def _fetch_and_filter_server_tools(
+            server: MCPServer,
+        ) -> "tuple[list[MCPTool], ServerOutcome]":
+            """Fetch and filter tools from a single server, classifying any failure into that
+            server's outcome so the aggregate can keep serving the healthy subset without a
+            broken server masquerading as an empty one."""
+            if server is None:
+                return [], ServerListOk(tool_count=0)
+
+            server_auth_header, extra_headers = _prepare_mcp_server_headers(
+                server=server,
+                mcp_server_auth_headers=mcp_server_auth_headers,
+                mcp_auth_header=mcp_auth_header,
+                oauth2_headers=oauth2_headers,
+                raw_headers=raw_headers,
+                user_api_key_auth=user_api_key_auth,
+                scope_servers=allowed_mcp_servers,
+            )
+
+            # Prefer server-stored per-user OAuth when configured, so a stale
+            # Authorization header from the MCP client cannot override Redis/DB
+            # (same issue as call_tool in mcp_server_manager: VS Code caches tokens).
+            from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import (  # noqa: PLC0415
+                to_server_spec,
+            )
+
+            # A server migrated to the v2 resolver gets its token from the resolver at connect
+            # time; building it here would double-resolve and be shadowed by the v2 graft. The
+            # preemptive 401 already challenged a missing token, so one exists for the connect.
+            migrated_to_v2: Final = to_server_spec(server) is not None
+            if (
+                not migrated_to_v2
+                and server.auth_type == MCPAuth.oauth2
+                and getattr(server, "needs_user_oauth_token", False)
+                and user_api_key_auth is not None
+            ):
+                db_headers: Final = await _get_user_oauth_extra_headers_from_db(
+                    server,
+                    user_api_key_auth,
+                    prefetched_creds=_prefetched_oauth_creds,
+                )
+                if db_headers:
+                    extra_headers = db_headers
+
+            # If still no OAuth2 token, fall back to pre-fetched creds (non-stale-client path)
+            elif not migrated_to_v2 and extra_headers is None and server.auth_type == MCPAuth.oauth2:
+                extra_headers = await _get_user_oauth_extra_headers_from_db(
+                    server,
+                    user_api_key_auth,
+                    prefetched_creds=_prefetched_oauth_creds,
+                )
+
+            if server.is_byok and server.auth_type != MCPAuth.oauth2 and server_auth_header is None:
+                server_auth_header = await _get_byok_credential(server, user_api_key_auth)
+
+            try:
+                tools: Final = await global_mcp_server_manager._get_tools_from_server(
+                    server=server,
+                    mcp_auth_header=server_auth_header,
+                    extra_headers=extra_headers,
+                    add_prefix=True,  # Always add server prefix
+                    raw_headers=raw_headers,
+                    client_ip=client_ip,
+                    user_api_key_auth=user_api_key_auth,
+                    oauth2_headers=oauth2_headers,
+                )
+                filtered_tools = filter_tools_by_allowed_tools(tools, server)
+
+                filtered_tools = await filter_tools_by_key_team_permissions(
+                    tools=filtered_tools,
+                    server_id=server.server_id,
+                    user_api_key_auth=user_api_key_auth,
+                )
+
+                if mcp_proxy_mode:
+                    from litellm.proxy._experimental.mcp_server.tool_search import with_mcp_proxy_identity
+
+                    filtered_tools = [  # mutable-ok: MCP tool pipeline
+                        with_mcp_proxy_identity(tool, server.server_id) for tool in filtered_tools
+                    ]
+                else:
+                    filtered_tools = apply_tool_overrides(filtered_tools, server)
+
+                verbose_logger.debug(
+                    "Successfully fetched %s tools from server %s, %s after filtering",
+                    len(tools),
+                    server.name,
+                    len(filtered_tools),
+                )
+                return filtered_tools, ServerListOk(tool_count=len(filtered_tools))
+            except MCPUpstreamAuthError as e:
+                # Absorb so one unauthenticated server does not empty every other server's
+                # tools. Surfacing the upstream 401 to the client as a re-auth challenge is
+                # intentionally not done here: raising from this list handler cannot produce a
+                # 401 + WWW-Authenticate (the MCP session manager serializes it as a JSON-RPC
+                # error). Single-server routes surface it via the request-scope preemptive
+                # check in _raise_preemptive_401_for_unauthenticated_servers instead.
+                verbose_logger.debug("MCP list_tools: omitting %s; it needs upstream auth", server.name)
+                return [], classify_list_exception(e)
+            except Exception as e:
+                verbose_logger.exception("Error getting tools from server %s: %s", server.name, e)
+                return [], classify_list_exception(e)
+
+        # Fetch tools from all servers in parallel
+        tasks: Final = [_fetch_and_filter_server_tools(server) for server in allowed_mcp_servers]
+        results: Final = await asyncio.gather(*tasks)
+
+        # Flatten results into single list
+        all_tools: Final[list[MCPTool]] = [tool for tools, _ in results for tool in tools]
+        server_outcomes: Final[dict[str, ServerOutcome]] = {
+            _aggregate_server_key(server): outcome
+            for server, (_, outcome) in zip(allowed_mcp_servers, results)
+            if server is not None
+        }
+
+        # If logging is enabled, enrich spend_logs_metadata with counts
+        if litellm_logging_obj:
+            per_server_tool_counts: Final[dict[str, int]] = {
+                _aggregate_server_key(server): len(server_tools)
+                for server, (server_tools, _) in zip(allowed_mcp_servers, results)
+                if server is not None
+            }
+
+            metadata_dict: Final = litellm_logging_obj.model_call_details.get("metadata")
+            if isinstance(metadata_dict, dict):
+                spend_meta = metadata_dict.get("spend_logs_metadata")
+                if not isinstance(spend_meta, dict):
+                    spend_meta = {}
+                    metadata_dict["spend_logs_metadata"] = spend_meta
+                spend_meta["allowed_server_count"] = len(allowed_mcp_servers)
+                spend_meta["tool_count_total"] = len(all_tools)
+                spend_meta["per_server_tool_counts"] = per_server_tool_counts
+                spend_meta["per_server_list_outcomes"] = {
+                    key: outcome_wire_value(outcome) for key, outcome in server_outcomes.items()
+                }
+
+            end_time: Final = datetime.now()
+            try:
+                await litellm_logging_obj.async_success_handler(
+                    result=[tool.model_dump(mode="json") if isinstance(tool, MCPTool) else tool for tool in all_tools],
+                    start_time=list_tools_start_time,
+                    end_time=end_time,
+                )
+            except Exception as log_exc:
+                # list_tools responses must not be dropped due to non-blocking
+                # observability/serialization failures.
+                verbose_logger.warning(
+                    "MCP list_tools success logging failed (continuing): %s",
+                    log_exc,
+                )
+
+        verbose_logger.info("Successfully fetched %s tools total from all MCP servers", len(all_tools))
+
+        return AggregateToolListing(tools=all_tools, outcomes=server_outcomes)
+    except Exception as e:
+        # Only fire failure hook if logging was requested for this list-tools execution
+        if log_list_tools_to_spendlogs and user_api_key_auth is not None:
+            try:
+                from litellm.proxy.proxy_server import proxy_logging_obj
+
+                if proxy_logging_obj:
+                    traceback_str: Final = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG)
+                    await proxy_logging_obj.post_call_failure_hook(
+                        request_data=list_tools_request_data or {},
+                        original_exception=e,
+                        user_api_key_dict=user_api_key_auth,
+                        route="/mcp/list_tools",
+                        traceback_str=traceback_str,
+                    )
+            except Exception:
+                verbose_logger.debug("Failed to log MCP list_tools failure via post_call_failure_hook")
+        raise
+
+
+async def _get_prompts_from_mcp_servers(
+    user_api_key_auth: UserAPIKeyAuth | None,
+    mcp_auth_header: str | None,
+    mcp_servers: list[str] | None,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
+    oauth2_headers: dict[str, str] | None = None,
+    raw_headers: dict[str, str] | None = None,
+    client_ip: str | None = None,
+) -> list[Prompt]:
+    """
+    Helper method to fetch prompt from MCP servers based on server filtering criteria.
+
+    Args:
+        user_api_key_auth: User authentication info for access control
+        mcp_auth_header: Optional auth header for MCP server (deprecated)
+        mcp_servers: Optional list of server names/aliases to filter by
+        mcp_server_auth_headers: Optional dict of server-specific auth headers
+        oauth2_headers: Optional dict of oauth2 headers
+
+    Returns:
+        List[Prompt]: Combined list of prompts from filtered servers
+    """
+
+    allowed_mcp_servers: Final = await _get_allowed_mcp_servers(
+        user_api_key_auth=user_api_key_auth,
+        mcp_servers=mcp_servers,
+        client_ip=client_ip,
+    )
+
+    # Get prompts from each allowed server
+    all_prompts: Final = []
+    for server in allowed_mcp_servers:
+        if server is None:
+            continue
+
+        server_auth_header, extra_headers = _prepare_mcp_server_headers(
+            server=server,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            mcp_auth_header=mcp_auth_header,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            user_api_key_auth=user_api_key_auth,
+            scope_servers=allowed_mcp_servers,
+        )
+
+        try:
+            prompts = await global_mcp_server_manager.get_prompts_from_server(
+                server=server,
+                user_api_key_auth=user_api_key_auth,
+                mcp_auth_header=server_auth_header,
+                extra_headers=extra_headers,
+                add_prefix=True,  # Always add server prefix
+                raw_headers=raw_headers,
+                client_ip=client_ip,
+            )
+
+            all_prompts.extend(prompts)
+
+            verbose_logger.debug("Successfully fetched %s prompts from server %s", len(prompts), server.name)
+        except Exception as e:
+            verbose_logger.exception("Error getting prompts from server %s: %s", server.name, e)
+            # Continue with other servers instead of failing completely
+
+    verbose_logger.info("Successfully fetched %s prompts total from all MCP servers", len(all_prompts))
+
+    return all_prompts
+
+
+async def _get_resources_from_mcp_servers(
+    user_api_key_auth: UserAPIKeyAuth | None,
+    mcp_auth_header: str | None,
+    mcp_servers: list[str] | None,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
+    oauth2_headers: dict[str, str] | None = None,
+    raw_headers: dict[str, str] | None = None,
+    client_ip: str | None = None,
+) -> list[Resource]:
+    """Fetch resources from allowed MCP servers."""
+
+    allowed_mcp_servers: Final = await _get_allowed_mcp_servers(
+        user_api_key_auth=user_api_key_auth,
+        mcp_servers=mcp_servers,
+        client_ip=client_ip,
+    )
+
+    all_resources: Final[list[Resource]] = []
+    for server in allowed_mcp_servers:
+        if server is None:
+            continue
+
+        server_auth_header, extra_headers = _prepare_mcp_server_headers(
+            server=server,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            mcp_auth_header=mcp_auth_header,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            user_api_key_auth=user_api_key_auth,
+            scope_servers=allowed_mcp_servers,
+        )
+
+        try:
+            resources = await global_mcp_server_manager.get_resources_from_server(
+                server=server,
+                user_api_key_auth=user_api_key_auth,
+                mcp_auth_header=server_auth_header,
+                extra_headers=extra_headers,
+                add_prefix=True,  # Always add server prefix
+                raw_headers=raw_headers,
+                client_ip=client_ip,
+            )
+            all_resources.extend(resources)
+
+            verbose_logger.debug("Successfully fetched %s resources from server %s", len(resources), server.name)
+        except Exception as e:
+            verbose_logger.exception("Error getting resources from server %s: %s", server.name, e)
+
+    verbose_logger.info("Successfully fetched %s resources total from all MCP servers", len(all_resources))
+
+    return all_resources
+
+
+async def _get_resource_templates_from_mcp_servers(
+    user_api_key_auth: UserAPIKeyAuth | None,
+    mcp_auth_header: str | None,
+    mcp_servers: list[str] | None,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
+    oauth2_headers: dict[str, str] | None = None,
+    raw_headers: dict[str, str] | None = None,
+    client_ip: str | None = None,
+) -> list[ResourceTemplate]:
+    """Fetch resource templates from allowed MCP servers."""
+
+    allowed_mcp_servers: Final = await _get_allowed_mcp_servers(
+        user_api_key_auth=user_api_key_auth,
+        mcp_servers=mcp_servers,
+        client_ip=client_ip,
+    )
+
+    all_resource_templates: Final[list[ResourceTemplate]] = []
+    for server in allowed_mcp_servers:
+        if server is None:
+            continue
+
+        server_auth_header, extra_headers = _prepare_mcp_server_headers(
+            server=server,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            mcp_auth_header=mcp_auth_header,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            user_api_key_auth=user_api_key_auth,
+            scope_servers=allowed_mcp_servers,
+        )
+
+        try:
+            resource_templates = await global_mcp_server_manager.get_resource_templates_from_server(
+                server=server,
+                user_api_key_auth=user_api_key_auth,
+                mcp_auth_header=server_auth_header,
+                extra_headers=extra_headers,
+                add_prefix=True,  # Always add server prefix
+                raw_headers=raw_headers,
+                client_ip=client_ip,
+            )
+            all_resource_templates.extend(resource_templates)
+            verbose_logger.debug(
+                "Successfully fetched %s resource templates from server %s",
+                len(resource_templates),
+                server.name,
+            )
+        except Exception as e:
+            verbose_logger.exception(
+                "Error getting resource templates from server %s: %s",
+                server.name,
+                str(e),
+            )
+
+    verbose_logger.info(
+        "Successfully fetched %s resource templates total from all MCP servers",
+        len(all_resource_templates),
+    )
+
+    return all_resource_templates
+
+
+async def filter_tools_by_key_team_permissions(
+    tools: list[MCPTool],
+    server_id: str,
+    user_api_key_auth: UserAPIKeyAuth | None,
+) -> list[MCPTool]:
+    """
+    Filter tools based on key/team mcp_tool_permissions.
+
+    Note: Tool names in the DB are stored without server prefixes,
+    but tool names from MCP servers are prefixed. We need to strip
+    the prefix before comparing.
+    """
+    # Filter by key/team tool-level permissions
+    allowed_tool_names: Final = await MCPRequestHandler.get_allowed_tools_for_server(
+        server_id=server_id,
+        user_api_key_auth=user_api_key_auth,
+    )
+
+    # Tools arrive prefixed with the server's own prefix; strip exactly that
+    # prefix (resolved from the server) rather than the first separator, so a
+    # prefix containing the separator still reduces to the stored bare name.
+    server: Final = global_mcp_server_manager.get_mcp_server_by_id(server_id)
+    return [
+        t
+        for t in tools
+        if MCPRequestHandler.tool_is_granted(strip_known_server_prefix(t.name, server), allowed_tool_names)
+    ]
+
+
+async def _list_mcp_tools(
+    user_api_key_auth: UserAPIKeyAuth | None = None,
+    mcp_auth_header: str | None = None,
+    mcp_servers: list[str] | None = None,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
+    oauth2_headers: dict[str, str] | None = None,
+    raw_headers: dict[str, str] | None = None,
+    log_list_tools_to_spendlogs: bool = False,
+    list_tools_log_source: str | None = None,
+    client_ip: str | None = None,
+    mcp_proxy_mode: bool = False,
+) -> AggregateToolListing:
+    """
+    List all available MCP tools.
+
+    Args:
+        user_api_key_auth: User authentication info for access control
+        mcp_auth_header: Optional auth header for MCP server (deprecated)
+        mcp_servers: Optional list of server names/aliases to filter by
+        mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value}
+        client_ip: Client IP for IP-based server access control
+
+    Returns:
+        AggregateToolListing: Combined tools from all accessible servers plus each server's
+        classified listing outcome
+    """
+
+    try:
+        listing: Final = await _get_tools_from_mcp_servers(
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=mcp_auth_header,
+            mcp_servers=mcp_servers,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            log_list_tools_to_spendlogs=log_list_tools_to_spendlogs,
+            list_tools_log_source=list_tools_log_source,
+            client_ip=client_ip,
+            mcp_proxy_mode=mcp_proxy_mode,
+        )
+        verbose_logger.debug("Successfully fetched %s tools from managed MCP servers", len(listing.tools))
+        return listing
+    except HTTPException:
+        raise
+    except Exception as e:
+        verbose_logger.exception("Error getting tools from managed MCP servers: %s", e)
+        # Continue with an empty listing instead of failing completely
+        return AggregateToolListing(tools=[], outcomes={})
+
+
+async def _list_mcp_prompts(
+    user_api_key_auth: UserAPIKeyAuth | None = None,
+    mcp_auth_header: str | None = None,
+    mcp_servers: list[str] | None = None,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
+    oauth2_headers: dict[str, str] | None = None,
+    raw_headers: dict[str, str] | None = None,
+    client_ip: str | None = None,
+) -> list[Prompt]:
+    """
+    List all available MCP prompts.
+
+    Args:
+        user_api_key_auth: User authentication info for access control
+        mcp_auth_header: Optional auth header for MCP server (deprecated)
+        mcp_servers: Optional list of server names/aliases to filter by
+        mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value}
+
+    Returns:
+        List[Prompt]: Combined list of tools from all accessible servers
+    """
+    # Get tools from managed MCP servers with error handling
+    managed_prompts = []
+    try:
+        managed_prompts = await _get_prompts_from_mcp_servers(
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=mcp_auth_header,
+            mcp_servers=mcp_servers,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            client_ip=client_ip,
+        )
+        verbose_logger.debug("Successfully fetched %s prompts from managed MCP servers", len(managed_prompts))
+    except Exception as e:
+        verbose_logger.exception("Error getting tools from managed MCP servers: %s", e)
+        # Continue with empty managed tools list instead of failing completely
+
+    return managed_prompts
+
+
+async def _list_mcp_resources(
+    user_api_key_auth: UserAPIKeyAuth | None = None,
+    mcp_auth_header: str | None = None,
+    mcp_servers: list[str] | None = None,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
+    oauth2_headers: dict[str, str] | None = None,
+    raw_headers: dict[str, str] | None = None,
+    client_ip: str | None = None,
+) -> list[Resource]:
+    """List all available MCP resources."""
+
+    managed_resources: list[Resource] = []
+    try:
+        managed_resources = await _get_resources_from_mcp_servers(
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=mcp_auth_header,
+            mcp_servers=mcp_servers,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            client_ip=client_ip,
+        )
+        verbose_logger.debug("Successfully fetched %s resources from managed MCP servers", len(managed_resources))
+    except Exception as e:
+        verbose_logger.exception("Error getting resources from managed MCP servers: %s", e)
+
+    return managed_resources
+
+
+async def _list_mcp_resource_templates(
+    user_api_key_auth: UserAPIKeyAuth | None = None,
+    mcp_auth_header: str | None = None,
+    mcp_servers: list[str] | None = None,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
+    oauth2_headers: dict[str, str] | None = None,
+    raw_headers: dict[str, str] | None = None,
+    client_ip: str | None = None,
+) -> list[ResourceTemplate]:
+    """List all available MCP resource templates."""
+
+    managed_resource_templates: list[ResourceTemplate] = []
+    try:
+        managed_resource_templates = await _get_resource_templates_from_mcp_servers(
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=mcp_auth_header,
+            mcp_servers=mcp_servers,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            client_ip=client_ip,
+        )
+        verbose_logger.debug(
+            "Successfully fetched %s resource templates from managed MCP servers",
+            len(managed_resource_templates),
+        )
+    except Exception as e:
+        verbose_logger.exception(
+            "Error getting resource templates from managed MCP servers: %s",
+            str(e),
+        )
+
+    return managed_resource_templates
+
+
+def _resolve_display_name_to_original(
+    name: str,
+    allowed_mcp_servers: list[MCPServer],
+) -> str:
+    """Translate a display-name override back to the original prefixed tool name.
+
+    When a client received a customised display name from tools/list (e.g.
+    "Get Pet") it will call tools/call with that same string.  We need to
+    reverse-map it to the original prefixed name (e.g.
+    "petstore_mcp-getPetById") before any routing or permission logic runs.
+    """
+    for server in allowed_mcp_servers:
+        display_map = server.tool_name_to_display_name or {}
+        for unprefixed_name, display_name in display_map.items():
+            if display_name == name:
+                return add_server_prefix_to_name(unprefixed_name, get_server_prefix(server))
+    return name
+
+
+async def _get_byok_credential(
+    mcp_server: MCPServer,
+    user_api_key_auth: UserAPIKeyAuth | None,
+) -> str | None:
+    """Retrieve the stored BYOK credential for a user+server pair, served from the worker cache within its TTL."""
+    if not mcp_server.is_byok:
+        return None
+    user_id: Final = (user_api_key_auth.user_id if user_api_key_auth else None) or ""
+    if not user_id:
+        return None
+
+    cached: Final = get_cached_byok_credential(user_id, mcp_server.server_id)
+    if cached is not None:
+        return cached.credential
+
+    from litellm.proxy._experimental.mcp_server.db import get_user_credential
+    from litellm.proxy.proxy_server import prisma_client
+
+    if prisma_client is None:
+        return None
+    credential: Final = await get_user_credential(
+        prisma_client=prisma_client,
+        user_id=user_id,
+        server_id=mcp_server.server_id,
+    )
+    cache_byok_credential(user_id, mcp_server.server_id, credential)
+    return credential
+
+
+async def _check_byok_credential(
+    mcp_server: MCPServer,
+    user_api_key_auth: UserAPIKeyAuth | None,
+) -> None:
+    """
+    If the MCP server is BYOK-enabled, verify that the requesting user has a
+    stored credential.  When no credential is found, raise an HTTP 401 with a
+    WWW-Authenticate header that points the MCP client to our OAuth metadata
+    endpoint so it can drive the authorization flow.
+    """
+    if not mcp_server.is_byok:
+        return
+
+    user_id: Final = (user_api_key_auth.user_id if user_api_key_auth else None) or ""
+    if not user_id:
+        raise HTTPException(
+            status_code=401,
+            detail={
+                "error": "byok_auth_required",
+                "server_id": mcp_server.server_id,
+                "server_name": mcp_server.server_name or mcp_server.name,
+                "message": "User identity is required for BYOK servers",
+            },
+            headers={"WWW-Authenticate": get_byok_www_authenticate()},
+        )
+
+    cached: Final = get_cached_byok_credential(user_id, mcp_server.server_id)
+    if cached is not None:
+        if cached.credential is None:
+            raise HTTPException(
+                status_code=401,
+                detail={
+                    "error": "byok_auth_required",
+                    "server_id": mcp_server.server_id,
+                    "server_name": mcp_server.server_name or mcp_server.name,
+                    "message": (
+                        "No stored credential found for this BYOK server. "
+                        "Complete the OAuth authorization flow to provide your API key."
+                    ),
+                },
+                headers={"WWW-Authenticate": get_byok_www_authenticate()},
+            )
+        return
+
+    from litellm.proxy._experimental.mcp_server.db import get_user_credential
+    from litellm.proxy.proxy_server import prisma_client
+
+    if prisma_client is None:
+        # Fail closed on DB unavailability: returning here previously
+        # bypassed the ownership check and let any proxy-authenticated
+        # caller invoke BYOK tools during outage windows.
+        raise HTTPException(
+            status_code=503,
+            detail={
+                "error": "byok_auth_unavailable",
+                "server_id": mcp_server.server_id,
+                "server_name": mcp_server.server_name or mcp_server.name,
+                "message": "BYOK credential check requires a database connection.",
+            },
+        )
+
+    credential: Final = await get_user_credential(
+        prisma_client=prisma_client,
+        user_id=user_id,
+        server_id=mcp_server.server_id,
+    )
+    cache_byok_credential(user_id, mcp_server.server_id, credential)
+    if credential is None:
+        raise HTTPException(
+            status_code=401,
+            detail={
+                "error": "byok_auth_required",
+                "server_id": mcp_server.server_id,
+                "server_name": mcp_server.server_name or mcp_server.name,
+                "message": (
+                    "No stored credential found for this BYOK server. "
+                    "Complete the OAuth authorization flow to provide your API key."
+                ),
+            },
+            headers={"WWW-Authenticate": get_byok_www_authenticate()},
+        )
+
+
+async def _list_tools_before_first_call(
+    server: MCPServer | None,
+    tool_name: str,
+    allowed_mcp_servers: list[MCPServer],
+    user_api_key_auth: UserAPIKeyAuth | None,
+    mcp_auth_header: str | None,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None,
+    oauth2_headers: dict[str, str] | None,
+    raw_headers: dict[str, str] | None,
+    client_ip: str | None = None,
+) -> None:
+    """List ``server`` with the caller's own credentials when it does not yet expose ``tool_name`` here.
+
+    The startup fill skips a server whose upstream wants the caller's token, and mcp 2 no
+    longer lists before an uncached tools/call, so a worker that has not served tools/list
+    for this caller would otherwise answer 404 for a tool the caller can see. Gating on the
+    requested tool, not on any prior listing, keeps callers with different upstream catalogs
+    from masking each other.
+    """
+    if server is None or global_mcp_server_manager.server_exposes_tool(server, tool_name):
+        return
+    if all(allowed.server_id != server.server_id for allowed in allowed_mcp_servers):
+        return
+    try:
+        await _get_tools_from_mcp_servers(
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=mcp_auth_header,
+            mcp_servers=[server.server_id],
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            client_ip=client_ip,
+        )
+    except Exception as e:  # noqa: BLE001  # best effort: resolution below answers as it did before
+        verbose_logger.debug("MCP tools/call: listing %s before its first call failed: %s", server.name, e)
+
+
+async def execute_mcp_tool(
+    name: str,
+    arguments: dict[str, object],
+    allowed_mcp_servers: list[MCPServer],
+    start_time: datetime,
+    user_api_key_auth: UserAPIKeyAuth | None = None,
+    mcp_auth_header: str | None = None,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
+    oauth2_headers: dict[str, str] | None = None,
+    raw_headers: dict[str, str] | None = None,
+    host_progress_callback: ProgressCallback | None = None,
+    guardrail_context: Mapping[str, object] | None = None,
+    client_ip: str | None = None,
+    **kwargs: object,  # kwargs-ok: preserves the existing REST and decorated logging call contract
+) -> CallToolResult:
+    context: Final = prepare_context(
+        user_api_key_auth=user_api_key_auth,
+        mcp_auth_header=mcp_auth_header,
+        mcp_server_auth_headers=mcp_server_auth_headers,
+        oauth2_headers=oauth2_headers,
+        raw_headers=raw_headers,
+        client_ip=client_ip,
+    )
+    operation: Final = AuthorizedToolCall(
+        name=name,
+        arguments=arguments,
+        allowed_mcp_servers=tuple(allowed_mcp_servers),
+        start_time=start_time,
+        host_progress_callback=host_progress_callback,
+        guardrail_context=guardrail_context,
+        logging_data=types.MappingProxyType(kwargs),
+    )
+    return await GatewayOperations().execute(operation, context)
+
+
+async def _execute_mcp_tool(
+    name: str,
+    arguments: dict[str, object],
+    allowed_mcp_servers: list[MCPServer],
+    start_time: datetime,
+    user_api_key_auth: UserAPIKeyAuth | None = None,
+    mcp_auth_header: str | None = None,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
+    oauth2_headers: dict[str, str] | None = None,
+    raw_headers: dict[str, str] | None = None,
+    host_progress_callback: ProgressCallback | None = None,
+    guardrail_context: Mapping[str, object] | None = None,
+    client_ip: str | None = None,
+    **kwargs: Any,
+) -> CallToolResult:
+    """
+    Execute MCP tool.
+
+    This function assumes permission checks have already been performed.
+
+    Args:
+        name: Tool name (may include server prefix)
+        arguments: Tool arguments
+        allowed_mcp_servers: Pre-validated list of servers the user can access
+        start_time: Start time for logging
+        user_api_key_auth: Optional user API key auth for logging
+        mcp_auth_header: Optional MCP auth header
+        mcp_server_auth_headers: Optional server-specific auth headers
+        oauth2_headers: Optional OAuth2 headers
+        raw_headers: Optional raw HTTP headers
+        **kwargs: Additional arguments (e.g., litellm_logging_obj)
+
+    Returns:
+        CallToolResult: Tool execution result
+    """
+    # Track resolved MCP server for both permission checks and dispatch
+    mcp_server: MCPServer | None = None
+    requested_server_id: Final[str | None] = kwargs.get("requested_server_id")
+
+    # If the client called with a display-name override (e.g. "Get Pet"),
+    # translate it back to the original prefixed name before any routing.
+    name = _resolve_display_name_to_original(name, allowed_mcp_servers)
+
+    # Remove prefix from tool name for logging and processing
+    original_tool_name, server_name = split_server_prefix_from_name(name)
+
+    requested_server: MCPServer | None = None
+    if requested_server_id:
+        requested_server = next(
+            (s for s in allowed_mcp_servers if s.server_id == requested_server_id),
+            None,
+        )
+
+    name_is_prefixed = False
+    if requested_server is not None and MCP_TOOL_PREFIX_SEPARATOR in name:
+        all_registry_prefixes: Final[set[str]] = set()
+        for registry_server in global_mcp_server_manager.get_registry().values():
+            for known_prefix in iter_known_server_prefixes(registry_server):
+                all_registry_prefixes.add(normalize_server_name(known_prefix))
+        name_is_prefixed = is_tool_name_prefixed(name, known_server_prefixes=all_registry_prefixes)
+
+    first_call_target: Final = (
+        requested_server
+        if requested_server is not None and not name_is_prefixed
+        else global_mcp_server_manager.server_owning_tool_name_prefix(name)
+    )
+    first_call_tool_name: Final = (
+        name
+        if first_call_target is None or (requested_server is not None and not name_is_prefixed)
+        else strip_known_server_prefix(name, first_call_target)
+    )
+    await _list_tools_before_first_call(
+        server=first_call_target,
+        tool_name=first_call_tool_name,
+        allowed_mcp_servers=allowed_mcp_servers,
+        user_api_key_auth=user_api_key_auth,
+        mcp_auth_header=mcp_auth_header,
+        mcp_server_auth_headers=mcp_server_auth_headers,
+        oauth2_headers=oauth2_headers,
+        raw_headers=raw_headers,
+        client_ip=client_ip,
+    )
+
+    if requested_server is not None and not name_is_prefixed:
+        # REST callers may pass server_id with the upstream tool name (no
+        # LiteLLM prefix). The first segment is not a registered server
+        # prefix, so the whole string is the upstream tool name and may
+        # legitimately contain the separator (e.g. "text-to-speech").
+        # server_id is authoritative for routing and auth.
+        mcp_server = requested_server
+        server_name = requested_server.name
+        original_tool_name = name
+    else:
+        # Resolve from tool name (MCP JSON-RPC or prefixed REST tool names).
+        mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(name)
+        if mcp_server is None and requested_server is not None:
+            for known_prefix in iter_known_server_prefixes(requested_server):
+                candidate = global_mcp_server_manager._get_mcp_server_from_tool_name(
+                    add_server_prefix_to_name(name, known_prefix)
+                )
+                if candidate is not None:
+                    mcp_server = candidate
+                    break
+        if mcp_server is not None:
+            server_name = mcp_server.name
+            original_tool_name = strip_known_server_prefix(name, mcp_server)
+
+        if requested_server is not None:
+            if mcp_server is not None and mcp_server.server_id != requested_server.server_id:
+                raise HTTPException(
+                    status_code=403,
+                    detail={
+                        "error": "tool_server_mismatch",
+                        "message": (
+                            f"Tool '{name}' belongs to MCP server "
+                            f"'{mcp_server.name}' but request specified "
+                            f"server_id for '{requested_server.name}'."
+                        ),
+                    },
+                )
+            if mcp_server is None:
+                mcp_server = requested_server
+                server_name = requested_server.name
+                original_tool_name = strip_known_server_prefix(name, requested_server)
+
+    # Only enforce server-level permissions when we can resolve a server
+    if server_name:
+        if not MCPRequestHandler.is_tool_allowed(
+            allowed_mcp_servers=[server.name for server in allowed_mcp_servers],
+            server_name=server_name,
+        ):
+            raise HTTPException(
+                status_code=403,
+                detail="User not allowed to call this tool.",
+            )
+
+    standard_logging_mcp_tool_call: Final[StandardLoggingMCPToolCall] = _get_standard_logging_mcp_tool_call(
+        name=original_tool_name,  # Use original name for logging
+        arguments=arguments,
+        server_name=server_name,
+        session_id=_mcp_session_id_from_headers(raw_headers),
+    )
+    litellm_logging_obj: Final[LiteLLMLoggingObj | None] = kwargs.get("litellm_logging_obj", None)
+    if litellm_logging_obj:
+        litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = standard_logging_mcp_tool_call
+        litellm_logging_obj.model = f"MCP: {name}"
+        litellm_logging_obj.model_call_details["model"] = f"MCP: {name}"
+    # Resolve the MCP server early so BYOK checks and credential injection
+    # apply to ALL dispatch paths (local tool registry AND managed MCP server).
+    if mcp_server is None:
+        mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(name)
+
+    if mcp_server:
+        standard_logging_mcp_tool_call["mcp_server_cost_info"] = (mcp_server.mcp_info or {}).get("mcp_server_cost_info")
+        if litellm_logging_obj:
+            litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = standard_logging_mcp_tool_call
+
+        # BYOK: retrieve the stored per-user credential.  A single DB call
+        # both checks existence and fetches the value, avoiding a double query.
+        if mcp_server.is_byok and not mcp_auth_header:
+            byok_cred: Final = await _get_byok_credential(mcp_server, user_api_key_auth)
+            if byok_cred is None:
+                raise HTTPException(
+                    status_code=401,
+                    detail={
+                        "error": "byok_auth_required",
+                        "server_id": mcp_server.server_id,
+                        "server_name": mcp_server.server_name or mcp_server.name,
+                        "message": (
+                            "No stored credential found for this BYOK server. "
+                            "Complete the OAuth authorization flow to provide your API key."
+                        ),
+                    },
+                    headers={"WWW-Authenticate": get_byok_www_authenticate()},
+                )
+            mcp_auth_header = byok_cred
+        elif mcp_server.is_byok:
+            # External auth header supplied; still enforce user-identity check.
+            await _check_byok_credential(mcp_server, user_api_key_auth)
+
+    # Check if tool exists in local registry first (for OpenAPI-based tools)
+    # These tools are registered with their prefixed names
+    #########################################################
+    local_tool: Final = global_mcp_tool_registry.get_tool(name)
+    if local_tool:
+        # OpenAPI-backed tools used to bypass `pre_call_tool_check` —
+        # only the managed path ran allowed/banned-tool checks, key/team
+        # tool permissions, and parameter validation. Run the same checks
+        # before dispatching to the local registry. Refuse the call if
+        # we cannot resolve a server: tools registered via
+        # openapi_to_mcp_generator are always tied to a server, so a
+        # missing mcp_server here means the tool->server mapping has
+        # not finished initializing or the registry entry is orphaned.
+        # Skipping the check would re-open the same authorization gap.
+        if mcp_server is None:
+            raise HTTPException(
+                status_code=503,
+                detail=(
+                    f"MCP server for tool '{name}' is not available; "
+                    "refusing to dispatch without authorization checks. "
+                    "Retry once the server is registered."
+                ),
+            )
+
+        # `pre_call_tool_check` calls into `proxy_logging_obj` for the
+        # pre-call guardrail hooks, so source it from the canonical
+        # `proxy_server` module the same way `_handle_managed_mcp_tool`
+        # does. `kwargs.get("proxy_logging_obj")` is None on the MCP
+        # entry path and would crash with AttributeError after the
+        # security checks pass.
+        from litellm.proxy.proxy_server import proxy_logging_obj
+
+        hook_result = await global_mcp_server_manager.pre_call_tool_check(
+            name=original_tool_name,
+            arguments=arguments or {},
+            server_name=server_name or mcp_server.name,
+            user_api_key_auth=user_api_key_auth,
+            proxy_logging_obj=proxy_logging_obj,
+            server=mcp_server,
+            raw_headers=raw_headers,
+            litellm_logging_obj=litellm_logging_obj,
+            guardrail_context=guardrail_context,
+        )
+        # `pre_call_tool_check` may return guardrail-modified
+        # arguments; honor them on the local path too.
+        if isinstance(hook_result, dict) and "arguments" in hook_result:
+            arguments = hook_result["arguments"]
+
+        verbose_logger.debug("Executing local registry tool: %s", name)
+        # The credential rides ContextVars because the tool function has its
+        # headers baked into the closure at registration time.
+        auth_header_value, openapi_forwarded_headers, upstream_credential = _resolve_openapi_tool_auth(
+            mcp_server=mcp_server,
+            mcp_auth_header=mcp_auth_header,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            raw_headers=raw_headers,
+            user_api_key_auth=user_api_key_auth,
+        )
+        (
+            resolved_auth_headers,
+            forwarded_headers,
+        ) = await global_mcp_server_manager.resolve_openapi_upstream_auth(
+            mcp_server=mcp_server,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            mcp_auth_header=upstream_credential,
+            user_api_key_auth=user_api_key_auth,
+            forwarded_headers=openapi_forwarded_headers,
+        )
+
+        _auth_token: Final = _request_auth_header.set(auth_header_value)
+        _extra_token: Final = _request_extra_headers.set(forwarded_headers)
+        _resolved_token: Final = _request_resolved_auth_headers.set(resolved_auth_headers)
+        try:
+            response = await _handle_local_mcp_tool(name, arguments)
+        finally:
+            _request_auth_header.reset(_auth_token)
+            _request_extra_headers.reset(_extra_token)
+            _request_resolved_auth_headers.reset(_resolved_token)
+
+    # Try managed MCP server tool (the name is bare; the prefix boundary was
+    # already resolved above against this server's registered prefixes)
+    # Primary and recommended way to use external MCP servers
+    #########################################################
+    elif mcp_server:
+        response = await _handle_managed_mcp_tool(
+            server_name=server_name,
+            name=original_tool_name,
+            arguments=arguments,
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=mcp_auth_header,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            client_ip=client_ip,
+            litellm_logging_obj=litellm_logging_obj,
+            guardrail_context=guardrail_context,
+            host_progress_callback=host_progress_callback,
+        )
+
+    # Fall back to local tool registry with original name (legacy support)
+    #########################################################
+    # Deprecated: Local MCP Server Tool
+    #########################################################
+    else:
+        # Gate only what can actually dispatch. When the unprefixed name is
+        # not in the registry either, `_handle_local_mcp_tool` below reports
+        # 404 and nothing runs, so demanding a server here would turn every
+        # unknown tool name into a misleading 503.
+        if global_mcp_tool_registry.get_tool(original_tool_name) is not None:
+            # `mcp_server` is None here because the tool name is not in the
+            # tool -> server mapping, but the name still carries a prefix
+            # that the server-level check above compared against the
+            # caller's `allowed_mcp_servers` by exact `name`. So the named
+            # server is in that list and can carry the tool-level checks,
+            # even with the mapping cold. Resolve it from
+            # `allowed_mcp_servers` rather than the registry: the registry
+            # would happily return a server the caller holds no grant for,
+            # and matching anything other than `name` would accept a server
+            # the check never validated.
+            prefix_server: Final = next(
+                (candidate for candidate in allowed_mcp_servers if candidate.name == server_name),
+                None,
+            )
+            if prefix_server is None:
+                # A non-empty prefix that passed the server-level check
+                # always matches here, so this arm only fires when the
+                # prefix was empty, which is exactly the case that check
+                # skips. Fail closed rather than dispatch with no server to
+                # evaluate a tool ceiling against.
+                raise HTTPException(
+                    status_code=503,
+                    detail=(
+                        f"MCP server for tool '{original_tool_name}' is not available; "
+                        "refusing to dispatch without authorization checks. "
+                        "Retry once the server is registered."
+                    ),
+                )
+
+            from litellm.proxy.proxy_server import proxy_logging_obj
+
+            hook_result = await global_mcp_server_manager.pre_call_tool_check(
+                name=original_tool_name,
+                arguments=arguments,
+                server_name=server_name,
+                user_api_key_auth=user_api_key_auth,
+                proxy_logging_obj=proxy_logging_obj,
+                server=prefix_server,
+                raw_headers=raw_headers,
+                litellm_logging_obj=litellm_logging_obj,
+                guardrail_context=guardrail_context,
+            )
+            if "arguments" in hook_result:
+                arguments = hook_result["arguments"]  # pyright: ignore[reportAny]  # hook returns untyped args
+
+        response = await _handle_local_mcp_tool(original_tool_name, arguments)
+
+    return await _run_post_mcp_call_guardrails(
+        result=response,
+        litellm_logging_obj=litellm_logging_obj,
+        user_api_key_auth=user_api_key_auth,
+        request_data=kwargs,
+    )
+
+
+async def _run_post_mcp_call_guardrails(
+    result: CallToolResult,
+    litellm_logging_obj: LiteLLMLoggingObj | None,
+    user_api_key_auth: UserAPIKeyAuth | None,
+    request_data: Mapping[str, object],
+) -> CallToolResult:
+    """Run ``post_mcp_call`` guardrails over an executed tool result.
+
+    Lives on ``execute_mcp_tool``'s return path rather than inside
+    ``_fire_mcp_tool_call_logging`` so enforcement never depends on logging
+    being configured, and so every dispatch route gets it: the MCP protocol
+    handler, the REST endpoint, and tool search all funnel through here.
+    A guardrail that rejects the result raises, matching ``pre_mcp_call``.
+    """
+    from litellm.proxy.proxy_server import proxy_logging_obj
+
+    if proxy_logging_obj is None:
+        return result
+    return await proxy_logging_obj.post_mcp_call_hook(
+        response=result,
+        request_data=(
+            litellm_logging_obj.model_call_details if litellm_logging_obj is not None else dict(request_data)
+        ),
+        user_api_key_dict=user_api_key_auth,
+    )
+
+
+async def _fire_mcp_tool_call_logging(
+    logging_obj: LiteLLMLoggingObj,
+    result: CallToolResult,
+    start_time: datetime,
+    end_time: datetime,
+    user_api_key_auth: UserAPIKeyAuth | None = None,
+    request_data: Mapping[str, object] | None = None,
+) -> CallToolResult:
+    """Fire post-call logging for an executed MCP tool call, returning the result to send.
+
+    The returned result is what the caller must forward to the client: a
+    ``post_mcp_call`` guardrail may rewrite the tool output (e.g. mask
+    sensitive values) or reject it, in which case its exception propagates.
+    Guardrails run before the success/failure logging so the masked text, not
+    the raw one, is what gets logged.
+
+    A result with ``is_error=True`` is logged as a failure (``status="failure"``
+    payload, so OTel marks the span ERROR) while the HTTP wire behavior stays
+    200 + ``isError: true`` per the MCP spec. The error check runs after
+    ``async_post_mcp_tool_call_hook`` because guardrails may flip the result
+    to ``is_error=True`` in that hook. Raised exceptions never reach here (the
+    ``@client`` wrapper and ``call_mcp_tool``'s except path log those), so
+    this cannot double-log a failure.
+
+    ``request_data`` may carry credential-bearing fields (the REST path puts
+    ``raw_headers``, ``mcp_auth_header``, ``mcp_server_auth_headers``, and
+    ``oauth2_headers`` at the top level of its data dict), so those are
+    stripped before the dict is handed to ``post_call_failure_hook``
+    callbacks.
+    """
+    from litellm.proxy.proxy_server import proxy_logging_obj
+
+    logging_obj.post_call(original_response=result)
+    await logging_obj.async_post_mcp_tool_call_hook(
+        kwargs=logging_obj.model_call_details,
+        response_obj=result,
+        start_time=start_time,
+        end_time=end_time,
+    )
+    logging_obj.call_type = CallTypes.call_mcp_tool.value
+    error_message: Final = extract_mcp_tool_result_error_message(result)
+    if error_message is None:
+        await logging_obj.async_success_handler(result=result, start_time=start_time, end_time=end_time)
+        return result
+
+    logging_obj.has_run_logging(event_type="sync_success")
+    logging_obj.has_run_logging(event_type="async_success")
+    tool_error: Final = MCPToolResultError(error_message)
+    logging_obj.failure_handler(tool_error, "", start_time, end_time)
+    await logging_obj.async_failure_handler(tool_error, "", start_time, end_time)
+
+    if user_api_key_auth is None:
+        return result
+
+    if proxy_logging_obj:
+        sanitized_request_data: Final = {
+            key: value for key, value in (request_data or {}).items() if key not in _MCP_CREDENTIAL_REQUEST_FIELDS
+        }
+        await proxy_logging_obj.post_call_failure_hook(
+            request_data=sanitized_request_data,
+            original_exception=tool_error,
+            user_api_key_dict=user_api_key_auth,
+            route="/mcp/call_tool",
+        )
+    return result
+
+
+async def fire_mcp_tool_call_failure_logging(
+    logging_obj: LiteLLMLoggingObj | None,
+    exception: Exception,
+    start_time: datetime,
+    user_api_key_auth: UserAPIKeyAuth | None,
+    request_data: Mapping[str, object],
+) -> None:
+    """Failure logging shared by the ``/mcp`` path and the REST endpoint. Call from
+    inside the ``except`` block so the traceback is still available.
+
+    The failure handlers run first because ``_ProxyDBLogger.async_post_call_failure_hook``
+    builds the failure spend-log row from the ``standard_logging_object`` they produce;
+    both gate on ``should_run_logging``, so the ``@client`` wrapper does not log twice.
+    A relayed upstream 401 (``MCPUpstreamAuthError``) is an expected caller-must-reauth
+    signal and skips ``post_call_failure_hook``, which fires the ``llm_exceptions`` alert.
+    """
+    from litellm.proxy.proxy_server import proxy_logging_obj
+
+    traceback_str: Final = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG)
+    if logging_obj is not None:
+        end_time: Final = datetime.now()  # noqa: DTZ005  # naive to match `start_time`, which it is subtracted from
+        logging_obj.failure_handler(exception, traceback_str, start_time, end_time)
+        await logging_obj.async_failure_handler(exception, traceback_str, start_time, end_time)
+
+    if isinstance(exception, MCPUpstreamAuthError) or not proxy_logging_obj or user_api_key_auth is None:
+        return
+    sanitized_request_data: Final = {
+        key: value for key, value in request_data.items() if key not in _MCP_CREDENTIAL_REQUEST_FIELDS
+    }
+    await proxy_logging_obj.post_call_failure_hook(
+        request_data=sanitized_request_data,
+        original_exception=exception,
+        user_api_key_dict=user_api_key_auth,
+        route="/mcp/call_tool",
+        traceback_str=traceback_str,
+    )
+
+
+@client
+async def call_mcp_tool(
+    name: str,
+    arguments: dict[str, object] | None = None,
+    user_api_key_auth: UserAPIKeyAuth | None = None,
+    mcp_auth_header: str | None = None,
+    mcp_servers: list[str] | None = None,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
+    oauth2_headers: dict[str, str] | None = None,
+    raw_headers: dict[str, str] | None = None,
+    client_ip: str | None = None,
+    **kwargs: Any,
+) -> CallToolResult:
+    """
+    Call a specific tool with the provided arguments (handles prefixed tool names).
+    """
+    start_time: Final = datetime.now()
+    litellm_logging_obj: Final[LiteLLMLoggingObj | None] = kwargs.get("litellm_logging_obj", None)
+
+    try:
+        if arguments is None:
+            raise HTTPException(status_code=400, detail="Request arguments are required")
+
+        ## CHECK IF USER IS ALLOWED TO CALL THIS TOOL
+        allowed_mcp_server_ids: Final = await global_mcp_server_manager.get_allowed_mcp_servers(
+            user_api_key_auth=user_api_key_auth,
+        )
+
+        allowed_mcp_servers: list[MCPServer] = []
+        for allowed_mcp_server_id in allowed_mcp_server_ids:
+            allowed_server = global_mcp_server_manager.get_mcp_server_by_id(allowed_mcp_server_id)
+            if allowed_server is not None:
+                # Same request-time oauth2_flow backstop the listing path applies,
+                # so a null-flow M2M-shape row is treated as M2M on tool calls too.
+                allowed_server = MCPServerManager.resolve_oauth2_flow_for_request(allowed_server)
+                allowed_mcp_servers.append(allowed_server)
+
+        allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names(
+            mcp_servers=mcp_servers,
+            allowed_mcp_servers=allowed_mcp_servers,
+        )
+        if mcp_servers and not allowed_mcp_servers:
+            await raise_denied_scoped_mcp_access(
+                requested_names=mcp_servers,
+                user_api_key_auth=user_api_key_auth,
+                client_ip=client_ip,
+            )
+        if not allowed_mcp_servers:
+            raise HTTPException(
+                status_code=403,
+                detail="User not allowed to call this tool.",
+            )
+
+        # Delegate to execute_mcp_tool for execution
+        response = await execute_mcp_tool(
+            name=name,
+            arguments=arguments,
+            allowed_mcp_servers=allowed_mcp_servers,
+            start_time=start_time,
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=mcp_auth_header,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            client_ip=client_ip,
+            **kwargs,
+        )
+    except Exception as e:
+        await fire_mcp_tool_call_failure_logging(litellm_logging_obj, e, start_time, user_api_key_auth, kwargs)
+        raise
+
+    if litellm_logging_obj:
+        response = await _fire_mcp_tool_call_logging(
+            logging_obj=litellm_logging_obj,
+            result=response,
+            start_time=start_time,
+            end_time=datetime.now(),
+            user_api_key_auth=user_api_key_auth,
+            request_data=kwargs,
+        )
+    return response
+
+
+async def mcp_get_prompt(
+    name: str,
+    arguments: dict[str, str] | None = None,
+    user_api_key_auth: UserAPIKeyAuth | None = None,
+    mcp_auth_header: str | None = None,
+    mcp_servers: list[str] | None = None,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
+    oauth2_headers: dict[str, str] | None = None,
+    raw_headers: dict[str, str] | None = None,
+    client_ip: str | None = None,
+) -> GetPromptResult:
+    """
+    Fetch a specific MCP prompt, handling both prefixed and unprefixed names.
+    """
+    allowed_mcp_servers: Final = await _get_allowed_mcp_servers(
+        user_api_key_auth=user_api_key_auth,
+        mcp_servers=mcp_servers,
+        client_ip=client_ip,
+    )
+
+    if not allowed_mcp_servers:
+        raise HTTPException(
+            status_code=403,
+            detail="User not allowed to get this prompt.",
+        )
+
+    # Extract server name from prefixed prompt name
+    original_prompt_name, server_name = split_server_prefix_from_name(name)
+
+    server: Final = next((s for s in allowed_mcp_servers if s.name == server_name), None)
+    if server is None:
+        raise HTTPException(
+            status_code=403,
+            detail="User not allowed to get this prompt.",
+        )
+
+    server_auth_header, extra_headers = _prepare_mcp_server_headers(
+        server=server,
+        mcp_server_auth_headers=mcp_server_auth_headers,
+        mcp_auth_header=mcp_auth_header,
+        oauth2_headers=oauth2_headers,
+        raw_headers=raw_headers,
+        user_api_key_auth=user_api_key_auth,
+    )
+
+    return await global_mcp_server_manager.get_prompt_from_server(
+        server=server,
+        user_api_key_auth=user_api_key_auth,
+        prompt_name=original_prompt_name,
+        arguments=arguments,
+        mcp_auth_header=server_auth_header,
+        extra_headers=extra_headers,
+        raw_headers=raw_headers,
+        client_ip=client_ip,
+    )
+
+
+async def mcp_read_resource(
+    url: AnyUrl,
+    user_api_key_auth: UserAPIKeyAuth | None = None,
+    mcp_auth_header: str | None = None,
+    mcp_servers: list[str] | None = None,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
+    oauth2_headers: dict[str, str] | None = None,
+    raw_headers: dict[str, str] | None = None,
+    client_ip: str | None = None,
+) -> ReadResourceResult:
+    """Read resource contents from upstream MCP servers."""
+
+    allowed_mcp_servers: Final = await _get_allowed_mcp_servers(
+        user_api_key_auth=user_api_key_auth,
+        mcp_servers=mcp_servers,
+        client_ip=client_ip,
+    )
+
+    if not allowed_mcp_servers:
+        raise HTTPException(
+            status_code=403,
+            detail="User not allowed to read this resource.",
+        )
+
+    if len(allowed_mcp_servers) != 1:
+        raise HTTPException(
+            status_code=400,
+            detail=("Multiple MCP servers configured; read_resource currently supports exactly one allowed server."),
+        )
+
+    server: Final = allowed_mcp_servers[0]
+
+    server_auth_header, extra_headers = _prepare_mcp_server_headers(
+        server=server,
+        mcp_server_auth_headers=mcp_server_auth_headers,
+        mcp_auth_header=mcp_auth_header,
+        oauth2_headers=oauth2_headers,
+        raw_headers=raw_headers,
+        user_api_key_auth=user_api_key_auth,
+    )
+
+    return await global_mcp_server_manager.read_resource_from_server(
+        server=server,
+        user_api_key_auth=user_api_key_auth,
+        url=url,
+        mcp_auth_header=server_auth_header,
+        extra_headers=extra_headers,
+        raw_headers=raw_headers,
+        client_ip=client_ip,
+    )
+
+
+def _get_standard_logging_mcp_tool_call(
+    name: str,
+    arguments: dict[str, object],
+    server_name: str | None,
+    session_id: str | None = None,
+) -> StandardLoggingMCPToolCall:
+    mcp_server: Final = global_mcp_server_manager._get_mcp_server_from_tool_name(
+        add_server_prefix_to_name(name, server_name) if server_name else name
+    )
+    namespaced_tool_name: Final = f"{server_name}/{name}" if server_name else name
+    if mcp_server:
+        mcp_info: Final = mcp_server.mcp_info or {}
+        return StandardLoggingMCPToolCall(
+            name=name,
+            arguments=arguments,
+            mcp_server_name=mcp_info.get("server_name"),
+            mcp_server_logo_url=mcp_info.get("logo_url"),
+            namespaced_tool_name=namespaced_tool_name,
+            mcp_session_id=session_id,
+            mcp_auth_mode=mcp_server.auth_type,
+            mcp_server_resource=_redact_mcp_resource_url(mcp_server.url),
+        )
+    else:
+        return StandardLoggingMCPToolCall(
+            name=name,
+            arguments=arguments,
+            namespaced_tool_name=namespaced_tool_name,
+            mcp_session_id=session_id,
+        )
+
+
+async def _handle_managed_mcp_tool(
+    server_name: str,
+    name: str,
+    arguments: dict[str, object],
+    user_api_key_auth: UserAPIKeyAuth | None = None,
+    mcp_auth_header: str | None = None,
+    mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
+    oauth2_headers: dict[str, str] | None = None,
+    raw_headers: dict[str, str] | None = None,
+    litellm_logging_obj: LiteLLMLoggingObj | None = None,
+    host_progress_callback: ProgressCallback | None = None,
+    guardrail_context: Mapping[str, object] | None = None,
+    client_ip: str | None = None,
+) -> CallToolResult:
+    """Handle tool execution for managed server tools"""
+    # Import here to avoid circular import
+    from litellm.proxy.proxy_server import proxy_logging_obj
+
+    call_tool_result: Final = await global_mcp_server_manager.call_tool(
+        server_name=server_name,
+        name=name,
+        arguments=arguments,
+        user_api_key_auth=user_api_key_auth,
+        mcp_auth_header=mcp_auth_header,
+        mcp_server_auth_headers=mcp_server_auth_headers,
+        oauth2_headers=oauth2_headers,
+        raw_headers=raw_headers,
+        client_ip=client_ip,
+        proxy_logging_obj=proxy_logging_obj,
+        host_progress_callback=host_progress_callback,
+        litellm_logging_obj=litellm_logging_obj,
+        guardrail_context=guardrail_context,
+    )
+    verbose_logger.debug("CALL TOOL RESULT: %s", call_tool_result)
+    return call_tool_result
+
+
+async def _handle_local_mcp_tool(name: str, arguments: dict[str, object]) -> CallToolResult:
+    """Execute a local-registry tool and report whether it succeeded.
+
+    Returns the result rather than bare content because the verdict is part of it: the content
+    alone cannot say whether the handler failed, so callers used to stamp is_error=False on every
+    outcome and an upstream rejection was served as tool output.
+
+    A failure is reported as ``is_error=True`` here rather than raised, because the REST surface
+    turns an unrecognized exception into a 500 and an upstream 403 or 429 is not a gateway crash.
+    ``MCPUpstreamAuthError`` is the exception: it propagates so the caller is told to
+    re-authenticate, which both renderers already know how to say.
+
+    Note: Local tools don't use prefixes, so we use the original name
+    """
+    import inspect
+
+    tool: Final = global_mcp_tool_registry.get_tool(name)
+    if not tool:
+        raise HTTPException(status_code=404, detail=f"Tool '{name}' not found")
+
+    try:
+        if inspect.iscoroutinefunction(tool.handler):
+            result = await tool.handler(**arguments)
+        else:
+            result = tool.handler(**arguments)
+    except MCPUpstreamAuthError:
+        raise
+    except Exception as e:
+        verbose_logger.exception("Error executing local tool %s: %s", name, e)
+        return CallToolResult(
+            content=[TextContent(text=f"Error: {e}", type="text")],  # mutable-ok: MCP result content
+            is_error=True,
+        )
+    return CallToolResult(
+        content=[TextContent(text=str(result), type="text")],  # mutable-ok: MCP result content
+        is_error=False,
+    )
+
+
+_MCP_CREDENTIAL_REQUEST_FIELDS: Final = frozenset(
+    {
+        "raw_headers",
+        "mcp_auth_header",
+        "mcp_server_auth_headers",
+        "oauth2_headers",
+        "user_api_key_auth",
+    }
+)
+
+
+class _McpDeniedDetail(TypedDict):
+    error: ReadOnly[str]
+
+
+async def _execute_handle_list_tools(
+    context: OperationContext, params: PaginatedRequestParams, host_progress_callback: ProgressCallback | None = None
+) -> ListToolsResult:
+    try:
+        (
+            user_api_key_auth,
+            mcp_auth_header,
+            mcp_servers,
+            mcp_server_auth_headers,
+            oauth2_headers,
+            raw_headers,
+            _client_ip,
+        ) = context.legacy_auth()
+        verbose_logger.debug("MCP list_tools - User API Key Auth from context: %s", user_api_key_auth)
+        verbose_logger.debug("MCP list_tools - MCP servers from context: %s", mcp_servers)
+        verbose_logger.debug(
+            "MCP list_tools - MCP server auth headers: %s",
+            list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None,
+        )
+        from mcp.types import Tool
+
+        from litellm.proxy._experimental.mcp_server.tool_search import (
+            get_mcp_proxy_tool_definitions,
+            get_virtual_tool_definitions,
+        )
+
+        if context.mcp_proxy_mode:
+            return ListToolsResult(tools=[Tool.model_validate(d) for d in get_mcp_proxy_tool_definitions()])
+        if getattr(
+            getattr(user_api_key_auth, "object_permission", None),
+            "mcp_tool_search_enabled",
+            False,
+        ):
+            return ListToolsResult(tools=[Tool.model_validate(d) for d in get_virtual_tool_definitions()])
+
+        # Get mcp_servers from context variable
+        verbose_logger.debug("MCP list_tools - Calling _list_mcp_tools")
+        listing: Final = await _list_mcp_tools(
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=mcp_auth_header,
+            mcp_servers=mcp_servers,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            log_list_tools_to_spendlogs=True,
+            list_tools_log_source="mcp_protocol",
+            client_ip=_client_ip,
+        )
+        verbose_logger.info("MCP list_tools - Successfully returned %s tools", len(listing.tools))
+        if not listing.outcomes:
+            return ListToolsResult(tools=listing.tools)
+        outcome_meta: Final = {
+            SERVER_OUTCOMES_META_KEY: {key: outcome_wire_value(outcome) for key, outcome in listing.outcomes.items()}
+        }
+        return ListToolsResult.model_validate({"tools": listing.tools, "_meta": outcome_meta})
+    except HTTPException as e:
+        from mcp.shared.exceptions import MCPError
+        from mcp.types import INVALID_REQUEST
+
+        raise MCPError(code=INVALID_REQUEST, message=_http_detail_message(e.detail)) from e
+    except Exception as e:
+        verbose_logger.exception("Error in list_tools endpoint: %s", e)
+        # Return empty list instead of failing completely
+        # This prevents the HTTP stream from failing and allows the client to get a response
+        return ListToolsResult(tools=[])  # mutable-ok: MCP result payload
+
+
+async def _execute_mcp_server_tool_call(
+    context: OperationContext, params: CallToolRequestParams, host_progress_callback: ProgressCallback | None = None
+) -> CallToolResult:
+    from mcp.types import CallToolResult
+
+    from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
+    from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
+    from litellm.proxy.proxy_server import proxy_config
+
+    (
+        user_api_key_auth,
+        mcp_auth_header,
+        mcp_servers,
+        mcp_server_auth_headers,
+        oauth2_headers,
+        raw_headers,
+        _client_ip,
+    ) = context.legacy_auth()
+    verbose_logger.debug(
+        "MCP mcp_server_tool_call - user_api_key_auth=%s, user_role=%s",
+        user_api_key_auth,
+        getattr(user_api_key_auth, "user_role", "N/A"),
+    )
+
+    verbose_logger.debug("MCP mcp_server_tool_call - User API Key Auth from context: %s", user_api_key_auth)
+
+    try:
+        # Inside this try so virtual-tool errors convert to isError
+        # CallToolResult instead of raising out of the protocol handler.
+        virtual_tool_result: Final = await _dispatch_virtual_mcp_tool(
+            name=params.name,
+            arguments=params.arguments,
+            user_api_key_auth=user_api_key_auth,
+            client_ip=_client_ip,
+            mcp_servers=mcp_servers,
+            mcp_auth_header=mcp_auth_header,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            mcp_proxy_mode=context.mcp_proxy_mode,
+        )
+        if virtual_tool_result is not None:
+            return virtual_tool_result
+
+        # Create a body date for logging
+        body_data: Final = {"name": params.name, "arguments": params.arguments}  # mutable-ok: logging payload
+        # Set trace/session id from raw_headers so spend logs and logging_obj stay consistent (same as A2A)
+        chain_id: Final = get_chain_id_from_headers(raw_headers)
+        if chain_id:
+            body_data["litellm_trace_id"] = chain_id
+            body_data["litellm_session_id"] = chain_id
+
+        request: Final = build_synthetic_mcp_request(
+            path="/mcp/tools/call",
+            raw_headers=raw_headers,
+            client_ip=_client_ip,
+        )
+        if user_api_key_auth is not None:
+            data = await add_litellm_data_to_request(
+                data=body_data,
+                request=request,
+                # Bill a team-derived call to the team that granted it. A keyless admitted
+                # subject carries no team_id, so spend skipped team updates entirely and
+                # charged the user's PRIMARY org — the granting team's budget never
+                # accumulated (so it could never begin to block) and, cross-org, the wrong
+                # organization was charged. This is the ACCOUNTING half; the enforcement
+                # half (an already-over-budget team stops granting) lives in the source gate.
+                # Authorization is unaffected: it ran before this, and the union is resolved
+                # from the untouched auth object passed to call_mcp_tool below.
+                user_api_key_dict=await MCPRequestHandler.billing_auth_for_tool_call(
+                    user_api_key_auth, tool_name=params.name
+                ),
+                proxy_config=proxy_config,
+            )
+        else:
+            data = body_data
+
+        response: Final = await call_mcp_tool(
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=mcp_auth_header,
+            mcp_servers=mcp_servers,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            client_ip=_client_ip,
+            host_progress_callback=host_progress_callback,
+            **data,  # for logging
+        )
+    except MCPMissingUserEnvVarsError as e:
+        verbose_logger.info(
+            "MCP mcp_server_tool_call missing per-user env vars: server_id=%s missing=%s",
+            e.server_id,
+            e.missing,
+        )
+        return CallToolResult(
+            content=[TextContent(text=str(e), type="text")],
+            is_error=True,
+        )
+    except BlockedPiiEntityError as e:
+        verbose_logger.error("BlockedPiiEntityError in MCP tool call: %s", e)
+        return CallToolResult(
+            content=[
+                TextContent(
+                    text=f"Error: Blocked PII entity detected - {e}",
+                    type="text",
+                )
+            ],
+            is_error=True,
+        )
+    except GuardrailRaisedException as e:
+        verbose_logger.error("GuardrailRaisedException in MCP tool call: %s", e)
+        return CallToolResult(
+            content=[TextContent(text=f"Error: Guardrail violation - {e}", type="text")],
+            is_error=True,
+        )
+    except HTTPException as e:
+        verbose_logger.error("HTTPException in MCP tool call: %s", e)
+        return CallToolResult(
+            content=[TextContent(text=f"Error: {_http_detail_message(e.detail)}", type="text")],
+            is_error=True,
+        )
+    except MCPUpstreamAuthError as e:
+        # The MCP session manager serializes handler exceptions as JSON-RPC errors, so a
+        # mid-session tool call cannot emit a raw 401 + WWW-Authenticate the way the REST
+        # call path and the connect-time preemptive check do. Return an explicit isError
+        # naming the upstream status (at info level, not a traceback) so the client still
+        # learns it must re-authenticate upstream and expected pass-through 401s don't spam.
+        verbose_logger.info("Upstream auth failure calling MCP tool: HTTP %s", e.status_code)
+        return CallToolResult(
+            content=[
+                TextContent(
+                    text=f"Error: upstream authentication required (HTTP {e.status_code})",
+                    type="text",
+                )
+            ],
+            is_error=True,
+        )
+    except Exception as e:
+        verbose_logger.exception("MCP mcp_server_tool_call - error: %s", e)
+        return CallToolResult(
+            content=[TextContent(text=f"Error: {e}", type="text")],
+            is_error=True,
+        )
+
+    return response
+
+
+async def _execute_list_prompts(
+    context: OperationContext, params: PaginatedRequestParams, host_progress_callback: ProgressCallback | None = None
+) -> ListPromptsResult:
+    if context.mcp_proxy_mode:
+        _reject_mcp_proxy_operation()
+    try:
+        (
+            user_api_key_auth,
+            mcp_auth_header,
+            mcp_servers,
+            mcp_server_auth_headers,
+            oauth2_headers,
+            raw_headers,
+            _client_ip,
+        ) = context.legacy_auth()
+        verbose_logger.debug("MCP list_prompts - User API Key Auth from context: %s", user_api_key_auth)
+        verbose_logger.debug("MCP list_prompts - MCP servers from context: %s", mcp_servers)
+        verbose_logger.debug(
+            "MCP list_prompts - MCP server auth headers: %s",
+            list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None,
+        )
+        # Get mcp_servers from context variable
+        verbose_logger.debug("MCP list_prompts - Calling _list_prompts")
+        prompts: Final = await _list_mcp_prompts(
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=mcp_auth_header,
+            mcp_servers=mcp_servers,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            client_ip=_client_ip,
+        )
+        verbose_logger.info("MCP list_prompts - Successfully returned %s prompts", len(prompts))
+        return ListPromptsResult(prompts=prompts)
+    except Exception as e:
+        verbose_logger.exception("Error in list_prompts endpoint: %s", e)
+        # Return empty list instead of failing completely
+        # This prevents the HTTP stream from failing and allows the client to get a response
+        return ListPromptsResult(prompts=[])  # mutable-ok: MCP result payload
+
+
+async def _execute_get_prompt(
+    context: OperationContext, params: GetPromptRequestParams, host_progress_callback: ProgressCallback | None = None
+) -> GetPromptResult:
+    if context.mcp_proxy_mode:
+        _reject_mcp_proxy_operation()
+    (
+        user_api_key_auth,
+        mcp_auth_header,
+        mcp_servers,
+        mcp_server_auth_headers,
+        oauth2_headers,
+        raw_headers,
+        _client_ip,
+    ) = context.legacy_auth()
+
+    verbose_logger.debug("MCP mcp_server_tool_call - User API Key Auth from context: %s", user_api_key_auth)
+    return await mcp_get_prompt(
+        name=params.name,
+        arguments=params.arguments,
+        user_api_key_auth=user_api_key_auth,
+        mcp_auth_header=mcp_auth_header,
+        mcp_servers=mcp_servers,
+        mcp_server_auth_headers=mcp_server_auth_headers,
+        oauth2_headers=oauth2_headers,
+        raw_headers=raw_headers,
+        client_ip=_client_ip,
+    )
+
+
+async def _execute_list_resources(
+    context: OperationContext, params: PaginatedRequestParams, host_progress_callback: ProgressCallback | None = None
+) -> ListResourcesResult:
+    if context.mcp_proxy_mode:
+        _reject_mcp_proxy_operation()
+    try:
+        (
+            user_api_key_auth,
+            mcp_auth_header,
+            mcp_servers,
+            mcp_server_auth_headers,
+            oauth2_headers,
+            raw_headers,
+            _client_ip,
+        ) = context.legacy_auth()
+        verbose_logger.debug("MCP list_resources - User API Key Auth from context: %s", user_api_key_auth)
+        verbose_logger.debug("MCP list_resources - MCP servers from context: %s", mcp_servers)
+        verbose_logger.debug(
+            "MCP list_resources - MCP server auth headers: %s",
+            list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None,
+        )
+
+        resources: Final = await _list_mcp_resources(
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=mcp_auth_header,
+            mcp_servers=mcp_servers,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            client_ip=_client_ip,
+        )
+        verbose_logger.info("MCP list_resources - Successfully returned %s resources", len(resources))
+        return ListResourcesResult(resources=resources)
+    except Exception as e:
+        verbose_logger.exception("Error in list_resources endpoint: %s", e)
+        return ListResourcesResult(resources=[])  # mutable-ok: MCP result payload
+
+
+async def _execute_list_resource_templates(
+    context: OperationContext, params: PaginatedRequestParams, host_progress_callback: ProgressCallback | None = None
+) -> ListResourceTemplatesResult:
+    if context.mcp_proxy_mode:
+        _reject_mcp_proxy_operation()
+    try:
+        (
+            user_api_key_auth,
+            mcp_auth_header,
+            mcp_servers,
+            mcp_server_auth_headers,
+            oauth2_headers,
+            raw_headers,
+            _client_ip,
+        ) = context.legacy_auth()
+        verbose_logger.debug("MCP list_resource_templates - User API Key Auth from context: %s", user_api_key_auth)
+        verbose_logger.debug("MCP list_resource_templates - MCP servers from context: %s", mcp_servers)
+        verbose_logger.debug(
+            "MCP list_resource_templates - MCP server auth headers: %s",
+            list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None,
+        )
+
+        resource_templates: Final = await _list_mcp_resource_templates(
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=mcp_auth_header,
+            mcp_servers=mcp_servers,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            client_ip=_client_ip,
+        )
+        verbose_logger.info(
+            "MCP list_resource_templates - Successfully returned %s resource templates", len(resource_templates)
+        )
+        return ListResourceTemplatesResult(resource_templates=resource_templates)
+    except Exception as e:
+        verbose_logger.exception("Error in list_resource_templates endpoint: %s", e)
+        return ListResourceTemplatesResult(resource_templates=[])  # mutable-ok: MCP result payload
+
+
+async def _execute_read_resource(
+    context: OperationContext, params: ReadResourceRequestParams, host_progress_callback: ProgressCallback | None = None
+) -> ReadResourceResult:
+    if context.mcp_proxy_mode:
+        _reject_mcp_proxy_operation()
+    (
+        user_api_key_auth,
+        mcp_auth_header,
+        mcp_servers,
+        mcp_server_auth_headers,
+        oauth2_headers,
+        raw_headers,
+        _client_ip,
+    ) = context.legacy_auth()
+
+    read_resource_result: Final = await mcp_read_resource(
+        url=params.uri,
+        user_api_key_auth=user_api_key_auth,
+        mcp_auth_header=mcp_auth_header,
+        mcp_servers=mcp_servers,
+        mcp_server_auth_headers=mcp_server_auth_headers,
+        oauth2_headers=oauth2_headers,
+        raw_headers=raw_headers,
+        client_ip=_client_ip,
+    )
+
+    return read_resource_result
+
+
+def _reject_mcp_proxy_operation() -> NoReturn:
+    from mcp.shared.exceptions import MCPError
+    from mcp.types import METHOD_NOT_FOUND
+
+    raise MCPError(code=METHOD_NOT_FOUND, message="Operation unavailable on /mcp/proxy")
+
+
+def prepare_context(
+    user_api_key_auth: UserAPIKeyAuth | None = None,
+    mcp_auth_header: str | None = None,
+    mcp_servers: Sequence[str] | None = None,
+    mcp_server_auth_headers: Mapping[str, Mapping[str, str]] | None = None,
+    oauth2_headers: Mapping[str, str] | None = None,
+    raw_headers: Mapping[str, str] | None = None,
+    client_ip: str | None = None,
+    mcp_proxy_mode: bool = False,
+) -> OperationContext:
+    return OperationContext(
+        _caller=user_api_key_auth,
+        mcp_auth_header=mcp_auth_header,
+        mcp_servers=tuple(mcp_servers) if mcp_servers is not None else None,
+        mcp_server_auth_headers=mcp_server_auth_headers,
+        oauth2_headers=oauth2_headers,
+        raw_headers=raw_headers,
+        client_ip=client_ip,
+        mcp_proxy_mode=mcp_proxy_mode,
+    )
+
+
+GatewayOperation: TypeAlias = (
+    AuthorizedToolCall
+    | ListToolsRequest
+    | CallToolRequest
+    | ListPromptsRequest
+    | GetPromptRequest
+    | ListResourcesRequest
+    | ListResourceTemplatesRequest
+    | ReadResourceRequest
+)
+GatewayResult: TypeAlias = (
+    ListToolsResult
+    | CallToolResult
+    | ListPromptsResult
+    | GetPromptResult
+    | ListResourcesResult
+    | ListResourceTemplatesResult
+    | ReadResourceResult
+)
+
+
+class GatewayOperations:
+    def __init__(self, host_progress_callback: ProgressCallback | None = None) -> None:
+        self._host_progress_callback = host_progress_callback
+
+    @overload
+    async def execute(self, operation: AuthorizedToolCall, context: OperationContext) -> CallToolResult: ...
+
+    @overload
+    async def execute(self, operation: ListToolsRequest, context: OperationContext) -> ListToolsResult: ...
+
+    @overload
+    async def execute(self, operation: CallToolRequest, context: OperationContext) -> CallToolResult: ...
+
+    @overload
+    async def execute(self, operation: ListPromptsRequest, context: OperationContext) -> ListPromptsResult: ...
+
+    @overload
+    async def execute(self, operation: GetPromptRequest, context: OperationContext) -> GetPromptResult: ...
+
+    @overload
+    async def execute(self, operation: ListResourcesRequest, context: OperationContext) -> ListResourcesResult: ...
+
+    @overload
+    async def execute(
+        self, operation: ListResourceTemplatesRequest, context: OperationContext
+    ) -> ListResourceTemplatesResult: ...
+
+    @overload
+    async def execute(self, operation: ReadResourceRequest, context: OperationContext) -> ReadResourceResult: ...
+
+    async def execute(self, operation: GatewayOperation, context: OperationContext) -> GatewayResult:
+        match operation:
+            case AuthorizedToolCall():
+                auth, token, _servers, server_headers, oauth_headers, headers, _client_ip = context.legacy_auth()
+                return await _execute_mcp_tool(
+                    name=operation.name,
+                    arguments=dict(operation.arguments),  # mutable-ok: existing tool hooks own mutable argument data
+                    allowed_mcp_servers=list(
+                        operation.allowed_mcp_servers
+                    ),  # mutable-ok: legacy dispatch list contract
+                    start_time=operation.start_time,
+                    user_api_key_auth=auth,
+                    mcp_auth_header=token,
+                    mcp_server_auth_headers=server_headers,
+                    oauth2_headers=oauth_headers,
+                    raw_headers=headers,
+                    client_ip=_client_ip,
+                    host_progress_callback=operation.host_progress_callback,
+                    guardrail_context=operation.guardrail_context,
+                    **operation.logging_data,
+                )
+            case ListToolsRequest(params=params):
+                return await _execute_handle_list_tools(
+                    context, params or PaginatedRequestParams(), self._host_progress_callback
+                )
+            case CallToolRequest(params=params):
+                return await _execute_mcp_server_tool_call(context, params, self._host_progress_callback)
+            case ListPromptsRequest(params=params):
+                return await _execute_list_prompts(
+                    context, params or PaginatedRequestParams(), self._host_progress_callback
+                )
+            case GetPromptRequest(params=params):
+                return await _execute_get_prompt(context, params, self._host_progress_callback)
+            case ListResourcesRequest(params=params):
+                return await _execute_list_resources(
+                    context, params or PaginatedRequestParams(), self._host_progress_callback
+                )
+            case ListResourceTemplatesRequest(params=params):
+                return await _execute_list_resource_templates(
+                    context, params or PaginatedRequestParams(), self._host_progress_callback
+                )
+            case ReadResourceRequest(params=params):
+                return await _execute_read_resource(context, params, self._host_progress_callback)
+            case _:
+                assert_never(operation)
diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py
index 15f97a15b73..c2f7bf7d531 100644
--- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py
+++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py
@@ -203,17 +203,19 @@ if MCP_AVAILABLE:
     from litellm.proxy._experimental.mcp_server.oauth_utils import (
         get_request_base_url,
     )
-    from litellm.proxy._experimental.mcp_server.server import (
+    from litellm.proxy._experimental.mcp_server.operations import (
         ListMCPToolsRestAPIResponseObject,
         MCPInfo,
         MCPServer,
-        _aggregate_server_key,  # pyright: ignore[reportPrivateUsage]  # same per-server key as the tools/list _meta outcomes
-        _apply_toolset_scope,
+        _aggregate_server_key,
         _fire_mcp_tool_call_logging,
         execute_mcp_tool,
         filter_tools_by_allowed_tools,
         filter_tools_by_key_team_permissions,
         fire_mcp_tool_call_failure_logging,
+    )
+    from litellm.proxy._experimental.mcp_server.server import (
+        _apply_toolset_scope,
         reject_disallowed_mcp_client,
     )
 
@@ -670,6 +672,7 @@ if MCP_AVAILABLE:
         user_api_key_auth: UserAPIKeyAuth | None = None,
         extra_headers: dict[str, str] | None = None,
         apply_tool_filters: bool = True,
+        client_ip: str | None = None,
     ):
         """Helper function to get tools for a single server.
 
@@ -684,6 +687,7 @@ if MCP_AVAILABLE:
             extra_headers=extra_headers,
             add_prefix=False,
             raw_headers=raw_headers,
+            client_ip=client_ip,
             user_api_key_auth=user_api_key_auth,
         )
 
@@ -797,6 +801,7 @@ if MCP_AVAILABLE:
                 user_api_key_dict,
                 extra_headers=user_oauth_extra_headers,
                 apply_tool_filters=apply_tool_filters,
+                client_ip=rest_client_ip,
             )
         except MCPUpstreamAuthError:
             # Surface the upstream 401/403 to the caller so it can emit the
@@ -1016,6 +1021,7 @@ if MCP_AVAILABLE:
                             user_api_key_dict,
                             extra_headers=user_oauth_extra_headers,
                             apply_tool_filters=apply_tool_filters,
+                            client_ip=_rest_client_ip,
                         )
                     except Exception as e:
                         verbose_logger.warning(
@@ -1193,6 +1199,7 @@ if MCP_AVAILABLE:
                     mcp_server_auth_headers=data.get("mcp_server_auth_headers"),
                     oauth2_headers=user_oauth_extra_headers or data.get("oauth2_headers"),
                     raw_headers=data.get("raw_headers"),
+                    client_ip=IPAddressUtils.get_mcp_client_ip(request),
                     litellm_logging_obj=data.get("litellm_logging_obj"),
                     guardrail_context=MCPRequestContext.resolve_guardrail_context(data),
                     requested_server_id=canonical_server_id,
diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py
index 3a9bca926b0..33978bb9182 100644
--- a/litellm/proxy/_experimental/mcp_server/server.py
+++ b/litellm/proxy/_experimental/mcp_server/server.py
@@ -11,28 +11,22 @@ import hashlib
 import json
 import os
 import time
-import traceback
 import types
-import uuid
 from collections import Counter
-from collections.abc import AsyncIterator, Callable, Iterable, Mapping, Sequence
-from datetime import datetime
-from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol
+from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterable, Mapping, Sequence
+from typing import TYPE_CHECKING, Final, NoReturn, Protocol
 
 import httpx
 from fastapi import FastAPI, HTTPException
-from pydantic import AnyUrl, ConfigDict, Field, TypeAdapter, ValidationError
+from pydantic import ConfigDict, TypeAdapter, ValidationError
 from starlette.requests import Request as StarletteRequest
 from starlette.responses import JSONResponse
 from starlette.types import Message, Receive, Scope, Send
-from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_logger
 from litellm.constants import (
-    MAXIMUM_TRACEBACK_LINES_TO_LOG,
     MCP_GATEWAY_SESSION_ID_PREFIX_LENGTH,
 )
-from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
@@ -41,12 +35,6 @@ from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
     MCPRequestHandler,
     _is_mcp_admitted_user_subject,
 )
-from litellm.proxy._experimental.mcp_server.byok_credential_cache import (
-    byok_credential_cache,
-    byok_credential_cache_key,
-    cache_byok_credential,
-    get_cached_byok_credential,
-)
 from litellm.proxy._experimental.mcp_server.client_allowlist import (
     MCPClientAllowlist,
     check_mcp_client_allowed,
@@ -56,7 +44,6 @@ from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
     get_request_base_url,
 )
 from litellm.proxy._experimental.mcp_server.exceptions import (
-    MCPToolResultError,
     MCPUpstreamAuthError,
 )
 from litellm.proxy._experimental.mcp_server.mcp_context import (
@@ -74,7 +61,6 @@ from litellm.proxy._experimental.mcp_server.mcp_debug import (
 )
 from litellm.proxy._experimental.mcp_server.oauth_utils import (
     _redact_mcp_resource_url,
-    get_byok_www_authenticate,
     get_passthrough_www_authenticate,
     get_route_relative_request_path,
     well_known_root_suffix,
@@ -84,14 +70,6 @@ from litellm.proxy._experimental.mcp_server.utils import (
     LITELLM_MCP_SERVER_DESCRIPTION,
     LITELLM_MCP_SERVER_NAME,
     LITELLM_MCP_SERVER_VERSION,
-    MCPMissingUserEnvVarsError,
-    add_server_prefix_to_name,
-    build_synthetic_mcp_request,
-    extract_mcp_tool_result_error_message,
-    get_server_prefix,
-    iter_known_server_prefixes,
-    logging_safe_mcp_headers,
-    match_known_tool_name,
 )
 from litellm.proxy._types import (
     ProxyException,
@@ -99,13 +77,6 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
 )
 from litellm.proxy.auth.ip_address_utils import IPAddressUtils
-from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
-    publish_auth_cache_invalidation,
-)
-from litellm.proxy.litellm_pre_call_utils import (
-    LiteLLMProxyRequestSetup,
-    get_chain_id_from_headers,
-)
 from litellm.types.mcp import (
     MCPAuth,
     MCPGatewaySession,
@@ -114,14 +85,11 @@ from litellm.types.mcp import (
     MCPGatewaySessionsTerminateResponse,
     MCPSpecVersion,
 )
-from litellm.types.mcp_server.mcp_server_manager import MCPInfo, MCPServer
-from litellm.types.utils import CallTypes, StandardLoggingMCPToolCall
-from litellm.utils import Rules, client, function_setup
+from litellm.types.mcp_server.mcp_server_manager import MCPServer
 
 if TYPE_CHECKING:
     from mcp.server.session import ServerSession as _McpServerSession
 
-    from litellm.proxy._experimental.mcp_server.db import OAuthCredentialPayload
 
 _STATEFUL_SESSION_IDLE_TIMEOUT_SECONDS: Final = 30 * 60
 # Upper bound on concurrent stateful sessions a single caller may hold. Each
@@ -159,13 +127,6 @@ def unsupported_protocol_version(scope: Scope) -> str | None:
     return None
 
 
-async def _invalidate_byok_cred_cache(user_id: str, server_id: str) -> None:
-    """Drop a stored-or-deleted BYOK credential from this worker's cache and from every peer worker's."""
-    cache_key: Final = byok_credential_cache_key(user_id, server_id)
-    byok_credential_cache.delete_cache(cache_key)
-    await publish_auth_cache_invalidation(cache_key=cache_key)
-
-
 # Check if MCP is available
 # "mcp" requires python 3.10 or higher, but several litellm users use python 3.8
 # We're making this conditional import to avoid breaking users who use python 3.8.
@@ -210,19 +171,6 @@ _SESSION_MANAGERS_INITIALIZED = False
 _INITIALIZATION_LOCK: Final = asyncio.Lock()
 
 
-def _mcp_session_id_from_headers(
-    raw_headers: dict[str, str] | None,
-) -> str | None:
-    """The ``mcp-session-id`` of a stateful MCP session, read case-insensitively
-    from the request headers. ``None`` for stateless calls (no such header)."""
-    if not raw_headers:
-        return None
-    for key, value in raw_headers.items():
-        if isinstance(key, str) and key.lower() == "mcp-session-id":
-            return value or None
-    return None
-
-
 def _jsonrpc_text_has_top_level_method(text: str) -> bool:
     """Whether a (possibly truncated) JSON-RPC envelope has a ``method`` key at
     the root object's top level.
@@ -466,6 +414,56 @@ def _proxy_exception_to_http_exception(exc: ProxyException) -> HTTPException:
 
 
 if MCP_AVAILABLE:
+    __all__ = (
+        "_MCP_CREDENTIAL_REQUEST_FIELDS",
+        "ListMCPToolsRestAPIResponseObject",
+        "_McpDeniedDetail",
+        "_aggregate_server_key",
+        "_build_virtual_call_logging_obj",
+        "_check_byok_credential",
+        "_client_has_passthrough_authorization",
+        "_client_has_per_server_auth_header",
+        "_dispatch_virtual_mcp_tool",
+        "_fire_mcp_tool_call_logging",
+        "_get_allowed_mcp_servers",
+        "_get_allowed_mcp_servers_from_mcp_server_names",
+        "_get_byok_credential",
+        "_get_prompts_from_mcp_servers",
+        "_get_resource_templates_from_mcp_servers",
+        "_get_resources_from_mcp_servers",
+        "_get_standard_logging_mcp_tool_call",
+        "_get_tools_from_mcp_servers",
+        "_get_user_oauth_extra_headers_from_db",
+        "_handle_local_mcp_tool",
+        "_handle_managed_mcp_tool",
+        "_http_detail_message",
+        "_invalidate_byok_cred_cache",
+        "_list_mcp_prompts",
+        "_list_mcp_resource_templates",
+        "_list_mcp_resources",
+        "_list_mcp_tools",
+        "_list_tools_before_first_call",
+        "_mcp_session_id_from_headers",
+        "_merge_gateway_initialize_instructions",
+        "_prefetch_oauth_creds_for_user",
+        "_prepare_mcp_server_headers",
+        "_raise_if_initialize_grants_no_mcp_servers",
+        "_redact_mcp_resource_url",
+        "_resolve_display_name_to_original",
+        "_run_post_mcp_call_guardrails",
+        "_server_answers_to",
+        "_tool_name_matches",
+        "apply_tool_overrides",
+        "call_mcp_tool",
+        "execute_mcp_tool",
+        "filter_tools_by_allowed_tools",
+        "filter_tools_by_key_team_permissions",
+        "fire_mcp_tool_call_failure_logging",
+        "global_mcp_server_manager",
+        "mcp_get_prompt",
+        "mcp_read_resource",
+        "raise_denied_scoped_mcp_access",
+    )
     from mcp.server import Server
 
     # Import auth context variables and middleware
@@ -476,6 +474,23 @@ if MCP_AVAILABLE:
     from mcp.server.context import ServerRequestContext
     from mcp.server.lowlevel.server import NotificationOptions
     from mcp.server.models import InitializationOptions
+    from mcp.shared.exceptions import MCPError
+    from mcp.types import (
+        CallToolRequest,
+        GetPromptRequest,
+        ListPromptsRequest,
+        ListResourcesRequest,
+        ListResourceTemplatesRequest,
+        ListToolsRequest,
+        ReadResourceRequest,
+    )
+
+    from litellm.proxy._experimental.mcp_server import operations
+    from litellm.proxy._experimental.mcp_server.contracts import OperationContext
+    from litellm.proxy._experimental.mcp_server.operations import (
+        _invalidate_byok_cred_cache,
+        _mcp_session_id_from_headers,
+    )
 
     try:
         from mcp.server.streamable_http_manager import StreamableHTTPSessionManager
@@ -493,62 +508,27 @@ if MCP_AVAILABLE:
         ListResourceTemplatesResult,
         ListToolsResult,
         PaginatedRequestParams,
-        Prompt,
         ReadResourceRequestParams,
-        TextContent,
     )
-    from mcp.types import Tool as MCPTool
     from mcp_types.version import HANDSHAKE_PROTOCOL_VERSIONS
 
     from litellm.proxy._experimental.mcp_server.auth.litellm_auth_handler import (
         MCPAuthenticatedUser,
     )
-    from litellm.proxy._experimental.mcp_server.faults.list_outcomes import (
-        SERVER_OUTCOMES_META_KEY,
-        AggregateToolListing,
-        ServerListOk,
-        ServerOutcome,
-        classify_list_exception,
-        outcome_wire_value,
-    )
     from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
         MCPServerManager,
-        _caller_authorization_fans_out,
-        _client_forwarded_authorization_headers,
-        _resolve_openapi_tool_auth,
-        _should_strip_caller_authorization,
         global_mcp_server_manager,
     )
-    from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
-        _request_auth_header,
-        _request_extra_headers,
-        _request_resolved_auth_headers,
-    )
-    from litellm.proxy._experimental.mcp_server.sse_transport import SseServerTransport
-    from litellm.proxy._experimental.mcp_server.tool_registry import (
-        global_mcp_tool_registry,
-    )
-    from litellm.proxy._experimental.mcp_server.utils import (
-        MCP_TOOL_PREFIX_SEPARATOR,
-        is_tool_name_prefixed,
-        normalize_server_name,
-        split_server_prefix_from_name,
-        strip_known_server_prefix,
-    )
-    from litellm.types.mcp import DEFAULT_CREDENTIAL_HEADER, without_header
 
     ######################################################
     ############ MCP Tools List REST API Response Object #
     # Defined here because we don't want to add `mcp` as a
     # required dependency for `litellm` pip package
     ######################################################
-    class ListMCPToolsRestAPIResponseObject(MCPTool):
-        """
-        Object returned by the /tools/list REST API route.
-        """
-
-        mcp_info: MCPInfo | None = Field(default=None, alias="mcp_info")
-        model_config = ConfigDict(arbitrary_types_allowed=True)
+    from litellm.proxy._experimental.mcp_server.operations import (
+        ListMCPToolsRestAPIResponseObject,
+    )
+    from litellm.proxy._experimental.mcp_server.sse_transport import SseServerTransport
 
     def _gateway_create_initialization_options(
         self,
@@ -818,94 +798,45 @@ if MCP_AVAILABLE:
     ############### MCP Server Routes #######################
     ########################################################
 
-    async def handle_list_tools(ctx: ServerRequestContext, params: PaginatedRequestParams) -> ListToolsResult:
-        """
-        List all available tools, with each server's listing outcome attached to the result's
-        ``_meta`` (SERVER_OUTCOMES_META_KEY) so a broken upstream is distinguishable from a healthy
-        server with no tools. Returning a ListToolsResult (rather than a bare list) makes the MCP SDK
-        pass the result through unwrapped, which is what lets the ``_meta`` survive to the client.
-        Also captures the active session for propagation to callbacks.
-        """
-        req_ctx: Final = ctx
-        _ctx_reset_token: Final = active_mcp_request_ctx_var.set(ctx)
-        _session_reset_token: Final = active_mcp_session_var.set(ctx.session)
-        _trace_token = None
-        _transport_token = None
-        _destinations_token = None
-
-        try:
-            _trace_token = _otel_set_mcp_trace_carrier(_mcp_meta_trace_carrier(req_ctx))
-            _transport_token = _otel_set_mcp_transport_span(_otel_transport_span_from_message(req_ctx))
-            _destinations_token = _otel_set_mcp_request_destinations(req_ctx)
-            # Get user authentication from context variable
+    @contextlib.asynccontextmanager
+    async def _legacy_operation_context(ctx: ServerRequestContext, *, trace: bool) -> AsyncGenerator[OperationContext]:
+        with contextlib.ExitStack() as cleanup:
+            cleanup.callback(active_mcp_request_ctx_var.reset, active_mcp_request_ctx_var.set(ctx))
+            cleanup.callback(active_mcp_session_var.reset, active_mcp_session_var.set(ctx.session))
+            if trace:
+                cleanup.callback(
+                    _otel_reset_mcp_trace_carrier, _otel_set_mcp_trace_carrier(_mcp_meta_trace_carrier(ctx))
+                )
+                cleanup.callback(
+                    _otel_reset_mcp_transport_span, _otel_set_mcp_transport_span(_otel_transport_span_from_message(ctx))
+                )
+                cleanup.callback(_otel_reset_mcp_request_destinations, _otel_set_mcp_request_destinations(ctx))
             (
-                user_api_key_auth,
-                mcp_auth_header,
-                mcp_servers,
-                mcp_server_auth_headers,
-                oauth2_headers,
-                raw_headers,
-                _client_ip,
+                auth,
+                token,
+                servers,
+                server_headers,
+                oauth_headers,
+                headers,
+                client_ip,
             ) = await get_or_extract_auth_context()
-            verbose_logger.debug("MCP list_tools - User API Key Auth from context: %s", user_api_key_auth)
-            verbose_logger.debug("MCP list_tools - MCP servers from context: %s", mcp_servers)
-            verbose_logger.debug(
-                "MCP list_tools - MCP server auth headers: %s",
-                list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None,
-            )
-            from mcp.types import Tool
-
-            from litellm.proxy._experimental.mcp_server.tool_search import (
-                get_mcp_proxy_tool_definitions,
-                get_virtual_tool_definitions,
+            yield operations.prepare_context(
+                auth, token, servers, server_headers, oauth_headers, headers, client_ip, _mcp_proxy_mode.get()
             )
 
-            if _mcp_proxy_mode.get():
-                return ListToolsResult(tools=[Tool.model_validate(d) for d in get_mcp_proxy_tool_definitions()])
-            if getattr(
-                getattr(user_api_key_auth, "object_permission", None),
-                "mcp_tool_search_enabled",
-                False,
-            ):
-                return ListToolsResult(tools=[Tool.model_validate(d) for d in get_virtual_tool_definitions()])
-
-            # Get mcp_servers from context variable
-            verbose_logger.debug("MCP list_tools - Calling _list_mcp_tools")
-            listing: Final = await _list_mcp_tools(
-                user_api_key_auth=user_api_key_auth,
-                mcp_auth_header=mcp_auth_header,
-                mcp_servers=mcp_servers,
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
-                log_list_tools_to_spendlogs=True,
-                list_tools_log_source="mcp_protocol",
-            )
-            verbose_logger.info("MCP list_tools - Successfully returned %s tools", len(listing.tools))
-            if not listing.outcomes:
-                return ListToolsResult(tools=listing.tools)
-            outcome_meta: Final = {
-                SERVER_OUTCOMES_META_KEY: {
-                    key: outcome_wire_value(outcome) for key, outcome in listing.outcomes.items()
-                }
-            }
-            return ListToolsResult.model_validate({"tools": listing.tools, "_meta": outcome_meta})
-        except HTTPException as e:
-            from mcp.shared.exceptions import MCPError
-            from mcp.types import INVALID_REQUEST
-
-            raise MCPError(code=INVALID_REQUEST, message=_http_detail_message(e.detail)) from e
-        except Exception as e:
-            verbose_logger.exception("Error in list_tools endpoint: %s", e)
-            # Return empty list instead of failing completely
-            # This prevents the HTTP stream from failing and allows the client to get a response
-            return ListToolsResult(tools=[])  # mutable-ok: MCP result payload
-        finally:
-            _otel_reset_mcp_request_destinations(_destinations_token)
-            _otel_reset_mcp_transport_span(_transport_token)
-            _otel_reset_mcp_trace_carrier(_trace_token)
-            active_mcp_session_var.reset(_session_reset_token)
-            active_mcp_request_ctx_var.reset(_ctx_reset_token)
+    async def handle_list_tools(ctx: ServerRequestContext, params: PaginatedRequestParams) -> ListToolsResult:
+        try:
+            async with _legacy_operation_context(ctx, trace=True) as context:
+                return await operations.GatewayOperations(_capture_host_progress_callback(ctx)).execute(
+                    ListToolsRequest(params=params), context
+                )
+        except MCPError:
+            raise
+        except HTTPException as exc:
+            raise MCPError(code=INVALID_REQUEST, message=operations._http_detail_message(exc.detail)) from exc
+        except Exception as exc:  # noqa: BLE001  # preserve native listing fallback for ingress failures
+            verbose_logger.exception("Error in list_tools endpoint: %s", exc)
+            return ListToolsResult(tools=[])
 
     def _capture_host_progress_callback(ctx: ServerRequestContext) -> Callable | None:
         """Return a progress-forwarding callback bound to the host MCP session.
@@ -942,581 +873,71 @@ if MCP_AVAILABLE:
 
         raise MCPError(code=METHOD_NOT_FOUND, message="Operation unavailable on /mcp/proxy")
 
-    async def _build_virtual_call_logging_obj(
-        name: str,
-        arguments: dict[str, object],
-        user_api_key_auth: UserAPIKeyAuth,
-        raw_headers: Mapping[str, str] | None = None,
-        client_ip: str | None = None,
-    ) -> LiteLLMLoggingObj | None:
-        """Run the pre-call pipeline (guardrails + logging setup) for a virtual
-        mcp_tool_call so the SSE path spend-logs like the REST path."""
-        from litellm.proxy.common_request_processing import (
-            ProxyBaseLLMRequestProcessing,
-        )
-        from litellm.proxy.proxy_server import (
-            general_settings,
-            proxy_config,
-            proxy_logging_obj,
-        )
-
-        request: Final = build_synthetic_mcp_request(
-            path="/mcp/tools/call",
-            raw_headers=raw_headers,
-            client_ip=client_ip,
-        )
-        _, virtual_logging_obj = await ProxyBaseLLMRequestProcessing(
-            data={"name": name, "arguments": arguments}
-        ).common_processing_pre_call_logic(
-            request=request,
-            user_api_key_dict=user_api_key_auth,
-            proxy_config=proxy_config,
-            route_type=CallTypes.call_mcp_tool.value,
-            proxy_logging_obj=proxy_logging_obj,
-            general_settings=general_settings,
-        )
-        return virtual_logging_obj
-
-    async def _dispatch_virtual_mcp_tool(
-        name: str,
-        arguments: dict[str, object] | None,
-        user_api_key_auth: UserAPIKeyAuth | None,
-        client_ip: str | None,
-        mcp_servers: list[str] | None = None,
-        mcp_auth_header: str | None = None,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
-        oauth2_headers: dict[str, str] | None = None,
-        raw_headers: dict[str, str] | None = None,
-    ) -> CallToolResult | None:
-        """Handle the mcp_tool_search / mcp_tool_call virtual tools.
-
-        Returns a CallToolResult when ``name`` is a virtual tool, else ``None`` so
-        the caller falls through to normal tool routing.
-        """
-        from litellm.llms.litellm_proxy.skills.skill_search import DEFAULT_SKILL_SEARCH_TOP_K
-        from litellm.proxy._experimental.mcp_server.tool_search import (
-            AGENT_SEARCH_TOOL_NAME,
-            DEFAULT_AGENT_SEARCH_TOP_K,
-            MCP_PROXY_CALL_TOOL_NAME,
-            MCP_PROXY_TOOL_NAMES,
-            MCP_TOOL_SEARCH_TOOL_NAME,
-            SKILL_SEARCH_TOOL_NAME,
-            VIRTUAL_TOOL_NAMES,
-            coerce_top_k,
-            handle_agent_search,
-            handle_mcp_proxy_tool,
-            handle_mcp_tool_call,
-            handle_mcp_tool_search,
-            handle_skill_search,
-        )
-
-        if _mcp_proxy_mode.get() and name not in MCP_PROXY_TOOL_NAMES:
-            return CallToolResult(
-                content=[  # mutable-ok: MCP result content
-                    TextContent(type="text", text=f"Tool {name} is unavailable on /mcp/proxy")
-                ],
-                is_error=True,
-            )
-
-        if _mcp_proxy_mode.get() and name in MCP_PROXY_TOOL_NAMES:
-            assert user_api_key_auth is not None
-            proxy_call_start: Final = datetime.now()  # noqa: DTZ005  # logging pipeline uses naive datetimes
-            proxy_logging_obj: Final = (
-                await _build_virtual_call_logging_obj(
-                    name=name,
-                    arguments=arguments or {},  # mutable-ok: logging pipeline payload
-                    user_api_key_auth=user_api_key_auth,
-                    raw_headers=raw_headers,
-                    client_ip=client_ip,
-                )
-                if name == MCP_PROXY_CALL_TOOL_NAME
-                else None
-            )
-            try:
-                proxy_result: Final = await handle_mcp_proxy_tool(
-                    name=name,
-                    arguments=arguments or {},  # mutable-ok: proxy handler payload
-                    user_api_key_dict=user_api_key_auth,
-                    client_ip=client_ip,
-                    mcp_servers=mcp_servers,
-                    mcp_auth_header=mcp_auth_header,
-                    mcp_server_auth_headers=mcp_server_auth_headers,
-                    oauth2_headers=oauth2_headers,
-                    raw_headers=raw_headers,
-                    litellm_logging_obj=proxy_logging_obj,
-                )
-            except Exception as exc:
-                if proxy_logging_obj is not None:
-                    from litellm.proxy.proxy_server import proxy_logging_obj as request_logging_obj
-
-                    failure_end: Final = datetime.now()  # noqa: DTZ005  # matches the logging pipeline start time
-                    failure_traceback: Final = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG)
-                    try:
-                        proxy_logging_obj.failure_handler(exc, failure_traceback, proxy_call_start, failure_end)
-                        await proxy_logging_obj.async_failure_handler(
-                            exc, failure_traceback, proxy_call_start, failure_end
-                        )
-                        if not isinstance(exc, MCPUpstreamAuthError):
-                            await request_logging_obj.post_call_failure_hook(
-                                request_data={  # mutable-ok: failure hook mutates its request payload
-                                    "name": name,
-                                    "arguments": arguments,
-                                    "litellm_logging_obj": proxy_logging_obj,
-                                },
-                                original_exception=exc,
-                                user_api_key_dict=user_api_key_auth,
-                                route="/mcp/call_tool",
-                                traceback_str=failure_traceback,
-                            )
-                    except Exception:  # noqa: BLE001  # a failing failure hook must not mask the tool call's own error
-                        verbose_logger.exception("Error logging failed MCP proxy tool call")
-                raise
-            if proxy_logging_obj is not None:
-                return await _fire_mcp_tool_call_logging(
-                    logging_obj=proxy_logging_obj,
-                    result=proxy_result,
-                    start_time=proxy_call_start,
-                    end_time=datetime.now(),  # noqa: DTZ005  # matches the logging pipeline start time
-                    user_api_key_auth=user_api_key_auth,
-                    request_data=types.MappingProxyType({"name": name, "arguments": arguments}),
-                )
-            return proxy_result
-
-        if name not in VIRTUAL_TOOL_NAMES:
-            return None
-
-        if not getattr(
-            getattr(user_api_key_auth, "object_permission", None),
-            "mcp_tool_search_enabled",
-            False,
-        ):
-            return CallToolResult(
-                content=[
-                    TextContent(
-                        type="text",
-                        text=f"Tool {name} requires mcp_tool_search_enabled on the key",
-                    )
-                ],
-                is_error=True,
-            )
-
-        args: Final = arguments or {}
-        if name == MCP_TOOL_SEARCH_TOOL_NAME:
-            return await handle_mcp_tool_search(
-                query=args.get("query", ""),
-                top_k=coerce_top_k(args.get("top_k", 5)),
-                user_api_key_dict=user_api_key_auth,
-                client_ip=client_ip,
-                mcp_servers=mcp_servers,
-                mcp_auth_header=mcp_auth_header,
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
-            )
-
-        assert user_api_key_auth is not None  # guaranteed by the flag check above
-        if name == AGENT_SEARCH_TOOL_NAME:
-            return await handle_agent_search(
-                query=str(args.get("query", "")),
-                top_k=coerce_top_k(args.get("top_k", DEFAULT_AGENT_SEARCH_TOP_K), default=DEFAULT_AGENT_SEARCH_TOP_K),
-                user_api_key_dict=user_api_key_auth,
-            )
-        if name == SKILL_SEARCH_TOOL_NAME:
-            return await handle_skill_search(
-                query=str(args.get("query", "")),
-                top_k=coerce_top_k(args.get("top_k", DEFAULT_SKILL_SEARCH_TOP_K), default=DEFAULT_SKILL_SEARCH_TOP_K),
-                user_api_key_dict=user_api_key_auth,
-            )
-        virtual_logging_obj: Final = await _build_virtual_call_logging_obj(
-            name=name,
-            arguments=args,
-            user_api_key_auth=user_api_key_auth,
-            raw_headers=raw_headers,
-            client_ip=client_ip,
-        )
-        return await handle_mcp_tool_call(
-            tool_name=args.get("tool_name", ""),
-            arguments=args.get("arguments") or {},
-            user_api_key_dict=user_api_key_auth,
-            client_ip=client_ip,
-            mcp_servers=mcp_servers,
-            mcp_auth_header=mcp_auth_header,
-            mcp_server_auth_headers=mcp_server_auth_headers,
-            oauth2_headers=oauth2_headers,
-            raw_headers=raw_headers,
-            litellm_logging_obj=virtual_logging_obj,
-        )
+    from litellm.proxy._experimental.mcp_server.operations import (
+        _build_virtual_call_logging_obj,
+        _dispatch_virtual_mcp_tool,
+    )
 
     async def mcp_server_tool_call(ctx: ServerRequestContext, params: CallToolRequestParams) -> CallToolResult:
-        """
-        Call a specific tool with the provided arguments
-        Args:
-            ctx: SDK request context carrying the client session and HTTP request
-            params (CallToolRequestParams): Tool name and arguments
-        Returns:
-            CallToolResult: Tool execution results
-        """
-        from mcp.types import CallToolResult
-
-        from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
-        from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
-        from litellm.proxy.proxy_server import proxy_config
-
-        req_ctx: Final = ctx
-        _ctx_reset_token: Final = active_mcp_request_ctx_var.set(ctx)
-        _session_reset_token: Final = active_mcp_session_var.set(ctx.session)
-        _trace_token = None
-        _transport_token = None
-        _destinations_token = None
-
-        try:
-            _trace_token = _otel_set_mcp_trace_carrier(_mcp_meta_trace_carrier(req_ctx))
-            _transport_token = _otel_set_mcp_transport_span(_otel_transport_span_from_message(req_ctx))
-            _destinations_token = _otel_set_mcp_request_destinations(req_ctx)
-            # Validate arguments
-            (
-                user_api_key_auth,
-                mcp_auth_header,
-                mcp_servers,
-                mcp_server_auth_headers,
-                oauth2_headers,
-                raw_headers,
-                _client_ip,
-            ) = await get_or_extract_auth_context()
-            verbose_logger.debug(
-                "MCP mcp_server_tool_call - user_api_key_auth=%s, user_role=%s",
-                user_api_key_auth,
-                getattr(user_api_key_auth, "user_role", "N/A"),
+        async with _legacy_operation_context(ctx, trace=True) as context:
+            return await operations.GatewayOperations(_capture_host_progress_callback(ctx)).execute(
+                CallToolRequest(params=params), context
             )
 
-            verbose_logger.debug("MCP mcp_server_tool_call - User API Key Auth from context: %s", user_api_key_auth)
-
-            try:
-                # Inside this try so virtual-tool errors convert to isError
-                # CallToolResult instead of raising out of the protocol handler.
-                virtual_tool_result: Final = await _dispatch_virtual_mcp_tool(
-                    name=params.name,
-                    arguments=params.arguments,
-                    user_api_key_auth=user_api_key_auth,
-                    client_ip=_client_ip,
-                    mcp_servers=mcp_servers,
-                    mcp_auth_header=mcp_auth_header,
-                    mcp_server_auth_headers=mcp_server_auth_headers,
-                    oauth2_headers=oauth2_headers,
-                    raw_headers=raw_headers,
-                )
-                if virtual_tool_result is not None:
-                    return virtual_tool_result
-
-                host_progress_callback: Final = _capture_host_progress_callback(ctx)
-                # Create a body date for logging
-                body_data: Final = {"name": params.name, "arguments": params.arguments}  # mutable-ok: logging payload
-                # Set trace/session id from raw_headers so spend logs and logging_obj stay consistent (same as A2A)
-                chain_id: Final = get_chain_id_from_headers(raw_headers)
-                if chain_id:
-                    body_data["litellm_trace_id"] = chain_id
-                    body_data["litellm_session_id"] = chain_id
-
-                request: Final = build_synthetic_mcp_request(
-                    path="/mcp/tools/call",
-                    raw_headers=raw_headers,
-                    client_ip=_client_ip,
-                )
-                if user_api_key_auth is not None:
-                    data = await add_litellm_data_to_request(
-                        data=body_data,
-                        request=request,
-                        # Bill a team-derived call to the team that granted it. A keyless admitted
-                        # subject carries no team_id, so spend skipped team updates entirely and
-                        # charged the user's PRIMARY org — the granting team's budget never
-                        # accumulated (so it could never begin to block) and, cross-org, the wrong
-                        # organization was charged. This is the ACCOUNTING half; the enforcement
-                        # half (an already-over-budget team stops granting) lives in the source gate.
-                        # Authorization is unaffected: it ran before this, and the union is resolved
-                        # from the untouched auth object passed to call_mcp_tool below.
-                        user_api_key_dict=await MCPRequestHandler.billing_auth_for_tool_call(
-                            user_api_key_auth, tool_name=params.name
-                        ),
-                        proxy_config=proxy_config,
-                    )
-                else:
-                    data = body_data
-
-                response: Final = await call_mcp_tool(
-                    user_api_key_auth=user_api_key_auth,
-                    mcp_auth_header=mcp_auth_header,
-                    mcp_servers=mcp_servers,
-                    mcp_server_auth_headers=mcp_server_auth_headers,
-                    oauth2_headers=oauth2_headers,
-                    raw_headers=raw_headers,
-                    client_ip=_client_ip,
-                    host_progress_callback=host_progress_callback,
-                    **data,  # for logging
-                )
-            except MCPMissingUserEnvVarsError as e:
-                verbose_logger.info(
-                    "MCP mcp_server_tool_call missing per-user env vars: server_id=%s missing=%s",
-                    e.server_id,
-                    e.missing,
-                )
-                return CallToolResult(
-                    content=[TextContent(text=str(e), type="text")],
-                    is_error=True,
-                )
-            except BlockedPiiEntityError as e:
-                verbose_logger.error("BlockedPiiEntityError in MCP tool call: %s", e)
-                return CallToolResult(
-                    content=[
-                        TextContent(
-                            text=f"Error: Blocked PII entity detected - {e}",
-                            type="text",
-                        )
-                    ],
-                    is_error=True,
-                )
-            except GuardrailRaisedException as e:
-                verbose_logger.error("GuardrailRaisedException in MCP tool call: %s", e)
-                return CallToolResult(
-                    content=[TextContent(text=f"Error: Guardrail violation - {e}", type="text")],
-                    is_error=True,
-                )
-            except HTTPException as e:
-                verbose_logger.error("HTTPException in MCP tool call: %s", e)
-                return CallToolResult(
-                    content=[TextContent(text=f"Error: {_http_detail_message(e.detail)}", type="text")],
-                    is_error=True,
-                )
-            except MCPUpstreamAuthError as e:
-                # The MCP session manager serializes handler exceptions as JSON-RPC errors, so a
-                # mid-session tool call cannot emit a raw 401 + WWW-Authenticate the way the REST
-                # call path and the connect-time preemptive check do. Return an explicit isError
-                # naming the upstream status (at info level, not a traceback) so the client still
-                # learns it must re-authenticate upstream and expected pass-through 401s don't spam.
-                verbose_logger.info("Upstream auth failure calling MCP tool: HTTP %s", e.status_code)
-                return CallToolResult(
-                    content=[
-                        TextContent(
-                            text=f"Error: upstream authentication required (HTTP {e.status_code})",
-                            type="text",
-                        )
-                    ],
-                    is_error=True,
-                )
-            except Exception as e:
-                verbose_logger.exception("MCP mcp_server_tool_call - error: %s", e)
-                return CallToolResult(
-                    content=[TextContent(text=f"Error: {e}", type="text")],
-                    is_error=True,
-                )
-
-            return response
-        finally:
-            _otel_reset_mcp_request_destinations(_destinations_token)
-            _otel_reset_mcp_transport_span(_transport_token)
-            _otel_reset_mcp_trace_carrier(_trace_token)
-            active_mcp_session_var.reset(_session_reset_token)
-            active_mcp_request_ctx_var.reset(_ctx_reset_token)
-
     async def list_prompts(ctx: ServerRequestContext, params: PaginatedRequestParams) -> ListPromptsResult:
-        """
-        List all available prompts
-        """
         if _mcp_proxy_mode.get():
             _reject_mcp_proxy_operation()
-        _ctx_reset_token: Final = active_mcp_request_ctx_var.set(ctx)
-        _session_reset_token: Final = active_mcp_session_var.set(ctx.session)
-
         try:
-            # Get user authentication from context variable
-            (
-                user_api_key_auth,
-                mcp_auth_header,
-                mcp_servers,
-                mcp_server_auth_headers,
-                oauth2_headers,
-                raw_headers,
-                _client_ip,
-            ) = await get_or_extract_auth_context()
-            verbose_logger.debug("MCP list_prompts - User API Key Auth from context: %s", user_api_key_auth)
-            verbose_logger.debug("MCP list_prompts - MCP servers from context: %s", mcp_servers)
-            verbose_logger.debug(
-                "MCP list_prompts - MCP server auth headers: %s",
-                list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None,
-            )
-            # Get mcp_servers from context variable
-            verbose_logger.debug("MCP list_prompts - Calling _list_prompts")
-            prompts: Final = await _list_mcp_prompts(
-                user_api_key_auth=user_api_key_auth,
-                mcp_auth_header=mcp_auth_header,
-                mcp_servers=mcp_servers,
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
-            )
-            verbose_logger.info("MCP list_prompts - Successfully returned %s prompts", len(prompts))
-            return ListPromptsResult(prompts=prompts)
-        except Exception as e:
-            verbose_logger.exception("Error in list_prompts endpoint: %s", e)
-            # Return empty list instead of failing completely
-            # This prevents the HTTP stream from failing and allows the client to get a response
-            return ListPromptsResult(prompts=[])  # mutable-ok: MCP result payload
-        finally:
-            active_mcp_session_var.reset(_session_reset_token)
-            active_mcp_request_ctx_var.reset(_ctx_reset_token)
+            async with _legacy_operation_context(ctx, trace=False) as context:
+                return await operations.GatewayOperations(_capture_host_progress_callback(ctx)).execute(
+                    ListPromptsRequest(params=params), context
+                )
+        except Exception as exc:  # noqa: BLE001  # preserve native listing fallback for ingress failures
+            verbose_logger.exception("Error in list_prompts endpoint: %s", exc)
+            return ListPromptsResult(prompts=[])
 
     async def get_prompt(ctx: ServerRequestContext, params: GetPromptRequestParams) -> GetPromptResult:
-        """
-        Get a specific prompt with the provided arguments
-        """
         if _mcp_proxy_mode.get():
             _reject_mcp_proxy_operation()
-        _ctx_reset_token: Final = active_mcp_request_ctx_var.set(ctx)
-        _session_reset_token: Final = active_mcp_session_var.set(ctx.session)
-
-        try:
-            (
-                user_api_key_auth,
-                mcp_auth_header,
-                mcp_servers,
-                mcp_server_auth_headers,
-                oauth2_headers,
-                raw_headers,
-                _client_ip,
-            ) = await get_or_extract_auth_context()
-
-            verbose_logger.debug("MCP mcp_server_tool_call - User API Key Auth from context: %s", user_api_key_auth)
-            return await mcp_get_prompt(
-                name=params.name,
-                arguments=params.arguments,
-                user_api_key_auth=user_api_key_auth,
-                mcp_auth_header=mcp_auth_header,
-                mcp_servers=mcp_servers,
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
+        async with _legacy_operation_context(ctx, trace=False) as context:
+            return await operations.GatewayOperations(_capture_host_progress_callback(ctx)).execute(
+                GetPromptRequest(params=params), context
             )
-        finally:
-            active_mcp_session_var.reset(_session_reset_token)
-            active_mcp_request_ctx_var.reset(_ctx_reset_token)
 
     async def list_resources(ctx: ServerRequestContext, params: PaginatedRequestParams) -> ListResourcesResult:
-        """List all available resources."""
         if _mcp_proxy_mode.get():
             _reject_mcp_proxy_operation()
-        _ctx_reset_token: Final = active_mcp_request_ctx_var.set(ctx)
-        _session_reset_token: Final = active_mcp_session_var.set(ctx.session)
-
         try:
-            (
-                user_api_key_auth,
-                mcp_auth_header,
-                mcp_servers,
-                mcp_server_auth_headers,
-                oauth2_headers,
-                raw_headers,
-                _client_ip,
-            ) = await get_or_extract_auth_context()
-            verbose_logger.debug("MCP list_resources - User API Key Auth from context: %s", user_api_key_auth)
-            verbose_logger.debug("MCP list_resources - MCP servers from context: %s", mcp_servers)
-            verbose_logger.debug(
-                "MCP list_resources - MCP server auth headers: %s",
-                list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None,
-            )
-
-            resources: Final = await _list_mcp_resources(
-                user_api_key_auth=user_api_key_auth,
-                mcp_auth_header=mcp_auth_header,
-                mcp_servers=mcp_servers,
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
-            )
-            verbose_logger.info("MCP list_resources - Successfully returned %s resources", len(resources))
-            return ListResourcesResult(resources=resources)
-        except Exception as e:
-            verbose_logger.exception("Error in list_resources endpoint: %s", e)
-            return ListResourcesResult(resources=[])  # mutable-ok: MCP result payload
-        finally:
-            active_mcp_session_var.reset(_session_reset_token)
-            active_mcp_request_ctx_var.reset(_ctx_reset_token)
+            async with _legacy_operation_context(ctx, trace=False) as context:
+                return await operations.GatewayOperations(_capture_host_progress_callback(ctx)).execute(
+                    ListResourcesRequest(params=params), context
+                )
+        except Exception as exc:  # noqa: BLE001  # preserve native listing fallback for ingress failures
+            verbose_logger.exception("Error in list_resources endpoint: %s", exc)
+            return ListResourcesResult(resources=[])
 
     async def list_resource_templates(
         ctx: ServerRequestContext, params: PaginatedRequestParams
     ) -> ListResourceTemplatesResult:
-        """List all available resource templates."""
         if _mcp_proxy_mode.get():
             _reject_mcp_proxy_operation()
-        _ctx_reset_token: Final = active_mcp_request_ctx_var.set(ctx)
-        _session_reset_token: Final = active_mcp_session_var.set(ctx.session)
-
         try:
-            (
-                user_api_key_auth,
-                mcp_auth_header,
-                mcp_servers,
-                mcp_server_auth_headers,
-                oauth2_headers,
-                raw_headers,
-                _client_ip,
-            ) = await get_or_extract_auth_context()
-            verbose_logger.debug("MCP list_resource_templates - User API Key Auth from context: %s", user_api_key_auth)
-            verbose_logger.debug("MCP list_resource_templates - MCP servers from context: %s", mcp_servers)
-            verbose_logger.debug(
-                "MCP list_resource_templates - MCP server auth headers: %s",
-                list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None,
-            )
-
-            resource_templates: Final = await _list_mcp_resource_templates(
-                user_api_key_auth=user_api_key_auth,
-                mcp_auth_header=mcp_auth_header,
-                mcp_servers=mcp_servers,
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
-            )
-            verbose_logger.info(
-                "MCP list_resource_templates - Successfully returned %s resource templates", len(resource_templates)
-            )
-            return ListResourceTemplatesResult(resource_templates=resource_templates)
-        except Exception as e:
-            verbose_logger.exception("Error in list_resource_templates endpoint: %s", e)
-            return ListResourceTemplatesResult(resource_templates=[])  # mutable-ok: MCP result payload
-        finally:
-            active_mcp_session_var.reset(_session_reset_token)
-            active_mcp_request_ctx_var.reset(_ctx_reset_token)
+            async with _legacy_operation_context(ctx, trace=False) as context:
+                return await operations.GatewayOperations(_capture_host_progress_callback(ctx)).execute(
+                    ListResourceTemplatesRequest(params=params), context
+                )
+        except Exception as exc:  # noqa: BLE001  # preserve native listing fallback for ingress failures
+            verbose_logger.exception("Error in list_resource_templates endpoint: %s", exc)
+            return ListResourceTemplatesResult(resource_templates=[])
 
     async def read_resource(ctx: ServerRequestContext, params: ReadResourceRequestParams) -> ReadResourceResult:
         if _mcp_proxy_mode.get():
             _reject_mcp_proxy_operation()
-        _ctx_reset_token: Final = active_mcp_request_ctx_var.set(ctx)
-        _session_reset_token: Final = active_mcp_session_var.set(ctx.session)
-
-        try:
-            (
-                user_api_key_auth,
-                mcp_auth_header,
-                mcp_servers,
-                mcp_server_auth_headers,
-                oauth2_headers,
-                raw_headers,
-                _client_ip,
-            ) = await get_or_extract_auth_context()
-
-            read_resource_result: Final = await mcp_read_resource(
-                url=params.uri,
-                user_api_key_auth=user_api_key_auth,
-                mcp_auth_header=mcp_auth_header,
-                mcp_servers=mcp_servers,
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
+        async with _legacy_operation_context(ctx, trace=False) as context:
+            return await operations.GatewayOperations(_capture_host_progress_callback(ctx)).execute(
+                ReadResourceRequest(params=params), context
             )
 
-            return read_resource_result
-        finally:
-            active_mcp_session_var.reset(_session_reset_token)
-            active_mcp_request_ctx_var.reset(_ctx_reset_token)
-
     server.add_request_handler("tools/list", PaginatedRequestParams, handle_list_tools)
     server.add_request_handler("tools/call", CallToolRequestParams, mcp_server_tool_call)
     server.add_request_handler("prompts/list", PaginatedRequestParams, list_prompts)
@@ -1533,527 +954,24 @@ if MCP_AVAILABLE:
     ############ Helper Functions ##########################
     ########################################################
 
-    async def _get_allowed_mcp_servers_from_mcp_server_names(
-        mcp_servers: Sequence[str] | None,
-        allowed_mcp_servers: list[MCPServer],
-    ) -> list[MCPServer]:
-        """
-        Get the filtered MCP servers from the MCP server names.
-
-        Fails closed when ``mcp_servers`` is explicitly provided (path- or
-        header-derived) but none of the names resolve to a server alias or
-        access group the caller can access. The previous behavior returned
-        the full ``allowed_mcp_servers`` set, which silently widened scope
-        when a client targeted ``/mcp/<unknown>/`` and made URL/header
-        namespacing appear to work when it did not.
-        """
-
-        filtered_server: Final[dict[str, MCPServer]] = {}
-        # Filter servers based on mcp_servers parameter if provided
-        if mcp_servers is not None:
-            for server_or_group in mcp_servers:
-                server_name_matched = False
-
-                for server in allowed_mcp_servers:
-                    if server and _server_answers_to(server, server_or_group):
-                        filtered_server[server.server_id] = server
-                        server_name_matched = True
-                        break
-
-                if not server_name_matched:
-                    try:
-                        access_group_server_ids = await MCPRequestHandler._get_mcp_servers_from_access_groups(
-                            [server_or_group]
-                        )
-                        # Only include servers that the user has access to
-                        for server_id in access_group_server_ids:
-                            for server in allowed_mcp_servers:
-                                if server_id == server.server_id:
-                                    filtered_server[server.server_id] = server
-                    except Exception as e:
-                        verbose_logger.debug("Could not resolve '%s' as access group: %s", server_or_group, e)
-
-        if filtered_server:
-            return list(filtered_server.values())
-
-        if mcp_servers is not None:
-            # Caller asked for a specific scope but nothing resolved. Fail
-            # closed so URL/header namespacing cannot silently fall back to
-            # the caller's full allowed-server set.
-            verbose_logger.debug(
-                "MCP scope filter resolved to no servers for requested names %s; returning empty list (fail-closed).",
-                mcp_servers,
-            )
-            return []
-
-        return allowed_mcp_servers
-
-    def _http_detail_message(detail: object) -> str:
-        return str(detail.get("error")) if isinstance(detail, dict) and detail.get("error") else str(detail)
-
-    def _server_answers_to(server: MCPServer, name: str) -> bool:
-        requested: Final = name.lower()
-        return any(requested == known.lower() for known in iter_known_server_prefixes(server) if known)
-
-    class _McpDeniedDetail(TypedDict):
-        error: ReadOnly[str]
-
-    async def raise_denied_scoped_mcp_access(
-        requested_names: Sequence[str],
-        user_api_key_auth: UserAPIKeyAuth | None,
-        client_ip: str | None = None,
-    ) -> None:
-        """A scoped request (``/mcp/<name>`` path or ``x-mcp-servers`` header) resolved to zero
-        allowed servers, so the denial must be loud: a silent 200 with no tools reads as a healthy
-        server with no tools. Unknown, unauthorized, and access-group names all share one generic
-        error so scoping cannot probe which servers exist; the agent variant fires only when the
-        same request resolves once the agent binding is stripped, proving the binding caused the veto."""
-        agent_id: Final = user_api_key_auth.agent_id if user_api_key_auth else None
-        if user_api_key_auth is not None and agent_id:
-            resolved_without_agent: Final = await _get_allowed_mcp_servers(
-                user_api_key_auth=user_api_key_auth.model_copy(update=types.MappingProxyType({"agent_id": None})),
-                mcp_servers=requested_names,
-                client_ip=client_ip,
-            )
-
-            def _resolved_to_server(name: str) -> bool:
-                return any(_server_answers_to(server, name) for server in resolved_without_agent)
-
-            vetoed_server: Final = next((name for name in requested_names if _resolved_to_server(name)), None)
-            if vetoed_server is not None:
-                agent_denial: Final[_McpDeniedDetail] = {
-                    "error": (
-                        f"MCP server '{vetoed_server}' is not available to this key: the key is bound to "
-                        f"agent '{agent_id}', whose MCP grants do not include this server. Add the server "
-                        f"to the agent's object_permission.mcp_servers (edit the agent in the Admin UI or "
-                        f"PATCH /v1/agents/{agent_id}), or use a key that is not bound to the agent."
-                    )
-                }
-                raise HTTPException(status_code=403, detail=agent_denial)
-            vetoed_group: Final = next(
-                (
-                    name
-                    for name in requested_names
-                    if not _resolved_to_server(name)
-                    and any(name in (server.access_groups or ()) for server in resolved_without_agent)
-                ),
-                None,
-            )
-            if vetoed_group is not None:
-                group_denial: Final[_McpDeniedDetail] = {
-                    "error": (
-                        f"MCP access group '{vetoed_group}' is not available to this key: the key is bound to "
-                        f"agent '{agent_id}', whose MCP grants do not include it. Add the group to the "
-                        f"agent's object_permission.mcp_access_groups (edit the agent in the Admin UI or "
-                        f"PATCH /v1/agents/{agent_id}), or use a key that is not bound to the agent."
-                    )
-                }
-                raise HTTPException(status_code=403, detail=group_denial)
-        generic_denial: Final[_McpDeniedDetail] = {
-            "error": f"The key is not allowed to access the requested MCP servers: {', '.join(requested_names)}"
-        }
-        raise HTTPException(status_code=403, detail=generic_denial)
-
-    def _tool_name_matches(tool_name: str, filter_list: list[str], mcp_server: MCPServer) -> bool:
-        """
-        Check if a tool name matches any name in the filter list.
-
-        Reads the same owner the server-level permission checks use, so discovery hides
-        exactly what dispatch refuses. ``mcp_server`` is required: guessing the boundary
-        at the first separator mismatches every tool on a server whose prefix contains
-        the separator.
-        """
-        bare_name: Final = strip_known_server_prefix(tool_name, mcp_server)
-        return match_known_tool_name(bare_name, mcp_server, filter_list) is not None
-
-    def filter_tools_by_allowed_tools(
-        tools: list[MCPTool],
-        mcp_server: MCPServer,
-    ) -> list[MCPTool]:
-        """
-        Filter tools by allowed/disallowed tools configuration.
-
-        If allowed_tools is set, only tools in that list are returned.
-        If disallowed_tools is set, tools in that list are excluded.
-        Tool names are matched with and without server prefixes for flexibility.
-
-        Args:
-            tools: List of tools to filter
-            mcp_server: Server configuration with allowed_tools/disallowed_tools
-
-        Returns:
-            Filtered list of tools
-        """
-        from litellm.proxy._experimental.mcp_server.utils import (
-            server_applies_tool_allowlist,
-        )
-
-        tools_to_return = tools
-
-        # Filter by allowed_tools (whitelist)
-        if server_applies_tool_allowlist(mcp_server):
-            if not mcp_server.allowed_tools:
-                return []
-            tools_to_return = [
-                tool for tool in tools if _tool_name_matches(tool.name, mcp_server.allowed_tools, mcp_server)
-            ]
-
-        # Filter by disallowed_tools (blacklist)
-        if mcp_server.disallowed_tools:
-            tools_to_return = [
-                tool
-                for tool in tools_to_return
-                if not _tool_name_matches(tool.name, mcp_server.disallowed_tools, mcp_server)
-            ]
-
-        return tools_to_return
-
-    def apply_tool_overrides(
-        tools: list[MCPTool],
-        mcp_server: MCPServer,
-    ) -> list[MCPTool]:
-        """Apply admin-configured display name/description overrides to tools.
-
-        Overrides are keyed by the unprefixed tool name, same convention as
-        allowed_tools configuration.
-        """
-        display_name_map: Final = mcp_server.tool_name_to_display_name or {}
-        description_map: Final = mcp_server.tool_name_to_description or {}
-        if not display_name_map and not description_map:
-            return tools
-
-        for tool in tools:
-            unprefixed = strip_known_server_prefix(tool.name, mcp_server)
-            lookup_key = unprefixed or tool.name
-            if lookup_key in display_name_map:
-                tool.name = display_name_map[lookup_key]
-            if lookup_key in description_map:
-                tool.description = description_map[lookup_key]
-        return tools
-
-    def _get_client_ip_from_context() -> str | None:
-        """
-        Extract client_ip from auth context.
-        Returns None if context not set (caller should handle this as "no IP filtering").
-        """
-        try:
-            auth_user: Final = auth_context_var.get()
-            if auth_user and isinstance(auth_user, MCPAuthenticatedUser):
-                return auth_user.client_ip
-        except Exception:
-            pass
-        return None
-
-    async def _get_allowed_mcp_servers(
-        user_api_key_auth: UserAPIKeyAuth | None,
-        mcp_servers: Sequence[str] | None,
-        client_ip: str | None = None,
-    ) -> list[MCPServer]:
-        """Return allowed MCP servers for a request after applying filters.
-
-        Args:
-            user_api_key_auth: The authenticated user's API key info.
-            mcp_servers: Optional list of server names to filter to.
-            client_ip: Client IP for IP-based access control. If None, falls back to
-                      auth context. Pass explicitly from request handlers for safety.
-        Note: If client_ip is None and auth context is not set, IP filtering is skipped.
-              This is intentional for internal callers but may indicate a bug if called
-              from a request handler without proper context setup.
-        """
-        # Use explicit client_ip if provided, otherwise try auth context
-        if client_ip is None:
-            client_ip = _get_client_ip_from_context()
-            if client_ip is None:
-                verbose_logger.debug(
-                    "MCP _get_allowed_mcp_servers called without client_ip and no auth context. "
-                    "IP filtering will be skipped. This is expected for internal calls."
-                )
-
-        allowed_mcp_server_ids = await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth)
-        (
-            allowed_mcp_server_ids,
-            _ip_blocked,
-        ) = global_mcp_server_manager.filter_server_ids_by_ip_with_info(allowed_mcp_server_ids, client_ip)
-        verbose_logger.debug(
-            "MCP IP filter: client_ip=%s, allowed_server_ids=%s",
-            client_ip,
-            allowed_mcp_server_ids,
-        )
-        if _ip_blocked > 0:
-            verbose_logger.debug(
-                "MCP IP filtering: %d server(s) are not accessible from client IP %s "
-                "because they are restricted to internal networks. "
-                "No tools from those servers will be returned. "
-                "To expose a server externally, set 'available_on_public_internet: true' "
-                "in its configuration.",
-                _ip_blocked,
-                client_ip,
-            )
-        allowed_mcp_servers: list[MCPServer] = []
-        for allowed_mcp_server_id in allowed_mcp_server_ids:
-            mcp_server = global_mcp_server_manager.get_mcp_server_by_id(allowed_mcp_server_id)
-            if mcp_server is not None:
-                # Apply the request-time oauth2_flow backstop for legacy null rows.
-                mcp_server = MCPServerManager.resolve_oauth2_flow_for_request(mcp_server)
-                allowed_mcp_servers.append(mcp_server)
-
-        if mcp_servers is not None:
-            allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names(
-                mcp_servers=mcp_servers,
-                allowed_mcp_servers=allowed_mcp_servers,
-            )
-
-        return allowed_mcp_servers
-
-    def _client_has_per_server_auth_header(
-        server: MCPServer,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None,
-    ) -> bool:
-        """True if the request carries a per-server ``x-mcp-{alias}-authorization``
-        header for this server. This is the multi-server binding: it names one
-        upstream, so it is unambiguously the caller's upstream token regardless of
-        auth mode (never the LiteLLM admission credential).
-
-        Resolves through the same ``lookup_mcp_server_auth_in_headers`` egress uses, so
-        the connect gate and egress agree on which per-server header names match: a
-        dashboard client sends ``x-mcp-{sanitize_mcp_alias_for_header(alias)}-authorization``,
-        and matching only the raw alias here would 401 a token egress would forward.
-        """
-        if not mcp_server_auth_headers:
-            return False
-        from litellm.proxy._experimental.mcp_server.utils import (
-            lookup_mcp_server_auth_in_headers,
-        )
-
-        server_headers: Final = lookup_mcp_server_auth_in_headers(
-            mcp_server_auth_headers,
-            alias=server.alias,
-            server_name=server.server_name,
-            access_groups=server.access_groups,
-        )
-        if isinstance(server_headers, str):
-            return bool(server_headers.strip())
-        if isinstance(server_headers, dict):
-            return any(isinstance(hk, str) and hk.lower() == "authorization" for hk in server_headers)
-        return False
-
-    def _client_has_passthrough_authorization(
-        server: MCPServer,
-        oauth2_headers: dict[str, str] | None,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None,
-    ) -> bool:
-        """True if the incoming request already carries an ``Authorization``
-        header the gateway will forward to this pass-through server.
-
-        The client may supply the bearer as either the top-level
-        ``Authorization`` header (surfaced via ``oauth2_headers``) or a
-        per-server ``x-mcp-auth-<alias>`` style header (surfaced via
-        ``mcp_server_auth_headers``). Either form skips the pre-emptive 401.
-        """
-        if oauth2_headers:
-            for k in oauth2_headers:
-                if k.lower() == "authorization":
-                    return True
-        return _client_has_per_server_auth_header(server, mcp_server_auth_headers)
-
-    async def _get_user_oauth_extra_headers_from_db(
-        server: MCPServer,
-        user_api_key_auth: UserAPIKeyAuth | None,
-        prefetched_creds: 'Mapping[str, "OAuthCredentialPayload"] | None' = None,
-    ) -> dict[str, str] | None:
-        """Stored OAuth2 token for (user, server) as an ``Authorization: Bearer`` header, or None.
-
-        Thin wrapper over ``resolve_user_oauth_access_token`` (Redis cache, else DB + refresh);
-        ``prefetched_creds`` skips the per-server Redis/DB lookups for the batch path.
-        """
-        if server.auth_type != MCPAuth.oauth2 or user_api_key_auth is None:
-            return None
-        from litellm.proxy._experimental.mcp_server.db import (  # noqa: PLC0415
-            resolve_user_oauth_access_token,
-        )
-
-        token: Final = await resolve_user_oauth_access_token(
-            getattr(user_api_key_auth, "user_id", None), server, prefetched_creds
-        )
-        return {"Authorization": f"Bearer {token}"} if token else None
-
-    async def _prefetch_oauth_creds_for_user(
-        user_api_key_auth: UserAPIKeyAuth | None,
-    ) -> dict[str, "OAuthCredentialPayload"]:
-        """Fetch all OAuth2 credentials for the user in one DB query.
-
-        Returns a dict keyed by server_id to avoid N+1 queries in asyncio.gather loops.
-        """
-        user_id: Final[str | None] = getattr(user_api_key_auth, "user_id", None) if user_api_key_auth else None
-        if not user_id:
-            return {}
-        try:
-            from litellm.proxy._experimental.mcp_server.db import (  # noqa: PLC0415
-                list_user_oauth_credentials,
-            )
-            from litellm.proxy.utils import get_prisma_client_or_throw  # noqa: PLC0415
-
-            prisma_client: Final = get_prisma_client_or_throw(
-                "Database not connected. Connect a database to use OAuth2 MCP tools."
-            )
-            creds: Final = await list_user_oauth_credentials(prisma_client, user_id)
-            return {c["server_id"]: c for c in creds if "server_id" in c}
-        except Exception as e:
-            verbose_logger.warning("_prefetch_oauth_creds_for_user: failed to prefetch for user=%s: %s", user_id, e)
-            return {}
-
-    def _prepare_mcp_server_headers(
-        server: MCPServer,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None,
-        mcp_auth_header: str | None,
-        oauth2_headers: dict[str, str] | None,
-        raw_headers: dict[str, str] | None,
-        user_api_key_auth: UserAPIKeyAuth | None = None,
-        scope_servers: list[MCPServer] | None = None,
-    ) -> tuple[dict[str, str] | str | None, dict[str, str] | None]:
-        """Build auth and extra headers for a server.
-
-        ``scope_servers`` is the full server list a fan-out handler iterates. Passing it lets the
-        client-forwarded token modes withhold the caller's request-wide ``Authorization`` when
-        another server in the scope would also receive it (``_caller_authorization_fans_out``);
-        explicitly-addressed operations leave it None. Per-server ``x-mcp-{alias}-authorization``
-        headers are unaffected — they bind one token to one server and are the multi-server shape.
-        """
-        server_auth_header: dict[str, str] | str | None = None
-        if mcp_server_auth_headers:
-            from litellm.proxy._experimental.mcp_server.utils import (
-                lookup_mcp_server_auth_in_headers,
-            )
-
-            server_auth_header = lookup_mcp_server_auth_in_headers(
-                mcp_server_auth_headers,
-                alias=server.alias,
-                server_name=server.server_name,
-                access_groups=server.access_groups,
-            )
-
-        extra_headers: dict[str, str] | None = None
-        is_client_forwarded_mode: Final = server.is_client_forwarded_token
-        # In a multi-server listing scope the request-wide Authorization can only carry one token,
-        # so it is withheld from a client-forwarded server when another server in scope also consumes
-        # it (RFC 9700 cross-resource replay); such scopes must bind per-server via
-        # x-mcp-{alias}-authorization. The decision is computed once so BOTH the forwarding branch and
-        # the extra_headers copy loop below honor it — otherwise a server that lists Authorization in
-        # extra_headers would re-copy the withheld bearer from raw_headers and replay it anyway.
-        withhold_forwarded_authorization: Final = is_client_forwarded_mode and _caller_authorization_fans_out(
-            server, scope_servers
-        )
-        if server.auth_type == MCPAuth.oauth2:
-            # For OAuth2 M2M servers, upstream Authorization must come from
-            # client_credentials token fetch, never from caller headers.
-            if server.has_client_credentials:
-                extra_headers = None
-            else:
-                # Copy to avoid mutating the original dict (important for parallel fetching)
-                extra_headers = oauth2_headers.copy() if oauth2_headers else None
-                # Migrated authorization_code: the v2 resolver injects the stored per-user
-                # token, so drop the caller-forwarded Authorization (apply-if-absent would
-                # otherwise let it shadow the resolved token). Delegate keeps it. Centralized
-                # via _should_strip_caller_authorization to match _call_regular_mcp_tool.
-                if extra_headers and _should_strip_caller_authorization(
-                    mcp_server=server,
-                    raw_headers=raw_headers,
-                    user_api_key_auth=user_api_key_auth,
-                ):
-                    extra_headers = without_header(extra_headers, DEFAULT_CREDENTIAL_HEADER)
-        elif is_client_forwarded_mode:
-            if not withhold_forwarded_authorization:
-                extra_headers = _client_forwarded_authorization_headers(
-                    mcp_server=server,
-                    oauth2_headers=oauth2_headers,
-                    raw_headers=raw_headers,
-                    user_api_key_auth=user_api_key_auth,
-                )
-
-        if server.extra_headers and raw_headers:
-            if extra_headers is None:
-                extra_headers = {}
-
-            normalized_raw_headers: Final = {str(k).lower(): v for k, v in raw_headers.items() if isinstance(k, str)}
-
-            # Centralized strip decision shared with
-            # ``MCPServerManager._call_regular_mcp_tool`` so the two
-            # code paths cannot drift on this security-sensitive choice.
-            # See ``_should_strip_caller_authorization`` for the rules.
-            strip_caller_authorization: Final = _should_strip_caller_authorization(
-                mcp_server=server,
-                raw_headers=raw_headers,
-                user_api_key_auth=user_api_key_auth,
-            )
-
-            for header in server.extra_headers:
-                if not isinstance(header, str):
-                    continue
-                if header.lower() == "authorization" and (
-                    strip_caller_authorization or withhold_forwarded_authorization
-                ):
-                    continue
-                header_value = normalized_raw_headers.get(header.lower())
-                if header_value is None:
-                    continue
-                extra_headers[header] = header_value
-
-        # Reset to None if no headers were actually added
-        if extra_headers is not None and len(extra_headers) == 0:
-            extra_headers = None
-
-        if server_auth_header is None:
-            server_auth_header = mcp_auth_header
-
-        return server_auth_header, extra_headers
-
-    def _merge_gateway_initialize_instructions(
-        allowed_mcp_servers: list[MCPServer],
-    ) -> str | None:
-        """YAML/DB override, else upstream text (prefetch on init, or list_tools / health_check / call_tool cache)."""
-        if not allowed_mcp_servers:
-            return None
-
-        texts: Final[list[tuple[str, str]]] = []
-        for server in allowed_mcp_servers:
-            label = server.alias or server.server_name or server.name or server.server_id or "mcp"
-            if server.instructions and server.instructions.strip():
-                texts.append((label, server.instructions.strip()))
-                continue
-            if server.spec_path:
-                continue
-            cached = global_mcp_server_manager._upstream_initialize_instructions_by_server_id.get(server.server_id)
-            if cached and cached.strip():
-                texts.append((label, cached.strip()))
-
-        if not texts:
-            return None
-        if len(texts) == 1:
-            return texts[0][1]
-        return "\n\n---\n\n".join(f"[{lbl}]\n{txt}" for lbl, txt in texts)
-
-    async def _raise_if_initialize_grants_no_mcp_servers(
-        allowed: Sequence[MCPServer],
-        user_api_key_auth: UserAPIKeyAuth | None,
-        mcp_servers: Sequence[str] | None,
-        client_ip: str | None,
-    ) -> None:
-        if allowed or user_api_key_auth is None or not user_api_key_auth.api_key:
-            return
-        if mcp_servers:
-            await raise_denied_scoped_mcp_access(
-                requested_names=mcp_servers,
-                user_api_key_auth=user_api_key_auth,
-                client_ip=client_ip,
-            )
-        no_servers_denial: Final[_McpDeniedDetail] = {
-            "error": (
-                "The key has no MCP servers granted, or none of its granted servers is loaded and allowed for "
-                "this client IP. Grant servers or access groups to the key, its team, or its organization "
-                "(object_permission.mcp_servers), check the server's allowed IPs, and reconnect."
-            )
-        }
-        raise HTTPException(status_code=403, detail=no_servers_denial)
+    from litellm.proxy._experimental.mcp_server.operations import (
+        _client_has_passthrough_authorization,
+        _client_has_per_server_auth_header,
+        _get_allowed_mcp_servers,
+        _get_allowed_mcp_servers_from_mcp_server_names,
+        _get_user_oauth_extra_headers_from_db,
+        _http_detail_message,
+        _McpDeniedDetail,
+        _merge_gateway_initialize_instructions,
+        _prefetch_oauth_creds_for_user,
+        _prepare_mcp_server_headers,
+        _raise_if_initialize_grants_no_mcp_servers,
+        _server_answers_to,
+        _tool_name_matches,
+        apply_tool_overrides,
+        filter_tools_by_allowed_tools,
+        raise_denied_scoped_mcp_access,
+    )
 
     @contextlib.asynccontextmanager
     async def _gateway_initialize_instructions_request_scope(
@@ -2063,26 +981,28 @@ if MCP_AVAILABLE:
         scoped_server_endpoint: bool = False,
         is_initialize: bool = False,
     ) -> AsyncIterator[None]:
-        allowed: Final = await _get_allowed_mcp_servers(
+        allowed: Final = await operations._get_allowed_mcp_servers(
             user_api_key_auth=user_api_key_auth,
             mcp_servers=mcp_servers,
             client_ip=client_ip,
         )
         if is_initialize:
-            await _raise_if_initialize_grants_no_mcp_servers(allowed, user_api_key_auth, mcp_servers, client_ip)
+            await operations._raise_if_initialize_grants_no_mcp_servers(
+                allowed, user_api_key_auth, mcp_servers, client_ip
+            )
         if allowed:
             # return_exceptions=True: a per-server probe failure (incl. CancelledError
             # bubbled from anyio task group teardown on connection refused) must not
             # cancel sibling probes or 500 the gateway initialize request.
             await asyncio.gather(
                 *[
-                    global_mcp_server_manager._ensure_upstream_initialize_instructions_cached(s)
+                    operations.global_mcp_server_manager._ensure_upstream_initialize_instructions_cached(s)
                     for s in allowed
                     if s is not None
                 ],
                 return_exceptions=True,
             )
-        merged: Final = _merge_gateway_initialize_instructions(allowed_mcp_servers=allowed)
+        merged: Final = operations._merge_gateway_initialize_instructions(allowed_mcp_servers=allowed)
         scoped_server_name = None
         if scoped_server_endpoint and len(allowed) == 1:
             scoped_server: Final = allowed[0]
@@ -2097,1599 +1017,34 @@ if MCP_AVAILABLE:
             _mcp_gateway_initialize_instructions.reset(instructions_token)
             _mcp_gateway_server_name.reset(server_name_token)
 
-    def _aggregate_server_key(server: MCPServer) -> str:
-        """The client-visible key for a server in listing outcomes and spend metadata: the same
-        display prefix (alias, or the short prefix when that mode is enabled) the caller already
-        sees on the tool names. Canonical internal server names never key a caller-readable
-        surface; when the display naming deliberately hides them, the outcome keys must too."""
-        return get_server_prefix(server) or "unknown"
-
-    async def _get_tools_from_mcp_servers(
-        user_api_key_auth: UserAPIKeyAuth | None,
-        mcp_auth_header: str | None,
-        mcp_servers: list[str] | None,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
-        oauth2_headers: dict[str, str] | None = None,
-        raw_headers: dict[str, str] | None = None,
-        log_list_tools_to_spendlogs: bool = False,
-        list_tools_log_source: str | None = None,
-        litellm_trace_id: str | None = None,
-        request_tags: list[str] | None = None,
-        client_ip: str | None = None,
-        mcp_proxy_mode: bool = False,
-    ) -> AggregateToolListing:
-        """
-        Helper method to fetch tools from MCP servers based on server filtering criteria.
-
-        Args:
-            user_api_key_auth: User authentication info for access control
-            mcp_auth_header: Optional auth header for MCP server (deprecated)
-            mcp_servers: Optional list of server names/aliases to filter by
-            mcp_server_auth_headers: Optional dict of server-specific auth headers
-            oauth2_headers: Optional dict of oauth2 headers
-
-        Returns:
-            AggregateToolListing: Combined tools from filtered servers plus each server's
-            classified listing outcome
-        """
-        if not MCP_AVAILABLE:
-            return AggregateToolListing(tools=[], outcomes={})
-
-        list_tools_start_time: Final = datetime.now()
-        litellm_logging_obj: LiteLLMLoggingObj | None = None
-        list_tools_request_data: dict[str, object] = {}
-
-        if log_list_tools_to_spendlogs:
-            # This is intentionally minimal: only async_success_handler / post_call_failure_hook
-            rules_obj: Final = Rules()
-            list_tools_call_id: Final = str(uuid.uuid4())
-            # Derive trace_id from raw_headers when not explicitly passed (same as A2A / MCP call_tool)
-            effective_litellm_trace_id: Final = litellm_trace_id or get_chain_id_from_headers(raw_headers)
-            spend_logs_metadata: Final[dict[str, object]] = {
-                "mcp_operation": "list_tools",
-            }
-            if isinstance(list_tools_log_source, str):
-                spend_logs_metadata["source"] = list_tools_log_source
-            if isinstance(mcp_servers, list):
-                spend_logs_metadata["requested_mcp_servers"] = mcp_servers
-
-            list_tools_request_data = {
-                "model": "MCP: list_tools",
-                "call_type": CallTypes.list_mcp_tools.value,
-                "litellm_call_id": list_tools_call_id,
-                "litellm_trace_id": effective_litellm_trace_id,
-                "metadata": {
-                    "spend_logs_metadata": spend_logs_metadata,
-                    "headers": logging_safe_mcp_headers(raw_headers),
-                    **({"tags": request_tags} if request_tags else {}),
-                },
-                # Provide a small input payload for standard logging
-                "input": [
-                    {
-                        "role": "system",
-                        "content": {
-                            "mcp_operation": "list_tools",
-                            "requested_mcp_servers": mcp_servers,
-                        },
-                    }
-                ],
-            }
-
-            # Attach user identifiers using the standard helper
-            if user_api_key_auth is not None:
-                LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata(
-                    data=list_tools_request_data,
-                    user_api_key_dict=user_api_key_auth,
-                    _metadata_variable_name="metadata",
-                )
-
-                user_identifier: Final = getattr(user_api_key_auth, "end_user_id", None) or getattr(
-                    user_api_key_auth, "user_id", None
-                )
-                if user_identifier:
-                    list_tools_request_data["user"] = user_identifier
-
-            try:
-                litellm_logging_obj, _ = function_setup(
-                    original_function="list_mcp_tools",
-                    rules_obj=rules_obj,
-                    start_time=list_tools_start_time,
-                    **list_tools_request_data,
-                )
-                if litellm_logging_obj:
-                    litellm_logging_obj.call_type = CallTypes.list_mcp_tools.value
-                    litellm_logging_obj.model = "MCP: list_tools"
-            except Exception as logging_error:
-                verbose_logger.debug("Failed to initialize logging for MCP list_tools: %s", logging_error)
-                litellm_logging_obj = None
-
-        try:
-            allowed_mcp_servers: Final = await _get_allowed_mcp_servers(
-                user_api_key_auth=user_api_key_auth,
-                mcp_servers=mcp_servers,
-                client_ip=client_ip,
-            )
-            if mcp_servers and not allowed_mcp_servers:
-                await raise_denied_scoped_mcp_access(
-                    requested_names=mcp_servers,
-                    user_api_key_auth=user_api_key_auth,
-                    client_ip=client_ip,
-                )
-
-            # Pre-fetch OAuth credentials only when at least one server uses OAuth2,
-            # to avoid an unnecessary DB round-trip on requests with no OAuth2 MCP servers.
-            _has_oauth2_server = any(getattr(s, "auth_type", None) == MCPAuth.oauth2 for s in allowed_mcp_servers)
-            _prefetched_oauth_creds: Final = (
-                await _prefetch_oauth_creds_for_user(user_api_key_auth) if _has_oauth2_server else {}
-            )
-
-            async def _fetch_and_filter_server_tools(
-                server: MCPServer,
-            ) -> "tuple[list[MCPTool], ServerOutcome]":
-                """Fetch and filter tools from a single server, classifying any failure into that
-                server's outcome so the aggregate can keep serving the healthy subset without a
-                broken server masquerading as an empty one."""
-                if server is None:
-                    return [], ServerListOk(tool_count=0)
-
-                server_auth_header, extra_headers = _prepare_mcp_server_headers(
-                    server=server,
-                    mcp_server_auth_headers=mcp_server_auth_headers,
-                    mcp_auth_header=mcp_auth_header,
-                    oauth2_headers=oauth2_headers,
-                    raw_headers=raw_headers,
-                    user_api_key_auth=user_api_key_auth,
-                    scope_servers=allowed_mcp_servers,
-                )
-
-                # Prefer server-stored per-user OAuth when configured, so a stale
-                # Authorization header from the MCP client cannot override Redis/DB
-                # (same issue as call_tool in mcp_server_manager: VS Code caches tokens).
-                from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import (  # noqa: PLC0415
-                    to_server_spec,
-                )
-
-                # A server migrated to the v2 resolver gets its token from the resolver at connect
-                # time; building it here would double-resolve and be shadowed by the v2 graft. The
-                # preemptive 401 already challenged a missing token, so one exists for the connect.
-                migrated_to_v2: Final = to_server_spec(server) is not None
-                if (
-                    not migrated_to_v2
-                    and server.auth_type == MCPAuth.oauth2
-                    and getattr(server, "needs_user_oauth_token", False)
-                    and user_api_key_auth is not None
-                ):
-                    db_headers: Final = await _get_user_oauth_extra_headers_from_db(
-                        server,
-                        user_api_key_auth,
-                        prefetched_creds=_prefetched_oauth_creds,
-                    )
-                    if db_headers:
-                        extra_headers = db_headers
-
-                # If still no OAuth2 token, fall back to pre-fetched creds (non-stale-client path)
-                elif not migrated_to_v2 and extra_headers is None and server.auth_type == MCPAuth.oauth2:
-                    extra_headers = await _get_user_oauth_extra_headers_from_db(
-                        server,
-                        user_api_key_auth,
-                        prefetched_creds=_prefetched_oauth_creds,
-                    )
-
-                if server.is_byok and server.auth_type != MCPAuth.oauth2 and server_auth_header is None:
-                    server_auth_header = await _get_byok_credential(server, user_api_key_auth)
-
-                try:
-                    tools: Final = await global_mcp_server_manager._get_tools_from_server(
-                        server=server,
-                        mcp_auth_header=server_auth_header,
-                        extra_headers=extra_headers,
-                        add_prefix=True,  # Always add server prefix
-                        raw_headers=raw_headers,
-                        user_api_key_auth=user_api_key_auth,
-                        oauth2_headers=oauth2_headers,
-                    )
-                    filtered_tools = filter_tools_by_allowed_tools(tools, server)
-
-                    filtered_tools = await filter_tools_by_key_team_permissions(
-                        tools=filtered_tools,
-                        server_id=server.server_id,
-                        user_api_key_auth=user_api_key_auth,
-                    )
-
-                    if mcp_proxy_mode:
-                        from litellm.proxy._experimental.mcp_server.tool_search import with_mcp_proxy_identity
-
-                        filtered_tools = [  # mutable-ok: MCP tool pipeline
-                            with_mcp_proxy_identity(tool, server.server_id) for tool in filtered_tools
-                        ]
-                    else:
-                        filtered_tools = apply_tool_overrides(filtered_tools, server)
-
-                    verbose_logger.debug(
-                        "Successfully fetched %s tools from server %s, %s after filtering",
-                        len(tools),
-                        server.name,
-                        len(filtered_tools),
-                    )
-                    return filtered_tools, ServerListOk(tool_count=len(filtered_tools))
-                except MCPUpstreamAuthError as e:
-                    # Absorb so one unauthenticated server does not empty every other server's
-                    # tools. Surfacing the upstream 401 to the client as a re-auth challenge is
-                    # intentionally not done here: raising from this list handler cannot produce a
-                    # 401 + WWW-Authenticate (the MCP session manager serializes it as a JSON-RPC
-                    # error). Single-server routes surface it via the request-scope preemptive
-                    # check in _raise_preemptive_401_for_unauthenticated_servers instead.
-                    verbose_logger.debug("MCP list_tools: omitting %s; it needs upstream auth", server.name)
-                    return [], classify_list_exception(e)
-                except Exception as e:
-                    verbose_logger.exception("Error getting tools from server %s: %s", server.name, e)
-                    return [], classify_list_exception(e)
-
-            # Fetch tools from all servers in parallel
-            tasks: Final = [_fetch_and_filter_server_tools(server) for server in allowed_mcp_servers]
-            results: Final = await asyncio.gather(*tasks)
-
-            # Flatten results into single list
-            all_tools: Final[list[MCPTool]] = [tool for tools, _ in results for tool in tools]
-            server_outcomes: Final[dict[str, ServerOutcome]] = {
-                _aggregate_server_key(server): outcome
-                for server, (_, outcome) in zip(allowed_mcp_servers, results)
-                if server is not None
-            }
-
-            # If logging is enabled, enrich spend_logs_metadata with counts
-            if litellm_logging_obj:
-                per_server_tool_counts: Final[dict[str, int]] = {
-                    _aggregate_server_key(server): len(server_tools)
-                    for server, (server_tools, _) in zip(allowed_mcp_servers, results)
-                    if server is not None
-                }
-
-                metadata_dict: Final = litellm_logging_obj.model_call_details.get("metadata")
-                if isinstance(metadata_dict, dict):
-                    spend_meta = metadata_dict.get("spend_logs_metadata")
-                    if not isinstance(spend_meta, dict):
-                        spend_meta = {}
-                        metadata_dict["spend_logs_metadata"] = spend_meta
-                    spend_meta["allowed_server_count"] = len(allowed_mcp_servers)
-                    spend_meta["tool_count_total"] = len(all_tools)
-                    spend_meta["per_server_tool_counts"] = per_server_tool_counts
-                    spend_meta["per_server_list_outcomes"] = {
-                        key: outcome_wire_value(outcome) for key, outcome in server_outcomes.items()
-                    }
-
-                end_time: Final = datetime.now()
-                try:
-                    await litellm_logging_obj.async_success_handler(
-                        result=[
-                            tool.model_dump(mode="json") if isinstance(tool, MCPTool) else tool for tool in all_tools
-                        ],
-                        start_time=list_tools_start_time,
-                        end_time=end_time,
-                    )
-                except Exception as log_exc:
-                    # list_tools responses must not be dropped due to non-blocking
-                    # observability/serialization failures.
-                    verbose_logger.warning(
-                        "MCP list_tools success logging failed (continuing): %s",
-                        log_exc,
-                    )
-
-            verbose_logger.info("Successfully fetched %s tools total from all MCP servers", len(all_tools))
-
-            return AggregateToolListing(tools=all_tools, outcomes=server_outcomes)
-        except Exception as e:
-            # Only fire failure hook if logging was requested for this list-tools execution
-            if log_list_tools_to_spendlogs and user_api_key_auth is not None:
-                try:
-                    from litellm.proxy.proxy_server import proxy_logging_obj
-
-                    if proxy_logging_obj:
-                        traceback_str: Final = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG)
-                        await proxy_logging_obj.post_call_failure_hook(
-                            request_data=list_tools_request_data or {},
-                            original_exception=e,
-                            user_api_key_dict=user_api_key_auth,
-                            route="/mcp/list_tools",
-                            traceback_str=traceback_str,
-                        )
-                except Exception:
-                    verbose_logger.debug("Failed to log MCP list_tools failure via post_call_failure_hook")
-            raise
-
-    async def _get_prompts_from_mcp_servers(
-        user_api_key_auth: UserAPIKeyAuth | None,
-        mcp_auth_header: str | None,
-        mcp_servers: list[str] | None,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
-        oauth2_headers: dict[str, str] | None = None,
-        raw_headers: dict[str, str] | None = None,
-    ) -> list[Prompt]:
-        """
-        Helper method to fetch prompt from MCP servers based on server filtering criteria.
-
-        Args:
-            user_api_key_auth: User authentication info for access control
-            mcp_auth_header: Optional auth header for MCP server (deprecated)
-            mcp_servers: Optional list of server names/aliases to filter by
-            mcp_server_auth_headers: Optional dict of server-specific auth headers
-            oauth2_headers: Optional dict of oauth2 headers
-
-        Returns:
-            List[Prompt]: Combined list of prompts from filtered servers
-        """
-        if not MCP_AVAILABLE:
-            return []
-
-        allowed_mcp_servers: Final = await _get_allowed_mcp_servers(
-            user_api_key_auth=user_api_key_auth,
-            mcp_servers=mcp_servers,
-        )
-
-        # Get prompts from each allowed server
-        all_prompts: Final = []
-        for server in allowed_mcp_servers:
-            if server is None:
-                continue
-
-            server_auth_header, extra_headers = _prepare_mcp_server_headers(
-                server=server,
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                mcp_auth_header=mcp_auth_header,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
-                user_api_key_auth=user_api_key_auth,
-                scope_servers=allowed_mcp_servers,
-            )
-
-            try:
-                prompts = await global_mcp_server_manager.get_prompts_from_server(
-                    server=server,
-                    user_api_key_auth=user_api_key_auth,
-                    mcp_auth_header=server_auth_header,
-                    extra_headers=extra_headers,
-                    add_prefix=True,  # Always add server prefix
-                    raw_headers=raw_headers,
-                )
-
-                all_prompts.extend(prompts)
-
-                verbose_logger.debug("Successfully fetched %s prompts from server %s", len(prompts), server.name)
-            except Exception as e:
-                verbose_logger.exception("Error getting prompts from server %s: %s", server.name, e)
-                # Continue with other servers instead of failing completely
-
-        verbose_logger.info("Successfully fetched %s prompts total from all MCP servers", len(all_prompts))
-
-        return all_prompts
-
-    async def _get_resources_from_mcp_servers(
-        user_api_key_auth: UserAPIKeyAuth | None,
-        mcp_auth_header: str | None,
-        mcp_servers: list[str] | None,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
-        oauth2_headers: dict[str, str] | None = None,
-        raw_headers: dict[str, str] | None = None,
-    ) -> list[Resource]:
-        """Fetch resources from allowed MCP servers."""
-
-        if not MCP_AVAILABLE:
-            return []
-
-        allowed_mcp_servers: Final = await _get_allowed_mcp_servers(
-            user_api_key_auth=user_api_key_auth,
-            mcp_servers=mcp_servers,
-        )
-
-        all_resources: Final[list[Resource]] = []
-        for server in allowed_mcp_servers:
-            if server is None:
-                continue
-
-            server_auth_header, extra_headers = _prepare_mcp_server_headers(
-                server=server,
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                mcp_auth_header=mcp_auth_header,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
-                user_api_key_auth=user_api_key_auth,
-                scope_servers=allowed_mcp_servers,
-            )
-
-            try:
-                resources = await global_mcp_server_manager.get_resources_from_server(
-                    server=server,
-                    user_api_key_auth=user_api_key_auth,
-                    mcp_auth_header=server_auth_header,
-                    extra_headers=extra_headers,
-                    add_prefix=True,  # Always add server prefix
-                    raw_headers=raw_headers,
-                )
-                all_resources.extend(resources)
-
-                verbose_logger.debug("Successfully fetched %s resources from server %s", len(resources), server.name)
-            except Exception as e:
-                verbose_logger.exception("Error getting resources from server %s: %s", server.name, e)
-
-        verbose_logger.info("Successfully fetched %s resources total from all MCP servers", len(all_resources))
-
-        return all_resources
-
-    async def _get_resource_templates_from_mcp_servers(
-        user_api_key_auth: UserAPIKeyAuth | None,
-        mcp_auth_header: str | None,
-        mcp_servers: list[str] | None,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
-        oauth2_headers: dict[str, str] | None = None,
-        raw_headers: dict[str, str] | None = None,
-    ) -> list[ResourceTemplate]:
-        """Fetch resource templates from allowed MCP servers."""
-
-        if not MCP_AVAILABLE:
-            return []
-
-        allowed_mcp_servers: Final = await _get_allowed_mcp_servers(
-            user_api_key_auth=user_api_key_auth,
-            mcp_servers=mcp_servers,
-        )
-
-        all_resource_templates: Final[list[ResourceTemplate]] = []
-        for server in allowed_mcp_servers:
-            if server is None:
-                continue
-
-            server_auth_header, extra_headers = _prepare_mcp_server_headers(
-                server=server,
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                mcp_auth_header=mcp_auth_header,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
-                user_api_key_auth=user_api_key_auth,
-                scope_servers=allowed_mcp_servers,
-            )
-
-            try:
-                resource_templates = await global_mcp_server_manager.get_resource_templates_from_server(
-                    server=server,
-                    user_api_key_auth=user_api_key_auth,
-                    mcp_auth_header=server_auth_header,
-                    extra_headers=extra_headers,
-                    add_prefix=True,  # Always add server prefix
-                    raw_headers=raw_headers,
-                )
-                all_resource_templates.extend(resource_templates)
-                verbose_logger.debug(
-                    "Successfully fetched %s resource templates from server %s",
-                    len(resource_templates),
-                    server.name,
-                )
-            except Exception as e:
-                verbose_logger.exception(
-                    "Error getting resource templates from server %s: %s",
-                    server.name,
-                    str(e),
-                )
-
-        verbose_logger.info(
-            "Successfully fetched %s resource templates total from all MCP servers",
-            len(all_resource_templates),
-        )
-
-        return all_resource_templates
-
-    async def filter_tools_by_key_team_permissions(
-        tools: list[MCPTool],
-        server_id: str,
-        user_api_key_auth: UserAPIKeyAuth | None,
-    ) -> list[MCPTool]:
-        """
-        Filter tools based on key/team mcp_tool_permissions.
-
-        Note: Tool names in the DB are stored without server prefixes,
-        but tool names from MCP servers are prefixed. We need to strip
-        the prefix before comparing.
-        """
-        # Filter by key/team tool-level permissions
-        allowed_tool_names: Final = await MCPRequestHandler.get_allowed_tools_for_server(
-            server_id=server_id,
-            user_api_key_auth=user_api_key_auth,
-        )
-
-        # Tools arrive prefixed with the server's own prefix; strip exactly that
-        # prefix (resolved from the server) rather than the first separator, so a
-        # prefix containing the separator still reduces to the stored bare name.
-        server: Final = global_mcp_server_manager.get_mcp_server_by_id(server_id)
-        return [
-            t
-            for t in tools
-            if MCPRequestHandler.tool_is_granted(strip_known_server_prefix(t.name, server), allowed_tool_names)
-        ]
-
-    async def _list_mcp_tools(
-        user_api_key_auth: UserAPIKeyAuth | None = None,
-        mcp_auth_header: str | None = None,
-        mcp_servers: list[str] | None = None,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
-        oauth2_headers: dict[str, str] | None = None,
-        raw_headers: dict[str, str] | None = None,
-        log_list_tools_to_spendlogs: bool = False,
-        list_tools_log_source: str | None = None,
-        client_ip: str | None = None,
-        mcp_proxy_mode: bool = False,
-    ) -> AggregateToolListing:
-        """
-        List all available MCP tools.
-
-        Args:
-            user_api_key_auth: User authentication info for access control
-            mcp_auth_header: Optional auth header for MCP server (deprecated)
-            mcp_servers: Optional list of server names/aliases to filter by
-            mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value}
-            client_ip: Client IP for IP-based server access control
-
-        Returns:
-            AggregateToolListing: Combined tools from all accessible servers plus each server's
-            classified listing outcome
-        """
-        if not MCP_AVAILABLE:
-            return AggregateToolListing(tools=[], outcomes={})
-
-        try:
-            listing: Final = await _get_tools_from_mcp_servers(
-                user_api_key_auth=user_api_key_auth,
-                mcp_auth_header=mcp_auth_header,
-                mcp_servers=mcp_servers,
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
-                log_list_tools_to_spendlogs=log_list_tools_to_spendlogs,
-                list_tools_log_source=list_tools_log_source,
-                client_ip=client_ip,
-                mcp_proxy_mode=mcp_proxy_mode,
-            )
-            verbose_logger.debug("Successfully fetched %s tools from managed MCP servers", len(listing.tools))
-            return listing
-        except HTTPException:
-            raise
-        except Exception as e:
-            verbose_logger.exception("Error getting tools from managed MCP servers: %s", e)
-            # Continue with an empty listing instead of failing completely
-            return AggregateToolListing(tools=[], outcomes={})
-
-    async def _list_mcp_prompts(
-        user_api_key_auth: UserAPIKeyAuth | None = None,
-        mcp_auth_header: str | None = None,
-        mcp_servers: list[str] | None = None,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
-        oauth2_headers: dict[str, str] | None = None,
-        raw_headers: dict[str, str] | None = None,
-    ) -> list[Prompt]:
-        """
-        List all available MCP prompts.
-
-        Args:
-            user_api_key_auth: User authentication info for access control
-            mcp_auth_header: Optional auth header for MCP server (deprecated)
-            mcp_servers: Optional list of server names/aliases to filter by
-            mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value}
-
-        Returns:
-            List[Prompt]: Combined list of tools from all accessible servers
-        """
-        if not MCP_AVAILABLE:
-            return []
-        # Get tools from managed MCP servers with error handling
-        managed_prompts = []
-        try:
-            managed_prompts = await _get_prompts_from_mcp_servers(
-                user_api_key_auth=user_api_key_auth,
-                mcp_auth_header=mcp_auth_header,
-                mcp_servers=mcp_servers,
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
-            )
-            verbose_logger.debug("Successfully fetched %s prompts from managed MCP servers", len(managed_prompts))
-        except Exception as e:
-            verbose_logger.exception("Error getting tools from managed MCP servers: %s", e)
-            # Continue with empty managed tools list instead of failing completely
-
-        return managed_prompts
-
-    async def _list_mcp_resources(
-        user_api_key_auth: UserAPIKeyAuth | None = None,
-        mcp_auth_header: str | None = None,
-        mcp_servers: list[str] | None = None,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
-        oauth2_headers: dict[str, str] | None = None,
-        raw_headers: dict[str, str] | None = None,
-    ) -> list[Resource]:
-        """List all available MCP resources."""
-
-        if not MCP_AVAILABLE:
-            return []
-
-        managed_resources: list[Resource] = []
-        try:
-            managed_resources = await _get_resources_from_mcp_servers(
-                user_api_key_auth=user_api_key_auth,
-                mcp_auth_header=mcp_auth_header,
-                mcp_servers=mcp_servers,
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
-            )
-            verbose_logger.debug("Successfully fetched %s resources from managed MCP servers", len(managed_resources))
-        except Exception as e:
-            verbose_logger.exception("Error getting resources from managed MCP servers: %s", e)
-
-        return managed_resources
-
-    async def _list_mcp_resource_templates(
-        user_api_key_auth: UserAPIKeyAuth | None = None,
-        mcp_auth_header: str | None = None,
-        mcp_servers: list[str] | None = None,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
-        oauth2_headers: dict[str, str] | None = None,
-        raw_headers: dict[str, str] | None = None,
-    ) -> list[ResourceTemplate]:
-        """List all available MCP resource templates."""
-
-        if not MCP_AVAILABLE:
-            return []
-
-        managed_resource_templates: list[ResourceTemplate] = []
-        try:
-            managed_resource_templates = await _get_resource_templates_from_mcp_servers(
-                user_api_key_auth=user_api_key_auth,
-                mcp_auth_header=mcp_auth_header,
-                mcp_servers=mcp_servers,
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
-            )
-            verbose_logger.debug(
-                "Successfully fetched %s resource templates from managed MCP servers",
-                len(managed_resource_templates),
-            )
-        except Exception as e:
-            verbose_logger.exception(
-                "Error getting resource templates from managed MCP servers: %s",
-                str(e),
-            )
-
-        return managed_resource_templates
-
-    def _resolve_display_name_to_original(
-        name: str,
-        allowed_mcp_servers: list[MCPServer],
-    ) -> str:
-        """Translate a display-name override back to the original prefixed tool name.
-
-        When a client received a customised display name from tools/list (e.g.
-        "Get Pet") it will call tools/call with that same string.  We need to
-        reverse-map it to the original prefixed name (e.g.
-        "petstore_mcp-getPetById") before any routing or permission logic runs.
-        """
-        for server in allowed_mcp_servers:
-            display_map = server.tool_name_to_display_name or {}
-            for unprefixed_name, display_name in display_map.items():
-                if display_name == name:
-                    return add_server_prefix_to_name(unprefixed_name, get_server_prefix(server))
-        return name
-
-    async def _get_byok_credential(
-        mcp_server: MCPServer,
-        user_api_key_auth: UserAPIKeyAuth | None,
-    ) -> str | None:
-        """Retrieve the stored BYOK credential for a user+server pair, served from the worker cache within its TTL."""
-        if not mcp_server.is_byok:
-            return None
-        user_id: Final = (user_api_key_auth.user_id if user_api_key_auth else None) or ""
-        if not user_id:
-            return None
-
-        cached: Final = get_cached_byok_credential(user_id, mcp_server.server_id)
-        if cached is not None:
-            return cached.credential
-
-        from litellm.proxy._experimental.mcp_server.db import get_user_credential
-        from litellm.proxy.proxy_server import prisma_client
-
-        if prisma_client is None:
-            return None
-        credential: Final = await get_user_credential(
-            prisma_client=prisma_client,
-            user_id=user_id,
-            server_id=mcp_server.server_id,
-        )
-        cache_byok_credential(user_id, mcp_server.server_id, credential)
-        return credential
-
-    async def _check_byok_credential(
-        mcp_server: MCPServer,
-        user_api_key_auth: UserAPIKeyAuth | None,
-    ) -> None:
-        """
-        If the MCP server is BYOK-enabled, verify that the requesting user has a
-        stored credential.  When no credential is found, raise an HTTP 401 with a
-        WWW-Authenticate header that points the MCP client to our OAuth metadata
-        endpoint so it can drive the authorization flow.
-        """
-        if not mcp_server.is_byok:
-            return
-
-        user_id: Final = (user_api_key_auth.user_id if user_api_key_auth else None) or ""
-        if not user_id:
-            raise HTTPException(
-                status_code=401,
-                detail={
-                    "error": "byok_auth_required",
-                    "server_id": mcp_server.server_id,
-                    "server_name": mcp_server.server_name or mcp_server.name,
-                    "message": "User identity is required for BYOK servers",
-                },
-                headers={"WWW-Authenticate": get_byok_www_authenticate()},
-            )
-
-        cached: Final = get_cached_byok_credential(user_id, mcp_server.server_id)
-        if cached is not None:
-            if cached.credential is None:
-                raise HTTPException(
-                    status_code=401,
-                    detail={
-                        "error": "byok_auth_required",
-                        "server_id": mcp_server.server_id,
-                        "server_name": mcp_server.server_name or mcp_server.name,
-                        "message": (
-                            "No stored credential found for this BYOK server. "
-                            "Complete the OAuth authorization flow to provide your API key."
-                        ),
-                    },
-                    headers={"WWW-Authenticate": get_byok_www_authenticate()},
-                )
-            return
-
-        from litellm.proxy._experimental.mcp_server.db import get_user_credential
-        from litellm.proxy.proxy_server import prisma_client
-
-        if prisma_client is None:
-            # Fail closed on DB unavailability: returning here previously
-            # bypassed the ownership check and let any proxy-authenticated
-            # caller invoke BYOK tools during outage windows.
-            raise HTTPException(
-                status_code=503,
-                detail={
-                    "error": "byok_auth_unavailable",
-                    "server_id": mcp_server.server_id,
-                    "server_name": mcp_server.server_name or mcp_server.name,
-                    "message": "BYOK credential check requires a database connection.",
-                },
-            )
-
-        credential: Final = await get_user_credential(
-            prisma_client=prisma_client,
-            user_id=user_id,
-            server_id=mcp_server.server_id,
-        )
-        cache_byok_credential(user_id, mcp_server.server_id, credential)
-        if credential is None:
-            raise HTTPException(
-                status_code=401,
-                detail={
-                    "error": "byok_auth_required",
-                    "server_id": mcp_server.server_id,
-                    "server_name": mcp_server.server_name or mcp_server.name,
-                    "message": (
-                        "No stored credential found for this BYOK server. "
-                        "Complete the OAuth authorization flow to provide your API key."
-                    ),
-                },
-                headers={"WWW-Authenticate": get_byok_www_authenticate()},
-            )
-
-    async def _list_tools_before_first_call(
-        server: MCPServer | None,
-        tool_name: str,
-        allowed_mcp_servers: list[MCPServer],
-        user_api_key_auth: UserAPIKeyAuth | None,
-        mcp_auth_header: str | None,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None,
-        oauth2_headers: dict[str, str] | None,
-        raw_headers: dict[str, str] | None,
-    ) -> None:
-        """List ``server`` with the caller's own credentials when it does not yet expose ``tool_name`` here.
-
-        The startup fill skips a server whose upstream wants the caller's token, and mcp 2 no
-        longer lists before an uncached tools/call, so a worker that has not served tools/list
-        for this caller would otherwise answer 404 for a tool the caller can see. Gating on the
-        requested tool, not on any prior listing, keeps callers with different upstream catalogs
-        from masking each other.
-        """
-        if server is None or global_mcp_server_manager.server_exposes_tool(server, tool_name):
-            return
-        if all(allowed.server_id != server.server_id for allowed in allowed_mcp_servers):
-            return
-        try:
-            await _get_tools_from_mcp_servers(
-                user_api_key_auth=user_api_key_auth,
-                mcp_auth_header=mcp_auth_header,
-                mcp_servers=[server.server_id],
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
-            )
-        except Exception as e:  # noqa: BLE001  # best effort: resolution below answers as it did before
-            verbose_logger.debug("MCP tools/call: listing %s before its first call failed: %s", server.name, e)
-
-    async def execute_mcp_tool(
-        name: str,
-        arguments: dict[str, object],
-        allowed_mcp_servers: list[MCPServer],
-        start_time: datetime,
-        user_api_key_auth: UserAPIKeyAuth | None = None,
-        mcp_auth_header: str | None = None,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
-        oauth2_headers: dict[str, str] | None = None,
-        raw_headers: dict[str, str] | None = None,
-        host_progress_callback: Callable | None = None,
-        guardrail_context: Mapping[str, object] | None = None,
-        **kwargs: Any,
-    ) -> CallToolResult:
-        """
-        Execute MCP tool.
-
-        This function assumes permission checks have already been performed.
-
-        Args:
-            name: Tool name (may include server prefix)
-            arguments: Tool arguments
-            allowed_mcp_servers: Pre-validated list of servers the user can access
-            start_time: Start time for logging
-            user_api_key_auth: Optional user API key auth for logging
-            mcp_auth_header: Optional MCP auth header
-            mcp_server_auth_headers: Optional server-specific auth headers
-            oauth2_headers: Optional OAuth2 headers
-            raw_headers: Optional raw HTTP headers
-            **kwargs: Additional arguments (e.g., litellm_logging_obj)
-
-        Returns:
-            CallToolResult: Tool execution result
-        """
-        # Track resolved MCP server for both permission checks and dispatch
-        mcp_server: MCPServer | None = None
-        requested_server_id: Final[str | None] = kwargs.get("requested_server_id")
-
-        # If the client called with a display-name override (e.g. "Get Pet"),
-        # translate it back to the original prefixed name before any routing.
-        name = _resolve_display_name_to_original(name, allowed_mcp_servers)
-
-        # Remove prefix from tool name for logging and processing
-        original_tool_name, server_name = split_server_prefix_from_name(name)
-
-        requested_server: MCPServer | None = None
-        if requested_server_id:
-            requested_server = next(
-                (s for s in allowed_mcp_servers if s.server_id == requested_server_id),
-                None,
-            )
-
-        name_is_prefixed = False
-        if requested_server is not None and MCP_TOOL_PREFIX_SEPARATOR in name:
-            all_registry_prefixes: Final[set[str]] = set()
-            for registry_server in global_mcp_server_manager.get_registry().values():
-                for known_prefix in iter_known_server_prefixes(registry_server):
-                    all_registry_prefixes.add(normalize_server_name(known_prefix))
-            name_is_prefixed = is_tool_name_prefixed(name, known_server_prefixes=all_registry_prefixes)
-
-        first_call_target: Final = (
-            requested_server
-            if requested_server is not None and not name_is_prefixed
-            else global_mcp_server_manager.server_owning_tool_name_prefix(name)
-        )
-        first_call_tool_name: Final = (
-            name
-            if first_call_target is None or (requested_server is not None and not name_is_prefixed)
-            else strip_known_server_prefix(name, first_call_target)
-        )
-        await _list_tools_before_first_call(
-            server=first_call_target,
-            tool_name=first_call_tool_name,
-            allowed_mcp_servers=allowed_mcp_servers,
-            user_api_key_auth=user_api_key_auth,
-            mcp_auth_header=mcp_auth_header,
-            mcp_server_auth_headers=mcp_server_auth_headers,
-            oauth2_headers=oauth2_headers,
-            raw_headers=raw_headers,
-        )
-
-        if requested_server is not None and not name_is_prefixed:
-            # REST callers may pass server_id with the upstream tool name (no
-            # LiteLLM prefix). The first segment is not a registered server
-            # prefix, so the whole string is the upstream tool name and may
-            # legitimately contain the separator (e.g. "text-to-speech").
-            # server_id is authoritative for routing and auth.
-            mcp_server = requested_server
-            server_name = requested_server.name
-            original_tool_name = name
-        else:
-            # Resolve from tool name (MCP JSON-RPC or prefixed REST tool names).
-            mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(name)
-            if mcp_server is None and requested_server is not None:
-                for known_prefix in iter_known_server_prefixes(requested_server):
-                    candidate = global_mcp_server_manager._get_mcp_server_from_tool_name(
-                        add_server_prefix_to_name(name, known_prefix)
-                    )
-                    if candidate is not None:
-                        mcp_server = candidate
-                        break
-            if mcp_server is not None:
-                server_name = mcp_server.name
-                original_tool_name = strip_known_server_prefix(name, mcp_server)
-
-            if requested_server is not None:
-                if mcp_server is not None and mcp_server.server_id != requested_server.server_id:
-                    raise HTTPException(
-                        status_code=403,
-                        detail={
-                            "error": "tool_server_mismatch",
-                            "message": (
-                                f"Tool '{name}' belongs to MCP server "
-                                f"'{mcp_server.name}' but request specified "
-                                f"server_id for '{requested_server.name}'."
-                            ),
-                        },
-                    )
-                if mcp_server is None:
-                    mcp_server = requested_server
-                    server_name = requested_server.name
-                    original_tool_name = strip_known_server_prefix(name, requested_server)
-
-        # Only enforce server-level permissions when we can resolve a server
-        if server_name:
-            if not MCPRequestHandler.is_tool_allowed(
-                allowed_mcp_servers=[server.name for server in allowed_mcp_servers],
-                server_name=server_name,
-            ):
-                raise HTTPException(
-                    status_code=403,
-                    detail="User not allowed to call this tool.",
-                )
-
-        standard_logging_mcp_tool_call: Final[StandardLoggingMCPToolCall] = _get_standard_logging_mcp_tool_call(
-            name=original_tool_name,  # Use original name for logging
-            arguments=arguments,
-            server_name=server_name,
-            session_id=_mcp_session_id_from_headers(raw_headers),
-        )
-        litellm_logging_obj: Final[LiteLLMLoggingObj | None] = kwargs.get("litellm_logging_obj", None)
-        if litellm_logging_obj:
-            litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = standard_logging_mcp_tool_call
-            litellm_logging_obj.model = f"MCP: {name}"
-            litellm_logging_obj.model_call_details["model"] = f"MCP: {name}"
-        # Resolve the MCP server early so BYOK checks and credential injection
-        # apply to ALL dispatch paths (local tool registry AND managed MCP server).
-        if mcp_server is None:
-            mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(name)
-
-        if mcp_server:
-            standard_logging_mcp_tool_call["mcp_server_cost_info"] = (mcp_server.mcp_info or {}).get(
-                "mcp_server_cost_info"
-            )
-            if litellm_logging_obj:
-                litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = standard_logging_mcp_tool_call
-
-            # BYOK: retrieve the stored per-user credential.  A single DB call
-            # both checks existence and fetches the value, avoiding a double query.
-            if mcp_server.is_byok and not mcp_auth_header:
-                byok_cred: Final = await _get_byok_credential(mcp_server, user_api_key_auth)
-                if byok_cred is None:
-                    raise HTTPException(
-                        status_code=401,
-                        detail={
-                            "error": "byok_auth_required",
-                            "server_id": mcp_server.server_id,
-                            "server_name": mcp_server.server_name or mcp_server.name,
-                            "message": (
-                                "No stored credential found for this BYOK server. "
-                                "Complete the OAuth authorization flow to provide your API key."
-                            ),
-                        },
-                        headers={"WWW-Authenticate": get_byok_www_authenticate()},
-                    )
-                mcp_auth_header = byok_cred
-            elif mcp_server.is_byok:
-                # External auth header supplied; still enforce user-identity check.
-                await _check_byok_credential(mcp_server, user_api_key_auth)
-
-        # Check if tool exists in local registry first (for OpenAPI-based tools)
-        # These tools are registered with their prefixed names
-        #########################################################
-        local_tool: Final = global_mcp_tool_registry.get_tool(name)
-        if local_tool:
-            # OpenAPI-backed tools used to bypass `pre_call_tool_check` —
-            # only the managed path ran allowed/banned-tool checks, key/team
-            # tool permissions, and parameter validation. Run the same checks
-            # before dispatching to the local registry. Refuse the call if
-            # we cannot resolve a server: tools registered via
-            # openapi_to_mcp_generator are always tied to a server, so a
-            # missing mcp_server here means the tool->server mapping has
-            # not finished initializing or the registry entry is orphaned.
-            # Skipping the check would re-open the same authorization gap.
-            if mcp_server is None:
-                raise HTTPException(
-                    status_code=503,
-                    detail=(
-                        f"MCP server for tool '{name}' is not available; "
-                        "refusing to dispatch without authorization checks. "
-                        "Retry once the server is registered."
-                    ),
-                )
-
-            # `pre_call_tool_check` calls into `proxy_logging_obj` for the
-            # pre-call guardrail hooks, so source it from the canonical
-            # `proxy_server` module the same way `_handle_managed_mcp_tool`
-            # does. `kwargs.get("proxy_logging_obj")` is None on the MCP
-            # entry path and would crash with AttributeError after the
-            # security checks pass.
-            from litellm.proxy.proxy_server import proxy_logging_obj
-
-            hook_result = await global_mcp_server_manager.pre_call_tool_check(
-                name=original_tool_name,
-                arguments=arguments or {},
-                server_name=server_name or mcp_server.name,
-                user_api_key_auth=user_api_key_auth,
-                proxy_logging_obj=proxy_logging_obj,
-                server=mcp_server,
-                raw_headers=raw_headers,
-                litellm_logging_obj=litellm_logging_obj,
-                guardrail_context=guardrail_context,
-            )
-            # `pre_call_tool_check` may return guardrail-modified
-            # arguments; honor them on the local path too.
-            if isinstance(hook_result, dict) and "arguments" in hook_result:
-                arguments = hook_result["arguments"]
-
-            verbose_logger.debug("Executing local registry tool: %s", name)
-            # The credential rides ContextVars because the tool function has its
-            # headers baked into the closure at registration time.
-            auth_header_value, openapi_forwarded_headers, upstream_credential = _resolve_openapi_tool_auth(
-                mcp_server=mcp_server,
-                mcp_auth_header=mcp_auth_header,
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                raw_headers=raw_headers,
-                user_api_key_auth=user_api_key_auth,
-            )
-            (
-                resolved_auth_headers,
-                forwarded_headers,
-            ) = await global_mcp_server_manager.resolve_openapi_upstream_auth(
-                mcp_server=mcp_server,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
-                mcp_auth_header=upstream_credential,
-                user_api_key_auth=user_api_key_auth,
-                forwarded_headers=openapi_forwarded_headers,
-            )
-
-            _auth_token: Final = _request_auth_header.set(auth_header_value)
-            _extra_token: Final = _request_extra_headers.set(forwarded_headers)
-            _resolved_token: Final = _request_resolved_auth_headers.set(resolved_auth_headers)
-            try:
-                response = await _handle_local_mcp_tool(name, arguments)
-            finally:
-                _request_auth_header.reset(_auth_token)
-                _request_extra_headers.reset(_extra_token)
-                _request_resolved_auth_headers.reset(_resolved_token)
-
-        # Try managed MCP server tool (the name is bare; the prefix boundary was
-        # already resolved above against this server's registered prefixes)
-        # Primary and recommended way to use external MCP servers
-        #########################################################
-        elif mcp_server:
-            response = await _handle_managed_mcp_tool(
-                server_name=server_name,
-                name=original_tool_name,
-                arguments=arguments,
-                user_api_key_auth=user_api_key_auth,
-                mcp_auth_header=mcp_auth_header,
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
-                litellm_logging_obj=litellm_logging_obj,
-                guardrail_context=guardrail_context,
-                host_progress_callback=host_progress_callback,
-            )
-
-        # Fall back to local tool registry with original name (legacy support)
-        #########################################################
-        # Deprecated: Local MCP Server Tool
-        #########################################################
-        else:
-            # Gate only what can actually dispatch. When the unprefixed name is
-            # not in the registry either, `_handle_local_mcp_tool` below reports
-            # 404 and nothing runs, so demanding a server here would turn every
-            # unknown tool name into a misleading 503.
-            if global_mcp_tool_registry.get_tool(original_tool_name) is not None:
-                # `mcp_server` is None here because the tool name is not in the
-                # tool -> server mapping, but the name still carries a prefix
-                # that the server-level check above compared against the
-                # caller's `allowed_mcp_servers` by exact `name`. So the named
-                # server is in that list and can carry the tool-level checks,
-                # even with the mapping cold. Resolve it from
-                # `allowed_mcp_servers` rather than the registry: the registry
-                # would happily return a server the caller holds no grant for,
-                # and matching anything other than `name` would accept a server
-                # the check never validated.
-                prefix_server: Final = next(
-                    (candidate for candidate in allowed_mcp_servers if candidate.name == server_name),
-                    None,
-                )
-                if prefix_server is None:
-                    # A non-empty prefix that passed the server-level check
-                    # always matches here, so this arm only fires when the
-                    # prefix was empty, which is exactly the case that check
-                    # skips. Fail closed rather than dispatch with no server to
-                    # evaluate a tool ceiling against.
-                    raise HTTPException(
-                        status_code=503,
-                        detail=(
-                            f"MCP server for tool '{original_tool_name}' is not available; "
-                            "refusing to dispatch without authorization checks. "
-                            "Retry once the server is registered."
-                        ),
-                    )
-
-                from litellm.proxy.proxy_server import proxy_logging_obj
-
-                hook_result = await global_mcp_server_manager.pre_call_tool_check(
-                    name=original_tool_name,
-                    arguments=arguments,
-                    server_name=server_name,
-                    user_api_key_auth=user_api_key_auth,
-                    proxy_logging_obj=proxy_logging_obj,
-                    server=prefix_server,
-                    raw_headers=raw_headers,
-                    litellm_logging_obj=litellm_logging_obj,
-                    guardrail_context=guardrail_context,
-                )
-                if "arguments" in hook_result:
-                    arguments = hook_result["arguments"]  # pyright: ignore[reportAny]  # hook returns untyped args
-
-            response = await _handle_local_mcp_tool(original_tool_name, arguments)
-
-        return await _run_post_mcp_call_guardrails(
-            result=response,
-            litellm_logging_obj=litellm_logging_obj,
-            user_api_key_auth=user_api_key_auth,
-            request_data=kwargs,
-        )
-
-    async def _run_post_mcp_call_guardrails(
-        result: CallToolResult,
-        litellm_logging_obj: LiteLLMLoggingObj | None,
-        user_api_key_auth: UserAPIKeyAuth | None,
-        request_data: Mapping[str, object],
-    ) -> CallToolResult:
-        """Run ``post_mcp_call`` guardrails over an executed tool result.
-
-        Lives on ``execute_mcp_tool``'s return path rather than inside
-        ``_fire_mcp_tool_call_logging`` so enforcement never depends on logging
-        being configured, and so every dispatch route gets it: the MCP protocol
-        handler, the REST endpoint, and tool search all funnel through here.
-        A guardrail that rejects the result raises, matching ``pre_mcp_call``.
-        """
-        from litellm.proxy.proxy_server import proxy_logging_obj
-
-        if proxy_logging_obj is None:
-            return result
-        return await proxy_logging_obj.post_mcp_call_hook(
-            response=result,
-            request_data=(
-                litellm_logging_obj.model_call_details if litellm_logging_obj is not None else dict(request_data)
-            ),
-            user_api_key_dict=user_api_key_auth,
-        )
-
-    _MCP_CREDENTIAL_REQUEST_FIELDS: Final = frozenset(
-        {
-            "raw_headers",
-            "mcp_auth_header",
-            "mcp_server_auth_headers",
-            "oauth2_headers",
-            "user_api_key_auth",
-        }
+    from litellm.proxy._experimental.mcp_server.operations import (
+        _MCP_CREDENTIAL_REQUEST_FIELDS,
+        _aggregate_server_key,
+        _check_byok_credential,
+        _fire_mcp_tool_call_logging,
+        _get_byok_credential,
+        _get_prompts_from_mcp_servers,
+        _get_resource_templates_from_mcp_servers,
+        _get_resources_from_mcp_servers,
+        _get_standard_logging_mcp_tool_call,
+        _get_tools_from_mcp_servers,
+        _handle_local_mcp_tool,
+        _handle_managed_mcp_tool,
+        _list_mcp_prompts,
+        _list_mcp_resource_templates,
+        _list_mcp_resources,
+        _list_mcp_tools,
+        _list_tools_before_first_call,
+        _resolve_display_name_to_original,
+        _run_post_mcp_call_guardrails,
+        call_mcp_tool,
+        execute_mcp_tool,
+        filter_tools_by_key_team_permissions,
+        fire_mcp_tool_call_failure_logging,
+        mcp_get_prompt,
+        mcp_read_resource,
     )
 
-    async def _fire_mcp_tool_call_logging(
-        logging_obj: LiteLLMLoggingObj,
-        result: CallToolResult,
-        start_time: datetime,
-        end_time: datetime,
-        user_api_key_auth: UserAPIKeyAuth | None = None,
-        request_data: Mapping[str, object] | None = None,
-    ) -> CallToolResult:
-        """Fire post-call logging for an executed MCP tool call, returning the result to send.
-
-        The returned result is what the caller must forward to the client: a
-        ``post_mcp_call`` guardrail may rewrite the tool output (e.g. mask
-        sensitive values) or reject it, in which case its exception propagates.
-        Guardrails run before the success/failure logging so the masked text, not
-        the raw one, is what gets logged.
-
-        A result with ``is_error=True`` is logged as a failure (``status="failure"``
-        payload, so OTel marks the span ERROR) while the HTTP wire behavior stays
-        200 + ``isError: true`` per the MCP spec. The error check runs after
-        ``async_post_mcp_tool_call_hook`` because guardrails may flip the result
-        to ``is_error=True`` in that hook. Raised exceptions never reach here (the
-        ``@client`` wrapper and ``call_mcp_tool``'s except path log those), so
-        this cannot double-log a failure.
-
-        ``request_data`` may carry credential-bearing fields (the REST path puts
-        ``raw_headers``, ``mcp_auth_header``, ``mcp_server_auth_headers``, and
-        ``oauth2_headers`` at the top level of its data dict), so those are
-        stripped before the dict is handed to ``post_call_failure_hook``
-        callbacks.
-        """
-        from litellm.proxy.proxy_server import proxy_logging_obj
-
-        logging_obj.post_call(original_response=result)
-        await logging_obj.async_post_mcp_tool_call_hook(
-            kwargs=logging_obj.model_call_details,
-            response_obj=result,
-            start_time=start_time,
-            end_time=end_time,
-        )
-        logging_obj.call_type = CallTypes.call_mcp_tool.value
-        error_message: Final = extract_mcp_tool_result_error_message(result)
-        if error_message is None:
-            await logging_obj.async_success_handler(result=result, start_time=start_time, end_time=end_time)
-            return result
-
-        logging_obj.has_run_logging(event_type="sync_success")
-        logging_obj.has_run_logging(event_type="async_success")
-        tool_error: Final = MCPToolResultError(error_message)
-        logging_obj.failure_handler(tool_error, "", start_time, end_time)
-        await logging_obj.async_failure_handler(tool_error, "", start_time, end_time)
-
-        if user_api_key_auth is None:
-            return result
-
-        if proxy_logging_obj:
-            sanitized_request_data: Final = {
-                key: value for key, value in (request_data or {}).items() if key not in _MCP_CREDENTIAL_REQUEST_FIELDS
-            }
-            await proxy_logging_obj.post_call_failure_hook(
-                request_data=sanitized_request_data,
-                original_exception=tool_error,
-                user_api_key_dict=user_api_key_auth,
-                route="/mcp/call_tool",
-            )
-        return result
-
-    async def fire_mcp_tool_call_failure_logging(
-        logging_obj: LiteLLMLoggingObj | None,
-        exception: Exception,
-        start_time: datetime,
-        user_api_key_auth: UserAPIKeyAuth | None,
-        request_data: Mapping[str, object],
-    ) -> None:
-        """Failure logging shared by the ``/mcp`` path and the REST endpoint. Call from
-        inside the ``except`` block so the traceback is still available.
-
-        The failure handlers run first because ``_ProxyDBLogger.async_post_call_failure_hook``
-        builds the failure spend-log row from the ``standard_logging_object`` they produce;
-        both gate on ``should_run_logging``, so the ``@client`` wrapper does not log twice.
-        A relayed upstream 401 (``MCPUpstreamAuthError``) is an expected caller-must-reauth
-        signal and skips ``post_call_failure_hook``, which fires the ``llm_exceptions`` alert.
-        """
-        from litellm.proxy.proxy_server import proxy_logging_obj
-
-        traceback_str: Final = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG)
-        if logging_obj is not None:
-            end_time: Final = datetime.now()  # noqa: DTZ005  # naive to match `start_time`, which it is subtracted from
-            logging_obj.failure_handler(exception, traceback_str, start_time, end_time)
-            await logging_obj.async_failure_handler(exception, traceback_str, start_time, end_time)
-
-        if isinstance(exception, MCPUpstreamAuthError) or not proxy_logging_obj or user_api_key_auth is None:
-            return
-        sanitized_request_data: Final = {
-            key: value for key, value in request_data.items() if key not in _MCP_CREDENTIAL_REQUEST_FIELDS
-        }
-        await proxy_logging_obj.post_call_failure_hook(
-            request_data=sanitized_request_data,
-            original_exception=exception,
-            user_api_key_dict=user_api_key_auth,
-            route="/mcp/call_tool",
-            traceback_str=traceback_str,
-        )
-
-    @client
-    async def call_mcp_tool(
-        name: str,
-        arguments: dict[str, object] | None = None,
-        user_api_key_auth: UserAPIKeyAuth | None = None,
-        mcp_auth_header: str | None = None,
-        mcp_servers: list[str] | None = None,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
-        oauth2_headers: dict[str, str] | None = None,
-        raw_headers: dict[str, str] | None = None,
-        client_ip: str | None = None,
-        **kwargs: Any,
-    ) -> CallToolResult:
-        """
-        Call a specific tool with the provided arguments (handles prefixed tool names).
-        """
-        start_time: Final = datetime.now()
-        litellm_logging_obj: Final[LiteLLMLoggingObj | None] = kwargs.get("litellm_logging_obj", None)
-
-        try:
-            if arguments is None:
-                raise HTTPException(status_code=400, detail="Request arguments are required")
-
-            ## CHECK IF USER IS ALLOWED TO CALL THIS TOOL
-            allowed_mcp_server_ids: Final = await global_mcp_server_manager.get_allowed_mcp_servers(
-                user_api_key_auth=user_api_key_auth,
-            )
-
-            allowed_mcp_servers: list[MCPServer] = []
-            for allowed_mcp_server_id in allowed_mcp_server_ids:
-                allowed_server = global_mcp_server_manager.get_mcp_server_by_id(allowed_mcp_server_id)
-                if allowed_server is not None:
-                    # Same request-time oauth2_flow backstop the listing path applies,
-                    # so a null-flow M2M-shape row is treated as M2M on tool calls too.
-                    allowed_server = MCPServerManager.resolve_oauth2_flow_for_request(allowed_server)
-                    allowed_mcp_servers.append(allowed_server)
-
-            allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names(
-                mcp_servers=mcp_servers,
-                allowed_mcp_servers=allowed_mcp_servers,
-            )
-            if mcp_servers and not allowed_mcp_servers:
-                await raise_denied_scoped_mcp_access(
-                    requested_names=mcp_servers,
-                    user_api_key_auth=user_api_key_auth,
-                    client_ip=client_ip,
-                )
-            if not allowed_mcp_servers:
-                raise HTTPException(
-                    status_code=403,
-                    detail="User not allowed to call this tool.",
-                )
-
-            # Delegate to execute_mcp_tool for execution
-            response = await execute_mcp_tool(
-                name=name,
-                arguments=arguments,
-                allowed_mcp_servers=allowed_mcp_servers,
-                start_time=start_time,
-                user_api_key_auth=user_api_key_auth,
-                mcp_auth_header=mcp_auth_header,
-                mcp_server_auth_headers=mcp_server_auth_headers,
-                oauth2_headers=oauth2_headers,
-                raw_headers=raw_headers,
-                **kwargs,
-            )
-        except Exception as e:
-            await fire_mcp_tool_call_failure_logging(litellm_logging_obj, e, start_time, user_api_key_auth, kwargs)
-            raise
-
-        if litellm_logging_obj:
-            response = await _fire_mcp_tool_call_logging(
-                logging_obj=litellm_logging_obj,
-                result=response,
-                start_time=start_time,
-                end_time=datetime.now(),
-                user_api_key_auth=user_api_key_auth,
-                request_data=kwargs,
-            )
-        return response
-
-    async def mcp_get_prompt(
-        name: str,
-        arguments: dict[str, object] | None = None,
-        user_api_key_auth: UserAPIKeyAuth | None = None,
-        mcp_auth_header: str | None = None,
-        mcp_servers: list[str] | None = None,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
-        oauth2_headers: dict[str, str] | None = None,
-        raw_headers: dict[str, str] | None = None,
-    ) -> GetPromptResult:
-        """
-        Fetch a specific MCP prompt, handling both prefixed and unprefixed names.
-        """
-        allowed_mcp_servers: Final = await _get_allowed_mcp_servers(
-            user_api_key_auth=user_api_key_auth,
-            mcp_servers=mcp_servers,
-        )
-
-        if not allowed_mcp_servers:
-            raise HTTPException(
-                status_code=403,
-                detail="User not allowed to get this prompt.",
-            )
-
-        # Extract server name from prefixed prompt name
-        original_prompt_name, server_name = split_server_prefix_from_name(name)
-
-        server: Final = next((s for s in allowed_mcp_servers if s.name == server_name), None)
-        if server is None:
-            raise HTTPException(
-                status_code=403,
-                detail="User not allowed to get this prompt.",
-            )
-
-        server_auth_header, extra_headers = _prepare_mcp_server_headers(
-            server=server,
-            mcp_server_auth_headers=mcp_server_auth_headers,
-            mcp_auth_header=mcp_auth_header,
-            oauth2_headers=oauth2_headers,
-            raw_headers=raw_headers,
-            user_api_key_auth=user_api_key_auth,
-        )
-
-        return await global_mcp_server_manager.get_prompt_from_server(
-            server=server,
-            user_api_key_auth=user_api_key_auth,
-            prompt_name=original_prompt_name,
-            arguments=arguments,
-            mcp_auth_header=server_auth_header,
-            extra_headers=extra_headers,
-            raw_headers=raw_headers,
-        )
-
-    async def mcp_read_resource(
-        url: AnyUrl,
-        user_api_key_auth: UserAPIKeyAuth | None = None,
-        mcp_auth_header: str | None = None,
-        mcp_servers: list[str] | None = None,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
-        oauth2_headers: dict[str, str] | None = None,
-        raw_headers: dict[str, str] | None = None,
-    ) -> ReadResourceResult:
-        """Read resource contents from upstream MCP servers."""
-
-        allowed_mcp_servers: Final = await _get_allowed_mcp_servers(
-            user_api_key_auth=user_api_key_auth,
-            mcp_servers=mcp_servers,
-        )
-
-        if not allowed_mcp_servers:
-            raise HTTPException(
-                status_code=403,
-                detail="User not allowed to read this resource.",
-            )
-
-        if len(allowed_mcp_servers) != 1:
-            raise HTTPException(
-                status_code=400,
-                detail=(
-                    "Multiple MCP servers configured; read_resource currently supports exactly one allowed server."
-                ),
-            )
-
-        server: Final = allowed_mcp_servers[0]
-
-        server_auth_header, extra_headers = _prepare_mcp_server_headers(
-            server=server,
-            mcp_server_auth_headers=mcp_server_auth_headers,
-            mcp_auth_header=mcp_auth_header,
-            oauth2_headers=oauth2_headers,
-            raw_headers=raw_headers,
-            user_api_key_auth=user_api_key_auth,
-        )
-
-        return await global_mcp_server_manager.read_resource_from_server(
-            server=server,
-            user_api_key_auth=user_api_key_auth,
-            url=url,
-            mcp_auth_header=server_auth_header,
-            extra_headers=extra_headers,
-            raw_headers=raw_headers,
-        )
-
-    def _get_standard_logging_mcp_tool_call(
-        name: str,
-        arguments: dict[str, object],
-        server_name: str | None,
-        session_id: str | None = None,
-    ) -> StandardLoggingMCPToolCall:
-        mcp_server: Final = global_mcp_server_manager._get_mcp_server_from_tool_name(
-            add_server_prefix_to_name(name, server_name) if server_name else name
-        )
-        namespaced_tool_name: Final = f"{server_name}/{name}" if server_name else name
-        if mcp_server:
-            mcp_info: Final = mcp_server.mcp_info or {}
-            return StandardLoggingMCPToolCall(
-                name=name,
-                arguments=arguments,
-                mcp_server_name=mcp_info.get("server_name"),
-                mcp_server_logo_url=mcp_info.get("logo_url"),
-                namespaced_tool_name=namespaced_tool_name,
-                mcp_session_id=session_id,
-                mcp_auth_mode=mcp_server.auth_type,
-                mcp_server_resource=_redact_mcp_resource_url(mcp_server.url),
-            )
-        else:
-            return StandardLoggingMCPToolCall(
-                name=name,
-                arguments=arguments,
-                namespaced_tool_name=namespaced_tool_name,
-                mcp_session_id=session_id,
-            )
-
-    async def _handle_managed_mcp_tool(
-        server_name: str,
-        name: str,
-        arguments: dict[str, object],
-        user_api_key_auth: UserAPIKeyAuth | None = None,
-        mcp_auth_header: str | None = None,
-        mcp_server_auth_headers: dict[str, dict[str, str]] | None = None,
-        oauth2_headers: dict[str, str] | None = None,
-        raw_headers: dict[str, str] | None = None,
-        litellm_logging_obj: LiteLLMLoggingObj | None = None,
-        host_progress_callback: Callable | None = None,
-        guardrail_context: Mapping[str, object] | None = None,
-    ) -> CallToolResult:
-        """Handle tool execution for managed server tools"""
-        # Import here to avoid circular import
-        from litellm.proxy.proxy_server import proxy_logging_obj
-
-        call_tool_result: Final = await global_mcp_server_manager.call_tool(
-            server_name=server_name,
-            name=name,
-            arguments=arguments,
-            user_api_key_auth=user_api_key_auth,
-            mcp_auth_header=mcp_auth_header,
-            mcp_server_auth_headers=mcp_server_auth_headers,
-            oauth2_headers=oauth2_headers,
-            raw_headers=raw_headers,
-            proxy_logging_obj=proxy_logging_obj,
-            host_progress_callback=host_progress_callback,
-            litellm_logging_obj=litellm_logging_obj,
-            guardrail_context=guardrail_context,
-        )
-        verbose_logger.debug("CALL TOOL RESULT: %s", call_tool_result)
-        return call_tool_result
-
-    async def _handle_local_mcp_tool(name: str, arguments: dict[str, object]) -> CallToolResult:
-        """Execute a local-registry tool and report whether it succeeded.
-
-        Returns the result rather than bare content because the verdict is part of it: the content
-        alone cannot say whether the handler failed, so callers used to stamp is_error=False on every
-        outcome and an upstream rejection was served as tool output.
-
-        A failure is reported as ``is_error=True`` here rather than raised, because the REST surface
-        turns an unrecognized exception into a 500 and an upstream 403 or 429 is not a gateway crash.
-        ``MCPUpstreamAuthError`` is the exception: it propagates so the caller is told to
-        re-authenticate, which both renderers already know how to say.
-
-        Note: Local tools don't use prefixes, so we use the original name
-        """
-        import inspect
-
-        tool: Final = global_mcp_tool_registry.get_tool(name)
-        if not tool:
-            raise HTTPException(status_code=404, detail=f"Tool '{name}' not found")
-
-        try:
-            if inspect.iscoroutinefunction(tool.handler):
-                result = await tool.handler(**arguments)
-            else:
-                result = tool.handler(**arguments)
-        except MCPUpstreamAuthError:
-            raise
-        except Exception as e:
-            verbose_logger.exception("Error executing local tool %s: %s", name, e)
-            return CallToolResult(
-                content=[TextContent(text=f"Error: {e}", type="text")],  # mutable-ok: MCP result content
-                is_error=True,
-            )
-        return CallToolResult(
-            content=[TextContent(text=str(result), type="text")],  # mutable-ok: MCP result content
-            is_error=False,
-        )
-
     def _get_mcp_servers_in_path(path: str) -> list[str] | None:
         """
         Get the MCP servers from the path
@@ -4178,7 +1533,9 @@ if MCP_AVAILABLE:
                     detail=f"API key does not have access to toolset '{toolset_id}'.",
                 )
 
-        tool_permissions = await global_mcp_server_manager.resolve_toolset_tool_permissions(toolset_ids=[toolset_id])
+        tool_permissions = await operations.global_mcp_server_manager.resolve_toolset_tool_permissions(
+            toolset_ids=[toolset_id]
+        )
         server_ids: Final = list(tool_permissions.keys())
         existing_op: Final = user_api_key_auth.object_permission
         if existing_op is not None:
@@ -4197,7 +1554,7 @@ if MCP_AVAILABLE:
                 mcp_servers=server_ids,
                 mcp_tool_permissions=tool_permissions,
             )
-        return user_api_key_auth.model_copy(update={"object_permission": updated_op})
+        return user_api_key_auth.model_copy(update={"object_permission": updated_op, "mcp_toolset_id": toolset_id})
 
     async def _raise_preemptive_401_for_unauthenticated_servers(
         scope: Scope,
@@ -4221,7 +1578,7 @@ if MCP_AVAILABLE:
         a server it will be 403'd on immediately after authentication.
         """
         for server_name in mcp_servers or []:
-            server = global_mcp_server_manager.get_mcp_server_by_name(server_name, client_ip=client_ip)
+            server = operations.global_mcp_server_manager.get_mcp_server_by_name(server_name, client_ip=client_ip)
             if server is not None and allowed_server_ids is not None and server.server_id not in allowed_server_ids:
                 # Caller's narrowed scope excludes this server — skip the
                 # preemptive challenge and let downstream authorization
@@ -4234,7 +1591,7 @@ if MCP_AVAILABLE:
                 # authorization_url/token_url can change their inferred flow.
                 continue
             if server is not None:
-                server = await global_mcp_server_manager.ensure_oauth_metadata_discovered(server)
+                server = await operations.global_mcp_server_manager.ensure_oauth_metadata_discovered(server)
             if server and server.auth_type == MCPAuth.oauth2:
                 # The challenge decision is per oauth2 sub-mode, not per header:
                 # gateway-managed modes (M2M and interactive authorization_code)
@@ -4262,7 +1619,7 @@ if MCP_AVAILABLE:
                     # authorization server is the gateway itself, vaulting via the
                     # authorize interlude); the per-server relay advertised below
                     # cannot vault without a litellm key on its token request.
-                    if await global_mcp_server_manager.has_user_oauth_token(server, user_api_key_auth):
+                    if await operations.global_mcp_server_manager.has_user_oauth_token(server, user_api_key_auth):
                         continue
 
                     if _is_mcp_admitted_user_subject(user_api_key_auth):
@@ -4345,12 +1702,12 @@ if MCP_AVAILABLE:
                 and server.server_id
                 in frozenset(
                     allowed.server_id
-                    for allowed in await _get_allowed_mcp_servers(
+                    for allowed in await operations._get_allowed_mcp_servers(
                         user_api_key_auth=user_api_key_auth, mcp_servers=mcp_servers, client_ip=client_ip
                     )
                 )
             ):
-                await global_mcp_server_manager.preflight_token_exchange(
+                await operations.global_mcp_server_manager.preflight_token_exchange(
                     server=server,
                     oauth2_headers=oauth2_headers,
                     user_api_key_auth=user_api_key_auth,
@@ -4366,7 +1723,9 @@ if MCP_AVAILABLE:
             if (
                 server
                 and server.is_oauth_passthrough
-                and not _client_has_passthrough_authorization(server, oauth2_headers, mcp_server_auth_headers)
+                and not operations._client_has_passthrough_authorization(
+                    server, oauth2_headers, mcp_server_auth_headers
+                )
             ):
                 www_authenticate = get_passthrough_www_authenticate(
                     scope=scope,
@@ -4383,7 +1742,7 @@ if MCP_AVAILABLE:
                 and server.is_oauth_delegate
                 and len(mcp_servers or []) == 1
                 and _get_forwarded_auth_from_scope(scope) is None
-                and not _client_has_per_server_auth_header(server, mcp_server_auth_headers)
+                and not operations._client_has_per_server_auth_header(server, mcp_server_auth_headers)
             ):
                 www_authenticate = get_passthrough_www_authenticate(
                     scope=scope,
@@ -4400,7 +1759,7 @@ if MCP_AVAILABLE:
                 and server.is_true_passthrough
                 and len(mcp_servers or []) == 1
                 and not _scope_has_authorization_header(scope)
-                and not _client_has_per_server_auth_header(server, mcp_server_auth_headers)
+                and not operations._client_has_per_server_auth_header(server, mcp_server_auth_headers)
             ):
                 if server.is_dcr_bridge:
                     raise HTTPException(
@@ -4528,7 +1887,7 @@ if MCP_AVAILABLE:
 
         # Use the authorized server set, not the raw user-supplied names, so that
         # a caller cannot force a probe to a server their key is not allowed to use.
-        allowed_servers: Final = await _get_allowed_mcp_servers(
+        allowed_servers: Final = await operations._get_allowed_mcp_servers(
             user_api_key_auth=user_api_key_auth,
             mcp_servers=mcp_servers,
             client_ip=client_ip,
diff --git a/litellm/proxy/_experimental/mcp_server/tool_search.py b/litellm/proxy/_experimental/mcp_server/tool_search.py
index a482d02c31d..3650c722103 100644
--- a/litellm/proxy/_experimental/mcp_server/tool_search.py
+++ b/litellm/proxy/_experimental/mcp_server/tool_search.py
@@ -463,8 +463,8 @@ async def handle_mcp_tool_search(
     oauth2_headers: dict[str, str] | None = None,
     raw_headers: dict[str, str] | None = None,
 ) -> CallToolResult:
-    from litellm.proxy._experimental.mcp_server.server import (
-        _list_mcp_tools,  # pyright: ignore[reportPrivateUsage]  # shared catalog owner
+    from litellm.proxy._experimental.mcp_server.operations import (
+        _list_mcp_tools,
     )
     from litellm.proxy.proxy_server import llm_router, proxy_logging_obj
 
@@ -519,8 +519,8 @@ async def handle_mcp_proxy_tool(
     from jsonschema import validate
 
     from litellm.proxy import proxy_server
-    from litellm.proxy._experimental.mcp_server.server import (  # pyright: ignore[reportPrivateUsage]  # shared catalog owner
-        _list_mcp_tools,  # pyright: ignore[reportPrivateUsage]  # shared catalog owner
+    from litellm.proxy._experimental.mcp_server.operations import (
+        _list_mcp_tools,
     )
 
     listing: Final = await _list_mcp_tools(
@@ -607,7 +607,7 @@ async def handle_mcp_tool_call(
     requested_server_id: str | None = None,
     guardrail_context: Mapping[str, object] | None = None,
 ) -> CallToolResult:
-    from litellm.proxy._experimental.mcp_server.server import (
+    from litellm.proxy._experimental.mcp_server.operations import (
         _get_allowed_mcp_servers,
         execute_mcp_tool,
         raise_denied_scoped_mcp_access,
@@ -643,6 +643,7 @@ async def handle_mcp_tool_call(
         mcp_server_auth_headers=mcp_server_auth_headers,
         oauth2_headers=oauth2_headers,
         raw_headers=raw_headers,
+        client_ip=client_ip,
         litellm_logging_obj=litellm_logging_obj,
         requested_server_id=requested_server_id,
         guardrail_context=guardrail_context,
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 3b34440d0bf..07338bcff52 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -3238,6 +3238,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView):  # the expected response ob
     # above; a forged value could at most narrow, but the stripping keeps the field's provenance
     # single-owner so its meaning stays trustworthy.
     mcp_session_resource_server_id: str | None = Field(default=None, exclude=True)
+    mcp_toolset_id: str | None = Field(default=None, exclude=True)
     via_virtual_key: bool = Field(
         default=False,
         exclude=True,
@@ -3279,6 +3280,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView):  # the expected response ob
         values.pop("mcp_admitted_user_subject", None)
         values.pop("mcp_source_team_rpm_limits", None)
         values.pop("mcp_session_resource_server_id", None)
+        values.pop("mcp_toolset_id", None)
         values.pop("via_virtual_key", None)
         if values.get("api_key") is not None:
             values.update({"token": cls._safe_hash_litellm_api_key(values.get("api_key"))})
diff --git a/scripts/check_mcp_operation_boundary.py b/scripts/check_mcp_operation_boundary.py
new file mode 100644
index 00000000000..b6c9dcefabf
--- /dev/null
+++ b/scripts/check_mcp_operation_boundary.py
@@ -0,0 +1,65 @@
+import ast
+import sys
+from pathlib import Path
+from typing import Final
+
+PACKAGE: Final = Path("litellm/proxy/_experimental/mcp_server")
+LEGACY_ADAPTERS: Final = frozenset({"server.py", "legacy_callbacks.py", "mcp_context.py", "mcp_debug.py"})
+CONFINED_NAMES: Final = frozenset(
+    {
+        "auth_context_var",
+        "active_mcp_session_var",
+        "active_mcp_request_ctx_var",
+        "get_active_auth_context",
+        "get_active_mcp_session",
+        "get_active_mcp_request_ctx",
+        "get_or_extract_auth_context",
+        "_session_obj_auth_storage",
+        "WeakKeyDictionary",
+        "_mcp_active_toolset_id",
+        "_mcp_gateway_initialize_instructions",
+        "_mcp_gateway_server_name",
+        "_mcp_proxy_mode",
+    }
+)
+
+
+def is_confined(name: str) -> bool:
+    return name in CONFINED_NAMES or name.startswith("_stateful_session_")
+
+
+def violations(path: Path, source: str) -> tuple[str, ...]:
+    if path.name in LEGACY_ADAPTERS:
+        return ()
+    tree: Final = ast.parse(source, filename=str(path))
+    return tuple(
+        f"{path}:{node.lineno}: MCP request/session state belongs in a legacy adapter"
+        for node in ast.walk(tree)
+        if (
+            isinstance(node, ast.ImportFrom)
+            and (
+                (node.module or "").endswith(".mcp_context")
+                or any(is_confined(alias.name) for alias in node.names)
+                or (path.name in {"operations.py", "contracts.py"} and (node.module or "").endswith(".server"))
+            )
+            or isinstance(node, ast.Name)
+            and is_confined(node.id)
+            or isinstance(node, ast.Attribute)
+            and is_confined(node.attr)
+        )
+    )
+
+
+def main() -> int:
+    findings: Final = tuple(
+        finding for path in sorted(PACKAGE.rglob("*.py")) for finding in violations(path, path.read_text())
+    )
+    if findings:
+        print("\n".join(findings), file=sys.stderr)
+        return 1
+    print("MCP operation boundary: passed")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/scripts/pre_commit_lint.sh b/scripts/pre_commit_lint.sh
index 1abd415d237..22cc38f841c 100755
--- a/scripts/pre_commit_lint.sh
+++ b/scripts/pre_commit_lint.sh
@@ -102,6 +102,9 @@ ui_prettier_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs|json|css|s
 ui_eslint_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs)$'
 
 litellm_py_files=$(scope_match "$litellm_py_pattern")
+if [ -n "$(scope_match '^(litellm/proxy/_experimental/mcp_server/|scripts/check_mcp_operation_boundary\.py)')" ]; then
+    uv run --no-sync python scripts/check_mcp_operation_boundary.py || exit 1
+fi
 e2e_py_files=$(scope_match "$e2e_py_pattern")
 test_tree_files=$(scope_match "$test_tree_pattern")
 # ruff format (and CI's format step) skip enterprise; the rest of make lint covers it.
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py
index 87e23893616..a77b4c8d565 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py
@@ -1,3 +1,4 @@
+from litellm.proxy._experimental.mcp_server import operations as mcp_operations
 """
 Unit tests for the BYOK OAuth 2.1 authorization server endpoints.
 
@@ -592,7 +593,7 @@ async def test_check_byok_credential_missing_credential(monkeypatch):
 
     monkeypatch.delenv("PROXY_BASE_URL", raising=False)
     monkeypatch.delenv("SERVER_ROOT_PATH", raising=False)
-    server_module.byok_credential_cache.flush_cache()
+    mcp_operations.byok_credential_cache.flush_cache()
     mock_prisma = MagicMock()
 
     with (
@@ -628,13 +629,13 @@ async def test_execute_byok_tool_missing_credential_advertises_api_key_flow(monk
     from litellm.types.mcp_server.mcp_server_manager import MCPServer
 
     monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com/proxy")
-    mcp_module.byok_credential_cache.flush_cache()
+    mcp_operations.byok_credential_cache.flush_cache()
     server = MCPServer(server_id="byok-discovery", name="byok-discovery", transport=MCPTransport.http, is_byok=True)
     prisma = MagicMock()
     prisma.db.litellm_mcpusercredentials.find_unique = AsyncMock(return_value=None)
     monkeypatch.setattr(proxy_server, "prisma_client", prisma)
     with pytest.raises(HTTPException) as exc_info:
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name="list_regions",
             arguments={},
             allowed_mcp_servers=[server],
@@ -687,7 +688,7 @@ async def test_invalidate_byok_cred_cache_evicts_locally_and_broadcasts_the_same
 
     server = MCPServer(server_id="byok-revoke", name="byok-server", transport=MCPTransport.http, is_byok=True)
     user_auth = UserAPIKeyAuth(user_id="mallory", api_key="sk-test")
-    server_module.byok_credential_cache.flush_cache()
+    mcp_operations.byok_credential_cache.flush_cache()
     db_lookup = AsyncMock(side_effect=["sk-before-revoke", None])
     publish = AsyncMock()
 
@@ -699,13 +700,13 @@ async def test_invalidate_byok_cred_cache_evicts_locally_and_broadcasts_the_same
             "litellm.proxy.proxy_server.prisma_client", MagicMock()
         ),
         patch.object(  # test-quality-ok: the redis publisher is module-level; asserting the broadcast without a redis
-            server_module, "publish_auth_cache_invalidation", new=publish
+            mcp_operations, "publish_auth_cache_invalidation", new=publish
         ),
     ):
-        assert await server_module._get_byok_credential(server, user_auth) == "sk-before-revoke"
-        assert await server_module._get_byok_credential(server, user_auth) == "sk-before-revoke"
-        await server_module._invalidate_byok_cred_cache("mallory", "byok-revoke")
-        assert await server_module._get_byok_credential(server, user_auth) is None
+        assert await mcp_operations._get_byok_credential(server, user_auth) == "sk-before-revoke"
+        assert await mcp_operations._get_byok_credential(server, user_auth) == "sk-before-revoke"
+        await mcp_operations._invalidate_byok_cred_cache("mallory", "byok-revoke")
+        assert await mcp_operations._get_byok_credential(server, user_auth) is None
 
     assert db_lookup.await_count == 2
     publish.assert_awaited_once_with(cache_key=byok_credential_cache_key("mallory", "byok-revoke"))
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_contracts.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_contracts.py
new file mode 100644
index 00000000000..e13ecdfcce9
--- /dev/null
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_contracts.py
@@ -0,0 +1,60 @@
+from dataclasses import FrozenInstanceError
+
+import pytest
+
+from litellm.proxy._experimental.mcp_server.operations import prepare_context
+from litellm.proxy._types import UserAPIKeyAuth
+
+
+def test_operation_context_isolates_nested_headers_and_caller_permissions():
+    caller = UserAPIKeyAuth(user_id="alpha", models=["allowed"])
+    caller.mcp_admitted_user_subject = True
+    caller.mcp_session_resource_server_id = "alpha-server"
+    caller.mcp_toolset_id = "toolset-alpha"
+    caller.mcp_source_team_rpm_limits = {"team": {"alpha-server": 2}}
+    headers = {"x-caller": "alpha"}
+    server_headers = {"alpha-server": {"authorization": "alpha-token"}}
+    context = prepare_context(caller, raw_headers=headers, mcp_server_auth_headers=server_headers)
+
+    caller.models.append("forbidden")
+    caller.mcp_source_team_rpm_limits["team"]["alpha-server"] = 999
+    headers["x-caller"] = "bravo"
+    server_headers["alpha-server"]["authorization"] = "bravo-token"
+    captured = context.user_api_key_auth
+    assert captured is not None
+    assert captured.models == ["allowed"]
+    assert captured.mcp_admitted_user_subject is True
+    assert captured.mcp_session_resource_server_id == "alpha-server"
+    assert captured.mcp_toolset_id == "toolset-alpha"
+    assert captured.mcp_source_team_rpm_limits == {"team": {"alpha-server": 2}}
+    captured.models.append("also-forbidden")
+    assert context.user_api_key_auth.models == ["allowed"]
+    assert context.raw_headers == {"x-caller": "alpha"}
+    assert context.mcp_server_auth_headers == {"alpha-server": {"authorization": "alpha-token"}}
+    with pytest.raises(TypeError):
+        context.raw_headers["x-caller"] = "changed"
+    with pytest.raises(TypeError):
+        context.mcp_server_auth_headers["alpha-server"]["authorization"] = "changed"
+    with pytest.raises(FrozenInstanceError):
+        context.client_ip = "untrusted"
+
+
+def test_operation_context_preserves_missing_and_empty_inputs():
+    missing = prepare_context()
+    empty = prepare_context(mcp_servers=[], raw_headers={}, oauth2_headers={}, mcp_server_auth_headers={})
+    assert missing.user_api_key_auth is None
+    assert missing.mcp_servers is None
+    assert missing.raw_headers is None
+    assert missing.oauth2_headers is None
+    assert missing.mcp_server_auth_headers is None
+    assert empty.mcp_servers == ()
+    assert empty.raw_headers == {}
+    assert empty.oauth2_headers == {}
+    assert empty.mcp_server_auth_headers == {}
+
+
+def test_toolset_request_marker_cannot_be_supplied_by_caller_or_serialized():
+    auth = UserAPIKeyAuth.model_validate({"user_id": "alpha", "mcp_toolset_id": "forged"})
+    assert auth.mcp_toolset_id is None
+    auth.mcp_toolset_id = "server-resolved"
+    assert "mcp_toolset_id" not in auth.model_dump()
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_block_recording.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_block_recording.py
index 64d926bc5e3..b8aadef430f 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_block_recording.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_block_recording.py
@@ -1,5 +1,6 @@
+from litellm.proxy._experimental.mcp_server import operations as mcp_operations
 """Tests for guardrail-block recording in
-``litellm.proxy._experimental.mcp_server.server.call_mcp_tool``.
+``litellm.proxy._experimental.mcp_server.operations.call_mcp_tool``.
 
 A pre-call MCP guardrail block *raises* into ``call_mcp_tool``'s
 ``except Exception``. The failure spend-log row that the Guardrails Monitor's
@@ -70,7 +71,7 @@ async def _call_block(logging_obj, order: list, *, user_api_key_auth=mock.sentin
 
     with mock.patch.dict(sys.modules, {"litellm.proxy.proxy_server": fake_proxy_server}):
         with contextlib.suppress(HTTPException):
-            await server.call_mcp_tool.__wrapped__(
+            await mcp_operations.call_mcp_tool.__wrapped__(
                 name="t",
                 arguments=None,
                 user_api_key_auth=user_api_key_auth,
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py
index 28faf375ab8..9659eb1cbc2 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py
@@ -1229,7 +1229,7 @@ class TestResolveByokMcpAuthHeader:
         user_auth = UserAPIKeyAuth(user_id="user-1", api_key="sk-dashboard")
 
         with patch(
-            "litellm.proxy._experimental.mcp_server.server._get_byok_credential",
+            "litellm.proxy._experimental.mcp_server.operations._get_byok_credential",
             new=AsyncMock(return_value="stored-cred"),
         ):
             result = await _resolve_byok_mcp_auth_header(server, user_auth, None)
@@ -1249,7 +1249,7 @@ class TestResolveByokMcpAuthHeader:
         user_auth = UserAPIKeyAuth(user_id="user-1", api_key="sk-dashboard")
 
         with patch(
-            "litellm.proxy._experimental.mcp_server.server._get_byok_credential",
+            "litellm.proxy._experimental.mcp_server.operations._get_byok_credential",
             new=AsyncMock(return_value=None),
         ):
             with pytest.raises(HTTPException) as exc_info:
@@ -1272,7 +1272,7 @@ class TestResolveByokMcpAuthHeader:
         check_mock = AsyncMock(return_value=None)
 
         with patch(
-            "litellm.proxy._experimental.mcp_server.server._check_byok_credential",
+            "litellm.proxy._experimental.mcp_server.operations._check_byok_credential",
             new=check_mock,
         ):
             result = await _resolve_byok_mcp_auth_header(server, user_auth, "caller-header")
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py
index 3f5d4ad83ea..1909e3306a2 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py
@@ -1,3 +1,4 @@
+from litellm.proxy._experimental.mcp_server import operations as mcp_operations
 """Unit tests for MCP OAuth passthrough tool-fetch behavior."""
 
 import logging
@@ -339,16 +340,16 @@ async def test_aggregate_list_tools_absorbs_one_unauthenticated_server():
             raise MCPUpstreamAuthError(status_code=401, www_authenticate=None, server_name=server.name)
         return [good_tool]
 
-    with patch.object(mcp_server, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate, working])), patch.object(
-        mcp_server, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={})
-    ), patch.object(mcp_server, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object(
-        mcp_server, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None)
+    with patch.object(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate, working])), patch.object(
+        mcp_operations, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={})
+    ), patch.object(mcp_operations, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object(
+        mcp_operations, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None)
     ), patch.object(
-        mcp_server, "filter_tools_by_key_team_permissions", AsyncMock(side_effect=lambda tools, **k: tools)
+        mcp_operations, "filter_tools_by_key_team_permissions", AsyncMock(side_effect=lambda tools, **k: tools)
     ), patch.object(
-        mcp_server.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools)
+        mcp_operations.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools)
     ):
-        listing = await mcp_server._get_tools_from_mcp_servers(
+        listing = await mcp_operations._get_tools_from_mcp_servers(
             user_api_key_auth=UserAPIKeyAuth(token="h", user_id="u1"),
             mcp_auth_header=None,
             mcp_servers=None,
@@ -382,14 +383,14 @@ async def test_single_server_route_also_absorbs_upstream_auth_error():
     # /<server>/mcp sets the path-derived single-server scope; absorption must hold even then.
     token = _mcp_gateway_server_name.set("delegate_docs")
     try:
-        with patch.object(mcp_server, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate])), patch.object(
-            mcp_server, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={})
-        ), patch.object(mcp_server, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object(
-            mcp_server, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None)
+        with patch.object(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate])), patch.object(
+            mcp_operations, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={})
+        ), patch.object(mcp_operations, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object(
+            mcp_operations, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None)
         ), patch.object(
-            mcp_server.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools)
+            mcp_operations.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools)
         ):
-            listing = await mcp_server._get_tools_from_mcp_servers(
+            listing = await mcp_operations._get_tools_from_mcp_servers(
                 user_api_key_auth=UserAPIKeyAuth(token="h", user_id="u1"),
                 mcp_auth_header=None,
                 mcp_servers=["delegate_docs"],
@@ -419,15 +420,15 @@ async def test_aggregate_with_single_accessible_server_still_absorbs():
     async def fake_get_tools(server, **kwargs):
         raise MCPUpstreamAuthError(status_code=401, www_authenticate=None, server_name=server.name)
 
-    with patch.object(mcp_server, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate])), patch.object(
-        mcp_server, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={})
-    ), patch.object(mcp_server, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object(
-        mcp_server, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None)
+    with patch.object(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate])), patch.object(
+        mcp_operations, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={})
+    ), patch.object(mcp_operations, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object(
+        mcp_operations, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None)
     ), patch.object(
-        mcp_server.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools)
+        mcp_operations.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools)
     ):
         # Aggregate route: no explicit server filter, even though only one server is accessible.
-        listing = await mcp_server._get_tools_from_mcp_servers(
+        listing = await mcp_operations._get_tools_from_mcp_servers(
             user_api_key_auth=UserAPIKeyAuth(token="h", user_id="u1"),
             mcp_auth_header=None,
             mcp_servers=None,
@@ -475,3 +476,25 @@ async def test_client_creation_failure_logs_sanitized_exchange(monkeypatch, capl
             await manager._get_tools_from_server(server)
     assert "POST https://upstream/ -> HTTP 500" in caplog.text
     assert "missing_scope" in caplog.text and "query-secret" not in caplog.text
+
+
+@pytest.mark.parametrize(
+    "oauth_headers,server_headers,authorized",
+    [
+        ({"Authorization": "Bearer upstream"}, None, True),
+        ({"AUTHORIZATION": "Bearer upstream"}, None, True),
+        ({"x-unrelated": "present"}, None, False),
+        (None, {"catalog": {"Authorization": "Bearer scoped"}}, True),
+        (None, {"other-server": {"Authorization": "Bearer unrelated"}}, False),
+        (None, {"catalog": {"x-unrelated": "present"}}, False),
+        (None, {"catalog": "Bearer legacy"}, True),
+        (None, {"catalog": "   "}, False),
+    ],
+)
+def test_passthrough_admission_recognizes_only_matching_authorization(oauth_headers, server_headers, authorized):
+    from litellm.proxy._experimental.mcp_server.operations import _client_has_passthrough_authorization
+    from litellm.types.mcp import MCPTransport
+    from litellm.types.mcp_server.mcp_server_manager import MCPServer
+
+    server = MCPServer(server_id="catalog", name="catalog", alias="catalog", transport=MCPTransport.http)
+    assert _client_has_passthrough_authorization(server, oauth_headers, server_headers) is authorized
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py
index 84d4f1fd083..ed5d67164bd 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py
@@ -1,3 +1,4 @@
+from litellm.proxy._experimental.mcp_server import operations as mcp_operations
 import json
 from datetime import datetime
 
@@ -27,8 +28,8 @@ def proxy_mode():
 @pytest.mark.asyncio
 @pytest.mark.usefixtures("proxy_mode")
 async def test_proxy_call_rejects_non_proxy_tool_names() -> None:
-    result = await server._dispatch_virtual_mcp_tool(
-        name="math_stdio-add", arguments={"a": 1, "b": 2}, user_api_key_auth=AUTH, client_ip=None
+    result = await mcp_operations._dispatch_virtual_mcp_tool(
+        name="math_stdio-add", arguments={"a": 1, "b": 2}, user_api_key_auth=AUTH, client_ip=None, mcp_proxy_mode=True
     )
 
     assert result is not None
@@ -105,12 +106,13 @@ async def test_proxy_scope_exception_emits_failure_log(monkeypatch: pytest.Monke
     arguments = {"tool_id": "denied-scope", "arguments": {}}
 
     with pytest.raises(HTTPException) as denied:
-        await server._dispatch_virtual_mcp_tool(
+        await mcp_operations._dispatch_virtual_mcp_tool(
             name="call_tool",
             arguments=arguments,
             user_api_key_auth=auth,
             client_ip=None,
             mcp_servers=["ungranted"],
+            mcp_proxy_mode=True,
             raw_headers={"authorization": "Bearer raw-scope-secret", "x-litellm-call-id": "scope-denial"},
         )
 
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py
index 3668a06203c..169de9095cf 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py
@@ -1,3 +1,4 @@
+from litellm.proxy._experimental.mcp_server import operations as mcp_operations
 import asyncio
 import contextlib
 import contextvars
@@ -138,7 +139,7 @@ async def test_mcp_server_tool_call_body_contains_request_data(_mcp_request_ctx)
         mock_add_litellm_data_to_request,
     ):
         with patch(
-            "litellm.proxy._experimental.mcp_server.server.call_mcp_tool",
+            "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool",
             mock_call_mcp_tool,
         ):
             with patch(
@@ -194,7 +195,7 @@ async def test_mcp_server_tool_call_forwards_client_headers_to_logging(_mcp_requ
         mock_add_litellm_data_to_request,
     ):
         with patch(
-            "litellm.proxy._experimental.mcp_server.server.call_mcp_tool",
+            "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool",
             mock_call_mcp_tool,
         ):
             with patch("litellm.proxy.proxy_server.proxy_config", MagicMock()):
@@ -241,7 +242,7 @@ async def test_mcp_server_tool_call_strips_custom_litellm_key_header(_mcp_reques
         capturing_add_litellm_data_to_request,
     ):
         with patch(
-            "litellm.proxy._experimental.mcp_server.server.call_mcp_tool",
+            "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool",
             mock_call_mcp_tool,
         ):
             with patch("litellm.proxy.proxy_server.proxy_config", MagicMock()):
@@ -287,11 +288,11 @@ async def test_mcp_server_tool_call_relays_upstream_auth_error_as_iserror(_mcp_r
         mock_add_litellm_data_to_request,
     ):
         with patch(
-            "litellm.proxy._experimental.mcp_server.server.call_mcp_tool",
+            "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool",
             mock_call_mcp_tool,
         ):
             with patch("litellm.proxy.proxy_server.proxy_config", MagicMock()):
-                with patch("litellm.proxy._experimental.mcp_server.server.verbose_logger", mock_logger):
+                with patch("litellm.proxy._experimental.mcp_server.operations.verbose_logger", mock_logger):
                     result = await mcp_server_tool_call(_mcp_request_ctx(), _call_tool_params("test_tool", {"param": "value"}))
 
     assert result.is_error is True
@@ -867,15 +868,15 @@ async def test_get_prompts_from_mcp_servers_success():
 
     with (
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             AsyncMock(return_value=[server_a, server_b]),
         ) as mock_allowed,
         patch(
-            "litellm.proxy._experimental.mcp_server.server._prepare_mcp_server_headers",
+            "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers",
             return_value=(None, None),
         ) as mock_headers,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         ) as mock_manager,
     ):
         mock_manager.get_prompts_from_server = AsyncMock(
@@ -927,15 +928,15 @@ async def test_get_resources_from_mcp_servers_success():
 
     with (
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             AsyncMock(return_value=[server_a, server_b]),
         ) as mock_allowed,
         patch(
-            "litellm.proxy._experimental.mcp_server.server._prepare_mcp_server_headers",
+            "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers",
             return_value=(None, None),
         ) as mock_headers,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         ) as mock_manager,
     ):
         mock_manager.get_resources_from_server = AsyncMock(
@@ -992,15 +993,15 @@ async def test_get_resource_templates_from_mcp_servers_success():
 
     with (
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             AsyncMock(return_value=[server]),
         ) as mock_allowed,
         patch(
-            "litellm.proxy._experimental.mcp_server.server._prepare_mcp_server_headers",
+            "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers",
             return_value=(None, None),
         ) as mock_headers,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         ) as mock_manager,
     ):
         mock_manager.get_resource_templates_from_server = AsyncMock(
@@ -1042,15 +1043,15 @@ async def test_mcp_get_prompt_success():
 
     with (
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             AsyncMock(return_value=[server]),
         ) as mock_allowed,
         patch(
-            "litellm.proxy._experimental.mcp_server.server._prepare_mcp_server_headers",
+            "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers",
             return_value=({"Authorization": "token"}, {"X-Test": "1"}),
         ) as mock_headers,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         ) as mock_manager,
     ):
         mock_manager.get_prompt_from_server = AsyncMock(return_value=prompt_result)
@@ -1078,6 +1079,7 @@ async def test_mcp_get_prompt_success():
         mcp_auth_header={"Authorization": "token"},
         extra_headers={"X-Test": "1"},
         raw_headers=None,
+        client_ip=None,
     )
     assert result is prompt_result
 
@@ -1106,15 +1108,15 @@ async def test_mcp_read_resource_success():
 
     with (
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             AsyncMock(return_value=[server]),
         ) as mock_allowed,
         patch(
-            "litellm.proxy._experimental.mcp_server.server._prepare_mcp_server_headers",
+            "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers",
             return_value=({"Authorization": "token"}, {"X-Test": "1"}),
         ) as mock_headers,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         ) as mock_manager,
     ):
         mock_manager.read_resource_from_server = AsyncMock(return_value=read_result)
@@ -1140,6 +1142,7 @@ async def test_mcp_read_resource_success():
         mcp_auth_header={"Authorization": "token"},
         extra_headers={"X-Test": "1"},
         raw_headers=None,
+        client_ip=None,
     )
     assert result is read_result
 
@@ -1264,7 +1267,7 @@ async def test_mcp_read_resource_multiple_servers_error():
     server_b.name = "server_b"
 
     with patch(
-        "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+        "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
         AsyncMock(return_value=[server_a, server_b]),
     ) as mock_allowed:
         with pytest.raises(HTTPException) as exc_info:
@@ -1354,11 +1357,11 @@ async def test_get_tools_from_mcp_servers_continues_when_one_server_fails():
     mock_manager._get_tools_from_server = mock_get_tools_from_server
 
     with patch(
-        "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+        "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         mock_manager,
     ):
         with patch(
-            "litellm.proxy._experimental.mcp_server.server.verbose_logger",
+            "litellm.proxy._experimental.mcp_server.operations.verbose_logger",
         ) as mock_logger:
             # Test with server-specific auth headers
             mcp_server_auth_headers = {
@@ -1450,11 +1453,11 @@ async def test_get_tools_from_mcp_servers_handles_all_servers_failing():
     mock_manager._get_tools_from_server = mock_get_tools_from_server
 
     with patch(
-        "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+        "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         mock_manager,
     ):
         with patch(
-            "litellm.proxy._experimental.mcp_server.server.verbose_logger",
+            "litellm.proxy._experimental.mcp_server.operations.verbose_logger",
         ) as mock_logger:
             # Test with server-specific auth headers
             mcp_server_auth_headers = {
@@ -1524,11 +1527,11 @@ async def _denied_scoped_list(
 
     with (
         patch(  # test-quality-ok: the permission resolver is a module-level function; the suite's only seam
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             resolver,
         ),
         patch(  # test-quality-ok: the server registry is a module-level singleton; the suite's only seam
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
             mock_manager,
         ),
     ):
@@ -1575,11 +1578,11 @@ async def test_empty_scope_lists_nothing_instead_of_raising_a_nameless_denial():
 
     with (
         patch(  # test-quality-ok: the permission resolver is a module-level function; the suite's only seam
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             resolver,
         ),
         patch(  # test-quality-ok: the server registry is a module-level singleton; the suite's only seam
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
             _denied_scope_manager({"github": "srv-github"}),
         ),
     ):
@@ -1721,7 +1724,8 @@ async def test_scoped_list_agent_veto_attributed_for_differently_cased_server_na
 
 
 @pytest.mark.asyncio
-async def test_handle_list_tools_converts_permission_httpexception_to_mcp_error(_mcp_request_ctx):
+@pytest.mark.parametrize("denial_at_auth", [False, True])
+async def test_handle_list_tools_converts_permission_httpexception_to_mcp_error(_mcp_request_ctx, denial_at_auth):
     """The MCP protocol handler surfaces a permission HTTPException as a clean JSON-RPC error
     (MCPError, INVALID_REQUEST) carrying the denial message, instead of a raw 500."""
     try:
@@ -1738,10 +1742,10 @@ async def test_handle_list_tools_converts_permission_httpexception_to_mcp_error(
     with (
         patch(  # test-quality-ok: the protocol handler reads auth from module context; no injection seam
             "litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context",
-            new=AsyncMock(return_value=(None, None, None, None, None, None, None)),
+            new=AsyncMock(return_value=(None, None, None, None, None, None, None), side_effect=denial if denial_at_auth else None),
         ),
         patch(  # test-quality-ok: the listing helper is the handler's only collaborator; the suite's seam
-            "litellm.proxy._experimental.mcp_server.server._list_mcp_tools",
+            "litellm.proxy._experimental.mcp_server.operations._list_mcp_tools",
             new=AsyncMock(side_effect=denial),
         ),
     ):
@@ -1768,7 +1772,7 @@ async def test_mcp_server_tool_call_renders_denial_message_not_detail_dict(_mcp_
             new=AsyncMock(return_value=(None, None, None, None, None, None, None)),
         ),
         patch(  # test-quality-ok: the tool-call helper is the handler's only collaborator; the suite's seam
-            "litellm.proxy._experimental.mcp_server.server.call_mcp_tool",
+            "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool",
             new=AsyncMock(side_effect=denial),
         ),
     ):
@@ -1819,7 +1823,7 @@ async def test_mcp_server_tool_call_body_with_none_arguments(_mcp_request_ctx):
         mock_add_litellm_data_to_request,
     ):
         with patch(
-            "litellm.proxy._experimental.mcp_server.server.call_mcp_tool",
+            "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool",
             mock_call_mcp_tool,
         ):
             with patch(
@@ -1893,7 +1897,7 @@ async def test_concurrent_initialize_session_managers():
                 "run",
                 return_value=mock_cm_sse,
             ) as mock_sse_run,
-            patch("litellm.proxy._experimental.mcp_server.server.verbose_logger"),
+            patch("litellm.proxy._experimental.mcp_server.operations.verbose_logger"),
         ):
             # Create multiple concurrent tasks that call initialize_session_managers
             async def init_task():
@@ -2333,7 +2337,7 @@ async def test_mcp_routing_chunked_initialize_to_stateful():
             "litellm.proxy._experimental.mcp_server.server.set_auth_context",
         ),
         patch(  # test-quality-ok: registry is empty in unit tests; key owns one server
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             new_callable=AsyncMock,
             return_value=[MagicMock()],
         ),
@@ -2823,7 +2827,7 @@ async def test_initialize_request_tracks_active_session_after_response_header():
                 return_value=(owner_auth, None, None, None, None, None),
             ),
             patch(  # test-quality-ok: registry is empty in unit tests; key owns one server
-                "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+                "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
                 new_callable=AsyncMock,
                 return_value=[MagicMock()],
             ),
@@ -2976,7 +2980,7 @@ async def test_initialize_request_records_client_name_in_gateway_sessions_report
                 return_value=(owner_auth, None, None, None, None, None),
             ),
             patch(  # test-quality-ok: registry is empty in unit tests; key owns one server
-                "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+                "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
                 new_callable=AsyncMock,
                 return_value=[MagicMock()],
             ),
@@ -3451,7 +3455,7 @@ async def test_initialize_request_with_existing_session_tracks_new_session():
                 ),
             ),
             patch(  # test-quality-ok: registry is empty in unit tests; key owns one server
-                "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+                "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
                 new_callable=AsyncMock,
                 return_value=[MagicMock()],
             ),
@@ -4164,7 +4168,7 @@ async def test_mcp_routing_with_conflicting_alias_and_group_name():
 
     with (
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_allowed_mcp_servers",
             mock_get_allowed,
         ),
         patch(
@@ -4172,7 +4176,7 @@ async def test_mcp_routing_with_conflicting_alias_and_group_name():
             mock_db_lookup,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager._get_tools_from_server",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager._get_tools_from_server",
             mock_get_tools_spy,
         ),
     ):
@@ -4281,16 +4285,16 @@ async def test_oauth2_caller_headers_not_forwarded_for_migrated_server():
             side_effect=mock_fetch_tools_with_timeout,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             AsyncMock(return_value=[oauth2_server]),
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._prefetch_oauth_creds_for_user",
+            "litellm.proxy._experimental.mcp_server.operations._prefetch_oauth_creds_for_user",
             new_callable=AsyncMock,
             return_value={},
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_user_oauth_extra_headers_from_db",
+            "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db",
             new_callable=AsyncMock,
             return_value=None,
         ),
@@ -4372,7 +4376,7 @@ async def test_list_tools_single_server_unprefixed_names():
     mock_manager._get_tools_from_server = mock_get_tools_from_server
 
     with patch(
-        "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+        "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         mock_manager,
     ):
         listing = await _get_tools_from_mcp_servers(
@@ -4451,7 +4455,7 @@ async def test_list_tools_multiple_servers_prefixed_names():
     mock_manager._get_tools_from_server = mock_get_tools_from_server
 
     with patch(
-        "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+        "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         mock_manager,
     ):
         listing = await _get_tools_from_mcp_servers(
@@ -4631,7 +4635,7 @@ async def test_call_mcp_tool_user_unauthorized_access():
             AsyncMock(return_value=["allowed_server", "another_server"]),
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_id",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_id",
             side_effect=mock_get_server_by_id,
         ),
     ):
@@ -4661,11 +4665,11 @@ async def test_call_mcp_tool_scoped_denial_names_the_binding_agent():
 
     with (
         patch(  # test-quality-ok: the server registry is a module-level singleton; the suite's only seam
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_allowed_mcp_servers",
             AsyncMock(return_value=[]),
         ),
         patch(  # test-quality-ok: the permission resolver is a module-level function; the suite's only seam
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             _scope_resolver({"github": "srv-github"}),
         ),
     ):
@@ -4737,7 +4741,7 @@ async def test_call_mcp_tool_unauthorized_403_does_not_leak_server_credentials()
             AsyncMock(return_value=["allowed_server"]),
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_id",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_id",
             side_effect=mock_get_server_by_id,
         ),
     ):
@@ -4880,7 +4884,7 @@ async def test_list_tools_filters_by_key_team_permissions():
     mock_manager._get_tools_from_server = mock_get_tools_from_server
 
     with patch(
-        "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+        "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         mock_manager,
     ):
         listing = await _get_tools_from_mcp_servers(
@@ -4991,7 +4995,7 @@ async def test_list_tools_with_team_tool_permissions_inheritance():
     mock_manager._get_tools_from_server = mock_get_tools_from_server
 
     with patch(
-        "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+        "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         mock_manager,
     ):
         # Mock the team object permission retrieval
@@ -5083,7 +5087,7 @@ async def test_list_tools_with_no_tool_permissions_shows_all():
     mock_manager._get_tools_from_server = mock_get_tools_from_server
 
     with patch(
-        "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+        "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         mock_manager,
     ):
         listing = await _get_tools_from_mcp_servers(
@@ -5189,7 +5193,7 @@ async def test_list_tools_strips_prefix_when_matching_permissions():
     mock_manager._get_tools_from_server = mock_get_tools_from_server
 
     with patch(
-        "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+        "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         mock_manager,
     ):
         listing = await _get_tools_from_mcp_servers(
@@ -5631,12 +5635,12 @@ async def test_call_mcp_tool_logs_failure_via_post_call_failure_hook():
             return_value=mock_server,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers_from_mcp_server_names",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers_from_mcp_server_names",
             new_callable=AsyncMock,
             return_value=[mock_server],
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool",
+            "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool",
             new_callable=AsyncMock,
             side_effect=Exception("boom"),
         ),
@@ -5700,26 +5704,26 @@ async def test_get_tools_from_mcp_servers_logs_list_tools_to_spendlogs_when_enab
 
     with (
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             new=AsyncMock(return_value=[server_a]),
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._prepare_mcp_server_headers",
+            "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers",
             return_value=(None, None),
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         ) as mock_manager,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.filter_tools_by_allowed_tools",
+            "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_allowed_tools",
             side_effect=lambda tools, _server: tools,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.filter_tools_by_key_team_permissions",
+            "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_key_team_permissions",
             new=AsyncMock(side_effect=lambda tools, **_: tools),
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.function_setup",
+            "litellm.proxy._experimental.mcp_server.operations.function_setup",
             side_effect=_capture_function_setup,
         ),
     ):
@@ -5782,26 +5786,26 @@ async def test_get_tools_from_mcp_servers_returns_tools_when_success_logging_fai
 
     with (
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             new=AsyncMock(return_value=[server_a]),
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._prepare_mcp_server_headers",
+            "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers",
             return_value=(None, None),
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         ) as mock_manager,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.filter_tools_by_allowed_tools",
+            "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_allowed_tools",
             side_effect=lambda tools, _server: tools,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.filter_tools_by_key_team_permissions",
+            "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_key_team_permissions",
             new=AsyncMock(side_effect=lambda tools, **_: tools),
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.function_setup",
+            "litellm.proxy._experimental.mcp_server.operations.function_setup",
             return_value=(dummy_logging_obj, None),
         ),
     ):
@@ -6102,23 +6106,23 @@ async def test_get_tools_from_mcp_servers_injects_stored_oauth2_token():
 
     with (
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             new=AsyncMock(return_value=[oauth2_server]),
         ),
         patch(
             # Patch the bulk prefetch so no real DB connection is needed
-            "litellm.proxy._experimental.mcp_server.server._prefetch_oauth_creds_for_user",
+            "litellm.proxy._experimental.mcp_server.operations._prefetch_oauth_creds_for_user",
             new=AsyncMock(return_value=prefetched_creds),
         ) as mock_prefetch,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         ) as mock_manager,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.filter_tools_by_allowed_tools",
+            "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_allowed_tools",
             side_effect=lambda tools, _server: tools,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.filter_tools_by_key_team_permissions",
+            "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_key_team_permissions",
             new=AsyncMock(side_effect=lambda tools, **_: tools),
         ),
     ):
@@ -6450,7 +6454,7 @@ class TestGatewayCreateInitializationOptions:
 
         with (
             patch(
-                "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+                "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
                 new_callable=AsyncMock,
                 return_value=[scoped_server],
             ),
@@ -6480,7 +6484,7 @@ class TestGatewayCreateInitializationOptions:
         from litellm.proxy._types import UserAPIKeyAuth
 
         with patch(  # test-quality-ok: grant resolution is the input under test
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             new_callable=AsyncMock,
             return_value=[],
         ):
@@ -6506,7 +6510,7 @@ class TestGatewayCreateInitializationOptions:
         from litellm.proxy._types import UserAPIKeyAuth
 
         with patch(  # test-quality-ok: grant resolution is the input under test
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             new_callable=AsyncMock,
             return_value=[],
         ):
@@ -6531,7 +6535,7 @@ class TestGatewayCreateInitializationOptions:
         from litellm.proxy._types import UserAPIKeyAuth
 
         with patch(  # test-quality-ok: grant resolution is the input under test
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             new_callable=AsyncMock,
             return_value=[],
         ):
@@ -6587,7 +6591,7 @@ class TestGatewayCreateInitializationOptions:
                 ),
             ),
             patch(
-                "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+                "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
                 new_callable=AsyncMock,
                 return_value=[scoped_server],
             ),
@@ -6722,14 +6726,14 @@ async def test_list_tools_with_legacy_db_m2m_server_resolves_oauth2_flow():
 
     with (
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         ) as mock_manager,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.filter_tools_by_allowed_tools",
+            "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_allowed_tools",
             side_effect=lambda tools, _server: tools,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.filter_tools_by_key_team_permissions",
+            "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_key_team_permissions",
             new=AsyncMock(side_effect=lambda tools, **_: tools),
         ),
     ):
@@ -6992,7 +6996,7 @@ def _patch_delegate_resolver(server: MCPServer, *resolvable_names: str):
         return server if name in resolvable_names else None
 
     return patch(
-        "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
+        "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
         side_effect=_resolve,
     )
 
@@ -7011,7 +7015,7 @@ async def test_legacy_delegate_bare_token_is_not_probed_upstream():  # test-qual
     with (
         _patch_delegate_resolver(server, "delegate_test"),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             new=AsyncMock(return_value=[server]),
         ),
         patch(
@@ -7047,7 +7051,7 @@ async def test_legacy_delegate_dual_credentials_are_not_probed_upstream():  # te
 
     with (
         patch(  # test-quality-ok: isolate authorized-server resolution so this test targets the preflight boundary
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             new=AsyncMock(return_value=[server]),
         ),
         patch(  # test-quality-ok: the removed probe call is the security regression under test
@@ -7094,7 +7098,7 @@ async def test_oauth_passthrough_preflight_preserves_status_contract(probe_statu
 
     with (
         patch(  # test-quality-ok: isolate authorized-server resolution so this test exercises the preflight contract
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             new=AsyncMock(return_value=[server]),
         ),
         patch(  # test-quality-ok: the upstream transport boundary is the behavior being mapped to an HTTP response
@@ -7140,7 +7144,7 @@ async def test_delegate_tokenless_request_not_probed():
     with (
         _patch_delegate_resolver(server, "delegate_test"),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             new=AsyncMock(return_value=[server]),
         ),
         patch(
@@ -7173,7 +7177,7 @@ async def test_delegate_preflight_skipped_on_multi_server_routes():
     with (
         _patch_delegate_resolver(servers[0], "delegate_test", "other_server"),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             new=AsyncMock(return_value=servers),
         ),
         patch(
@@ -7216,7 +7220,7 @@ async def test_bare_authorization_never_probes_passthrough_servers():
     with (
         _patch_delegate_resolver(passthrough_server, "pt_server"),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             new=AsyncMock(return_value=[passthrough_server]),
         ),
         patch(
@@ -7262,7 +7266,7 @@ async def test_delegate_not_probed_when_named_only_via_server_id():
     with (
         _patch_delegate_resolver(server, "delegate_test"),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             new=AsyncMock(return_value=[server]),
         ),
         patch(
@@ -7295,7 +7299,7 @@ async def test_delegate_probe_not_fanned_out_to_access_group_members():
     with (
         _patch_delegate_resolver(group_member, "delegate_test"),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             new=AsyncMock(return_value=[group_member]),
         ),
         patch(
@@ -7391,11 +7395,11 @@ async def test_execute_mcp_tool_rest_server_id_authoritative_for_unprefixed_tool
 
     with (
         patch.dict(
-            mcp_module.global_mcp_server_manager.tool_name_to_mcp_server_name_mapping,
+            mcp_operations.global_mcp_server_manager.tool_name_to_mcp_server_name_mapping,
             {"echo": oauth_server.name},
         ),
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "get_registry",
             return_value={
                 api_key_server.server_id: api_key_server,
@@ -7403,13 +7407,12 @@ async def test_execute_mcp_tool_rest_server_id_authoritative_for_unprefixed_tool
             },
         ),
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "_get_mcp_server_from_tool_name",
             return_value=oauth_server,
         ),
         patch.object(
-            mcp_module,
-            "_handle_managed_mcp_tool",
+            mcp_operations, "_handle_managed_mcp_tool",
             new=fake_handle_managed_mcp_tool,
         ),
         patch.object(
@@ -7418,12 +7421,12 @@ async def test_execute_mcp_tool_rest_server_id_authoritative_for_unprefixed_tool
             return_value=True,
         ),
         patch.object(
-            mcp_module.global_mcp_tool_registry,
+            mcp_operations.global_mcp_tool_registry,
             "get_tool",
             return_value=None,
         ),
     ):
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name="echo",
             arguments={"message": "hello"},
             allowed_mcp_servers=[api_key_server, oauth_server],
@@ -7456,7 +7459,7 @@ def _worker_that_never_listed(server: MCPServer, upstream_tools: tuple[str, ...]
 
     from litellm.proxy._experimental.mcp_server import server as mcp_module
 
-    mcp_module.global_mcp_server_manager.registry[server.server_id] = server
+    mcp_operations.global_mcp_server_manager.registry[server.server_id] = server
     dispatched: dict[str, object] = {}
 
     async def fake_handle_managed_mcp_tool(**kwargs):
@@ -7468,17 +7471,17 @@ def _worker_that_never_listed(server: MCPServer, upstream_tools: tuple[str, ...]
 
     with (
         patch.object(  # test-quality-ok: the upstream MCP session is the boundary; a real one needs an initialize handshake over a live server
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "_create_mcp_client",
             new=AsyncMock(return_value=MagicMock()),
         ) as create_client,
         patch.object(  # test-quality-ok: same boundary, this is the tools/list answer the upstream would give
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "_fetch_tools_with_timeout",
             side_effect=fake_fetch_tools,
         ) as fetch_tools,
         patch.object(  # test-quality-ok: records the resolved server and bare name the managed call would forward upstream
-            mcp_module, "_handle_managed_mcp_tool", new=fake_handle_managed_mcp_tool
+            mcp_operations, "_handle_managed_mcp_tool", new=fake_handle_managed_mcp_tool
         ),
     ):
         yield SimpleNamespace(create_client=create_client, fetch_tools=fetch_tools, dispatched=dispatched)
@@ -7492,7 +7495,7 @@ async def test_execute_mcp_tool_lists_never_listed_passthrough_server_with_calle
 
     server = _never_listed_passthrough_server()
     with _worker_that_never_listed(server, upstream_tools=("add",)) as worker:
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name="lazy_map-add",
             arguments={"a": 1, "b": 2},
             allowed_mcp_servers=[server],
@@ -7513,7 +7516,7 @@ async def test_execute_mcp_tool_rest_server_id_lists_never_listed_server_first()
 
     server = _never_listed_passthrough_server()
     with _worker_that_never_listed(server, upstream_tools=("add",)) as worker:
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name="add",
             arguments={"a": 1, "b": 2},
             allowed_mcp_servers=[server],
@@ -7536,7 +7539,7 @@ async def test_execute_mcp_tool_unknown_tool_on_never_listed_server_lists_once_t
         _worker_that_never_listed(server, upstream_tools=("add",)) as worker,
         pytest.raises(HTTPException) as exc_info,
     ):
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name="lazy_map-nope",
             arguments={},
             allowed_mcp_servers=[server],
@@ -7557,8 +7560,8 @@ async def test_execute_mcp_tool_does_not_relist_a_server_this_worker_already_lis
 
     server = _never_listed_passthrough_server()
     with _worker_that_never_listed(server, upstream_tools=("add",)) as worker:
-        mcp_module.global_mcp_server_manager._create_prefixed_tools([MCPTool(name="add", inputSchema={})], server)
-        await mcp_module.execute_mcp_tool(
+        mcp_operations.global_mcp_server_manager._create_prefixed_tools([MCPTool(name="add", inputSchema={})], server)
+        await mcp_operations.execute_mcp_tool(
             name="lazy_map-add",
             arguments={"a": 1, "b": 2},
             allowed_mcp_servers=[server],
@@ -7580,8 +7583,8 @@ async def test_execute_mcp_tool_lists_a_tool_this_worker_has_not_yet_seen_on_a_l
 
     server = _never_listed_passthrough_server()
     with _worker_that_never_listed(server, upstream_tools=("add", "multiply")) as worker:
-        mcp_module.global_mcp_server_manager._create_prefixed_tools([MCPTool(name="add", inputSchema={})], server)
-        await mcp_module.execute_mcp_tool(
+        mcp_operations.global_mcp_server_manager._create_prefixed_tools([MCPTool(name="add", inputSchema={})], server)
+        await mcp_operations.execute_mcp_tool(
             name="lazy_map-multiply",
             arguments={"a": 1, "b": 2},
             allowed_mcp_servers=[server],
@@ -7604,7 +7607,7 @@ async def test_execute_mcp_tool_never_lists_a_server_the_caller_cannot_access():
         _worker_that_never_listed(server, upstream_tools=("add",)) as worker,
         pytest.raises(HTTPException) as exc_info,
     ):
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name="lazy_map-add",
             arguments={"a": 1, "b": 2},
             allowed_mcp_servers=[other_server],
@@ -7650,13 +7653,12 @@ async def test_execute_mcp_tool_strips_a_prefix_that_contains_the_separator():
 
     with (
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "_get_mcp_server_from_tool_name",
             return_value=alias_less_server,
         ),
         patch.object(
-            mcp_module,
-            "_handle_managed_mcp_tool",
+            mcp_operations, "_handle_managed_mcp_tool",
             new=fake_handle_managed_mcp_tool,
         ),
         patch.object(
@@ -7665,12 +7667,12 @@ async def test_execute_mcp_tool_strips_a_prefix_that_contains_the_separator():
             return_value=True,
         ),
         patch.object(
-            mcp_module.global_mcp_tool_registry,
+            mcp_operations.global_mcp_tool_registry,
             "get_tool",
             return_value=None,
         ),
     ):
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name=f"{server_id}-read_wiki_contents",
             arguments={"repoName": "acme/wiki"},
             allowed_mcp_servers=[alias_less_server],
@@ -7724,11 +7726,11 @@ async def test_execute_mcp_tool_rest_server_id_injects_requested_server_credenti
 
     with (
         patch.dict(
-            mcp_module.global_mcp_server_manager.tool_name_to_mcp_server_name_mapping,
+            mcp_operations.global_mcp_server_manager.tool_name_to_mcp_server_name_mapping,
             {"echo": collision_server.name, "echo_requested-echo": requested_server.name},
         ),
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "get_registry",
             return_value={
                 requested_server.server_id: requested_server,
@@ -7736,7 +7738,7 @@ async def test_execute_mcp_tool_rest_server_id_injects_requested_server_credenti
             },
         ),
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "_create_mcp_client",
             new=fake_create_mcp_client,
         ),
@@ -7746,13 +7748,13 @@ async def test_execute_mcp_tool_rest_server_id_injects_requested_server_credenti
             return_value=True,
         ),
         patch.object(
-            mcp_module.global_mcp_tool_registry,
+            mcp_operations.global_mcp_tool_registry,
             "get_tool",
             return_value=None,
         ),
         patch("litellm.proxy.proxy_server.proxy_logging_obj", None),
     ):
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name="echo",
             arguments={"message": "hello"},
             allowed_mcp_servers=[requested_server, collision_server],
@@ -7795,7 +7797,7 @@ async def test_execute_mcp_tool_rest_prefixed_tool_still_validates_server_id():
 
     with (
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "get_registry",
             return_value={
                 api_key_server.server_id: api_key_server,
@@ -7803,7 +7805,7 @@ async def test_execute_mcp_tool_rest_prefixed_tool_still_validates_server_id():
             },
         ),
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "_get_mcp_server_from_tool_name",
             return_value=oauth_server,
         ),
@@ -7813,13 +7815,13 @@ async def test_execute_mcp_tool_rest_prefixed_tool_still_validates_server_id():
             return_value=True,
         ),
         patch.object(
-            mcp_module.global_mcp_tool_registry,
+            mcp_operations.global_mcp_tool_registry,
             "get_tool",
             return_value=None,
         ),
         pytest.raises(HTTPException) as exc_info,
     ):
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name="echo_oauth_m2m-echo",
             arguments={"message": "hello"},
             allowed_mcp_servers=[api_key_server, oauth_server],
@@ -7857,7 +7859,7 @@ async def test_execute_mcp_tool_rest_unauthorized_prefix_still_mismatches():
 
     with (
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "get_registry",
             return_value={
                 api_key_server.server_id: api_key_server,
@@ -7865,7 +7867,7 @@ async def test_execute_mcp_tool_rest_unauthorized_prefix_still_mismatches():
             },
         ),
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "_get_mcp_server_from_tool_name",
             return_value=restricted_server,
         ),
@@ -7875,13 +7877,13 @@ async def test_execute_mcp_tool_rest_unauthorized_prefix_still_mismatches():
             return_value=True,
         ),
         patch.object(
-            mcp_module.global_mcp_tool_registry,
+            mcp_operations.global_mcp_tool_registry,
             "get_tool",
             return_value=None,
         ),
         pytest.raises(HTTPException) as exc_info,
     ):
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name="restricted_server-echo",
             arguments={"message": "hello"},
             allowed_mcp_servers=[api_key_server],
@@ -7921,18 +7923,17 @@ async def test_execute_mcp_tool_rest_hyphenated_upstream_tool_name_routes_to_req
 
     with (
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "get_registry",
             return_value={api_key_server.server_id: api_key_server},
         ),
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "_get_mcp_server_from_tool_name",
             return_value=None,
         ),
         patch.object(
-            mcp_module,
-            "_handle_managed_mcp_tool",
+            mcp_operations, "_handle_managed_mcp_tool",
             new=fake_handle_managed_mcp_tool,
         ),
         patch.object(
@@ -7941,12 +7942,12 @@ async def test_execute_mcp_tool_rest_hyphenated_upstream_tool_name_routes_to_req
             return_value=True,
         ),
         patch.object(
-            mcp_module.global_mcp_tool_registry,
+            mcp_operations.global_mcp_tool_registry,
             "get_tool",
             return_value=None,
         ),
     ):
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name="text-to-speech",
             arguments={"message": "hello"},
             allowed_mcp_servers=[api_key_server],
@@ -8005,22 +8006,22 @@ async def test_execute_mcp_tool_sets_model_in_model_call_details():
 
     with (
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "_get_mcp_server_from_tool_name",
             return_value=fake_server,
         ),
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "pre_call_tool_check",
             new=AsyncMock(return_value={}),
         ),
         patch.object(
-            mcp_module.global_mcp_tool_registry,
+            mcp_operations.global_mcp_tool_registry,
             "get_tool",
             return_value=fake_tool,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool",
+            "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool",
             new=AsyncMock(return_value=[]),
         ),
         patch(
@@ -8028,7 +8029,7 @@ async def test_execute_mcp_tool_sets_model_in_model_call_details():
             return_value=True,
         ),
     ):
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name="list_pets",
             arguments={"limit": 10},
             allowed_mcp_servers=[fake_server],
@@ -8084,7 +8085,7 @@ async def test_execute_mcp_tool_rest_unresolved_prefixed_name_routes_to_requeste
 
     with (
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "get_registry",
             return_value={
                 requested_server.server_id: requested_server,
@@ -8092,13 +8093,12 @@ async def test_execute_mcp_tool_rest_unresolved_prefixed_name_routes_to_requeste
             },
         ),
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "_get_mcp_server_from_tool_name",
             return_value=None,
         ),
         patch.object(
-            mcp_module,
-            "_handle_managed_mcp_tool",
+            mcp_operations, "_handle_managed_mcp_tool",
             new=fake_handle_managed_mcp_tool,
         ),
         patch.object(
@@ -8107,12 +8107,12 @@ async def test_execute_mcp_tool_rest_unresolved_prefixed_name_routes_to_requeste
             return_value=True,
         ),
         patch.object(
-            mcp_module.global_mcp_tool_registry,
+            mcp_operations.global_mcp_tool_registry,
             "get_tool",
             return_value=None,
         ),
     ):
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name="known_prefix-list_things",
             arguments={"message": "hello"},
             allowed_mcp_servers=[requested_server, prefix_owner],
@@ -8164,7 +8164,7 @@ async def test_execute_mcp_tool_rest_prefix_retry_resolution_still_enforces_serv
 
     with (
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "get_registry",
             return_value={
                 requested_server.server_id: requested_server,
@@ -8172,7 +8172,7 @@ async def test_execute_mcp_tool_rest_prefix_retry_resolution_still_enforces_serv
             },
         ),
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "_get_mcp_server_from_tool_name",
             side_effect=resolve_only_when_requested_prefix_added,
         ),
@@ -8182,13 +8182,13 @@ async def test_execute_mcp_tool_rest_prefix_retry_resolution_still_enforces_serv
             return_value=True,
         ),
         patch.object(
-            mcp_module.global_mcp_tool_registry,
+            mcp_operations.global_mcp_tool_registry,
             "get_tool",
             return_value=None,
         ),
         pytest.raises(HTTPException) as exc_info,
     ):
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name="known_prefix-echo",
             arguments={"message": "hello"},
             allowed_mcp_servers=[requested_server, prefix_owner],
@@ -9091,14 +9091,14 @@ async def test_call_tool_with_legacy_db_m2m_server_resolves_oauth2_flow():
 
     with (
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         ) as mock_manager,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool",
+            "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool",
             side_effect=capture_execute,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers_from_mcp_server_names",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers_from_mcp_server_names",
             new=AsyncMock(side_effect=lambda mcp_servers, allowed_mcp_servers: allowed_mcp_servers),
         ),
     ):
@@ -9176,12 +9176,12 @@ async def test_call_mcp_tool_skips_failure_hook_for_upstream_auth_error():
         ),
         patch.object(global_mcp_server_manager, "get_mcp_server_by_id", return_value=mock_server),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers_from_mcp_server_names",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers_from_mcp_server_names",
             new_callable=AsyncMock,
             return_value=[mock_server],
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool",
+            "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool",
             new_callable=AsyncMock,
             side_effect=MCPUpstreamAuthError(status_code=401, www_authenticate="Bearer", server_name="test_server"),
         ),
@@ -9261,7 +9261,7 @@ async def test_aggregate_listing_reports_per_server_outcomes():
     mock_manager._get_tools_from_server = mock_get_tools_from_server
 
     with patch(
-        "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+        "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
         mock_manager,
     ):
         listing = await _get_tools_from_mcp_servers(
@@ -9335,7 +9335,7 @@ async def test_handle_list_tools_attaches_outcome_meta(_mcp_request_ctx):
             new=AsyncMock(return_value=(None, None, None, None, None, None, None)),
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._list_mcp_tools",
+            "litellm.proxy._experimental.mcp_server.operations._list_mcp_tools",
             new=AsyncMock(return_value=listing),
         ),
     ):
@@ -9401,12 +9401,12 @@ class TestPreemptive401ModeAware:
 
         with (
             patch.object(
-                server_module.global_mcp_server_manager,
+                mcp_operations.global_mcp_server_manager,
                 "get_mcp_server_by_name",
                 return_value=server,
             ),
             patch.object(
-                server_module.global_mcp_server_manager,
+                mcp_operations.global_mcp_server_manager,
                 "has_user_oauth_token",
                 new_callable=AsyncMock,
                 return_value=has_stored_token,
@@ -9425,7 +9425,7 @@ class TestPreemptive401ModeAware:
     async def test_deferred_discovery_runs_before_delegate_challenge(self):
         from litellm.proxy._experimental.mcp_server import server as server_module
 
-        manager = server_module.global_mcp_server_manager
+        manager = mcp_operations.global_mcp_server_manager
         server = _make_oauth2_server(
             "lazy_delegate",
             oauth2_flow="authorization_code",
@@ -9457,7 +9457,7 @@ class TestPreemptive401ModeAware:
     async def test_stamped_m2m_challenge_skips_deferred_discovery(self):
         from litellm.proxy._experimental.mcp_server import server as server_module
 
-        manager = server_module.global_mcp_server_manager
+        manager = mcp_operations.global_mcp_server_manager
         server = _make_oauth2_server("stamped_m2m", oauth2_flow="client_credentials")
 
         with patch.object(
@@ -9500,12 +9500,12 @@ class TestPreemptive401ModeAware:
         with (
             patch.dict(os.environ, {"SERVER_ROOT_PATH": "/litellm"}),
             patch.object(
-                server_module.global_mcp_server_manager,
+                mcp_operations.global_mcp_server_manager,
                 "get_mcp_server_by_name",
                 return_value=server,
             ),
             patch.object(
-                server_module.global_mcp_server_manager,
+                mcp_operations.global_mcp_server_manager,
                 "has_user_oauth_token",
                 new_callable=AsyncMock,
                 return_value=False,
@@ -9607,17 +9607,17 @@ class TestSingleServerPreflightReachesIdJag:
 
         with (
             patch.object(  # test-quality-ok: route wiring must use the manager's configured server
-                server_module.global_mcp_server_manager,
+                mcp_operations.global_mcp_server_manager,
                 "get_mcp_server_by_name",
                 return_value=server,
             ),
             patch.object(  # test-quality-ok: route wiring must invoke the manager preflight
-                server_module.global_mcp_server_manager,
+                mcp_operations.global_mcp_server_manager,
                 "preflight_token_exchange",
                 preflight,
             ),
             patch.object(  # test-quality-ok: allowed-set resolution needs the DB; the test controls its answer
-                server_module, "_get_allowed_mcp_servers", AsyncMock(return_value=[server])
+                mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[server])
             ),
         ):
             await server_module._raise_preemptive_401_for_unauthenticated_servers(
@@ -9667,12 +9667,12 @@ class TestSingleServerPreflightReachesIdJag:
 
         with (
             patch.object(  # test-quality-ok: route wiring must use the manager's configured server
-                server_module.global_mcp_server_manager,
+                mcp_operations.global_mcp_server_manager,
                 "get_mcp_server_by_name",
                 return_value=token_exchange,
             ),
             patch.object(  # test-quality-ok: route wiring must invoke the manager preflight
-                server_module.global_mcp_server_manager,
+                mcp_operations.global_mcp_server_manager,
                 "preflight_token_exchange",
                 preflight,
             ),
@@ -9733,13 +9733,13 @@ class TestOboPreflightScopedToAllowedServers:
         preflight = AsyncMock()
         with (
             patch.object(  # test-quality-ok: route handler reads the module-level manager, no injection seam
-                server_module.global_mcp_server_manager, "get_mcp_server_by_name", return_value=requested
+                mcp_operations.global_mcp_server_manager, "get_mcp_server_by_name", return_value=requested
             ),
             patch.object(  # test-quality-ok: the exchanger is the observable; a real one would call an IdP
-                server_module.global_mcp_server_manager, "preflight_token_exchange", preflight
+                mcp_operations.global_mcp_server_manager, "preflight_token_exchange", preflight
             ),
             patch.object(  # test-quality-ok: allowed-set resolution needs the DB; the test controls its answer
-                server_module, "_get_allowed_mcp_servers", allowed_lookup
+                mcp_operations, "_get_allowed_mcp_servers", allowed_lookup
             ),
         ):
             await server_module._raise_preemptive_401_for_unauthenticated_servers(
@@ -10048,7 +10048,7 @@ class TestListFiltersHonorThePrefixBoundary:
 
         with (
             patch.object(MCPRequestHandler, "get_allowed_tools_for_server", AsyncMock(return_value=grants)),
-            patch("litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager") as mock_manager,
+            patch("litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager") as mock_manager,
         ):
             mock_manager.get_mcp_server_by_id.return_value = server
 
@@ -10111,11 +10111,11 @@ async def test_list_tools_injects_byok_credential_for_non_oauth2_auth_types(auth
 
     with (
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
             mock_manager,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_byok_credential",
+            "litellm.proxy._experimental.mcp_server.operations._get_byok_credential",
             AsyncMock(return_value="personal-api-key"),
         ),
     ):
@@ -10208,3 +10208,44 @@ async def test_streamable_http_rejects_modern_protocol_version(header_value: str
     assert header_value in body["error"]["message"]
     for version in body["error"]["message"].split("supported: ")[1].split(", "):
         assert version in HANDSHAKE_PROTOCOL_VERSIONS
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("handler_name,field", [
+    ("handle_list_tools", "tools"),
+    ("list_prompts", "prompts"),
+    ("list_resources", "resources"),
+    ("list_resource_templates", "resource_templates"),
+])
+async def test_native_listing_preserves_empty_result_on_auth_failure(_mcp_request_ctx, handler_name, field):
+    from litellm.proxy._experimental.mcp_server import server
+
+    with patch.object(server, "get_or_extract_auth_context", AsyncMock(side_effect=RuntimeError("auth failure"))):
+        result = await getattr(server, handler_name)(_mcp_request_ctx(), _paged_params())
+    assert getattr(result, field) == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("failure_hook_raises", [False, True])
+async def test_tool_listing_preserves_permission_denial_when_failure_logging_fails(failure_hook_raises):
+    from litellm.proxy._experimental.mcp_server import operations
+    from litellm.proxy import proxy_server
+
+    auth = UserAPIKeyAuth(user_id="denied-caller")
+    denial = HTTPException(status_code=403, detail="scope denied")
+    logger = MagicMock()
+    logger.post_call_failure_hook = AsyncMock(side_effect=RuntimeError("log unavailable") if failure_hook_raises else None)
+    upstream = AsyncMock()
+    with (
+        patch.object(operations, "_get_allowed_mcp_servers", AsyncMock(side_effect=denial)),
+        patch.object(operations, "function_setup", return_value=(None, None)),
+        patch.object(proxy_server, "proxy_logging_obj", logger),
+        patch.object(operations.global_mcp_server_manager, "_get_tools_from_server", upstream),
+    ):
+        with pytest.raises(HTTPException) as rejected:
+            await operations._get_tools_from_mcp_servers(user_api_key_auth=auth, mcp_auth_header=None, mcp_servers=["catalog"], log_list_tools_to_spendlogs=True)
+    assert rejected.value is denial
+    upstream.assert_not_awaited()
+    logger.post_call_failure_hook.assert_awaited_once()
+    assert logger.post_call_failure_hook.await_args.kwargs["original_exception"] is denial
+    assert logger.post_call_failure_hook.await_args.kwargs["user_api_key_dict"] == auth
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
index 9140ac61f1a..7be79f9b514 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
@@ -71,6 +71,135 @@ from litellm.proxy.utils import ProxyLogging
 from litellm.types.guardrails import GuardrailEventHooks
 
 
+@pytest.mark.asyncio
+async def test_manager_sampling_preserves_explicit_headers_without_ambient_context():
+    from litellm.proxy._experimental.mcp_server import server as legacy_server
+
+    caller = UserAPIKeyAuth(user_id="sampling-caller")
+    upstream = MCPServer(
+        server_id="sampling-context",
+        name="sampling_context",
+        url="https://example.invalid/mcp",
+        transport=MCPTransport.http,
+        allow_sampling=True,
+    )
+    sampling = AsyncMock()
+    client = MagicMock()
+    client.call_tool = AsyncMock(return_value=CallToolResult(content=[]))
+    assert legacy_server.get_active_auth_context() is None
+    with (
+        patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPClient", return_value=client) as factory,
+        patch("litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", sampling),
+    ):
+        await MCPServerManager()._call_regular_mcp_tool(
+            mcp_server=upstream,
+            original_tool_name="probe",
+            arguments={},
+            tasks=[],
+            mcp_auth_header=None,
+            mcp_server_auth_headers=None,
+            oauth2_headers=None,
+            raw_headers={"x-test-caller": "sampling-caller"},
+            proxy_logging_obj=None,
+            user_api_key_auth=caller,
+        )
+        callback = factory.call_args.kwargs["sampling_callback"]
+        await callback(None, None)
+    assert sampling.await_args.kwargs["user_api_key_auth"].user_id == "sampling-caller"
+    assert sampling.await_args.kwargs["raw_headers"] == {"x-test-caller": "sampling-caller"}
+
+
+
+@pytest.mark.asyncio
+async def test_sampling_callback_keeps_creation_context_after_caller_switch():
+    from mcp.server.auth.middleware.auth_context import auth_context_var
+
+    from litellm.proxy._experimental.mcp_server import server as legacy_server
+    from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_sampling_callback
+
+    token = auth_context_var.set(None)
+    recorder = AsyncMock()
+    try:
+        original = UserAPIKeyAuth(user_id="alpha", models=["alpha-model"])
+        original.mcp_admitted_user_subject = True
+        headers = {"x-caller": "alpha"}
+        legacy_server.set_auth_context(original, raw_headers=headers, client_ip="192.0.2.1")
+        callback = _create_sampling_callback()
+        original.models.append("bravo-model")
+        headers["x-caller"] = "bravo"
+        legacy_server.set_auth_context(UserAPIKeyAuth(user_id="bravo"), raw_headers={"x-caller": "bravo"})
+        with patch("litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", recorder):
+            await callback(None, None)
+        observed = recorder.await_args.kwargs
+        assert observed["user_api_key_auth"].user_id == "alpha"
+        assert observed["user_api_key_auth"].models == ["alpha-model"]
+        assert observed["user_api_key_auth"].mcp_admitted_user_subject is True
+        assert observed["raw_headers"] == {"x-caller": "alpha"}
+        assert observed["client_ip"] == "192.0.2.1"
+    finally:
+        auth_context_var.reset(token)
+
+
+@pytest.mark.asyncio
+async def test_elicitation_callback_keeps_initiating_session():
+    from litellm.proxy._experimental.mcp_server import server as legacy_server
+    from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_elicitation_callback
+
+    initiating = MagicMock()
+    replacement = MagicMock()
+    recorder = AsyncMock()
+    token = legacy_server.active_mcp_session_var.set(initiating)
+    try:
+        callback = _create_elicitation_callback()
+        legacy_server.active_mcp_session_var.set(replacement)
+        with patch("litellm.proxy._experimental.mcp_server.elicitation_handler.handle_elicitation_request", recorder):
+            await callback(None, None)
+        assert recorder.await_args.kwargs["downstream_session"] is initiating
+        assert recorder.await_args.kwargs["downstream_capabilities"] is initiating.capabilities
+    finally:
+        legacy_server.active_mcp_session_var.reset(token)
+
+
+@pytest.mark.asyncio
+async def test_sampling_callbacks_isolate_callers_and_cancellation():
+    from mcp.types import ErrorData
+
+    from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_sampling_callback
+
+    started = asyncio.Event()
+    cancelled = asyncio.Event()
+    observed = {}
+
+    async def record_sampling(*, user_api_key_auth, raw_headers, **kwargs):
+        label = user_api_key_auth.user_id
+        if label == "cancelled":
+            started.set()
+            try:
+                await asyncio.Event().wait()
+            except asyncio.CancelledError:
+                cancelled.set()
+                raise
+        await asyncio.sleep(0)
+        observed[label] = raw_headers["x-caller"]
+        return ErrorData(code=-1, message=label)
+
+    callbacks = tuple(
+        _create_sampling_callback(UserAPIKeyAuth(user_id=label), raw_headers={"x-caller": label})
+        for label in ("alpha", "bravo", "cancelled")
+    )
+    with patch(
+        "litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", record_sampling
+    ):
+        tasks = tuple(asyncio.create_task(callback(None, None)) for callback in callbacks)
+        await asyncio.wait_for(started.wait(), timeout=2)
+        tasks[2].cancel()
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+    assert observed == {"alpha": "alpha", "bravo": "bravo"}
+    assert [result.message for result in results[:2]] == ["alpha", "bravo"]
+    assert isinstance(results[2], asyncio.CancelledError)
+    assert cancelled.is_set()
+
+
 def _reload_mcp_manager_module():
     utils_module = sys.modules["litellm.proxy._experimental.mcp_server.utils"]
     manager_module = sys.modules["litellm.proxy._experimental.mcp_server.mcp_server_manager"]
@@ -82,6 +211,9 @@ def _reload_mcp_manager_module():
     server_module = sys.modules.get("litellm.proxy._experimental.mcp_server.server")
     if server_module is not None and hasattr(server_module, "global_mcp_server_manager"):
         server_module.global_mcp_server_manager = reloaded.global_mcp_server_manager
+    operations_module = sys.modules.get("litellm.proxy._experimental.mcp_server.operations")
+    if operations_module is not None:
+        operations_module.global_mcp_server_manager = reloaded.global_mcp_server_manager
     return reloaded
 
 
@@ -3921,6 +4053,7 @@ class TestMCPServerManager:
             result = await manager.get_resource_templates_from_server(
                 server=server,
                 user_api_key_auth=None,
+                raw_headers=None,
                 mcp_auth_header="auth",
                 extra_headers=None,
                 add_prefix=False,
@@ -3933,6 +4066,8 @@ class TestMCPServerManager:
             stdio_env=None,
             subject_token=None,
             user_api_key_auth=None,
+            raw_headers=None,
+            client_ip=None,
         )
         mock_client.list_resource_templates.assert_awaited_once()
         assert result == expected_templates
@@ -5847,7 +5982,7 @@ class TestMCPServerManager:
         stored = {"Authorization": "Bearer stored-user-token"}
 
         with patch(
-            "litellm.proxy._experimental.mcp_server.server._get_user_oauth_extra_headers_from_db",
+            "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db",
             new=AsyncMock(return_value=stored),
         ) as mock_lookup:
             result = await manager._resolve_oauth2_headers_for_tool_call(
@@ -5874,7 +6009,7 @@ class TestMCPServerManager:
         user_auth = UserAPIKeyAuth(api_key="sk-test", user_id="alice")
 
         with patch(
-            "litellm.proxy._experimental.mcp_server.server._get_user_oauth_extra_headers_from_db",
+            "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db",
             new=AsyncMock(return_value={"Authorization": "Bearer should-not-be-used"}),
         ) as mock_lookup:
             result = await manager._resolve_oauth2_headers_for_tool_call(
@@ -5900,7 +6035,7 @@ class TestMCPServerManager:
         user_auth = UserAPIKeyAuth(api_key="sk-test", user_id="alice")
 
         with patch(
-            "litellm.proxy._experimental.mcp_server.server._get_user_oauth_extra_headers_from_db",
+            "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db",
             new=AsyncMock(side_effect=RuntimeError("redis down")),
         ):
             result = await manager._resolve_oauth2_headers_for_tool_call(
@@ -6056,7 +6191,7 @@ class TestMCPServerManager:
         user_auth = UserAPIKeyAuth(api_key="sk-test")
 
         with patch(
-            "litellm.proxy._experimental.mcp_server.server._get_user_oauth_extra_headers_from_db",
+            "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db",
             new=AsyncMock(return_value={"Authorization": "Bearer x"}),
         ) as mock_lookup:
             result = await manager._resolve_oauth2_headers_for_tool_call(
@@ -6860,7 +6995,8 @@ class TestMCPServerManager:
         }
         user_api_key_auth = UserAPIKeyAuth(api_key="sk-test", user_id="user-123")
 
-        token = _mcp_active_toolset_id.set("toolset-abc")
+        user_api_key_auth.mcp_toolset_id = "toolset-abc"
+        token = _mcp_active_toolset_id.set("unrelated-ambient-toolset")
         try:
             with (
                 patch.object(proxy_server_module, "user_api_key_cache", cache),
@@ -14075,3 +14211,31 @@ async def test_request_selected_during_guardrail_runs_concurrently_with_tool(mon
     assert guardrail_started.is_set() is selected
     assert result.is_error is False
     assert result.content[0].text == "executed"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("with_caller", [True, False])
+async def test_client_sampling_does_not_fill_explicit_context_from_another_ambient_caller(with_caller):
+    from mcp.server.auth.middleware.auth_context import auth_context_var
+    from litellm.proxy._experimental.mcp_server import server as legacy_server
+
+    upstream = MCPServer(server_id="explicit-empty", name="explicit_empty", url="https://example.invalid/mcp", transport=MCPTransport.http, allow_sampling=True)
+    token = auth_context_var.set(None)
+    sampling = AsyncMock()
+    try:
+        legacy_server.set_auth_context(UserAPIKeyAuth(user_id="unrelated"), raw_headers={"authorization": "unrelated-credential"}, client_ip="192.0.2.99")
+        with (
+            patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPClient") as factory,
+            patch("litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", sampling),
+        ):
+            await MCPServerManager()._create_mcp_client(upstream, user_api_key_auth=UserAPIKeyAuth(user_id="explicit") if with_caller else None)
+            await factory.call_args.kwargs["sampling_callback"](None, None)
+        captured = sampling.await_args.kwargs
+        if with_caller:
+            assert captured["user_api_key_auth"].user_id == "explicit"
+        else:
+            assert captured["user_api_key_auth"] is None
+        assert captured["raw_headers"] is None
+        assert captured["client_ip"] is None
+    finally:
+        auth_context_var.reset(token)
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py
index 9420eecd222..ec6fdef69ee 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py
@@ -639,12 +639,12 @@ async def test_per_user_oauth_missing_stored_token_returns_preemptive_401():
             return_value=False,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.has_user_oauth_token",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token",
             new_callable=AsyncMock,
             return_value=False,
         ) as mock_has_token,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
             return_value=oauth_server,
         ),
         patch.object(
@@ -727,12 +727,12 @@ async def test_admitted_subject_missing_stored_token_challenged_with_resource_me
             return_value=False,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.has_user_oauth_token",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token",
             new_callable=AsyncMock,
             return_value=False,
         ) as mock_has_token,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
             return_value=oauth_server,
         ),
         patch.object(
@@ -833,11 +833,11 @@ async def test_client_credentials_server_is_not_preemptively_challenged(m2m_fiel
             return_value=False,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.has_user_oauth_token",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token",
             new_callable=AsyncMock,
         ) as mock_has_token,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
             return_value=m2m_server,
         ),
         patch.object(session_manager_stateless, "handle_request", new_callable=AsyncMock) as mock_handle_request,
@@ -929,16 +929,16 @@ async def test_handle_streamable_http_mcp_delegated_server_surfaces_upstream_cha
             return_value=False,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._get_user_oauth_extra_headers_from_db",
+            "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db",
             new_callable=AsyncMock,
             return_value=None,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
             return_value=delegated_server,
         ),
         patch(  # test-quality-ok: registry is empty in unit tests; key owns the delegated server
-            "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+            "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
             new_callable=AsyncMock,
             return_value=[delegated_server],
         ),
@@ -1022,12 +1022,12 @@ async def test_per_user_oauth_with_stored_token_skips_preemptive_401():
             return_value=False,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.has_user_oauth_token",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token",
             new_callable=AsyncMock,
             return_value=True,
         ) as mock_has_token,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
             return_value=oauth_server,
         ),
         patch.object(
@@ -1126,11 +1126,11 @@ async def test_handle_streamable_http_mcp_delegated_server_without_token_returns
             return_value=False,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.has_user_oauth_token",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token",
             new_callable=AsyncMock,
         ) as mock_has_token,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
             return_value=delegated_server,
         ),
         patch.object(
@@ -1218,7 +1218,7 @@ async def test_handle_streamable_http_mcp_token_exchange_without_subject_returns
             return_value=False,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
             return_value=obo_server,
         ),
         patch.object(
@@ -1317,7 +1317,7 @@ async def test_handle_streamable_http_mcp_oauth_delegate_without_token_returns_g
             True,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
             return_value=od_server,
         ),
         patch.object(
@@ -1391,7 +1391,7 @@ async def test_handle_streamable_http_mcp_oauth_delegate_with_forwarded_token_sk
             new_callable=AsyncMock,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
             return_value=od_server,
         ),
         patch.object(
@@ -1453,7 +1453,7 @@ async def _run_passthrough_connect(
             new_callable=AsyncMock,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
             return_value=server,
         ),
         patch.object(session_manager_stateless, "handle_request", new_callable=AsyncMock) as mock_handle_request,
@@ -1574,7 +1574,7 @@ async def test_handle_streamable_http_mcp_true_passthrough_without_token_surface
             return_value=probe_client,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
             return_value=tp_server,
         ),
         patch.object(
@@ -1642,7 +1642,7 @@ async def test_handle_streamable_http_mcp_true_passthrough_dcr_bridge_challenges
             return_value=probe_client,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
             return_value=bridge_server,
         ),
         patch.object(
@@ -1720,7 +1720,7 @@ async def test_handle_streamable_http_mcp_true_passthrough_with_token_skips_prob
             return_value=probe_client,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
             return_value=tp_server,
         ),
         patch.object(
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py
index cb43d2c2592..4575741aa8b 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py
@@ -1,3 +1,4 @@
+from litellm.proxy._experimental.mcp_server import operations as mcp_operations
 """
 Tests for MCP tool search feature.
 
@@ -572,7 +573,7 @@ class TestCallToolRestApiVirtualTools:
         mock_tool.input_schema = {"type": "object", "properties": {}}
 
         with patch(
-            "litellm.proxy._experimental.mcp_server.server._list_mcp_tools",
+            "litellm.proxy._experimental.mcp_server.operations._list_mcp_tools",
             new_callable=AsyncMock,
             return_value=AggregateToolListing(tools=[mock_tool], outcomes={}),
         ):
@@ -616,12 +617,12 @@ class TestCallToolRestApiVirtualTools:
 
         with (
             patch(
-                "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+                "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
                 new_callable=AsyncMock,
                 return_value=[MagicMock()],
             ),
             patch(
-                "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool",
+                "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool",
                 new_callable=AsyncMock,
                 return_value=fake_result,
             ) as mock_execute,
@@ -669,12 +670,12 @@ class TestCallToolRestApiVirtualTools:
                 return_value="203.0.113.7",
             ),
             patch(
-                "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+                "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
                 new_callable=AsyncMock,
                 return_value=[MagicMock()],
             ) as mock_allowed,
             patch(
-                "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool",
+                "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool",
                 new_callable=AsyncMock,
                 return_value=fake_result,
             ),
@@ -699,7 +700,7 @@ class TestCallToolRestApiVirtualTools:
                 return_value="203.0.113.7",
             ),
             patch(
-                "litellm.proxy._experimental.mcp_server.server._list_mcp_tools",
+                "litellm.proxy._experimental.mcp_server.operations._list_mcp_tools",
                 new_callable=AsyncMock,
                 return_value=AggregateToolListing(tools=[], outcomes={}),
             ) as mock_list,
@@ -832,7 +833,7 @@ class TestCallToolRestApiVirtualTools:
                 "litellm.proxy.proxy_server.proxy_logging_obj", key_limits
             ),
             patch(  # test-quality-ok: the authorized catalog is the seam every virtual tool shares; the ranking under test stays real
-                "litellm.proxy._experimental.mcp_server.server._list_mcp_tools",
+                "litellm.proxy._experimental.mcp_server.operations._list_mcp_tools",
                 new_callable=AsyncMock,
                 return_value=AggregateToolListing(tools=list(CATALOG), outcomes={}),
             ) as mock_list,
@@ -939,7 +940,7 @@ class TestDispatchVirtualMcpTool:
             new_callable=AsyncMock,
             return_value="SEARCH_RESULT",
         ) as mock_search:
-            result = await srv._dispatch_virtual_mcp_tool(
+            result = await mcp_operations._dispatch_virtual_mcp_tool(
                 name=MCP_TOOL_SEARCH_TOOL_NAME,
                 arguments={"query": "q", "top_k": 3},
                 user_api_key_auth=uak,
@@ -961,7 +962,7 @@ class TestDispatchVirtualMcpTool:
             new_callable=AsyncMock,
             return_value="AGENT_RESULT",
         ) as mock_agent_search:
-            result = await srv._dispatch_virtual_mcp_tool(
+            result = await mcp_operations._dispatch_virtual_mcp_tool(
                 name=AGENT_SEARCH_TOOL_NAME,
                 arguments={"query": "translate a document", "top_k": "2"},
                 user_api_key_auth=uak,
@@ -996,7 +997,7 @@ class TestDispatchVirtualMcpTool:
             new_callable=AsyncMock,
             return_value="CALL_RESULT",
         ) as mock_call:
-            result = await srv._dispatch_virtual_mcp_tool(
+            result = await mcp_operations._dispatch_virtual_mcp_tool(
                 name=MCP_TOOL_CALL_TOOL_NAME,
                 arguments={"tool_name": "math-add", "arguments": {"a": 1, "b": 2}},
                 user_api_key_auth=uak,
@@ -1027,8 +1028,7 @@ class TestDispatchVirtualMcpTool:
         sentinel_logging_obj = object()
         with (
             patch.object(
-                srv,
-                "_build_virtual_call_logging_obj",
+                mcp_operations, "_build_virtual_call_logging_obj",
                 new_callable=AsyncMock,
                 return_value=sentinel_logging_obj,
             ) as mock_build,
@@ -1038,7 +1038,7 @@ class TestDispatchVirtualMcpTool:
                 return_value="CALL_RESULT",
             ) as mock_call,
         ):
-            await srv._dispatch_virtual_mcp_tool(
+            await mcp_operations._dispatch_virtual_mcp_tool(
                 name=MCP_TOOL_CALL_TOOL_NAME,
                 arguments={"tool_name": "math-add", "arguments": {"a": 1}},
                 user_api_key_auth=uak,
@@ -1060,7 +1060,7 @@ class TestDispatchVirtualMcpTool:
             new_callable=AsyncMock,
             return_value="SEARCH_RESULT",
         ) as mock_search:
-            await srv._dispatch_virtual_mcp_tool(
+            await mcp_operations._dispatch_virtual_mcp_tool(
                 name=MCP_TOOL_SEARCH_TOOL_NAME,
                 arguments={"query": "issue", "top_k": "not-a-number"},
                 user_api_key_auth=uak,
@@ -1083,12 +1083,12 @@ class TestDispatchVirtualMcpTool:
         fake = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False)
         with (
             patch(
-                "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+                "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
                 new_callable=AsyncMock,
                 return_value=[MagicMock()],
             ) as mock_allowed,
             patch(
-                "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool",
+                "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool",
                 new_callable=AsyncMock,
                 return_value=fake,
             ) as mock_exec,
@@ -1130,12 +1130,12 @@ class TestDispatchVirtualMcpTool:
         uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True))
         with (
             patch(
-                "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+                "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
                 new_callable=AsyncMock,
                 return_value=[],
             ),
             patch(
-                "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool",
+                "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool",
                 new_callable=AsyncMock,
             ) as mock_exec,
         ):
@@ -1217,7 +1217,7 @@ class TestMcpServerToolCallErrorHandling:
                 return_value=(uak, None, None, None, None, None, None),
             ),
             patch(
-                "litellm.proxy._experimental.mcp_server.server._dispatch_virtual_mcp_tool",
+                "litellm.proxy._experimental.mcp_server.operations._dispatch_virtual_mcp_tool",
                 new_callable=AsyncMock,
                 side_effect=HTTPException(status_code=403, detail="User not allowed to call this tool"),
             ),
@@ -1254,7 +1254,7 @@ async def test_handle_mcp_tool_call_scoped_denial_names_the_binding_agent() -> N
         ]
 
     with patch(  # test-quality-ok: the permission resolver is a module-level function; the suite's only seam
-        "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+        "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
         new=AsyncMock(side_effect=resolve),
     ):
         with pytest.raises(HTTPException) as exc_info:
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py
index 519acc241c6..60098b1f656 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py
@@ -564,7 +564,7 @@ class TestMCPActiveToolsetContextVar:
                 MagicMock(get_mcp_client_ip=MagicMock(return_value="127.0.0.1")),
             ),
             patch(
-                "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+                "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
                 MagicMock(get_mcp_server_by_name=MagicMock(return_value=None)),
             ),
             patch(
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py
index ac716bace3c..bb70f38285c 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py
@@ -1,3 +1,4 @@
+from litellm.proxy._experimental.mcp_server import operations as mcp_operations
 """
 VERIA-7 regression: OpenAPI-backed (local-registry) MCP tools must run
 through `pre_call_tool_check` before dispatch, the same as managed
@@ -49,22 +50,22 @@ async def test_openapi_local_tool_runs_pre_call_tool_check():
 
     with (
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "_get_mcp_server_from_tool_name",
             return_value=fake_server,
         ),
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "pre_call_tool_check",
             new=pre_call,
         ),
         patch.object(
-            mcp_module.global_mcp_tool_registry,
+            mcp_operations.global_mcp_tool_registry,
             "get_tool",
             return_value=fake_tool,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool",
+            "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool",
             new=handle_local,
         ),
         patch(
@@ -72,7 +73,7 @@ async def test_openapi_local_tool_runs_pre_call_tool_check():
             return_value=True,
         ),
     ):
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name="list_pets",
             arguments={"limit": 10},
             allowed_mcp_servers=[fake_server],
@@ -92,7 +93,7 @@ async def test_openapi_local_tool_runs_pre_call_tool_check():
     assert pre_call_kwargs["guardrail_context"] == {"metadata": {"guardrails": ("block-all",)}}
     assert pre_call_kwargs["name"] == "list_pets"
     assert pre_call_kwargs["server"] is fake_server
-    assert pre_call_kwargs["user_api_key_auth"] is user
+    assert pre_call_kwargs["user_api_key_auth"] == user
     # `proxy_logging_obj` must be sourced from the canonical proxy_server
     # module (same as the managed path) — passing None would crash the
     # downstream `_create_mcp_request_object_from_kwargs` call with
@@ -134,22 +135,22 @@ async def test_openapi_local_tool_blocked_when_pre_call_check_raises():
 
     with (
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "_get_mcp_server_from_tool_name",
             return_value=fake_server,
         ),
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "pre_call_tool_check",
             new=pre_call,
         ),
         patch.object(
-            mcp_module.global_mcp_tool_registry,
+            mcp_operations.global_mcp_tool_registry,
             "get_tool",
             return_value=fake_tool,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool",
+            "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool",
             new=handle_local,
         ),
         patch(
@@ -158,7 +159,7 @@ async def test_openapi_local_tool_blocked_when_pre_call_check_raises():
         ),
     ):
         with pytest.raises(HTTPException) as exc:
-            await mcp_module.execute_mcp_tool(
+            await mcp_operations.execute_mcp_tool(
                 name="delete_pet",
                 arguments={},
                 allowed_mcp_servers=[fake_server],
@@ -195,24 +196,24 @@ async def test_openapi_local_tool_denied_when_server_not_resolvable():
 
     # `_get_mcp_server_from_tool_name` returns None — no server context.
     with (
-        patch.object(mcp_module, "_resolve_openapi_tool_auth", new=resolve_auth),
+        patch.object(mcp_operations, "_resolve_openapi_tool_auth", new=resolve_auth),
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "_get_mcp_server_from_tool_name",
             return_value=None,
         ),
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "pre_call_tool_check",
             new=pre_call,
         ),
         patch.object(
-            mcp_module.global_mcp_tool_registry,
+            mcp_operations.global_mcp_tool_registry,
             "get_tool",
             return_value=fake_tool,
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool",
+            "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool",
             new=handle_local,
         ),
         patch(
@@ -221,7 +222,7 @@ async def test_openapi_local_tool_denied_when_server_not_resolvable():
         ),
     ):
         with pytest.raises(HTTPException) as exc:
-            await mcp_module.execute_mcp_tool(
+            await mcp_operations.execute_mcp_tool(
                 name="list_pets",
                 arguments={},
                 allowed_mcp_servers=[],
@@ -280,27 +281,27 @@ async def test_openapi_local_tool_injects_resolved_oauth_token():
 
     with (
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "_get_mcp_server_from_tool_name",
             return_value=oauth_server,
         ),
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "pre_call_tool_check",
             new=AsyncMock(return_value={}),
         ),
         patch.object(
-            mcp_module.global_mcp_tool_registry,
+            mcp_operations.global_mcp_tool_registry,
             "get_tool",
             return_value=fake_tool,
         ),
         patch.object(
-            mcp_module.global_mcp_server_manager._cred_provider,
+            mcp_operations.global_mcp_server_manager._cred_provider,
             "resolve_credentials",
             new=AsyncMock(return_value=Ok(StaticHeaderAuth("Bearer stored-user-token"))),
         ),
         patch(
-            "litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool",
+            "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool",
             new=handle_local,
         ),
         patch(
@@ -308,7 +309,7 @@ async def test_openapi_local_tool_injects_resolved_oauth_token():
             return_value=True,
         ),
     ):
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name="get_values",
             arguments={},
             allowed_mcp_servers=[oauth_server],
@@ -417,7 +418,7 @@ async def test_legacy_local_tool_fallback_refuses_unentitled_caller(legacy_local
     )
 
     with pytest.raises(HTTPException) as exc:
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name=f"{LEGACY_SERVER_NAME}-{LEGACY_TOOL}",
             arguments={},
             allowed_mcp_servers=[server],
@@ -451,7 +452,7 @@ async def test_legacy_local_tool_fallback_still_dispatches_entitled_caller(
     server, executed = legacy_local_tool
     user = _caller_entitled_to([LEGACY_TOOL])
 
-    result = await mcp_module.execute_mcp_tool(
+    result = await mcp_operations.execute_mcp_tool(
         name=f"{LEGACY_SERVER_NAME}-{LEGACY_TOOL}",
         arguments={},
         allowed_mcp_servers=[server],
@@ -481,7 +482,7 @@ async def test_legacy_local_tool_fallback_fails_closed_on_empty_prefix(
     _server, executed = legacy_local_tool
 
     with pytest.raises(HTTPException) as exc:
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name=f"-{LEGACY_TOOL}",
             arguments={},
             allowed_mcp_servers=[],
@@ -523,7 +524,7 @@ async def test_legacy_local_tool_fallback_fails_closed_when_prefix_names_no_serv
         return_value=True,
     ):
         with pytest.raises(HTTPException) as exc:
-            await mcp_module.execute_mcp_tool(
+            await mcp_operations.execute_mcp_tool(
                 name=f"{LEGACY_SERVER_NAME}-{LEGACY_TOOL}",
                 arguments={},
                 allowed_mcp_servers=[other_server],
@@ -546,7 +547,7 @@ async def test_unknown_tool_name_still_reports_not_found():
     from litellm.proxy._experimental.mcp_server import server as mcp_module
 
     with pytest.raises(HTTPException) as exc:
-        await mcp_module.execute_mcp_tool(
+        await mcp_operations.execute_mcp_tool(
             name="tool_no_registry_knows",
             arguments={},
             allowed_mcp_servers=[],
@@ -610,7 +611,7 @@ async def test_per_server_auth_header_reaches_both_openapi_dispatch_arms(dispatc
         captured["injected"] = _request_auth_header.get()
         return []
 
-    manager = mcp_module.global_mcp_server_manager
+    manager = mcp_operations.global_mcp_server_manager
     with (
         patch.object(manager, "resolve_openapi_upstream_auth", new=fake_resolver),
         patch.object(manager, "pre_call_tool_check", new=AsyncMock(return_value={})),
@@ -620,9 +621,9 @@ async def test_per_server_auth_header_reaches_both_openapi_dispatch_arms(dispatc
             fake_tool.name = "list_reports"
             with (
                 patch.object(manager, "_get_mcp_server_from_tool_name", return_value=server),
-                patch.object(mcp_module.global_mcp_tool_registry, "get_tool", return_value=fake_tool),
+                patch.object(mcp_operations.global_mcp_tool_registry, "get_tool", return_value=fake_tool),
                 patch(
-                    "litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool",
+                    "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool",
                     new=capture_local,
                 ),
                 patch(
@@ -630,7 +631,7 @@ async def test_per_server_auth_header_reaches_both_openapi_dispatch_arms(dispatc
                     return_value=True,
                 ),
             ):
-                await mcp_module.execute_mcp_tool(
+                await mcp_operations.execute_mcp_tool(
                     name="list_reports",
                     arguments={},
                     allowed_mcp_servers=[server],
@@ -702,11 +703,11 @@ async def test_local_dispatch_reports_the_outcome_instead_of_success(failure: st
     user = UserAPIKeyAuth(api_key="sk-user", user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value)
 
     with (
-        patch.object(mcp_module.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=server),
-        patch.object(mcp_module.global_mcp_server_manager, "pre_call_tool_check", new=AsyncMock(return_value={})),
-        patch.object(mcp_module.global_mcp_tool_registry, "get_tool", return_value=fake_tool),
+        patch.object(mcp_operations.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=server),
+        patch.object(mcp_operations.global_mcp_server_manager, "pre_call_tool_check", new=AsyncMock(return_value={})),
+        patch.object(mcp_operations.global_mcp_tool_registry, "get_tool", return_value=fake_tool),
         patch.object(
-            mcp_module.global_mcp_server_manager,
+            mcp_operations.global_mcp_server_manager,
             "resolve_openapi_upstream_auth",
             new=AsyncMock(return_value=(None, None)),
         ),
@@ -715,7 +716,7 @@ async def test_local_dispatch_reports_the_outcome_instead_of_success(failure: st
             return_value=True,
         ),
     ):
-        call = mcp_module.execute_mcp_tool(
+        call = mcp_operations.execute_mcp_tool(
             name="list_reports",
             arguments={},
             allowed_mcp_servers=[server],
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_operations.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_operations.py
new file mode 100644
index 00000000000..870f2cd5f8f
--- /dev/null
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_operations.py
@@ -0,0 +1,343 @@
+import asyncio
+from unittest.mock import AsyncMock, patch
+
+import pytest
+from mcp.types import GetPromptRequest, GetPromptRequestParams, GetPromptResult
+
+from litellm.proxy._experimental.mcp_server.operations import GatewayOperations, prepare_context
+from litellm.proxy._types import UserAPIKeyAuth
+
+
+@pytest.mark.asyncio
+async def test_dispatch_uses_explicit_context_when_ambient_caller_differs():
+    from mcp.server.auth.middleware.auth_context import auth_context_var
+    from litellm.proxy._experimental.mcp_server.server import set_auth_context
+
+    context = prepare_context(
+        UserAPIKeyAuth(user_id="alpha"),
+        raw_headers={"x-caller": "alpha"},
+        mcp_servers=["alpha-server"],
+        client_ip="192.0.2.1",
+    )
+    token = auth_context_var.set(None)
+    handler = AsyncMock(return_value=GetPromptResult(messages=[]))
+    try:
+        set_auth_context(UserAPIKeyAuth(user_id="bravo"), raw_headers={"x-caller": "bravo"})
+        with patch("litellm.proxy._experimental.mcp_server.operations.mcp_get_prompt", handler):
+            result = await GatewayOperations().execute(
+                GetPromptRequest(params=GetPromptRequestParams(name="alpha-prompt")), context
+            )
+        assert result.messages == []
+        assert handler.await_args.kwargs["name"] == "alpha-prompt"
+        assert handler.await_args.kwargs["user_api_key_auth"].user_id == "alpha"
+        assert handler.await_args.kwargs["raw_headers"] == {"x-caller": "alpha"}
+        assert handler.await_args.kwargs["mcp_servers"] == ["alpha-server"]
+        assert handler.await_args.kwargs["client_ip"] == "192.0.2.1"
+    finally:
+        auth_context_var.reset(token)
+
+
+@pytest.mark.asyncio
+async def test_legacy_adapter_cleans_context_after_cancelled_operation():
+    from types import SimpleNamespace
+    from litellm.proxy._experimental.mcp_server import server
+    from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var
+
+    previous_session = server.active_mcp_session_var.get()
+    previous_request = active_mcp_request_ctx_var.get()
+    request = SimpleNamespace(session=object())
+    auth = (None, None, None, None, None, None, None)
+
+    async def cancelled_operation():
+        async with server._legacy_operation_context(request, trace=False):
+            assert server.active_mcp_session_var.get() is request.session
+            assert active_mcp_request_ctx_var.get() is request
+            raise asyncio.CancelledError
+
+    with patch(
+        "litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context", AsyncMock(return_value=auth)
+    ):
+        with pytest.raises(asyncio.CancelledError):
+            await cancelled_operation()
+    assert server.active_mcp_session_var.get() is previous_session
+    assert active_mcp_request_ctx_var.get() is previous_request
+
+
+@pytest.mark.asyncio
+async def test_legacy_adapter_cleans_context_when_trace_setup_fails():
+    from types import SimpleNamespace
+    from litellm.proxy._experimental.mcp_server import server
+    from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var
+
+    previous_session = server.active_mcp_session_var.get()
+    previous_request = active_mcp_request_ctx_var.get()
+    request = SimpleNamespace(session=object())
+
+    async def enter_operation():
+        async with server._legacy_operation_context(request, trace=True):
+            pytest.fail("Trace setup failure must prevent dispatch")
+
+    with patch.object(server, "_otel_set_mcp_transport_span", side_effect=RuntimeError("trace failure")):
+        with pytest.raises(RuntimeError, match="trace failure"):
+            await enter_operation()
+    assert server.active_mcp_session_var.get() is previous_session
+    assert active_mcp_request_ctx_var.get() is previous_request
+
+
+@pytest.mark.asyncio
+async def test_prompt_sampling_receives_explicit_operation_caller_headers_and_ip():
+    from unittest.mock import MagicMock
+    from litellm.proxy._experimental.mcp_server import operations
+    from litellm.types.mcp import MCPTransport
+    from litellm.types.mcp_server.mcp_server_manager import MCPServer
+
+    upstream = MCPServer(
+        server_id="explicit-prompt",
+        name="explicit_prompt",
+        url="https://example.invalid/mcp",
+        transport=MCPTransport.http,
+        allow_sampling=True,
+    )
+    context = prepare_context(
+        UserAPIKeyAuth(user_id="prompt-caller"),
+        raw_headers={"x-caller": "prompt-caller"},
+        client_ip="192.0.2.41",
+    )
+    client = MagicMock()
+    client.get_prompt = AsyncMock(return_value=GetPromptResult(messages=[]))
+    sampling = AsyncMock()
+    with (
+        patch.object(operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[upstream])),
+        patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPClient", return_value=client) as factory,
+        patch("litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", sampling),
+    ):
+        result = await GatewayOperations().execute(
+            GetPromptRequest(params=GetPromptRequestParams(name="explicit_prompt-prompt")), context
+        )
+        assert result.messages == []
+        await factory.call_args.kwargs["sampling_callback"](None, None)
+    captured = sampling.await_args.kwargs
+    assert captured["user_api_key_auth"] is not None
+    assert captured["user_api_key_auth"].user_id == "prompt-caller"
+    assert captured["raw_headers"] == {"x-caller": "prompt-caller"}
+    assert captured["client_ip"] == "192.0.2.41"
+
+
+def _catalog_case(method):
+    from mcp import types
+
+    cases = {
+        "prompts/list": (
+            types.ListPromptsRequest(),
+            "list_prompts",
+            "get_prompts_from_server",
+            [types.Prompt(name="catalog-prompt")],
+            "prompts",
+        ),
+        "prompts/get": (
+            types.GetPromptRequest(
+                params=types.GetPromptRequestParams(name="catalog-prompt", arguments={"topic": "test"})
+            ),
+            "get_prompt",
+            "get_prompt_from_server",
+            types.GetPromptResult(messages=[]),
+            None,
+        ),
+        "resources/list": (
+            types.ListResourcesRequest(),
+            "list_resources",
+            "get_resources_from_server",
+            [types.Resource(name="document", uri="https://example.com/document")],
+            "resources",
+        ),
+        "resources/templates/list": (
+            types.ListResourceTemplatesRequest(),
+            "list_resource_templates",
+            "get_resource_templates_from_server",
+            [types.ResourceTemplate(name="document", uri_template="https://example.com/{name}")],
+            "resource_templates",
+        ),
+        "resources/read": (
+            types.ReadResourceRequest(params=types.ReadResourceRequestParams(uri="https://example.com/document")),
+            "read_resource",
+            "read_resource_from_server",
+            types.ReadResourceResult(
+                contents=[types.TextResourceContents(uri="https://example.com/document", text="document body")]
+            ),
+            None,
+        ),
+    }
+    return cases[method]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "method", ["prompts/list", "prompts/get", "resources/list", "resources/templates/list", "resources/read"]
+)
+@pytest.mark.parametrize("state", ["success", "denied", "upstream_failure", "scope_failure"])
+async def test_native_catalog_operations_preserve_context_results_and_failure_policy(method, state):
+    from types import SimpleNamespace
+    from fastapi import HTTPException
+    from mcp.server.context import ServerRequestContext
+    from mcp.types import PaginatedRequestParams
+    from litellm.proxy._experimental.mcp_server import operations, server
+    from litellm.types.mcp import MCPTransport
+    from litellm.types.mcp_server.mcp_server_manager import MCPServer
+
+    operation, handler_name, manager_method, payload, collection = _catalog_case(method)
+    caller = UserAPIKeyAuth(user_id="catalog-caller")
+    headers = {"x-caller": "catalog-caller"}
+    upstream_server = MCPServer(server_id="catalog", name="catalog", transport=MCPTransport.http)
+    allowed = AsyncMock(
+        return_value=[] if state == "denied" else [upstream_server],
+        side_effect=HTTPException(status_code=403, detail="scope denied") if state == "scope_failure" else None,
+    )
+    upstream = AsyncMock(
+        return_value=payload, side_effect=RuntimeError("upstream unavailable") if state == "upstream_failure" else None
+    )
+    ctx = ServerRequestContext(
+        session=SimpleNamespace(), lifespan_context={}, protocol_version="2025-06-18", method=method
+    )
+    auth = (caller, None, ["catalog"], None, None, headers, "192.0.2.41")
+    with (
+        patch.object(server, "get_or_extract_auth_context", AsyncMock(return_value=auth)),
+        patch.object(operations, "_get_allowed_mcp_servers", allowed),
+        patch.object(operations.global_mcp_server_manager, manager_method, upstream),
+    ):
+        if collection is None and state != "success":
+            expected_error = RuntimeError if state == "upstream_failure" else HTTPException
+            with pytest.raises(expected_error):
+                await getattr(server, handler_name)(ctx, operation.params)
+        else:
+            result = await getattr(server, handler_name)(ctx, operation.params or PaginatedRequestParams())
+            if collection:
+                assert getattr(result, collection) == (payload if state == "success" else [])
+            else:
+                assert result == payload
+    assert allowed.await_args.kwargs == {
+        "user_api_key_auth": caller,
+        "mcp_servers": ["catalog"],
+        "client_ip": "192.0.2.41",
+    }
+    if state in ("denied", "scope_failure"):
+        upstream.assert_not_awaited()
+    else:
+        upstream.assert_awaited_once()
+        forwarded = upstream.await_args.kwargs
+        assert forwarded["user_api_key_auth"] == caller
+        assert forwarded["raw_headers"] == headers
+        assert forwarded["client_ip"] == "192.0.2.41"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "method", ["prompts/list", "prompts/get", "resources/list", "resources/templates/list", "resources/read"]
+)
+async def test_explicit_proxy_context_rejects_catalog_operations_before_upstream_access(method):
+    from mcp.shared.exceptions import MCPError
+    from mcp.types import METHOD_NOT_FOUND
+    from litellm.proxy._experimental.mcp_server import operations
+
+    operation, _, manager_method, _, _ = _catalog_case(method)
+    upstream = AsyncMock()
+    with patch.object(operations.global_mcp_server_manager, manager_method, upstream):
+        with pytest.raises(MCPError) as rejected:
+            await GatewayOperations().execute(operation, prepare_context(mcp_proxy_mode=True))
+    assert rejected.value.error.code == METHOD_NOT_FOUND
+    assert rejected.value.error.message == "Operation unavailable on /mcp/proxy"
+    upstream.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("failure", ["missing_env", "pii", "guardrail", "unexpected"])
+async def test_tool_operation_preserves_failure_messages_and_request_trace(failure):
+    from mcp.types import CallToolRequest, CallToolRequestParams
+    from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
+    from litellm.proxy._experimental.mcp_server import operations
+    from litellm.proxy._experimental.mcp_server.utils import MCPMissingUserEnvVarsError
+
+    failures = {
+        "missing_env": (
+            MCPMissingUserEnvVarsError(
+                server_id="server", server_name="server", missing=["TOKEN"], setup_url="https://example.com/setup"
+            ),
+            "https://example.com/setup",
+        ),
+        "pii": (
+            BlockedPiiEntityError(entity_type="EMAIL_ADDRESS", guardrail_name="test"),
+            "Blocked PII entity detected",
+        ),
+        "guardrail": (GuardrailRaisedException(message="request denied"), "Guardrail violation"),
+        "unexpected": (RuntimeError("upstream unavailable"), "Error: upstream unavailable"),
+    }
+    error, expected = failures[failure]
+    dispatch = AsyncMock(side_effect=error)
+    context = prepare_context(
+        raw_headers={"x-litellm-trace-id": "operation-trace", "authorization": "private-test-header"}
+    )
+    with patch.object(operations, "call_mcp_tool", dispatch):
+        result = await GatewayOperations().execute(
+            CallToolRequest(params=CallToolRequestParams(name="catalog-tool", arguments={})), context
+        )
+    assert result.is_error is True
+    assert expected in result.content[0].text
+    assert "private-test-header" not in result.content[0].text
+    dispatch.assert_awaited_once()
+    assert dispatch.await_args.kwargs["litellm_trace_id"] == "operation-trace"
+    assert dispatch.await_args.kwargs["litellm_session_id"] == "operation-trace"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "method,helper",
+    [
+        ("prompts/list", "_list_mcp_prompts"),
+        ("resources/list", "_list_mcp_resources"),
+        ("resources/templates/list", "_list_mcp_resource_templates"),
+    ],
+)
+async def test_catalog_operation_preserves_empty_result_for_malformed_upstream_items(method, helper):
+    from litellm.proxy._experimental.mcp_server import operations
+
+    operation, _, _, _, collection = _catalog_case(method)
+    with patch.object(operations, helper, AsyncMock(return_value=[{"unexpected": "item"}])):
+        result = await GatewayOperations().execute(operation, prepare_context())
+    assert getattr(result, collection) == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("catalog_unavailable", [False, True])
+async def test_tool_listing_returns_empty_result_without_dispatch_for_unavailable_catalog(catalog_unavailable):
+    from mcp.types import ListToolsRequest
+    from litellm.proxy._experimental.mcp_server import operations
+
+    allowed = AsyncMock(
+        return_value=[], side_effect=RuntimeError("catalog unavailable") if catalog_unavailable else None
+    )
+    upstream = AsyncMock()
+    with (
+        patch.object(operations, "_get_allowed_mcp_servers", allowed),
+        patch.object(operations.global_mcp_server_manager, "_get_tools_from_server", upstream),
+    ):
+        result = await GatewayOperations().execute(ListToolsRequest(), prepare_context())
+    assert result.tools == []
+    allowed.assert_awaited_once()
+    upstream.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_explicit_proxy_context_lists_builtin_tools_and_blocks_direct_tool_dispatch():
+    from mcp.types import CallToolRequest, CallToolRequestParams, ListToolsRequest
+    from litellm.proxy._experimental.mcp_server import operations
+
+    context = prepare_context(mcp_proxy_mode=True)
+    allowed = AsyncMock()
+    with patch.object(operations, "_get_allowed_mcp_servers", allowed):
+        listing = await GatewayOperations().execute(ListToolsRequest(), context)
+        denied = await GatewayOperations().execute(
+            CallToolRequest(params=CallToolRequestParams(name="catalog-tool", arguments={})), context
+        )
+    assert {tool.name for tool in listing.tools} == {"search_tools", "get_tool_schema", "call_tool"}
+    assert denied.is_error is True
+    assert "unavailable on /mcp/proxy" in denied.content[0].text
+    allowed.assert_not_awaited()
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py
index 13af58c15c0..233a8cc96ba 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py
@@ -1,3 +1,4 @@
+from litellm.proxy._experimental.mcp_server import operations as mcp_operations
 import asyncio
 import inspect
 import json
@@ -1253,6 +1254,7 @@ class TestListToolsRestAPI:
             user_api_key_auth=None,
             extra_headers=None,
             apply_tool_filters=True,
+            client_ip=None,
         ):
             captured["called"] = True
             captured["server"] = server
@@ -1338,6 +1340,7 @@ class TestListToolsRestAPI:
             user_api_key_auth=None,
             extra_headers=None,
             apply_tool_filters=True,
+            client_ip=None,
         ):
             captured["user_api_key_auth"] = user_api_key_auth
             return ["tool-1"]
@@ -1891,6 +1894,7 @@ class TestListToolsRestAPI:
             user_api_key_auth=None,
             extra_headers=None,
             apply_tool_filters=True,
+            client_ip=None,
         ):
             captured["called"] = True
             captured["server_arg"] = server
@@ -2027,6 +2031,7 @@ class TestListToolsRestAPI:
             user_api_key_auth=None,
             extra_headers=None,
             apply_tool_filters=True,
+            client_ip=None,
         ):
             captured["called"] = True
             captured["server_arg"] = server
@@ -2112,6 +2117,7 @@ class TestListToolsRestAPI:
             user_api_key_auth=None,
             extra_headers=None,
             apply_tool_filters=True,
+            client_ip=None,
         ):
             return ["scoped-tool"]
 
@@ -2319,6 +2325,7 @@ class TestListToolsRestAPI:
             user_api_key_auth=None,
             extra_headers=None,
             apply_tool_filters=True,
+            client_ip=None,
         ):
             captured["server"] = server
             captured["auth_header"] = server_auth_header
@@ -3145,10 +3152,10 @@ async def test_request_selected_tool_specific_guardrail_applies_to_virtual_execu
     monkeypatch.setattr(litellm, "callbacks", [guardrail])
     monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry)
     monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager)
-    monkeypatch.setattr(server, "global_mcp_tool_registry", registry)
-    monkeypatch.setattr(server, "global_mcp_server_manager", manager)
+    monkeypatch.setattr(mcp_operations, "global_mcp_tool_registry", registry)
+    monkeypatch.setattr(mcp_operations, "global_mcp_server_manager", manager)
     monkeypatch.setattr(rest_endpoints, "global_mcp_server_manager", manager)
-    monkeypatch.setattr(server, "_get_allowed_mcp_servers", AsyncMock(return_value=[managed_server]))
+    monkeypatch.setattr(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[managed_server]))
     monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache()))
     monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", passthrough_request_data)
     monkeypatch.setattr(proxy_server, "proxy_config", {})
diff --git a/tests/test_litellm/test_check_mcp_operation_boundary.py b/tests/test_litellm/test_check_mcp_operation_boundary.py
new file mode 100644
index 00000000000..d7ac72de9f0
--- /dev/null
+++ b/tests/test_litellm/test_check_mcp_operation_boundary.py
@@ -0,0 +1,52 @@
+from pathlib import Path
+
+import pytest
+
+from scripts.check_mcp_operation_boundary import main, violations
+
+
+@pytest.mark.parametrize(
+    "source",
+    (
+        "from mcp.server.auth.middleware.auth_context import auth_context_var as hidden",
+        "from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_proxy_mode as mode",
+        "caller = legacy.get_active_auth_context()",
+        "owners = transport._stateful_session_owners",
+        "from weakref import WeakKeyDictionary",
+        "from litellm.proxy._experimental.mcp_server.server import get_auth_context",
+    ),
+)
+def test_shared_operation_boundary_rejects_ambient_state(source):
+    assert violations(Path("operations.py"), source)
+
+
+def test_legacy_adapter_may_resolve_context_but_policy_must_receive_it():
+    source = "from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_proxy_mode"
+    assert violations(Path("server.py"), source) == ()
+    assert violations(Path("legacy_callbacks.py"), source) == ()
+    assert violations(Path("operations.py"), "def execute(context):\n    return context.client_ip") == ()
+    assert violations(Path("mcp_server_manager.py"), "def _mcp_registry_key(server):\n    return server.name") == ()
+
+
+def test_boundary_command_rejects_shared_state_and_accepts_explicit_context(tmp_path, monkeypatch, capsys):
+    import subprocess
+    import sys
+
+    package = tmp_path / "litellm/proxy/_experimental/mcp_server"
+    package.mkdir(parents=True)
+    module = package / "operations.py"
+    module.write_text("from mcp.server.auth.middleware.auth_context import auth_context_var as hidden\n")
+    command = [sys.executable, str(Path(__file__).resolve().parents[2] / "scripts/check_mcp_operation_boundary.py")]
+    monkeypatch.chdir(tmp_path)
+    assert main() == 1
+    assert "operations.py:1:" in capsys.readouterr().err
+    rejected = subprocess.run(command, cwd=tmp_path, capture_output=True, text=True, check=False)
+    assert rejected.returncode == 1
+    assert "operations.py:1: MCP request/session state belongs in a legacy adapter" in rejected.stderr
+
+    module.write_text("def execute(context):\n    return context.client_ip\n")
+    assert main() == 0
+    assert "MCP operation boundary: passed" in capsys.readouterr().out
+    accepted = subprocess.run(command, cwd=tmp_path, capture_output=True, text=True, check=False)
+    assert accepted.returncode == 0
+    assert "MCP operation boundary: passed" in accepted.stdout

From bf4fccc937175999d9327051479274bfb8c6d5fd Mon Sep 17 00:00:00 2001
From: yuneng <yuneng@berri.ai>
Date: Mon, 21 Sep 2026 19:25:52 +0000
Subject: [PATCH 047/160] feat(ui): expose remaining complexity router advanced
 settings

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../add_model/ClassificationMethodConfig.tsx  | 40 ++++++++-
 .../add_model/ComplexityRouterConfig.tsx      | 36 ++++++++
 .../add_model/HeuristicKeywordOverrides.tsx   | 42 +++++++++
 .../add_model/HousekeepingRoutingControls.tsx | 44 ++++++++++
 .../add_model/PlanModeOverrideControls.tsx    | 18 ++++
 .../components/add_model/ReminderMarkers.tsx  | 77 +++++++++++++++++
 .../add_model/ResponseFormatControls.tsx      | 12 +++
 .../add_model/add_auto_router_tab.tsx         | 14 +++
 .../build_complexity_router_config.test.ts    | 58 +++++++++++++
 .../build_complexity_router_config.ts         | 85 +++++++++++++++++++
 .../components/add_model/classifier_types.ts  |  3 +-
 ...d_updated_complexity_router_config.test.ts | 17 ++++
 .../edit_auto_router_modal.tsx                | 54 ++++++++++++
 13 files changed, 498 insertions(+), 2 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/components/add_model/HeuristicKeywordOverrides.tsx
 create mode 100644 ui/litellm-dashboard/src/components/add_model/HousekeepingRoutingControls.tsx
 create mode 100644 ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx

diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
index b72b29a29f4..2564137fb02 100644
--- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
@@ -19,7 +19,10 @@ import HeuristicScoringConfig from "./HeuristicScoringConfig";
 import ClassifierReasoningEffortSelect from "./ClassifierReasoningEffortSelect";
 import ClassifierCircuitBreakerConfig from "./ClassifierCircuitBreakerConfig";
 import ClassifierVisionConfig from "./ClassifierVisionConfig";
-import { getHeuristicV2SuccessThresholdError } from "./build_complexity_router_config";
+import {
+  getClassifierPluginTimeoutError,
+  getHeuristicV2SuccessThresholdError,
+} from "./build_complexity_router_config";
 import type { ReasoningEffort } from "./complexity_router_tiers";
 import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults";
 import {
@@ -61,6 +64,7 @@ const CLASSIFIER_CONTEXT_WINDOW_SIZE_ID = "classifier-context-window-size";
 const CLASSIFIER_CONTEXT_BUDGET_CHARS_ID = "classifier-context-budget-chars";
 const HYBRID_BOUNDARY_MARGIN_ID = "hybrid-boundary-margin";
 const HEURISTIC_V2_SUCCESS_THRESHOLD_ID = "heuristic-v2-success-threshold";
+const CLASSIFIER_PLUGIN_TIMEOUT_ID = "classifier-plugin-timeout-ms";
 
 const CUSTOM_PROMPT_WITH_HEURISTIC_FALLBACK =
   "This router classifies with your own prompt, so the tier comes from whatever rubric it states. The four tier " +
@@ -467,6 +471,40 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
     <>
       <ClassifierTypeRadios value={value} classifierType={classifierType} onTypeChange={handleClassifierTypeChange} />
 
+      {classifierType === "custom" && (
+        <div className="mt-4 space-y-2">
+          <p className="text-sm text-muted-foreground">
+            This router uses a custom classifier plugin set in config.yaml. Pick a classifier below to replace it.
+          </p>
+          <Label htmlFor={CLASSIFIER_PLUGIN_TIMEOUT_ID} className="block font-semibold">
+            Classifier plugin timeout (ms)
+          </Label>
+          <Input
+            id={CLASSIFIER_PLUGIN_TIMEOUT_ID}
+            inputMode="numeric"
+            placeholder="3000"
+            value={value.classifier_plugin_timeout_ms ?? ""}
+            onChange={(event) =>
+              onChange({
+                ...value,
+                classifier_plugin_timeout_ms: event.target.value.trim() === "" ? undefined : Number(event.target.value),
+              })
+            }
+            aria-invalid={Boolean(
+              showValidationErrors && getClassifierPluginTimeoutError(classifierType, value.classifier_plugin_timeout_ms),
+            )}
+          />
+          <p className="text-sm text-muted-foreground">
+            Time budget for the plugin call. On expiry the fallback path decides the tier.
+          </p>
+          {showValidationErrors && getClassifierPluginTimeoutError(classifierType, value.classifier_plugin_timeout_ms) && (
+            <p className="text-sm text-destructive" role="alert">
+              {getClassifierPluginTimeoutError(classifierType, value.classifier_plugin_timeout_ms)}
+            </p>
+          )}
+        </div>
+      )}
+
       {classifierType === "heuristic_v2" && (
         <div className="mt-4 space-y-2">
           <Label htmlFor={HEURISTIC_V2_SUCCESS_THRESHOLD_ID} className="block font-semibold">
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
index 9fa4e762015..a97e7a77a21 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
@@ -63,9 +63,14 @@ import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from
 import { type CustomDimensionRow } from "./custom_dimensions";
 import CompressionControls from "./CompressionControls";
 import { type AutoRouterCompressionState, DEFAULT_AUTO_ROUTER_COMPRESSION } from "./buildAutoRouterCompression";
+import HeuristicKeywordOverrides from "./HeuristicKeywordOverrides";
+import HousekeepingRoutingControls from "./HousekeepingRoutingControls";
+import ReminderMarkers from "./ReminderMarkers";
+import { type ReminderMarkerPair } from "./build_complexity_router_config";
 
 export type { DimensionWeights, TierBoundaries, TokenThresholds };
 export type { CustomTierSet, TierRow } from "./tier_rows";
+export type { ReminderMarkerPair } from "./build_complexity_router_config";
 
 export const DEFAULT_CLASSIFIER_TIMEOUT_MS = 3000;
 export const DEFAULT_TIER_DISTANCE_PENALTY = 0.5;
@@ -429,6 +434,16 @@ export interface ComplexityRouterConfigValue {
    * edit round-trip.
    */
   tier_model_params?: TierModelParamsByTier;
+  code_keywords?: string[];
+  reasoning_keywords?: string[];
+  technical_keywords?: string[];
+  simple_keywords?: string[];
+  plan_mode_patterns?: string[];
+  route_housekeeping_to_cheapest_tier?: boolean;
+  housekeeping_patterns?: string[];
+  reminder_markers?: ReminderMarkerPair[];
+  max_tokens_from_tier_model?: boolean;
+  classifier_plugin_timeout_ms?: number;
 }
 
 /** Session affinity wins where a hand-authored config sets both, matching the backend's own `or`. */
@@ -786,6 +801,17 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
                       />
                     ),
                   },
+              ]
+              : []),
+            ...(!forecast
+              ? [
+                  {
+                    key: "keyword-overrides",
+                    label: (
+                      <strong className="text-foreground font-semibold">Advanced: Heuristic Keyword Overrides</strong>
+                    ),
+                    children: <HeuristicKeywordOverrides value={value} onChange={onChange} />,
+                  },
                 ]
               : []),
             {
@@ -814,6 +840,16 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
                 <PlanModeOverrideControls value={value} onChange={onChange} planModeTierOptions={planModeTierOptions} />
               ),
             },
+            {
+              key: "housekeeping",
+              label: <strong className="text-foreground font-semibold">Advanced: Housekeeping Routing</strong>,
+              children: <HousekeepingRoutingControls value={value} onChange={onChange} />,
+            },
+            {
+              key: "reminder-markers",
+              label: <strong className="text-foreground font-semibold">Advanced: Reminder Markers</strong>,
+              children: <ReminderMarkers value={value} onChange={onChange} showValidationErrors={showValidationErrors} />,
+            },
             {
               key: "context-window",
               label: <strong className="text-foreground font-semibold">Advanced: Context Window Escalation</strong>,
diff --git a/ui/litellm-dashboard/src/components/add_model/HeuristicKeywordOverrides.tsx b/ui/litellm-dashboard/src/components/add_model/HeuristicKeywordOverrides.tsx
new file mode 100644
index 00000000000..696924f4e77
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/HeuristicKeywordOverrides.tsx
@@ -0,0 +1,42 @@
+import React from "react";
+import { MultiSelect } from "@/components/shared/MultiSelect";
+import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
+
+const fields = [
+  ["code_keywords", "Code keywords"],
+  ["reasoning_keywords", "Reasoning keywords"],
+  ["technical_keywords", "Technical keywords"],
+  ["simple_keywords", "Simple keywords"],
+] as const;
+
+const HeuristicKeywordOverrides: React.FC<{
+  value: ComplexityRouterConfigValue;
+  onChange: (value: ComplexityRouterConfigValue) => void;
+}> = ({ value, onChange }) => (
+  <div className="space-y-3">
+    <p className="text-sm text-muted-foreground">
+      Each list replaces the built-in keyword list of the same name for the heuristic scorer. Leave a list empty to
+      keep the built-in one. To add technical terms without replacing the list, use custom technical keywords under
+      Classification Method.
+    </p>
+    {fields.map(([key, label]) => {
+      const keywords = value[key] ?? [];
+      return (
+        <div key={key}>
+          <strong className="mb-1 block font-semibold">{label}</strong>
+          <MultiSelect
+            options={keywords.map((keyword) => ({ label: keyword, value: keyword }))}
+            value={keywords}
+            onValueChange={(next) => onChange({ ...value, [key]: next.length > 0 ? next : undefined })}
+            placeholder={`Add ${label.toLowerCase()}`}
+            emptyText="Type to add a keyword"
+            allowCustomValues
+            className="w-full"
+          />
+        </div>
+      );
+    })}
+  </div>
+);
+
+export default HeuristicKeywordOverrides;
diff --git a/ui/litellm-dashboard/src/components/add_model/HousekeepingRoutingControls.tsx b/ui/litellm-dashboard/src/components/add_model/HousekeepingRoutingControls.tsx
new file mode 100644
index 00000000000..2b6caba6291
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/HousekeepingRoutingControls.tsx
@@ -0,0 +1,44 @@
+import React from "react";
+import { MultiSelect } from "@/components/shared/MultiSelect";
+import { Switch } from "@/components/ui/switch";
+import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
+
+const HousekeepingRoutingControls: React.FC<{
+  value: ComplexityRouterConfigValue;
+  onChange: (value: ComplexityRouterConfigValue) => void;
+}> = ({ value, onChange }) => {
+  const enabled = value.route_housekeeping_to_cheapest_tier ?? true;
+  const patterns = value.housekeeping_patterns ?? [];
+  return (
+    <>
+      <div className="mb-2 flex items-center gap-2">
+        <Switch
+          checked={enabled}
+          onCheckedChange={(next) => onChange({ ...value, route_housekeeping_to_cheapest_tier: next })}
+          aria-label="Route housekeeping calls to the cheapest tier"
+        />
+        <strong className="font-semibold">Route housekeeping calls to the cheapest tier</strong>
+      </div>
+      <span className="mb-3 block text-xs text-muted-foreground">
+        Conversation-title style calls skip the classifier and go to the cheapest tier.
+      </span>
+      <strong className="mb-1 block font-semibold">Additional housekeeping sentinels</strong>
+      <MultiSelect
+        options={patterns.map((pattern) => ({ label: pattern, value: pattern }))}
+        value={patterns}
+        onValueChange={(next) => onChange({ ...value, housekeeping_patterns: next.length > 0 ? next : undefined })}
+        placeholder="e.g., conversation title"
+        emptyText="Type to add a sentinel"
+        allowCustomValues
+        disabled={!enabled}
+        className="w-full"
+      />
+      <span className="mt-1 block text-xs text-muted-foreground">
+        Case-sensitive literal strings added to the built-in conversation-title sentinels.
+        {!enabled && " Turn housekeeping routing on for these to take effect."}
+      </span>
+    </>
+  );
+};
+
+export default HousekeepingRoutingControls;
diff --git a/ui/litellm-dashboard/src/components/add_model/PlanModeOverrideControls.tsx b/ui/litellm-dashboard/src/components/add_model/PlanModeOverrideControls.tsx
index 83fd0c42d16..e195fb62222 100644
--- a/ui/litellm-dashboard/src/components/add_model/PlanModeOverrideControls.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/PlanModeOverrideControls.tsx
@@ -1,5 +1,6 @@
 import React from "react";
 import { Switch } from "@/components/ui/switch";
+import { MultiSelect } from "@/components/shared/MultiSelect";
 import TierRowSelect from "./TierRowSelect";
 import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
 
@@ -38,6 +39,23 @@ const PlanModeOverrideControls: React.FC<{
         />
       </div>
     )}
+    <div className="mt-4">
+      <strong className="mb-1 block font-semibold">Additional plan-mode sentinels</strong>
+      <MultiSelect
+        options={(value.plan_mode_patterns ?? []).map((pattern) => ({ label: pattern, value: pattern }))}
+        value={value.plan_mode_patterns ?? []}
+        onValueChange={(patterns) =>
+          onChange({ ...value, plan_mode_patterns: patterns.length > 0 ? patterns : undefined })
+        }
+        placeholder="e.g., enter plan mode"
+        emptyText="Type to add a sentinel"
+        allowCustomValues
+        className="w-full"
+      />
+      <span className="mt-1 block text-xs text-muted-foreground">
+        Case-sensitive literal strings added to the built-in Claude Code and Copilot plan-mode markers.
+      </span>
+    </div>
   </>
 );
 
diff --git a/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx b/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx
new file mode 100644
index 00000000000..452f4dc727f
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx
@@ -0,0 +1,77 @@
+import React from "react";
+import { Plus, Trash2 } from "lucide-react";
+import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
+import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
+import { getReminderMarkersError, type ReminderMarkerPair } from "./build_complexity_router_config";
+
+const ReminderMarkers: React.FC<{
+  value: ComplexityRouterConfigValue;
+  onChange: (value: ComplexityRouterConfigValue) => void;
+  showValidationErrors?: boolean;
+}> = ({ value, onChange, showValidationErrors = false }) => {
+  const markers = value.reminder_markers ?? [];
+  const update = (index: number, patch: Partial<ReminderMarkerPair>) =>
+    onChange({
+      ...value,
+      reminder_markers: markers.map((marker, markerIndex) => (markerIndex === index ? { ...marker, ...patch } : marker)),
+    });
+  const remove = (index: number) => {
+    const next = markers.filter((_, markerIndex) => markerIndex !== index);
+    onChange({ ...value, reminder_markers: next.length > 0 ? next : undefined });
+  };
+  const error = getReminderMarkersError(value.reminder_markers);
+  return (
+    <div>
+      <p className="mb-4 text-sm text-muted-foreground">
+        Delimiter pairs that wrap harness-injected reminder blocks, which are stripped before classification. Setting any
+        pair replaces the built-in pairs, so list every pair your harness emits. Matching is case-insensitive and values
+        are saved lowercased.
+      </p>
+      <div className="space-y-3">
+        {markers.map((marker, index) => (
+          <div className="flex items-end gap-2" key={`${index}-${marker.open}-${marker.close}`}>
+            <div className="flex-1">
+              <label className="mb-1 block text-sm font-medium" htmlFor={`reminder-marker-${index}-open`}>
+                Opening delimiter
+              </label>
+              <Input
+                id={`reminder-marker-${index}-open`}
+                aria-label={`Reminder marker ${index + 1} opening delimiter`}
+                placeholder="<system-reminder>"
+                value={marker.open}
+                onChange={(event) => update(index, { open: event.target.value })}
+              />
+            </div>
+            <div className="flex-1">
+              <label className="mb-1 block text-sm font-medium" htmlFor={`reminder-marker-${index}-close`}>
+                Closing delimiter
+              </label>
+              <Input
+                id={`reminder-marker-${index}-close`}
+                aria-label={`Reminder marker ${index + 1} closing delimiter`}
+                placeholder="</system-reminder>"
+                value={marker.close}
+                onChange={(event) => update(index, { close: event.target.value })}
+              />
+            </div>
+            <Button variant="ghost" size="icon" aria-label={`Remove reminder marker pair ${index + 1}`} onClick={() => remove(index)}>
+              <Trash2 />
+            </Button>
+          </div>
+        ))}
+      </div>
+      <Button
+        className="mt-3"
+        variant="outline"
+        onClick={() => onChange({ ...value, reminder_markers: [...markers, { open: "", close: "" }] })}
+      >
+        <Plus />
+        Add marker pair
+      </Button>
+      {showValidationErrors && error && <p className="mt-2 text-xs text-destructive">{error}</p>}
+    </div>
+  );
+};
+
+export default ReminderMarkers;
diff --git a/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx b/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx
index 68dd880a684..9edbf204a6d 100644
--- a/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx
@@ -18,6 +18,18 @@ const ResponseFormatControls: React.FC<{
     <span className="block text-xs text-muted-foreground">
       Return the resolved underlying model name in responses instead of the autorouter alias.
     </span>
+    <div className="mt-4 flex items-center gap-2 mb-2">
+      <Switch
+        checked={value.max_tokens_from_tier_model ?? true}
+        onCheckedChange={(enabled) => onChange({ ...value, max_tokens_from_tier_model: enabled })}
+        aria-label="Cap max_tokens at the tier model's output ceiling"
+      />
+      <strong className="font-semibold">Cap max_tokens at the tier model&apos;s output ceiling</strong>
+    </div>
+    <span className="block text-xs text-muted-foreground">
+      Replace the caller&apos;s max_tokens with the routed tier model&apos;s output ceiling so one client value fits every
+      tier. Off forwards the caller&apos;s value unchanged.
+    </span>
   </>
 );
 
diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
index 84e44fee9c3..2fa28f761e3 100644
--- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
@@ -48,6 +48,8 @@ import {
   getKeywordTierRulesError,
   getClassifierModelError,
   getHeuristicV2SuccessThresholdError,
+  getReminderMarkersError,
+  getClassifierPluginTimeoutError,
   getClassifierReasoningEffortError,
   getMissingTiersError,
   getPlanModeTierError,
@@ -152,6 +154,8 @@ export const getSubmitBlockedReason = (
     getKeywordTierRulesError(keywordTierRules, activeTierRows(config)) ??
     getClassifierModelError(config) ??
     getHeuristicV2SuccessThresholdError(config.heuristic_v2_success_threshold) ??
+    getReminderMarkersError(config.reminder_markers) ??
+    getClassifierPluginTimeoutError(config.classifier_type, config.classifier_plugin_timeout_ms) ??
     (heuristicScoringRole(config) === "decides" ? customDimensionsError(config.custom_dimensions) : null) ??
     getClassifierReasoningEffortError(config, modelInfo) ??
     getReferencedModelsError(referencedModelsParams, availability)
@@ -448,6 +452,16 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
     enableContextWindowEscalation: complexityRouterConfig.enable_context_window_escalation,
     contextWindowEscalationBuffer: complexityRouterConfig.context_window_escalation_buffer,
     sessionAffinityTtlSeconds: complexityRouterConfig.session_affinity_ttl_seconds,
+    codeKeywords: complexityRouterConfig.code_keywords,
+    reasoningKeywords: complexityRouterConfig.reasoning_keywords,
+    technicalKeywords: complexityRouterConfig.technical_keywords,
+    simpleKeywords: complexityRouterConfig.simple_keywords,
+    planModePatterns: complexityRouterConfig.plan_mode_patterns,
+    routeHousekeepingToCheapestTier: complexityRouterConfig.route_housekeeping_to_cheapest_tier,
+    housekeepingPatterns: complexityRouterConfig.housekeeping_patterns,
+    reminderMarkers: complexityRouterConfig.reminder_markers,
+    maxTokensFromTierModel: complexityRouterConfig.max_tokens_from_tier_model,
+    classifierPluginTimeoutMs: complexityRouterConfig.classifier_plugin_timeout_ms,
   };
 
   const submitRecommendedRouter = async (name: string) => {
diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
index 054aba7a6aa..cf5164d91c4 100644
--- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
+++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
@@ -6,6 +6,8 @@ import {
   getKeywordTierRulesError,
   getClassifierModelError,
   getHeuristicV2SuccessThresholdError,
+  getReminderMarkersError,
+  getClassifierPluginTimeoutError,
   getClassifierReasoningEffortError,
   getMissingTiersError,
   hydrateCustomTierSet,
@@ -1482,3 +1484,59 @@ describe("classifier vision wire payload", () => {
     expect(payload.classifier_llm_config).not.toHaveProperty("vision");
   });
 });
+
+describe("advanced complexity router fields", () => {
+  it("normalizes lists, reminder markers, and explicit false values", () => {
+    const payload = buildComplexityRouterConfig({
+      ...baseParams,
+      codeKeywords: [" async ", "  "],
+      reasoningKeywords: ["prove"],
+      technicalKeywords: ["api"],
+      simpleKeywords: ["hello"],
+      planModePatterns: [" plan "],
+      routeHousekeepingToCheapestTier: false,
+      housekeepingPatterns: [" title "],
+      reminderMarkers: [{ open: " <SYSTEM> ", close: " </SYSTEM> " }],
+      maxTokensFromTierModel: false,
+      classifierType: "custom",
+      classifierPluginTimeoutMs: 3000,
+    });
+    expect(payload).toMatchObject({
+      code_keywords: ["async"],
+      reasoning_keywords: ["prove"],
+      technical_keywords: ["api"],
+      simple_keywords: ["hello"],
+      plan_mode_patterns: ["plan"],
+      route_housekeeping_to_cheapest_tier: false,
+      housekeeping_patterns: ["title"],
+      reminder_markers: [{ open: "<system>", close: "</system>" }],
+      max_tokens_from_tier_model: false,
+      classifier_plugin_timeout_ms: 3000,
+    });
+  });
+
+  it("omits defaults, empty lists, and timeout values for non-custom classifiers", () => {
+    const payload = buildComplexityRouterConfig({
+      ...baseParams,
+      codeKeywords: [" ", ""],
+      reminderMarkers: [],
+      routeHousekeepingToCheapestTier: true,
+      maxTokensFromTierModel: true,
+      classifierPluginTimeoutMs: 3000,
+    });
+    expect(payload).not.toHaveProperty("code_keywords");
+    expect(payload).not.toHaveProperty("reminder_markers");
+    expect(payload).not.toHaveProperty("route_housekeeping_to_cheapest_tier");
+    expect(payload).not.toHaveProperty("max_tokens_from_tier_model");
+    expect(payload).not.toHaveProperty("classifier_plugin_timeout_ms");
+  });
+
+  it("validates marker pairs and custom classifier timeout", () => {
+    expect(getReminderMarkersError([{ open: " <X> ", close: " <x> " }])).toContain("different");
+    expect(getReminderMarkersError([{ open: "", close: "</x>" }])).toContain("needs both");
+    expect(getReminderMarkersError([{ open: "<x>", close: "</x>" }])).toBeNull();
+    expect(getClassifierPluginTimeoutError("custom", 0)).toContain("whole number");
+    expect(getClassifierPluginTimeoutError("custom", 3000)).toBeNull();
+    expect(getClassifierPluginTimeoutError("heuristic", 0)).toBeNull();
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
index 15ae2b4c0b0..96fbe7f2c2e 100644
--- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
+++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
@@ -54,6 +54,10 @@ import {
 
 export type ClassifierVisionConfig = { enabled?: boolean; max_images?: number };
 export type ClassifierLLMConfigWire = ClassifierLLMConfig & { vision?: ClassifierVisionConfig };
+export interface ReminderMarkerPair {
+  open: string;
+  close: string;
+}
 
 /**
  * Drop an empty system_prompt so the payload carries an override only when there is one. The
@@ -181,6 +185,16 @@ export interface StoredComplexityRouterConfig {
   stall_escalation_enabled?: unknown;
   stall_escalation_window?: unknown;
   stall_escalation_repeat_threshold?: unknown;
+  code_keywords?: unknown;
+  reasoning_keywords?: unknown;
+  technical_keywords?: unknown;
+  simple_keywords?: unknown;
+  plan_mode_patterns?: unknown;
+  route_housekeeping_to_cheapest_tier?: unknown;
+  housekeeping_patterns?: unknown;
+  reminder_markers?: unknown;
+  max_tokens_from_tier_model?: unknown;
+  classifier_plugin_timeout_ms?: unknown;
 }
 
 export interface BuildComplexityRouterConfigParams {
@@ -233,6 +247,16 @@ export interface BuildComplexityRouterConfigParams {
   enableContextWindowEscalation?: boolean;
   contextWindowEscalationBuffer?: number;
   sessionAffinityTtlSeconds?: number;
+  codeKeywords?: string[];
+  reasoningKeywords?: string[];
+  technicalKeywords?: string[];
+  simpleKeywords?: string[];
+  planModePatterns?: string[];
+  routeHousekeepingToCheapestTier?: boolean;
+  housekeepingPatterns?: string[];
+  reminderMarkers?: ReminderMarkerPair[];
+  maxTokensFromTierModel?: boolean;
+  classifierPluginTimeoutMs?: number;
 }
 
 /**
@@ -302,6 +326,16 @@ export interface ComplexityRouterConfigPayload {
   enable_context_window_escalation?: boolean;
   context_window_escalation_buffer?: number;
   tier_model_configs?: Record<string, { model_name: string; litellm_params: TierModelParams }[]>;
+  code_keywords?: string[];
+  reasoning_keywords?: string[];
+  technical_keywords?: string[];
+  simple_keywords?: string[];
+  plan_mode_patterns?: string[];
+  route_housekeeping_to_cheapest_tier?: boolean;
+  housekeeping_patterns?: string[];
+  reminder_markers?: ReminderMarkerPair[];
+  max_tokens_from_tier_model?: boolean;
+  classifier_plugin_timeout_ms?: number;
 }
 
 export const serializeTierLabels = (tierLabels: ComplexityTierLabels | undefined): ComplexityTierLabels | undefined => {
@@ -376,6 +410,26 @@ export const getHeuristicV2SuccessThresholdError = (threshold: number | undefine
   return validProbability ? null : "Success threshold must be a number between 0 and 1";
 };
 
+export const getReminderMarkersError = (pairs: ReminderMarkerPair[] | undefined): string | null => {
+  for (const [index, pair] of (pairs ?? []).entries()) {
+    const open = pair.open.trim().toLowerCase();
+    const close = pair.close.trim().toLowerCase();
+    if (!open || !close) return `Reminder marker pair ${index + 1} needs both an opening and a closing delimiter`;
+    if (open === close) return `Reminder marker pair ${index + 1} must use different opening and closing delimiters`;
+  }
+  return null;
+};
+
+export const getClassifierPluginTimeoutError = (
+  classifierType: ClassifierType,
+  timeoutMs: number | undefined,
+): string | null => {
+  if (classifierType !== "custom" || timeoutMs === undefined) return null;
+  return Number.isInteger(timeoutMs) && timeoutMs > 0
+    ? null
+    : "Classifier plugin timeout must be a whole number of milliseconds greater than 0";
+};
+
 export const getClassifierModelError = (
   config: Pick<
     ComplexityRouterConfigValue,
@@ -640,6 +694,16 @@ export const buildComplexityRouterConfig = ({
   enableContextWindowEscalation,
   contextWindowEscalationBuffer,
   sessionAffinityTtlSeconds,
+  codeKeywords,
+  reasoningKeywords,
+  technicalKeywords,
+  simpleKeywords,
+  planModePatterns,
+  routeHousekeepingToCheapestTier,
+  housekeepingPatterns,
+  reminderMarkers,
+  maxTokensFromTierModel,
+  classifierPluginTimeoutMs,
 }: BuildComplexityRouterConfigParams): ComplexityRouterConfigPayload => {
   const serializedTierModelConfigs = customTierSet
     ? serializeTierModelConfigs(
@@ -672,6 +736,14 @@ export const buildComplexityRouterConfig = ({
   };
   const effectiveType = effectiveClassifierType({ custom_tier_set: customTierSet, classifier_type: classifierType });
   const forecast = isForecastClassifier(effectiveType);
+  const cleanList = (items: string[] | undefined): string[] | undefined => {
+    const cleaned = (items ?? []).map((item) => item.trim()).filter(Boolean);
+    return cleaned.length > 0 ? cleaned : undefined;
+  };
+  const cleanedReminderMarkers = reminderMarkers?.map(({ open, close }) => ({
+    open: open.trim().toLowerCase(),
+    close: close.trim().toLowerCase(),
+  }));
 
   const supportsOpeningPrompt = !customTierSet && !forecast && usesLlmClassifier(effectiveType);
   const payload: ComplexityRouterConfigPayload = {
@@ -740,6 +812,19 @@ export const buildComplexityRouterConfig = ({
     ...(sessionAffinityTtlSeconds !== undefined && {
       session_affinity_ttl_seconds: sessionAffinityTtlSeconds,
     }),
+    ...(cleanList(codeKeywords) && { code_keywords: cleanList(codeKeywords) }),
+    ...(cleanList(reasoningKeywords) && { reasoning_keywords: cleanList(reasoningKeywords) }),
+    ...(cleanList(technicalKeywords) && { technical_keywords: cleanList(technicalKeywords) }),
+    ...(cleanList(simpleKeywords) && { simple_keywords: cleanList(simpleKeywords) }),
+    ...(cleanList(planModePatterns) && { plan_mode_patterns: cleanList(planModePatterns) }),
+    ...(routeHousekeepingToCheapestTier === false && { route_housekeeping_to_cheapest_tier: false }),
+    ...(cleanList(housekeepingPatterns) && { housekeeping_patterns: cleanList(housekeepingPatterns) }),
+    ...(cleanedReminderMarkers && cleanedReminderMarkers.length > 0 && { reminder_markers: cleanedReminderMarkers }),
+    ...(maxTokensFromTierModel === false && { max_tokens_from_tier_model: false }),
+    ...(classifierType === "custom" &&
+      classifierPluginTimeoutMs !== undefined &&
+      Number.isInteger(classifierPluginTimeoutMs) &&
+      classifierPluginTimeoutMs > 0 && { classifier_plugin_timeout_ms: classifierPluginTimeoutMs }),
     ...scorerKnobs,
   };
   if (!customTierSet) return payload;
diff --git a/ui/litellm-dashboard/src/components/add_model/classifier_types.ts b/ui/litellm-dashboard/src/components/add_model/classifier_types.ts
index ec88166ed2e..aa9d5619052 100644
--- a/ui/litellm-dashboard/src/components/add_model/classifier_types.ts
+++ b/ui/litellm-dashboard/src/components/add_model/classifier_types.ts
@@ -6,7 +6,8 @@ export type ClassifierType =
   | "heuristic_first"
   | "hybrid"
   | "capability"
-  | "llm_v2";
+  | "llm_v2"
+  | "custom";
 
 export const usesLlmClassifier = (classifierType: ClassifierType): boolean =>
   (["llm", "heuristic_first", "hybrid", "capability", "llm_v2"] as const).some((type) => type === classifierType);
diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts
index 2450f7bce27..eaa6595d8a3 100644
--- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts
+++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts
@@ -854,6 +854,16 @@ describe("managed keys survive an untouched open-and-save", () => {
     reasoning_override_min_score: 0.3,
     enable_context_window_escalation: false,
     context_window_escalation_buffer: 0.9,
+    code_keywords: ["async", "await"],
+    reasoning_keywords: ["prove"],
+    technical_keywords: ["api"],
+    simple_keywords: ["hello"],
+    plan_mode_patterns: ["plan now"],
+    route_housekeeping_to_cheapest_tier: false,
+    housekeeping_patterns: ["conversation title"],
+    reminder_markers: [{ open: "<system-reminder>", close: "</system-reminder>" }],
+    max_tokens_from_tier_model: false,
+    classifier_plugin_timeout_ms: 3000,
   };
 
   // tier_definitions and fallback_tier cannot sit beside heuristic_first, which this fixture uses,
@@ -864,6 +874,7 @@ describe("managed keys survive an untouched open-and-save", () => {
     "fallback_tier",
     "hybrid_boundary_margin",
     "jev_classifier_config",
+    "classifier_plugin_timeout_ms",
   ]);
 
   // The stall keys are rejected beside the session pinning and user-turn classification this
@@ -894,6 +905,12 @@ describe("managed keys survive an untouched open-and-save", () => {
     expect(dropped).toEqual([]);
   });
 
+  it("keeps the custom classifier plugin timeout through an untouched save", () => {
+    const stored = { ...STORED_ALL_MANAGED, classifier_type: "custom", classifier_plugin_timeout_ms: 3000 };
+    const hydrated = hydrateComplexityRouterConfig(stored, undefined);
+    expect(buildUpdatedComplexityRouterConfig(stored, hydrated).classifier_plugin_timeout_ms).toBe(3000);
+  });
+
   it("carries an enabled non-reasoning tier and its models through their own round trip", () => {
     // `tiers` is rewritten wholesale on save, so this is the regression that matters: opening an
     // enabled router and saving an unrelated edit must not delete the tier or its pool.
diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
index c88bbb101f7..476207553f7 100644
--- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
+++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
@@ -45,6 +45,8 @@ import {
   buildComplexityRouterConfig,
   getClassifierModelError,
   getHeuristicV2SuccessThresholdError,
+  getReminderMarkersError,
+  getClassifierPluginTimeoutError,
   getClassifierReasoningEffortError,
   getKeywordTierRulesError,
   getMissingTiersError,
@@ -113,6 +115,8 @@ export const hydrateComplexityRouterConfig = (
   parsedConfig: StoredComplexityRouterConfig,
   complexityRouterDefaultModel: string | null | undefined,
 ): ComplexityRouterConfigValue => {
+  const stringList = (input: unknown): string[] | undefined =>
+    Array.isArray(input) ? input.filter((item): item is string => typeof item === "string") : undefined;
   const builtIn = hydrateBuiltInTiers(parsedConfig.tiers, parsedConfig.enable_non_reasoning_tier);
   const { tiers: hydratedTiers, enable_non_reasoning_tier } = builtIn;
   const custom_tier_set = hydrateCustomTierSet(parsedConfig);
@@ -219,6 +223,31 @@ export const hydrateComplexityRouterConfig = (
       typeof parsedConfig.stall_escalation_repeat_threshold === "number"
         ? parsedConfig.stall_escalation_repeat_threshold
         : undefined,
+    code_keywords: stringList(parsedConfig.code_keywords),
+    reasoning_keywords: stringList(parsedConfig.reasoning_keywords),
+    technical_keywords: stringList(parsedConfig.technical_keywords),
+    simple_keywords: stringList(parsedConfig.simple_keywords),
+    plan_mode_patterns: stringList(parsedConfig.plan_mode_patterns),
+    route_housekeeping_to_cheapest_tier:
+      typeof parsedConfig.route_housekeeping_to_cheapest_tier === "boolean"
+        ? parsedConfig.route_housekeeping_to_cheapest_tier
+        : undefined,
+    housekeeping_patterns: stringList(parsedConfig.housekeeping_patterns),
+    reminder_markers: Array.isArray(parsedConfig.reminder_markers)
+      ? parsedConfig.reminder_markers.filter(
+          (pair): pair is { open: string; close: string } =>
+            typeof pair === "object" &&
+            pair !== null &&
+            typeof (pair as { open?: unknown }).open === "string" &&
+            typeof (pair as { close?: unknown }).close === "string",
+        )
+      : undefined,
+    max_tokens_from_tier_model:
+      typeof parsedConfig.max_tokens_from_tier_model === "boolean" ? parsedConfig.max_tokens_from_tier_model : undefined,
+    classifier_plugin_timeout_ms:
+      typeof parsedConfig.classifier_plugin_timeout_ms === "number" && Number.isFinite(parsedConfig.classifier_plugin_timeout_ms)
+        ? parsedConfig.classifier_plugin_timeout_ms
+        : undefined,
   };
 };
 
@@ -266,6 +295,16 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
   "stall_escalation_enabled",
   "stall_escalation_window",
   "stall_escalation_repeat_threshold",
+  "code_keywords",
+  "reasoning_keywords",
+  "technical_keywords",
+  "simple_keywords",
+  "plan_mode_patterns",
+  "route_housekeeping_to_cheapest_tier",
+  "housekeeping_patterns",
+  "reminder_markers",
+  "max_tokens_from_tier_model",
+  "classifier_plugin_timeout_ms",
 ]);
 
 // Managed only when the caller passes the corresponding state. A caller that does not render
@@ -387,6 +426,16 @@ export const buildUpdatedComplexityRouterConfig = (
     stallEscalationEnabled: value.stall_escalation_enabled,
     stallEscalationWindow: value.stall_escalation_window,
     stallEscalationRepeatThreshold: value.stall_escalation_repeat_threshold,
+    codeKeywords: value.code_keywords,
+    reasoningKeywords: value.reasoning_keywords,
+    technicalKeywords: value.technical_keywords,
+    simpleKeywords: value.simple_keywords,
+    planModePatterns: value.plan_mode_patterns,
+    routeHousekeepingToCheapestTier: value.route_housekeeping_to_cheapest_tier,
+    housekeepingPatterns: value.housekeeping_patterns,
+    reminderMarkers: value.reminder_markers,
+    maxTokensFromTierModel: value.max_tokens_from_tier_model,
+    classifierPluginTimeoutMs: value.classifier_plugin_timeout_ms,
   };
   const built = buildComplexityRouterConfig(builderParams);
 
@@ -585,6 +634,11 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
       const classifierError =
         getClassifierModelError(complexityRouterConfig) ??
         getHeuristicV2SuccessThresholdError(complexityRouterConfig.heuristic_v2_success_threshold) ??
+        getReminderMarkersError(complexityRouterConfig.reminder_markers) ??
+        getClassifierPluginTimeoutError(
+          complexityRouterConfig.classifier_type,
+          complexityRouterConfig.classifier_plugin_timeout_ms,
+        ) ??
         getForecastConfigError(complexityRouterConfig) ??
         (heuristicScoringRole(complexityRouterConfig) === "decides"
           ? customDimensionsError(complexityRouterConfig.custom_dimensions)

From d2f30a77fded3a39c32b5dfc2f7f464bcd8f0409 Mon Sep 17 00:00:00 2001
From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
Date: Mon, 21 Sep 2026 12:31:48 -0700
Subject: [PATCH 048/160] test(mcp): preserve toolset scope across explicit
 context

---
 .../mcp_server/test_mcp_toolset_scope.py         | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py
index 60098b1f656..1398884783e 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py
@@ -58,6 +58,22 @@ class TestApplyToolsetScope:
         assert set(op.mcp_servers or []) == {"server-a", "server-b"}
         assert op.mcp_tool_permissions == toolset_perms
 
+        from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager
+        from litellm.proxy._experimental.mcp_server.operations import prepare_context
+
+        manager = MCPServerManager()
+        unscoped_open = await manager.operator_open_server_ids(
+            auth, allow_all_server_ids=["operator-open-outside-toolset"], submitted_server_ids=[]
+        )
+        scoped_open = await manager.operator_open_server_ids(
+            prepare_context(result).user_api_key_auth,
+            allow_all_server_ids=["operator-open-outside-toolset"],
+            submitted_server_ids=[],
+        )
+        assert unscoped_open == {"operator-open-outside-toolset"}
+        assert scoped_open == set()
+        assert auth.mcp_toolset_id is None
+
     @pytest.mark.asyncio
     async def test_admin_creates_object_permission_when_none(self):
         """Admin key with object_permission=None can access any toolset."""

From 9c411dd6f2e569ea7ac54c08f03847d5c70cddba Mon Sep 17 00:00:00 2001
From: yucheng <yucheng@berri.ai>
Date: Mon, 21 Sep 2026 19:35:29 +0000
Subject: [PATCH 049/160] refactor(proxy): make scheduled job shutdown timeouts
 configurable via env

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/constants.py                          |  2 ++
 litellm/proxy/shutdown/scheduled_jobs.py      | 23 +++++++++++--------
 .../proxy/shutdown/test_scheduled_jobs.py     | 15 ++++++------
 3 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/litellm/constants.py b/litellm/constants.py
index bbeb4846e27..842adf62f6b 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -1742,6 +1742,8 @@ SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS: Final = float(
 SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS: Final = float(os.getenv("SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS", "300"))
 SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS: Final = float(os.getenv("SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS", "30"))
 SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP: Final = int(os.getenv("SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP", "100000"))
+SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS: Final = float(os.getenv("SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS", "5"))
+SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS: Final = float(os.getenv("SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS", "5"))
 TOOL_SPEND_TOP_TOOLS: Final = 100
 SPEND_LOG_PARTITION_INTERVAL: Final = os.getenv("SPEND_LOG_PARTITION_INTERVAL", "day")
 SPEND_LOG_PARTITION_PRECREATE_AHEAD: Final = int(os.getenv("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7))
diff --git a/litellm/proxy/shutdown/scheduled_jobs.py b/litellm/proxy/shutdown/scheduled_jobs.py
index 5345d380112..cf4937780b8 100644
--- a/litellm/proxy/shutdown/scheduled_jobs.py
+++ b/litellm/proxy/shutdown/scheduled_jobs.py
@@ -7,9 +7,10 @@ from typing import Final, Protocol
 from apscheduler.executors.asyncio import AsyncIOExecutor
 
 from litellm._logging import verbose_proxy_logger
-
-JOB_FINISH_TIMEOUT_SECONDS: Final = 5.0
-JOB_CANCEL_TIMEOUT_SECONDS: Final = 5.0
+from litellm.constants import (
+    SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS,
+    SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS,
+)
 
 
 class StoppableScheduler(Protocol):
@@ -41,8 +42,8 @@ def pause_scheduled_jobs(scheduler: StoppableScheduler) -> None:
 
 async def stop_in_flight_scheduler_jobs(scheduler: StoppableScheduler, executor: AwaitableAsyncIOExecutor) -> None:
     """
-    Let in-flight jobs finish for up to JOB_FINISH_TIMEOUT_SECONDS, then stop the scheduler and
-    wait, bounded by JOB_CANCEL_TIMEOUT_SECONDS, for the jobs it cancels.
+    Let in-flight jobs finish for up to SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS, then stop the scheduler and
+    wait, bounded by SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS, for the jobs it cancels.
 
     Must run before the database is disconnected: a write job that finishes needs its connection,
     and a job's cancellation handler is what records the run's outcome.
@@ -52,19 +53,23 @@ async def stop_in_flight_scheduler_jobs(scheduler: StoppableScheduler, executor:
     in_flight: Final = executor.in_flight_jobs()
     if in_flight:
         verbose_proxy_logger.info(
-            "Waiting up to %ss for %d in-flight scheduled job(s) to finish", JOB_FINISH_TIMEOUT_SECONDS, len(in_flight)
+            "Waiting up to %ss for %d in-flight scheduled job(s) to finish",
+            SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS,
+            len(in_flight),
         )
     still_running: Final = (
-        (await asyncio.wait(in_flight, timeout=JOB_FINISH_TIMEOUT_SECONDS))[1] if in_flight else frozenset()
+        (await asyncio.wait(in_flight, timeout=SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS))[1]
+        if in_flight
+        else frozenset()
     )
     scheduler.shutdown(wait=False)
     if not still_running:
         return
     verbose_proxy_logger.info("Cancelling %d in-flight scheduled job(s) for shutdown", len(still_running))
-    _done, pending = await asyncio.wait(still_running, timeout=JOB_CANCEL_TIMEOUT_SECONDS)
+    _done, pending = await asyncio.wait(still_running, timeout=SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS)
     if pending:
         verbose_proxy_logger.warning(
             "%d scheduled job(s) did not finish within %ss of cancellation; giving up on them",
             len(pending),
-            JOB_CANCEL_TIMEOUT_SECONDS,
+            SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS,
         )
diff --git a/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py b/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
index 7defd6cef6c..1f94cee04ed 100644
--- a/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
+++ b/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
@@ -7,11 +7,11 @@ from datetime import datetime, timedelta
 import pytest
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 
-import litellm.proxy.shutdown.scheduled_jobs as scheduled_jobs
+from litellm.constants import SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS
 from litellm.proxy.shutdown.scheduled_jobs import (
     AwaitableAsyncIOExecutor,
-    stop_in_flight_scheduler_jobs,
     pause_scheduled_jobs,
+    stop_in_flight_scheduler_jobs,
 )
 
 
@@ -75,9 +75,8 @@ async def test_in_flight_jobs_observe_cancellation_before_shutdown_returns():
 
 
 @pytest.mark.asyncio
-async def test_a_job_that_is_finishing_is_allowed_to_finish_rather_than_cancelled(monkeypatch):
+async def test_a_job_that_is_finishing_is_allowed_to_finish_rather_than_cancelled():
     """A spend write cancelled mid-commit drops the rows it popped, so short jobs get to finish first"""
-    monkeypatch.setattr(scheduled_jobs, "JOB_FINISH_TIMEOUT_SECONDS", 2.0)
     write = _Job(work_seconds=0.2)
     stuck = _Job()
     async with _running_scheduler(write, stuck) as (scheduler, executor):
@@ -99,16 +98,18 @@ async def test_every_in_flight_job_is_cancelled_not_only_the_first():
 
 
 @pytest.mark.asyncio
-async def test_a_job_that_ignores_cancellation_is_abandoned_after_the_timeout(monkeypatch, caplog):
+async def test_a_job_that_ignores_cancellation_is_abandoned_after_the_timeout(caplog):
     """A job that swallows CancelledError must not hold the pod past its termination grace period"""
-    monkeypatch.setattr(scheduled_jobs, "JOB_CANCEL_TIMEOUT_SECONDS", 0.05)
     job = _Job(swallow_cancellation=True)
     async with _running_scheduler(job) as (scheduler, executor):
         with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
             await stop_in_flight_scheduler_jobs(scheduler, executor)
 
         assert job.events == ["cancelled"]
-        assert "1 scheduled job(s) did not finish within 0.05s of cancellation" in caplog.text
+        assert (
+            f"1 scheduled job(s) did not finish within {SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS}s of cancellation"
+            in caplog.text
+        )
 
 
 @pytest.mark.asyncio

From 4e388e6aea52ad6c9c2939997f860689e69716aa Mon Sep 17 00:00:00 2001
From: yucheng <yucheng@berri.ai>
Date: Mon, 21 Sep 2026 19:37:28 +0000
Subject: [PATCH 050/160] refactor(proxy): inject scheduled job shutdown
 timeouts

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/shutdown/scheduled_jobs.py      | 20 ++++++++++++-------
 .../proxy/shutdown/test_scheduled_jobs.py     | 10 +++-------
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/litellm/proxy/shutdown/scheduled_jobs.py b/litellm/proxy/shutdown/scheduled_jobs.py
index cf4937780b8..e920ce19eb9 100644
--- a/litellm/proxy/shutdown/scheduled_jobs.py
+++ b/litellm/proxy/shutdown/scheduled_jobs.py
@@ -40,10 +40,16 @@ def pause_scheduled_jobs(scheduler: StoppableScheduler) -> None:
         scheduler.pause()
 
 
-async def stop_in_flight_scheduler_jobs(scheduler: StoppableScheduler, executor: AwaitableAsyncIOExecutor) -> None:
+async def stop_in_flight_scheduler_jobs(
+    scheduler: StoppableScheduler,
+    executor: AwaitableAsyncIOExecutor,
+    *,
+    finish_timeout_seconds: float = SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS,
+    cancel_timeout_seconds: float = SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS,
+) -> None:
     """
-    Let in-flight jobs finish for up to SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS, then stop the scheduler and
-    wait, bounded by SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS, for the jobs it cancels.
+    Let in-flight jobs finish for up to finish_timeout_seconds, then stop the scheduler and wait, bounded by
+    cancel_timeout_seconds, for the jobs it cancels.
 
     Must run before the database is disconnected: a write job that finishes needs its connection,
     and a job's cancellation handler is what records the run's outcome.
@@ -54,11 +60,11 @@ async def stop_in_flight_scheduler_jobs(scheduler: StoppableScheduler, executor:
     if in_flight:
         verbose_proxy_logger.info(
             "Waiting up to %ss for %d in-flight scheduled job(s) to finish",
-            SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS,
+            finish_timeout_seconds,
             len(in_flight),
         )
     still_running: Final = (
-        (await asyncio.wait(in_flight, timeout=SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS))[1]
+        (await asyncio.wait(in_flight, timeout=finish_timeout_seconds))[1]
         if in_flight
         else frozenset()
     )
@@ -66,10 +72,10 @@ async def stop_in_flight_scheduler_jobs(scheduler: StoppableScheduler, executor:
     if not still_running:
         return
     verbose_proxy_logger.info("Cancelling %d in-flight scheduled job(s) for shutdown", len(still_running))
-    _done, pending = await asyncio.wait(still_running, timeout=SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS)
+    _done, pending = await asyncio.wait(still_running, timeout=cancel_timeout_seconds)
     if pending:
         verbose_proxy_logger.warning(
             "%d scheduled job(s) did not finish within %ss of cancellation; giving up on them",
             len(pending),
-            SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS,
+            cancel_timeout_seconds,
         )
diff --git a/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py b/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
index 1f94cee04ed..fbce38db39f 100644
--- a/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
+++ b/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py
@@ -7,7 +7,6 @@ from datetime import datetime, timedelta
 import pytest
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 
-from litellm.constants import SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS
 from litellm.proxy.shutdown.scheduled_jobs import (
     AwaitableAsyncIOExecutor,
     pause_scheduled_jobs,
@@ -80,7 +79,7 @@ async def test_a_job_that_is_finishing_is_allowed_to_finish_rather_than_cancelle
     write = _Job(work_seconds=0.2)
     stuck = _Job()
     async with _running_scheduler(write, stuck) as (scheduler, executor):
-        await stop_in_flight_scheduler_jobs(scheduler, executor)
+        await stop_in_flight_scheduler_jobs(scheduler, executor, finish_timeout_seconds=2.0)
 
         assert write.events == ["committed", "finished"]
         assert stuck.events == ["cancelled", "finished"]
@@ -103,13 +102,10 @@ async def test_a_job_that_ignores_cancellation_is_abandoned_after_the_timeout(ca
     job = _Job(swallow_cancellation=True)
     async with _running_scheduler(job) as (scheduler, executor):
         with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
-            await stop_in_flight_scheduler_jobs(scheduler, executor)
+            await stop_in_flight_scheduler_jobs(scheduler, executor, cancel_timeout_seconds=0.05)
 
         assert job.events == ["cancelled"]
-        assert (
-            f"1 scheduled job(s) did not finish within {SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS}s of cancellation"
-            in caplog.text
-        )
+        assert "1 scheduled job(s) did not finish within 0.05s of cancellation" in caplog.text
 
 
 @pytest.mark.asyncio

From 9644032cb806a8bef55d1bcf4219ddec4886b758 Mon Sep 17 00:00:00 2001
From: yuneng <yuneng@berri.ai>
Date: Mon, 21 Sep 2026 19:37:45 +0000
Subject: [PATCH 051/160] refactor(ui): split complexity router form files and
 cover advanced fields

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../add_model/ClassificationMethodConfig.tsx  | 118 +--------
 .../ClassifierPluginTimeoutField.tsx          |  54 ++++
 .../add_model/ClassifierTypeRadios.tsx        |  89 +++++++
 .../ComplexityRouterAdvancedSections.tsx      | 240 +++++++++++++++++
 .../add_model/ComplexityRouterConfig.test.tsx |  58 +++++
 .../add_model/ComplexityRouterConfig.tsx      | 210 +++------------
 .../components/add_model/ReminderMarkers.tsx  |   4 +-
 .../add_model/add_auto_router_tab.tsx         |  59 +----
 .../build_complexity_router_config.test.ts    |  16 ++
 .../build_complexity_router_config.ts         |  17 +-
 .../complexity_router_builder_params.ts       |  74 ++++++
 ...dit_auto_router_modal.integration.test.tsx |  71 +++++
 .../edit_auto_router_modal.tsx                | 243 +-----------------
 .../hydrate_complexity_router_config.ts       | 183 +++++++++++++
 14 files changed, 839 insertions(+), 597 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/components/add_model/ClassifierPluginTimeoutField.tsx
 create mode 100644 ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx
 create mode 100644 ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx
 create mode 100644 ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts
 create mode 100644 ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts

diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
index 2564137fb02..b7a0fd67443 100644
--- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
@@ -19,10 +19,9 @@ import HeuristicScoringConfig from "./HeuristicScoringConfig";
 import ClassifierReasoningEffortSelect from "./ClassifierReasoningEffortSelect";
 import ClassifierCircuitBreakerConfig from "./ClassifierCircuitBreakerConfig";
 import ClassifierVisionConfig from "./ClassifierVisionConfig";
-import {
-  getClassifierPluginTimeoutError,
-  getHeuristicV2SuccessThresholdError,
-} from "./build_complexity_router_config";
+import { getHeuristicV2SuccessThresholdError } from "./build_complexity_router_config";
+import ClassifierPluginTimeoutField from "./ClassifierPluginTimeoutField";
+import ClassifierTypeRadios from "./ClassifierTypeRadios";
 import type { ReasoningEffort } from "./complexity_router_tiers";
 import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults";
 import {
@@ -64,7 +63,6 @@ const CLASSIFIER_CONTEXT_WINDOW_SIZE_ID = "classifier-context-window-size";
 const CLASSIFIER_CONTEXT_BUDGET_CHARS_ID = "classifier-context-budget-chars";
 const HYBRID_BOUNDARY_MARGIN_ID = "hybrid-boundary-margin";
 const HEURISTIC_V2_SUCCESS_THRESHOLD_ID = "heuristic-v2-success-threshold";
-const CLASSIFIER_PLUGIN_TIMEOUT_ID = "classifier-plugin-timeout-ms";
 
 const CUSTOM_PROMPT_WITH_HEURISTIC_FALLBACK =
   "This router classifies with your own prompt, so the tier comes from whatever rubric it states. The four tier " +
@@ -208,84 +206,6 @@ export const InactiveHeuristicV2Threshold: React.FC<Pick<ClassificationMethodCon
   );
 };
 
-const ClassifierTypeRadios: React.FC<{
-  value: ComplexityRouterConfigValue;
-  classifierType: ClassifierType;
-  onTypeChange: (classifierType: ClassifierType) => void;
-}> = ({ value, classifierType, onTypeChange }) => {
-  const scorerLocked = Boolean(value.custom_tier_set);
-  const scorerLockedReason = restrictedBy(value, "heuristicClassifier")?.reason;
-  return (
-    <RadioGroup
-      value={classifierType}
-      onValueChange={(classifierType: unknown) => onTypeChange(classifierType as ClassifierType)}
-      className="w-full"
-    >
-      <div className="flex w-full flex-col items-start gap-2">
-        <SimpleTooltip content={scorerLockedReason}>
-          <Label className="items-start font-normal leading-normal has-data-disabled:cursor-not-allowed has-data-disabled:opacity-50">
-            <RadioGroupItem value="heuristic" className="mt-0.5" disabled={scorerLocked} />
-            <span>
-              <strong className="font-semibold">Heuristic</strong>{" "}
-              <span className="text-muted-foreground">
-                (default), rule-based scoring with no API calls and &lt;1ms latency
-              </span>
-            </span>
-          </Label>
-        </SimpleTooltip>
-        <SimpleTooltip content={scorerLockedReason}>
-          <Label className="items-start font-normal leading-normal has-data-disabled:cursor-not-allowed has-data-disabled:opacity-50">
-            <RadioGroupItem value="heuristic_v2" className="mt-0.5" disabled={scorerLocked} />
-            <span>
-              <strong className="font-semibold">Heuristic v2</strong>{" "}
-              <span className="text-muted-foreground">
-                uses bundled calibrated four-tier probabilities with no API call
-              </span>
-            </span>
-          </Label>
-        </SimpleTooltip>
-        <Label className="items-start font-normal leading-normal">
-          <RadioGroupItem value="llm" className="mt-0.5" />
-          <span>
-            <strong className="font-semibold">LLM Classifier</strong>{" "}
-            <span className="text-muted-foreground">calls a model to decide the tier (e.g. a small/fast model)</span>
-          </span>
-        </Label>
-        <Label className="items-start font-normal leading-normal">
-          <RadioGroupItem value="jev" className="mt-0.5" />
-          <span>
-            <strong className="font-semibold">JEV Classifier</strong>{" "}
-            <span className="text-muted-foreground">uses TypeSafe System One Choice to decide the tier</span>
-          </span>
-        </Label>
-        <SimpleTooltip content={scorerLockedReason}>
-          <Label className="items-start font-normal leading-normal has-data-disabled:cursor-not-allowed has-data-disabled:opacity-50">
-            <RadioGroupItem value="heuristic_first" className="mt-0.5" disabled={scorerLocked} />
-            <span>
-              <strong className="font-semibold">Heuristic first</strong>{" "}
-              <span className="text-muted-foreground">
-                scores locally, and only pays for the classifier when the score does not confidently land a cheap tier
-              </span>
-            </span>
-          </Label>
-        </SimpleTooltip>
-        <SimpleTooltip content={scorerLockedReason}>
-          <Label className="items-start font-normal leading-normal has-data-disabled:cursor-not-allowed has-data-disabled:opacity-50">
-            <RadioGroupItem value="hybrid" className="mt-0.5" disabled={scorerLocked} />
-            <span>
-              <strong className="font-semibold">Hybrid</strong>{" "}
-              <span className="text-muted-foreground">
-                keeps the local score at any tier, and only pays for the classifier when that score lands near a tier
-                boundary
-              </span>
-            </span>
-          </Label>
-        </SimpleTooltip>
-      </div>
-    </RadioGroup>
-  );
-};
-
 const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
   value,
   onChange,
@@ -472,37 +392,7 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
       <ClassifierTypeRadios value={value} classifierType={classifierType} onTypeChange={handleClassifierTypeChange} />
 
       {classifierType === "custom" && (
-        <div className="mt-4 space-y-2">
-          <p className="text-sm text-muted-foreground">
-            This router uses a custom classifier plugin set in config.yaml. Pick a classifier below to replace it.
-          </p>
-          <Label htmlFor={CLASSIFIER_PLUGIN_TIMEOUT_ID} className="block font-semibold">
-            Classifier plugin timeout (ms)
-          </Label>
-          <Input
-            id={CLASSIFIER_PLUGIN_TIMEOUT_ID}
-            inputMode="numeric"
-            placeholder="3000"
-            value={value.classifier_plugin_timeout_ms ?? ""}
-            onChange={(event) =>
-              onChange({
-                ...value,
-                classifier_plugin_timeout_ms: event.target.value.trim() === "" ? undefined : Number(event.target.value),
-              })
-            }
-            aria-invalid={Boolean(
-              showValidationErrors && getClassifierPluginTimeoutError(classifierType, value.classifier_plugin_timeout_ms),
-            )}
-          />
-          <p className="text-sm text-muted-foreground">
-            Time budget for the plugin call. On expiry the fallback path decides the tier.
-          </p>
-          {showValidationErrors && getClassifierPluginTimeoutError(classifierType, value.classifier_plugin_timeout_ms) && (
-            <p className="text-sm text-destructive" role="alert">
-              {getClassifierPluginTimeoutError(classifierType, value.classifier_plugin_timeout_ms)}
-            </p>
-          )}
-        </div>
+        <ClassifierPluginTimeoutField value={value} onChange={onChange} showValidationErrors={showValidationErrors} />
       )}
 
       {classifierType === "heuristic_v2" && (
diff --git a/ui/litellm-dashboard/src/components/add_model/ClassifierPluginTimeoutField.tsx b/ui/litellm-dashboard/src/components/add_model/ClassifierPluginTimeoutField.tsx
new file mode 100644
index 00000000000..45decf09a09
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/ClassifierPluginTimeoutField.tsx
@@ -0,0 +1,54 @@
+import React from "react";
+import { Input } from "@/components/ui/input";
+import { Label } from "@/components/ui/label";
+import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
+import { getClassifierPluginTimeoutError } from "./build_complexity_router_config";
+
+const CLASSIFIER_PLUGIN_TIMEOUT_ID = "classifier-plugin-timeout-ms";
+
+interface ClassifierPluginTimeoutFieldProps {
+  value: ComplexityRouterConfigValue;
+  onChange: (value: ComplexityRouterConfigValue) => void;
+  showValidationErrors?: boolean;
+}
+
+const ClassifierPluginTimeoutField: React.FC<ClassifierPluginTimeoutFieldProps> = ({
+  value,
+  onChange,
+  showValidationErrors = false,
+}) => {
+  const error = getClassifierPluginTimeoutError("custom", value.classifier_plugin_timeout_ms);
+  return (
+    <div className="mt-4 space-y-2">
+      <p className="text-sm text-muted-foreground">
+        This router uses a custom classifier plugin set in config.yaml. Pick a classifier below to replace it.
+      </p>
+      <Label htmlFor={CLASSIFIER_PLUGIN_TIMEOUT_ID} className="block font-semibold">
+        Classifier plugin timeout (ms)
+      </Label>
+      <Input
+        id={CLASSIFIER_PLUGIN_TIMEOUT_ID}
+        inputMode="numeric"
+        placeholder="3000"
+        value={value.classifier_plugin_timeout_ms ?? ""}
+        onChange={(event) =>
+          onChange({
+            ...value,
+            classifier_plugin_timeout_ms: event.target.value.trim() === "" ? undefined : Number(event.target.value),
+          })
+        }
+        aria-invalid={Boolean(showValidationErrors && error)}
+      />
+      <p className="text-sm text-muted-foreground">
+        Time budget for the plugin call. On expiry the fallback path decides the tier.
+      </p>
+      {showValidationErrors && error && (
+        <p className="text-sm text-destructive" role="alert">
+          {error}
+        </p>
+      )}
+    </div>
+  );
+};
+
+export default ClassifierPluginTimeoutField;
diff --git a/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx b/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx
new file mode 100644
index 00000000000..1602e19069a
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx
@@ -0,0 +1,89 @@
+import React from "react";
+import { Label } from "@/components/ui/label";
+import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group";
+import { SimpleTooltip } from "@/components/ui/tooltip";
+import type { ClassifierType } from "./classifier_types";
+import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
+import { restrictedBy } from "./TierRestrictions";
+
+interface ClassifierTypeRadiosProps {
+  value: ComplexityRouterConfigValue;
+  classifierType: ClassifierType;
+  onTypeChange: (classifierType: ClassifierType) => void;
+}
+
+const ClassifierTypeRadios: React.FC<ClassifierTypeRadiosProps> = ({ value, classifierType, onTypeChange }) => {
+  const scorerLocked = Boolean(value.custom_tier_set);
+  const scorerLockedReason = restrictedBy(value, "heuristicClassifier")?.reason;
+  return (
+    <RadioGroup
+      value={classifierType}
+      onValueChange={(nextType: unknown) => onTypeChange(nextType as ClassifierType)}
+      className="w-full"
+    >
+      <div className="flex w-full flex-col items-start gap-2">
+        <SimpleTooltip content={scorerLockedReason}>
+          <Label className="items-start font-normal leading-normal has-data-disabled:cursor-not-allowed has-data-disabled:opacity-50">
+            <RadioGroupItem value="heuristic" className="mt-0.5" disabled={scorerLocked} />
+            <span>
+              <strong className="font-semibold">Heuristic</strong>{" "}
+              <span className="text-muted-foreground">
+                (default), rule-based scoring with no API calls and &lt;1ms latency
+              </span>
+            </span>
+          </Label>
+        </SimpleTooltip>
+        <SimpleTooltip content={scorerLockedReason}>
+          <Label className="items-start font-normal leading-normal has-data-disabled:cursor-not-allowed has-data-disabled:opacity-50">
+            <RadioGroupItem value="heuristic_v2" className="mt-0.5" disabled={scorerLocked} />
+            <span>
+              <strong className="font-semibold">Heuristic v2</strong>{" "}
+              <span className="text-muted-foreground">
+                uses bundled calibrated four-tier probabilities with no API call
+              </span>
+            </span>
+          </Label>
+        </SimpleTooltip>
+        <Label className="items-start font-normal leading-normal">
+          <RadioGroupItem value="llm" className="mt-0.5" />
+          <span>
+            <strong className="font-semibold">LLM Classifier</strong>{" "}
+            <span className="text-muted-foreground">calls a model to decide the tier (e.g. a small/fast model)</span>
+          </span>
+        </Label>
+        <Label className="items-start font-normal leading-normal">
+          <RadioGroupItem value="jev" className="mt-0.5" />
+          <span>
+            <strong className="font-semibold">JEV Classifier</strong>{" "}
+            <span className="text-muted-foreground">uses TypeSafe System One Choice to decide the tier</span>
+          </span>
+        </Label>
+        <SimpleTooltip content={scorerLockedReason}>
+          <Label className="items-start font-normal leading-normal has-data-disabled:cursor-not-allowed has-data-disabled:opacity-50">
+            <RadioGroupItem value="heuristic_first" className="mt-0.5" disabled={scorerLocked} />
+            <span>
+              <strong className="font-semibold">Heuristic first</strong>{" "}
+              <span className="text-muted-foreground">
+                scores locally, and only pays for the classifier when the score does not confidently land a cheap tier
+              </span>
+            </span>
+          </Label>
+        </SimpleTooltip>
+        <SimpleTooltip content={scorerLockedReason}>
+          <Label className="items-start font-normal leading-normal has-data-disabled:cursor-not-allowed has-data-disabled:opacity-50">
+            <RadioGroupItem value="hybrid" className="mt-0.5" disabled={scorerLocked} />
+            <span>
+              <strong className="font-semibold">Hybrid</strong>{" "}
+              <span className="text-muted-foreground">
+                keeps the local score at any tier, and only pays for the classifier when that score lands near a tier
+                boundary
+              </span>
+            </span>
+          </Label>
+        </SimpleTooltip>
+      </div>
+    </RadioGroup>
+  );
+};
+
+export default ClassifierTypeRadios;
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx
new file mode 100644
index 00000000000..dd822907735
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx
@@ -0,0 +1,240 @@
+import React from "react";
+import { ChevronRight } from "lucide-react";
+import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
+import { Separator } from "@/components/ui/separator";
+import type { ModelGroup } from "@/components/llm_calls/fetch_models";
+import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
+import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig";
+import ClassificationMethodConfig from "./ClassificationMethodConfig";
+import ContextWindowEscalationConfig from "./ContextWindowEscalationConfig";
+import ResponseFormatControls from "./ResponseFormatControls";
+import StallEscalationConfig from "./StallEscalationConfig";
+import { Restricted, restrictedBy } from "./TierRestrictions";
+import EscalationKeywords from "./EscalationKeywords";
+import KeywordTierRules, { type KeywordTierRule } from "./KeywordTierRules";
+import SemanticKeywordMatching from "./SemanticKeywordMatching";
+import CompressionControls from "./CompressionControls";
+import PlanModeOverrideControls from "./PlanModeOverrideControls";
+import { AffinityControls } from "./AffinityControls";
+import { ModalityRoutingControls } from "./ModalityRoutingControls";
+import HeuristicKeywordOverrides from "./HeuristicKeywordOverrides";
+import HousekeepingRoutingControls from "./HousekeepingRoutingControls";
+import ReminderMarkers from "./ReminderMarkers";
+import type { AutoRouterCompressionState } from "./buildAutoRouterCompression";
+import { activeTierName, type TierRow } from "./tier_rows";
+
+interface ComplexityRouterAdvancedSectionsProps {
+  value: ComplexityRouterConfigValue;
+  onChange: (value: ComplexityRouterConfigValue) => void;
+  forecast: boolean;
+  modelOptions: { value: string; label: string }[];
+  classifierEffortOptionsByModel: Record<string, string[]>;
+  customTechnicalKeywords?: string[];
+  onCustomTechnicalKeywordsChange?: (keywords: string[]) => void;
+  showValidationErrors: boolean;
+  defaultModel?: string;
+  planModeTierOptions: { value: string; label: string }[];
+  keywordTierRules: KeywordTierRule[];
+  onKeywordTierRulesChange?: (rules: KeywordTierRule[]) => void;
+  semanticMatchingEnabled: boolean;
+  onSemanticMatchingEnabledChange?: (enabled: boolean) => void;
+  embeddingModel?: string;
+  onEmbeddingModelChange: (model: string) => void;
+  matchThreshold: number;
+  onMatchThresholdChange: (threshold: number) => void;
+  escalationKeywords: string[];
+  onEscalationKeywordsChange?: (keywords: string[]) => void;
+  autoRouterCompression: AutoRouterCompressionState;
+  onAutoRouterCompressionChange?: (state: AutoRouterCompressionState) => void;
+  modelInfo: ModelGroup[];
+  tierRows: TierRow[];
+  customTierSet: ComplexityRouterConfigValue["custom_tier_set"];
+}
+
+const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSectionsProps> = ({
+  value,
+  onChange,
+  forecast,
+  modelOptions,
+  classifierEffortOptionsByModel,
+  customTechnicalKeywords,
+  onCustomTechnicalKeywordsChange,
+  showValidationErrors,
+  defaultModel,
+  planModeTierOptions,
+  keywordTierRules,
+  onKeywordTierRulesChange,
+  semanticMatchingEnabled,
+  onSemanticMatchingEnabledChange,
+  embeddingModel,
+  onEmbeddingModelChange,
+  matchThreshold,
+  onMatchThresholdChange,
+  escalationKeywords,
+  onEscalationKeywordsChange,
+  autoRouterCompression,
+  onAutoRouterCompressionChange,
+  modelInfo,
+  tierRows,
+  customTierSet,
+}) => {
+  const sections = [
+    ...(!forecast
+      ? [
+          {
+            key: "classifier",
+            label: <strong className="text-foreground font-semibold">Advanced: Classification Method</strong>,
+            children: (
+              <ClassificationMethodConfig
+                value={value}
+                onChange={onChange}
+                modelOptions={modelOptions}
+                effortOptionsByModel={classifierEffortOptionsByModel}
+                customTechnicalKeywords={customTechnicalKeywords}
+                onCustomTechnicalKeywordsChange={onCustomTechnicalKeywordsChange}
+                showValidationErrors={showValidationErrors}
+                defaultModel={defaultModel}
+              />
+            ),
+          },
+        ]
+      : []),
+    ...(!forecast
+      ? [
+          {
+            key: "keyword-overrides",
+            label: <strong className="text-foreground font-semibold">Advanced: Heuristic Keyword Overrides</strong>,
+            children: <HeuristicKeywordOverrides value={value} onChange={onChange} />,
+          },
+        ]
+      : []),
+    {
+      key: "adaptive",
+      label: <strong className="text-foreground font-semibold">Advanced: Adaptive Routing</strong>,
+      children: (
+        <Restricted by={restrictedBy(value, "adaptive")}>
+          <AdaptiveRoutingConfig value={value} onChange={onChange} />
+        </Restricted>
+      ),
+    },
+    {
+      key: "affinity",
+      label: <strong className="text-foreground font-semibold">Advanced: Affinity</strong>,
+      children: <AffinityControls value={value} onChange={onChange} />,
+    },
+    {
+      key: "modality",
+      label: <strong className="text-foreground font-semibold">Advanced: Modality Routing</strong>,
+      children: <ModalityRoutingControls value={value} onChange={onChange} />,
+    },
+    {
+      key: "plan-mode",
+      label: <strong className="text-foreground font-semibold">Advanced: Plan-Mode Override</strong>,
+      children: <PlanModeOverrideControls value={value} onChange={onChange} planModeTierOptions={planModeTierOptions} />,
+    },
+    {
+      key: "housekeeping",
+      label: <strong className="text-foreground font-semibold">Advanced: Housekeeping Routing</strong>,
+      children: <HousekeepingRoutingControls value={value} onChange={onChange} />,
+    },
+    {
+      key: "reminder-markers",
+      label: <strong className="text-foreground font-semibold">Advanced: Reminder Markers</strong>,
+      children: <ReminderMarkers value={value} onChange={onChange} showValidationErrors={showValidationErrors} />,
+    },
+    {
+      key: "context-window",
+      label: <strong className="text-foreground font-semibold">Advanced: Context Window Escalation</strong>,
+      children: <ContextWindowEscalationConfig value={value} onChange={onChange} />,
+    },
+    {
+      key: "stall-escalation",
+      label: <strong className="text-foreground font-semibold">Advanced: Stalled Task Escalation</strong>,
+      children: (
+        <Restricted by={restrictedBy(value, "stallEscalation")}>
+          <StallEscalationConfig value={value} onChange={onChange} />
+        </Restricted>
+      ),
+    },
+    {
+      key: "response",
+      label: <strong className="text-foreground font-semibold">Advanced: Response Format</strong>,
+      children: <ResponseFormatControls value={value} onChange={onChange} />,
+    },
+    ...(onEscalationKeywordsChange
+      ? [
+          {
+            key: "escalation",
+            label: <strong className="text-foreground font-semibold">Advanced: Escalation Keywords</strong>,
+            children: (
+              <Restricted by={restrictedBy(value, "escalation")}>
+                <EscalationKeywords keywords={escalationKeywords} onChange={onEscalationKeywordsChange} />
+              </Restricted>
+            ),
+          },
+        ]
+      : []),
+    ...(onAutoRouterCompressionChange
+      ? [
+          {
+            key: "compression",
+            label: <strong className="text-foreground font-semibold">Advanced: Compression</strong>,
+            children: <CompressionControls value={autoRouterCompression} onChange={onAutoRouterCompressionChange} />,
+          },
+        ]
+      : []),
+    ...(onKeywordTierRulesChange || onSemanticMatchingEnabledChange
+      ? [
+          {
+            key: "keyword-semantic",
+            label: <strong className="text-foreground font-semibold">Advanced: Keyword/Semantic Matching</strong>,
+            children: (
+              <>
+                {onKeywordTierRulesChange && (
+                  <KeywordTierRules
+                    rules={keywordTierRules}
+                    onChange={onKeywordTierRulesChange}
+                    tierLabels={value.tier_labels}
+                    tierNames={
+                      customTierSet || forecast ? tierRows.map(activeTierName).filter(Boolean) : undefined
+                    }
+                  />
+                )}
+                {onKeywordTierRulesChange && onSemanticMatchingEnabledChange && <Separator className="my-4" />}
+                {onSemanticMatchingEnabledChange && (
+                  <SemanticKeywordMatching
+                    enabled={semanticMatchingEnabled}
+                    onEnabledChange={onSemanticMatchingEnabledChange}
+                    embeddingModel={embeddingModel}
+                    onEmbeddingModelChange={onEmbeddingModelChange}
+                    matchThreshold={matchThreshold}
+                    onMatchThresholdChange={onMatchThresholdChange}
+                    modelInfo={modelInfo}
+                    showValidationErrors={showValidationErrors}
+                  />
+                )}
+              </>
+            ),
+          },
+        ]
+      : []),
+  ];
+
+  return (
+    <>
+      {sections
+        .filter(({ key }) => !forecast || !["adaptive", "context-window", "escalation"].includes(key))
+        .map(({ key, label, children }) => (
+          <Collapsible key={key} className="border-b border-border last:border-b-0">
+            <CollapsibleTrigger className="group flex w-full items-center gap-2 px-4 py-3 text-left">
+              <ChevronRight className="size-4 shrink-0 text-muted-foreground transition-transform group-data-panel-open:rotate-90" />
+              {label}
+            </CollapsibleTrigger>
+            <CollapsibleContent className="px-4 pb-4">{children}</CollapsibleContent>
+          </Collapsible>
+        ))}
+    </>
+  );
+};
+
+export default ComplexityRouterAdvancedSections;
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
index 70658b787f0..9a8577100df 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
@@ -96,6 +96,64 @@ describe("ComplexityRouterConfig", () => {
     expect(screen.queryByText("Classifier Model")).not.toBeInTheDocument();
   });
 
+  it("shows heuristic advanced sections and hides keyword overrides for capability classifiers", () => {
+    const { rerender } = renderWithProviders(<ComplexityRouterConfig {...baseProps} />);
+
+    expect(screen.getByText("Advanced: Heuristic Keyword Overrides")).toBeInTheDocument();
+    expect(screen.getByText("Advanced: Housekeeping Routing")).toBeInTheDocument();
+    expect(screen.getByText("Advanced: Reminder Markers")).toBeInTheDocument();
+
+    const capabilityValue = { ...defaultValue, classifier_type: "capability" as const };
+    rerender(<ComplexityRouterConfig {...baseProps} value={capabilityValue} />);
+    expect(screen.queryByText("Advanced: Heuristic Keyword Overrides")).not.toBeInTheDocument();
+
+  });
+
+  it.each([
+    ["custom", true],
+    ["heuristic", false],
+  ] as const)("shows plugin timeout only for %s classifiers", (classifierType, visible) => {
+    renderWithProviders(
+      <ComplexityRouterConfig {...baseProps} value={{ ...defaultValue, classifier_type: classifierType }} />,
+    );
+    fireEvent.click(screen.getByText("Advanced: Classification Method"));
+    if (visible) {
+      expect(screen.getByLabelText("Classifier plugin timeout (ms)")).toBeInTheDocument();
+    } else {
+      expect(screen.queryByLabelText("Classifier plugin timeout (ms)")).not.toBeInTheDocument();
+    }
+  });
+
+  it.each([true, false])("shows reminder marker validation only when requested: %s", (showValidationErrors) => {
+    const value = { ...defaultValue, reminder_markers: [{ open: "", close: "x" }] };
+    renderWithProviders(
+      <ComplexityRouterConfig
+        {...baseProps}
+        value={value}
+        showValidationErrors={showValidationErrors}
+      />,
+    );
+    fireEvent.click(screen.getByText("Advanced: Reminder Markers"));
+    const validation = screen.queryByText(/needs both/i);
+    if (showValidationErrors) {
+      expect(validation).toBeInTheDocument();
+    } else {
+      expect(validation).not.toBeInTheDocument();
+    }
+  });
+
+  it("disables housekeeping sentinels when cheapest-tier routing is off", () => {
+    renderWithProviders(
+      <ComplexityRouterConfig
+        {...baseProps}
+        value={{ ...defaultValue, route_housekeeping_to_cheapest_tier: false }}
+      />,
+    );
+    fireEvent.click(screen.getByText("Advanced: Housekeeping Routing"));
+    const sentinelInput = screen.getByRole("combobox", { name: "e.g., conversation title" });
+    expect(sentinelInput).toBeDisabled();
+  });
+
   it("should toggle returning the raw model name", async () => {
     const user = userEvent.setup();
     const onChange = vi.fn();
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
index a97e7a77a21..acf6b62a95a 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
@@ -2,21 +2,17 @@ import RoutingOptions from "./RoutingOptions";
 import type { JevClassifierConfig } from "./jev_classifier_config";
 import { type ClassifierType } from "./classifier_types";
 export { type ClassifierType, usesLlmClassifier, usesClassifierContext } from "./classifier_types";
-import PlanModeOverrideControls from "./PlanModeOverrideControls";
 import ForecastClassifierConfig, { ForecastSolverModels } from "./ForecastClassifierConfig";
 import { isForecastClassifier, type CapabilitySettings, type FuseSettings } from "./forecast_classifier_config";
 import { SimpleTooltip } from "@/components/ui/tooltip";
 import { MultiSelect } from "@/components/shared/MultiSelect";
 import DefaultModelField from "./DefaultModelField";
-import { ChevronRight, Info, Plus, Trash2, X } from "lucide-react";
+import { Info, Plus, Trash2, X } from "lucide-react";
 
-import { AffinityControls } from "./AffinityControls";
 import NonReasoningTierToggle from "./NonReasoningTierToggle";
 import TierConfigIntro from "./TierConfigIntro";
 import TierRowSelect from "./TierRowSelect";
-import { ModalityRoutingControls } from "./ModalityRoutingControls";
 import { Card, CardContent } from "@/components/ui/card";
-import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
 import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group";
 import { Separator } from "@/components/ui/separator";
 import { Button } from "@/components/ui/button";
@@ -39,12 +35,8 @@ import {
 } from "./tier_rows";
 import React from "react";
 import { ModelGroup } from "@/components/llm_calls/fetch_models";
-import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig";
-import ClassificationMethodConfig, { InactiveHeuristicV2Threshold } from "./ClassificationMethodConfig";
-import ContextWindowEscalationConfig from "./ContextWindowEscalationConfig";
-import ResponseFormatControls from "./ResponseFormatControls";
-import StallEscalationConfig from "./StallEscalationConfig";
-import { Restricted, restrictedBy } from "./TierRestrictions";
+import { InactiveHeuristicV2Threshold } from "./ClassificationMethodConfig";
+import ComplexityRouterAdvancedSections from "./ComplexityRouterAdvancedSections";
 import { type TierSetAction, applyTierSetAction, setFallbackTier } from "./tier_set_actions";
 import {
   ReasoningEffort,
@@ -56,16 +48,10 @@ import {
   tierRowLabel,
 } from "./complexity_router_tiers";
 import TierModelEffortRows from "./TierModelEffortRows";
-import EscalationKeywords from "./EscalationKeywords";
-import KeywordTierRules, { KeywordTierRule } from "./KeywordTierRules";
-import SemanticKeywordMatching from "./SemanticKeywordMatching";
+import { KeywordTierRule } from "./KeywordTierRules";
 import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from "./heuristic_scoring_knobs";
 import { type CustomDimensionRow } from "./custom_dimensions";
-import CompressionControls from "./CompressionControls";
 import { type AutoRouterCompressionState, DEFAULT_AUTO_ROUTER_COMPRESSION } from "./buildAutoRouterCompression";
-import HeuristicKeywordOverrides from "./HeuristicKeywordOverrides";
-import HousekeepingRoutingControls from "./HousekeepingRoutingControls";
-import ReminderMarkers from "./ReminderMarkers";
 import { type ReminderMarkerPair } from "./build_complexity_router_config";
 
 export type { DimensionWeights, TierBoundaries, TokenThresholds };
@@ -782,167 +768,33 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
           </>
         )}
         <div className="rounded-lg border border-border bg-muted">
-          {[
-            ...(!forecast
-              ? [
-                  {
-                    key: "classifier",
-                    label: <strong className="text-foreground font-semibold">Advanced: Classification Method</strong>,
-                    children: (
-                      <ClassificationMethodConfig
-                        value={value}
-                        onChange={onChange}
-                        modelOptions={modelOptions}
-                        effortOptionsByModel={classifierEffortOptionsByModel}
-                        customTechnicalKeywords={customTechnicalKeywords}
-                        onCustomTechnicalKeywordsChange={onCustomTechnicalKeywordsChange}
-                        showValidationErrors={showValidationErrors}
-                        defaultModel={defaultModel}
-                      />
-                    ),
-                  },
-              ]
-              : []),
-            ...(!forecast
-              ? [
-                  {
-                    key: "keyword-overrides",
-                    label: (
-                      <strong className="text-foreground font-semibold">Advanced: Heuristic Keyword Overrides</strong>
-                    ),
-                    children: <HeuristicKeywordOverrides value={value} onChange={onChange} />,
-                  },
-                ]
-              : []),
-            {
-              key: "adaptive",
-              label: <strong className="text-foreground font-semibold">Advanced: Adaptive Routing</strong>,
-              children: (
-                <Restricted by={restrictedBy(value, "adaptive")}>
-                  <AdaptiveRoutingConfig value={value} onChange={onChange} />
-                </Restricted>
-              ),
-            },
-            {
-              key: "affinity",
-              label: <strong className="text-foreground font-semibold">Advanced: Affinity</strong>,
-              children: <AffinityControls value={value} onChange={onChange} />,
-            },
-            {
-              key: "modality",
-              label: <strong className="text-foreground font-semibold">Advanced: Modality Routing</strong>,
-              children: <ModalityRoutingControls value={value} onChange={onChange} />,
-            },
-            {
-              key: "plan-mode",
-              label: <strong className="text-foreground font-semibold">Advanced: Plan-Mode Override</strong>,
-              children: (
-                <PlanModeOverrideControls value={value} onChange={onChange} planModeTierOptions={planModeTierOptions} />
-              ),
-            },
-            {
-              key: "housekeeping",
-              label: <strong className="text-foreground font-semibold">Advanced: Housekeeping Routing</strong>,
-              children: <HousekeepingRoutingControls value={value} onChange={onChange} />,
-            },
-            {
-              key: "reminder-markers",
-              label: <strong className="text-foreground font-semibold">Advanced: Reminder Markers</strong>,
-              children: <ReminderMarkers value={value} onChange={onChange} showValidationErrors={showValidationErrors} />,
-            },
-            {
-              key: "context-window",
-              label: <strong className="text-foreground font-semibold">Advanced: Context Window Escalation</strong>,
-              children: <ContextWindowEscalationConfig value={value} onChange={onChange} />,
-            },
-            {
-              key: "stall-escalation",
-              label: <strong className="text-foreground font-semibold">Advanced: Stalled Task Escalation</strong>,
-              children: (
-                <Restricted by={restrictedBy(value, "stallEscalation")}>
-                  <StallEscalationConfig value={value} onChange={onChange} />
-                </Restricted>
-              ),
-            },
-            {
-              key: "response",
-              label: <strong className="text-foreground font-semibold">Advanced: Response Format</strong>,
-              children: <ResponseFormatControls value={value} onChange={onChange} />,
-            },
-            ...(onEscalationKeywordsChange
-              ? [
-                  {
-                    key: "escalation",
-                    label: <strong className="text-foreground font-semibold">Advanced: Escalation Keywords</strong>,
-                    children: (
-                      <Restricted by={restrictedBy(value, "escalation")}>
-                        <EscalationKeywords keywords={escalationKeywords} onChange={onEscalationKeywordsChange} />
-                      </Restricted>
-                    ),
-                  },
-                ]
-              : []),
-            ...(onAutoRouterCompressionChange
-              ? [
-                  {
-                    key: "compression",
-                    label: <strong className="text-foreground font-semibold">Advanced: Compression</strong>,
-                    children: (
-                      <CompressionControls value={autoRouterCompression} onChange={onAutoRouterCompressionChange} />
-                    ),
-                  },
-                ]
-              : []),
-            ...(onKeywordTierRulesChange || onSemanticMatchingEnabledChange
-              ? [
-                  {
-                    key: "keyword-semantic",
-                    label: (
-                      <strong className="text-foreground font-semibold">Advanced: Keyword/Semantic Matching</strong>
-                    ),
-                    children: (
-                      <>
-                        {onKeywordTierRulesChange && (
-                          <KeywordTierRules
-                            rules={keywordTierRules}
-                            onChange={onKeywordTierRulesChange}
-                            tierLabels={value.tier_labels}
-                            tierNames={
-                              customTierSet || isForecastClassifier(value.classifier_type)
-                                ? tierRows.map(activeTierName).filter(Boolean)
-                                : undefined
-                            }
-                          />
-                        )}
-                        {onKeywordTierRulesChange && onSemanticMatchingEnabledChange && <Separator className="my-4" />}
-                        {onSemanticMatchingEnabledChange && (
-                          <SemanticKeywordMatching
-                            enabled={semanticMatchingEnabled}
-                            onEnabledChange={onSemanticMatchingEnabledChange}
-                            embeddingModel={embeddingModel}
-                            onEmbeddingModelChange={onEmbeddingModelChange}
-                            matchThreshold={matchThreshold}
-                            onMatchThresholdChange={onMatchThresholdChange}
-                            modelInfo={modelInfo}
-                            showValidationErrors={showValidationErrors}
-                          />
-                        )}
-                      </>
-                    ),
-                  },
-                ]
-              : []),
-          ]
-            .filter(({ key }) => !forecast || !["adaptive", "context-window", "escalation"].includes(key))
-            .map(({ key, label, children }) => (
-              <Collapsible key={key} className="border-b border-border last:border-b-0">
-                <CollapsibleTrigger className="group flex w-full items-center gap-2 px-4 py-3 text-left">
-                  <ChevronRight className="size-4 shrink-0 text-muted-foreground transition-transform group-data-panel-open:rotate-90" />
-                  {label}
-                </CollapsibleTrigger>
-                <CollapsibleContent className="px-4 pb-4">{children}</CollapsibleContent>
-              </Collapsible>
-            ))}
+          <ComplexityRouterAdvancedSections
+            value={value}
+            onChange={onChange}
+            forecast={forecast}
+            modelOptions={modelOptions}
+            classifierEffortOptionsByModel={classifierEffortOptionsByModel}
+            customTechnicalKeywords={customTechnicalKeywords}
+            onCustomTechnicalKeywordsChange={onCustomTechnicalKeywordsChange}
+            showValidationErrors={showValidationErrors}
+            defaultModel={defaultModel}
+            planModeTierOptions={planModeTierOptions}
+            keywordTierRules={keywordTierRules}
+            onKeywordTierRulesChange={onKeywordTierRulesChange}
+            semanticMatchingEnabled={semanticMatchingEnabled}
+            onSemanticMatchingEnabledChange={onSemanticMatchingEnabledChange}
+            embeddingModel={embeddingModel}
+            onEmbeddingModelChange={onEmbeddingModelChange}
+            matchThreshold={matchThreshold}
+            onMatchThresholdChange={onMatchThresholdChange}
+            escalationKeywords={escalationKeywords}
+            onEscalationKeywordsChange={onEscalationKeywordsChange}
+            autoRouterCompression={autoRouterCompression}
+            onAutoRouterCompressionChange={onAutoRouterCompressionChange}
+            modelInfo={modelInfo}
+            tierRows={tierRows}
+            customTierSet={customTierSet}
+          />
         </div>
       </RoutingOptions>
     </div>
diff --git a/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx b/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx
index 452f4dc727f..970394291d4 100644
--- a/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx
@@ -30,14 +30,13 @@ const ReminderMarkers: React.FC<{
       </p>
       <div className="space-y-3">
         {markers.map((marker, index) => (
-          <div className="flex items-end gap-2" key={`${index}-${marker.open}-${marker.close}`}>
+          <div className="flex items-end gap-2" key={index}>
             <div className="flex-1">
               <label className="mb-1 block text-sm font-medium" htmlFor={`reminder-marker-${index}-open`}>
                 Opening delimiter
               </label>
               <Input
                 id={`reminder-marker-${index}-open`}
-                aria-label={`Reminder marker ${index + 1} opening delimiter`}
                 placeholder="<system-reminder>"
                 value={marker.open}
                 onChange={(event) => update(index, { open: event.target.value })}
@@ -49,7 +48,6 @@ const ReminderMarkers: React.FC<{
               </label>
               <Input
                 id={`reminder-marker-${index}-close`}
-                aria-label={`Reminder marker ${index + 1} closing delimiter`}
                 placeholder="</system-reminder>"
                 value={marker.close}
                 onChange={(event) => update(index, { close: event.target.value })}
diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
index 2fa28f761e3..f805a5b5511 100644
--- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
@@ -28,10 +28,6 @@ import ComplexityRouterConfig, {
   effectiveClassifierType,
   usesLlmClassifier,
   heuristicScoringRole,
-  DEFAULT_ADAPTIVE_WEIGHTS,
-  DEFAULT_SESSION_AFFINITY,
-  DEFAULT_DEPLOYMENT_AFFINITY,
-  DEFAULT_TIER_DISTANCE_PENALTY,
 } from "./ComplexityRouterConfig";
 import { KeywordTierRule } from "./KeywordTierRules";
 import { customDimensionsError } from "./custom_dimensions";
@@ -57,6 +53,7 @@ import {
   getTierLabelsError,
   dryRunRejection,
 } from "./build_complexity_router_config";
+import { builderParamsFromValue } from "./complexity_router_builder_params";
 import { activeTierName, activeTierRows, getCustomTierRowsError, resolveComplexityDefaultModel } from "./tier_rows";
 import { tierRowLabel } from "./complexity_router_tiers";
 import { buildAutoRouterTestTargets, AutoRouterTestTarget } from "./build_auto_router_test_targets";
@@ -403,65 +400,13 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
   );
 
   const complexityRouterConfigParams: BuildComplexityRouterConfigParams = {
-    tiers: complexityRouterConfig.tiers,
-    enableNonReasoningTier: complexityRouterConfig.enable_non_reasoning_tier,
-    customTierSet: complexityRouterConfig.custom_tier_set,
-    defaultModel: complexityRouterConfig.default_model,
-    planModeMinTier: complexityRouterConfig.plan_mode_min_tier,
-    classificationPrompt: complexityRouterConfig.classification_prompt,
-    classificationExamples: complexityRouterConfig.classification_examples,
-    heuristicFirstMaxTier: complexityRouterConfig.heuristic_first_max_tier,
-    hybridBoundaryMargin: complexityRouterConfig.hybrid_boundary_margin,
-    classificationMode: complexityRouterConfig.classification_mode,
-    tierLabels: complexityRouterConfig.tier_labels,
-    classifierType: complexityRouterConfig.classifier_type,
-    jevClassifierConfig: complexityRouterConfig.jev_classifier_config,
-    heuristicV2SuccessThreshold: complexityRouterConfig.heuristic_v2_success_threshold,
-    capabilityClassifierConfig: complexityRouterConfig.capability_classifier_config,
-    llmV2Config: complexityRouterConfig.llm_v2_config,
-    classifierLlmConfig: complexityRouterConfig.classifier_llm_config,
-    classifierContextWindowSize: complexityRouterConfig.classifier_context_window_size,
-    classifierContextBudgetChars: complexityRouterConfig.classifier_context_budget_chars,
-    classifierContextPerTurnChars: complexityRouterConfig.classifier_context_per_turn_chars,
-    classifierContextIncludeAssistantTurns: complexityRouterConfig.classifier_context_include_assistant_turns,
-    classifierFallback: complexityRouterConfig.classifier_fallback,
-    sessionAffinity: complexityRouterConfig.session_affinity ?? DEFAULT_SESSION_AFFINITY,
-    modalityRouting: complexityRouterConfig.modality_routing ?? false,
-    modalityPinOverride: complexityRouterConfig.modality_pin_override ?? false,
-    deploymentAffinity: complexityRouterConfig.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY,
+    ...builderParamsFromValue(complexityRouterConfig),
     customTechnicalKeywords,
     keywordTierRules,
     semanticMatchingEnabled,
     embeddingModel,
     matchThreshold,
     escalationKeywords,
-    stallEscalationEnabled: complexityRouterConfig.stall_escalation_enabled,
-    stallEscalationWindow: complexityRouterConfig.stall_escalation_window,
-    stallEscalationRepeatThreshold: complexityRouterConfig.stall_escalation_repeat_threshold,
-    adaptive: complexityRouterConfig.adaptive ?? false,
-    adaptiveWeights: complexityRouterConfig.adaptive_weights ?? DEFAULT_ADAPTIVE_WEIGHTS,
-    tierDistancePenalty: complexityRouterConfig.tier_distance_penalty ?? DEFAULT_TIER_DISTANCE_PENALTY,
-    adaptiveEligible: complexityRouterConfig.adaptive_eligible ?? "all",
-    returnRawModelName: complexityRouterConfig.return_raw_model_name ?? false,
-    tierModelParams: complexityRouterConfig.tier_model_params,
-    tierBoundaries: complexityRouterConfig.tier_boundaries,
-    tokenThresholds: complexityRouterConfig.token_thresholds,
-    dimensionWeights: complexityRouterConfig.dimension_weights,
-    customDimensions: complexityRouterConfig.custom_dimensions,
-    reasoningOverrideMinScore: complexityRouterConfig.reasoning_override_min_score,
-    enableContextWindowEscalation: complexityRouterConfig.enable_context_window_escalation,
-    contextWindowEscalationBuffer: complexityRouterConfig.context_window_escalation_buffer,
-    sessionAffinityTtlSeconds: complexityRouterConfig.session_affinity_ttl_seconds,
-    codeKeywords: complexityRouterConfig.code_keywords,
-    reasoningKeywords: complexityRouterConfig.reasoning_keywords,
-    technicalKeywords: complexityRouterConfig.technical_keywords,
-    simpleKeywords: complexityRouterConfig.simple_keywords,
-    planModePatterns: complexityRouterConfig.plan_mode_patterns,
-    routeHousekeepingToCheapestTier: complexityRouterConfig.route_housekeeping_to_cheapest_tier,
-    housekeepingPatterns: complexityRouterConfig.housekeeping_patterns,
-    reminderMarkers: complexityRouterConfig.reminder_markers,
-    maxTokensFromTierModel: complexityRouterConfig.max_tokens_from_tier_model,
-    classifierPluginTimeoutMs: complexityRouterConfig.classifier_plugin_timeout_ms,
   };
 
   const submitRecommendedRouter = async (name: string) => {
diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
index cf5164d91c4..6db2b8213d1 100644
--- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
+++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
@@ -1531,6 +1531,22 @@ describe("advanced complexity router fields", () => {
     expect(payload).not.toHaveProperty("classifier_plugin_timeout_ms");
   });
 
+  it.each([
+    "code_keywords",
+    "reasoning_keywords",
+    "technical_keywords",
+    "simple_keywords",
+    "plan_mode_patterns",
+    "route_housekeeping_to_cheapest_tier",
+    "housekeeping_patterns",
+    "reminder_markers",
+    "max_tokens_from_tier_model",
+    "classifier_plugin_timeout_ms",
+  ])("omits unset advanced field %s", (key) => {
+    const payload = buildComplexityRouterConfig(baseParams);
+    expect(payload).not.toHaveProperty(key);
+  });
+
   it("validates marker pairs and custom classifier timeout", () => {
     expect(getReminderMarkersError([{ open: " <X> ", close: " <x> " }])).toContain("different");
     expect(getReminderMarkersError([{ open: "", close: "</x>" }])).toContain("needs both");
diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
index 96fbe7f2c2e..ab71b6b3cce 100644
--- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
+++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
@@ -744,6 +744,16 @@ export const buildComplexityRouterConfig = ({
     open: open.trim().toLowerCase(),
     close: close.trim().toLowerCase(),
   }));
+  const cleanedLists = Object.fromEntries(
+    Object.entries({
+      code_keywords: cleanList(codeKeywords),
+      reasoning_keywords: cleanList(reasoningKeywords),
+      technical_keywords: cleanList(technicalKeywords),
+      simple_keywords: cleanList(simpleKeywords),
+      plan_mode_patterns: cleanList(planModePatterns),
+      housekeeping_patterns: cleanList(housekeepingPatterns),
+    }).filter(([, list]) => list !== undefined),
+  );
 
   const supportsOpeningPrompt = !customTierSet && !forecast && usesLlmClassifier(effectiveType);
   const payload: ComplexityRouterConfigPayload = {
@@ -812,13 +822,8 @@ export const buildComplexityRouterConfig = ({
     ...(sessionAffinityTtlSeconds !== undefined && {
       session_affinity_ttl_seconds: sessionAffinityTtlSeconds,
     }),
-    ...(cleanList(codeKeywords) && { code_keywords: cleanList(codeKeywords) }),
-    ...(cleanList(reasoningKeywords) && { reasoning_keywords: cleanList(reasoningKeywords) }),
-    ...(cleanList(technicalKeywords) && { technical_keywords: cleanList(technicalKeywords) }),
-    ...(cleanList(simpleKeywords) && { simple_keywords: cleanList(simpleKeywords) }),
-    ...(cleanList(planModePatterns) && { plan_mode_patterns: cleanList(planModePatterns) }),
+    ...cleanedLists,
     ...(routeHousekeepingToCheapestTier === false && { route_housekeeping_to_cheapest_tier: false }),
-    ...(cleanList(housekeepingPatterns) && { housekeeping_patterns: cleanList(housekeepingPatterns) }),
     ...(cleanedReminderMarkers && cleanedReminderMarkers.length > 0 && { reminder_markers: cleanedReminderMarkers }),
     ...(maxTokensFromTierModel === false && { max_tokens_from_tier_model: false }),
     ...(classifierType === "custom" &&
diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts
new file mode 100644
index 00000000000..124a85ce9a3
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts
@@ -0,0 +1,74 @@
+import type { BuildComplexityRouterConfigParams } from "./build_complexity_router_config";
+import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
+import {
+  DEFAULT_ADAPTIVE_WEIGHTS,
+  DEFAULT_DEPLOYMENT_AFFINITY,
+  DEFAULT_SESSION_AFFINITY,
+  DEFAULT_TIER_DISTANCE_PENALTY,
+} from "./ComplexityRouterConfig";
+
+export const builderParamsFromValue = (
+  value: ComplexityRouterConfigValue,
+): Omit<
+  BuildComplexityRouterConfigParams,
+  | "customTechnicalKeywords"
+  | "keywordTierRules"
+  | "semanticMatchingEnabled"
+  | "embeddingModel"
+  | "matchThreshold"
+  | "escalationKeywords"
+> => ({
+  tiers: value.tiers,
+  enableNonReasoningTier: value.enable_non_reasoning_tier,
+  customTierSet: value.custom_tier_set,
+  defaultModel: value.default_model,
+  planModeMinTier: value.plan_mode_min_tier,
+  classificationPrompt: value.classification_prompt,
+  classificationExamples: value.classification_examples,
+  heuristicFirstMaxTier: value.heuristic_first_max_tier,
+  hybridBoundaryMargin: value.hybrid_boundary_margin,
+  classificationMode: value.classification_mode,
+  tierLabels: value.tier_labels,
+  classifierType: value.classifier_type,
+  jevClassifierConfig: value.jev_classifier_config,
+  heuristicV2SuccessThreshold: value.heuristic_v2_success_threshold,
+  capabilityClassifierConfig: value.capability_classifier_config,
+  llmV2Config: value.llm_v2_config,
+  classifierLlmConfig: value.classifier_llm_config,
+  classifierContextWindowSize: value.classifier_context_window_size,
+  classifierContextBudgetChars: value.classifier_context_budget_chars,
+  classifierContextPerTurnChars: value.classifier_context_per_turn_chars,
+  classifierContextIncludeAssistantTurns: value.classifier_context_include_assistant_turns,
+  classifierFallback: value.classifier_fallback,
+  sessionAffinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY,
+  sessionAffinityTtlSeconds: value.session_affinity_ttl_seconds,
+  modalityRouting: value.modality_routing ?? false,
+  modalityPinOverride: value.modality_pin_override ?? false,
+  deploymentAffinity: value.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY,
+  adaptive: value.adaptive ?? false,
+  adaptiveWeights: value.adaptive_weights ?? DEFAULT_ADAPTIVE_WEIGHTS,
+  tierDistancePenalty: value.tier_distance_penalty ?? DEFAULT_TIER_DISTANCE_PENALTY,
+  adaptiveEligible: value.adaptive_eligible ?? "all",
+  returnRawModelName: value.return_raw_model_name ?? false,
+  tierBoundaries: value.tier_boundaries,
+  tokenThresholds: value.token_thresholds,
+  dimensionWeights: value.dimension_weights,
+  customDimensions: value.custom_dimensions,
+  reasoningOverrideMinScore: value.reasoning_override_min_score,
+  tierModelParams: value.tier_model_params,
+  enableContextWindowEscalation: value.enable_context_window_escalation,
+  contextWindowEscalationBuffer: value.context_window_escalation_buffer,
+  stallEscalationEnabled: value.stall_escalation_enabled,
+  stallEscalationWindow: value.stall_escalation_window,
+  stallEscalationRepeatThreshold: value.stall_escalation_repeat_threshold,
+  codeKeywords: value.code_keywords,
+  reasoningKeywords: value.reasoning_keywords,
+  technicalKeywords: value.technical_keywords,
+  simpleKeywords: value.simple_keywords,
+  planModePatterns: value.plan_mode_patterns,
+  routeHousekeepingToCheapestTier: value.route_housekeeping_to_cheapest_tier,
+  housekeepingPatterns: value.housekeeping_patterns,
+  reminderMarkers: value.reminder_markers,
+  maxTokensFromTierModel: value.max_tokens_from_tier_model,
+  classifierPluginTimeoutMs: value.classifier_plugin_timeout_ms,
+});
diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx
index 34db61483cf..aa6cf92ceb9 100644
--- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx
+++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx
@@ -347,6 +347,77 @@ describe("EditAutoRouterModal keyword matching", () => {
   });
 });
 
+describe("EditAutoRouterModal advanced field round trips", () => {
+  const storedAdvancedConfig = {
+    ...STORED_CONFIG,
+    route_housekeeping_to_cheapest_tier: false,
+    housekeeping_patterns: ["conversation title"],
+    reminder_markers: [{ open: "<a>", close: "</a>" }],
+    max_tokens_from_tier_model: false,
+  };
+
+  const renderAdvancedModal = (props: Partial<React.ComponentProps<typeof EditAutoRouterModal>> = {}) =>
+    renderModal({
+      modelData: {
+        ...MODEL_DATA,
+        litellm_params: { ...MODEL_DATA.litellm_params, complexity_router_config: storedAdvancedConfig },
+      },
+      ...props,
+    });
+
+  beforeEach(() => {
+    modelPatchUpdateCall.mockClear();
+  });
+
+  it("hydrates housekeeping and reminder fields, then omits the default max-token value after editing", async () => {
+    const user = userEvent.setup();
+    renderAdvancedModal();
+
+    await user.click(await screen.findByText("Advanced: Housekeeping Routing"));
+    expect(screen.getByRole("switch", { name: "Route housekeeping calls to the cheapest tier" })).not.toBeChecked();
+    expect(screen.getByRole("combobox", { name: "e.g., conversation title" })).toHaveValue("");
+
+    await user.click(screen.getByText("Advanced: Reminder Markers"));
+    expect(screen.getByLabelText("Opening delimiter")).toHaveValue("<a>");
+    expect(screen.getByLabelText("Closing delimiter")).toHaveValue("</a>");
+
+    await user.click(screen.getByText("Advanced: Response Format"));
+    const maxTokensSwitch = screen.getByRole("switch", { name: "Cap max_tokens at the tier model's output ceiling" });
+    await user.click(maxTokensSwitch);
+    await user.click(screen.getByRole("button", { name: /save changes/i }));
+    await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
+
+    expect(savedConfig()).not.toHaveProperty("max_tokens_from_tier_model");
+    expect(savedConfig()).toMatchObject({
+      route_housekeeping_to_cheapest_tier: false,
+      housekeeping_patterns: ["conversation title"],
+      reminder_markers: [{ open: "<a>", close: "</a>" }],
+    });
+  });
+
+  it("does not PATCH when the edit is cancelled", async () => {
+    const user = userEvent.setup();
+    const onCancel = vi.fn();
+    renderAdvancedModal({ onCancel });
+    await user.click(screen.getByRole("button", { name: /cancel/i }));
+    expect(onCancel).toHaveBeenCalledOnce();
+    expect(modelPatchUpdateCall).not.toHaveBeenCalled();
+  });
+
+  it("preserves all stored advanced fields through an untouched save", async () => {
+    const user = userEvent.setup();
+    renderAdvancedModal();
+    await user.click(screen.getByRole("button", { name: /save changes/i }));
+    await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
+    expect(savedConfig()).toMatchObject({
+      route_housekeeping_to_cheapest_tier: false,
+      housekeeping_patterns: ["conversation title"],
+      reminder_markers: [{ open: "<a>", close: "</a>" }],
+      max_tokens_from_tier_model: false,
+    });
+  });
+});
+
 describe("EditAutoRouterModal classifier context window", () => {
   beforeEach(() => {
     modelPatchUpdateCall.mockClear();
diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
index 476207553f7..e3a2df39b88 100644
--- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
+++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
@@ -1,13 +1,9 @@
 import AutoRouterClassifierTabs from "../add_model/AutoRouterClassifierTabs";
 import { usesClassifierContext } from "../add_model/classifier_types";
-import { defaultJevClassifierConfig, jevClassifierConfigSchema } from "../add_model/jev_classifier_config";
-import type { StoredComplexityRouterConfig } from "../add_model/build_complexity_router_config";
 export type { StoredComplexityRouterConfig } from "../add_model/build_complexity_router_config";
 import {
   getForecastConfigError,
   isForecastClassifier,
-  capabilitySettingsSchema,
-  fuseSettingsSchema,
 } from "../add_model/forecast_classifier_config";
 import React, { useEffect, useMemo, useState } from "react";
 import {
@@ -30,13 +26,10 @@ import ModelChoiceCombobox, { type ModelChoice } from "../add_model/ModelChoiceC
 import { modelAvailableCall, modelPatchUpdateCall, validateAutoRouterConfig } from "../networking";
 import { fetchAutoRouterModels, fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models";
 import RouterConfigBuilder, { type RouterConfig, serializeRouterConfig } from "../add_model/RouterConfigBuilder";
-import { hydrateTierModelParams } from "../add_model/complexity_router_tiers";
 import {
-  type ActiveTierSet,
   CUSTOM_TIER_OMITTED_KEYS,
   activeTierRows,
   getCustomTierRowsError,
-  tierParamsByRowId,
   resolveComplexityDefaultModel,
 } from "../add_model/tier_rows";
 import { isComplexityRouter } from "../add_model/auto_router_strategies";
@@ -53,10 +46,6 @@ import {
   getSemanticConfigError,
   getPlanModeTierError,
   getTierLabelsError,
-  hydrateBuiltInTiers,
-  hydrateCustomTierSet,
-  hydratePlanModeMinTier,
-  hydrateTierLabels,
   dryRunRejection,
 } from "../add_model/build_complexity_router_config";
 import { KeywordTierRule } from "../add_model/KeywordTierRules";
@@ -68,22 +57,14 @@ import {
   hydrateAutoRouterCompression,
 } from "../add_model/buildAutoRouterCompression";
 import { hydrateKeywordTierRules } from "../add_model/complexity_router_keywords";
-import { customDimensionsError, hydrateCustomDimensions } from "../add_model/custom_dimensions";
-import {
-  hydrateDimensionWeights,
-  hydrateReasoningOverrideMinScore,
-  hydrateTierBoundaries,
-  hydrateTokenThresholds,
-} from "../add_model/heuristic_scoring_knobs";
+import { customDimensionsError } from "../add_model/custom_dimensions";
 import ComplexityRouterConfig, {
   ComplexityRouterConfigValue,
   effectiveClassifierType,
   heuristicScoringRole,
-  DEFAULT_ADAPTIVE_WEIGHTS,
-  DEFAULT_SESSION_AFFINITY,
-  DEFAULT_DEPLOYMENT_AFFINITY,
-  DEFAULT_TIER_DISTANCE_PENALTY,
 } from "../add_model/ComplexityRouterConfig";
+import { builderParamsFromValue } from "../add_model/complexity_router_builder_params";
+import { hydrateComplexityRouterConfig, hydratePinnedDefaultModel } from "./hydrate_complexity_router_config";
 import {
   Dialog,
   DialogContent,
@@ -106,151 +87,7 @@ interface EditAutoRouterModalProps {
 // Keys this modal rewrites from its own form state on save. Anything absent from this set is
 // carried through untouched from the stored config, so a key only belongs here once the modal
 // actually renders a control that can set it.
-
-/**
- * The stored complexity_router_config as form state. Every key in MANAGED_COMPLEXITY_ROUTER_KEYS is
- * rewritten from this state on save, so a key missing here is silently dropped from the saved config.
- */
-export const hydrateComplexityRouterConfig = (
-  parsedConfig: StoredComplexityRouterConfig,
-  complexityRouterDefaultModel: string | null | undefined,
-): ComplexityRouterConfigValue => {
-  const stringList = (input: unknown): string[] | undefined =>
-    Array.isArray(input) ? input.filter((item): item is string => typeof item === "string") : undefined;
-  const builtIn = hydrateBuiltInTiers(parsedConfig.tiers, parsedConfig.enable_non_reasoning_tier);
-  const { tiers: hydratedTiers, enable_non_reasoning_tier } = builtIn;
-  const custom_tier_set = hydrateCustomTierSet(parsedConfig);
-  const activeTiers = { ...builtIn, custom_tier_set };
-
-  return {
-    tiers: hydratedTiers,
-    enable_non_reasoning_tier,
-    custom_tier_set,
-    tier_model_params: tierParamsByRowId(
-      hydrateTierModelParams(parsedConfig.tiers, parsedConfig.tier_model_configs),
-      activeTierRows(activeTiers),
-    ),
-    default_model: hydratePinnedDefaultModel(parsedConfig.default_model, complexityRouterDefaultModel, activeTiers),
-    plan_mode_min_tier: hydratePlanModeMinTier(parsedConfig.plan_mode_min_tier, custom_tier_set),
-    tier_labels: hydrateTierLabels(parsedConfig.tier_labels),
-    classifier_type: parsedConfig.classifier_type || "heuristic",
-    heuristic_v2_success_threshold:
-      typeof parsedConfig.heuristic_v2_success_threshold === "number"
-        ? parsedConfig.heuristic_v2_success_threshold
-        : undefined,
-    capability_classifier_config: capabilitySettingsSchema.safeParse(parsedConfig.capability_classifier_config).data,
-    llm_v2_config: fuseSettingsSchema.safeParse(parsedConfig.llm_v2_config).data,
-    classifier_llm_config: parsedConfig.classifier_type === "jev" ? undefined : parsedConfig.classifier_llm_config,
-    jev_classifier_config:
-      parsedConfig.classifier_type === "jev"
-        ? jevClassifierConfigSchema.safeParse(parsedConfig.jev_classifier_config ?? {}).data ??
-          defaultJevClassifierConfig()
-        : undefined,
-    classifier_context_window_size:
-      typeof parsedConfig.classifier_context_window_size === "number"
-        ? parsedConfig.classifier_context_window_size
-        : undefined,
-    classifier_context_budget_chars:
-      typeof parsedConfig.classifier_context_budget_chars === "number"
-        ? parsedConfig.classifier_context_budget_chars
-        : undefined,
-    classifier_context_per_turn_chars:
-      typeof parsedConfig.classifier_context_per_turn_chars === "number"
-        ? parsedConfig.classifier_context_per_turn_chars
-        : undefined,
-    classifier_context_include_assistant_turns:
-      typeof parsedConfig.classifier_context_include_assistant_turns === "boolean"
-        ? parsedConfig.classifier_context_include_assistant_turns
-        : undefined,
-    classifier_fallback:
-      parsedConfig.classifier_fallback === "default_model" || parsedConfig.classifier_fallback === "heuristic"
-        ? parsedConfig.classifier_fallback
-        : undefined,
-    classification_prompt:
-      typeof parsedConfig.classification_prompt === "string" && parsedConfig.classification_prompt.trim() !== ""
-        ? parsedConfig.classification_prompt
-        : undefined,
-    classification_examples:
-      typeof parsedConfig.classification_examples === "string" && parsedConfig.classification_examples.trim() !== ""
-        ? parsedConfig.classification_examples
-        : undefined,
-    heuristic_first_max_tier:
-      typeof parsedConfig.heuristic_first_max_tier === "string" && parsedConfig.heuristic_first_max_tier.trim() !== ""
-        ? parsedConfig.heuristic_first_max_tier
-        : undefined,
-    hybrid_boundary_margin:
-      typeof parsedConfig.hybrid_boundary_margin === "number" ? parsedConfig.hybrid_boundary_margin : undefined,
-    classification_mode:
-      parsedConfig.classification_mode === "user_turn" || parsedConfig.classification_mode === "every_request"
-        ? parsedConfig.classification_mode
-        : undefined,
-    tier_boundaries: hydrateTierBoundaries(parsedConfig.tier_boundaries),
-    token_thresholds: hydrateTokenThresholds(parsedConfig.token_thresholds),
-    dimension_weights: hydrateDimensionWeights(parsedConfig.dimension_weights),
-    custom_dimensions: hydrateCustomDimensions(parsedConfig.custom_dimensions),
-    reasoning_override_min_score: hydrateReasoningOverrideMinScore(parsedConfig.reasoning_override_min_score),
-    session_affinity:
-      typeof parsedConfig.session_affinity === "boolean" ? parsedConfig.session_affinity : DEFAULT_SESSION_AFFINITY,
-    session_affinity_ttl_seconds:
-      typeof parsedConfig.session_affinity_ttl_seconds === "number" &&
-      Number.isFinite(parsedConfig.session_affinity_ttl_seconds)
-        ? parsedConfig.session_affinity_ttl_seconds
-        : undefined,
-    modality_routing: typeof parsedConfig.modality_routing === "boolean" ? parsedConfig.modality_routing : false,
-    modality_pin_override:
-      typeof parsedConfig.modality_pin_override === "boolean" ? parsedConfig.modality_pin_override : false,
-    deployment_affinity:
-      typeof parsedConfig.deployment_affinity === "boolean"
-        ? parsedConfig.deployment_affinity
-        : DEFAULT_DEPLOYMENT_AFFINITY,
-    adaptive: parsedConfig.adaptive || false,
-    adaptive_weights: parsedConfig.adaptive_weights,
-    tier_distance_penalty: parsedConfig.tier_distance_penalty,
-    adaptive_eligible: parsedConfig.adaptive_eligible || "all",
-    return_raw_model_name: parsedConfig.return_raw_model_name || false,
-    enable_context_window_escalation:
-      typeof parsedConfig.enable_context_window_escalation === "boolean"
-        ? parsedConfig.enable_context_window_escalation
-        : undefined,
-    context_window_escalation_buffer:
-      typeof parsedConfig.context_window_escalation_buffer === "number"
-        ? parsedConfig.context_window_escalation_buffer
-        : undefined,
-    stall_escalation_enabled: parsedConfig.stall_escalation_enabled === true || undefined,
-    stall_escalation_window:
-      typeof parsedConfig.stall_escalation_window === "number" ? parsedConfig.stall_escalation_window : undefined,
-    stall_escalation_repeat_threshold:
-      typeof parsedConfig.stall_escalation_repeat_threshold === "number"
-        ? parsedConfig.stall_escalation_repeat_threshold
-        : undefined,
-    code_keywords: stringList(parsedConfig.code_keywords),
-    reasoning_keywords: stringList(parsedConfig.reasoning_keywords),
-    technical_keywords: stringList(parsedConfig.technical_keywords),
-    simple_keywords: stringList(parsedConfig.simple_keywords),
-    plan_mode_patterns: stringList(parsedConfig.plan_mode_patterns),
-    route_housekeeping_to_cheapest_tier:
-      typeof parsedConfig.route_housekeeping_to_cheapest_tier === "boolean"
-        ? parsedConfig.route_housekeeping_to_cheapest_tier
-        : undefined,
-    housekeeping_patterns: stringList(parsedConfig.housekeeping_patterns),
-    reminder_markers: Array.isArray(parsedConfig.reminder_markers)
-      ? parsedConfig.reminder_markers.filter(
-          (pair): pair is { open: string; close: string } =>
-            typeof pair === "object" &&
-            pair !== null &&
-            typeof (pair as { open?: unknown }).open === "string" &&
-            typeof (pair as { close?: unknown }).close === "string",
-        )
-      : undefined,
-    max_tokens_from_tier_model:
-      typeof parsedConfig.max_tokens_from_tier_model === "boolean" ? parsedConfig.max_tokens_from_tier_model : undefined,
-    classifier_plugin_timeout_ms:
-      typeof parsedConfig.classifier_plugin_timeout_ms === "number" && Number.isFinite(parsedConfig.classifier_plugin_timeout_ms)
-        ? parsedConfig.classifier_plugin_timeout_ms
-        : undefined,
-  };
-};
-
+export { hydrateComplexityRouterConfig, hydratePinnedDefaultModel };
 export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
   "tiers",
   "enable_non_reasoning_tier",
@@ -324,24 +161,6 @@ const toRecord = (value: unknown): Record<string, unknown> => {
     : {};
 };
 
-// A pin lives in two places: complexity_router_config.default_model (this UI's own marker, added
-// by PR #36615) and litellm_params.complexity_router_default_model (what the backend reads). Only
-// the marker proves an operator picked it, because before #36615 every save wrote a tier-derived
-// value into litellm_params. So with no marker, a litellm_params value counts as a pin only when
-// it diverges from what the tiers alone derive; a match stays unpinned and keeps tracking tiers.
-export const hydratePinnedDefaultModel = (
-  storedConfigDefaultModel: unknown,
-  litellmParamsDefaultModel: string | null | undefined,
-  activeTiers: ActiveTierSet,
-): string | undefined => {
-  if (typeof storedConfigDefaultModel === "string" && storedConfigDefaultModel.trim()) {
-    return storedConfigDefaultModel;
-  }
-  const tierDerived = resolveComplexityDefaultModel(activeTiers);
-  const externalOverride = litellmParamsDefaultModel?.trim();
-  return externalOverride && externalOverride !== tierDerived ? externalOverride : undefined;
-};
-
 export interface KeywordMatchingState {
   keywordTierRules: KeywordTierRule[];
   escalationKeywords: string[];
@@ -377,65 +196,13 @@ export const buildUpdatedComplexityRouterConfig = (
   );
 
   const builderParams: BuildComplexityRouterConfigParams = {
-    tiers: value.tiers,
-    enableNonReasoningTier: value.enable_non_reasoning_tier,
-    customTierSet: value.custom_tier_set,
-    defaultModel: value.default_model,
-    planModeMinTier: value.plan_mode_min_tier,
-    classificationPrompt: value.classification_prompt,
-    classificationExamples: value.classification_examples,
-    heuristicFirstMaxTier: value.heuristic_first_max_tier,
-    hybridBoundaryMargin: value.hybrid_boundary_margin,
-    classificationMode: value.classification_mode,
-    tierLabels: value.tier_labels,
-    classifierType: value.classifier_type,
-    jevClassifierConfig: value.jev_classifier_config,
-    heuristicV2SuccessThreshold: value.heuristic_v2_success_threshold,
-    capabilityClassifierConfig: value.capability_classifier_config,
-    llmV2Config: value.llm_v2_config,
-    classifierLlmConfig: value.classifier_llm_config,
-    classifierContextWindowSize: value.classifier_context_window_size,
-    classifierContextBudgetChars: value.classifier_context_budget_chars,
-    classifierContextPerTurnChars: value.classifier_context_per_turn_chars,
-    classifierContextIncludeAssistantTurns: value.classifier_context_include_assistant_turns,
-    classifierFallback: value.classifier_fallback,
-    sessionAffinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY,
-    sessionAffinityTtlSeconds: value.session_affinity_ttl_seconds,
-    modalityRouting: value.modality_routing ?? false,
-    modalityPinOverride: value.modality_pin_override ?? false,
-    deploymentAffinity: value.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY,
+    ...builderParamsFromValue(value),
     customTechnicalKeywords: customTechnicalKeywords ?? [],
     keywordTierRules: keywordMatching?.keywordTierRules ?? [],
     semanticMatchingEnabled: keywordMatching?.semanticMatchingEnabled ?? false,
     embeddingModel: keywordMatching?.embeddingModel,
     matchThreshold: keywordMatching?.matchThreshold ?? DEFAULT_MATCH_THRESHOLD,
     escalationKeywords: keywordMatching?.escalationKeywords ?? [],
-    adaptive: value.adaptive ?? false,
-    adaptiveWeights: value.adaptive_weights ?? DEFAULT_ADAPTIVE_WEIGHTS,
-    tierDistancePenalty: value.tier_distance_penalty ?? DEFAULT_TIER_DISTANCE_PENALTY,
-    adaptiveEligible: value.adaptive_eligible ?? "all",
-    returnRawModelName: value.return_raw_model_name ?? false,
-    tierBoundaries: value.tier_boundaries,
-    tokenThresholds: value.token_thresholds,
-    dimensionWeights: value.dimension_weights,
-    customDimensions: value.custom_dimensions,
-    reasoningOverrideMinScore: value.reasoning_override_min_score,
-    tierModelParams: value.tier_model_params,
-    enableContextWindowEscalation: value.enable_context_window_escalation,
-    contextWindowEscalationBuffer: value.context_window_escalation_buffer,
-    stallEscalationEnabled: value.stall_escalation_enabled,
-    stallEscalationWindow: value.stall_escalation_window,
-    stallEscalationRepeatThreshold: value.stall_escalation_repeat_threshold,
-    codeKeywords: value.code_keywords,
-    reasoningKeywords: value.reasoning_keywords,
-    technicalKeywords: value.technical_keywords,
-    simpleKeywords: value.simple_keywords,
-    planModePatterns: value.plan_mode_patterns,
-    routeHousekeepingToCheapestTier: value.route_housekeeping_to_cheapest_tier,
-    housekeepingPatterns: value.housekeeping_patterns,
-    reminderMarkers: value.reminder_markers,
-    maxTokensFromTierModel: value.max_tokens_from_tier_model,
-    classifierPluginTimeoutMs: value.classifier_plugin_timeout_ms,
   };
   const built = buildComplexityRouterConfig(builderParams);
 
diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts b/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts
new file mode 100644
index 00000000000..c7c7bcb3382
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts
@@ -0,0 +1,183 @@
+import { defaultJevClassifierConfig, jevClassifierConfigSchema } from "../add_model/jev_classifier_config";
+import { capabilitySettingsSchema, fuseSettingsSchema } from "../add_model/forecast_classifier_config";
+import type { StoredComplexityRouterConfig } from "../add_model/build_complexity_router_config";
+import {
+  hydrateBuiltInTiers,
+  hydrateCustomTierSet,
+  hydratePlanModeMinTier,
+  hydrateTierLabels,
+} from "../add_model/build_complexity_router_config";
+import { hydrateTierModelParams } from "../add_model/complexity_router_tiers";
+import { hydrateCustomDimensions } from "../add_model/custom_dimensions";
+import {
+  hydrateDimensionWeights,
+  hydrateReasoningOverrideMinScore,
+  hydrateTierBoundaries,
+  hydrateTokenThresholds,
+} from "../add_model/heuristic_scoring_knobs";
+import type { ComplexityRouterConfigValue } from "../add_model/ComplexityRouterConfig";
+import { DEFAULT_DEPLOYMENT_AFFINITY, DEFAULT_SESSION_AFFINITY } from "../add_model/ComplexityRouterConfig";
+import {
+  type ActiveTierSet,
+  activeTierRows,
+  tierParamsByRowId,
+  resolveComplexityDefaultModel,
+} from "../add_model/tier_rows";
+
+const isReminderMarkerPair = (
+  input: unknown,
+): input is { open: string; close: string } =>
+  typeof input === "object" &&
+  input !== null &&
+  "open" in input &&
+  "close" in input &&
+  typeof input.open === "string" &&
+  typeof input.close === "string";
+
+const stringList = (input: unknown): string[] | undefined =>
+  Array.isArray(input) ? input.filter((item): item is string => typeof item === "string") : undefined;
+
+export const hydratePinnedDefaultModel = (
+  storedConfigDefaultModel: unknown,
+  litellmParamsDefaultModel: string | null | undefined,
+  activeTiers: ActiveTierSet,
+): string | undefined => {
+  if (typeof storedConfigDefaultModel === "string" && storedConfigDefaultModel.trim()) {
+    return storedConfigDefaultModel;
+  }
+  const tierDerived = resolveComplexityDefaultModel(activeTiers);
+  const externalOverride = litellmParamsDefaultModel?.trim();
+  return externalOverride && externalOverride !== tierDerived ? externalOverride : undefined;
+};
+
+export const hydrateComplexityRouterConfig = (
+  parsedConfig: StoredComplexityRouterConfig,
+  complexityRouterDefaultModel: string | null | undefined,
+): ComplexityRouterConfigValue => {
+  const builtIn = hydrateBuiltInTiers(parsedConfig.tiers, parsedConfig.enable_non_reasoning_tier);
+  const { tiers: hydratedTiers, enable_non_reasoning_tier } = builtIn;
+  const custom_tier_set = hydrateCustomTierSet(parsedConfig);
+  const activeTiers = { ...builtIn, custom_tier_set };
+
+  return {
+    tiers: hydratedTiers,
+    enable_non_reasoning_tier,
+    custom_tier_set,
+    tier_model_params: tierParamsByRowId(
+      hydrateTierModelParams(parsedConfig.tiers, parsedConfig.tier_model_configs),
+      activeTierRows(activeTiers),
+    ),
+    default_model: hydratePinnedDefaultModel(parsedConfig.default_model, complexityRouterDefaultModel, activeTiers),
+    plan_mode_min_tier: hydratePlanModeMinTier(parsedConfig.plan_mode_min_tier, custom_tier_set),
+    tier_labels: hydrateTierLabels(parsedConfig.tier_labels),
+    classifier_type: parsedConfig.classifier_type || "heuristic",
+    heuristic_v2_success_threshold:
+      typeof parsedConfig.heuristic_v2_success_threshold === "number"
+        ? parsedConfig.heuristic_v2_success_threshold
+        : undefined,
+    capability_classifier_config: capabilitySettingsSchema.safeParse(parsedConfig.capability_classifier_config).data,
+    llm_v2_config: fuseSettingsSchema.safeParse(parsedConfig.llm_v2_config).data,
+    classifier_llm_config: parsedConfig.classifier_type === "jev" ? undefined : parsedConfig.classifier_llm_config,
+    jev_classifier_config:
+      parsedConfig.classifier_type === "jev"
+        ? jevClassifierConfigSchema.safeParse(parsedConfig.jev_classifier_config ?? {}).data ??
+          defaultJevClassifierConfig()
+        : undefined,
+    classifier_context_window_size:
+      typeof parsedConfig.classifier_context_window_size === "number"
+        ? parsedConfig.classifier_context_window_size
+        : undefined,
+    classifier_context_budget_chars:
+      typeof parsedConfig.classifier_context_budget_chars === "number"
+        ? parsedConfig.classifier_context_budget_chars
+        : undefined,
+    classifier_context_per_turn_chars:
+      typeof parsedConfig.classifier_context_per_turn_chars === "number"
+        ? parsedConfig.classifier_context_per_turn_chars
+        : undefined,
+    classifier_context_include_assistant_turns:
+      typeof parsedConfig.classifier_context_include_assistant_turns === "boolean"
+        ? parsedConfig.classifier_context_include_assistant_turns
+        : undefined,
+    classifier_fallback:
+      parsedConfig.classifier_fallback === "default_model" || parsedConfig.classifier_fallback === "heuristic"
+        ? parsedConfig.classifier_fallback
+        : undefined,
+    classification_prompt:
+      typeof parsedConfig.classification_prompt === "string" && parsedConfig.classification_prompt.trim() !== ""
+        ? parsedConfig.classification_prompt
+        : undefined,
+    classification_examples:
+      typeof parsedConfig.classification_examples === "string" && parsedConfig.classification_examples.trim() !== ""
+        ? parsedConfig.classification_examples
+        : undefined,
+    heuristic_first_max_tier:
+      typeof parsedConfig.heuristic_first_max_tier === "string" && parsedConfig.heuristic_first_max_tier.trim() !== ""
+        ? parsedConfig.heuristic_first_max_tier
+        : undefined,
+    hybrid_boundary_margin:
+      typeof parsedConfig.hybrid_boundary_margin === "number" ? parsedConfig.hybrid_boundary_margin : undefined,
+    classification_mode:
+      parsedConfig.classification_mode === "user_turn" || parsedConfig.classification_mode === "every_request"
+        ? parsedConfig.classification_mode
+        : undefined,
+    tier_boundaries: hydrateTierBoundaries(parsedConfig.tier_boundaries),
+    token_thresholds: hydrateTokenThresholds(parsedConfig.token_thresholds),
+    dimension_weights: hydrateDimensionWeights(parsedConfig.dimension_weights),
+    custom_dimensions: hydrateCustomDimensions(parsedConfig.custom_dimensions),
+    reasoning_override_min_score: hydrateReasoningOverrideMinScore(parsedConfig.reasoning_override_min_score),
+    session_affinity:
+      typeof parsedConfig.session_affinity === "boolean" ? parsedConfig.session_affinity : DEFAULT_SESSION_AFFINITY,
+    session_affinity_ttl_seconds:
+      typeof parsedConfig.session_affinity_ttl_seconds === "number" &&
+      Number.isFinite(parsedConfig.session_affinity_ttl_seconds)
+        ? parsedConfig.session_affinity_ttl_seconds
+        : undefined,
+    modality_routing: typeof parsedConfig.modality_routing === "boolean" ? parsedConfig.modality_routing : false,
+    modality_pin_override:
+      typeof parsedConfig.modality_pin_override === "boolean" ? parsedConfig.modality_pin_override : false,
+    deployment_affinity:
+      typeof parsedConfig.deployment_affinity === "boolean"
+        ? parsedConfig.deployment_affinity
+        : DEFAULT_DEPLOYMENT_AFFINITY,
+    adaptive: parsedConfig.adaptive || false,
+    adaptive_weights: parsedConfig.adaptive_weights,
+    tier_distance_penalty: parsedConfig.tier_distance_penalty,
+    adaptive_eligible: parsedConfig.adaptive_eligible || "all",
+    return_raw_model_name: parsedConfig.return_raw_model_name || false,
+    enable_context_window_escalation:
+      typeof parsedConfig.enable_context_window_escalation === "boolean"
+        ? parsedConfig.enable_context_window_escalation
+        : undefined,
+    context_window_escalation_buffer:
+      typeof parsedConfig.context_window_escalation_buffer === "number"
+        ? parsedConfig.context_window_escalation_buffer
+        : undefined,
+    stall_escalation_enabled: parsedConfig.stall_escalation_enabled === true || undefined,
+    stall_escalation_window:
+      typeof parsedConfig.stall_escalation_window === "number" ? parsedConfig.stall_escalation_window : undefined,
+    stall_escalation_repeat_threshold:
+      typeof parsedConfig.stall_escalation_repeat_threshold === "number"
+        ? parsedConfig.stall_escalation_repeat_threshold
+        : undefined,
+    code_keywords: stringList(parsedConfig.code_keywords),
+    reasoning_keywords: stringList(parsedConfig.reasoning_keywords),
+    technical_keywords: stringList(parsedConfig.technical_keywords),
+    simple_keywords: stringList(parsedConfig.simple_keywords),
+    plan_mode_patterns: stringList(parsedConfig.plan_mode_patterns),
+    route_housekeeping_to_cheapest_tier:
+      typeof parsedConfig.route_housekeeping_to_cheapest_tier === "boolean"
+        ? parsedConfig.route_housekeeping_to_cheapest_tier
+        : undefined,
+    housekeeping_patterns: stringList(parsedConfig.housekeeping_patterns),
+    reminder_markers: Array.isArray(parsedConfig.reminder_markers)
+      ? parsedConfig.reminder_markers.filter(isReminderMarkerPair)
+      : undefined,
+    max_tokens_from_tier_model:
+      typeof parsedConfig.max_tokens_from_tier_model === "boolean" ? parsedConfig.max_tokens_from_tier_model : undefined,
+    classifier_plugin_timeout_ms:
+      typeof parsedConfig.classifier_plugin_timeout_ms === "number" && Number.isFinite(parsedConfig.classifier_plugin_timeout_ms)
+        ? parsedConfig.classifier_plugin_timeout_ms
+        : undefined,
+  };
+};

From 68074da1d1658e2c1a8337e090ceccf8488994c4 Mon Sep 17 00:00:00 2001
From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
Date: Mon, 21 Sep 2026 12:39:47 -0700
Subject: [PATCH 052/160] test(mcp): cover stable server ordering and sort
 priorities

---
 .../test_mcp_management_endpoints.py          | 83 ++++++++++---------
 .../_components/mcp_servers.test.tsx          | 72 +++++++++++++++-
 2 files changed, 112 insertions(+), 43 deletions(-)

diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py
index a526bfc90aa..b745c35ba1f 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py
@@ -6,7 +6,7 @@ import logging
 from contextlib import ExitStack
 from datetime import datetime, timedelta
 from types import SimpleNamespace
-from typing import List, Optional
+from typing import Final, List, Optional
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
@@ -1539,54 +1539,57 @@ class TestTeamScopedMCPServerAccess:
 
 
 class TestFetchAllMCPServersOrdering:
-    def test_display_order_is_case_insensitive_name_then_id(self):
-        servers = [
-            generate_mock_mcp_server_db_record(server_id="s-2", alias="github"),
-            generate_mock_mcp_server_db_record(server_id="s-1", alias="github"),
-            generate_mock_mcp_server_db_record(server_id="s-0", alias="Slack"),
-            generate_mock_mcp_server_db_record(server_id="s-3", alias="confluence"),
-        ]
+    def test_display_order_is_case_insensitive_name_then_id(self) -> None:
+        servers: Final = (
+            LiteLLM_MCPServerTable(server_id="s-2", server_name="GitHub", alias="aaa", transport=MCPTransport.http),
+            LiteLLM_MCPServerTable(server_id="s-1", alias="github", transport=MCPTransport.http),
+            LiteLLM_MCPServerTable(server_id="s-0", server_name="Slack", alias="zzz", transport=MCPTransport.http),
+            LiteLLM_MCPServerTable(server_id="confluence", server_name="", alias="", transport=MCPTransport.http),
+        )
 
-        ordered = sorted(servers, key=mgmt_endpoints._mcp_server_display_order)
-        assert [s.server_id for s in ordered] == ["s-3", "s-1", "s-2", "s-0"]
+        ordered: Final = sorted(servers, key=mgmt_endpoints._mcp_server_display_order)
+        assert [s.server_id for s in ordered] == ["confluence", "s-1", "s-2", "s-0"]
 
+    @pytest.mark.parametrize("team_id", [None, "team-1"])
+    @pytest.mark.parametrize("reverse", [False, True])
     @pytest.mark.asyncio
-    async def test_list_is_sorted_by_display_name_regardless_of_resolution_order(self):
-        """The registry resolves ids through a set, so the response must impose its own order."""
-        mock_user_auth = generate_mock_user_api_key_auth(
+    async def test_list_is_sorted_by_display_name_regardless_of_resolution_order(
+        self, team_id: str | None, reverse: bool
+    ) -> None:
+        mock_user_auth: Final = generate_mock_user_api_key_auth(
             user_role=LitellmUserRoles.PROXY_ADMIN,
             user_id="admin_user",
         )
-        first_order = [
+        servers: Final = (
             generate_mock_mcp_server_db_record(server_id="s-zeta", alias="zeta"),
             generate_mock_mcp_server_db_record(server_id="s-alpha", alias="Alpha"),
             generate_mock_mcp_server_db_record(server_id="s-mid", alias="mid"),
-        ]
-        second_order = list(reversed(first_order))
-
-        for resolved in (first_order, second_order):
-            mock_manager = MagicMock()
-            mock_manager.get_all_allowed_mcp_servers = AsyncMock(return_value=resolved)
-            with (
-                patch(  # test-quality-ok: the route reads a module-global manager with no injection seam
-                    "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager",
-                    mock_manager,
-                ),
-                patch(  # test-quality-ok: admin view is derived from module-global proxy settings
-                    "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view",
-                    return_value=True,
-                ),
-                patch(  # test-quality-ok: auth contexts need a live prisma client
-                    "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts",
-                    AsyncMock(return_value=[mock_user_auth]),
-                ),
-            ):
-                from litellm.proxy.management_endpoints.mcp_management_endpoints import (
-                    fetch_all_mcp_servers,
-                )
-
-                result = await fetch_all_mcp_servers(user_api_key_dict=mock_user_auth)
-                assert [s.server_id for s in result] == ["s-alpha", "s-mid", "s-zeta"]
+        )
+        resolved: Final = list(reversed(servers) if reverse else servers)
+        mock_manager: Final = MagicMock()
+        mock_manager.get_all_allowed_mcp_servers = AsyncMock(return_value=resolved)
+        with (
+            patch(  # test-quality-ok: the route reads a module-global manager with no injection seam
+                "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager",
+                mock_manager,
+            ),
+            patch(  # test-quality-ok: admin view is derived from module-global proxy settings
+                "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view",
+                return_value=True,
+            ),
+            patch(  # test-quality-ok: auth contexts need a live prisma client
+                "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts",
+                AsyncMock(return_value=[mock_user_auth]),
+            ),
+            patch(  # test-quality-ok: isolate the route's ordering from team database resolution
+                "litellm.proxy.management_endpoints.mcp_management_endpoints._get_team_scoped_mcp_server_list",
+                AsyncMock(return_value=resolved),
+            ),
+        ):
+            result: Final = await mgmt_endpoints.fetch_all_mcp_servers(
+                user_api_key_dict=mock_user_auth, team_id=team_id
+            )
+            assert [s.server_id for s in result] == ["s-alpha", "s-mid", "s-zeta"]
 
     @pytest.mark.asyncio
     async def test_restricted_virtual_key_cannot_use_team_id_filter(self):
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx
index 94a2c75016d..61880363234 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx
@@ -3,7 +3,7 @@ import { render, waitFor, screen, act, within } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import { describe, it, expect, vi, beforeEach } from "vitest";
 import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
-import MCPServers, { compareServers } from "./mcp_servers";
+import MCPServers, { compareServers, type SortKey } from "./mcp_servers";
 import type { MCPServer } from "@/components/mcp_tools/types";
 import * as networking from "@/components/networking";
 
@@ -33,8 +33,14 @@ const createQueryClient = () =>
   });
 
 describe("compareServers", () => {
-  const server = (server_id: string, name: string, created_at = ""): MCPServer =>
-    ({ server_id, server_name: name, created_at, updated_at: created_at }) as MCPServer;
+  const server = (server_id: string, name: string, created_at = ""): MCPServer => ({
+    server_id,
+    server_name: name,
+    created_at,
+    updated_at: created_at,
+    created_by: "user",
+    updated_by: "user",
+  });
 
   const shuffled = [server("c", "github"), server("a", "slack"), server("b", "Jira")];
 
@@ -56,6 +62,66 @@ describe("compareServers", () => {
       "old",
     ]);
   });
+
+  it.each<SortKey>(["created_desc", "updated_desc", "name_asc", "health"])(
+    "breaks equal timestamps and names by ID for %s regardless of input order",
+    (sort) => {
+      const servers = [
+        server("b", "GitHub", "2026-01-01T00:00:00Z"),
+        server("c", "Slack", "2026-01-01T00:00:00Z"),
+        server("a", "github", "2026-01-01T00:00:00Z"),
+      ];
+      for (const input of [servers, [...servers].reverse()]) {
+        expect([...input].sort((a, b) => compareServers(a, b, sort)).map((s) => s.server_id)).toEqual(["a", "b", "c"]);
+      }
+    },
+  );
+
+  it("uses the display name before alias, then falls back to alias and ID", () => {
+    const servers: MCPServer[] = [
+      { ...server("s-slack", "Slack"), alias: "aaa" },
+      { ...server("s-github", ""), server_name: null, alias: "GitHub" },
+      { ...server("confluence", ""), alias: "" },
+    ];
+    for (const input of [servers, [...servers].reverse()]) {
+      expect([...input].sort((a, b) => compareServers(a, b, "name_asc")).map((s) => s.server_id)).toEqual([
+        "confluence",
+        "s-github",
+        "s-slack",
+      ]);
+    }
+  });
+
+  it.each<SortKey>(["created_desc", "updated_desc", "health"])(
+    "keeps timestamped servers before missing timestamps for %s",
+    (sort) => {
+      const servers = [
+        server("config", "aaa"),
+        server("older", "bbb", "2026-01-01T00:00:00Z"),
+        server("newer", "zzz", "2026-02-01T00:00:00Z"),
+      ];
+      for (const input of [servers, [...servers].reverse()]) {
+        expect([...input].sort((a, b) => compareServers(a, b, sort)).map((s) => s.server_id)).toEqual([
+          "newer",
+          "older",
+          "config",
+        ]);
+      }
+    },
+  );
+
+  it("sorts health before recency and display name", () => {
+    const servers: MCPServer[] = [
+      { ...server("healthy", "aaa", "2026-03-01T00:00:00Z"), status: "healthy" },
+      { ...server("unknown", "bbb", "2026-02-01T00:00:00Z"), status: "unknown" },
+      { ...server("unhealthy", "zzz", "2026-01-01T00:00:00Z"), status: "unhealthy" },
+    ];
+    expect(servers.sort((a, b) => compareServers(a, b, "health")).map((s) => s.server_id)).toEqual([
+      "unhealthy",
+      "unknown",
+      "healthy",
+    ]);
+  });
 });
 
 describe("MCPServers", () => {

From e208b4e89ed41c6e4239c638f8e51a1239560959 Mon Sep 17 00:00:00 2001
From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
Date: Mon, 21 Sep 2026 12:40:12 -0700
Subject: [PATCH 053/160] fix(mcp): keep explicit legacy sampling callers
 isolated

---
 .../_experimental/mcp_server/legacy_callbacks.py    |  2 +-
 .../mcp_server/test_mcp_server_manager.py           | 13 +++++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py b/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py
index 4424907c28a..9e321062643 100644
--- a/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py
+++ b/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py
@@ -33,7 +33,7 @@ def create_sampling_callback(
 ) -> SamplingCallback:
     from litellm.proxy._experimental.mcp_server.server import get_active_auth_context
 
-    auth: Final = get_active_auth_context() if operation_context is None else None
+    auth: Final = get_active_auth_context() if operation_context is None and user_api_key_auth is None else None
     captured: Final = (
         operation_context
         if operation_context is not None
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
index 7be79f9b514..2dc7e01966a 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
@@ -14214,10 +14214,11 @@ async def test_request_selected_during_guardrail_runs_concurrently_with_tool(mon
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize("with_caller", [True, False])
-async def test_client_sampling_does_not_fill_explicit_context_from_another_ambient_caller(with_caller):
+@pytest.mark.parametrize("with_caller,legacy_factory", [(True, False), (False, False), (True, True)])
+async def test_client_sampling_does_not_fill_explicit_context_from_another_ambient_caller(with_caller, legacy_factory):
     from mcp.server.auth.middleware.auth_context import auth_context_var
     from litellm.proxy._experimental.mcp_server import server as legacy_server
+    from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_sampling_callback
 
     upstream = MCPServer(server_id="explicit-empty", name="explicit_empty", url="https://example.invalid/mcp", transport=MCPTransport.http, allow_sampling=True)
     token = auth_context_var.set(None)
@@ -14228,8 +14229,12 @@ async def test_client_sampling_does_not_fill_explicit_context_from_another_ambie
             patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPClient") as factory,
             patch("litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", sampling),
         ):
-            await MCPServerManager()._create_mcp_client(upstream, user_api_key_auth=UserAPIKeyAuth(user_id="explicit") if with_caller else None)
-            await factory.call_args.kwargs["sampling_callback"](None, None)
+            if legacy_factory:
+                callback = _create_sampling_callback(user_api_key_auth=UserAPIKeyAuth(user_id="explicit"))
+            else:
+                await MCPServerManager()._create_mcp_client(upstream, user_api_key_auth=UserAPIKeyAuth(user_id="explicit") if with_caller else None)
+                callback = factory.call_args.kwargs["sampling_callback"]
+            await callback(None, None)
         captured = sampling.await_args.kwargs
         if with_caller:
             assert captured["user_api_key_auth"].user_id == "explicit"

From ee07f710630bdadb7035f08cfb9cd728ec4425db Mon Sep 17 00:00:00 2001
From: yucheng <yucheng@berri.ai>
Date: Mon, 21 Sep 2026 19:43:11 +0000
Subject: [PATCH 054/160] style(proxy): format scheduled job timeout
 configuration

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/constants.py                     | 8 ++++++--
 litellm/proxy/shutdown/scheduled_jobs.py | 4 +---
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/litellm/constants.py b/litellm/constants.py
index 842adf62f6b..1971c336a96 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -1742,8 +1742,12 @@ SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS: Final = float(
 SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS: Final = float(os.getenv("SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS", "300"))
 SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS: Final = float(os.getenv("SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS", "30"))
 SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP: Final = int(os.getenv("SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP", "100000"))
-SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS: Final = float(os.getenv("SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS", "5"))
-SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS: Final = float(os.getenv("SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS", "5"))
+SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS: Final = float(
+    os.getenv("SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS", "5")
+)
+SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS: Final = float(
+    os.getenv("SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS", "5")
+)
 TOOL_SPEND_TOP_TOOLS: Final = 100
 SPEND_LOG_PARTITION_INTERVAL: Final = os.getenv("SPEND_LOG_PARTITION_INTERVAL", "day")
 SPEND_LOG_PARTITION_PRECREATE_AHEAD: Final = int(os.getenv("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7))
diff --git a/litellm/proxy/shutdown/scheduled_jobs.py b/litellm/proxy/shutdown/scheduled_jobs.py
index e920ce19eb9..7889c35cf4e 100644
--- a/litellm/proxy/shutdown/scheduled_jobs.py
+++ b/litellm/proxy/shutdown/scheduled_jobs.py
@@ -64,9 +64,7 @@ async def stop_in_flight_scheduler_jobs(
             len(in_flight),
         )
     still_running: Final = (
-        (await asyncio.wait(in_flight, timeout=finish_timeout_seconds))[1]
-        if in_flight
-        else frozenset()
+        (await asyncio.wait(in_flight, timeout=finish_timeout_seconds))[1] if in_flight else frozenset()
     )
     scheduler.shutdown(wait=False)
     if not still_running:

From f70683ae92748a9e43c5c0faade2e81275e23a8c Mon Sep 17 00:00:00 2001
From: yuneng <yuneng@berri.ai>
Date: Mon, 21 Sep 2026 19:56:30 +0000
Subject: [PATCH 055/160] fix(ui): satisfy complexity router CI lint budgets

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../ComplexityRouterAdvancedSections.tsx      |  2 +-
 .../add_model/add_auto_router_tab.tsx         | 26 +++++++++---------
 .../build_complexity_router_config.ts         | 27 ++++++++++---------
 .../edit_auto_router_modal.tsx                | 13 +++++----
 .../hydrate_complexity_router_config.ts       | 16 ++++++-----
 5 files changed, 45 insertions(+), 39 deletions(-)

diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx
index dd822907735..0412298ccdd 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx
@@ -28,7 +28,7 @@ interface ComplexityRouterAdvancedSectionsProps {
   onChange: (value: ComplexityRouterConfigValue) => void;
   forecast: boolean;
   modelOptions: { value: string; label: string }[];
-  classifierEffortOptionsByModel: Record<string, string[]>;
+  classifierEffortOptionsByModel: Record<string, string[] | null | undefined>;
   customTechnicalKeywords?: string[];
   onCustomTechnicalKeywordsChange?: (keywords: string[]) => void;
   showValidationErrors: boolean;
diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
index f805a5b5511..9be49edf08d 100644
--- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
@@ -408,6 +408,17 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
     matchThreshold,
     escalationKeywords,
   };
+  const jevRequestParams =
+    effectiveClassifierType(complexityRouterConfig) === "jev"
+      ? {
+          prompt: JEV_CONNECTION_TEST_PROMPT,
+          config: buildComplexityRouterConfig(complexityRouterConfigParams),
+          defaultModel: resolveComplexityDefaultModel(complexityRouterConfig, complexityRouterConfig.default_model),
+          routerName: watchedName,
+          teamId: requiresTeamScope ? watchedTeamId ?? undefined : undefined,
+        }
+      : undefined;
+  const jevRequest = jevRequestParams ? buildAutoRouterRoutingTestRequest(jevRequestParams) : undefined;
 
   const submitRecommendedRouter = async (name: string) => {
     // The one answer the submit button reads, so a disabled button and a refused submit cannot
@@ -816,20 +827,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
         testId={connectionTestId}
         accessToken={accessToken}
         targets={testTargets}
-        jevRequest={
-          effectiveClassifierType(complexityRouterConfig) === "jev"
-            ? buildAutoRouterRoutingTestRequest({
-                prompt: JEV_CONNECTION_TEST_PROMPT,
-                config: buildComplexityRouterConfig(complexityRouterConfigParams),
-                defaultModel: resolveComplexityDefaultModel(
-                  complexityRouterConfig,
-                  complexityRouterConfig.default_model,
-                ),
-                routerName: watchedName,
-                teamId: requiresTeamScope ? watchedTeamId ?? undefined : undefined,
-              })
-            : undefined
-        }
+        jevRequest={jevRequest}
         onTestComplete={() => setIsTestingConnection(false)}
       />
     </TooltipProvider>
diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
index ab71b6b3cce..4782a58513d 100644
--- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
+++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
@@ -744,16 +744,22 @@ export const buildComplexityRouterConfig = ({
     open: open.trim().toLowerCase(),
     close: close.trim().toLowerCase(),
   }));
+  const cleanedListValues = {
+    code_keywords: cleanList(codeKeywords),
+    reasoning_keywords: cleanList(reasoningKeywords),
+    technical_keywords: cleanList(technicalKeywords),
+    simple_keywords: cleanList(simpleKeywords),
+    plan_mode_patterns: cleanList(planModePatterns),
+    housekeeping_patterns: cleanList(housekeepingPatterns),
+  };
   const cleanedLists = Object.fromEntries(
-    Object.entries({
-      code_keywords: cleanList(codeKeywords),
-      reasoning_keywords: cleanList(reasoningKeywords),
-      technical_keywords: cleanList(technicalKeywords),
-      simple_keywords: cleanList(simpleKeywords),
-      plan_mode_patterns: cleanList(planModePatterns),
-      housekeeping_patterns: cleanList(housekeepingPatterns),
-    }).filter(([, list]) => list !== undefined),
+    Object.entries(cleanedListValues).filter(([, list]) => list !== undefined),
   );
+  const hasValidCustomClassifierTimeout =
+    classifierType === "custom" &&
+    classifierPluginTimeoutMs !== undefined &&
+    Number.isInteger(classifierPluginTimeoutMs) &&
+    classifierPluginTimeoutMs > 0;
 
   const supportsOpeningPrompt = !customTierSet && !forecast && usesLlmClassifier(effectiveType);
   const payload: ComplexityRouterConfigPayload = {
@@ -826,10 +832,7 @@ export const buildComplexityRouterConfig = ({
     ...(routeHousekeepingToCheapestTier === false && { route_housekeeping_to_cheapest_tier: false }),
     ...(cleanedReminderMarkers && cleanedReminderMarkers.length > 0 && { reminder_markers: cleanedReminderMarkers }),
     ...(maxTokensFromTierModel === false && { max_tokens_from_tier_model: false }),
-    ...(classifierType === "custom" &&
-      classifierPluginTimeoutMs !== undefined &&
-      Number.isInteger(classifierPluginTimeoutMs) &&
-      classifierPluginTimeoutMs > 0 && { classifier_plugin_timeout_ms: classifierPluginTimeoutMs }),
+    ...(hasValidCustomClassifierTimeout && { classifier_plugin_timeout_ms: classifierPluginTimeoutMs }),
     ...scorerKnobs,
   };
   if (!customTierSet) return payload;
diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
index e3a2df39b88..e6f1ccd0e17 100644
--- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
+++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
@@ -373,12 +373,13 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
       setRouterConfig(parsedConfig);
 
       // Set form values
-      form.reset({
+      const routerFormValues = {
         auto_router_name: modelData.model_name,
         auto_router_default_model: modelData.litellm_params?.auto_router_default_model || null,
         auto_router_embedding_model: modelData.litellm_params?.auto_router_embedding_model || null,
         model_access_group: modelData.model_info?.access_groups || [],
-      });
+      };
+      form.reset(routerFormValues);
     } catch (error) {
       console.error("Error parsing auto router config:", error);
       toast.fromError("Error loading auto router configuration");
@@ -456,11 +457,12 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
       // Dual write: complexity_router_config.default_model (the pin marker hydratePinnedDefaultModel
       // reads back) and complexity_router_default_model (what the backend routes on) must always be
       // written together from the same value. Same pairing in add_auto_router_tab.tsx.
+      const keywordMatching = { keywordTierRules, escalationKeywords, semanticMatchingEnabled, embeddingModel, matchThreshold };
       const updatedConfig = buildUpdatedComplexityRouterConfig(
         modelData.litellm_params?.complexity_router_config,
         complexityRouterConfig,
         customTechnicalKeywords,
-        { keywordTierRules, escalationKeywords, semanticMatchingEnabled, embeddingModel, matchThreshold },
+        keywordMatching,
       );
       const serverVerdict = await validateAutoRouterConfig(accessToken, updatedConfig, modelData?.model_info?.team_id);
       const dryRunError = dryRunRejection(serverVerdict);
@@ -497,12 +499,13 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
       );
 
       toast.success("Auto router configuration updated successfully");
-      onSuccess({
+      const updatedModelData = {
         ...modelData,
         model_name: values.auto_router_name,
         litellm_params: updatedLitellmParams,
         model_info: updatedModelInfo,
-      });
+      };
+      onSuccess(updatedModelData);
       onCancel();
       return;
     }
diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts b/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts
index c7c7bcb3382..b3a9ae0a504 100644
--- a/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts
+++ b/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts
@@ -26,13 +26,15 @@ import {
 
 const isReminderMarkerPair = (
   input: unknown,
-): input is { open: string; close: string } =>
-  typeof input === "object" &&
-  input !== null &&
-  "open" in input &&
-  "close" in input &&
-  typeof input.open === "string" &&
-  typeof input.close === "string";
+): input is { open: string; close: string } => {
+  if (typeof input !== "object" || input === null) {
+    return false;
+  }
+  if (!("open" in input) || !("close" in input)) {
+    return false;
+  }
+  return typeof input.open === "string" && typeof input.close === "string";
+};
 
 const stringList = (input: unknown): string[] | undefined =>
   Array.isArray(input) ? input.filter((item): item is string => typeof item === "string") : undefined;

From 9002749e29bcff3de2f8991ee988f923ffc5a3c5 Mon Sep 17 00:00:00 2001
From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
Date: Mon, 21 Sep 2026 12:58:06 -0700
Subject: [PATCH 056/160] fix(mcp): preserve Python 3.10 imports and
 integration test seams

---
 .../_experimental/mcp_server/operations.py    |  4 ++--
 tests/mcp_tests/test_mcp_logging.py           |  6 ++---
 tests/mcp_tests/test_mcp_server.py            | 24 ++++++++++---------
 3 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/litellm/proxy/_experimental/mcp_server/operations.py b/litellm/proxy/_experimental/mcp_server/operations.py
index 3ee4f37add7..a579c33e702 100644
--- a/litellm/proxy/_experimental/mcp_server/operations.py
+++ b/litellm/proxy/_experimental/mcp_server/operations.py
@@ -6,7 +6,7 @@ import types
 import uuid
 from collections.abc import Mapping, Sequence
 from datetime import datetime
-from typing import Any, Final, NoReturn, TypeAlias, assert_never, overload
+from typing import Any, Final, NoReturn, TypeAlias, overload
 
 from fastapi import HTTPException
 from mcp import ReadResourceResult, Resource
@@ -34,7 +34,7 @@ from mcp.types import (
 )
 from mcp.types import Tool as MCPTool
 from pydantic import AnyUrl, ConfigDict, Field, TypeAdapter
-from typing_extensions import ReadOnly, TypedDict
+from typing_extensions import ReadOnly, TypedDict, assert_never
 
 from litellm._logging import verbose_logger
 from litellm.constants import (
diff --git a/tests/mcp_tests/test_mcp_logging.py b/tests/mcp_tests/test_mcp_logging.py
index ed8829945e5..41d0e2cb59b 100644
--- a/tests/mcp_tests/test_mcp_logging.py
+++ b/tests/mcp_tests/test_mcp_logging.py
@@ -142,7 +142,7 @@ async def test_mcp_cost_tracking():
                 local_mcp_server_manager,
             ),
             patch(
-                "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+                "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
                 local_mcp_server_manager,
             ),
         ):
@@ -293,7 +293,7 @@ async def test_mcp_cost_tracking_per_tool():
                 local_mcp_server_manager,
             ),
             patch(
-                "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+                "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
                 local_mcp_server_manager,
             ),
         ):
@@ -451,7 +451,7 @@ async def test_mcp_tool_call_hook():
                 local_mcp_server_manager,
             ),
             patch(
-                "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+                "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
                 local_mcp_server_manager,
             ),
         ):
diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py
index 94cf35b675d..2b92367f186 100644
--- a/tests/mcp_tests/test_mcp_server.py
+++ b/tests/mcp_tests/test_mcp_server.py
@@ -922,7 +922,7 @@ async def test_get_tools_from_mcp_servers():
         )
 
         with patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
             mock_manager,
         ):
             # Test with specific servers
@@ -950,6 +950,7 @@ async def test_get_tools_from_mcp_servers():
                 extra_headers=None,
                 add_prefix=False,
                 raw_headers=None,
+                client_ip=None,
                 user_api_key_auth=None,
                 oauth2_headers=None,
             ):
@@ -966,7 +967,7 @@ async def test_get_tools_from_mcp_servers():
             )
 
         with patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
             mock_manager_2,
         ):
             result = await _get_tools_from_mcp_servers(
@@ -998,7 +999,7 @@ async def test_get_tools_from_mcp_servers():
         )
 
         with patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
             mock_manager,
         ):
             with patch(
@@ -1981,6 +1982,7 @@ async def test_get_tools_for_single_server():
             extra_headers=None,
             add_prefix=False,
             raw_headers=None,
+            client_ip=None,
             user_api_key_auth=None,
         )
 
@@ -2076,7 +2078,7 @@ async def test_rest_listing_hides_key_grants_dispatch_would_refuse():
     with patch(
         "litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager"
     ) as mock_manager, patch(
-        "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager"
+        "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager"
     ) as mock_server_manager, patch.object(
         MCPRequestHandler,
         "get_allowed_tools_for_server",
@@ -2473,7 +2475,7 @@ async def test_filter_tools_by_allowed_tools_integration():
 
     # Mock the global MCP server manager
     with patch(
-        "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager"
+        "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager"
     ) as mock_manager:
         # Mock manager methods
         mock_manager.get_allowed_mcp_servers = AsyncMock(
@@ -2588,7 +2590,7 @@ async def test_filter_tools_by_disallowed_tools_integration():
 
     # Mock the global MCP server manager
     with patch(
-        "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager"
+        "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager"
     ) as mock_manager:
         # Mock manager methods
         mock_manager.get_allowed_mcp_servers = AsyncMock(
@@ -2689,7 +2691,7 @@ async def test_filter_tools_no_restrictions_integration():
 
     # Mock the global MCP server manager
     with patch(
-        "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager"
+        "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager"
     ) as mock_manager:
         # Mock manager methods
         mock_manager.get_allowed_mcp_servers = AsyncMock(
@@ -2970,10 +2972,10 @@ async def test_call_mcp_tool_uses_manager_permission_lookup():
             return_value=mock_server,
         ) as mock_get_server,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_tool_registry"
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_tool_registry"
         ) as mock_tool_registry,
         patch(
-            "litellm.proxy._experimental.mcp_server.server._handle_managed_mcp_tool",
+            "litellm.proxy._experimental.mcp_server.operations._handle_managed_mcp_tool",
             new_callable=AsyncMock,
         ) as mock_handle_managed,
         patch(
@@ -3046,10 +3048,10 @@ async def test_call_mcp_tool_resolves_unprefixed_tool_name_and_checks_permission
             return_value=mock_server,
         ) as mock_get_server,
         patch(
-            "litellm.proxy._experimental.mcp_server.server.global_mcp_tool_registry"
+            "litellm.proxy._experimental.mcp_server.operations.global_mcp_tool_registry"
         ) as mock_tool_registry,
         patch(
-            "litellm.proxy._experimental.mcp_server.server._handle_managed_mcp_tool",
+            "litellm.proxy._experimental.mcp_server.operations._handle_managed_mcp_tool",
             new_callable=AsyncMock,
         ) as mock_handle_managed,
         patch(

From 4117d9f7860bc7bba6250682654f5cddeca4cd3f Mon Sep 17 00:00:00 2001
From: yuneng <yuneng@berri.ai>
Date: Mon, 21 Sep 2026 20:01:08 +0000
Subject: [PATCH 057/160] style(ui): format complexity router files

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../ComplexityRouterAdvancedSections.tsx        |  8 ++++----
 .../add_model/ComplexityRouterConfig.test.tsx   | 12 ++----------
 .../add_model/HeuristicKeywordOverrides.tsx     |  4 ++--
 .../components/add_model/ReminderMarkers.tsx    | 17 ++++++++++++-----
 .../add_model/ResponseFormatControls.tsx        |  4 ++--
 .../add_model/build_complexity_router_config.ts |  4 +---
 .../edit_auto_router/edit_auto_router_modal.tsx | 13 ++++++++-----
 .../hydrate_complexity_router_config.ts         | 11 ++++++-----
 8 files changed, 37 insertions(+), 36 deletions(-)

diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx
index 0412298ccdd..71f8bce76b5 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx
@@ -130,7 +130,9 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
     {
       key: "plan-mode",
       label: <strong className="text-foreground font-semibold">Advanced: Plan-Mode Override</strong>,
-      children: <PlanModeOverrideControls value={value} onChange={onChange} planModeTierOptions={planModeTierOptions} />,
+      children: (
+        <PlanModeOverrideControls value={value} onChange={onChange} planModeTierOptions={planModeTierOptions} />
+      ),
     },
     {
       key: "housekeeping",
@@ -195,9 +197,7 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
                     rules={keywordTierRules}
                     onChange={onKeywordTierRulesChange}
                     tierLabels={value.tier_labels}
-                    tierNames={
-                      customTierSet || forecast ? tierRows.map(activeTierName).filter(Boolean) : undefined
-                    }
+                    tierNames={customTierSet || forecast ? tierRows.map(activeTierName).filter(Boolean) : undefined}
                   />
                 )}
                 {onKeywordTierRulesChange && onSemanticMatchingEnabledChange && <Separator className="my-4" />}
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
index 9a8577100df..56b37b92af3 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
@@ -106,7 +106,6 @@ describe("ComplexityRouterConfig", () => {
     const capabilityValue = { ...defaultValue, classifier_type: "capability" as const };
     rerender(<ComplexityRouterConfig {...baseProps} value={capabilityValue} />);
     expect(screen.queryByText("Advanced: Heuristic Keyword Overrides")).not.toBeInTheDocument();
-
   });
 
   it.each([
@@ -127,11 +126,7 @@ describe("ComplexityRouterConfig", () => {
   it.each([true, false])("shows reminder marker validation only when requested: %s", (showValidationErrors) => {
     const value = { ...defaultValue, reminder_markers: [{ open: "", close: "x" }] };
     renderWithProviders(
-      <ComplexityRouterConfig
-        {...baseProps}
-        value={value}
-        showValidationErrors={showValidationErrors}
-      />,
+      <ComplexityRouterConfig {...baseProps} value={value} showValidationErrors={showValidationErrors} />,
     );
     fireEvent.click(screen.getByText("Advanced: Reminder Markers"));
     const validation = screen.queryByText(/needs both/i);
@@ -144,10 +139,7 @@ describe("ComplexityRouterConfig", () => {
 
   it("disables housekeeping sentinels when cheapest-tier routing is off", () => {
     renderWithProviders(
-      <ComplexityRouterConfig
-        {...baseProps}
-        value={{ ...defaultValue, route_housekeeping_to_cheapest_tier: false }}
-      />,
+      <ComplexityRouterConfig {...baseProps} value={{ ...defaultValue, route_housekeeping_to_cheapest_tier: false }} />,
     );
     fireEvent.click(screen.getByText("Advanced: Housekeeping Routing"));
     const sentinelInput = screen.getByRole("combobox", { name: "e.g., conversation title" });
diff --git a/ui/litellm-dashboard/src/components/add_model/HeuristicKeywordOverrides.tsx b/ui/litellm-dashboard/src/components/add_model/HeuristicKeywordOverrides.tsx
index 696924f4e77..185f187bbfc 100644
--- a/ui/litellm-dashboard/src/components/add_model/HeuristicKeywordOverrides.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/HeuristicKeywordOverrides.tsx
@@ -15,8 +15,8 @@ const HeuristicKeywordOverrides: React.FC<{
 }> = ({ value, onChange }) => (
   <div className="space-y-3">
     <p className="text-sm text-muted-foreground">
-      Each list replaces the built-in keyword list of the same name for the heuristic scorer. Leave a list empty to
-      keep the built-in one. To add technical terms without replacing the list, use custom technical keywords under
+      Each list replaces the built-in keyword list of the same name for the heuristic scorer. Leave a list empty to keep
+      the built-in one. To add technical terms without replacing the list, use custom technical keywords under
       Classification Method.
     </p>
     {fields.map(([key, label]) => {
diff --git a/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx b/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx
index 970394291d4..c7f9ee48e0b 100644
--- a/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx
@@ -14,7 +14,9 @@ const ReminderMarkers: React.FC<{
   const update = (index: number, patch: Partial<ReminderMarkerPair>) =>
     onChange({
       ...value,
-      reminder_markers: markers.map((marker, markerIndex) => (markerIndex === index ? { ...marker, ...patch } : marker)),
+      reminder_markers: markers.map((marker, markerIndex) =>
+        markerIndex === index ? { ...marker, ...patch } : marker,
+      ),
     });
   const remove = (index: number) => {
     const next = markers.filter((_, markerIndex) => markerIndex !== index);
@@ -24,9 +26,9 @@ const ReminderMarkers: React.FC<{
   return (
     <div>
       <p className="mb-4 text-sm text-muted-foreground">
-        Delimiter pairs that wrap harness-injected reminder blocks, which are stripped before classification. Setting any
-        pair replaces the built-in pairs, so list every pair your harness emits. Matching is case-insensitive and values
-        are saved lowercased.
+        Delimiter pairs that wrap harness-injected reminder blocks, which are stripped before classification. Setting
+        any pair replaces the built-in pairs, so list every pair your harness emits. Matching is case-insensitive and
+        values are saved lowercased.
       </p>
       <div className="space-y-3">
         {markers.map((marker, index) => (
@@ -53,7 +55,12 @@ const ReminderMarkers: React.FC<{
                 onChange={(event) => update(index, { close: event.target.value })}
               />
             </div>
-            <Button variant="ghost" size="icon" aria-label={`Remove reminder marker pair ${index + 1}`} onClick={() => remove(index)}>
+            <Button
+              variant="ghost"
+              size="icon"
+              aria-label={`Remove reminder marker pair ${index + 1}`}
+              onClick={() => remove(index)}
+            >
               <Trash2 />
             </Button>
           </div>
diff --git a/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx b/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx
index 9edbf204a6d..6e7b1489a4c 100644
--- a/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx
@@ -27,8 +27,8 @@ const ResponseFormatControls: React.FC<{
       <strong className="font-semibold">Cap max_tokens at the tier model&apos;s output ceiling</strong>
     </div>
     <span className="block text-xs text-muted-foreground">
-      Replace the caller&apos;s max_tokens with the routed tier model&apos;s output ceiling so one client value fits every
-      tier. Off forwards the caller&apos;s value unchanged.
+      Replace the caller&apos;s max_tokens with the routed tier model&apos;s output ceiling so one client value fits
+      every tier. Off forwards the caller&apos;s value unchanged.
     </span>
   </>
 );
diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
index 4782a58513d..ca46e171970 100644
--- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
+++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
@@ -752,9 +752,7 @@ export const buildComplexityRouterConfig = ({
     plan_mode_patterns: cleanList(planModePatterns),
     housekeeping_patterns: cleanList(housekeepingPatterns),
   };
-  const cleanedLists = Object.fromEntries(
-    Object.entries(cleanedListValues).filter(([, list]) => list !== undefined),
-  );
+  const cleanedLists = Object.fromEntries(Object.entries(cleanedListValues).filter(([, list]) => list !== undefined));
   const hasValidCustomClassifierTimeout =
     classifierType === "custom" &&
     classifierPluginTimeoutMs !== undefined &&
diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
index e6f1ccd0e17..0dafa9b330a 100644
--- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
+++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
@@ -1,10 +1,7 @@
 import AutoRouterClassifierTabs from "../add_model/AutoRouterClassifierTabs";
 import { usesClassifierContext } from "../add_model/classifier_types";
 export type { StoredComplexityRouterConfig } from "../add_model/build_complexity_router_config";
-import {
-  getForecastConfigError,
-  isForecastClassifier,
-} from "../add_model/forecast_classifier_config";
+import { getForecastConfigError, isForecastClassifier } from "../add_model/forecast_classifier_config";
 import React, { useEffect, useMemo, useState } from "react";
 import {
   complexityRouterSchema,
@@ -457,7 +454,13 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
       // Dual write: complexity_router_config.default_model (the pin marker hydratePinnedDefaultModel
       // reads back) and complexity_router_default_model (what the backend routes on) must always be
       // written together from the same value. Same pairing in add_auto_router_tab.tsx.
-      const keywordMatching = { keywordTierRules, escalationKeywords, semanticMatchingEnabled, embeddingModel, matchThreshold };
+      const keywordMatching = {
+        keywordTierRules,
+        escalationKeywords,
+        semanticMatchingEnabled,
+        embeddingModel,
+        matchThreshold,
+      };
       const updatedConfig = buildUpdatedComplexityRouterConfig(
         modelData.litellm_params?.complexity_router_config,
         complexityRouterConfig,
diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts b/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts
index b3a9ae0a504..6dbd2b19b52 100644
--- a/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts
+++ b/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts
@@ -24,9 +24,7 @@ import {
   resolveComplexityDefaultModel,
 } from "../add_model/tier_rows";
 
-const isReminderMarkerPair = (
-  input: unknown,
-): input is { open: string; close: string } => {
+const isReminderMarkerPair = (input: unknown): input is { open: string; close: string } => {
   if (typeof input !== "object" || input === null) {
     return false;
   }
@@ -176,9 +174,12 @@ export const hydrateComplexityRouterConfig = (
       ? parsedConfig.reminder_markers.filter(isReminderMarkerPair)
       : undefined,
     max_tokens_from_tier_model:
-      typeof parsedConfig.max_tokens_from_tier_model === "boolean" ? parsedConfig.max_tokens_from_tier_model : undefined,
+      typeof parsedConfig.max_tokens_from_tier_model === "boolean"
+        ? parsedConfig.max_tokens_from_tier_model
+        : undefined,
     classifier_plugin_timeout_ms:
-      typeof parsedConfig.classifier_plugin_timeout_ms === "number" && Number.isFinite(parsedConfig.classifier_plugin_timeout_ms)
+      typeof parsedConfig.classifier_plugin_timeout_ms === "number" &&
+      Number.isFinite(parsedConfig.classifier_plugin_timeout_ms)
         ? parsedConfig.classifier_plugin_timeout_ms
         : undefined,
   };

From ef67412e50c9f3250eb473a0a14fbf00617e36ea Mon Sep 17 00:00:00 2001
From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
Date: Mon, 21 Sep 2026 13:13:20 -0700
Subject: [PATCH 058/160] fix(mcp): keep OAuth prefetch failure logs free of
 caller data

---
 .../_experimental/mcp_server/operations.py    |  6 ++---
 .../proxy/_experimental/mcp_server/server.py  |  3 +++
 .../mcp_server/test_operations.py             | 22 +++++++++++++++++++
 3 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/litellm/proxy/_experimental/mcp_server/operations.py b/litellm/proxy/_experimental/mcp_server/operations.py
index a579c33e702..fcee3483e15 100644
--- a/litellm/proxy/_experimental/mcp_server/operations.py
+++ b/litellm/proxy/_experimental/mcp_server/operations.py
@@ -763,8 +763,8 @@ async def _prefetch_oauth_creds_for_user(
         )
         creds: Final = await list_user_oauth_credentials(prisma_client, user_id)
         return {c["server_id"]: c for c in creds if "server_id" in c}
-    except Exception as e:
-        verbose_logger.warning("_prefetch_oauth_creds_for_user: failed to prefetch for user=%s: %s", user_id, e)
+    except Exception:
+        verbose_logger.warning("_prefetch_oauth_creds_for_user: failed to prefetch OAuth credentials")
         return {}
 
 
@@ -3099,4 +3099,4 @@ class GatewayOperations:
             case ReadResourceRequest(params=params):
                 return await _execute_read_resource(context, params, self._host_progress_callback)
             case _:
-                assert_never(operation)
+                return assert_never(operation)
diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py
index 33978bb9182..a57eed04419 100644
--- a/litellm/proxy/_experimental/mcp_server/server.py
+++ b/litellm/proxy/_experimental/mcp_server/server.py
@@ -416,7 +416,10 @@ def _proxy_exception_to_http_exception(exc: ProxyException) -> HTTPException:
 if MCP_AVAILABLE:
     __all__ = (
         "_MCP_CREDENTIAL_REQUEST_FIELDS",
+        "BlobResourceContents",
         "ListMCPToolsRestAPIResponseObject",
+        "ResourceTemplate",
+        "TextResourceContents",
         "_McpDeniedDetail",
         "_aggregate_server_key",
         "_build_virtual_call_logging_obj",
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_operations.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_operations.py
index 870f2cd5f8f..abb925ddc77 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_operations.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_operations.py
@@ -8,6 +8,28 @@ from litellm.proxy._experimental.mcp_server.operations import GatewayOperations,
 from litellm.proxy._types import UserAPIKeyAuth
 
 
+@pytest.mark.asyncio
+async def test_oauth_prefetch_failure_does_not_log_caller_or_exception_text(caplog):
+    from litellm.proxy._experimental.mcp_server.operations import _prefetch_oauth_creds_for_user
+
+    user_id = "caller\nFORGED-USER-LINE"
+    fetch = AsyncMock(side_effect=RuntimeError("database\nFORGED-ERROR-LINE"))
+    database = object()
+    with (
+        patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=database),
+        patch("litellm.proxy._experimental.mcp_server.db.list_user_oauth_credentials", fetch),
+        caplog.at_level("WARNING", logger="LiteLLM"),
+    ):
+        result = await _prefetch_oauth_creds_for_user(UserAPIKeyAuth(user_id=user_id))
+    assert result == {}
+    fetch.assert_awaited_once_with(database, user_id)
+    warnings = [record.getMessage() for record in caplog.records if "prefetch" in record.getMessage()]
+    assert len(warnings) == 1
+    assert "failed" in warnings[0]
+    assert "\n" not in warnings[0]
+    assert "FORGED" not in warnings[0]
+
+
 @pytest.mark.asyncio
 async def test_dispatch_uses_explicit_context_when_ambient_caller_differs():
     from mcp.server.auth.middleware.auth_context import auth_context_var

From d2f457f144a430ad2848b33839d298d9312c8d4e Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:20:31 +0000
Subject: [PATCH 059/160] feat(cache): add semantic cache context and
 unsupported operation error

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/cache/src/base_cache.rs | 50 +++++++++++++++++++++
 litellm-rust/crates/cache/src/error.rs      |  2 +
 litellm-rust/crates/cache/src/lib.rs        |  2 +-
 3 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/litellm-rust/crates/cache/src/base_cache.rs b/litellm-rust/crates/cache/src/base_cache.rs
index 8bd69ba5ad6..5c10e7fd5c3 100644
--- a/litellm-rust/crates/cache/src/base_cache.rs
+++ b/litellm-rust/crates/cache/src/base_cache.rs
@@ -32,6 +32,28 @@ impl CacheContext for ExactCacheContext {
     }
 }
 
+#[derive(Clone, Debug, Default, PartialEq)]
+pub struct SemanticCacheContext {
+    pub input: Option<serde_json::Value>,
+    pub messages: Option<serde_json::Value>,
+    pub metadata: Option<serde_json::Value>,
+    pub scope: Option<String>,
+    pub ttl: Option<Duration>,
+}
+
+impl CacheContext for SemanticCacheContext {
+    fn ttl(&self) -> Option<Duration> {
+        self.ttl
+    }
+
+    fn with_ttl(&self, ttl: Option<Duration>) -> Self {
+        Self {
+            ttl,
+            ..self.clone()
+        }
+    }
+}
+
 #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
 #[serde(rename_all = "lowercase")]
 pub enum CacheConnectionStatus {
@@ -105,3 +127,31 @@ pub trait BaseCache: Send + Sync {
 
     fn test_connection(&self) -> impl Future<Output = Result<CacheConnectionResult, Error>> + Send;
 }
+
+#[cfg(test)]
+mod tests {
+    use std::time::Duration;
+
+    use serde_json::json;
+
+    use super::{CacheContext, SemanticCacheContext};
+
+    #[test]
+    fn semantic_context_with_ttl_only_replaces_ttl() {
+        let context = SemanticCacheContext {
+            input: Some(json!({"input": "hello"})),
+            messages: Some(json!([{"role": "user", "content": "hello"}])),
+            metadata: Some(json!({"tenant": "team"})),
+            scope: Some("scope".into()),
+            ttl: Some(Duration::from_secs(10)),
+        };
+
+        let updated = context.with_ttl(Some(Duration::from_secs(20)));
+
+        assert_eq!(updated.ttl, Some(Duration::from_secs(20)));
+        assert_eq!(updated.input, context.input);
+        assert_eq!(updated.messages, context.messages);
+        assert_eq!(updated.metadata, context.metadata);
+        assert_eq!(updated.scope, context.scope);
+    }
+}
diff --git a/litellm-rust/crates/cache/src/error.rs b/litellm-rust/crates/cache/src/error.rs
index ff3ff6572d4..51e4fe2d66a 100644
--- a/litellm-rust/crates/cache/src/error.rs
+++ b/litellm-rust/crates/cache/src/error.rs
@@ -6,4 +6,6 @@ pub enum Error {
     InvalidEntry,
     #[error("flushing Redis requires an explicit namespace")]
     UnscopedFlush,
+    #[error("cache operation is not supported by this backend")]
+    UnsupportedOperation,
 }
diff --git a/litellm-rust/crates/cache/src/lib.rs b/litellm-rust/crates/cache/src/lib.rs
index ce9f93b6dc4..8364c635e3a 100644
--- a/litellm-rust/crates/cache/src/lib.rs
+++ b/litellm-rust/crates/cache/src/lib.rs
@@ -8,7 +8,7 @@ mod error;
 
 pub use base_cache::{
     BaseCache, BatchEntry, CacheConnectionResult, CacheConnectionStatus, CacheContext,
-    ExactCacheContext,
+    ExactCacheContext, SemanticCacheContext,
 };
 pub use cache_type::CacheType;
 pub use caching::{Cache, CacheBackend, get_cache, set_cache};

From df97b274fc78ab261064f0a691016488c6c709dc Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:21:19 +0000
Subject: [PATCH 060/160] refactor(cache-response): generalize ResponseCache
 over the backend context

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/cache-response/README.md  |  2 +-
 .../crates/cache-response/src/response.rs     | 43 +++++++++++--------
 .../crates/python-bridge/src/cache/request.rs |  3 +-
 3 files changed, 28 insertions(+), 20 deletions(-)

diff --git a/litellm-rust/crates/cache-response/README.md b/litellm-rust/crates/cache-response/README.md
index 56c1646d343..46e561ddad1 100644
--- a/litellm-rust/crates/cache-response/README.md
+++ b/litellm-rust/crates/cache-response/README.md
@@ -58,4 +58,4 @@ Verify typed values, TTL precedence, missing entries, serialization failures, na
 
 Public SDK, Router, and proxy activation still need constructor parity, stream replay, embedding partial-batch integration, response reconstruction, callback scheduling, and failure-policy integration. This foundation does not switch those request paths
 
-Redis cluster, disk, cloud stores, and semantic caching remain follow-ups. The generic dual cache takes read, write, and remote-failure policies, runs its async operations through the async L2 methods, and provides L2-first counters and atomic affinity claims. Errors propagate by default, and `RemoteFailurePolicy::UseLocal` opts key-value operations and claims into the local tier when L2 is unavailable. Claims compare decoded values, so a pin written by Python still matches. Public Router integration remains follow-up work. Reservations and pubsub still need explicit capabilities owned by their consuming features. Adding a cache backend does not establish those guarantees
+Redis cluster, disk, and cloud stores remain follow-ups. Semantic backends plug in through `SemanticCacheContext`, which carries the prompt inputs and metadata alongside the cache TTL. The generic dual cache takes read, write, and remote-failure policies, runs its async operations through the async L2 methods, and provides L2-first counters and atomic affinity claims. Errors propagate by default, and `RemoteFailurePolicy::UseLocal` opts key-value operations and claims into the local tier when L2 is unavailable. Claims compare decoded values, so a pin written by Python still matches. Public Router integration remains follow-up work. Reservations and pubsub still need explicit capabilities owned by their consuming features. Adding a cache backend does not establish those guarantees
diff --git a/litellm-rust/crates/cache-response/src/response.rs b/litellm-rust/crates/cache-response/src/response.rs
index e50e68cdabb..e70e07a5d26 100644
--- a/litellm-rust/crates/cache-response/src/response.rs
+++ b/litellm-rust/crates/cache-response/src/response.rs
@@ -1,21 +1,21 @@
 use std::{sync::Arc, time::Duration};
 
 use litellm_cache::{
-    BaseCache, BatchCache, BatchEntry, CacheConnectionResult, Error, ExactCacheContext, FlushCache,
+    BaseCache, BatchCache, BatchEntry, CacheConnectionResult, CacheContext, Error, FlushCache,
 };
 use serde_json::Value;
 
 use crate::{CacheControls, CacheEntry, CacheKeyInput, PartialHits, cache_key};
 
 #[derive(Clone)]
-pub struct ResponseCacheRequest {
+pub struct ResponseCacheRequest<C: CacheContext = litellm_cache::ExactCacheContext> {
     pub key: CacheKeyInput,
     pub controls: CacheControls,
-    pub context: ExactCacheContext,
+    pub context: C,
     pub max_age: Option<Duration>,
 }
 
-impl ResponseCacheRequest {
+impl<C: CacheContext + Default> ResponseCacheRequest<C> {
     pub fn new(key: CacheKeyInput) -> Self {
         Self {
             key,
@@ -26,17 +26,24 @@ impl ResponseCacheRequest {
                 default_on: true,
                 ..Default::default()
             },
-            context: ExactCacheContext::default(),
+            context: C::default(),
             max_age: None,
         }
     }
 }
 
-pub struct ResponseCache<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> {
+pub struct ResponseCache<B: BaseCache<Value = CacheEntry>>
+where
+    B::Context: Default + PartialEq,
+{
     backend: Arc<B>,
 }
 
-impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCache<B> {
+impl<B> ResponseCache<B>
+where
+    B: BaseCache<Value = CacheEntry>,
+    B::Context: Default + PartialEq,
+{
     pub fn new(backend: Arc<B>) -> Self {
         Self { backend }
     }
@@ -46,7 +53,7 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
     }
 
     pub fn default_ttl(&self) -> Option<Duration> {
-        self.backend.get_ttl(&ExactCacheContext::default())
+        self.backend.get_ttl(&B::Context::default())
     }
 
     pub async fn async_flush(&self) -> Result<(), Error>
@@ -62,7 +69,7 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
 
     pub fn lookup(
         &self,
-        request: &ResponseCacheRequest,
+        request: &ResponseCacheRequest<B::Context>,
         now: Duration,
     ) -> Result<Option<Value>, Error> {
         if !request.controls.reads() {
@@ -81,7 +88,7 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
 
     pub async fn async_lookup(
         &self,
-        request: &ResponseCacheRequest,
+        request: &ResponseCacheRequest<B::Context>,
         now: Duration,
     ) -> Result<Option<Value>, Error> {
         if !request.controls.reads() {
@@ -101,7 +108,7 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
 
     pub fn lookup_batch(
         &self,
-        requests: &[ResponseCacheRequest],
+        requests: &[ResponseCacheRequest<B::Context>],
         now: Duration,
     ) -> Result<PartialHits, Error>
     where
@@ -126,7 +133,7 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
 
     pub async fn async_lookup_batch(
         &self,
-        requests: &[ResponseCacheRequest],
+        requests: &[ResponseCacheRequest<B::Context>],
         now: Duration,
     ) -> Result<PartialHits, Error>
     where
@@ -153,7 +160,7 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
 
     pub fn store(
         &self,
-        request: &ResponseCacheRequest,
+        request: &ResponseCacheRequest<B::Context>,
         response: Value,
         now: Duration,
     ) -> Result<(), Error> {
@@ -172,7 +179,7 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
 
     pub async fn async_store(
         &self,
-        request: &ResponseCacheRequest,
+        request: &ResponseCacheRequest<B::Context>,
         response: Value,
         now: Duration,
     ) -> Result<(), Error> {
@@ -193,7 +200,7 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
 
     pub async fn async_store_batch(
         &self,
-        entries: Vec<(ResponseCacheRequest, Value)>,
+        entries: Vec<(ResponseCacheRequest<B::Context>, Value)>,
         now: Duration,
     ) -> Result<(), Error> {
         self.async_store_entries(
@@ -209,7 +216,7 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
     /// the freshness of its original response.
     pub async fn async_store_entries(
         &self,
-        entries: Vec<(ResponseCacheRequest, Value, Duration)>,
+        entries: Vec<(ResponseCacheRequest<B::Context>, Value, Duration)>,
     ) -> Result<(), Error> {
         let writable = entries
             .into_iter()
@@ -249,8 +256,8 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
     }
 
     fn partial_hits(
-        requests: &[ResponseCacheRequest],
-        readable: Vec<(usize, &ResponseCacheRequest)>,
+        requests: &[ResponseCacheRequest<B::Context>],
+        readable: Vec<(usize, &ResponseCacheRequest<B::Context>)>,
         entries: Vec<BatchEntry<CacheEntry>>,
         now: Duration,
     ) -> Result<PartialHits, Error> {
diff --git a/litellm-rust/crates/python-bridge/src/cache/request.rs b/litellm-rust/crates/python-bridge/src/cache/request.rs
index 0c5343a63d0..52a5f7d9055 100644
--- a/litellm-rust/crates/python-bridge/src/cache/request.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/request.rs
@@ -1,5 +1,6 @@
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
+use litellm_cache::ExactCacheContext;
 use litellm_cache_response::{CacheControls, CacheKeyInput, ResponseCacheRequest};
 use litellm_host_python::from_py;
 use pyo3::{exceptions::PyValueError, prelude::*};
@@ -20,7 +21,7 @@ pub(super) fn request(value: &Bound<'_, PyAny>) -> PyResult<ResponseCacheRequest
 }
 
 fn request_input(input: RequestInput) -> PyResult<ResponseCacheRequest> {
-    let mut request = ResponseCacheRequest::new(input.key);
+    let mut request = ResponseCacheRequest::<ExactCacheContext>::new(input.key);
     if let Some(controls) = input.controls {
         request.controls = controls;
     }

From 8dc960c928614c2276c8f5e7cc8d1658009ba796 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:22:13 +0000
Subject: [PATCH 061/160] feat(cache): add SemanticCacheContext

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/cache/src/base_cache.rs | 25 +++++++++++++++++++++
 litellm-rust/crates/cache/src/lib.rs        |  2 +-
 litellm-rust/crates/cache/tests/caching.rs  | 24 +++++++++++++++++++-
 3 files changed, 49 insertions(+), 2 deletions(-)

diff --git a/litellm-rust/crates/cache/src/base_cache.rs b/litellm-rust/crates/cache/src/base_cache.rs
index 8bd69ba5ad6..6f961798795 100644
--- a/litellm-rust/crates/cache/src/base_cache.rs
+++ b/litellm-rust/crates/cache/src/base_cache.rs
@@ -32,6 +32,31 @@ impl CacheContext for ExactCacheContext {
     }
 }
 
+#[derive(Clone, Debug, Default, PartialEq)]
+pub struct SemanticCacheContext {
+    pub input: Option<serde_json::Value>,
+    pub messages: Vec<serde_json::Value>,
+    pub metadata: serde_json::Map<String, serde_json::Value>,
+    pub scope: Option<String>,
+    pub ttl: Option<Duration>,
+}
+
+impl CacheContext for SemanticCacheContext {
+    fn ttl(&self) -> Option<Duration> {
+        self.ttl
+    }
+
+    fn with_ttl(&self, ttl: Option<Duration>) -> Self {
+        Self {
+            input: self.input.clone(),
+            messages: self.messages.clone(),
+            metadata: self.metadata.clone(),
+            scope: self.scope.clone(),
+            ttl,
+        }
+    }
+}
+
 #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
 #[serde(rename_all = "lowercase")]
 pub enum CacheConnectionStatus {
diff --git a/litellm-rust/crates/cache/src/lib.rs b/litellm-rust/crates/cache/src/lib.rs
index ce9f93b6dc4..8364c635e3a 100644
--- a/litellm-rust/crates/cache/src/lib.rs
+++ b/litellm-rust/crates/cache/src/lib.rs
@@ -8,7 +8,7 @@ mod error;
 
 pub use base_cache::{
     BaseCache, BatchEntry, CacheConnectionResult, CacheConnectionStatus, CacheContext,
-    ExactCacheContext,
+    ExactCacheContext, SemanticCacheContext,
 };
 pub use cache_type::CacheType;
 pub use caching::{Cache, CacheBackend, get_cache, set_cache};
diff --git a/litellm-rust/crates/cache/tests/caching.rs b/litellm-rust/crates/cache/tests/caching.rs
index 9180ee9d0dc..2e65b4eeae5 100644
--- a/litellm-rust/crates/cache/tests/caching.rs
+++ b/litellm-rust/crates/cache/tests/caching.rs
@@ -1,7 +1,8 @@
 use std::{sync::Mutex, time::Duration};
 
 use litellm_cache::{
-    BaseCache, CacheConnectionResult, CacheContext, Error, ExactCacheContext, get_cache,
+    BaseCache, CacheConnectionResult, CacheContext, Error, ExactCacheContext,
+    SemanticCacheContext, get_cache,
 };
 
 struct TestCache {
@@ -126,6 +127,27 @@ fn associated_context_preserves_backend_specific_lookup_inputs() {
     );
 }
 
+#[test]
+fn semantic_context_with_ttl_preserves_lookup_inputs() {
+    let context = SemanticCacheContext {
+        input: Some(serde_json::json!("text")),
+        messages: vec![serde_json::json!({"role": "user", "content": "hi"})],
+        metadata: serde_json::Map::from_iter([(
+            "key".into(),
+            serde_json::json!("value"),
+        )]),
+        scope: Some("scope".into()),
+        ttl: None,
+    };
+    let updated = context.with_ttl(Some(Duration::from_secs(30)));
+    assert_eq!(updated.ttl(), Some(Duration::from_secs(30)));
+    assert_eq!(updated.input, context.input);
+    assert_eq!(updated.messages, context.messages);
+    assert_eq!(updated.metadata, context.metadata);
+    assert_eq!(updated.scope, context.scope);
+    assert_eq!(context.with_ttl(None).ttl(), None);
+}
+
 #[tokio::test]
 async fn default_batch_operations_use_async_writes_and_stop_on_failure() {
     let cache = TestCache {

From 1320eeeb41fbcd2a5842889e39f768d446fa6a05 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:23:29 +0000
Subject: [PATCH 062/160] refactor(cache-response): generalize ResponseCache
 over the backend context

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../crates/cache-response/src/response.rs     | 50 +++++++++++--------
 1 file changed, 30 insertions(+), 20 deletions(-)

diff --git a/litellm-rust/crates/cache-response/src/response.rs b/litellm-rust/crates/cache-response/src/response.rs
index e50e68cdabb..a27cc1967d5 100644
--- a/litellm-rust/crates/cache-response/src/response.rs
+++ b/litellm-rust/crates/cache-response/src/response.rs
@@ -1,21 +1,22 @@
 use std::{sync::Arc, time::Duration};
 
 use litellm_cache::{
-    BaseCache, BatchCache, BatchEntry, CacheConnectionResult, Error, ExactCacheContext, FlushCache,
+    BaseCache, BatchCache, BatchEntry, CacheConnectionResult, CacheContext, Error,
+    ExactCacheContext, FlushCache,
 };
 use serde_json::Value;
 
 use crate::{CacheControls, CacheEntry, CacheKeyInput, PartialHits, cache_key};
 
 #[derive(Clone)]
-pub struct ResponseCacheRequest {
+pub struct ResponseCacheRequest<C: CacheContext = ExactCacheContext> {
     pub key: CacheKeyInput,
     pub controls: CacheControls,
-    pub context: ExactCacheContext,
+    pub context: C,
     pub max_age: Option<Duration>,
 }
 
-impl ResponseCacheRequest {
+impl ResponseCacheRequest<ExactCacheContext> {
     pub fn new(key: CacheKeyInput) -> Self {
         Self {
             key,
@@ -32,11 +33,11 @@ impl ResponseCacheRequest {
     }
 }
 
-pub struct ResponseCache<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> {
+pub struct ResponseCache<B: BaseCache<Value = CacheEntry>> {
     backend: Arc<B>,
 }
 
-impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCache<B> {
+impl<B: BaseCache<Value = CacheEntry>> ResponseCache<B> {
     pub fn new(backend: Arc<B>) -> Self {
         Self { backend }
     }
@@ -45,8 +46,11 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
         &self.backend
     }
 
-    pub fn default_ttl(&self) -> Option<Duration> {
-        self.backend.get_ttl(&ExactCacheContext::default())
+    pub fn default_ttl(&self) -> Option<Duration>
+    where
+        B::Context: Default,
+    {
+        self.backend.get_ttl(&B::Context::default())
     }
 
     pub async fn async_flush(&self) -> Result<(), Error>
@@ -62,7 +66,7 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
 
     pub fn lookup(
         &self,
-        request: &ResponseCacheRequest,
+        request: &ResponseCacheRequest<B::Context>,
         now: Duration,
     ) -> Result<Option<Value>, Error> {
         if !request.controls.reads() {
@@ -81,7 +85,7 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
 
     pub async fn async_lookup(
         &self,
-        request: &ResponseCacheRequest,
+        request: &ResponseCacheRequest<B::Context>,
         now: Duration,
     ) -> Result<Option<Value>, Error> {
         if !request.controls.reads() {
@@ -101,7 +105,7 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
 
     pub fn lookup_batch(
         &self,
-        requests: &[ResponseCacheRequest],
+        requests: &[ResponseCacheRequest<B::Context>],
         now: Duration,
     ) -> Result<PartialHits, Error>
     where
@@ -126,7 +130,7 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
 
     pub async fn async_lookup_batch(
         &self,
-        requests: &[ResponseCacheRequest],
+        requests: &[ResponseCacheRequest<B::Context>],
         now: Duration,
     ) -> Result<PartialHits, Error>
     where
@@ -153,7 +157,7 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
 
     pub fn store(
         &self,
-        request: &ResponseCacheRequest,
+        request: &ResponseCacheRequest<B::Context>,
         response: Value,
         now: Duration,
     ) -> Result<(), Error> {
@@ -172,7 +176,7 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
 
     pub async fn async_store(
         &self,
-        request: &ResponseCacheRequest,
+        request: &ResponseCacheRequest<B::Context>,
         response: Value,
         now: Duration,
     ) -> Result<(), Error> {
@@ -193,9 +197,12 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
 
     pub async fn async_store_batch(
         &self,
-        entries: Vec<(ResponseCacheRequest, Value)>,
+        entries: Vec<(ResponseCacheRequest<B::Context>, Value)>,
         now: Duration,
-    ) -> Result<(), Error> {
+    ) -> Result<(), Error>
+    where
+        B::Context: PartialEq,
+    {
         self.async_store_entries(
             entries
                 .into_iter()
@@ -209,8 +216,11 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
     /// the freshness of its original response.
     pub async fn async_store_entries(
         &self,
-        entries: Vec<(ResponseCacheRequest, Value, Duration)>,
-    ) -> Result<(), Error> {
+        entries: Vec<(ResponseCacheRequest<B::Context>, Value, Duration)>,
+    ) -> Result<(), Error>
+    where
+        B::Context: PartialEq,
+    {
         let writable = entries
             .into_iter()
             .filter(|(request, _, _)| request.controls.writes())
@@ -249,8 +259,8 @@ impl<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>> ResponseCach
     }
 
     fn partial_hits(
-        requests: &[ResponseCacheRequest],
-        readable: Vec<(usize, &ResponseCacheRequest)>,
+        requests: &[ResponseCacheRequest<B::Context>],
+        readable: Vec<(usize, &ResponseCacheRequest<B::Context>)>,
         entries: Vec<BatchEntry<CacheEntry>>,
         now: Duration,
     ) -> Result<PartialHits, Error> {

From 8d9ab9eeaafe88efdf19fe67809e5fd21b2adb03 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:24:23 +0000
Subject: [PATCH 063/160] feat(cache-redis): expose the pooled connection
 handling for reuse

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/cache-redis/src/cache.rs  | 93 +++++++++++--------
 .../cache-redis/src/cache/operations.rs       | 28 +++---
 litellm-rust/crates/cache-redis/src/lib.rs    |  4 +
 3 files changed, 72 insertions(+), 53 deletions(-)

diff --git a/litellm-rust/crates/cache-redis/src/cache.rs b/litellm-rust/crates/cache-redis/src/cache.rs
index a960c383bf4..6388448accc 100644
--- a/litellm-rust/crates/cache-redis/src/cache.rs
+++ b/litellm-rust/crates/cache-redis/src/cache.rs
@@ -19,7 +19,7 @@ const DEFAULT_TTL: Duration = Duration::from_secs(600);
 const REDIS_TIMEOUT: Duration = Duration::from_secs(5);
 const REDIS_POOL_SIZE: u32 = 16;
 
-struct PooledConnection {
+pub struct PooledConnection {
     connection: redis::Connection,
     failed: bool,
 }
@@ -27,16 +27,19 @@ struct PooledConnection {
 /// Pools connections without a checkout PING, which would double every operation's round trips.
 /// A timed-out command leaves its reply on the socket while redis still reports the connection
 /// open, so any connection whose operation failed is discarded instead of being reused.
-struct ConnectionManager(redis::Client);
+pub struct ConnectionManager {
+    client: redis::Client,
+    timeout: Duration,
+}
 
 impl r2d2::ManageConnection for ConnectionManager {
     type Connection = PooledConnection;
     type Error = redis::RedisError;
 
     fn connect(&self) -> Result<PooledConnection, redis::RedisError> {
-        let connection = self.0.get_connection()?;
-        connection.set_read_timeout(Some(REDIS_TIMEOUT))?;
-        connection.set_write_timeout(Some(REDIS_TIMEOUT))?;
+        let connection = self.client.get_connection()?;
+        connection.set_read_timeout(Some(self.timeout))?;
+        connection.set_write_timeout(Some(self.timeout))?;
         Ok(PooledConnection {
             connection,
             failed: false,
@@ -68,12 +71,12 @@ const CLAIM_SCRIPT: &str = concat!(
 );
 const CLAIM_ATTEMPTS: usize = 8;
 
-enum Connections<C> {
+pub enum Connections<C> {
     Pool(r2d2::Pool<ConnectionManager>),
     Fixed(Mutex<C>),
 }
 
-struct ConnectionRef<'a>(&'a mut dyn redis::ConnectionLike);
+pub struct ConnectionRef<'a>(&'a mut dyn redis::ConnectionLike);
 
 impl redis::ConnectionLike for ConnectionRef<'_> {
     fn req_packed_command(&mut self, cmd: &[u8]) -> redis::RedisResult<redis::Value> {
@@ -110,7 +113,23 @@ impl<C> Connections<C>
 where
     C: redis::ConnectionLike + Send + 'static,
 {
-    fn execute<T>(
+    pub fn pooled(url: &str, timeout: Duration, pool_size: u32) -> Result<Self, Error> {
+        let client = redis::Client::open(url).map_err(|_| Error::Unavailable)?;
+        let pool = r2d2::Pool::builder()
+            .max_size(pool_size)
+            .min_idle(Some(0))
+            .connection_timeout(timeout)
+            .test_on_check_out(false)
+            .build(ConnectionManager { client, timeout })
+            .map_err(|_| Error::Unavailable)?;
+        Ok(Self::Pool(pool))
+    }
+
+    pub fn fixed(connection: C) -> Self {
+        Self::Fixed(Mutex::new(connection))
+    }
+
+    pub fn execute<T>(
         &self,
         operation: impl FnOnce(&mut ConnectionRef<'_>) -> Result<T, Error>,
     ) -> Result<T, Error> {
@@ -127,6 +146,16 @@ where
             }
         }
     }
+
+    pub async fn run_blocking<T, F>(connections: Arc<Self>, operation: F) -> Result<T, Error>
+    where
+        T: Send + 'static,
+        F: FnOnce(&mut ConnectionRef<'_>) -> Result<T, Error> + Send + 'static,
+    {
+        tokio::task::spawn_blocking(move || connections.execute(operation))
+            .await
+            .map_err(|_| Error::Unavailable)?
+    }
 }
 
 pub struct RedisCache<S, C = redis::Connection> {
@@ -138,16 +167,8 @@ pub struct RedisCache<S, C = redis::Connection> {
 
 impl<S: CacheCodec> RedisCache<S> {
     pub fn new(url: &str, default_ttl: Option<Duration>, codec: S) -> Result<Self, Error> {
-        let client = redis::Client::open(url).map_err(|_| Error::Unavailable)?;
-        let pool = r2d2::Pool::builder()
-            .max_size(REDIS_POOL_SIZE)
-            .min_idle(Some(0))
-            .connection_timeout(REDIS_TIMEOUT)
-            .test_on_check_out(false)
-            .build(ConnectionManager(client))
-            .map_err(|_| Error::Unavailable)?;
         Ok(Self {
-            connections: Arc::new(Connections::Pool(pool)),
+            connections: Arc::new(Connections::pooled(url, REDIS_TIMEOUT, REDIS_POOL_SIZE)?),
             default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
             codec,
             namespace: None,
@@ -162,7 +183,7 @@ where
 {
     pub fn with_connection(connection: C, default_ttl: Option<Duration>, codec: S) -> Self {
         Self {
-            connections: Arc::new(Connections::Fixed(Mutex::new(connection))),
+            connections: Arc::new(Connections::fixed(connection)),
             default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
             codec,
             namespace: None,
@@ -241,20 +262,14 @@ where
     }
 
     fn ttl_seconds(ttl: Duration) -> u64 {
-        ttl.as_secs()
-            .saturating_add(u64::from(ttl.subsec_nanos() > 0))
-            .max(1)
+        ttl_seconds(ttl)
     }
+}
 
-    async fn run_blocking<T, F>(connections: Arc<Connections<C>>, operation: F) -> Result<T, Error>
-    where
-        T: Send + 'static,
-        F: FnOnce(&mut ConnectionRef<'_>) -> Result<T, Error> + Send + 'static,
-    {
-        tokio::task::spawn_blocking(move || connections.execute(operation))
-            .await
-            .map_err(|_| Error::Unavailable)?
-    }
+pub fn ttl_seconds(ttl: Duration) -> u64 {
+    ttl.as_secs()
+        .saturating_add(u64::from(ttl.subsec_nanos() > 0))
+        .max(1)
 }
 
 fn namespaced_key(namespace: Option<&str>, key: &str) -> String {
@@ -313,7 +328,7 @@ where
         let payload = self.codec.encode(&value)?;
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             connection
                 .set_ex::<_, _, ()>(key, payload, ttl)
                 .map_err(|_| Error::Unavailable)
@@ -327,7 +342,7 @@ where
         _: &ExactCacheContext,
     ) -> Result<Option<Self::Value>, Error> {
         let key = self.namespaced_key(key);
-        let value = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let value = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             connection
                 .get::<_, redis::Value>(key)
                 .map_err(|_| Error::Unavailable)
@@ -350,7 +365,7 @@ where
             })
             .collect::<Result<Vec<_>, _>>()?;
         let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut pipeline = redis::pipe();
             for (key, payload) in entries {
                 pipeline
@@ -372,7 +387,7 @@ where
     }
 
     async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
-        match Self::run_blocking(Arc::clone(&self.connections), |connection| {
+        match Connections::run_blocking(Arc::clone(&self.connections), |connection| {
             Ok(match redis::cmd("PING").query::<String>(connection) {
                 Ok(_) => CacheConnectionResult {
                     status: CacheConnectionStatus::Success,
@@ -433,7 +448,7 @@ where
             .iter()
             .map(|key| self.namespaced_key(key))
             .collect::<Vec<_>>();
-        let values = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("MGET")
                 .arg(keys)
                 .query::<Vec<redis::Value>>(connection)
@@ -460,7 +475,7 @@ where
 
     async fn async_delete_cache(&self, key: &str) -> Result<(), Error> {
         let key = self.namespaced_key(key);
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             connection.del::<_, ()>(key).map_err(|_| Error::Unavailable)
         })
         .await
@@ -480,7 +495,7 @@ where
 
     async fn async_flush_cache(&self) -> Result<(), Error> {
         let pattern = self.namespaced_pattern()?;
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             Self::flush_matching(connection, &pattern)
         })
         .await
@@ -512,7 +527,7 @@ where
     ) -> Result<f64, Error> {
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             increment(connection, key, amount, ttl)
         })
         .await
@@ -623,7 +638,7 @@ where
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
         let codec = self.codec.clone();
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             claim(connection, &codec, &key, candidate, &eligible, ttl)
         })
         .await
diff --git a/litellm-rust/crates/cache-redis/src/cache/operations.rs b/litellm-rust/crates/cache-redis/src/cache/operations.rs
index d8d9ae24c4c..f27a7802bab 100644
--- a/litellm-rust/crates/cache-redis/src/cache/operations.rs
+++ b/litellm-rust/crates/cache-redis/src/cache/operations.rs
@@ -144,7 +144,7 @@ where
             .into_iter()
             .map(|key| self.namespaced_key(&key))
             .collect::<Vec<_>>();
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             connection.del(keys).map_err(|_| Error::Unavailable)
         })
         .await
@@ -172,7 +172,7 @@ where
             .iter()
             .map(|key| self.namespaced_key(key))
             .collect::<Vec<_>>();
-        let values = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("MGET")
                 .arg(keys)
                 .query::<Vec<redis::Value>>(connection)
@@ -192,7 +192,7 @@ where
     }
 
     pub async fn ping(&self) -> Result<bool, Error> {
-        Self::run_blocking(Arc::clone(&self.connections), |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), |connection| {
             redis::cmd("PING")
                 .query::<String>(connection)
                 .map(|response| response == "PONG")
@@ -203,7 +203,7 @@ where
 
     pub async fn async_get_ttl(&self, key: &str) -> Result<Option<i64>, Error> {
         let key = self.namespaced_key(key);
-        let ttl = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let ttl = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("TTL")
                 .arg(key)
                 .query::<i64>(connection)
@@ -215,7 +215,7 @@ where
 
     pub async fn async_scan_iter(&self, pattern: &str, count: usize) -> Result<Vec<String>, Error> {
         let pattern = format!("{}*", self.namespaced_key(pattern));
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut cursor = 0u64;
             let mut matches = Vec::new();
             loop {
@@ -249,7 +249,7 @@ where
         }
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(ttl.unwrap_or(self.default_ttl));
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut pipeline = redis::pipe();
             pipeline.cmd("SADD").arg(&key).arg(values);
             pipeline.cmd("EXPIRE").arg(&key).arg(ttl).ignore();
@@ -266,7 +266,7 @@ where
             return Err(Error::InvalidEntry);
         }
         let key = self.namespaced_key(key);
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("RPUSH")
                 .arg(key)
                 .arg(values)
@@ -292,7 +292,7 @@ where
         if operations.is_empty() {
             return Ok(Vec::new());
         }
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut pipeline = redis::pipe();
             for (key, values) in operations {
                 pipeline.cmd("RPUSH").arg(key).arg(values);
@@ -309,7 +309,7 @@ where
     ) -> Result<RedisLpopResult, Error> {
         let key = self.namespaced_key(key);
         let multiple = count.is_some();
-        let value = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let value = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut command = redis::cmd("LPOP");
             command.arg(key);
             if let Some(count) = count {
@@ -338,7 +338,7 @@ where
             .iter()
             .map(|(_, count)| count.is_some())
             .collect::<Vec<_>>();
-        let values = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut pipeline = redis::pipe();
             for (key, count) in operations {
                 let command = pipeline.cmd("LPOP").arg(key);
@@ -368,7 +368,7 @@ where
             .into_iter()
             .map(|key| self.namespaced_key(&key))
             .collect::<Vec<_>>();
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("EVAL")
                 .arg(script)
                 .arg(keys.len())
@@ -440,7 +440,7 @@ where
         if operations.is_empty() {
             return Ok(Vec::new());
         }
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut pipeline = redis::pipe();
             for (key, amount, ttl) in operations {
                 pipeline.cmd("INCRBYFLOAT").arg(&key).arg(amount);
@@ -461,7 +461,7 @@ where
     ) -> Result<i64, Error> {
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(ttl);
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             increment_with_floor(connection, key, amount, ttl)
         })
         .await
@@ -475,7 +475,7 @@ where
     ) -> Result<f64, Error> {
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(ttl.unwrap_or(self.default_ttl));
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("EVAL")
                 .arg(SET_MAX_SCRIPT)
                 .arg(1)
diff --git a/litellm-rust/crates/cache-redis/src/lib.rs b/litellm-rust/crates/cache-redis/src/lib.rs
index 98f6bfd8ce5..ea75906e9c9 100644
--- a/litellm-rust/crates/cache-redis/src/lib.rs
+++ b/litellm-rust/crates/cache-redis/src/lib.rs
@@ -1,6 +1,10 @@
 mod cache;
 mod topology;
 
+pub mod connection {
+    pub use crate::cache::{ConnectionRef, Connections, ttl_seconds};
+}
+
 pub use cache::{
     RedisArg, RedisCache, RedisLpopOperation, RedisLpopResult, RedisRpushOperation, RedisScript,
 };

From 1f86bb8e4640fd7e758e10f66106fd7c1bda01de Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:24:34 +0000
Subject: [PATCH 064/160] feat(cache-valkey-semantic): add native Valkey
 semantic cache backend

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.lock                       |  16 +
 .../crates/cache-valkey-semantic/Cargo.toml   |  20 +
 .../crates/cache-valkey-semantic/src/lib.rs   | 844 ++++++++++++++++++
 3 files changed, 880 insertions(+)
 create mode 100644 litellm-rust/crates/cache-valkey-semantic/Cargo.toml
 create mode 100644 litellm-rust/crates/cache-valkey-semantic/src/lib.rs

diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index ed4ae4e3353..5daf691d4c3 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -2502,6 +2502,22 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "litellm-cache-valkey-semantic"
+version = "0.1.0"
+dependencies = [
+ "litellm-cache",
+ "litellm-cache-response",
+ "r2d2",
+ "redis",
+ "redis-test",
+ "rstest",
+ "serde_json",
+ "sha2 0.10.9",
+ "tokio",
+ "uuid",
+]
+
 [[package]]
 name = "litellm-callbacks-legacy-python"
 version = "0.1.0"
diff --git a/litellm-rust/crates/cache-valkey-semantic/Cargo.toml b/litellm-rust/crates/cache-valkey-semantic/Cargo.toml
new file mode 100644
index 00000000000..9a0a566ca3b
--- /dev/null
+++ b/litellm-rust/crates/cache-valkey-semantic/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "litellm-cache-valkey-semantic"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-cache.workspace = true
+litellm-cache-response.workspace = true
+r2d2 = "0.8.10"
+redis = { version = "1.7.0", features = ["tls-rustls"] }
+serde_json.workspace = true
+sha2.workspace = true
+tokio.workspace = true
+uuid = { version = "1", features = ["v4"] }
+
+[dev-dependencies]
+redis-test = "1.0.4"
+rstest.workspace = true
diff --git a/litellm-rust/crates/cache-valkey-semantic/src/lib.rs b/litellm-rust/crates/cache-valkey-semantic/src/lib.rs
new file mode 100644
index 00000000000..85c4c9af15c
--- /dev/null
+++ b/litellm-rust/crates/cache-valkey-semantic/src/lib.rs
@@ -0,0 +1,844 @@
+use std::{
+    future::Future,
+    sync::{Arc, Mutex},
+    time::Duration,
+};
+
+use litellm_cache::{BaseCache, CacheCodec, CacheConnectionResult, Error, SemanticCacheContext};
+use litellm_cache_response::CacheEntry;
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+use uuid::Uuid;
+
+pub trait Embedder: Send + Sync + 'static {
+    fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, Error>;
+
+    fn async_embed(
+        &self,
+        prompt: &str,
+        metadata: Option<&Value>,
+    ) -> impl Future<Output = Result<Vec<f32>, Error>> + Send;
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct ValkeySemanticConfig {
+    pub similarity_threshold: f64,
+    pub index_name: String,
+}
+
+pub const DEFAULT_INDEX_NAME: &str = "litellm_semantic_cache_index";
+
+struct PooledConnection {
+    connection: redis::Connection,
+    failed: bool,
+}
+
+struct ConnectionManager(redis::Client);
+
+impl r2d2::ManageConnection for ConnectionManager {
+    type Connection = PooledConnection;
+    type Error = redis::RedisError;
+
+    fn connect(&self) -> Result<Self::Connection, Self::Error> {
+        let connection = self.0.get_connection()?;
+        Ok(PooledConnection {
+            connection,
+            failed: false,
+        })
+    }
+
+    fn is_valid(&self, connection: &mut Self::Connection) -> Result<(), Self::Error> {
+        redis::cmd("PING").query::<String>(&mut connection.connection)?;
+        Ok(())
+    }
+
+    fn has_broken(&self, connection: &mut Self::Connection) -> bool {
+        connection.failed || !redis::ConnectionLike::is_open(&connection.connection)
+    }
+}
+
+enum Connections<C> {
+    Pool(r2d2::Pool<ConnectionManager>),
+    Fixed(Mutex<C>),
+}
+
+struct ConnectionRef<'a>(&'a mut dyn redis::ConnectionLike);
+
+impl redis::ConnectionLike for ConnectionRef<'_> {
+    fn req_packed_command(&mut self, cmd: &[u8]) -> redis::RedisResult<redis::Value> {
+        self.0.req_packed_command(cmd)
+    }
+
+    fn req_packed_commands(
+        &mut self,
+        cmd: &[u8],
+        offset: usize,
+        count: usize,
+    ) -> redis::RedisResult<Vec<redis::Value>> {
+        self.0.req_packed_commands(cmd, offset, count)
+    }
+
+    fn get_db(&self) -> i64 {
+        self.0.get_db()
+    }
+
+    fn supports_pipelining(&self) -> bool {
+        self.0.supports_pipelining()
+    }
+
+    fn check_connection(&mut self) -> bool {
+        self.0.check_connection()
+    }
+
+    fn is_open(&self) -> bool {
+        self.0.is_open()
+    }
+}
+
+impl<C> Connections<C>
+where
+    C: redis::ConnectionLike + Send + 'static,
+{
+    fn execute<T>(
+        &self,
+        operation: impl FnOnce(&mut ConnectionRef<'_>) -> Result<T, Error>,
+    ) -> Result<T, Error> {
+        match self {
+            Self::Pool(pool) => {
+                let mut pooled = pool.get().map_err(|_| Error::Unavailable)?;
+                let result = operation(&mut ConnectionRef(&mut pooled.connection));
+                pooled.failed = matches!(result, Err(Error::Unavailable));
+                result
+            }
+            Self::Fixed(connection) => {
+                let mut connection = connection.lock().map_err(|_| Error::Unavailable)?;
+                operation(&mut ConnectionRef(&mut *connection))
+            }
+        }
+    }
+}
+
+pub struct ValkeySemanticCache<
+    E: Embedder,
+    S: CacheCodec<Value = CacheEntry>,
+    C = redis::Connection,
+> {
+    connections: Arc<Connections<C>>,
+    embedder: E,
+    codec: S,
+    config: ValkeySemanticConfig,
+    index_dimension: Arc<Mutex<Option<usize>>>,
+}
+
+impl<E, S> ValkeySemanticCache<E, S>
+where
+    E: Embedder,
+    S: CacheCodec<Value = CacheEntry>,
+{
+    pub fn new(
+        url: &str,
+        embedder: E,
+        codec: S,
+        config: ValkeySemanticConfig,
+    ) -> Result<Self, Error> {
+        let client = redis::Client::open(url).map_err(|_| Error::Unavailable)?;
+        let pool = r2d2::Pool::builder()
+            .max_size(16)
+            .min_idle(Some(0))
+            .test_on_check_out(false)
+            .build(ConnectionManager(client))
+            .map_err(|_| Error::Unavailable)?;
+        Ok(Self {
+            connections: Arc::new(Connections::Pool(pool)),
+            embedder,
+            codec,
+            config,
+            index_dimension: Arc::new(Mutex::new(None)),
+        })
+    }
+}
+
+impl<E, S, C> ValkeySemanticCache<E, S, C>
+where
+    E: Embedder,
+    S: CacheCodec<Value = CacheEntry>,
+    C: redis::ConnectionLike + Send + 'static,
+{
+    pub fn with_connection(
+        connection: C,
+        embedder: E,
+        codec: S,
+        config: ValkeySemanticConfig,
+    ) -> Self {
+        Self {
+            connections: Arc::new(Connections::Fixed(Mutex::new(connection))),
+            embedder,
+            codec,
+            config,
+            index_dimension: Arc::new(Mutex::new(None)),
+        }
+    }
+
+    pub fn similarity_threshold(&self) -> f64 {
+        self.config.similarity_threshold
+    }
+
+    pub fn index_name(&self) -> &str {
+        &self.config.index_name
+    }
+
+    fn key_prefix(&self) -> String {
+        format!("{}:", self.config.index_name)
+    }
+
+    fn ensure_index(&self, dimension: usize) -> Result<(), Error> {
+        ensure_index(
+            &self.connections,
+            &self.config.index_name,
+            &self.key_prefix(),
+            &self.index_dimension,
+            dimension,
+        )
+    }
+}
+
+impl<E, S, C> BaseCache for ValkeySemanticCache<E, S, C>
+where
+    E: Embedder,
+    S: CacheCodec<Value = CacheEntry>,
+    C: redis::ConnectionLike + Send + 'static,
+{
+    type Value = CacheEntry;
+    type Context = SemanticCacheContext;
+
+    fn get_ttl(&self, context: &Self::Context) -> Option<Duration> {
+        context.ttl
+    }
+
+    fn set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: &Self::Context,
+    ) -> Result<(), Error> {
+        let Some(prompt) = prompt_from_context(context) else {
+            return Ok(());
+        };
+        let embedding = self.embedder.embed(&prompt, context.metadata.as_ref())?;
+        self.ensure_index(embedding.len())?;
+        let scope = scope_tag(key);
+        let document = format!("{}{}:{}", self.key_prefix(), scope, Uuid::new_v4());
+        let response = self.codec.encode(&value)?;
+        let vector = embedding_bytes(&embedding);
+        let ttl = self.get_ttl(context);
+        self.connections.execute(|connection| {
+            let mut pipeline = redis::pipe();
+            pipeline
+                .cmd("HSET")
+                .arg(&document)
+                .arg("litellm_cache_key")
+                .arg(&scope)
+                .arg("prompt")
+                .arg(prompt)
+                .arg("response")
+                .arg(response)
+                .arg("embedding")
+                .arg(vector)
+                .ignore();
+            if let Some(ttl) = ttl {
+                pipeline
+                    .cmd("EXPIRE")
+                    .arg(&document)
+                    .arg(ttl.as_secs())
+                    .ignore();
+            }
+            pipeline
+                .query::<()>(connection)
+                .map_err(|_| Error::Unavailable)
+        })
+    }
+
+    fn get_cache(&self, key: &str, context: &Self::Context) -> Result<Option<Self::Value>, Error> {
+        let Some(prompt) = prompt_from_context(context) else {
+            return Ok(None);
+        };
+        let embedding = self.embedder.embed(&prompt, context.metadata.as_ref())?;
+        self.ensure_index(embedding.len())?;
+        let scope = scope_tag(key);
+        let query =
+            format!("(@litellm_cache_key:{{{scope}}})=>[KNN 1 @embedding $vec AS vector_distance]");
+        let vector = embedding_bytes(&embedding);
+        let response = self.connections.execute(|connection| {
+            redis::cmd("FT.SEARCH")
+                .arg(&self.config.index_name)
+                .arg(query)
+                .arg("PARAMS")
+                .arg(2)
+                .arg("vec")
+                .arg(vector)
+                .arg("RETURN")
+                .arg(2)
+                .arg("response")
+                .arg("vector_distance")
+                .arg("DIALECT")
+                .arg(2)
+                .query::<redis::Value>(connection)
+                .map_err(|_| Error::Unavailable)
+        })?;
+        let Some(fields) = search_fields(response)? else {
+            return Ok(None);
+        };
+        let response = fields
+            .iter()
+            .find_map(|(name, value)| (name == "response").then(|| value.clone()))
+            .ok_or(Error::InvalidEntry)?;
+        let distance = fields
+            .iter()
+            .find_map(|(name, value)| (name == "vector_distance").then(|| value.clone()))
+            .ok_or(Error::InvalidEntry)?;
+        let distance = parse_f64(&distance)?;
+        if 1.0 - distance < self.config.similarity_threshold {
+            return Ok(None);
+        }
+        self.codec.decode(&response).map(Some)
+    }
+
+    fn async_set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: Self::Context,
+    ) -> impl Future<Output = Result<(), Error>> + Send {
+        let key = key.to_owned();
+        let prompt = prompt_from_context(&context);
+        let metadata = context.metadata.clone();
+        async move {
+            let Some(prompt) = prompt else {
+                return Ok(());
+            };
+            let embedding = self
+                .embedder
+                .async_embed(&prompt, metadata.as_ref())
+                .await?;
+            let connections = Arc::clone(&self.connections);
+            let config = self.config.clone();
+            let index_dimension = Arc::clone(&self.index_dimension);
+            let response = self.codec.encode(&value)?;
+            let vector = embedding_bytes(&embedding);
+            let prefix = format!("{}:", config.index_name);
+            let scope = scope_tag(&key);
+            let document = format!("{prefix}{scope}:{}", Uuid::new_v4());
+            let ttl = context.ttl;
+            tokio::task::spawn_blocking(move || {
+                ensure_index(
+                    &connections,
+                    &config.index_name,
+                    &prefix,
+                    &index_dimension,
+                    embedding.len(),
+                )?;
+                connections.execute(|connection| {
+                    let mut pipeline = redis::pipe();
+                    pipeline
+                        .cmd("HSET")
+                        .arg(&document)
+                        .arg("litellm_cache_key")
+                        .arg(&scope)
+                        .arg("prompt")
+                        .arg(prompt)
+                        .arg("response")
+                        .arg(response)
+                        .arg("embedding")
+                        .arg(vector)
+                        .ignore();
+                    if let Some(ttl) = ttl {
+                        pipeline
+                            .cmd("EXPIRE")
+                            .arg(&document)
+                            .arg(ttl.as_secs())
+                            .ignore();
+                    }
+                    pipeline
+                        .query::<()>(connection)
+                        .map_err(|_| Error::Unavailable)
+                })
+            })
+            .await
+            .map_err(|_| Error::Unavailable)?
+        }
+    }
+
+    fn async_get_cache(
+        &self,
+        key: &str,
+        context: &Self::Context,
+    ) -> impl Future<Output = Result<Option<Self::Value>, Error>> + Send {
+        let key = key.to_owned();
+        let prompt = prompt_from_context(context);
+        let metadata = context.metadata.clone();
+        async move {
+            let Some(prompt) = prompt else {
+                return Ok(None);
+            };
+            let embedding = self
+                .embedder
+                .async_embed(&prompt, metadata.as_ref())
+                .await?;
+            let connections = Arc::clone(&self.connections);
+            let config = self.config.clone();
+            let index_dimension = Arc::clone(&self.index_dimension);
+            let threshold = config.similarity_threshold;
+            tokio::task::spawn_blocking(move || {
+                let prefix = format!("{}:", config.index_name);
+                ensure_index(
+                    &connections,
+                    &config.index_name,
+                    &prefix,
+                    &index_dimension,
+                    embedding.len(),
+                )?;
+                let scope = scope_tag(&key);
+                let query = format!(
+                    "(@litellm_cache_key:{{{scope}}})=>[KNN 1 @embedding $vec AS vector_distance]"
+                );
+                let vector = embedding_bytes(&embedding);
+                let response = connections.execute(|connection| {
+                    redis::cmd("FT.SEARCH")
+                        .arg(&config.index_name)
+                        .arg(query)
+                        .arg("PARAMS")
+                        .arg(2)
+                        .arg("vec")
+                        .arg(vector)
+                        .arg("RETURN")
+                        .arg(2)
+                        .arg("response")
+                        .arg("vector_distance")
+                        .arg("DIALECT")
+                        .arg(2)
+                        .query::<redis::Value>(connection)
+                        .map_err(|_| Error::Unavailable)
+                })?;
+                let Some(fields) = search_fields(response)? else {
+                    return Ok(None);
+                };
+                let response = fields
+                    .iter()
+                    .find_map(|(name, value)| (name == "response").then(|| value.clone()))
+                    .ok_or(Error::InvalidEntry)?;
+                let distance = fields
+                    .iter()
+                    .find_map(|(name, value)| (name == "vector_distance").then(|| value.clone()))
+                    .ok_or(Error::InvalidEntry)?;
+                let distance = parse_f64(&distance)?;
+                if 1.0 - distance < threshold {
+                    return Ok(None);
+                }
+                Ok(Some(response))
+            })
+            .await
+            .map_err(|_| Error::Unavailable)?
+            .and_then(|response| response.map(|bytes| self.codec.decode(&bytes)).transpose())
+        }
+    }
+
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        Err(Error::UnsupportedOperation)
+    }
+}
+
+pub fn prompt_from_context(context: &SemanticCacheContext) -> Option<String> {
+    if let Some(Value::Array(messages)) = context.messages.as_ref()
+        && !messages.is_empty()
+    {
+        return Some(
+            messages
+                .iter()
+                .filter_map(Value::as_object)
+                .map(message_text)
+                .collect(),
+        );
+    }
+    let input = context.input.as_ref()?;
+    let mut parts = Vec::new();
+    collect_input_text(input, &mut parts);
+    let prompt = parts.join("\n").trim().to_owned();
+    (!prompt.is_empty()).then_some(prompt)
+}
+
+fn message_text(message: &serde_json::Map<String, Value>) -> String {
+    let content = match message.get("content") {
+        Some(Value::String(value)) => value.clone(),
+        Some(Value::Array(parts)) => parts
+            .iter()
+            .filter_map(Value::as_object)
+            .filter_map(|part| part.get("text").and_then(Value::as_str))
+            .filter(|text| !text.is_empty())
+            .collect(),
+        _ => String::new(),
+    };
+    format!(
+        "{content}{}",
+        search_results_text(message.get("search_results"))
+    )
+}
+
+fn search_results_text(value: Option<&Value>) -> String {
+    let Some(Value::Array(results)) = value else {
+        return String::new();
+    };
+    results
+        .iter()
+        .filter_map(Value::as_object)
+        .map(|result| {
+            let source = result.get("source").and_then(Value::as_str).unwrap_or("");
+            let title = result.get("title").and_then(Value::as_str).unwrap_or("");
+            let content = result
+                .get("content")
+                .and_then(Value::as_array)
+                .map(|blocks| {
+                    blocks
+                        .iter()
+                        .filter_map(Value::as_object)
+                        .filter_map(|block| block.get("text").and_then(Value::as_str))
+                        .collect::<String>()
+                })
+                .unwrap_or_default();
+            let citations = result
+                .get("citations")
+                .filter(|value| !value.is_null())
+                .and_then(|value| serde_json::to_string(value).ok())
+                .unwrap_or_default();
+            format!("{source}{title}{content}{citations}")
+        })
+        .collect()
+}
+
+fn collect_input_text(value: &Value, parts: &mut Vec<String>) {
+    match value {
+        Value::String(value) => {
+            let value = value.trim();
+            if !value.is_empty() {
+                parts.push(value.to_owned());
+            }
+        }
+        Value::Array(values) => values
+            .iter()
+            .for_each(|value| collect_input_text(value, parts)),
+        Value::Object(object) => {
+            if let Some(content) = object.get("content").filter(|value| !value.is_null()) {
+                collect_input_text(content, parts);
+                return;
+            }
+            for key in ["text", "output", "input_text", "output_text"] {
+                if let Some(Value::String(value)) = object.get(key) {
+                    let value = value.trim();
+                    if !value.is_empty() {
+                        parts.push(value.to_owned());
+                        return;
+                    }
+                }
+            }
+        }
+        _ => {}
+    }
+}
+
+fn scope_tag(key: &str) -> String {
+    let digest = Sha256::digest(key.as_bytes());
+    digest.iter().map(|byte| format!("{byte:02x}")).collect()
+}
+
+fn embedding_bytes(embedding: &[f32]) -> Vec<u8> {
+    embedding
+        .iter()
+        .flat_map(|value| value.to_le_bytes())
+        .collect()
+}
+
+fn ensure_index<C>(
+    connections: &Connections<C>,
+    index_name: &str,
+    prefix: &str,
+    index_dimension: &Mutex<Option<usize>>,
+    dimension: usize,
+) -> Result<(), Error>
+where
+    C: redis::ConnectionLike + Send + 'static,
+{
+    if index_dimension
+        .lock()
+        .map_err(|_| Error::Unavailable)?
+        .is_some_and(|existing| existing == dimension)
+    {
+        return Ok(());
+    }
+    let create = connections.execute(|connection| {
+        Ok(redis::cmd("FT.CREATE")
+            .arg(index_name)
+            .arg("ON")
+            .arg("HASH")
+            .arg("PREFIX")
+            .arg(1)
+            .arg(prefix)
+            .arg("SCHEMA")
+            .arg("litellm_cache_key")
+            .arg("TAG")
+            .arg("embedding")
+            .arg("VECTOR")
+            .arg("HNSW")
+            .arg(6)
+            .arg("TYPE")
+            .arg("FLOAT32")
+            .arg("DIM")
+            .arg(dimension)
+            .arg("DISTANCE_METRIC")
+            .arg("COSINE")
+            .query::<String>(connection)
+            .map(|_| ())
+            .map_err(|error| error.to_string()))
+    })?;
+    if let Err(message) = create {
+        if !message.to_ascii_lowercase().contains("already exists") {
+            return Err(Error::Unavailable);
+        }
+        let info = connections.execute(|connection| {
+            redis::cmd("FT.INFO")
+                .arg(index_name)
+                .query::<redis::Value>(connection)
+                .map_err(|_| Error::Unavailable)
+        })?;
+        let existing = index_dimension_from_info(&info).ok_or(Error::Unavailable)?;
+        if existing != dimension {
+            return Err(Error::Unavailable);
+        }
+    }
+    *index_dimension.lock().map_err(|_| Error::Unavailable)? = Some(dimension);
+    Ok(())
+}
+
+fn index_dimension_from_info(value: &redis::Value) -> Option<usize> {
+    let redis::Value::Array(values) = value else {
+        return None;
+    };
+    let attributes = values.windows(2).find_map(|pair| {
+        (value_text(&pair[0]).as_deref() == Some("attributes")).then_some(&pair[1])
+    })?;
+    let redis::Value::Array(fields) = attributes else {
+        return None;
+    };
+    fields.iter().find_map(|field| {
+        let redis::Value::Array(values) = field else {
+            return None;
+        };
+        let flattened = values.iter().flat_map(|value| match value {
+            redis::Value::Array(values) => values.as_slice(),
+            _ => std::slice::from_ref(value),
+        });
+        let values = flattened.collect::<Vec<_>>();
+        values.windows(2).find_map(|pair| {
+            if value_text(pair[0]).as_deref() == Some("dimensions") {
+                return value_text(pair[1]).and_then(|value| value.parse().ok());
+            }
+            None
+        })
+    })
+}
+
+type SearchFields = Vec<(String, Vec<u8>)>;
+
+fn search_fields(value: redis::Value) -> Result<Option<SearchFields>, Error> {
+    let redis::Value::Array(values) = value else {
+        return Err(Error::InvalidEntry);
+    };
+    let total = parse_i64(values.first().ok_or(Error::InvalidEntry)?)?;
+    if total <= 0 || values.len() < 3 {
+        return Ok(None);
+    }
+    let redis::Value::Array(fields) = &values[2] else {
+        return Err(Error::InvalidEntry);
+    };
+    let (pairs, remainder) = fields.as_chunks::<2>();
+    if !remainder.is_empty() {
+        return Err(Error::InvalidEntry);
+    }
+    let pairs = pairs
+        .iter()
+        .map(|pair| {
+            Ok((
+                value_text(&pair[0]).ok_or(Error::InvalidEntry)?,
+                value_bytes(&pair[1])?,
+            ))
+        })
+        .collect::<Result<Vec<_>, Error>>()?;
+    Ok(Some(pairs))
+}
+
+fn parse_i64(value: &redis::Value) -> Result<i64, Error> {
+    value_text(value)
+        .ok_or(Error::InvalidEntry)?
+        .parse()
+        .map_err(|_| Error::InvalidEntry)
+}
+
+fn parse_f64(value: &[u8]) -> Result<f64, Error> {
+    std::str::from_utf8(value)
+        .map_err(|_| Error::InvalidEntry)?
+        .parse()
+        .map_err(|_| Error::InvalidEntry)
+}
+
+fn value_text(value: &redis::Value) -> Option<String> {
+    match value {
+        redis::Value::BulkString(bytes) => String::from_utf8(bytes.clone()).ok(),
+        redis::Value::SimpleString(value) => Some(value.clone()),
+        redis::Value::Int(value) => Some(value.to_string()),
+        _ => None,
+    }
+}
+
+fn value_bytes(value: &redis::Value) -> Result<Vec<u8>, Error> {
+    match value {
+        redis::Value::BulkString(bytes) => Ok(bytes.clone()),
+        redis::Value::SimpleString(value) => Ok(value.as_bytes().to_vec()),
+        redis::Value::Int(value) => Ok(value.to_string().into_bytes()),
+        _ => Err(Error::InvalidEntry),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::{Arc, Mutex};
+
+    use litellm_cache::BaseCache;
+    use litellm_cache_response::ResponseCacheCodec;
+    use redis_test::MockRedisConnection;
+    use rstest::rstest;
+    use serde_json::{Value, json};
+
+    use super::{
+        Embedder, ValkeySemanticCache, ValkeySemanticConfig, index_dimension_from_info,
+        prompt_from_context, scope_tag,
+    };
+
+    #[derive(Clone)]
+    struct FixedEmbedder {
+        vector: Vec<f32>,
+        calls: EmbedderCalls,
+    }
+
+    type EmbedderCalls = Arc<Mutex<Vec<(String, Option<Value>)>>>;
+
+    impl Embedder for FixedEmbedder {
+        fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, super::Error> {
+            self.calls
+                .lock()
+                .unwrap()
+                .push((prompt.into(), metadata.cloned()));
+            Ok(self.vector.clone())
+        }
+
+        async fn async_embed(
+            &self,
+            prompt: &str,
+            metadata: Option<&Value>,
+        ) -> Result<Vec<f32>, super::Error> {
+            self.embed(prompt, metadata)
+        }
+    }
+
+    fn context(
+        messages: Option<Value>,
+        input: Option<Value>,
+    ) -> litellm_cache::SemanticCacheContext {
+        litellm_cache::SemanticCacheContext {
+            messages,
+            input,
+            ..Default::default()
+        }
+    }
+
+    #[rstest]
+    #[case(json!([{"content": "hello"}]), None, Some("hello"))]
+    #[case(json!([{"content": [{"text": "hello"}, {"text": " world"}]}]), None, Some("hello world"))]
+    #[case(json!([{"search_results": [{"source": "s", "title": "t", "content": [{"text": "c"}], "citations": ["x"]}]}]), None, Some(r#"stc["x"]"#))]
+    #[case(Value::Array(vec![]), Some(json!(" hello ")), Some("hello"))]
+    #[case(Value::Array(vec![]), Some(json!([{"content": "first"}, {"text": "second"}])), Some("first\nsecond"))]
+    #[case(Value::Array(vec![]), Some(json!("   ")), None)]
+    fn prompt_shapes(
+        #[case] messages: Value,
+        #[case] input: Option<Value>,
+        #[case] expected: Option<&str>,
+    ) {
+        assert_eq!(
+            prompt_from_context(&context(Some(messages), input)),
+            expected.map(str::to_owned)
+        );
+    }
+
+    #[test]
+    fn scope_tags_are_lowercase_sha256() {
+        assert_eq!(
+            scope_tag("key"),
+            "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683"
+        );
+    }
+
+    #[test]
+    fn existing_index_dimension_is_read_from_attributes() {
+        let info = redis::Value::Array(vec![
+            redis::Value::SimpleString("attributes".into()),
+            redis::Value::Array(vec![redis::Value::Array(vec![
+                redis::Value::SimpleString("identifier".into()),
+                redis::Value::SimpleString("embedding".into()),
+                redis::Value::Array(vec![
+                    redis::Value::SimpleString("dimensions".into()),
+                    redis::Value::SimpleString("2".into()),
+                ]),
+            ])]),
+        ]);
+        assert_eq!(index_dimension_from_info(&info), Some(2));
+    }
+
+    #[tokio::test]
+    async fn unsupported_connection_test_is_reported() {
+        let cache = ValkeySemanticCache::with_connection(
+            MockRedisConnection::new([]).assert_all_commands_consumed(),
+            FixedEmbedder {
+                vector: vec![1.0, 0.0],
+                calls: Arc::default(),
+            },
+            ResponseCacheCodec,
+            ValkeySemanticConfig {
+                similarity_threshold: 0.8,
+                index_name: "test".into(),
+            },
+        );
+        assert_eq!(
+            cache.test_connection().await,
+            Err(super::Error::UnsupportedOperation)
+        );
+    }
+
+    #[test]
+    fn missing_prompt_does_not_touch_redis() {
+        let cache = ValkeySemanticCache::with_connection(
+            MockRedisConnection::new([]).assert_all_commands_consumed(),
+            FixedEmbedder {
+                vector: vec![1.0, 0.0],
+                calls: Arc::default(),
+            },
+            ResponseCacheCodec,
+            ValkeySemanticConfig {
+                similarity_threshold: 0.8,
+                index_name: "test".into(),
+            },
+        );
+        assert_eq!(cache.get_cache("key", &context(None, None)).unwrap(), None);
+        assert_eq!(cache.get_ttl(&context(None, None)), None);
+    }
+}

From be2f0d081b6c7ac41090ad9300b18402f226ef5f Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Mon, 21 Sep 2026 13:25:39 -0700
Subject: [PATCH 065/160] fix(proxy): report sources only on the read endpoints
 main does not cover

/config/field/info and /config/list already report per-key source on main,
so this drops the branch's versions of those and keeps /alerting/settings,
/get/ui_settings and /router/settings.

Read endpoints no longer write the freshly read database row back into the
shared settings store; the reload path already keeps it current, and a GET
that mutates global state leaks across callers.

Regenerates the lazy OpenAPI snapshot on Python 3.12, matching CI, and the
dashboard API types for the two new response fields.
---
 litellm/proxy/_lazy_openapi_snapshot.json     |  2 +-
 .../router_settings_endpoints.py              | 31 +++++++-------
 litellm/proxy/proxy_server.py                 |  9 ++--
 .../proxy_setting_endpoints.py                | 41 +++++++++++--------
 .../test_router_settings_endpoints.py         | 41 ++++++++++---------
 ui/litellm-dashboard/src/lib/http/schema.d.ts | 11 +++++
 6 files changed, 80 insertions(+), 55 deletions(-)

diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json
index 391f0042ed0..06e157498aa 100644
--- a/litellm/proxy/_lazy_openapi_snapshot.json
+++ b/litellm/proxy/_lazy_openapi_snapshot.json
@@ -19632,7 +19632,7 @@
                   }
                 }
               },
-              "description": "\nUnified rate-limit error.\n\nEvery rate-limit condition surfaced by litellm \u2014 whether it originated from\nan upstream LLM provider, a vendor batch endpoint, or one of litellm's own\nproxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\nmax-iterations, etc.) \u2014 is raised as an instance of this class.\n\nThe :attr:`category` attribute lets callers distinguish the source. See\n:class:`RateLimitErrorCategory` for the available values.\n"
+              "description": "\n    Unified rate-limit error.\n\n    Every rate-limit condition surfaced by litellm \u2014 whether it originated from\n    an upstream LLM provider, a vendor batch endpoint, or one of litellm's own\n    proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\n    max-iterations, etc.) \u2014 is raised as an instance of this class.\n\n    The :attr:`category` attribute lets callers distinguish the source. See\n    :class:`RateLimitErrorCategory` for the available values.\n    "
             },
             "500": {
               "content": {
diff --git a/litellm/proxy/management_endpoints/router_settings_endpoints.py b/litellm/proxy/management_endpoints/router_settings_endpoints.py
index 019ac68ae23..d6d74ada35a 100644
--- a/litellm/proxy/management_endpoints/router_settings_endpoints.py
+++ b/litellm/proxy/management_endpoints/router_settings_endpoints.py
@@ -8,7 +8,9 @@ GET /router/fields - Get router settings field definitions without values (for U
 """
 
 import inspect
-from typing import Any, Final, cast, get_args
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Any, Final, get_args
 
 from fastapi import APIRouter, Depends
 from pydantic import BaseModel, Field
@@ -127,19 +129,20 @@ async def get_router_settings(
             if field.field_name in current_values:
                 field.field_value = current_values[field.field_name]
 
-        field_defaults: Final[dict[str, object]] = {
-            field.field_name: cast(object, field.field_default)  # cast-ok: Pydantic field defaults are untyped
-            for field in router_fields
-        }
-        source: Final[dict[str, FieldSource]] = {
-            key: _router_setting_source(
-                proxy_config.router_settings,
-                key,
-                cast(object, current_values[key]),  # cast-ok: current values are stored in a typed response map
-                field_defaults.get(key),
-            )
-            for key in current_values
-        }
+        field_defaults: Final[Mapping[str, object]] = MappingProxyType(
+            {field.field_name: field.field_default for field in router_fields}
+        )
+        source: Final[Mapping[str, FieldSource]] = MappingProxyType(
+            {
+                key: _router_setting_source(
+                    proxy_config.router_settings,
+                    key,
+                    current_values[key],
+                    field_defaults.get(key),
+                )
+                for key in current_values
+            }
+        )
         return RouterSettingsResponse(
             fields=router_fields,
             current_values=current_values,
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 4b2781e031b..24af6d7f7d3 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -15991,7 +15991,7 @@ def _nested_setting_source(
     field_default: JsonValue,
 ) -> FieldSource:
     db_value: Final = db_values.get(field_name)
-    if db_value is not None and db_value != []:
+    if db_value is not None and not (isinstance(db_value, list) and len(db_value) == 0):
         return "db"
     parent_value: Final = settings.config_value(parent_key)
     if isinstance(parent_value, Mapping) and field_name in parent_value:
@@ -16036,13 +16036,13 @@ async def alerting_settings(
         where={"param_name": "general_settings"}
     )
 
-    db_general_settings_dict: Final[Mapping[str, JsonValue]] = (
-        dict(db_general_settings.param_value)
+    db_general_settings_dict: Final[Mapping[str, JsonValue]] = MappingProxyType(
+        dict(db_general_settings.param_value)  # mutable-ok: Prisma returns the JSON column as a plain dict
         if db_general_settings is not None and db_general_settings.param_value is not None
         else {}
     )
     alerting_args_value: Final = db_general_settings_dict.get("alerting_args")
-    alerting_args_dict: Final[Mapping[str, JsonValue]] = (
+    alerting_args_dict: Final[Mapping[str, JsonValue]] = MappingProxyType(
         alerting_args_value if isinstance(alerting_args_value, dict) else {}
     )
     alerting_values: Final = cast(  # cast-ok: alerting is stored as a JSON list when present
@@ -16050,7 +16050,6 @@ async def alerting_settings(
     )
 
     settings: Final = proxy_config.settings
-    settings.apply_db_row("general_settings", db_general_settings_dict)
 
     allowed_args: Final = MappingProxyType(
         {
diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
index 4505f3144ee..ed626bdb624 100644
--- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
+++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
@@ -1755,18 +1755,21 @@ async def get_ui_settings():
     ui_settings: Final = {k: v for k, v in parsed.items() if k in ALLOWED_UI_SETTINGS_FIELDS}
 
     apply_runtime_general_settings_flags(ui_settings)
-    proxy_config.settings.apply_db_row("ui_settings", ui_settings)
 
     # Refresh DualCache so other code paths (e.g. /user/filter/ui) see fresh values
     from litellm.proxy.proxy_server import user_api_key_cache
 
     await user_api_key_cache.async_set_cache(key=UI_SETTINGS_CACHE_KEY, value=ui_settings, ttl=UI_SETTINGS_CACHE_TTL)
 
-    effective_ui_settings: Final = {
-        **{key: proxy_config.settings[key] for key in ALLOWED_UI_SETTINGS_FIELDS if key in proxy_config.settings},
-        **ui_settings,
-    }
-    config: Final[dict[str, object]] = {"litellm_settings": {"ui_settings": effective_ui_settings}}
+    effective_ui_settings: Final[Mapping[str, object]] = MappingProxyType(
+        {
+            **{key: proxy_config.settings[key] for key in ALLOWED_UI_SETTINGS_FIELDS if key in proxy_config.settings},
+            **ui_settings,
+        }
+    )
+    config: Final[Mapping[str, object]] = MappingProxyType(
+        {"litellm_settings": MappingProxyType({"ui_settings": effective_ui_settings})}
+    )
     settings_class: Final = _get_effective_ui_settings_class()
     resolved_settings: Final = _SettingsWithSchema.model_validate(
         await _get_settings_with_schema(
@@ -1775,16 +1778,22 @@ async def get_ui_settings():
             config=config,
         )
     )
-    values: Final = {
-        **resolved_settings.values,
-        ENABLE_PTU_COST_ATTRIBUTION_UI_SETTING: is_ptu_cost_attribution_enabled(),
-    }
-    source: Final[dict[str, FieldSource]] = {
-        key: (
-            "db" if key in ui_settings else _ui_setting_source(key, values[key], proxy_config.settings, settings_class)
-        )
-        for key in values
-    }
+    values: Final[Mapping[str, object]] = MappingProxyType(
+        {
+            **resolved_settings.values,
+            ENABLE_PTU_COST_ATTRIBUTION_UI_SETTING: is_ptu_cost_attribution_enabled(),
+        }
+    )
+    source: Final[Mapping[str, FieldSource]] = MappingProxyType(
+        {
+            key: (
+                "db"
+                if key in ui_settings
+                else _ui_setting_source(key, values[key], proxy_config.settings, settings_class)
+            )
+            for key in values
+        }
+    )
     return UISettingsResponse(
         values=values,
         field_schema=resolved_settings.field_schema,
diff --git a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py
index 3af7de62abe..51c8679e89e 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py
@@ -15,12 +15,24 @@ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
 from litellm.proxy.management_endpoints.router_settings_endpoints import (
     get_router_settings,
 )
+from litellm.proxy.config_resolvers import SettingsStore
 from litellm.proxy.proxy_server import app
 from litellm.router import Router
 
 client = TestClient(app)
 
 
+def _stub_proxy_config(router_settings, config_router_settings):
+    class _StubProxyConfig:
+        def __init__(self):
+            self.router_settings = router_settings
+
+        async def get_config(self, config_file_path=None):
+            return {"router_settings": dict(config_router_settings)}
+
+    return _StubProxyConfig()
+
+
 class TestRouterSettingsEndpoints:
     """Test suite for router settings endpoints"""
 
@@ -77,25 +89,18 @@ class TestRouterSettingsEndpoints:
 
     @pytest.mark.asyncio
     async def test_get_router_settings_reports_sources(self, monkeypatch):
-        from litellm.proxy.config_resolvers import SettingsStore
-
         store = SettingsStore("router_settings")
         store.load_yaml({"routing_strategy": "simple-shuffle"})
         store.apply_db_row("router_settings", {"num_retries": 3})
-        monkeypatch.setattr(proxy_server.proxy_config, "router_settings", store)
-        monkeypatch.setattr(proxy_server, "llm_router", None)
-
-        async def fake_get_config(self, config_file_path=None):
-            return {
-                "router_settings": {
-                    "routing_strategy": "simple-shuffle",
-                    "num_retries": 3,
-                }
-            }
-
         monkeypatch.setattr(
-            proxy_server.ProxyConfig, "get_config", fake_get_config, raising=True
+            proxy_server,
+            "proxy_config",
+            _stub_proxy_config(
+                store,
+                {"routing_strategy": "simple-shuffle", "num_retries": 3},
+            ),
         )
+        monkeypatch.setattr(proxy_server, "llm_router", None)
 
         admin_user = UserAPIKeyAuth(
             user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-x"
@@ -132,12 +137,10 @@ class TestRouterSettingsEndpoints:
         )
 
         monkeypatch.setattr(proxy_server, "llm_router", llm_router)
-
-        async def fake_get_config(self, config_file_path=None):
-            return {}
-
         monkeypatch.setattr(
-            proxy_server.ProxyConfig, "get_config", fake_get_config, raising=True
+            proxy_server,
+            "proxy_config",
+            _stub_proxy_config(SettingsStore("router_settings"), {}),
         )
 
         admin_user = UserAPIKeyAuth(
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 6211eeaf962..ac89d676921 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -37069,6 +37069,13 @@ export interface components {
             routing_strategy_descriptions: {
                 [key: string]: string;
             };
+            /**
+             * Source
+             * @description Source of each current router setting
+             */
+            source: {
+                [key: string]: "config" | "db" | "env" | "default" | "unset";
+            };
         };
         /**
          * RoutingGroup
@@ -39532,6 +39539,10 @@ export interface components {
             field_schema: {
                 [key: string]: unknown;
             };
+            /** Source */
+            source: {
+                [key: string]: "config" | "db" | "env" | "default" | "unset";
+            };
             /** Values */
             values: {
                 [key: string]: unknown;

From 0a88658227f8e0d7e2d4928df7c5ee63bd83fd1d Mon Sep 17 00:00:00 2001
From: yucheng <yucheng@berri.ai>
Date: Mon, 21 Sep 2026 20:25:56 +0000
Subject: [PATCH 066/160] chore: retrigger ci after docs merge

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

From 3ba4a60d5ed5eeeb217b63ac7898746927f0db12 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:26:28 +0000
Subject: [PATCH 067/160] feat(rust): add HashiCorp Vault secret manager crate

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .github/workflows/test-rust.yml               |   2 +-
 litellm-rust/Cargo.lock                       |  19 +
 litellm-rust/Cargo.toml                       |   1 +
 .../crates/secrets-hashicorp/Cargo.toml       |  23 +
 .../crates/secrets-hashicorp/src/config.rs    | 169 +++++++
 .../crates/secrets-hashicorp/src/error.rs     |  32 ++
 .../crates/secrets-hashicorp/src/lib.rs       |   9 +
 .../secrets-hashicorp/src/secret_manager.rs   | 333 ++++++++++++++
 .../secrets-hashicorp/tests/secret_manager.rs | 424 ++++++++++++++++++
 litellm-rust/crates/secrets/Cargo.toml        |   2 +
 litellm-rust/crates/secrets/README.md         |   2 +
 litellm-rust/crates/secrets/src/error.rs      |   3 +
 litellm-rust/crates/secrets/src/handler.rs    |  10 +
 litellm-rust/crates/secrets/src/lib.rs        |   2 +
 litellm-rust/crates/secrets/tests/handler.rs  | 128 ++++++
 .../hashicorp_secret_manager.py               |  16 +-
 .../hashicorp_vault_parity.json               |  97 ++++
 .../test_hashicorp_secret_manager.py          |  50 +++
 18 files changed, 1314 insertions(+), 8 deletions(-)
 create mode 100644 litellm-rust/crates/secrets-hashicorp/Cargo.toml
 create mode 100644 litellm-rust/crates/secrets-hashicorp/src/config.rs
 create mode 100644 litellm-rust/crates/secrets-hashicorp/src/error.rs
 create mode 100644 litellm-rust/crates/secrets-hashicorp/src/lib.rs
 create mode 100644 litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs
 create mode 100644 litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs
 create mode 100644 tests/test_litellm/secret_managers/hashicorp_vault_parity.json

diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml
index 278fa7c425f..56bb9a568fc 100644
--- a/.github/workflows/test-rust.yml
+++ b/.github/workflows/test-rust.yml
@@ -130,7 +130,7 @@ jobs:
       - name: Test secret manager feature combinations
         run: |
           cargo test -p litellm-auth-gcp --locked --no-default-features
-          for features in '' aws google aws,google; do
+          for features in '' aws google hashicorp aws,google,hashicorp; do
             cargo test -p litellm-secrets --locked --no-default-features --features "$features"
           done
 
diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index ed4ae4e3353..6f05bba9417 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -2698,6 +2698,7 @@ dependencies = [
  "litellm-core-utils",
  "litellm-secrets-aws",
  "litellm-secrets-google",
+ "litellm-secrets-hashicorp",
  "litellm-secrets-types",
  "moka",
  "reqwest 0.12.28",
@@ -2755,6 +2756,24 @@ dependencies = [
  "wiremock",
 ]
 
+[[package]]
+name = "litellm-secrets-hashicorp"
+version = "0.1.0"
+dependencies = [
+ "litellm-core-utils",
+ "litellm-secrets-types",
+ "moka",
+ "reqwest 0.12.28",
+ "rstest",
+ "serde",
+ "serde_json",
+ "tempfile",
+ "thiserror 2.0.19",
+ "tokio",
+ "veil",
+ "wiremock",
+]
+
 [[package]]
 name = "litellm-secrets-types"
 version = "0.1.0"
diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
index 570d0dd3568..5030a94b140 100644
--- a/litellm-rust/Cargo.toml
+++ b/litellm-rust/Cargo.toml
@@ -22,6 +22,7 @@ litellm-secrets = { path = "crates/secrets" }
 litellm-secrets-types = { path = "crates/secrets-types" }
 litellm-secrets-aws = { path = "crates/secrets-aws" }
 litellm-secrets-google = { path = "crates/secrets-google" }
+litellm-secrets-hashicorp = { path = "crates/secrets-hashicorp" }
 litellm-http = { path = "crates/http" }
 litellm-llms = { path = "crates/llms" }
 litellm-types = { path = "crates/types" }
diff --git a/litellm-rust/crates/secrets-hashicorp/Cargo.toml b/litellm-rust/crates/secrets-hashicorp/Cargo.toml
new file mode 100644
index 00000000000..0656980a033
--- /dev/null
+++ b/litellm-rust/crates/secrets-hashicorp/Cargo.toml
@@ -0,0 +1,23 @@
+[package]
+name = "litellm-secrets-hashicorp"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-core-utils.workspace = true
+litellm-secrets-types.workspace = true
+moka.workspace = true
+reqwest.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+tokio.workspace = true
+veil.workspace = true
+
+[dev-dependencies]
+rstest.workspace = true
+tempfile = "3"
+tokio.workspace = true
+wiremock = "0.6.5"
diff --git a/litellm-rust/crates/secrets-hashicorp/src/config.rs b/litellm-rust/crates/secrets-hashicorp/src/config.rs
new file mode 100644
index 00000000000..f9491f71afb
--- /dev/null
+++ b/litellm-rust/crates/secrets-hashicorp/src/config.rs
@@ -0,0 +1,169 @@
+use std::{path::PathBuf, time::Duration};
+
+use litellm_core_utils::settings::Lookup;
+use litellm_secrets_types::KeyManagementSettings;
+use litellm_secrets_types::SecretValue;
+
+use crate::Error;
+
+const DEFAULT_ADDRESS: &str = "http://127.0.0.1:8200";
+const DEFAULT_MOUNT: &str = "secret";
+const DEFAULT_APPROLE_MOUNT_PATH: &str = "approle";
+const DEFAULT_REFRESH_INTERVAL: Duration = Duration::from_secs(86400);
+const HCP_VAULT_ADDR: &str = "HCP_VAULT_ADDR";
+const HCP_VAULT_TOKEN: &str = "HCP_VAULT_TOKEN";
+const HCP_VAULT_NAMESPACE: &str = "HCP_VAULT_NAMESPACE";
+const HCP_VAULT_LOGIN_NAMESPACE: &str = "HCP_VAULT_LOGIN_NAMESPACE";
+const HCP_VAULT_SECRET_NAMESPACE: &str = "HCP_VAULT_SECRET_NAMESPACE";
+const HCP_VAULT_MOUNT_NAME: &str = "HCP_VAULT_MOUNT_NAME";
+const HCP_VAULT_PATH_PREFIX: &str = "HCP_VAULT_PATH_PREFIX";
+const HCP_VAULT_APPROLE_ROLE_ID: &str = "HCP_VAULT_APPROLE_ROLE_ID";
+const HCP_VAULT_APPROLE_SECRET_ID: &str = "HCP_VAULT_APPROLE_SECRET_ID";
+const HCP_VAULT_APPROLE_MOUNT_PATH: &str = "HCP_VAULT_APPROLE_MOUNT_PATH";
+const HCP_VAULT_CLIENT_CERT: &str = "HCP_VAULT_CLIENT_CERT";
+const HCP_VAULT_CLIENT_KEY: &str = "HCP_VAULT_CLIENT_KEY";
+const HCP_VAULT_CERT_ROLE: &str = "HCP_VAULT_CERT_ROLE";
+const HCP_VAULT_REFRESH_INTERVAL: &str = "HCP_VAULT_REFRESH_INTERVAL";
+const SECRET_MANAGER_REFRESH_INTERVAL: &str = "SECRET_MANAGER_REFRESH_INTERVAL";
+
+#[derive(Clone, Debug)]
+pub struct AppRoleAuth {
+    pub role_id: String,
+    pub secret_id: SecretValue,
+    pub mount_path: String,
+}
+
+#[derive(Clone, Debug)]
+pub struct TlsCertAuth {
+    pub cert_path: PathBuf,
+    pub key_path: PathBuf,
+    pub role: Option<String>,
+}
+
+#[derive(Clone, Debug)]
+pub struct HashicorpVaultConfig {
+    pub address: String,
+    pub token: Option<SecretValue>,
+    pub namespace: Option<String>,
+    pub login_namespace: Option<String>,
+    pub secret_namespace: Option<String>,
+    pub mount: String,
+    pub path_prefix: Option<String>,
+    pub approle: Option<AppRoleAuth>,
+    pub tls_cert: Option<TlsCertAuth>,
+    pub refresh_interval: Duration,
+}
+
+impl HashicorpVaultConfig {
+    pub fn from_environment(environment: &dyn Lookup) -> Result<Self, Error> {
+        let address: String = environment
+            .get(HCP_VAULT_ADDR)
+            .and_then(|value| nonempty(value.trim()))
+            .map(|value| value.trim_end_matches('/').to_owned())
+            .filter(|value| !value.is_empty())
+            .unwrap_or_else(|| DEFAULT_ADDRESS.to_owned());
+        let token: Option<SecretValue> = environment
+            .get(HCP_VAULT_TOKEN)
+            .and_then(nonempty)
+            .map(SecretValue::new);
+        let namespace: Option<String> = path_component(environment.get(HCP_VAULT_NAMESPACE));
+        let login_namespace: Option<String> =
+            path_component(environment.get(HCP_VAULT_LOGIN_NAMESPACE));
+        let secret_namespace: Option<String> =
+            path_component(environment.get(HCP_VAULT_SECRET_NAMESPACE));
+        let mount: String = path_component(environment.get(HCP_VAULT_MOUNT_NAME))
+            .unwrap_or_else(|| DEFAULT_MOUNT.to_owned());
+        let path_prefix: Option<String> = path_component(environment.get(HCP_VAULT_PATH_PREFIX));
+        let approle: Option<AppRoleAuth> = match (
+            environment
+                .get(HCP_VAULT_APPROLE_ROLE_ID)
+                .and_then(nonempty),
+            environment
+                .get(HCP_VAULT_APPROLE_SECRET_ID)
+                .and_then(nonempty)
+                .map(SecretValue::new),
+        ) {
+            (Some(role_id), Some(secret_id)) => Some(AppRoleAuth {
+                role_id,
+                secret_id,
+                mount_path: path_component(environment.get(HCP_VAULT_APPROLE_MOUNT_PATH))
+                    .unwrap_or_else(|| DEFAULT_APPROLE_MOUNT_PATH.to_owned()),
+            }),
+            _ => None,
+        };
+        let tls_cert: Option<TlsCertAuth> = match (
+            environment.get(HCP_VAULT_CLIENT_CERT).and_then(nonempty),
+            environment.get(HCP_VAULT_CLIENT_KEY).and_then(nonempty),
+        ) {
+            (Some(cert_path), Some(key_path)) => Some(TlsCertAuth {
+                cert_path: PathBuf::from(cert_path),
+                key_path: PathBuf::from(key_path),
+                role: environment.get(HCP_VAULT_CERT_ROLE).and_then(nonempty),
+            }),
+            _ => None,
+        };
+        let refresh_interval: Duration = refresh_interval(environment)?;
+        Ok(Self {
+            address,
+            token,
+            namespace,
+            login_namespace,
+            secret_namespace,
+            mount,
+            path_prefix,
+            approle,
+            tls_cert,
+            refresh_interval,
+        })
+    }
+
+    pub fn from_settings(
+        _settings: &KeyManagementSettings,
+        environment: &dyn Lookup,
+    ) -> Result<Self, Error> {
+        Self::from_environment(environment)
+    }
+
+    pub fn login_namespace(&self) -> Option<&str> {
+        self.login_namespace
+            .as_deref()
+            .or(self.namespace.as_deref())
+    }
+
+    pub fn secret_namespace(&self) -> Option<&str> {
+        self.secret_namespace
+            .as_deref()
+            .or(self.namespace.as_deref())
+    }
+}
+
+fn nonempty(value: impl AsRef<str>) -> Option<String> {
+    let value: &str = value.as_ref();
+    (!value.is_empty()).then(|| value.to_owned())
+}
+
+fn path_component(value: Option<String>) -> Option<String> {
+    value
+        .and_then(|value| nonempty(value.trim()))
+        .map(|value| value.trim_matches('/').to_owned())
+        .filter(|value| !value.is_empty())
+}
+
+fn refresh_interval(environment: &dyn Lookup) -> Result<Duration, Error> {
+    let value: Option<String> = environment
+        .get(HCP_VAULT_REFRESH_INTERVAL)
+        .and_then(nonempty)
+        .or_else(|| {
+            environment
+                .get(SECRET_MANAGER_REFRESH_INTERVAL)
+                .and_then(nonempty)
+        });
+    let Some(value) = value else {
+        return Ok(DEFAULT_REFRESH_INTERVAL);
+    };
+    let seconds: i64 = value.parse().map_err(|_| Error::RefreshInterval)?;
+    if seconds < 0 {
+        return Ok(Duration::from_nanos(1));
+    }
+    Ok(Duration::from_secs(seconds as u64))
+}
diff --git a/litellm-rust/crates/secrets-hashicorp/src/error.rs b/litellm-rust/crates/secrets-hashicorp/src/error.rs
new file mode 100644
index 00000000000..3f6a5b66475
--- /dev/null
+++ b/litellm-rust/crates/secrets-hashicorp/src/error.rs
@@ -0,0 +1,32 @@
+#[derive(thiserror::Error, veil::Redact)]
+pub enum Error {
+    #[error("HashiCorp Vault requires an enterprise license")]
+    EnterpriseRequired,
+    #[error("invalid secret name")]
+    InvalidSecretName(#[from] litellm_secrets_types::Error),
+    #[error("HashiCorp Vault request failed")]
+    Request(
+        #[from]
+        #[redact]
+        reqwest::Error,
+    ),
+    #[error("HashiCorp Vault TLS identity could not be configured for {path}: {message}")]
+    TlsIdentity {
+        path: std::path::PathBuf,
+        message: String,
+    },
+    #[error("HashiCorp Vault login returned HTTP {status}")]
+    LoginStatus { status: u16 },
+    #[error("HashiCorp Vault login response is malformed")]
+    MalformedLogin,
+    #[error("HashiCorp Vault authentication is not configured")]
+    NoAuthConfigured,
+    #[error("HashiCorp Vault returned HTTP {status}")]
+    Status { status: u16 },
+    #[error("HashiCorp Vault response payload is malformed")]
+    MalformedPayload,
+    #[error("HashiCorp Vault secret value is not a string")]
+    NonStringValue,
+    #[error("invalid HashiCorp Vault refresh interval")]
+    RefreshInterval,
+}
diff --git a/litellm-rust/crates/secrets-hashicorp/src/lib.rs b/litellm-rust/crates/secrets-hashicorp/src/lib.rs
new file mode 100644
index 00000000000..a561a58b59e
--- /dev/null
+++ b/litellm-rust/crates/secrets-hashicorp/src/lib.rs
@@ -0,0 +1,9 @@
+#![forbid(unsafe_code)]
+
+mod config;
+mod error;
+pub mod secret_manager;
+
+pub use config::{AppRoleAuth, HashicorpVaultConfig, TlsCertAuth};
+pub use error::Error;
+pub use secret_manager::HashicorpVault;
diff --git a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs
new file mode 100644
index 00000000000..6b887289443
--- /dev/null
+++ b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs
@@ -0,0 +1,333 @@
+use std::{
+    fmt,
+    sync::Arc,
+    time::{Duration, Instant},
+};
+
+use litellm_core_utils::settings::Lookup;
+use litellm_secrets_types::{
+    BaseSecretManager, SecretValue, async_rotate_secret, validate_secret_name,
+};
+use moka::future::Cache;
+use reqwest::Client;
+use serde_json::{Value, json};
+use tokio::sync::Mutex;
+
+use crate::{Error, HashicorpVaultConfig};
+
+const CACHE_CAPACITY: u64 = 200;
+
+#[derive(Clone)]
+struct CachedToken {
+    token: SecretValue,
+    expires_at: Option<Instant>,
+}
+
+#[derive(Clone)]
+pub struct HashicorpVault {
+    client: Arc<Client>,
+    config: HashicorpVaultConfig,
+    cache: Cache<String, SecretValue>,
+    auth_token: Arc<Mutex<Option<CachedToken>>>,
+}
+
+impl fmt::Debug for HashicorpVault {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter
+            .debug_struct("HashicorpVault")
+            .field("config", &self.config)
+            .finish_non_exhaustive()
+    }
+}
+
+impl HashicorpVault {
+    pub fn new(
+        environment: Arc<dyn Lookup + Send + Sync>,
+        enterprise_enabled: bool,
+    ) -> Result<Self, Error> {
+        if !enterprise_enabled {
+            return Err(Error::EnterpriseRequired);
+        }
+        let config: HashicorpVaultConfig =
+            HashicorpVaultConfig::from_environment(environment.as_ref())?;
+        let client: Client = client_for_config(&config)?;
+        Self::with_client(client, config, enterprise_enabled)
+    }
+
+    pub fn with_client(
+        client: Client,
+        config: HashicorpVaultConfig,
+        enterprise_enabled: bool,
+    ) -> Result<Self, Error> {
+        if !enterprise_enabled {
+            return Err(Error::EnterpriseRequired);
+        }
+        let cache: Cache<String, SecretValue> = Cache::builder()
+            .max_capacity(CACHE_CAPACITY)
+            .time_to_live(config.refresh_interval)
+            .build();
+        Ok(Self {
+            client: Arc::new(client),
+            config,
+            cache,
+            auth_token: Arc::new(Mutex::new(None)),
+        })
+    }
+
+    pub fn secret_url(&self, secret_name: &str) -> Result<String, Error> {
+        validate_secret_name(secret_name).map_err(Error::InvalidSecretName)?;
+        let namespace: String = self
+            .config
+            .secret_namespace()
+            .map(|value| format!("{value}/"))
+            .unwrap_or_default();
+        let path_prefix: String = self
+            .config
+            .path_prefix
+            .as_deref()
+            .map(|value| format!("{value}/"))
+            .unwrap_or_default();
+        Ok(format!(
+            "{}/v1/{}{}/data/{}{}",
+            self.config.address, namespace, self.config.mount, path_prefix, secret_name
+        ))
+    }
+
+    pub fn login_url(&self) -> Option<String> {
+        self.config.approle.as_ref().map_or_else(
+            || {
+                self.config
+                    .tls_cert
+                    .as_ref()
+                    .map(|_| format!("{}/v1/auth/cert/login", self.config.address))
+            },
+            |approle| {
+                Some(format!(
+                    "{}/v1/auth/{}/login",
+                    self.config.address, approle.mount_path
+                ))
+            },
+        )
+    }
+
+    pub fn config(&self) -> &HashicorpVaultConfig {
+        &self.config
+    }
+
+    pub async fn async_read_secret(&self, secret_name: &str) -> Result<Option<SecretValue>, Error> {
+        let url: String = self.secret_url(secret_name)?;
+        if let Some(value) = self.cache.get(&url).await {
+            return Ok(Some(value));
+        }
+        let token: SecretValue = self.vault_token().await?;
+        let mut request = self
+            .client
+            .get(&url)
+            .header("X-Vault-Token", token.expose());
+        if let Some(namespace) = self.config.secret_namespace() {
+            request = request.header("X-Vault-Namespace", namespace);
+        }
+        let response: reqwest::Response = request.send().await?;
+        if response.status() == reqwest::StatusCode::NOT_FOUND {
+            return Ok(None);
+        }
+        if !response.status().is_success() {
+            return Err(Error::Status {
+                status: response.status().as_u16(),
+            });
+        }
+        let body = response.bytes().await?;
+        let body: Value = serde_json::from_slice(&body).map_err(|_| Error::MalformedPayload)?;
+        let data: &Value = body
+            .get("data")
+            .and_then(Value::as_object)
+            .and_then(|value| value.get("data"))
+            .ok_or(Error::MalformedPayload)?;
+        let data: &serde_json::Map<String, Value> =
+            data.as_object().ok_or(Error::MalformedPayload)?;
+        let Some(value) = data.get("key") else {
+            return Ok(None);
+        };
+        let value: &str = value.as_str().ok_or(Error::NonStringValue)?;
+        let value: SecretValue = SecretValue::new(value);
+        self.cache.insert(url, value.clone()).await;
+        Ok(Some(value))
+    }
+
+    pub async fn async_write_secret(
+        &self,
+        secret_name: &str,
+        value: SecretValue,
+        description: Option<&str>,
+    ) -> Result<Value, Error> {
+        let url: String = self.secret_url(secret_name)?;
+        let data: Value = match description {
+            Some(description) => json!({"key": value.expose(), "description": description}),
+            None => json!({"key": value.expose()}),
+        };
+        let token: SecretValue = self.vault_token().await?;
+        let mut request = self
+            .client
+            .post(&url)
+            .header("X-Vault-Token", token.expose());
+        if let Some(namespace) = self.config.secret_namespace() {
+            request = request.header("X-Vault-Namespace", namespace);
+        }
+        let response: reqwest::Response = request.json(&json!({"data": data})).send().await?;
+        if !response.status().is_success() {
+            return Err(Error::Status {
+                status: response.status().as_u16(),
+            });
+        }
+        self.cache.invalidate(&url).await;
+        let body = response.bytes().await?;
+        if body.is_empty() {
+            return Ok(Value::Null);
+        }
+        serde_json::from_slice(&body).map_err(|_| Error::MalformedPayload)
+    }
+
+    pub async fn async_delete_secret(&self, secret_name: &str) -> Result<(), Error> {
+        let url: String = self.secret_url(secret_name)?;
+        let token: SecretValue = self.vault_token().await?;
+        let mut request = self
+            .client
+            .delete(&url)
+            .header("X-Vault-Token", token.expose());
+        if let Some(namespace) = self.config.secret_namespace() {
+            request = request.header("X-Vault-Namespace", namespace);
+        }
+        let response: reqwest::Response = request.send().await?;
+        if !response.status().is_success() {
+            return Err(Error::Status {
+                status: response.status().as_u16(),
+            });
+        }
+        self.cache.invalidate(&url).await;
+        Ok(())
+    }
+
+    pub async fn async_rotate_secret(
+        &self,
+        current_name: &str,
+        new_name: &str,
+        value: &SecretValue,
+    ) -> Result<Value, Error> {
+        async_rotate_secret(self, current_name, new_name, value).await
+    }
+
+    async fn vault_token(&self) -> Result<SecretValue, Error> {
+        let mut cached: tokio::sync::MutexGuard<'_, Option<CachedToken>> =
+            self.auth_token.lock().await;
+        if let Some(entry) = cached.as_ref()
+            && entry
+                .expires_at
+                .is_none_or(|expires_at| expires_at > Instant::now())
+        {
+            return Ok(entry.token.clone());
+        }
+        let Some(login_url) = self.login_url() else {
+            let Some(token) = self.config.token.clone() else {
+                return Err(Error::NoAuthConfigured);
+            };
+            return Ok(token);
+        };
+        let body: Value = match (self.config.approle.as_ref(), self.config.tls_cert.as_ref()) {
+            (Some(approle), _) => {
+                json!({"role_id": approle.role_id, "secret_id": approle.secret_id.expose()})
+            }
+            (None, Some(tls)) => tls
+                .role
+                .as_deref()
+                .map_or_else(|| json!({}), |role| json!({"name": role})),
+            (None, None) => {
+                let Some(token) = self.config.token.clone() else {
+                    return Err(Error::NoAuthConfigured);
+                };
+                return Ok(token);
+            }
+        };
+        let mut request = self.client.post(login_url).json(&body);
+        if let Some(namespace) = self.config.login_namespace() {
+            request = request.header("X-Vault-Namespace", namespace);
+        }
+        let response: reqwest::Response = request.send().await?;
+        if !response.status().is_success() {
+            return Err(Error::LoginStatus {
+                status: response.status().as_u16(),
+            });
+        }
+        let body = response.bytes().await?;
+        let payload: Value = serde_json::from_slice(&body).map_err(|_| Error::MalformedLogin)?;
+        let auth: &serde_json::Map<String, Value> = payload
+            .get("auth")
+            .and_then(Value::as_object)
+            .ok_or(Error::MalformedLogin)?;
+        let token: SecretValue = SecretValue::new(
+            auth.get("client_token")
+                .and_then(Value::as_str)
+                .ok_or(Error::MalformedLogin)?,
+        );
+        let lease_duration: u64 = auth
+            .get("lease_duration")
+            .and_then(Value::as_u64)
+            .ok_or(Error::MalformedLogin)?;
+        let expires_at: Option<Instant> =
+            (lease_duration > 0).then(|| Instant::now() + Duration::from_secs(lease_duration));
+        *cached = Some(CachedToken {
+            token: token.clone(),
+            expires_at,
+        });
+        Ok(token)
+    }
+}
+
+impl BaseSecretManager for HashicorpVault {
+    type Error = Error;
+    type WriteResponse = Value;
+    type DeleteResponse = ();
+
+    async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
+        HashicorpVault::async_read_secret(self, name).await
+    }
+
+    async fn async_write_secret(
+        &self,
+        name: &str,
+        value: &SecretValue,
+        description: Option<&str>,
+    ) -> Result<Value, Error> {
+        HashicorpVault::async_write_secret(self, name, value.clone(), description).await
+    }
+
+    async fn async_delete_secret(
+        &self,
+        name: &str,
+        _recovery_window_in_days: i64,
+    ) -> Result<(), Error> {
+        HashicorpVault::async_delete_secret(self, name).await
+    }
+}
+
+fn client_for_config(config: &HashicorpVaultConfig) -> Result<Client, Error> {
+    let mut builder: reqwest::ClientBuilder = Client::builder();
+    if let Some(tls) = config.tls_cert.as_ref() {
+        let cert: Vec<u8> = std::fs::read(&tls.cert_path).map_err(|source| Error::TlsIdentity {
+            path: tls.cert_path.clone(),
+            message: source.to_string(),
+        })?;
+        let key: Vec<u8> = std::fs::read(&tls.key_path).map_err(|source| Error::TlsIdentity {
+            path: tls.key_path.clone(),
+            message: source.to_string(),
+        })?;
+        let identity: reqwest::Identity = reqwest::Identity::from_pem(
+            &[cert.as_slice(), key.as_slice()].concat(),
+        )
+        .map_err(|source| Error::TlsIdentity {
+            path: tls.cert_path.clone(),
+            message: source.to_string(),
+        })?;
+        builder = builder.identity(identity);
+    }
+    builder.build().map_err(Error::Request)
+}
diff --git a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs
new file mode 100644
index 00000000000..1cf13a79c77
--- /dev/null
+++ b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs
@@ -0,0 +1,424 @@
+use std::{collections::HashMap, sync::Arc, time::Duration};
+
+use litellm_core_utils::settings::Lookup;
+use litellm_secrets_hashicorp::{Error, HashicorpVault, HashicorpVaultConfig};
+use litellm_secrets_types::SecretValue;
+use serde::Deserialize;
+use serde_json::json;
+use wiremock::{
+    Mock, MockServer, ResponseTemplate,
+    matchers::{body_json, header, method, path},
+};
+
+fn config(server: &MockServer, values: &[(&str, &str)]) -> HashicorpVaultConfig {
+    let mut environment_values: HashMap<String, String> = values
+        .iter()
+        .map(|(name, value)| ((*name).to_owned(), (*value).to_owned()))
+        .collect();
+    environment_values.insert("HCP_VAULT_ADDR".to_owned(), server.uri());
+    let environment: Arc<dyn Lookup + Send + Sync> =
+        Arc::new(move |name: &str| environment_values.get(name).cloned());
+    HashicorpVaultConfig::from_environment(environment.as_ref()).unwrap()
+}
+
+fn manager(server: &MockServer, values: &[(&str, &str)]) -> HashicorpVault {
+    HashicorpVault::with_client(reqwest::Client::new(), config(server, values), true).unwrap()
+}
+
+#[tokio::test]
+async fn token_reads_use_vault_headers_and_cache_values() {
+    let server: MockServer = MockServer::start().await;
+    Mock::given(method("GET"))
+        .and(path("/v1/secret/data/name"))
+        .and(header("X-Vault-Token", "token"))
+        .respond_with(
+            ResponseTemplate::new(200).set_body_json(json!({"data": {"data": {"key": "value"}}})),
+        )
+        .expect(1)
+        .mount(&server)
+        .await;
+    let manager: HashicorpVault = manager(&server, &[("HCP_VAULT_TOKEN", "token")]);
+
+    assert_eq!(
+        manager
+            .async_read_secret("name")
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "value"
+    );
+    assert_eq!(
+        manager
+            .async_read_secret("name")
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "value"
+    );
+}
+
+#[tokio::test]
+async fn namespace_mount_and_prefix_are_sanitized_in_the_url() {
+    let server: MockServer = MockServer::start().await;
+    Mock::given(method("GET"))
+        .and(path("/v1/team-a/kv-prod/data/virtual-keys/name"))
+        .respond_with(
+            ResponseTemplate::new(200).set_body_json(json!({"data": {"data": {"key": "value"}}})),
+        )
+        .expect(1)
+        .mount(&server)
+        .await;
+    let manager: HashicorpVault = manager(
+        &server,
+        &[
+            ("HCP_VAULT_TOKEN", "token"),
+            ("HCP_VAULT_SECRET_NAMESPACE", " /team-a/ "),
+            ("HCP_VAULT_MOUNT_NAME", " /kv-prod/ "),
+            ("HCP_VAULT_PATH_PREFIX", " /virtual-keys/ "),
+        ],
+    );
+
+    assert_eq!(
+        manager.secret_url("name").unwrap(),
+        format!("{}/v1/team-a/kv-prod/data/virtual-keys/name", server.uri())
+    );
+    assert!(manager.async_read_secret("name").await.unwrap().is_some());
+}
+
+#[tokio::test]
+async fn approle_login_uses_namespace_and_reuses_the_token() {
+    let server: MockServer = MockServer::start().await;
+    Mock::given(method("POST"))
+        .and(path("/v1/auth/custom-approle/login"))
+        .and(header("X-Vault-Namespace", "login-root"))
+        .and(body_json(json!({"role_id": "role", "secret_id": "secret"})))
+        .respond_with(ResponseTemplate::new(200).set_body_json(
+            json!({"auth": {"client_token": "login-token", "lease_duration": 3600}}),
+        ))
+        .expect(1)
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/v1/secret-root/secret/data/name"))
+        .and(header("X-Vault-Token", "login-token"))
+        .respond_with(
+            ResponseTemplate::new(200).set_body_json(json!({"data": {"data": {"key": "value"}}})),
+        )
+        .expect(1)
+        .mount(&server)
+        .await;
+    let manager: HashicorpVault = manager(
+        &server,
+        &[
+            ("HCP_VAULT_APPROLE_ROLE_ID", "role"),
+            ("HCP_VAULT_APPROLE_SECRET_ID", "secret"),
+            ("HCP_VAULT_APPROLE_MOUNT_PATH", "custom-approle"),
+            ("HCP_VAULT_NAMESPACE", "secret-root"),
+            ("HCP_VAULT_LOGIN_NAMESPACE", "login-root"),
+        ],
+    );
+
+    assert!(manager.async_read_secret("name").await.unwrap().is_some());
+    assert!(manager.async_read_secret("name-2").await.unwrap().is_none());
+}
+
+#[tokio::test]
+async fn approle_tokens_expire_after_the_vault_lease() {
+    let server: MockServer = MockServer::start().await;
+    Mock::given(method("POST"))
+        .and(path("/v1/auth/approle/login"))
+        .respond_with(
+            ResponseTemplate::new(200).set_body_json(
+                json!({"auth": {"client_token": "login-token", "lease_duration": 1}}),
+            ),
+        )
+        .expect(2)
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .respond_with(
+            ResponseTemplate::new(200).set_body_json(json!({"data": {"data": {"key": "value"}}})),
+        )
+        .expect(2)
+        .mount(&server)
+        .await;
+    let manager: HashicorpVault = manager(
+        &server,
+        &[
+            ("HCP_VAULT_APPROLE_ROLE_ID", "role"),
+            ("HCP_VAULT_APPROLE_SECRET_ID", "secret"),
+            ("HCP_VAULT_REFRESH_INTERVAL", "0"),
+        ],
+    );
+
+    assert!(manager.async_read_secret("first").await.unwrap().is_some());
+    tokio::time::sleep(Duration::from_secs(1) + Duration::from_millis(50)).await;
+    assert!(manager.async_read_secret("second").await.unwrap().is_some());
+}
+
+#[tokio::test]
+async fn tls_login_posts_the_role_and_uses_the_client_identity() {
+    let server: MockServer = MockServer::start().await;
+    let directory: tempfile::TempDir = tempfile::tempdir().unwrap();
+    let cert_path = directory.path().join("client.crt");
+    let key_path = directory.path().join("client.key");
+    std::fs::write(&cert_path, TEST_CERTIFICATE).unwrap();
+    std::fs::write(&key_path, TEST_PRIVATE_KEY).unwrap();
+    Mock::given(method("POST"))
+        .and(path("/v1/auth/cert/login"))
+        .and(body_json(json!({"name": "vault-role"})))
+        .respond_with(
+            ResponseTemplate::new(200).set_body_json(
+                json!({"auth": {"client_token": "cert-token", "lease_duration": 0}}),
+            ),
+        )
+        .expect(1)
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/v1/secret/data/name"))
+        .respond_with(
+            ResponseTemplate::new(200).set_body_json(json!({"data": {"data": {"key": "value"}}})),
+        )
+        .mount(&server)
+        .await;
+    let manager: HashicorpVault = HashicorpVault::new(
+        {
+            let environment_values: HashMap<String, String> = HashMap::from([
+                ("HCP_VAULT_ADDR".to_owned(), server.uri()),
+                (
+                    "HCP_VAULT_CLIENT_CERT".to_owned(),
+                    cert_path.to_str().unwrap().to_owned(),
+                ),
+                (
+                    "HCP_VAULT_CLIENT_KEY".to_owned(),
+                    key_path.to_str().unwrap().to_owned(),
+                ),
+                ("HCP_VAULT_CERT_ROLE".to_owned(), "vault-role".to_owned()),
+            ]);
+            Arc::new(move |name: &str| environment_values.get(name).cloned())
+        },
+        true,
+    )
+    .unwrap();
+
+    assert!(manager.async_read_secret("name").await.unwrap().is_some());
+    assert_eq!(
+        manager.login_url().as_deref(),
+        Some(format!("{}/v1/auth/cert/login", server.uri()).as_str())
+    );
+}
+
+#[rstest::rstest]
+#[case::missing(404, json!({}), 0)]
+#[case::malformed(200, json!({}), 1)]
+#[case::missing_key(200, json!({"data": {"data": {}}}), 0)]
+#[case::non_string(200, json!({"data": {"data": {"key": 1}}}), 2)]
+#[tokio::test]
+async fn read_responses_distinguish_absence_and_malformed_payloads(
+    #[case] status: u16,
+    #[case] body: serde_json::Value,
+    #[case] expected: u8,
+) {
+    let server: MockServer = MockServer::start().await;
+    Mock::given(method("GET"))
+        .respond_with(ResponseTemplate::new(status).set_body_json(body))
+        .expect(1)
+        .mount(&server)
+        .await;
+    let result: Result<Option<SecretValue>, Error> =
+        manager(&server, &[("HCP_VAULT_TOKEN", "token")])
+            .async_read_secret("name")
+            .await;
+    match expected {
+        0 => assert!(result.unwrap().is_none()),
+        1 => assert!(matches!(result, Err(Error::MalformedPayload))),
+        2 => assert!(matches!(result, Err(Error::NonStringValue))),
+        _ => unreachable!(),
+    }
+}
+
+#[tokio::test]
+async fn write_and_delete_invalidate_the_read_cache() {
+    let server: MockServer = MockServer::start().await;
+    Mock::given(method("GET"))
+        .and(path("/v1/secret/data/name"))
+        .respond_with(
+            ResponseTemplate::new(200).set_body_json(json!({"data": {"data": {"key": "value"}}})),
+        )
+        .expect(2)
+        .mount(&server)
+        .await;
+    Mock::given(method("POST"))
+        .and(path("/v1/secret/data/name"))
+        .and(body_json(
+            json!({"data": {"key": "updated", "description": "description"}}),
+        ))
+        .respond_with(ResponseTemplate::new(200).set_body_json(json!({"data": {"version": 2}})))
+        .expect(1)
+        .mount(&server)
+        .await;
+    Mock::given(method("DELETE"))
+        .and(path("/v1/secret/data/name"))
+        .respond_with(ResponseTemplate::new(204))
+        .expect(1)
+        .mount(&server)
+        .await;
+    let manager: HashicorpVault = manager(&server, &[("HCP_VAULT_TOKEN", "token")]);
+
+    assert!(manager.async_read_secret("name").await.unwrap().is_some());
+    assert!(
+        manager
+            .async_write_secret("name", SecretValue::new("updated"), Some("description"))
+            .await
+            .is_ok()
+    );
+    assert!(manager.async_read_secret("name").await.unwrap().is_some());
+    manager.async_delete_secret("name").await.unwrap();
+}
+
+#[tokio::test]
+async fn no_auth_and_invalid_names_fail_without_requests() {
+    let server: MockServer = MockServer::start().await;
+    let manager: HashicorpVault = manager(&server, &[]);
+
+    assert!(matches!(
+        manager.async_read_secret("name").await,
+        Err(Error::NoAuthConfigured)
+    ));
+    assert!(matches!(
+        manager.async_read_secret("../name").await,
+        Err(Error::InvalidSecretName(_))
+    ));
+    assert!(server.received_requests().await.unwrap().is_empty());
+}
+
+#[tokio::test]
+async fn debug_output_redacts_authentication_values() {
+    let server: MockServer = MockServer::start().await;
+    let manager: HashicorpVault = HashicorpVault::with_client(
+        reqwest::Client::new(),
+        config(&server, &[("HCP_VAULT_TOKEN", "token-value")]),
+        true,
+    )
+    .unwrap();
+    let debug: String = format!("{manager:?}");
+    assert!(!debug.contains("token-value"));
+    assert!(!debug.contains("secret-id"));
+}
+
+#[derive(Deserialize)]
+struct ParityCase {
+    env: HashMap<String, String>,
+    expected_secret_url: String,
+    expected_login_url: Option<String>,
+    expected_login_namespace: Option<String>,
+    expected_secret_namespace: Option<String>,
+    secret_name: String,
+}
+
+#[test]
+fn configuration_matches_python_parity_fixture() {
+    let cases: Vec<ParityCase> = serde_json::from_str(include_str!(concat!(
+        env!("CARGO_MANIFEST_DIR"),
+        "/../../../tests/test_litellm/secret_managers/hashicorp_vault_parity.json"
+    )))
+    .unwrap();
+    for case in cases {
+        let values: HashMap<String, String> = case.env.clone();
+        let environment: Arc<dyn Lookup + Send + Sync> =
+            Arc::new(move |name: &str| values.get(name).cloned());
+        let config: HashicorpVaultConfig =
+            HashicorpVaultConfig::from_environment(environment.as_ref()).unwrap();
+        let manager: HashicorpVault =
+            HashicorpVault::with_client(reqwest::Client::new(), config, true).unwrap();
+        assert_eq!(
+            manager.secret_url(&case.secret_name).unwrap(),
+            case.expected_secret_url
+        );
+        assert_eq!(manager.login_url(), case.expected_login_url);
+        assert_eq!(
+            manager.config().login_namespace(),
+            case.expected_login_namespace.as_deref()
+        );
+        assert_eq!(
+            manager.config().secret_namespace(),
+            case.expected_secret_namespace.as_deref()
+        );
+    }
+}
+
+#[tokio::test]
+#[ignore]
+async fn live_vault_round_trip() {
+    let environment: Arc<dyn Lookup + Send + Sync> =
+        Arc::new(litellm_core_utils::settings::ProcessEnvironment);
+    let manager: HashicorpVault = HashicorpVault::new(environment, true).unwrap();
+    let name: String = std::env::var("LITELLM_VAULT_LIVE_SECRET_NAME").unwrap();
+    let value: SecretValue = SecretValue::new("native-live-value");
+    let url: String = manager.secret_url(&name).unwrap();
+    println!("native provenance: {} {}", module_path!(), url);
+    manager
+        .async_write_secret(&name, value.clone(), None)
+        .await
+        .unwrap();
+    assert_eq!(
+        manager.async_read_secret(&name).await.unwrap().unwrap(),
+        value
+    );
+    manager.async_delete_secret(&name).await.unwrap();
+    assert!(manager.async_read_secret(&name).await.unwrap().is_none());
+}
+
+const TEST_CERTIFICATE: &str = "-----BEGIN CERTIFICATE-----
+MIIDDzCCAfegAwIBAgIUeMzLFLM/mRbPGbNAew5N2UTscocwDQYJKoZIhvcNAQEL
+BQAwFzEVMBMGA1UEAwwMbGl0ZWxsbS10ZXN0MB4XDTI2MDkyMTIwMjA1OVoXDTI2
+MDkyMjIwMjA1OVowFzEVMBMGA1UEAwwMbGl0ZWxsbS10ZXN0MIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEAveYoSUJXybmkHmQsBfhBcv2Ob5Oy8ejZu+B3
+vTnrPumW4ANi1XXKBSazRGB3fEtAgr+3KhKeHaSKEQeBwJkAEBfdmQv0tpXICwHs
+1kFNtU0owy54HVW5/ia+LMszsFcPzVIoMnbUOuiKr9RaV7P+IEFzILPBVuV4DoYH
+yocjD3+9QNqokWgNL8LK37JijmNEFVaKFz0X6SyL2VRDlfPWTEBK52Gp/pvDgA6G
+eTSfyI+kCm9h5ECTYUAtmatk9WPVS8sWOqV1EXVanFyYBU+mDxoywAS1/6CHeIPh
+bNmCOZjPoO9qWBJ7ZyGhOconBigXY8qnlXymev+44IPHrx4urwIDAQABo1MwUTAd
+BgNVHQ4EFgQUvaZrZ6HKtbr3ekeZmgy4b5Pq95QwHwYDVR0jBBgwFoAUvaZrZ6HK
+tbr3ekeZmgy4b5Pq95QwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC
+AQEAEejrD8d1qDxW55XxQ4IC31rufoEvDV955jyvh2kALPaN/i5oWsBGI+UAQZna
+aaoQXwzlmHrtDUBWl0LztVTUamIleUep2+PLLauqqt43vxppxMX8Jn2mnPO20YE/
+hIzGx0jN/LBG8PDyLSvHdlgjP9ofA4Vg4rTQugdXRgOvlCE/epnH/MADcg9KYJtJ
+C1RObCIkL3LcdUbjStJRCY/U/FeWcgyncEPz95OFDkbrlNDajb6o6CkYfouqvhTc
+8XlgjjAVKIbAbRgbVu3elsquuFM97x2DzWDjkrMNmDt1FJ9ubK36gL6B3o0UMaoQ
+00R7x/eqvH+EkWa/2ekW9lpleQ==
+-----END CERTIFICATE-----
+";
+
+const TEST_PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC95ihJQlfJuaQe
+ZCwF+EFy/Y5vk7Lx6Nm74He9Oes+6ZbgA2LVdcoFJrNEYHd8S0CCv7cqEp4dpIoR
+B4HAmQAQF92ZC/S2lcgLAezWQU21TSjDLngdVbn+Jr4syzOwVw/NUigydtQ66Iqv
+1FpXs/4gQXMgs8FW5XgOhgfKhyMPf71A2qiRaA0vwsrfsmKOY0QVVooXPRfpLIvZ
+VEOV89ZMQErnYan+m8OADoZ5NJ/Ij6QKb2HkQJNhQC2Zq2T1Y9VLyxY6pXURdVqc
+XJgFT6YPGjLABLX/oId4g+Fs2YI5mM+g72pYEntnIaE5yicGKBdjyqeVfKZ6/7jg
+g8evHi6vAgMBAAECggEAGdJjlP6b8Fa5bdaCM/ebcrbuuNZVJVbb0JPHxGfNSLs7
+pE9hj5QaOdQW2Uviw3h6F61ZCzQH4xD+Iy2po5ZKb2XHYKnDB1bboj+LRGER337T
+9aJqe9at2VTMVEv3Rdm40NsEk0QcPLxlK16NQFK90gYEUSSQPDAswJDSG2R/zHn+
+vADI907mW/goEJHeLn8PWGlNlSiR6x+5JJtq+GXCzUzVvJYQSCLGxCSl2x2H+0g7
+NhFI0zPpdzNmO/h+yhzaFb6Rp5U8+ZsnZ3qYjQ/03gw1myTDKJt1YaO9JvArnNYX
+hcJQQ8Rt0bHhcrZA16bBOpqZlo5pKCicwI/netgN8QKBgQDcFz7AzdJ26sMSV32V
+rwrMgIoggt8qDjO1ARwqW35A1TIge0FoW4M4KpsXQGGfT341uU1esXEcyZ/1L/5X
+3ql2gX4DbOYLZLWYzZGR2hq33oi8HkhN98QrEwL9emSH8NqYX3Xxja3PrmCrSYJe
+Zbnd9TIm2XkxyMoyXJu6M/QvnwKBgQDc4dzqTbxoGEGa5MuJoGmMwPnqgdG9UM5J
+eExVnh7osxc2sOdsiPeRjjQTxs9v2kJwctC359OJoo9yGaaJeSghU4LEWJo1sqnA
+fzSCLammYvtVAtniyNv5Mxk/6Uimi4NNDKaAKB+m4K2uSn3U9AmY7KPYMGaSbS9W
+XSnobjxm8QKBgC8bPpAvvWs8ZhIn7bY659nLbUT2HeO3dHO6UBf0yzn/J6JyHxbB
+93zvCZDZc8uQTRgcmCW7XtVlhjoJUqvl+Wlm39zF0xr/LCsPXKfWAb/2/lcdOCaP
+8Emz4QD10EyUTYUtcWYJB/mafhBLRH8F0Nlj4J8WDu2L51MOJTqeYhZLAoGAWffN
+icocAbJPlo22sdoa4+/+W5yBF8GAJMDRJtZ+9H1t6SLpQHYRkMIBSETkXUTjZvX9
+Ocs9iIQkNW9pO/mTdO+VBfCo71JUfknR02xR+6m5gYjlws/ZeYlssXGN2/hbhNiw
+QOcW7Vv6olFJK6Iy/oz0t6wPO3kpnN3Zogi0paECgYEAwo44M1DdYCtV0snhmYM9
+5u0mPfYt5P2SVLXyUbr+vFTfrTL/WKnXIJgbsnj3Gvf+GIZv9tKcXhSNmEHQCYX4
+X3w9iTPddCHuvZ1fpufi2TyArJh0OkoNtLXJHTKrHjf2N+61AQzFiv5WieJrdE+H
+qr32PTUuVGPyO9LyTY4/RL0=
+-----END PRIVATE KEY-----
+";
diff --git a/litellm-rust/crates/secrets/Cargo.toml b/litellm-rust/crates/secrets/Cargo.toml
index a7e7ec80636..7369f9e1db6 100644
--- a/litellm-rust/crates/secrets/Cargo.toml
+++ b/litellm-rust/crates/secrets/Cargo.toml
@@ -9,11 +9,13 @@ repository.workspace = true
 default = []
 aws = ["dep:litellm-secrets-aws"]
 google = ["dep:litellm-secrets-google"]
+hashicorp = ["dep:litellm-secrets-hashicorp"]
 
 [dependencies]
 litellm-secrets-types.workspace = true
 litellm-secrets-aws = { workspace = true, optional = true }
 litellm-secrets-google = { workspace = true, optional = true }
+litellm-secrets-hashicorp = { workspace = true, optional = true }
 litellm-core-utils.workspace = true
 base64.workspace = true
 serde.workspace = true
diff --git a/litellm-rust/crates/secrets/README.md b/litellm-rust/crates/secrets/README.md
index 183a39e15bb..0d333c7d116 100644
--- a/litellm-rust/crates/secrets/README.md
+++ b/litellm-rust/crates/secrets/README.md
@@ -9,3 +9,5 @@ Backend failures propagate by default. To allow fallback during a backend failur
 `get_secret` preserves value types. `get_secret_str` accepts a string default and rejects boolean or JSON values with `Error::TypeMismatch`. `get_secret_bool` accepts a boolean default and converts strings containing `true` or `false`, ignoring surrounding whitespace and ASCII case. Other strings and JSON values produce `Error::TypeMismatch`. Conversion failures never activate fallback or replace a found value with the default
 
 Provider payloads remain strings unless explicitly selecting a field from an AWS primary JSON secret. Google caches only successfully decoded string payloads, so reads have identical values and types before and after caching. Confirmed absence and failed reads are not cached. AWS resource-not-found responses and Google HTTP 404 responses indicate absence. Other provider errors remain errors, and successful responses without the required payload are malformed responses rather than missing secrets
+
+The HashiCorp Vault backend is enabled with the `hashicorp` feature and reads KV v2 values from `HCP_VAULT_*` environment variables. It supports static tokens, AppRole authentication, and TLS certificate authentication
diff --git a/litellm-rust/crates/secrets/src/error.rs b/litellm-rust/crates/secrets/src/error.rs
index 0c6e681b8aa..e5b82580cfe 100644
--- a/litellm-rust/crates/secrets/src/error.rs
+++ b/litellm-rust/crates/secrets/src/error.rs
@@ -30,4 +30,7 @@ pub enum Error {
     #[cfg(feature = "google")]
     #[error(transparent)]
     Google(#[from] litellm_secrets_google::Error),
+    #[cfg(feature = "hashicorp")]
+    #[error(transparent)]
+    Hashicorp(#[from] litellm_secrets_hashicorp::Error),
 }
diff --git a/litellm-rust/crates/secrets/src/handler.rs b/litellm-rust/crates/secrets/src/handler.rs
index 943ffdf6158..c035d4cb586 100644
--- a/litellm-rust/crates/secrets/src/handler.rs
+++ b/litellm-rust/crates/secrets/src/handler.rs
@@ -13,6 +13,8 @@ pub enum SecretManager {
     GoogleKms(crate::google::GoogleKms),
     #[cfg(feature = "google")]
     GoogleSecretManager(crate::google::GoogleSecretManager),
+    #[cfg(feature = "hashicorp")]
+    HashicorpVault(crate::hashicorp::HashicorpVault),
 }
 
 impl SecretManager {
@@ -27,6 +29,8 @@ impl SecretManager {
             Self::GoogleKms(_) => KeyManagementSystem::GoogleKms,
             #[cfg(feature = "google")]
             Self::GoogleSecretManager(_) => KeyManagementSystem::GoogleSecretManager,
+            #[cfg(feature = "hashicorp")]
+            Self::HashicorpVault(_) => KeyManagementSystem::HashicorpVault,
         }
     }
 }
@@ -78,6 +82,12 @@ pub async fn get_secret_from_manager(
             .get_secret_from_google_secret_manager(secret_name)
             .await
             .map_err(Error::from),
+        #[cfg(feature = "hashicorp")]
+        SecretManager::HashicorpVault(client) => client
+            .async_read_secret(secret_name)
+            .await
+            .map(|value| value.map(Secret::String))
+            .map_err(Error::from),
     }
 }
 
diff --git a/litellm-rust/crates/secrets/src/lib.rs b/litellm-rust/crates/secrets/src/lib.rs
index ff2e95f7b2f..6cb391ff844 100644
--- a/litellm-rust/crates/secrets/src/lib.rs
+++ b/litellm-rust/crates/secrets/src/lib.rs
@@ -19,3 +19,5 @@ pub use state::{SecretManagerState, secret_manager_would_be_consulted};
 pub use litellm_secrets_aws as aws;
 #[cfg(feature = "google")]
 pub use litellm_secrets_google as google;
+#[cfg(feature = "hashicorp")]
+pub use litellm_secrets_hashicorp as hashicorp;
diff --git a/litellm-rust/crates/secrets/tests/handler.rs b/litellm-rust/crates/secrets/tests/handler.rs
index a2cbbd843e1..58941171fd9 100644
--- a/litellm-rust/crates/secrets/tests/handler.rs
+++ b/litellm-rust/crates/secrets/tests/handler.rs
@@ -105,3 +105,131 @@ async fn google_handler_requires_canonical_base64_and_preserves_plaintext_whites
         Err(Error::MissingCiphertext)
     ));
 }
+
+#[cfg(feature = "hashicorp")]
+#[tokio::test]
+async fn hashicorp_handler_resolves_found_missing_and_failed_values() {
+    use std::sync::Arc;
+
+    use litellm_core_utils::settings::Lookup;
+    use litellm_secrets::{
+        Error, FailurePolicy, KeyManagementSettings, SecretManager, SecretManagerState,
+        SecretResolver, hashicorp::HashicorpVault, hashicorp::HashicorpVaultConfig,
+    };
+    use wiremock::{
+        Mock, MockServer, ResponseTemplate,
+        matchers::{method, path},
+    };
+
+    let found_server = MockServer::start().await;
+    Mock::given(method("GET"))
+        .and(path("/v1/secret/data/KEY"))
+        .respond_with(
+            ResponseTemplate::new(200)
+                .set_body_json(serde_json::json!({"data": {"data": {"key": "remote"}}})),
+        )
+        .mount(&found_server)
+        .await;
+    let found_environment: Arc<dyn Lookup + Send + Sync> = Arc::new({
+        let address = found_server.uri();
+        move |name: &str| match name {
+            "HCP_VAULT_ADDR" => Some(address.clone()),
+            "HCP_VAULT_TOKEN" => Some("token".into()),
+            _ => None,
+        }
+    });
+    let found_config = HashicorpVaultConfig::from_environment(found_environment.as_ref()).unwrap();
+    let found_manager =
+        HashicorpVault::with_client(reqwest::Client::new(), found_config, true).unwrap();
+    let found_resolver = SecretResolver::new(
+        Arc::new(SecretManagerState::new(
+            SecretManager::HashicorpVault(found_manager),
+            KeyManagementSettings {
+                hosted_keys: Some(vec!["KEY".into()]),
+                ..Default::default()
+            },
+        )),
+        Arc::new(|_: &str| None),
+        litellm_secrets::OidcResolver::default(),
+    );
+    assert_eq!(
+        found_resolver
+            .get_secret_str("KEY", None)
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "remote"
+    );
+
+    let missing_server = MockServer::start().await;
+    Mock::given(method("GET"))
+        .respond_with(ResponseTemplate::new(404))
+        .mount(&missing_server)
+        .await;
+    let missing_environment: Arc<dyn Lookup + Send + Sync> = Arc::new({
+        let address = missing_server.uri();
+        move |name: &str| match name {
+            "HCP_VAULT_ADDR" => Some(address.clone()),
+            "HCP_VAULT_TOKEN" => Some("token".into()),
+            _ => None,
+        }
+    });
+    let missing_config =
+        HashicorpVaultConfig::from_environment(missing_environment.as_ref()).unwrap();
+    let missing_manager =
+        HashicorpVault::with_client(reqwest::Client::new(), missing_config, true).unwrap();
+    let missing_state = SecretManagerState::new(
+        SecretManager::HashicorpVault(missing_manager),
+        KeyManagementSettings {
+            hosted_keys: Some(vec!["KEY".into()]),
+            ..Default::default()
+        },
+    );
+    let missing = litellm_secrets::get_secret_from_manager(
+        missing_state.backend().unwrap(),
+        "KEY",
+        missing_state.settings().unwrap(),
+        &|_: &str| None,
+    )
+    .await
+    .unwrap();
+    assert!(missing.is_none());
+
+    let failed_server = MockServer::start().await;
+    Mock::given(method("GET"))
+        .respond_with(ResponseTemplate::new(500))
+        .mount(&failed_server)
+        .await;
+    let failed_environment: Arc<dyn Lookup + Send + Sync> = Arc::new({
+        let address = failed_server.uri();
+        move |name: &str| match name {
+            "HCP_VAULT_ADDR" => Some(address.clone()),
+            "HCP_VAULT_TOKEN" => Some("token".into()),
+            _ => None,
+        }
+    });
+    let failed_config =
+        HashicorpVaultConfig::from_environment(failed_environment.as_ref()).unwrap();
+    let failed_manager =
+        HashicorpVault::with_client(reqwest::Client::new(), failed_config, true).unwrap();
+    let failed_state = SecretManagerState::new(
+        SecretManager::HashicorpVault(failed_manager),
+        KeyManagementSettings {
+            hosted_keys: Some(vec!["KEY".into()]),
+            ..Default::default()
+        },
+    );
+    let failed_resolver = SecretResolver::new(
+        Arc::new(failed_state),
+        Arc::new(|_: &str| None),
+        litellm_secrets::OidcResolver::default(),
+    )
+    .with_failure_policy(FailurePolicy::Propagate);
+    assert!(matches!(
+        failed_resolver.get_secret_str("KEY", None).await,
+        Err(Error::Hashicorp(
+            litellm_secrets::hashicorp::Error::Status { status: 500 }
+        ))
+    ));
+}
diff --git a/litellm/secret_managers/hashicorp_secret_manager.py b/litellm/secret_managers/hashicorp_secret_manager.py
index e37a912c7e1..259b2416d7f 100644
--- a/litellm/secret_managers/hashicorp_secret_manager.py
+++ b/litellm/secret_managers/hashicorp_secret_manager.py
@@ -95,16 +95,16 @@ class HashicorpSecretManager(BaseSecretManager):
         from litellm.proxy.proxy_server import CommonProxyErrors, premium_user
 
         # Vault-specific config
-        self.vault_addr = os.getenv("HCP_VAULT_ADDR", "http://127.0.0.1:8200")
+        self.vault_addr = (os.getenv("HCP_VAULT_ADDR") or "http://127.0.0.1:8200").rstrip("/")
         self.vault_token = os.getenv("HCP_VAULT_TOKEN", "")
-        self.vault_namespace = os.getenv("HCP_VAULT_NAMESPACE", None)
-        self.login_namespace_override = os.getenv("HCP_VAULT_LOGIN_NAMESPACE", None)
-        self.secret_namespace_override = os.getenv("HCP_VAULT_SECRET_NAMESPACE", None)
+        self.vault_namespace = self._sanitize_path_component(os.getenv("HCP_VAULT_NAMESPACE"))
+        self.login_namespace_override = self._sanitize_path_component(os.getenv("HCP_VAULT_LOGIN_NAMESPACE"))
+        self.secret_namespace_override = self._sanitize_path_component(os.getenv("HCP_VAULT_SECRET_NAMESPACE"))
         # KV engine mount name (default: "secret")
         # If your KV engine is mounted somewhere other than "secret", set HCP_VAULT_MOUNT_NAME
-        self.vault_mount_name = os.getenv("HCP_VAULT_MOUNT_NAME", "secret")
+        self.vault_mount_name = self._sanitize_path_component(os.getenv("HCP_VAULT_MOUNT_NAME")) or "secret"
         # Optional path prefix for secrets (e.g., "myapp" -> secret/data/myapp/{secret_name})
-        self.vault_path_prefix = os.getenv("HCP_VAULT_PATH_PREFIX", None)
+        self.vault_path_prefix = self._sanitize_path_component(os.getenv("HCP_VAULT_PATH_PREFIX"))
 
         # Optional config for TLS cert auth
         self.tls_cert_path = os.getenv("HCP_VAULT_CLIENT_CERT", "")
@@ -114,7 +114,9 @@ class HashicorpSecretManager(BaseSecretManager):
         # Optional config for AppRole auth
         self.approle_role_id = os.getenv("HCP_VAULT_APPROLE_ROLE_ID", "")
         self.approle_secret_id = os.getenv("HCP_VAULT_APPROLE_SECRET_ID", "")
-        self.approle_mount_path = os.getenv("HCP_VAULT_APPROLE_MOUNT_PATH", "approle")
+        self.approle_mount_path = self._sanitize_path_component(
+            os.getenv("HCP_VAULT_APPROLE_MOUNT_PATH")
+        ) or "approle"
 
         self._verify_required_credentials_exist()
 
diff --git a/tests/test_litellm/secret_managers/hashicorp_vault_parity.json b/tests/test_litellm/secret_managers/hashicorp_vault_parity.json
new file mode 100644
index 00000000000..5f28d9602a0
--- /dev/null
+++ b/tests/test_litellm/secret_managers/hashicorp_vault_parity.json
@@ -0,0 +1,97 @@
+[
+  {
+    "name": "defaults",
+    "env": {
+      "HCP_VAULT_TOKEN": "token"
+    },
+    "secret_name": "OPENAI_API_KEY",
+    "expected_secret_url": "http://127.0.0.1:8200/v1/secret/data/OPENAI_API_KEY",
+    "expected_login_url": null,
+    "expected_login_namespace": null,
+    "expected_secret_namespace": null
+  },
+  {
+    "name": "global_namespace",
+    "env": {
+      "HCP_VAULT_ADDR": "http://vault.test:8200",
+      "HCP_VAULT_TOKEN": "token",
+      "HCP_VAULT_NAMESPACE": " admin/ "
+    },
+    "secret_name": "OPENAI_API_KEY",
+    "expected_secret_url": "http://vault.test:8200/v1/admin/secret/data/OPENAI_API_KEY",
+    "expected_login_url": null,
+    "expected_login_namespace": "admin",
+    "expected_secret_namespace": "admin"
+  },
+  {
+    "name": "namespace_overrides",
+    "env": {
+      "HCP_VAULT_ADDR": "http://vault.test:8200",
+      "HCP_VAULT_TOKEN": "token",
+      "HCP_VAULT_NAMESPACE": "admin",
+      "HCP_VAULT_LOGIN_NAMESPACE": " /root/ ",
+      "HCP_VAULT_SECRET_NAMESPACE": " /teams/team-a/ "
+    },
+    "secret_name": "OPENAI_API_KEY",
+    "expected_secret_url": "http://vault.test:8200/v1/teams/team-a/secret/data/OPENAI_API_KEY",
+    "expected_login_url": null,
+    "expected_login_namespace": "root",
+    "expected_secret_namespace": "teams/team-a"
+  },
+  {
+    "name": "custom_mount_and_prefix",
+    "env": {
+      "HCP_VAULT_ADDR": "http://vault.test:8200",
+      "HCP_VAULT_TOKEN": "token",
+      "HCP_VAULT_MOUNT_NAME": " /kv-prod/ ",
+      "HCP_VAULT_PATH_PREFIX": " /virtual-keys/ "
+    },
+    "secret_name": "DB_PASSWORD",
+    "expected_secret_url": "http://vault.test:8200/v1/kv-prod/data/virtual-keys/DB_PASSWORD",
+    "expected_login_url": null,
+    "expected_login_namespace": null,
+    "expected_secret_namespace": null
+  },
+  {
+    "name": "approle_custom_mount",
+    "env": {
+      "HCP_VAULT_ADDR": "http://vault.test:8200",
+      "HCP_VAULT_APPROLE_ROLE_ID": "role-id",
+      "HCP_VAULT_APPROLE_SECRET_ID": "secret-id",
+      "HCP_VAULT_APPROLE_MOUNT_PATH": " /custom-approle/ ",
+      "HCP_VAULT_NAMESPACE": "admin"
+    },
+    "secret_name": "OPENAI_API_KEY",
+    "expected_secret_url": "http://vault.test:8200/v1/admin/secret/data/OPENAI_API_KEY",
+    "expected_login_url": "http://vault.test:8200/v1/auth/custom-approle/login",
+    "expected_login_namespace": "admin",
+    "expected_secret_namespace": "admin"
+  },
+  {
+    "name": "tls_cert",
+    "env": {
+      "HCP_VAULT_ADDR": "http://vault.test:8200",
+      "HCP_VAULT_CLIENT_CERT": "/tmp/client.crt",
+      "HCP_VAULT_CLIENT_KEY": "/tmp/client.key",
+      "HCP_VAULT_CERT_ROLE": "vault-role",
+      "HCP_VAULT_NAMESPACE": "admin"
+    },
+    "secret_name": "OPENAI_API_KEY",
+    "expected_secret_url": "http://vault.test:8200/v1/admin/secret/data/OPENAI_API_KEY",
+    "expected_login_url": "http://vault.test:8200/v1/auth/cert/login",
+    "expected_login_namespace": "admin",
+    "expected_secret_namespace": "admin"
+  },
+  {
+    "name": "trailing_address_slash",
+    "env": {
+      "HCP_VAULT_ADDR": "http://vault.test:8200///",
+      "HCP_VAULT_TOKEN": "token"
+    },
+    "secret_name": "OPENAI_API_KEY",
+    "expected_secret_url": "http://vault.test:8200/v1/secret/data/OPENAI_API_KEY",
+    "expected_login_url": null,
+    "expected_login_namespace": null,
+    "expected_secret_namespace": null
+  }
+]
diff --git a/tests/test_litellm/secret_managers/test_hashicorp_secret_manager.py b/tests/test_litellm/secret_managers/test_hashicorp_secret_manager.py
index 1676540e4ec..fc18cb8b8f7 100644
--- a/tests/test_litellm/secret_managers/test_hashicorp_secret_manager.py
+++ b/tests/test_litellm/secret_managers/test_hashicorp_secret_manager.py
@@ -1,4 +1,5 @@
 import datetime
+import json
 from collections.abc import Mapping
 from pathlib import Path
 from typing import Final
@@ -18,6 +19,23 @@ LOGIN_RESPONSE: Final = {"auth": {"client_token": "hvs.login-token", "lease_dura
 SECRET_RESPONSE: Final = {"data": {"data": {"key": "sk-from-vault", "password": "pw-from-vault"}}}
 
 NAMESPACE_ENV_VARS: Final = ("HCP_VAULT_NAMESPACE", "HCP_VAULT_LOGIN_NAMESPACE", "HCP_VAULT_SECRET_NAMESPACE")
+PARITY_ENV_VARS: Final = (
+    "HCP_VAULT_ADDR",
+    "HCP_VAULT_TOKEN",
+    "HCP_VAULT_NAMESPACE",
+    "HCP_VAULT_LOGIN_NAMESPACE",
+    "HCP_VAULT_SECRET_NAMESPACE",
+    "HCP_VAULT_MOUNT_NAME",
+    "HCP_VAULT_PATH_PREFIX",
+    "HCP_VAULT_APPROLE_ROLE_ID",
+    "HCP_VAULT_APPROLE_SECRET_ID",
+    "HCP_VAULT_APPROLE_MOUNT_PATH",
+    "HCP_VAULT_CLIENT_CERT",
+    "HCP_VAULT_CLIENT_KEY",
+    "HCP_VAULT_CERT_ROLE",
+    "HCP_VAULT_REFRESH_INTERVAL",
+    "SECRET_MANAGER_REFRESH_INTERVAL",
+)
 
 
 def _build_manager(monkeypatch: pytest.MonkeyPatch, env: Mapping[str, str]) -> HashicorpSecretManager:
@@ -236,3 +254,35 @@ def test_tls_login_uses_login_namespace(monkeypatch: pytest.MonkeyPatch, tmp_pat
 
     assert manager._auth_via_tls_cert() == "hvs.login-token"
     assert login_route.calls.last.request.headers["X-Vault-Namespace"] == "root"
+
+
+with Path(__file__).with_name("hashicorp_vault_parity.json").open() as parity_file:
+    PARITY_CASES: Final = json.load(parity_file)
+
+
+@pytest.mark.parametrize("case", PARITY_CASES, ids=lambda case: case["name"])
+def test_configuration_matches_native_parity_fixture(
+    monkeypatch: pytest.MonkeyPatch, case: Mapping[str, object]
+) -> None:
+    monkeypatch.setattr(litellm.proxy.proxy_server, "premium_user", True)
+    for name in PARITY_ENV_VARS:
+        monkeypatch.delenv(name, raising=False)
+    for name, value in case["env"].items():
+        monkeypatch.setenv(name, value)
+
+    manager: Final = HashicorpSecretManager()
+    env: Final = case["env"]
+    expected_login_url: Final = case["expected_login_url"]
+    if env.get("HCP_VAULT_APPROLE_ROLE_ID") and env.get("HCP_VAULT_APPROLE_SECRET_ID"):
+        login_url: str | None = (
+            f"{manager.vault_addr}/v1/auth/{manager.approle_mount_path}/login"
+        )
+    elif env.get("HCP_VAULT_CLIENT_CERT") and env.get("HCP_VAULT_CLIENT_KEY"):
+        login_url = f"{manager.vault_addr}/v1/auth/cert/login"
+    else:
+        login_url = None
+
+    assert manager.get_url(case["secret_name"]) == case["expected_secret_url"]
+    assert manager.vault_login_namespace == case["expected_login_namespace"]
+    assert manager.vault_secret_namespace == case["expected_secret_namespace"]
+    assert login_url == expected_login_url

From ca31149040170297c1a00044d6bb9650862dd895 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:28:27 +0000
Subject: [PATCH 068/160] feat(rust): add native GCS object-store cache backend

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.lock                       |  17 +
 litellm-rust/Cargo.toml                       |   2 +
 litellm-rust/crates/auth-gcp/src/lib.rs       |   8 +
 litellm-rust/crates/cache-gcs/Cargo.toml      |  21 ++
 litellm-rust/crates/cache-gcs/src/cache.rs    | 285 +++++++++++++++++
 litellm-rust/crates/cache-gcs/src/lib.rs      |   5 +
 litellm-rust/crates/cache-gcs/src/token.rs    |  44 +++
 litellm-rust/crates/cache-gcs/tests/cache.rs  | 290 ++++++++++++++++++
 litellm-rust/crates/cache/src/error.rs        |   2 +
 litellm-rust/crates/python-bridge/Cargo.toml  |   1 +
 .../crates/python-bridge/src/cache/config.rs  | 143 ++++++++-
 .../crates/python-bridge/src/cache/facade.rs  |   4 +
 .../crates/python-bridge/src/cache/handle.rs  |  27 ++
 .../crates/python-bridge/src/cache/mod.rs     |   3 +-
 .../crates/python-bridge/src/cache/native.rs  |  35 +++
 15 files changed, 876 insertions(+), 11 deletions(-)
 create mode 100644 litellm-rust/crates/cache-gcs/Cargo.toml
 create mode 100644 litellm-rust/crates/cache-gcs/src/cache.rs
 create mode 100644 litellm-rust/crates/cache-gcs/src/lib.rs
 create mode 100644 litellm-rust/crates/cache-gcs/src/token.rs
 create mode 100644 litellm-rust/crates/cache-gcs/tests/cache.rs

diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index ed4ae4e3353..99a6133b4f1 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -2464,6 +2464,22 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "litellm-cache-gcs"
+version = "0.1.0"
+dependencies = [
+ "futures-util",
+ "litellm-auth-gcp",
+ "litellm-auth-types",
+ "litellm-cache",
+ "percent-encoding",
+ "reqwest 0.12.28",
+ "serde_json",
+ "tokio",
+ "url",
+ "wiremock",
+]
+
 [[package]]
 name = "litellm-cache-memory"
 version = "0.1.0"
@@ -2666,6 +2682,7 @@ dependencies = [
  "litellm-auth",
  "litellm-auth-gcp",
  "litellm-cache",
+ "litellm-cache-gcs",
  "litellm-cache-memory",
  "litellm-cache-redis",
  "litellm-cache-response",
diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
index 570d0dd3568..80425d567c0 100644
--- a/litellm-rust/Cargo.toml
+++ b/litellm-rust/Cargo.toml
@@ -29,6 +29,7 @@ litellm-core-utils = { path = "crates/core-utils" }
 litellm-cache = { path = "crates/cache" }
 litellm-cache-memory = { path = "crates/cache-memory" }
 litellm-cache-redis = { path = "crates/cache-redis" }
+litellm-cache-gcs = { path = "crates/cache-gcs" }
 litellm-cache-response = { path = "crates/cache-response" }
 litellm-token-counter = { path = "crates/token-counter" }
 litellm-token-counter-fast = { path = "crates/token-counter-fast" }
@@ -66,6 +67,7 @@ base64 = "0.22"
 moka = { version = "0.12.16", features = ["future"] }
 strum = { version = "0.28.0", features = ["derive"] }
 url = "2.5.8"
+percent-encoding = "2.3"
 webpki-roots = "1"
 time = { version = "0.3.53", features = ["parsing"] }
 criterion = "0.8.2"
diff --git a/litellm-rust/crates/auth-gcp/src/lib.rs b/litellm-rust/crates/auth-gcp/src/lib.rs
index 8aeddae9efc..534d85acdb0 100644
--- a/litellm-rust/crates/auth-gcp/src/lib.rs
+++ b/litellm-rust/crates/auth-gcp/src/lib.rs
@@ -128,6 +128,14 @@ impl VertexAuth {
         }
     }
 
+    pub async fn access_token(
+        &self,
+        config: &VertexConfig,
+        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
+    ) -> Result<String, Error> {
+        self.load_provider(config, env_lookup).await?.token().await
+    }
+
     pub async fn validate_environment(
         &self,
         headers: Vec<(String, String)>,
diff --git a/litellm-rust/crates/cache-gcs/Cargo.toml b/litellm-rust/crates/cache-gcs/Cargo.toml
new file mode 100644
index 00000000000..3890e60843c
--- /dev/null
+++ b/litellm-rust/crates/cache-gcs/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "litellm-cache-gcs"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+futures-util.workspace = true
+litellm-auth-gcp.workspace = true
+litellm-auth-types.workspace = true
+litellm-cache.workspace = true
+percent-encoding.workspace = true
+reqwest.workspace = true
+tokio.workspace = true
+url.workspace = true
+
+[dev-dependencies]
+serde_json.workspace = true
+tokio.workspace = true
+wiremock = "0.6.5"
diff --git a/litellm-rust/crates/cache-gcs/src/cache.rs b/litellm-rust/crates/cache-gcs/src/cache.rs
new file mode 100644
index 00000000000..1097f3252b1
--- /dev/null
+++ b/litellm-rust/crates/cache-gcs/src/cache.rs
@@ -0,0 +1,285 @@
+use std::{future::Future, sync::Arc, time::Duration};
+
+use futures_util::future::try_join_all;
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, Error, ExactCacheContext,
+    FlushCache,
+};
+use percent_encoding::{AsciiSet, CONTROLS, percent_encode};
+use reqwest::Client;
+
+use crate::{GcpTokenSource, TokenSource};
+
+pub const DEFAULT_ENDPOINT: &str = "https://storage.googleapis.com";
+
+const OBJECT_NAME_ENCODE_SET: &AsciiSet = &CONTROLS
+    .add(b' ')
+    .add(b'!')
+    .add(b'"')
+    .add(b'#')
+    .add(b'$')
+    .add(b'%')
+    .add(b'&')
+    .add(b'\'')
+    .add(b'(')
+    .add(b')')
+    .add(b'*')
+    .add(b'+')
+    .add(b',')
+    .add(b'/')
+    .add(b':')
+    .add(b';')
+    .add(b'<')
+    .add(b'=')
+    .add(b'>')
+    .add(b'?')
+    .add(b'@')
+    .add(b'[')
+    .add(b'\\')
+    .add(b']')
+    .add(b'^')
+    .add(b'`')
+    .add(b'{')
+    .add(b'|')
+    .add(b'}');
+
+pub fn key_prefix(gcs_path: Option<&str>) -> String {
+    match gcs_path {
+        Some(path) if !path.is_empty() => format!("{}/", path.trim_end_matches('/')),
+        _ => String::new(),
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct GcsConfig {
+    pub bucket_name: String,
+    pub gcs_path: Option<String>,
+    pub path_service_account: Option<String>,
+    pub endpoint: String,
+}
+
+impl GcsConfig {
+    pub fn new(bucket_name: impl Into<String>) -> Self {
+        Self {
+            bucket_name: bucket_name.into(),
+            gcs_path: None,
+            path_service_account: None,
+            endpoint: DEFAULT_ENDPOINT.to_string(),
+        }
+    }
+}
+
+pub struct GcsCache<S: CacheCodec> {
+    config: GcsConfig,
+    key_prefix: String,
+    client: Client,
+    token: Arc<dyn TokenSource>,
+    codec: S,
+}
+
+impl<S: CacheCodec> GcsCache<S> {
+    pub fn new(config: GcsConfig, codec: S) -> Result<Self, Error> {
+        let token = Arc::new(GcpTokenSource::new(config.path_service_account.clone()));
+        Self::with_token_source(config, codec, token)
+    }
+
+    pub fn with_token_source(
+        config: GcsConfig,
+        codec: S,
+        token: Arc<dyn TokenSource>,
+    ) -> Result<Self, Error> {
+        let client = Client::builder().build().map_err(|_| Error::Unavailable)?;
+        let key_prefix = key_prefix(config.gcs_path.as_deref());
+        Ok(Self {
+            config,
+            key_prefix,
+            client,
+            token,
+            codec,
+        })
+    }
+
+    pub fn bucket_name(&self) -> &str {
+        &self.config.bucket_name
+    }
+
+    pub fn key_prefix(&self) -> &str {
+        &self.key_prefix
+    }
+
+    pub fn path_service_account(&self) -> Option<&str> {
+        self.config.path_service_account.as_deref()
+    }
+
+    pub fn object_name(&self, key: &str) -> String {
+        format!("{}{}", self.key_prefix, key)
+    }
+
+    fn encoded_object_name(&self, key: &str) -> String {
+        percent_encode(self.object_name(key).as_bytes(), OBJECT_NAME_ENCODE_SET).to_string()
+    }
+
+    fn endpoint(&self, path: &str) -> String {
+        format!("{}{}", self.config.endpoint.trim_end_matches('/'), path)
+    }
+
+    async fn async_set(&self, key: &str, value: S::Value) -> Result<(), Error> {
+        let token = self.token.bearer_token().await?;
+        let payload = self.codec.encode(&value)?;
+        let url = self.endpoint(&format!(
+            "/upload/storage/v1/b/{}/o?uploadType=media&name={}",
+            self.config.bucket_name,
+            self.encoded_object_name(key)
+        ));
+        let response = self
+            .client
+            .post(url)
+            .bearer_auth(token)
+            .header(reqwest::header::CONTENT_TYPE, "application/json")
+            .body(payload)
+            .send()
+            .await
+            .map_err(|_| Error::Unavailable)?;
+        if !response.status().is_success() {
+            return Err(Error::Unavailable);
+        }
+        Ok(())
+    }
+
+    async fn async_get(&self, key: &str) -> Result<Option<S::Value>, Error> {
+        let token = self.token.bearer_token().await?;
+        let url = self.endpoint(&format!(
+            "/storage/v1/b/{}/o/{}?alt=media",
+            self.config.bucket_name,
+            self.encoded_object_name(key)
+        ));
+        let response = self
+            .client
+            .get(url)
+            .bearer_auth(token)
+            .header(reqwest::header::CONTENT_TYPE, "application/json")
+            .send()
+            .await
+            .map_err(|_| Error::Unavailable)?;
+        if response.status() == reqwest::StatusCode::NOT_FOUND {
+            return Ok(None);
+        }
+        if !response.status().is_success() {
+            return Err(Error::Unavailable);
+        }
+        let body = response.bytes().await.map_err(|_| Error::Unavailable)?;
+        self.codec
+            .decode(&body)
+            .map(Some)
+            .map_err(|_| Error::InvalidEntry)
+    }
+
+    fn run_sync<T, F>(future: F) -> Result<T, Error>
+    where
+        F: Future<Output = Result<T, Error>> + Send,
+        T: Send,
+    {
+        let run = || {
+            tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .map_err(|_| Error::Unavailable)
+                .and_then(|runtime| runtime.block_on(future))
+        };
+        if let Ok(handle) = tokio::runtime::Handle::try_current() {
+            if handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread {
+                return tokio::task::block_in_place(run);
+            }
+            return std::thread::scope(|scope| {
+                scope
+                    .spawn(run)
+                    .join()
+                    .map_err(|_| Error::Unavailable)
+                    .and_then(|result| result)
+            });
+        }
+        run()
+    }
+}
+
+impl<S: CacheCodec> BaseCache for GcsCache<S> {
+    type Value = S::Value;
+    type Context = ExactCacheContext;
+
+    fn get_ttl(&self, _: &Self::Context) -> Option<Duration> {
+        None
+    }
+
+    fn set_cache(&self, key: &str, value: Self::Value, _: &Self::Context) -> Result<(), Error> {
+        Self::run_sync(self.async_set(key, value))
+    }
+
+    fn get_cache(&self, key: &str, _: &Self::Context) -> Result<Option<Self::Value>, Error> {
+        Self::run_sync(self.async_get(key))
+    }
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        _: Self::Context,
+    ) -> Result<(), Error> {
+        self.async_set(key, value).await
+    }
+
+    async fn async_get_cache(
+        &self,
+        key: &str,
+        _: &Self::Context,
+    ) -> Result<Option<Self::Value>, Error> {
+        self.async_get(key).await
+    }
+
+    async fn async_set_cache_pipeline(
+        &self,
+        entries: Vec<(String, Self::Value)>,
+        context: Self::Context,
+    ) -> Result<(), Error> {
+        try_join_all(entries.into_iter().map(|(key, value)| {
+            let context = context.clone();
+            async move { self.async_set_cache(&key, value, context).await }
+        }))
+        .await
+        .map(|_| ())
+    }
+
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        Err(Error::UnsupportedOperation)
+    }
+}
+
+impl<S: CacheCodec> BatchCache for GcsCache<S> {
+    async fn async_batch_get_cache(
+        &self,
+        keys: Vec<String>,
+        context: Self::Context,
+    ) -> Result<Vec<BatchEntry<Self::Value>>, Error> {
+        try_join_all(keys.into_iter().map(|key| {
+            let context = context.clone();
+            async move {
+                match self.async_get_cache(&key, &context).await {
+                    Ok(Some(value)) => Ok(BatchEntry::Hit(value)),
+                    Ok(None) => Ok(BatchEntry::Miss),
+                    Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid),
+                    Err(error) => Err(error),
+                }
+            }
+        }))
+        .await
+    }
+}
+
+impl<S: CacheCodec> FlushCache for GcsCache<S> {
+    fn flush_cache(&self) -> Result<(), Error> {
+        Ok(())
+    }
+}
diff --git a/litellm-rust/crates/cache-gcs/src/lib.rs b/litellm-rust/crates/cache-gcs/src/lib.rs
new file mode 100644
index 00000000000..cbb61cf0685
--- /dev/null
+++ b/litellm-rust/crates/cache-gcs/src/lib.rs
@@ -0,0 +1,5 @@
+mod cache;
+mod token;
+
+pub use cache::{DEFAULT_ENDPOINT, GcsCache, GcsConfig, key_prefix};
+pub use token::{GcpTokenSource, StaticTokenSource, TokenSource};
diff --git a/litellm-rust/crates/cache-gcs/src/token.rs b/litellm-rust/crates/cache-gcs/src/token.rs
new file mode 100644
index 00000000000..adb601c276c
--- /dev/null
+++ b/litellm-rust/crates/cache-gcs/src/token.rs
@@ -0,0 +1,44 @@
+use std::{future::Future, pin::Pin};
+
+use litellm_auth_gcp::{VertexAuth, VertexConfig};
+use litellm_auth_types::{InputSource, SecretValue, Sourced};
+use litellm_cache::Error;
+
+pub trait TokenSource: Send + Sync + 'static {
+    fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>>;
+}
+
+pub struct GcpTokenSource {
+    auth: VertexAuth,
+    config: VertexConfig,
+}
+
+impl GcpTokenSource {
+    pub fn new(path_service_account: Option<String>) -> Self {
+        let credentials = path_service_account
+            .map(|path| Sourced::new(SecretValue::new(path), InputSource::Deployment));
+        Self {
+            auth: VertexAuth::default(),
+            config: VertexConfig::new(credentials, None, None),
+        }
+    }
+}
+
+impl TokenSource for GcpTokenSource {
+    fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>> {
+        Box::pin(async move {
+            self.auth
+                .access_token(&self.config, &|name| std::env::var(name).ok())
+                .await
+                .map_err(|_| Error::Unavailable)
+        })
+    }
+}
+
+pub struct StaticTokenSource(pub String);
+
+impl TokenSource for StaticTokenSource {
+    fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>> {
+        Box::pin(async move { Ok(self.0.clone()) })
+    }
+}
diff --git a/litellm-rust/crates/cache-gcs/tests/cache.rs b/litellm-rust/crates/cache-gcs/tests/cache.rs
new file mode 100644
index 00000000000..79e4ed100f8
--- /dev/null
+++ b/litellm-rust/crates/cache-gcs/tests/cache.rs
@@ -0,0 +1,290 @@
+use std::{sync::Arc, time::Duration};
+
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheContext, Error, ExactCacheContext, FlushCache,
+    JsonCodec,
+};
+use litellm_cache_gcs::{GcsCache, GcsConfig, StaticTokenSource, TokenSource, key_prefix};
+use serde_json::json;
+use wiremock::{
+    Mock, MockServer, ResponseTemplate,
+    matchers::{body_bytes, header, method, path, query_param},
+};
+
+fn config(server: &MockServer, gcs_path: Option<&str>) -> GcsConfig {
+    GcsConfig {
+        bucket_name: "bucket".into(),
+        gcs_path: gcs_path.map(str::to_string),
+        path_service_account: None,
+        endpoint: server.uri(),
+    }
+}
+
+fn cache(server: &MockServer, gcs_path: Option<&str>) -> GcsCache<JsonCodec<serde_json::Value>> {
+    GcsCache::with_token_source(
+        config(server, gcs_path),
+        JsonCodec::new(),
+        Arc::new(StaticTokenSource("tok".into())),
+    )
+    .unwrap()
+}
+
+#[tokio::test]
+async fn set_writes_encoded_object_and_headers() {
+    let server = MockServer::start().await;
+    Mock::given(method("POST"))
+        .and(path("/upload/storage/v1/b/bucket/o"))
+        .and(query_param("uploadType", "media"))
+        .and(header("authorization", "Bearer tok"))
+        .and(header("content-type", "application/json"))
+        .and(body_bytes(br#"{"value":"entry"}"#))
+        .respond_with(ResponseTemplate::new(200))
+        .expect(1)
+        .mount(&server)
+        .await;
+    cache(&server, Some("cache/"))
+        .set_cache(
+            "team:a b/c",
+            json!({"value": "entry"}),
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    let requests = server.received_requests().await.unwrap();
+    assert_eq!(requests.len(), 1);
+    assert_eq!(
+        requests[0].url.query(),
+        Some("uploadType=media&name=cache%2Fteam%3Aa%20b%2Fc")
+    );
+}
+
+#[tokio::test]
+async fn get_maps_statuses_and_decode_failures() {
+    let server = MockServer::start().await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/hit"))
+        .and(query_param("alt", "media"))
+        .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"})))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/missing"))
+        .respond_with(ResponseTemplate::new(404))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/server-error"))
+        .respond_with(ResponseTemplate::new(500))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/invalid"))
+        .respond_with(ResponseTemplate::new(200).set_body_string("not json"))
+        .mount(&server)
+        .await;
+
+    let cache = cache(&server, None);
+    assert_eq!(
+        cache
+            .get_cache("hit", &ExactCacheContext::default())
+            .unwrap(),
+        Some(json!({"value": "entry"}))
+    );
+    assert_eq!(
+        cache
+            .get_cache("missing", &ExactCacheContext::default())
+            .unwrap(),
+        None
+    );
+    assert_eq!(
+        cache
+            .get_cache("server-error", &ExactCacheContext::default())
+            .unwrap_err(),
+        Error::Unavailable
+    );
+    assert_eq!(
+        cache
+            .get_cache("invalid", &ExactCacheContext::default())
+            .unwrap_err(),
+        Error::InvalidEntry
+    );
+}
+
+#[test]
+fn key_prefix_normalizes_paths() {
+    assert_eq!(key_prefix(None), "");
+    assert_eq!(key_prefix(Some("a/b/")), "a/b/");
+    assert_eq!(key_prefix(Some("a/b")), "a/b/");
+    assert_eq!(key_prefix(Some("")), "");
+}
+
+#[tokio::test]
+async fn ignores_ttl_and_writes_pipeline_concurrently() {
+    let server = MockServer::start().await;
+    for key in ["one", "two", "three"] {
+        Mock::given(method("POST"))
+            .and(path("/upload/storage/v1/b/bucket/o"))
+            .and(query_param("uploadType", "media"))
+            .and(query_param("name", key))
+            .respond_with(ResponseTemplate::new(200))
+            .expect(1)
+            .mount(&server)
+            .await;
+    }
+    let cache = cache(&server, None);
+    assert_eq!(cache.get_ttl(&ExactCacheContext::default()), None);
+    assert_eq!(
+        cache.get_ttl(&ExactCacheContext::default().with_ttl(Some(Duration::from_secs(5)))),
+        None
+    );
+    cache
+        .async_set_cache_pipeline(
+            vec![
+                ("one".into(), json!({"key": "one"})),
+                ("two".into(), json!({"key": "two"})),
+                ("three".into(), json!({"key": "three"})),
+            ],
+            ExactCacheContext::default().with_ttl(Some(Duration::from_secs(5))),
+        )
+        .await
+        .unwrap();
+}
+
+#[tokio::test]
+async fn async_batch_get_preserves_hits_misses_and_invalid_entries() {
+    let server = MockServer::start().await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/hit"))
+        .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"})))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/missing"))
+        .respond_with(ResponseTemplate::new(404))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/invalid"))
+        .respond_with(ResponseTemplate::new(200).set_body_string("not json"))
+        .mount(&server)
+        .await;
+
+    assert_eq!(
+        cache(&server, None)
+            .async_batch_get_cache(
+                vec!["hit".into(), "missing".into(), "invalid".into()],
+                ExactCacheContext::default(),
+            )
+            .await
+            .unwrap(),
+        vec![
+            BatchEntry::Hit(json!({"value": "entry"})),
+            BatchEntry::Miss,
+            BatchEntry::Invalid,
+        ]
+    );
+}
+
+#[tokio::test]
+async fn lifecycle_operations_are_noops_and_connection_test_is_unsupported() {
+    let server = MockServer::start().await;
+    let cache = cache(&server, None);
+    assert_eq!(cache.flush_cache(), Ok(()));
+    assert_eq!(cache.disconnect().await, Ok(()));
+    assert_eq!(
+        cache.test_connection().await,
+        Err(Error::UnsupportedOperation)
+    );
+}
+
+#[test]
+fn sync_operations_work_without_an_active_runtime() {
+    let runtime = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap();
+    let server = runtime.block_on(MockServer::start());
+    runtime.block_on(
+        Mock::given(method("POST"))
+            .and(path("/upload/storage/v1/b/bucket/o"))
+            .respond_with(ResponseTemplate::new(200))
+            .mount(&server),
+    );
+    runtime.block_on(
+        Mock::given(method("GET"))
+            .and(path("/storage/v1/b/bucket/o/key"))
+            .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"})))
+            .mount(&server),
+    );
+    let cache = cache(&server, None);
+    cache
+        .set_cache(
+            "key",
+            json!({"value": "entry"}),
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    assert_eq!(
+        cache
+            .get_cache("key", &ExactCacheContext::default())
+            .unwrap(),
+        Some(json!({"value": "entry"}))
+    );
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn sync_operations_work_inside_a_multi_thread_runtime() {
+    let server = MockServer::start().await;
+    Mock::given(method("POST"))
+        .and(path("/upload/storage/v1/b/bucket/o"))
+        .respond_with(ResponseTemplate::new(200))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/key"))
+        .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"})))
+        .mount(&server)
+        .await;
+    let cache = cache(&server, None);
+    cache
+        .set_cache(
+            "key",
+            json!({"value": "entry"}),
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    assert_eq!(
+        cache
+            .get_cache("key", &ExactCacheContext::default())
+            .unwrap(),
+        Some(json!({"value": "entry"}))
+    );
+}
+
+struct FailingTokenSource;
+
+impl TokenSource for FailingTokenSource {
+    fn bearer_token(
+        &self,
+    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<String, Error>> + Send + '_>>
+    {
+        Box::pin(async { Err(Error::Unavailable) })
+    }
+}
+
+#[tokio::test]
+async fn token_source_failure_skips_http() {
+    let server = MockServer::start().await;
+    let cache = GcsCache::with_token_source(
+        config(&server, None),
+        JsonCodec::<serde_json::Value>::new(),
+        Arc::new(FailingTokenSource),
+    )
+    .unwrap();
+    assert_eq!(
+        cache
+            .get_cache("key", &ExactCacheContext::default())
+            .unwrap_err(),
+        Error::Unavailable
+    );
+    assert_eq!(server.received_requests().await.unwrap().len(), 0);
+}
diff --git a/litellm-rust/crates/cache/src/error.rs b/litellm-rust/crates/cache/src/error.rs
index ff3ff6572d4..1a381d0afd8 100644
--- a/litellm-rust/crates/cache/src/error.rs
+++ b/litellm-rust/crates/cache/src/error.rs
@@ -6,4 +6,6 @@ pub enum Error {
     InvalidEntry,
     #[error("flushing Redis requires an explicit namespace")]
     UnscopedFlush,
+    #[error("operation is not supported by this cache")]
+    UnsupportedOperation,
 }
diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml
index 1eb2ec28036..0f060c801a5 100644
--- a/litellm-rust/crates/python-bridge/Cargo.toml
+++ b/litellm-rust/crates/python-bridge/Cargo.toml
@@ -23,6 +23,7 @@ bytes.workspace = true
 litellm-cache.workspace = true
 litellm-cache-memory.workspace = true
 litellm-cache-redis.workspace = true
+litellm-cache-gcs.workspace = true
 litellm-cache-response.workspace = true
 serde.workspace = true
 litellm-auth.workspace = true
diff --git a/litellm-rust/crates/python-bridge/src/cache/config.rs b/litellm-rust/crates/python-bridge/src/cache/config.rs
index 0e7d6aee11d..66f493dd760 100644
--- a/litellm-rust/crates/python-bridge/src/cache/config.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/config.rs
@@ -73,9 +73,17 @@ pub(super) struct RedisCacheConfig {
     pub(super) connection: RedisConnectionConfig,
 }
 
+#[derive(Debug, PartialEq)]
+pub(super) struct GcsCacheConfig {
+    pub(super) bucket_name: String,
+    pub(super) key_prefix: String,
+    pub(super) path_service_account: Option<String>,
+}
+
 pub(super) enum CacheBackendConfig {
     Memory(MemoryCacheConfig),
     Redis(Box<RedisCacheConfig>),
+    Gcs(GcsCacheConfig),
 }
 
 #[allow(dead_code, reason = "consumed by the cache activation follow-up")]
@@ -90,6 +98,7 @@ pub(super) enum UnsupportedCacheConfig {
     RedisCredentials,
     RedisConnection,
     RedisOption,
+    GcsBucket,
 }
 
 impl UnsupportedCacheConfig {
@@ -100,6 +109,7 @@ impl UnsupportedCacheConfig {
             Self::RedisCredentials => "native Redis credentials require Python",
             Self::RedisConnection => "native Redis connection type is not implemented",
             Self::RedisOption => "native Redis configuration requires Python",
+            Self::GcsBucket => "native GCS cache requires a configured bucket name",
         }
     }
 }
@@ -142,14 +152,20 @@ impl NativeCacheConfig {
                 }))),
                 Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)),
             },
+            Some(CacheType::Gcs) => match project_gcs(&backend)? {
+                Ok(backend) => Ok(CacheConfigProjection::Native(Box::new(Self {
+                    policy,
+                    backend: CacheBackendConfig::Gcs(backend),
+                }))),
+                Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)),
+            },
             Some(
                 CacheType::RedisSemantic
                 | CacheType::ValkeySemantic
                 | CacheType::S3
                 | CacheType::Disk
                 | CacheType::QdrantSemantic
-                | CacheType::AzureBlob
-                | CacheType::Gcs,
+                | CacheType::AzureBlob,
             )
             | None => Ok(CacheConfigProjection::Unsupported(
                 UnsupportedCacheConfig::Backend,
@@ -158,12 +174,12 @@ impl NativeCacheConfig {
     }
 
     pub(super) fn service_mismatch(&self, service: &NativeResponseCache) -> Option<&'static str> {
-        if service.default_ttl()
-            != Some(match &self.backend {
-                CacheBackendConfig::Memory(config) => config.default_ttl,
-                CacheBackendConfig::Redis(config) => config.default_ttl,
-            })
-        {
+        let expected = match &self.backend {
+            CacheBackendConfig::Memory(config) => Some(config.default_ttl),
+            CacheBackendConfig::Redis(config) => Some(config.default_ttl),
+            CacheBackendConfig::Gcs(_) => None,
+        };
+        if service.default_ttl() != expected {
             return Some("facade and native backend default TTLs must match");
         }
         match &self.backend {
@@ -185,6 +201,31 @@ impl NativeCacheConfig {
             CacheBackendConfig::Redis(config) => (service.namespace()
                 != config.namespace.as_deref())
             .then_some("facade and native backend namespaces must match"),
+            CacheBackendConfig::Gcs(_) if service.kind() != "gcs" => {
+                Some("facade and native backend types must match")
+            }
+            CacheBackendConfig::Gcs(config)
+                if service
+                    .gcs_backend()
+                    .is_none_or(|backend| backend.bucket_name() != config.bucket_name) =>
+            {
+                Some("facade and native backend buckets must match")
+            }
+            CacheBackendConfig::Gcs(config)
+                if service
+                    .gcs_backend()
+                    .is_none_or(|backend| backend.key_prefix() != config.key_prefix) =>
+            {
+                Some("facade and native backend key prefixes must match")
+            }
+            CacheBackendConfig::Gcs(config)
+                if service.gcs_backend().is_none_or(|backend| {
+                    backend.path_service_account() != config.path_service_account.as_deref()
+                }) =>
+            {
+                Some("facade and native backend credentials must match")
+            }
+            CacheBackendConfig::Gcs(_) => None,
         }
     }
 }
@@ -201,6 +242,23 @@ fn project_memory(backend: &Bound<'_, PyAny>) -> PyResult<MemoryCacheConfig> {
     })
 }
 
+#[inline(never)]
+fn project_gcs(
+    backend: &Bound<'_, PyAny>,
+) -> PyResult<Result<GcsCacheConfig, UnsupportedCacheConfig>> {
+    let bucket_name = match backend.getattr("bucket_name")?.extract::<Option<String>>() {
+        Ok(Some(bucket_name)) if !bucket_name.is_empty() => bucket_name,
+        _ => return Ok(Err(UnsupportedCacheConfig::GcsBucket)),
+    };
+    Ok(Ok(GcsCacheConfig {
+        bucket_name,
+        key_prefix: backend.getattr("key_prefix")?.extract::<String>()?,
+        path_service_account: backend
+            .getattr("path_service_account")?
+            .extract::<Option<String>>()?,
+    }))
+}
+
 #[inline(never)]
 fn project_redis(
     backend: &Bound<'_, PyAny>,
@@ -468,8 +526,8 @@ mod tests {
     use pyo3::{prelude::*, types::PyDict};
 
     use super::{
-        CacheBackendConfig, CacheConfigProjection, CertificateRequirement, NativeCacheConfig,
-        RedisProtocol,
+        CacheBackendConfig, CacheConfigProjection, CertificateRequirement, GcsCacheConfig,
+        NativeCacheConfig, RedisProtocol, UnsupportedCacheConfig,
     };
     use crate::cache::native::NativeResponseCache;
 
@@ -531,6 +589,71 @@ mod tests {
         });
     }
 
+    #[test]
+    fn projects_gcs_configuration() {
+        Python::initialize();
+        Python::attach(|py| {
+            let facade = facade(
+                py,
+                "backend = SimpleNamespace(bucket_name='bucket', key_prefix='cache/', path_service_account='credentials.json')\n\
+                 facade = SimpleNamespace(type='gcs', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)",
+            );
+            let CacheConfigProjection::Native(config) =
+                NativeCacheConfig::project(&facade).unwrap()
+            else {
+                panic!("GCS cache should be supported");
+            };
+            let CacheBackendConfig::Gcs(gcs) = config.backend else {
+                panic!("expected GCS configuration");
+            };
+            assert_eq!(
+                gcs,
+                GcsCacheConfig {
+                    bucket_name: "bucket".into(),
+                    key_prefix: "cache/".into(),
+                    path_service_account: Some("credentials.json".into()),
+                }
+            );
+            let matching = NativeResponseCache::gcs(
+                litellm_cache_gcs::GcsConfig {
+                    bucket_name: "bucket".into(),
+                    gcs_path: Some("cache/".into()),
+                    path_service_account: Some("credentials.json".into()),
+                    endpoint: litellm_cache_gcs::DEFAULT_ENDPOINT.into(),
+                },
+                Some("token".into()),
+            )
+            .unwrap();
+            let matching_config = NativeCacheConfig {
+                policy: config.policy,
+                backend: CacheBackendConfig::Gcs(gcs),
+            };
+            assert_eq!(matching_config.service_mismatch(&matching), None);
+        });
+    }
+
+    #[test]
+    fn rejects_gcs_without_a_bucket_name() {
+        Python::initialize();
+        Python::attach(|py| {
+            let facade = facade(
+                py,
+                "backend = SimpleNamespace(bucket_name=None, key_prefix='', path_service_account=None)\n\
+                 facade = SimpleNamespace(type='gcs', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)",
+            );
+            let CacheConfigProjection::Unsupported(reason) =
+                NativeCacheConfig::project(&facade).unwrap()
+            else {
+                panic!("GCS cache without a bucket should be unsupported");
+            };
+            assert!(matches!(&reason, UnsupportedCacheConfig::GcsBucket));
+            assert_eq!(
+                reason.message(),
+                "native GCS cache requires a configured bucket name"
+            );
+        });
+    }
+
     #[test]
     fn projects_resolved_redis_tls_configuration() {
         Python::initialize();
diff --git a/litellm-rust/crates/python-bridge/src/cache/facade.rs b/litellm-rust/crates/python-bridge/src/cache/facade.rs
index f2f86c14b37..6aa94754e2d 100644
--- a/litellm-rust/crates/python-bridge/src/cache/facade.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/facade.rs
@@ -192,6 +192,7 @@ impl FacadeGuard {
         let (module, name, cache_kind) = match kind {
             "memory" => ("litellm.caching.in_memory_cache", "InMemoryCache", "local"),
             "redis" => ("litellm.caching.redis_cache", "RedisCache", "redis"),
+            "gcs" => ("litellm.caching.gcs_cache", "GCSCache", "gcs"),
             _ => unreachable!(),
         };
         let backend = facade.getattr("cache")?;
@@ -235,6 +236,9 @@ impl FacadeGuard {
                     "max_size_per_item",
                     "redis_kwargs",
                     "redis_flush_size",
+                    "bucket_name",
+                    "key_prefix",
+                    "path_service_account",
                 ],
             )?,
             redis_pool: (kind == "redis")
diff --git a/litellm-rust/crates/python-bridge/src/cache/handle.rs b/litellm-rust/crates/python-bridge/src/cache/handle.rs
index 8251b3df06c..87d9300b08b 100644
--- a/litellm-rust/crates/python-bridge/src/cache/handle.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/handle.rs
@@ -1,6 +1,8 @@
 use litellm_host_python::release_gil;
 use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*};
 
+use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig};
+
 use super::{cache_error, facade::FacadeGuard, native::NativeResponseCache, request::duration};
 
 #[pyclass(frozen, name = "_CacheTestHandle")]
@@ -51,6 +53,31 @@ impl CacheTestHandle {
         })
     }
 
+    #[staticmethod]
+    #[pyo3(signature = (bucket_name, *, gcs_path=None, path_service_account=None, endpoint=None, token=None))]
+    fn gcs(
+        py: Python<'_>,
+        bucket_name: String,
+        gcs_path: Option<String>,
+        path_service_account: Option<String>,
+        endpoint: Option<String>,
+        token: Option<String>,
+    ) -> PyResult<Self> {
+        let config = GcsConfig {
+            bucket_name,
+            gcs_path,
+            path_service_account,
+            endpoint: endpoint.unwrap_or_else(|| DEFAULT_ENDPOINT.to_string()),
+        };
+        let service = release_gil(py, move || NativeResponseCache::gcs(config, token))
+            .map_err(cache_error)?;
+        Ok(Self {
+            service,
+            guard: None,
+            pid: std::process::id(),
+        })
+    }
+
     #[getter]
     fn backend(&self) -> &'static str {
         self.service.kind()
diff --git a/litellm-rust/crates/python-bridge/src/cache/mod.rs b/litellm-rust/crates/python-bridge/src/cache/mod.rs
index aec08610f6e..ac4494150d9 100644
--- a/litellm-rust/crates/python-bridge/src/cache/mod.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/mod.rs
@@ -10,7 +10,7 @@ mod resolver;
 
 use litellm_cache::Error;
 use pyo3::{
-    exceptions::{PyRuntimeError, PyValueError},
+    exceptions::{PyNotImplementedError, PyRuntimeError, PyValueError},
     prelude::*,
 };
 
@@ -21,6 +21,7 @@ pub(crate) use self::{
 fn cache_error(error: Error) -> PyErr {
     match error {
         Error::InvalidEntry => PyValueError::new_err(error.to_string()),
+        Error::UnsupportedOperation => PyNotImplementedError::new_err(error.to_string()),
         _ => PyRuntimeError::new_err(error.to_string()),
     }
 }
diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs
index a9475429e45..9a030ac0434 100644
--- a/litellm-rust/crates/python-bridge/src/cache/native.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/native.rs
@@ -1,6 +1,7 @@
 use std::{sync::Arc, time::Duration};
 
 use litellm_cache::{CacheCodec, CacheConnectionResult, Error};
+use litellm_cache_gcs::{GcsCache, GcsConfig, StaticTokenSource};
 use litellm_cache_memory::InMemoryCache;
 use litellm_cache_redis::RedisCache;
 use litellm_cache_response::{
@@ -15,6 +16,7 @@ pub(super) enum NativeResponseCache {
         cache: Arc<ResponseCache<RedisCache<ResponseCacheCodec>>>,
         buffer: Option<Arc<WriteBuffer>>,
     },
+    Gcs(Arc<ResponseCache<GcsCache<ResponseCacheCodec>>>),
 }
 
 impl NativeResponseCache {
@@ -43,6 +45,18 @@ impl NativeResponseCache {
             buffer: None,
         })
     }
+
+    pub fn gcs(config: GcsConfig, token: Option<String>) -> Result<Self, Error> {
+        let backend = match token {
+            Some(token) => GcsCache::with_token_source(
+                config,
+                ResponseCacheCodec,
+                Arc::new(StaticTokenSource(token)),
+            )?,
+            None => GcsCache::new(config, ResponseCacheCodec)?,
+        };
+        Ok(Self::Gcs(Arc::new(ResponseCache::new(Arc::new(backend)))))
+    }
 }
 
 impl NativeResponseCache {
@@ -50,6 +64,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(_) => "memory",
             Self::Redis { .. } => "redis",
+            Self::Gcs(_) => "gcs",
         }
     }
 
@@ -57,6 +72,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.default_ttl(),
             Self::Redis { cache, .. } => cache.default_ttl(),
+            Self::Gcs(cache) => cache.default_ttl(),
         }
     }
 
@@ -64,6 +80,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(_) => None,
             Self::Redis { cache, .. } => cache.backend().namespace(),
+            Self::Gcs(_) => None,
         }
     }
 
@@ -71,6 +88,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => Some(cache.backend().max_size_in_memory()),
             Self::Redis { .. } => None,
+            Self::Gcs(_) => None,
         }
     }
 
@@ -78,6 +96,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.backend().max_entry_bytes(),
             Self::Redis { .. } => None,
+            Self::Gcs(_) => None,
         }
     }
 
@@ -99,6 +118,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.lookup(request, now),
             Self::Redis { cache, .. } => cache.lookup(request, now),
+            Self::Gcs(cache) => cache.lookup(request, now),
         }
     }
 
@@ -111,6 +131,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.store(request, response, now),
             Self::Redis { cache, .. } => cache.store(request, response, now),
+            Self::Gcs(cache) => cache.store(request, response, now),
         }
     }
 
@@ -122,6 +143,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.lookup_batch(requests, now),
             Self::Redis { cache, .. } => cache.lookup_batch(requests, now),
+            Self::Gcs(cache) => cache.lookup_batch(requests, now),
         }
     }
 
@@ -133,6 +155,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.async_lookup(request, now).await,
             Self::Redis { cache, .. } => cache.async_lookup(request, now).await,
+            Self::Gcs(cache) => cache.async_lookup(request, now).await,
         }
     }
 
@@ -152,6 +175,7 @@ impl NativeResponseCache {
                 cache,
                 buffer: Some(buffer),
             } => buffer.async_store(cache, request, response, now).await,
+            Self::Gcs(cache) => cache.async_store(request, response, now).await,
         }
     }
 
@@ -163,6 +187,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.async_lookup_batch(requests, now).await,
             Self::Redis { cache, .. } => cache.async_lookup_batch(requests, now).await,
+            Self::Gcs(cache) => cache.async_lookup_batch(requests, now).await,
         }
     }
 
@@ -174,6 +199,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.async_store_batch(entries, now).await,
             Self::Redis { cache, .. } => cache.async_store_batch(entries, now).await,
+            Self::Gcs(cache) => cache.async_store_batch(entries, now).await,
         }
     }
 
@@ -186,6 +212,7 @@ impl NativeResponseCache {
                 }
                 cache.async_flush().await
             }
+            Self::Gcs(cache) => cache.async_flush().await,
         }
     }
 
@@ -193,6 +220,14 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.test_connection().await,
             Self::Redis { cache, .. } => cache.test_connection().await,
+            Self::Gcs(cache) => cache.test_connection().await,
+        }
+    }
+
+    pub fn gcs_backend(&self) -> Option<&GcsCache<ResponseCacheCodec>> {
+        match self {
+            Self::Gcs(cache) => Some(cache.backend()),
+            _ => None,
         }
     }
 }

From beba2576be4b015574dccb3f916af5dfaf5d189f Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:29:06 +0000
Subject: [PATCH 069/160] fix(rust): align vault namespace handling with python
 and drop unused settings hook

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../crates/secrets-hashicorp/src/config.rs    | 10 +-----
 .../secrets-hashicorp/src/secret_manager.rs   | 31 ++++++++---------
 .../secrets-hashicorp/tests/secret_manager.rs | 33 +++++++++++++++++++
 .../hashicorp_secret_manager.py               | 16 ++++-----
 .../hashicorp_vault_parity.json               | 20 +++--------
 5 files changed, 58 insertions(+), 52 deletions(-)

diff --git a/litellm-rust/crates/secrets-hashicorp/src/config.rs b/litellm-rust/crates/secrets-hashicorp/src/config.rs
index f9491f71afb..d32491199c4 100644
--- a/litellm-rust/crates/secrets-hashicorp/src/config.rs
+++ b/litellm-rust/crates/secrets-hashicorp/src/config.rs
@@ -1,7 +1,6 @@
 use std::{path::PathBuf, time::Duration};
 
 use litellm_core_utils::settings::Lookup;
-use litellm_secrets_types::KeyManagementSettings;
 use litellm_secrets_types::SecretValue;
 
 use crate::Error;
@@ -117,13 +116,6 @@ impl HashicorpVaultConfig {
         })
     }
 
-    pub fn from_settings(
-        _settings: &KeyManagementSettings,
-        environment: &dyn Lookup,
-    ) -> Result<Self, Error> {
-        Self::from_environment(environment)
-    }
-
     pub fn login_namespace(&self) -> Option<&str> {
         self.login_namespace
             .as_deref()
@@ -163,7 +155,7 @@ fn refresh_interval(environment: &dyn Lookup) -> Result<Duration, Error> {
     };
     let seconds: i64 = value.parse().map_err(|_| Error::RefreshInterval)?;
     if seconds < 0 {
-        return Ok(Duration::from_nanos(1));
+        return Err(Error::RefreshInterval);
     }
     Ok(Duration::from_secs(seconds as u64))
 }
diff --git a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs
index 6b887289443..e4ca8c387c3 100644
--- a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs
+++ b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs
@@ -120,14 +120,12 @@ impl HashicorpVault {
             return Ok(Some(value));
         }
         let token: SecretValue = self.vault_token().await?;
-        let mut request = self
+        let response: reqwest::Response = self
             .client
             .get(&url)
-            .header("X-Vault-Token", token.expose());
-        if let Some(namespace) = self.config.secret_namespace() {
-            request = request.header("X-Vault-Namespace", namespace);
-        }
-        let response: reqwest::Response = request.send().await?;
+            .header("X-Vault-Token", token.expose())
+            .send()
+            .await?;
         if response.status() == reqwest::StatusCode::NOT_FOUND {
             return Ok(None);
         }
@@ -166,14 +164,13 @@ impl HashicorpVault {
             None => json!({"key": value.expose()}),
         };
         let token: SecretValue = self.vault_token().await?;
-        let mut request = self
+        let response: reqwest::Response = self
             .client
             .post(&url)
-            .header("X-Vault-Token", token.expose());
-        if let Some(namespace) = self.config.secret_namespace() {
-            request = request.header("X-Vault-Namespace", namespace);
-        }
-        let response: reqwest::Response = request.json(&json!({"data": data})).send().await?;
+            .header("X-Vault-Token", token.expose())
+            .json(&json!({"data": data}))
+            .send()
+            .await?;
         if !response.status().is_success() {
             return Err(Error::Status {
                 status: response.status().as_u16(),
@@ -190,14 +187,12 @@ impl HashicorpVault {
     pub async fn async_delete_secret(&self, secret_name: &str) -> Result<(), Error> {
         let url: String = self.secret_url(secret_name)?;
         let token: SecretValue = self.vault_token().await?;
-        let mut request = self
+        let response: reqwest::Response = self
             .client
             .delete(&url)
-            .header("X-Vault-Token", token.expose());
-        if let Some(namespace) = self.config.secret_namespace() {
-            request = request.header("X-Vault-Namespace", namespace);
-        }
-        let response: reqwest::Response = request.send().await?;
+            .header("X-Vault-Token", token.expose())
+            .send()
+            .await?;
         if !response.status().is_success() {
             return Err(Error::Status {
                 status: response.status().as_u16(),
diff --git a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs
index 1cf13a79c77..b7a81f49ad4 100644
--- a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs
+++ b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs
@@ -87,6 +87,39 @@ async fn namespace_mount_and_prefix_are_sanitized_in_the_url() {
     assert!(manager.async_read_secret("name").await.unwrap().is_some());
 }
 
+#[test]
+fn trailing_address_slashes_are_removed() {
+    let environment: Arc<dyn Lookup + Send + Sync> = Arc::new(|name: &str| match name {
+        "HCP_VAULT_ADDR" => Some("http://vault.test:8200///".to_owned()),
+        "HCP_VAULT_TOKEN" => Some("token".to_owned()),
+        _ => None,
+    });
+    let config: HashicorpVaultConfig =
+        HashicorpVaultConfig::from_environment(environment.as_ref()).unwrap();
+    let manager: HashicorpVault =
+        HashicorpVault::with_client(reqwest::Client::new(), config, true).unwrap();
+
+    assert_eq!(
+        manager.secret_url("name").unwrap(),
+        "http://vault.test:8200/v1/secret/data/name"
+    );
+}
+
+#[rstest::rstest]
+#[case("-1")]
+#[case("not-a-number")]
+fn invalid_refresh_intervals_are_rejected(#[case] value: &str) {
+    let environment: Arc<dyn Lookup + Send + Sync> = Arc::new(move |name: &str| match name {
+        "HCP_VAULT_REFRESH_INTERVAL" => Some(value.to_owned()),
+        _ => None,
+    });
+
+    assert!(matches!(
+        HashicorpVaultConfig::from_environment(environment.as_ref()),
+        Err(Error::RefreshInterval)
+    ));
+}
+
 #[tokio::test]
 async fn approle_login_uses_namespace_and_reuses_the_token() {
     let server: MockServer = MockServer::start().await;
diff --git a/litellm/secret_managers/hashicorp_secret_manager.py b/litellm/secret_managers/hashicorp_secret_manager.py
index 259b2416d7f..e37a912c7e1 100644
--- a/litellm/secret_managers/hashicorp_secret_manager.py
+++ b/litellm/secret_managers/hashicorp_secret_manager.py
@@ -95,16 +95,16 @@ class HashicorpSecretManager(BaseSecretManager):
         from litellm.proxy.proxy_server import CommonProxyErrors, premium_user
 
         # Vault-specific config
-        self.vault_addr = (os.getenv("HCP_VAULT_ADDR") or "http://127.0.0.1:8200").rstrip("/")
+        self.vault_addr = os.getenv("HCP_VAULT_ADDR", "http://127.0.0.1:8200")
         self.vault_token = os.getenv("HCP_VAULT_TOKEN", "")
-        self.vault_namespace = self._sanitize_path_component(os.getenv("HCP_VAULT_NAMESPACE"))
-        self.login_namespace_override = self._sanitize_path_component(os.getenv("HCP_VAULT_LOGIN_NAMESPACE"))
-        self.secret_namespace_override = self._sanitize_path_component(os.getenv("HCP_VAULT_SECRET_NAMESPACE"))
+        self.vault_namespace = os.getenv("HCP_VAULT_NAMESPACE", None)
+        self.login_namespace_override = os.getenv("HCP_VAULT_LOGIN_NAMESPACE", None)
+        self.secret_namespace_override = os.getenv("HCP_VAULT_SECRET_NAMESPACE", None)
         # KV engine mount name (default: "secret")
         # If your KV engine is mounted somewhere other than "secret", set HCP_VAULT_MOUNT_NAME
-        self.vault_mount_name = self._sanitize_path_component(os.getenv("HCP_VAULT_MOUNT_NAME")) or "secret"
+        self.vault_mount_name = os.getenv("HCP_VAULT_MOUNT_NAME", "secret")
         # Optional path prefix for secrets (e.g., "myapp" -> secret/data/myapp/{secret_name})
-        self.vault_path_prefix = self._sanitize_path_component(os.getenv("HCP_VAULT_PATH_PREFIX"))
+        self.vault_path_prefix = os.getenv("HCP_VAULT_PATH_PREFIX", None)
 
         # Optional config for TLS cert auth
         self.tls_cert_path = os.getenv("HCP_VAULT_CLIENT_CERT", "")
@@ -114,9 +114,7 @@ class HashicorpSecretManager(BaseSecretManager):
         # Optional config for AppRole auth
         self.approle_role_id = os.getenv("HCP_VAULT_APPROLE_ROLE_ID", "")
         self.approle_secret_id = os.getenv("HCP_VAULT_APPROLE_SECRET_ID", "")
-        self.approle_mount_path = self._sanitize_path_component(
-            os.getenv("HCP_VAULT_APPROLE_MOUNT_PATH")
-        ) or "approle"
+        self.approle_mount_path = os.getenv("HCP_VAULT_APPROLE_MOUNT_PATH", "approle")
 
         self._verify_required_credentials_exist()
 
diff --git a/tests/test_litellm/secret_managers/hashicorp_vault_parity.json b/tests/test_litellm/secret_managers/hashicorp_vault_parity.json
index 5f28d9602a0..f1faefd48e8 100644
--- a/tests/test_litellm/secret_managers/hashicorp_vault_parity.json
+++ b/tests/test_litellm/secret_managers/hashicorp_vault_parity.json
@@ -15,7 +15,7 @@
     "env": {
       "HCP_VAULT_ADDR": "http://vault.test:8200",
       "HCP_VAULT_TOKEN": "token",
-      "HCP_VAULT_NAMESPACE": " admin/ "
+      "HCP_VAULT_NAMESPACE": "admin"
     },
     "secret_name": "OPENAI_API_KEY",
     "expected_secret_url": "http://vault.test:8200/v1/admin/secret/data/OPENAI_API_KEY",
@@ -29,8 +29,8 @@
       "HCP_VAULT_ADDR": "http://vault.test:8200",
       "HCP_VAULT_TOKEN": "token",
       "HCP_VAULT_NAMESPACE": "admin",
-      "HCP_VAULT_LOGIN_NAMESPACE": " /root/ ",
-      "HCP_VAULT_SECRET_NAMESPACE": " /teams/team-a/ "
+      "HCP_VAULT_LOGIN_NAMESPACE": "root",
+      "HCP_VAULT_SECRET_NAMESPACE": "teams/team-a"
     },
     "secret_name": "OPENAI_API_KEY",
     "expected_secret_url": "http://vault.test:8200/v1/teams/team-a/secret/data/OPENAI_API_KEY",
@@ -58,7 +58,7 @@
       "HCP_VAULT_ADDR": "http://vault.test:8200",
       "HCP_VAULT_APPROLE_ROLE_ID": "role-id",
       "HCP_VAULT_APPROLE_SECRET_ID": "secret-id",
-      "HCP_VAULT_APPROLE_MOUNT_PATH": " /custom-approle/ ",
+      "HCP_VAULT_APPROLE_MOUNT_PATH": "custom-approle",
       "HCP_VAULT_NAMESPACE": "admin"
     },
     "secret_name": "OPENAI_API_KEY",
@@ -81,17 +81,5 @@
     "expected_login_url": "http://vault.test:8200/v1/auth/cert/login",
     "expected_login_namespace": "admin",
     "expected_secret_namespace": "admin"
-  },
-  {
-    "name": "trailing_address_slash",
-    "env": {
-      "HCP_VAULT_ADDR": "http://vault.test:8200///",
-      "HCP_VAULT_TOKEN": "token"
-    },
-    "secret_name": "OPENAI_API_KEY",
-    "expected_secret_url": "http://vault.test:8200/v1/secret/data/OPENAI_API_KEY",
-    "expected_login_url": null,
-    "expected_login_namespace": null,
-    "expected_secret_namespace": null
   }
 ]

From 6ab121a3e7364178544bcbee529f6c57b3115a0a Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:29:49 +0000
Subject: [PATCH 070/160] feat(cache-redis-semantic): add native Redis Semantic
 cache backend

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.lock                       |  15 +
 litellm-rust/Cargo.toml                       |   1 +
 .../crates/cache-redis-semantic/Cargo.toml    |  21 +
 .../crates/cache-redis-semantic/src/cache.rs  | 599 ++++++++++++++
 .../crates/cache-redis-semantic/src/lib.rs    |   4 +
 .../crates/cache-redis-semantic/src/prompt.rs |  95 +++
 .../cache-redis-semantic/tests/cache.rs       | 751 ++++++++++++++++++
 litellm-rust/crates/cache/tests/caching.rs    |   9 +-
 8 files changed, 1489 insertions(+), 6 deletions(-)
 create mode 100644 litellm-rust/crates/cache-redis-semantic/Cargo.toml
 create mode 100644 litellm-rust/crates/cache-redis-semantic/src/cache.rs
 create mode 100644 litellm-rust/crates/cache-redis-semantic/src/lib.rs
 create mode 100644 litellm-rust/crates/cache-redis-semantic/src/prompt.rs
 create mode 100644 litellm-rust/crates/cache-redis-semantic/tests/cache.rs

diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index ed4ae4e3353..e911d0d9c45 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -2486,6 +2486,21 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "litellm-cache-redis-semantic"
+version = "0.1.0"
+dependencies = [
+ "litellm-cache",
+ "litellm-cache-redis",
+ "litellm-cache-response",
+ "r2d2",
+ "redis",
+ "redis-test",
+ "serde_json",
+ "sha2 0.10.9",
+ "tokio",
+]
+
 [[package]]
 name = "litellm-cache-response"
 version = "0.1.0"
diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
index 570d0dd3568..05eea6bc299 100644
--- a/litellm-rust/Cargo.toml
+++ b/litellm-rust/Cargo.toml
@@ -29,6 +29,7 @@ litellm-core-utils = { path = "crates/core-utils" }
 litellm-cache = { path = "crates/cache" }
 litellm-cache-memory = { path = "crates/cache-memory" }
 litellm-cache-redis = { path = "crates/cache-redis" }
+litellm-cache-redis-semantic = { path = "crates/cache-redis-semantic" }
 litellm-cache-response = { path = "crates/cache-response" }
 litellm-token-counter = { path = "crates/token-counter" }
 litellm-token-counter-fast = { path = "crates/token-counter-fast" }
diff --git a/litellm-rust/crates/cache-redis-semantic/Cargo.toml b/litellm-rust/crates/cache-redis-semantic/Cargo.toml
new file mode 100644
index 00000000000..9a8755a189e
--- /dev/null
+++ b/litellm-rust/crates/cache-redis-semantic/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "litellm-cache-redis-semantic"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-cache.workspace = true
+litellm-cache-redis.workspace = true
+litellm-cache-response.workspace = true
+redis = { version = "1.7.0", features = ["tls-rustls"] }
+r2d2 = "0.8.10"
+serde_json.workspace = true
+sha2.workspace = true
+tokio.workspace = true
+
+[dev-dependencies]
+redis-test = "1.0.4"
+serde_json.workspace = true
+tokio.workspace = true
diff --git a/litellm-rust/crates/cache-redis-semantic/src/cache.rs b/litellm-rust/crates/cache-redis-semantic/src/cache.rs
new file mode 100644
index 00000000000..26d19d34670
--- /dev/null
+++ b/litellm-rust/crates/cache-redis-semantic/src/cache.rs
@@ -0,0 +1,599 @@
+use std::{
+    future::Future,
+    sync::{Arc, OnceLock},
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+
+use litellm_cache::{
+    BaseCache, CacheCodec, CacheConnectionResult, CacheConnectionStatus, Error,
+    SemanticCacheContext,
+};
+use litellm_cache_redis::connection::{ConnectionRef, Connections, ttl_seconds};
+use litellm_cache_response::{CacheEntry, ResponseCacheCodec};
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+
+use crate::prompt::prompt_from_context;
+
+const REDIS_TIMEOUT: Duration = Duration::from_secs(5);
+const REDIS_POOL_SIZE: u32 = 16;
+const DEFAULT_INDEX_NAME: &str = "litellm_semantic_cache_index";
+const CACHE_KEY_FIELD: &str = "litellm_cache_key";
+const VECTOR_FIELD: &str = "prompt_vector";
+
+pub trait Embedder: Send + Sync + 'static {
+    fn embed(
+        &self,
+        prompt: &str,
+        metadata: &serde_json::Map<String, Value>,
+    ) -> Result<Vec<f32>, Error>;
+
+    fn async_embed(
+        &self,
+        prompt: &str,
+        metadata: &serde_json::Map<String, Value>,
+    ) -> impl Future<Output = Result<Vec<f32>, Error>> + Send;
+}
+
+#[derive(Clone, Debug)]
+pub struct RedisSemanticConfig {
+    pub index_name: String,
+    pub similarity_threshold: f32,
+}
+
+impl Default for RedisSemanticConfig {
+    fn default() -> Self {
+        Self {
+            index_name: DEFAULT_INDEX_NAME.into(),
+            similarity_threshold: 0.9,
+        }
+    }
+}
+
+struct Inner {
+    index_name: String,
+    distance_threshold: f64,
+    resolved_index: OnceLock<String>,
+    codec: ResponseCacheCodec,
+    clock: fn() -> f64,
+}
+
+impl Inner {
+    fn new(config: RedisSemanticConfig) -> Self {
+        Self {
+            index_name: config.index_name,
+            distance_threshold: 1.0 - f64::from(config.similarity_threshold),
+            resolved_index: OnceLock::new(),
+            codec: ResponseCacheCodec,
+            clock: timestamp,
+        }
+    }
+
+    fn ensure_index(
+        &self,
+        connection: &mut ConnectionRef<'_>,
+        dims: usize,
+    ) -> Result<String, Error> {
+        if let Some(name) = self.resolved_index.get() {
+            return Ok(name.clone());
+        }
+        let name = match index_compatible(connection, &self.index_name, dims)? {
+            Some(true) => self.index_name.clone(),
+            Some(false) => self.isolated_index(connection, dims)?,
+            None => {
+                create_index(connection, &self.index_name, dims)?;
+                self.index_name.clone()
+            }
+        };
+        let _ = self.resolved_index.set(name.clone());
+        Ok(name)
+    }
+
+    fn isolated_index(
+        &self,
+        connection: &mut ConnectionRef<'_>,
+        dims: usize,
+    ) -> Result<String, Error> {
+        let name = format!("{}_isolated", self.index_name);
+        match index_compatible(connection, &name, dims)? {
+            Some(true) => Ok(name),
+            Some(false) => {
+                redis::cmd("FT.DROPINDEX")
+                    .arg(&name)
+                    .query::<()>(connection)
+                    .map_err(|_| Error::Unavailable)?;
+                create_index(connection, &name, dims)?;
+                Ok(name)
+            }
+            None => {
+                create_index(connection, &name, dims)?;
+                Ok(name)
+            }
+        }
+    }
+
+    fn store(
+        &self,
+        connection: &mut ConnectionRef<'_>,
+        tag: &str,
+        value: &CacheEntry,
+        prompt: &str,
+        vector: &[f32],
+        ttl: Option<Duration>,
+    ) -> Result<(), Error> {
+        let index = self.ensure_index(connection, vector.len())?;
+        let entry_id = entry_id(prompt, tag);
+        let hash_key = format!("{index}:{entry_id}");
+        let response = self.codec.encode(value)?;
+        redis::cmd("HSET")
+            .arg(&hash_key)
+            .arg("entry_id")
+            .arg(&entry_id)
+            .arg("prompt")
+            .arg(prompt)
+            .arg("response")
+            .arg(response)
+            .arg(VECTOR_FIELD)
+            .arg(vector_buffer(vector))
+            .arg("inserted_at")
+            .arg(format!("{}", (self.clock)()))
+            .arg("updated_at")
+            .arg(format!("{}", (self.clock)()))
+            .arg(CACHE_KEY_FIELD)
+            .arg(tag)
+            .query::<()>(connection)
+            .map_err(|_| Error::Unavailable)?;
+        if let Some(ttl) = ttl {
+            redis::cmd("EXPIRE")
+                .arg(&hash_key)
+                .arg(ttl_seconds(ttl))
+                .query::<()>(connection)
+                .map_err(|_| Error::Unavailable)?;
+        }
+        Ok(())
+    }
+
+    fn lookup(
+        &self,
+        connection: &mut ConnectionRef<'_>,
+        tag: &str,
+        vector: &[f32],
+    ) -> Result<Option<CacheEntry>, Error> {
+        let index = self.ensure_index(connection, vector.len())?;
+        let query = format!(
+            "(@{CACHE_KEY_FIELD}:{{{}}})=>[KNN 1 @{VECTOR_FIELD} $vector AS vector_distance]",
+            escape_tag(tag)
+        );
+        let result = redis::cmd("FT.SEARCH")
+            .arg(&index)
+            .arg(query)
+            .arg("RETURN")
+            .arg(8)
+            .arg("entry_id")
+            .arg("prompt")
+            .arg("response")
+            .arg("inserted_at")
+            .arg("updated_at")
+            .arg("metadata")
+            .arg(CACHE_KEY_FIELD)
+            .arg("vector_distance")
+            .arg("SORTBY")
+            .arg("vector_distance")
+            .arg("ASC")
+            .arg("DIALECT")
+            .arg(2)
+            .arg("LIMIT")
+            .arg(0)
+            .arg(1)
+            .arg("PARAMS")
+            .arg(2)
+            .arg("vector")
+            .arg(vector_buffer(vector))
+            .query::<redis::Value>(connection)
+            .map_err(|_| Error::Unavailable)?;
+        let Some(fields) = first_document(&result) else {
+            return Ok(None);
+        };
+        if string_field(fields, CACHE_KEY_FIELD).as_deref() != Some(tag) {
+            return Ok(None);
+        }
+        if number_field(fields, "vector_distance")
+            .is_none_or(|distance| distance > self.distance_threshold)
+        {
+            return Ok(None);
+        }
+        let Some(response) = bytes_field(fields, "response") else {
+            return Ok(None);
+        };
+        self.codec.decode(&response).map(Some)
+    }
+}
+
+pub struct RedisSemanticCache<E: Embedder, C = redis::Connection> {
+    connections: Arc<Connections<C>>,
+    embedder: E,
+    inner: Arc<Inner>,
+}
+
+impl<E: Embedder> RedisSemanticCache<E> {
+    pub fn new(url: &str, embedder: E, config: RedisSemanticConfig) -> Result<Self, Error> {
+        Ok(Self {
+            connections: Arc::new(Connections::pooled(url, REDIS_TIMEOUT, REDIS_POOL_SIZE)?),
+            embedder,
+            inner: Arc::new(Inner::new(config)),
+        })
+    }
+}
+
+impl<E: Embedder, C: redis::ConnectionLike + Send + 'static> RedisSemanticCache<E, C> {
+    pub fn with_connection(connection: C, embedder: E, config: RedisSemanticConfig) -> Self {
+        Self {
+            connections: Arc::new(Connections::fixed(connection)),
+            embedder,
+            inner: Arc::new(Inner::new(config)),
+        }
+    }
+
+    pub fn with_clock(self, clock: fn() -> f64) -> Self {
+        Self {
+            inner: Arc::new(Inner {
+                index_name: self.inner.index_name.clone(),
+                distance_threshold: self.inner.distance_threshold,
+                resolved_index: OnceLock::new(),
+                codec: self.inner.codec,
+                clock,
+            }),
+            ..self
+        }
+    }
+
+    fn tag<'a>(key: &'a str, context: &'a SemanticCacheContext) -> &'a str {
+        context.scope.as_deref().unwrap_or(key)
+    }
+}
+
+impl<E: Embedder, C: redis::ConnectionLike + Send + 'static> BaseCache
+    for RedisSemanticCache<E, C>
+{
+    type Value = CacheEntry;
+    type Context = SemanticCacheContext;
+
+    fn get_ttl(&self, context: &Self::Context) -> Option<Duration> {
+        context.ttl
+    }
+
+    fn set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: &Self::Context,
+    ) -> Result<(), Error> {
+        let Some(prompt) = prompt_from_context(context) else {
+            return Ok(());
+        };
+        let vector = self.embedder.embed(&prompt, &context.metadata)?;
+        let tag = Self::tag(key, context).to_string();
+        self.connections.execute(|connection| {
+            self.inner
+                .store(connection, &tag, &value, &prompt, &vector, context.ttl)
+        })
+    }
+
+    fn get_cache(&self, key: &str, context: &Self::Context) -> Result<Option<Self::Value>, Error> {
+        let Some(prompt) = prompt_from_context(context) else {
+            return Ok(None);
+        };
+        let vector = self.embedder.embed(&prompt, &context.metadata)?;
+        let tag = Self::tag(key, context).to_string();
+        self.connections
+            .execute(|connection| self.inner.lookup(connection, &tag, &vector))
+    }
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: Self::Context,
+    ) -> Result<(), Error> {
+        let Some(prompt) = prompt_from_context(&context) else {
+            return Ok(());
+        };
+        let vector = self
+            .embedder
+            .async_embed(&prompt, &context.metadata)
+            .await?;
+        let tag = Self::tag(key, &context).to_string();
+        let inner = Arc::clone(&self.inner);
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            inner.store(connection, &tag, &value, &prompt, &vector, context.ttl)
+        })
+        .await
+    }
+
+    async fn async_get_cache(
+        &self,
+        key: &str,
+        context: &Self::Context,
+    ) -> Result<Option<Self::Value>, Error> {
+        let Some(prompt) = prompt_from_context(context) else {
+            return Ok(None);
+        };
+        let vector = self
+            .embedder
+            .async_embed(&prompt, &context.metadata)
+            .await?;
+        let tag = Self::tag(key, context).to_string();
+        let inner = Arc::clone(&self.inner);
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            inner.lookup(connection, &tag, &vector)
+        })
+        .await
+    }
+
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        match Connections::run_blocking(Arc::clone(&self.connections), |connection| {
+            Ok(match redis::cmd("PING").query::<String>(connection) {
+                Ok(_) => CacheConnectionResult {
+                    status: CacheConnectionStatus::Success,
+                    message: "Redis cache connection test successful".into(),
+                    error: None,
+                },
+                Err(error) => CacheConnectionResult {
+                    status: CacheConnectionStatus::Failed,
+                    message: format!("Redis connection failed: {error}"),
+                    error: Some(error.to_string()),
+                },
+            })
+        })
+        .await
+        {
+            Ok(result) => Ok(result),
+            Err(error) => Ok(CacheConnectionResult {
+                status: CacheConnectionStatus::Failed,
+                message: format!("Redis connection failed: {error}"),
+                error: Some(error.to_string()),
+            }),
+        }
+    }
+}
+
+fn timestamp() -> f64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|duration| duration.as_secs_f64())
+        .unwrap_or_default()
+}
+
+fn entry_id(prompt: &str, tag: &str) -> String {
+    let mut digest = Sha256::new();
+    digest.update(prompt.as_bytes());
+    digest.update(CACHE_KEY_FIELD.as_bytes());
+    digest.update(tag.as_bytes());
+    format!("{:x}", digest.finalize())
+}
+
+fn vector_buffer(vector: &[f32]) -> Vec<u8> {
+    vector
+        .iter()
+        .flat_map(|component| component.to_le_bytes())
+        .collect()
+}
+
+fn escape_tag(value: &str) -> String {
+    value
+        .chars()
+        .flat_map(|ch| {
+            if matches!(
+                ch,
+                ',' | '.'
+                    | '<'
+                    | '>'
+                    | '{'
+                    | '}'
+                    | '['
+                    | ']'
+                    | '\\'
+                    | '"'
+                    | '\''
+                    | ':'
+                    | ';'
+                    | '!'
+                    | '@'
+                    | '#'
+                    | '$'
+                    | '%'
+                    | '^'
+                    | '&'
+                    | '*'
+                    | '('
+                    | ')'
+                    | '-'
+                    | '+'
+                    | '='
+                    | '~'
+                    | '|'
+                    | '/'
+                    | ' '
+                    | '?'
+            ) {
+                vec!['\\', ch]
+            } else {
+                vec![ch]
+            }
+        })
+        .collect()
+}
+
+fn create_index(connection: &mut ConnectionRef<'_>, name: &str, dims: usize) -> Result<(), Error> {
+    redis::cmd("FT.CREATE")
+        .arg(name)
+        .arg("ON")
+        .arg("HASH")
+        .arg("PREFIX")
+        .arg(1)
+        .arg(name)
+        .arg("SCORE")
+        .arg(1.0)
+        .arg("SCHEMA")
+        .arg("prompt")
+        .arg("TEXT")
+        .arg("WEIGHT")
+        .arg(1)
+        .arg("response")
+        .arg("TEXT")
+        .arg("WEIGHT")
+        .arg(1)
+        .arg("inserted_at")
+        .arg("NUMERIC")
+        .arg("updated_at")
+        .arg("NUMERIC")
+        .arg(VECTOR_FIELD)
+        .arg("VECTOR")
+        .arg("FLAT")
+        .arg(6)
+        .arg("TYPE")
+        .arg("FLOAT32")
+        .arg("DIM")
+        .arg(dims)
+        .arg("DISTANCE_METRIC")
+        .arg("COSINE")
+        .arg(CACHE_KEY_FIELD)
+        .arg("TAG")
+        .arg("SEPARATOR")
+        .arg(",")
+        .query::<()>(connection)
+        .map_err(|_| Error::Unavailable)
+}
+
+fn index_compatible(
+    connection: &mut ConnectionRef<'_>,
+    name: &str,
+    dims: usize,
+) -> Result<Option<bool>, Error> {
+    let info = match redis::cmd("FT.INFO")
+        .arg(name)
+        .query::<redis::Value>(connection)
+    {
+        Ok(info) => info,
+        Err(error) if unknown_index(&error) => return Ok(None),
+        Err(_) => return Err(Error::Unavailable),
+    };
+    Ok(Some(schema_compatible(&info, dims)))
+}
+
+fn unknown_index(error: &redis::RedisError) -> bool {
+    let message = error.to_string().to_lowercase();
+    message.contains("unknown") && message.contains("index")
+}
+
+fn schema_compatible(info: &redis::Value, dims: usize) -> bool {
+    let redis::Value::Array(entries) = info else {
+        return false;
+    };
+    let attributes = entries
+        .as_chunks::<2>()
+        .0
+        .iter()
+        .find(|pair| string_value(&pair[0]).as_deref() == Some("attributes"))
+        .map(|pair| &pair[1]);
+    let Some(redis::Value::Array(attributes)) = attributes else {
+        return false;
+    };
+    let fields = attributes
+        .iter()
+        .map(|attribute| {
+            let redis::Value::Array(attribute) = attribute else {
+                return (None, None, None);
+            };
+            let mut name = None;
+            let mut field_type = None;
+            let mut dim = None;
+            for pair in attribute.as_chunks::<2>().0 {
+                match string_value(&pair[0]).as_deref() {
+                    Some("identifier") => name = string_value(&pair[1]),
+                    Some("type") => field_type = string_value(&pair[1]),
+                    Some("dim") => dim = number_value(&pair[1]),
+                    _ => {}
+                }
+            }
+            (name, field_type, dim)
+        })
+        .collect::<Vec<_>>();
+    let has_field = |name: &str, field_type: &str| {
+        fields
+            .iter()
+            .any(|(n, t, _)| n.as_deref() == Some(name) && t.as_deref() == Some(field_type))
+    };
+    has_field("prompt", "TEXT")
+        && has_field("response", "TEXT")
+        && has_field("inserted_at", "NUMERIC")
+        && has_field("updated_at", "NUMERIC")
+        && has_field(CACHE_KEY_FIELD, "TAG")
+        && fields.iter().any(|(n, t, d)| {
+            n.as_deref() == Some(VECTOR_FIELD)
+                && t.as_deref() == Some("VECTOR")
+                && *d == Some(dims as f64)
+        })
+}
+
+fn string_value(value: &redis::Value) -> Option<String> {
+    match value {
+        redis::Value::BulkString(bytes) => String::from_utf8(bytes.clone()).ok(),
+        redis::Value::SimpleString(text) => Some(text.clone()),
+        redis::Value::VerbatimString { text, .. } => Some(text.clone()),
+        _ => None,
+    }
+}
+
+fn number_value(value: &redis::Value) -> Option<f64> {
+    match value {
+        redis::Value::Int(number) => Some(*number as f64),
+        redis::Value::Double(number) => Some(*number),
+        _ => string_value(value).and_then(|text| text.parse().ok()),
+    }
+}
+
+fn first_document(result: &redis::Value) -> Option<&[redis::Value]> {
+    let redis::Value::Array(items) = result else {
+        return None;
+    };
+    let [count, _document_id, fields, ..] = items.as_slice() else {
+        return None;
+    };
+    if !matches!(count, redis::Value::Int(count) if *count > 0) {
+        return None;
+    }
+    match fields {
+        redis::Value::Array(fields) => Some(fields.as_slice()),
+        _ => None,
+    }
+}
+
+fn field_value<'a>(fields: &'a [redis::Value], name: &str) -> Option<&'a redis::Value> {
+    fields
+        .as_chunks::<2>()
+        .0
+        .iter()
+        .find(|pair| string_value(&pair[0]).as_deref() == Some(name))
+        .map(|pair| &pair[1])
+}
+
+fn string_field(fields: &[redis::Value], name: &str) -> Option<String> {
+    field_value(fields, name).and_then(string_value)
+}
+
+fn number_field(fields: &[redis::Value], name: &str) -> Option<f64> {
+    field_value(fields, name).and_then(number_value)
+}
+
+fn bytes_field(fields: &[redis::Value], name: &str) -> Option<Vec<u8>> {
+    match field_value(fields, name)? {
+        redis::Value::BulkString(bytes) => Some(bytes.clone()),
+        redis::Value::SimpleString(text) => Some(text.clone().into_bytes()),
+        _ => None,
+    }
+}
diff --git a/litellm-rust/crates/cache-redis-semantic/src/lib.rs b/litellm-rust/crates/cache-redis-semantic/src/lib.rs
new file mode 100644
index 00000000000..a34603cd18f
--- /dev/null
+++ b/litellm-rust/crates/cache-redis-semantic/src/lib.rs
@@ -0,0 +1,4 @@
+mod cache;
+mod prompt;
+
+pub use cache::{Embedder, RedisSemanticCache, RedisSemanticConfig};
diff --git a/litellm-rust/crates/cache-redis-semantic/src/prompt.rs b/litellm-rust/crates/cache-redis-semantic/src/prompt.rs
new file mode 100644
index 00000000000..fc99898d847
--- /dev/null
+++ b/litellm-rust/crates/cache-redis-semantic/src/prompt.rs
@@ -0,0 +1,95 @@
+use litellm_cache::SemanticCacheContext;
+use serde_json::Value;
+
+pub fn prompt_from_context(context: &SemanticCacheContext) -> Option<String> {
+    if !context.messages.is_empty() {
+        return Some(messages_text(&context.messages));
+    }
+    let input = context.input.as_ref()?;
+    let mut parts = Vec::new();
+    collect_input_text(input, &mut parts);
+    let prompt = parts.join("\n").trim().to_string();
+    (!prompt.is_empty()).then_some(prompt)
+}
+
+fn messages_text(messages: &[Value]) -> String {
+    let mut text = String::new();
+    for message in messages {
+        let Some(message) = message.as_object() else {
+            continue;
+        };
+        match message.get("content") {
+            Some(Value::String(content)) => text.push_str(content),
+            Some(Value::Array(parts)) => {
+                for part in parts {
+                    if let Some(text_content) = part.get("text").and_then(Value::as_str) {
+                        text.push_str(text_content);
+                    }
+                }
+            }
+            _ => {}
+        }
+        text.push_str(&search_results_text(message.get("search_results")));
+    }
+    text
+}
+
+fn search_results_text(search_results: Option<&Value>) -> String {
+    let Some(Value::Array(results)) = search_results else {
+        return String::new();
+    };
+    let mut text = String::new();
+    for result in results {
+        let Some(result) = result.as_object() else {
+            continue;
+        };
+        for key in ["source", "title"] {
+            if let Some(value) = result.get(key).and_then(Value::as_str) {
+                text.push_str(value);
+            }
+        }
+        if let Some(Value::Array(content)) = result.get("content") {
+            for block in content {
+                if let Some(value) = block.get("text").and_then(Value::as_str) {
+                    text.push_str(value);
+                }
+            }
+        }
+        if let Some(citations) = result.get("citations") {
+            text.push_str(&citations.to_string());
+        }
+    }
+    text
+}
+
+fn collect_input_text(value: &Value, parts: &mut Vec<String>) {
+    match value {
+        Value::String(text) => {
+            let trimmed = text.trim();
+            if !trimmed.is_empty() {
+                parts.push(trimmed.to_string());
+            }
+        }
+        Value::Array(items) => {
+            for item in items {
+                collect_input_text(item, parts);
+            }
+        }
+        Value::Object(map) => {
+            if let Some(content) = map.get("content").filter(|content| !content.is_null()) {
+                collect_input_text(content, parts);
+                return;
+            }
+            for key in ["text", "output", "input_text", "output_text"] {
+                if let Some(Value::String(text)) = map.get(key) {
+                    let trimmed = text.trim();
+                    if !trimmed.is_empty() {
+                        parts.push(trimmed.to_string());
+                        return;
+                    }
+                }
+            }
+        }
+        _ => {}
+    }
+}
diff --git a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs
new file mode 100644
index 00000000000..77b057ae3b9
--- /dev/null
+++ b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs
@@ -0,0 +1,751 @@
+use std::{
+    collections::HashMap,
+    sync::{Arc, Mutex},
+    time::Duration,
+};
+
+use litellm_cache::{BaseCache, CacheCodec, Error, SemanticCacheContext};
+use litellm_cache_redis_semantic::{Embedder, RedisSemanticCache, RedisSemanticConfig};
+use litellm_cache_response::{CacheEntry, ResponseCacheCodec};
+use redis_test::{MockCmd, MockRedisConnection};
+use serde_json::{Value, json};
+use sha2::{Digest, Sha256};
+
+const INDEX: &str = "litellm_semantic_cache_index";
+
+struct FakeEmbedder {
+    vectors: HashMap<String, Vec<f32>>,
+    calls: Arc<Mutex<Vec<String>>>,
+}
+
+impl FakeEmbedder {
+    fn new(vectors: &[(&str, &[f32])]) -> (Self, Arc<Mutex<Vec<String>>>) {
+        let calls = Arc::new(Mutex::new(Vec::new()));
+        (
+            Self {
+                vectors: vectors
+                    .iter()
+                    .map(|(prompt, vector)| (prompt.to_string(), vector.to_vec()))
+                    .collect(),
+                calls: Arc::clone(&calls),
+            },
+            calls,
+        )
+    }
+}
+
+impl Embedder for FakeEmbedder {
+    fn embed(&self, prompt: &str, _: &serde_json::Map<String, Value>) -> Result<Vec<f32>, Error> {
+        self.calls.lock().unwrap().push(prompt.to_string());
+
+        Ok(self
+            .vectors
+            .get(prompt)
+            .cloned()
+            .unwrap_or_else(|| vec![0.1, 0.2, 0.3]))
+    }
+
+    async fn async_embed(
+        &self,
+        prompt: &str,
+        metadata: &serde_json::Map<String, Value>,
+    ) -> Result<Vec<f32>, Error> {
+        self.embed(prompt, metadata)
+    }
+}
+
+fn config() -> RedisSemanticConfig {
+    RedisSemanticConfig {
+        index_name: INDEX.into(),
+        similarity_threshold: 0.9,
+    }
+}
+
+fn messages_context(messages: Vec<Value>) -> SemanticCacheContext {
+    SemanticCacheContext {
+        messages,
+        ..Default::default()
+    }
+}
+
+fn entry() -> CacheEntry {
+    CacheEntry {
+        timestamp: Some(1.0),
+        response: json!({"answer": "yes"}),
+    }
+}
+
+fn encoded(entry: &CacheEntry) -> Vec<u8> {
+    ResponseCacheCodec.encode(entry).unwrap()
+}
+
+fn vector_bytes(vector: &[f32]) -> Vec<u8> {
+    vector
+        .iter()
+        .flat_map(|component| component.to_le_bytes())
+        .collect()
+}
+
+fn entry_id(prompt: &str, tag: &str) -> String {
+    let mut digest = Sha256::new();
+    digest.update(prompt.as_bytes());
+    digest.update(b"litellm_cache_key");
+    digest.update(tag.as_bytes());
+    format!("{:x}", digest.finalize())
+}
+
+fn s(value: &str) -> redis::Value {
+    redis::Value::BulkString(value.as_bytes().to_vec())
+}
+
+fn unknown_index_error() -> redis::RedisError {
+    redis::RedisError::from((redis::ErrorKind::Extension, "Unknown index name"))
+}
+
+fn attribute(name: &str, field_type: &str, extra: Vec<redis::Value>) -> redis::Value {
+    let mut parts = vec![
+        s("identifier"),
+        s(name),
+        s("attribute"),
+        s(name),
+        s("type"),
+        s(field_type),
+    ];
+    parts.extend(extra);
+    redis::Value::Array(parts)
+}
+
+fn index_info(attributes: Vec<redis::Value>) -> redis::Value {
+    redis::Value::Array(vec![
+        s("index_name"),
+        s(INDEX),
+        s("attributes"),
+        redis::Value::Array(attributes),
+    ])
+}
+
+fn vector_attribute(dims: i64) -> redis::Value {
+    attribute(
+        "prompt_vector",
+        "VECTOR",
+        vec![
+            s("algorithm"),
+            s("FLAT"),
+            s("data_type"),
+            s("FLOAT32"),
+            s("dim"),
+            redis::Value::Int(dims),
+            s("distance_metric"),
+            s("COSINE"),
+        ],
+    )
+}
+
+fn compatible_info(dims: i64) -> redis::Value {
+    index_info(vec![
+        attribute("prompt", "TEXT", vec![]),
+        attribute("response", "TEXT", vec![]),
+        attribute("inserted_at", "NUMERIC", vec![]),
+        attribute("updated_at", "NUMERIC", vec![]),
+        vector_attribute(dims),
+        attribute("litellm_cache_key", "TAG", vec![]),
+    ])
+}
+
+fn unscoped_info(dims: i64) -> redis::Value {
+    index_info(vec![
+        attribute("prompt", "TEXT", vec![]),
+        attribute("response", "TEXT", vec![]),
+        attribute("inserted_at", "NUMERIC", vec![]),
+        attribute("updated_at", "NUMERIC", vec![]),
+        vector_attribute(dims),
+    ])
+}
+
+fn create_index_command(name: &str, dims: usize) -> redis::Cmd {
+    let mut command = redis::cmd("FT.CREATE");
+    command
+        .arg(name)
+        .arg("ON")
+        .arg("HASH")
+        .arg("PREFIX")
+        .arg(1)
+        .arg(name)
+        .arg("SCORE")
+        .arg(1.0)
+        .arg("SCHEMA")
+        .arg("prompt")
+        .arg("TEXT")
+        .arg("WEIGHT")
+        .arg(1)
+        .arg("response")
+        .arg("TEXT")
+        .arg("WEIGHT")
+        .arg(1)
+        .arg("inserted_at")
+        .arg("NUMERIC")
+        .arg("updated_at")
+        .arg("NUMERIC")
+        .arg("prompt_vector")
+        .arg("VECTOR")
+        .arg("FLAT")
+        .arg(6)
+        .arg("TYPE")
+        .arg("FLOAT32")
+        .arg("DIM")
+        .arg(dims)
+        .arg("DISTANCE_METRIC")
+        .arg("COSINE")
+        .arg("litellm_cache_key")
+        .arg("TAG")
+        .arg("SEPARATOR")
+        .arg(",");
+    command
+}
+
+fn search_command(index: &str, tag: &str, vector: &[f32]) -> redis::Cmd {
+    let mut command = redis::cmd("FT.SEARCH");
+    command
+        .arg(index)
+        .arg(format!(
+            "(@litellm_cache_key:{{{tag}}})=>[KNN 1 @prompt_vector $vector AS vector_distance]"
+        ))
+        .arg("RETURN")
+        .arg(8)
+        .arg("entry_id")
+        .arg("prompt")
+        .arg("response")
+        .arg("inserted_at")
+        .arg("updated_at")
+        .arg("metadata")
+        .arg("litellm_cache_key")
+        .arg("vector_distance")
+        .arg("SORTBY")
+        .arg("vector_distance")
+        .arg("ASC")
+        .arg("DIALECT")
+        .arg(2)
+        .arg("LIMIT")
+        .arg(0)
+        .arg(1)
+        .arg("PARAMS")
+        .arg(2)
+        .arg("vector")
+        .arg(vector_bytes(vector));
+    command
+}
+
+fn hit_fields(tag: &str, distance: &str, response: Vec<u8>) -> redis::Value {
+    redis::Value::Array(vec![
+        s("entry_id"),
+        s("stored-id"),
+        s("prompt"),
+        s("hello prompt"),
+        s("response"),
+        redis::Value::BulkString(response),
+        s("inserted_at"),
+        s("1700000000.5"),
+        s("updated_at"),
+        s("1700000000.5"),
+        s("litellm_cache_key"),
+        s(tag),
+        s("vector_distance"),
+        s(distance),
+    ])
+}
+
+fn search_result(fields: redis::Value) -> redis::Value {
+    redis::Value::Array(vec![
+        redis::Value::Int(1),
+        s("litellm_semantic_cache_index:stored-id"),
+        fields,
+    ])
+}
+
+fn empty_result() -> redis::Value {
+    redis::Value::Array(vec![redis::Value::Int(0)])
+}
+
+#[test]
+fn store_creates_index_and_writes_hash_with_expire() {
+    let vector = vec![0.1f32, 0.2, 0.3];
+    let prompt = "hello prompt";
+    let tag = "key1";
+    let hash_key = format!("{INDEX}:{}", entry_id(prompt, tag));
+    let value = entry();
+    let connection = MockRedisConnection::new([
+        MockCmd::new(
+            redis::cmd("FT.INFO").arg(INDEX),
+            Err::<redis::Value, _>(unknown_index_error()),
+        ),
+        MockCmd::new(create_index_command(INDEX, 3), Ok("OK")),
+        MockCmd::new(
+            redis::cmd("HSET")
+                .arg(&hash_key)
+                .arg("entry_id")
+                .arg(entry_id(prompt, tag))
+                .arg("prompt")
+                .arg(prompt)
+                .arg("response")
+                .arg(encoded(&value))
+                .arg("prompt_vector")
+                .arg(vector_bytes(&vector))
+                .arg("inserted_at")
+                .arg("1700000000.5")
+                .arg("updated_at")
+                .arg("1700000000.5")
+                .arg("litellm_cache_key")
+                .arg(tag),
+            Ok(7),
+        ),
+        MockCmd::new(redis::cmd("EXPIRE").arg(&hash_key).arg(5), Ok(1)),
+    ])
+    .assert_all_commands_consumed();
+    let (embedder, _) = FakeEmbedder::new(&[(prompt, &vector)]);
+    let cache = RedisSemanticCache::with_connection(connection, embedder, config())
+        .with_clock(|| 1700000000.5);
+
+    let context = SemanticCacheContext {
+        ttl: Some(Duration::from_secs(5)),
+        ..messages_context(vec![json!({"role": "user", "content": prompt})])
+    };
+    cache.set_cache(tag, value, &context).unwrap();
+}
+
+#[test]
+fn store_without_ttl_skips_expire() {
+    let prompt = "hello prompt";
+    let connection = MockRedisConnection::new([
+        MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))),
+        MockCmd::new(
+            redis::cmd("HSET")
+                .arg(format!("{INDEX}:{}", entry_id(prompt, "key1")))
+                .arg("entry_id")
+                .arg(entry_id(prompt, "key1"))
+                .arg("prompt")
+                .arg(prompt)
+                .arg("response")
+                .arg(encoded(&entry()))
+                .arg("prompt_vector")
+                .arg(vector_bytes(&[0.1f32, 0.2, 0.3]))
+                .arg("inserted_at")
+                .arg("1700000000.5")
+                .arg("updated_at")
+                .arg("1700000000.5")
+                .arg("litellm_cache_key")
+                .arg("key1"),
+            Ok(7),
+        ),
+    ])
+    .assert_all_commands_consumed();
+    let (embedder, _) = FakeEmbedder::new(&[]);
+    let cache = RedisSemanticCache::with_connection(connection, embedder, config())
+        .with_clock(|| 1700000000.5);
+
+    cache
+        .set_cache(
+            "key1",
+            entry(),
+            &messages_context(vec![json!({"role": "user", "content": prompt})]),
+        )
+        .unwrap();
+}
+
+#[test]
+fn lookup_returns_hit_below_distance_threshold() {
+    let vector = vec![0.1f32, 0.2, 0.3];
+    let value = entry();
+    let connection = MockRedisConnection::new([
+        MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))),
+        MockCmd::new(
+            search_command(INDEX, "key1", &vector),
+            Ok(search_result(hit_fields("key1", "0.05", encoded(&value)))),
+        ),
+    ])
+    .assert_all_commands_consumed();
+    let (embedder, _) = FakeEmbedder::new(&[]);
+    let cache = RedisSemanticCache::with_connection(connection, embedder, config());
+
+    let hit = cache
+        .get_cache(
+            "key1",
+            &messages_context(vec![json!({"role": "user", "content": "hello prompt"})]),
+        )
+        .unwrap();
+    assert_eq!(hit, Some(value));
+}
+
+#[test]
+fn lookup_misses_above_distance_threshold_and_on_tag_mismatch() {
+    let vector = vec![0.1f32, 0.2, 0.3];
+    let connection = MockRedisConnection::new([
+        MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))),
+        MockCmd::new(
+            search_command(INDEX, "key1", &vector),
+            Ok(search_result(hit_fields("key1", "0.5", encoded(&entry())))),
+        ),
+        MockCmd::new(
+            search_command(INDEX, "key1", &vector),
+            Ok(search_result(hit_fields(
+                "other",
+                "0.05",
+                encoded(&entry()),
+            ))),
+        ),
+    ])
+    .assert_all_commands_consumed();
+    let (embedder, _) = FakeEmbedder::new(&[]);
+    let cache = RedisSemanticCache::with_connection(connection, embedder, config());
+    let context = messages_context(vec![json!({"role": "user", "content": "hello prompt"})]);
+
+    assert_eq!(cache.get_cache("key1", &context).unwrap(), None);
+    assert_eq!(cache.get_cache("key1", &context).unwrap(), None);
+}
+
+#[test]
+fn lookup_returns_invalid_entry_on_malformed_response() {
+    let vector = vec![0.1f32, 0.2, 0.3];
+    let connection = MockRedisConnection::new([
+        MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))),
+        MockCmd::new(
+            search_command(INDEX, "key1", &vector),
+            Ok(search_result(hit_fields(
+                "key1",
+                "0.05",
+                b"not json!".to_vec(),
+            ))),
+        ),
+    ])
+    .assert_all_commands_consumed();
+    let (embedder, _) = FakeEmbedder::new(&[]);
+    let cache = RedisSemanticCache::with_connection(connection, embedder, config());
+
+    assert_eq!(
+        cache
+            .get_cache(
+                "key1",
+                &messages_context(vec![json!({"role": "user", "content": "hello prompt"})])
+            )
+            .unwrap_err(),
+        Error::InvalidEntry
+    );
+}
+
+#[test]
+fn missing_prompt_is_noop_and_never_embeds() {
+    let connection = MockRedisConnection::new(Vec::<MockCmd>::new()).assert_all_commands_consumed();
+    let (embedder, calls) = FakeEmbedder::new(&[]);
+    let cache = RedisSemanticCache::with_connection(connection, embedder, config());
+
+    let context = SemanticCacheContext::default();
+    cache.set_cache("key1", entry(), &context).unwrap();
+    assert_eq!(cache.get_cache("key1", &context).unwrap(), None);
+    assert!(calls.lock().unwrap().is_empty());
+}
+
+#[test]
+fn scope_overrides_key_as_filter_tag() {
+    let vector = vec![0.1f32, 0.2, 0.3];
+    let prompt = "hello prompt";
+    let value = entry();
+    let connection = MockRedisConnection::new([
+        MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))),
+        MockCmd::new(
+            redis::cmd("HSET")
+                .arg(format!("{INDEX}:{}", entry_id(prompt, "scope-a")))
+                .arg("entry_id")
+                .arg(entry_id(prompt, "scope-a"))
+                .arg("prompt")
+                .arg(prompt)
+                .arg("response")
+                .arg(encoded(&value))
+                .arg("prompt_vector")
+                .arg(vector_bytes(&vector))
+                .arg("inserted_at")
+                .arg("1700000000.5")
+                .arg("updated_at")
+                .arg("1700000000.5")
+                .arg("litellm_cache_key")
+                .arg("scope-a"),
+            Ok(7),
+        ),
+        MockCmd::new(
+            search_command(INDEX, "scope\\-a", &vector),
+            Ok(search_result(hit_fields(
+                "scope-a",
+                "0.05",
+                encoded(&value),
+            ))),
+        ),
+    ])
+    .assert_all_commands_consumed();
+    let (embedder, _) = FakeEmbedder::new(&[]);
+    let cache = RedisSemanticCache::with_connection(connection, embedder, config())
+        .with_clock(|| 1700000000.5);
+    let context = SemanticCacheContext {
+        scope: Some("scope-a".into()),
+        ..messages_context(vec![json!({"role": "user", "content": prompt})])
+    };
+
+    cache.set_cache("key1", value.clone(), &context).unwrap();
+    assert_eq!(cache.get_cache("key1", &context).unwrap(), Some(value));
+}
+
+#[test]
+fn incompatible_schema_falls_back_to_isolated_index() {
+    let prompt = "hello prompt";
+    let tag = "key1";
+    let isolated = format!("{INDEX}_isolated");
+    let connection = MockRedisConnection::new([
+        MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(unscoped_info(3))),
+        MockCmd::new(
+            redis::cmd("FT.INFO").arg(&isolated),
+            Err::<redis::Value, _>(unknown_index_error()),
+        ),
+        MockCmd::new(create_index_command(&isolated, 3), Ok("OK")),
+        MockCmd::new(
+            redis::cmd("HSET")
+                .arg(format!("{isolated}:{}", entry_id(prompt, tag)))
+                .arg("entry_id")
+                .arg(entry_id(prompt, tag))
+                .arg("prompt")
+                .arg(prompt)
+                .arg("response")
+                .arg(encoded(&entry()))
+                .arg("prompt_vector")
+                .arg(vector_bytes(&[0.1f32, 0.2, 0.3]))
+                .arg("inserted_at")
+                .arg("1700000000.5")
+                .arg("updated_at")
+                .arg("1700000000.5")
+                .arg("litellm_cache_key")
+                .arg(tag),
+            Ok(7),
+        ),
+    ])
+    .assert_all_commands_consumed();
+    let (embedder, _) = FakeEmbedder::new(&[]);
+    let cache = RedisSemanticCache::with_connection(connection, embedder, config())
+        .with_clock(|| 1700000000.5);
+
+    cache
+        .set_cache(
+            tag,
+            entry(),
+            &messages_context(vec![json!({"role": "user", "content": prompt})]),
+        )
+        .unwrap();
+}
+
+#[test]
+fn tag_special_characters_are_escaped_in_search_filter() {
+    let vector = vec![0.1f32, 0.2, 0.3];
+    let tag = "a:b, c|d";
+    let connection = MockRedisConnection::new([
+        MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))),
+        MockCmd::new(
+            search_command(INDEX, "a\\:b\\,\\ c\\|d", &vector),
+            Ok(empty_result()),
+        ),
+    ])
+    .assert_all_commands_consumed();
+    let (embedder, _) = FakeEmbedder::new(&[]);
+    let cache = RedisSemanticCache::with_connection(connection, embedder, config());
+
+    assert_eq!(
+        cache
+            .get_cache(
+                tag,
+                &messages_context(vec![json!({"role": "user", "content": "hello prompt"})])
+            )
+            .unwrap(),
+        None
+    );
+}
+
+#[test]
+fn prompt_extraction_matches_python_message_and_input_shapes() {
+    let vector = vec![0.1f32, 0.2, 0.3];
+    let lookups = 5;
+    let mut commands = vec![MockCmd::new(
+        redis::cmd("FT.INFO").arg(INDEX),
+        Ok(compatible_info(3)),
+    )];
+    for _ in 0..lookups {
+        commands.push(MockCmd::new(
+            search_command(INDEX, "key1", &vector),
+            Ok(empty_result()),
+        ));
+    }
+    let connection = MockRedisConnection::new(commands).assert_all_commands_consumed();
+    let (embedder, calls) = FakeEmbedder::new(&[]);
+    let cache = RedisSemanticCache::with_connection(connection, embedder, config());
+
+    cache
+        .get_cache(
+            "key1",
+            &messages_context(vec![
+                json!({"role": "user", "content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}]}),
+                json!({"role": "assistant", "content": "reply"}),
+            ]),
+        )
+        .unwrap();
+    cache
+        .get_cache(
+            "key1",
+            &SemanticCacheContext {
+                input: Some(json!("  plain input  ")),
+                ..Default::default()
+            },
+        )
+        .unwrap();
+    cache
+        .get_cache(
+            "key1",
+            &SemanticCacheContext {
+                input: Some(
+                    json!([{"content": [{"type": "input_text", "text": "nested"}]}, "tail"]),
+                ),
+                ..Default::default()
+            },
+        )
+        .unwrap();
+    cache
+        .get_cache(
+            "key1",
+            &SemanticCacheContext {
+                input: Some(json!({"output_text": "  result text  "})),
+                ..Default::default()
+            },
+        )
+        .unwrap();
+    cache
+        .get_cache(
+            "key1",
+            &messages_context(vec![json!({
+                "role": "user",
+                "content": "question",
+                "search_results": [{"source": "src", "title": "t", "content": [{"text": "found"}], "citations": {"a": 1}}],
+            })]),
+        )
+        .unwrap();
+
+    assert_eq!(
+        *calls.lock().unwrap(),
+        vec![
+            "firstsecondreply",
+            "plain input",
+            "nested\ntail",
+            "result text",
+            "questionsrctfound{\"a\":1}",
+        ]
+    );
+}
+
+#[test]
+fn ttl_passes_through_context_only() {
+    let (embedder, _) = FakeEmbedder::new(&[]);
+    let cache = RedisSemanticCache::with_connection(
+        MockRedisConnection::new(Vec::<MockCmd>::new()),
+        embedder,
+        config(),
+    );
+    assert_eq!(cache.get_ttl(&SemanticCacheContext::default()), None);
+    assert_eq!(
+        cache.get_ttl(&SemanticCacheContext {
+            ttl: Some(Duration::from_secs(9)),
+            ..Default::default()
+        }),
+        Some(Duration::from_secs(9))
+    );
+}
+
+#[tokio::test]
+async fn async_paths_embed_then_run_blocking_redis_work() {
+    let vector = vec![0.1f32, 0.2, 0.3];
+    let prompt = "hello prompt";
+    let tag = "key1";
+    let hash_key = format!("{INDEX}:{}", entry_id(prompt, tag));
+    let value = entry();
+    let connection = MockRedisConnection::new([
+        MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))),
+        MockCmd::new(
+            redis::cmd("HSET")
+                .arg(&hash_key)
+                .arg("entry_id")
+                .arg(entry_id(prompt, tag))
+                .arg("prompt")
+                .arg(prompt)
+                .arg("response")
+                .arg(encoded(&value))
+                .arg("prompt_vector")
+                .arg(vector_bytes(&vector))
+                .arg("inserted_at")
+                .arg("1700000000.5")
+                .arg("updated_at")
+                .arg("1700000000.5")
+                .arg("litellm_cache_key")
+                .arg(tag),
+            Ok(7),
+        ),
+        MockCmd::new(
+            search_command(INDEX, tag, &vector),
+            Ok(search_result(hit_fields(tag, "0.05", encoded(&value)))),
+        ),
+    ])
+    .assert_all_commands_consumed();
+    let (embedder, _) = FakeEmbedder::new(&[]);
+    let cache = RedisSemanticCache::with_connection(connection, embedder, config())
+        .with_clock(|| 1700000000.5);
+    let context = messages_context(vec![json!({"role": "user", "content": prompt})]);
+
+    cache
+        .async_set_cache(tag, value.clone(), context.clone())
+        .await
+        .unwrap();
+    assert_eq!(
+        cache.async_get_cache(tag, &context).await.unwrap(),
+        Some(value)
+    );
+}
+
+#[test]
+fn live_store_lookup_and_ttl_against_redis_stack() {
+    let Ok(url) = std::env::var("LITELLM_REDIS_STACK_URL") else {
+        return;
+    };
+    let vector = vec![0.1f32, 0.2, 0.3, 0.4];
+    let prompt = "rust semantic cache live prompt";
+    let tag = "live-key";
+    let index_name = format!("rust_semantic_test_{}", std::process::id());
+    let (embedder, _) = FakeEmbedder::new(&[(prompt, &vector)]);
+    let cache = RedisSemanticCache::new(
+        &url,
+        embedder,
+        RedisSemanticConfig {
+            index_name: index_name.clone(),
+            similarity_threshold: 0.9,
+        },
+    )
+    .unwrap();
+    let context = SemanticCacheContext {
+        ttl: Some(Duration::from_secs(120)),
+        ..messages_context(vec![json!({"role": "user", "content": prompt})])
+    };
+    let value = entry();
+
+    cache.set_cache(tag, value.clone(), &context).unwrap();
+    assert_eq!(cache.get_cache(tag, &context).unwrap(), Some(value));
+    assert_eq!(cache.get_cache("other-key", &context).unwrap(), None);
+
+    let mut connection = redis::Client::open(url).unwrap().get_connection().unwrap();
+    let ttl: i64 = redis::Commands::ttl(
+        &mut connection,
+        format!("{index_name}:{}", entry_id(prompt, tag)),
+    )
+    .unwrap();
+    assert!(
+        ttl > 0,
+        "expected stored hash to carry an expiry, got {ttl}"
+    );
+}
diff --git a/litellm-rust/crates/cache/tests/caching.rs b/litellm-rust/crates/cache/tests/caching.rs
index 2e65b4eeae5..33171f36f46 100644
--- a/litellm-rust/crates/cache/tests/caching.rs
+++ b/litellm-rust/crates/cache/tests/caching.rs
@@ -1,8 +1,8 @@
 use std::{sync::Mutex, time::Duration};
 
 use litellm_cache::{
-    BaseCache, CacheConnectionResult, CacheContext, Error, ExactCacheContext,
-    SemanticCacheContext, get_cache,
+    BaseCache, CacheConnectionResult, CacheContext, Error, ExactCacheContext, SemanticCacheContext,
+    get_cache,
 };
 
 struct TestCache {
@@ -132,10 +132,7 @@ fn semantic_context_with_ttl_preserves_lookup_inputs() {
     let context = SemanticCacheContext {
         input: Some(serde_json::json!("text")),
         messages: vec![serde_json::json!({"role": "user", "content": "hi"})],
-        metadata: serde_json::Map::from_iter([(
-            "key".into(),
-            serde_json::json!("value"),
-        )]),
+        metadata: serde_json::Map::from_iter([("key".into(), serde_json::json!("value"))]),
         scope: Some("scope".into()),
         ttl: None,
     };

From 4db34812449038fed2c724a3ef099fefb3198ac2 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:33:32 +0000
Subject: [PATCH 071/160] feat(python-bridge): serve ValkeySemanticCache
 natively

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.lock                       |   2 +
 litellm-rust/crates/python-bridge/Cargo.toml  |   2 +
 .../crates/python-bridge/src/cache/config.rs  |  84 +++++-
 .../python-bridge/src/cache/embedder.rs       |  58 ++++
 .../crates/python-bridge/src/cache/facade.rs  |  58 +++-
 .../crates/python-bridge/src/cache/handle.rs  |  44 ++-
 .../crates/python-bridge/src/cache/mod.rs     |   4 +-
 .../crates/python-bridge/src/cache/native.rs  | 253 +++++++++++++-----
 .../crates/python-bridge/src/cache/request.rs |  39 ++-
 .../test_valkey_semantic_cache.py             | 149 +++++++++++
 10 files changed, 598 insertions(+), 95 deletions(-)
 create mode 100644 litellm-rust/crates/python-bridge/src/cache/embedder.rs
 create mode 100644 tests/test_litellm_rust/test_valkey_semantic_cache.py

diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index 5daf691d4c3..e4c9c385f1f 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -2685,6 +2685,7 @@ dependencies = [
  "litellm-cache-memory",
  "litellm-cache-redis",
  "litellm-cache-response",
+ "litellm-cache-valkey-semantic",
  "litellm-callbacks-legacy-python",
  "litellm-core",
  "litellm-core-utils",
@@ -2695,6 +2696,7 @@ dependencies = [
  "litellm-types",
  "pyo3",
  "pyo3-async-runtimes",
+ "redis",
  "rstest",
  "serde",
  "serde_json",
diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml
index 1eb2ec28036..ce2405f33c4 100644
--- a/litellm-rust/crates/python-bridge/Cargo.toml
+++ b/litellm-rust/crates/python-bridge/Cargo.toml
@@ -24,6 +24,7 @@ litellm-cache.workspace = true
 litellm-cache-memory.workspace = true
 litellm-cache-redis.workspace = true
 litellm-cache-response.workspace = true
+litellm-cache-valkey-semantic = { path = "../cache-valkey-semantic" }
 serde.workspace = true
 litellm-auth.workspace = true
 litellm-callbacks-legacy-python.workspace = true
@@ -37,6 +38,7 @@ litellm-host-python.workspace = true
 litellm-token-counter = { path = "../token-counter", default-features = false }
 pyo3.workspace = true
 pyo3-async-runtimes.workspace = true
+redis = { version = "1.7.0", features = ["tls-rustls"] }
 serde_json.workspace = true
 tokio = { workspace = true, features = ["sync"] }
 
diff --git a/litellm-rust/crates/python-bridge/src/cache/config.rs b/litellm-rust/crates/python-bridge/src/cache/config.rs
index 0e7d6aee11d..7218805b8df 100644
--- a/litellm-rust/crates/python-bridge/src/cache/config.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/config.rs
@@ -73,9 +73,18 @@ pub(super) struct RedisCacheConfig {
     pub(super) connection: RedisConnectionConfig,
 }
 
+#[allow(dead_code, reason = "consumed by the cache activation follow-up")]
+pub(super) struct ValkeySemanticCacheConfig {
+    pub(super) similarity_threshold: f64,
+    pub(super) index_name: String,
+    pub(super) embedding_model: String,
+    pub(super) connection: RedisConnectionConfig,
+}
+
 pub(super) enum CacheBackendConfig {
     Memory(MemoryCacheConfig),
     Redis(Box<RedisCacheConfig>),
+    ValkeySemantic(Box<ValkeySemanticCacheConfig>),
 }
 
 #[allow(dead_code, reason = "consumed by the cache activation follow-up")]
@@ -142,9 +151,15 @@ impl NativeCacheConfig {
                 }))),
                 Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)),
             },
+            Some(CacheType::ValkeySemantic) => match project_valkey_semantic(&backend)? {
+                Ok(backend) => Ok(CacheConfigProjection::Native(Box::new(Self {
+                    policy,
+                    backend: CacheBackendConfig::ValkeySemantic(Box::new(backend)),
+                }))),
+                Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)),
+            },
             Some(
                 CacheType::RedisSemantic
-                | CacheType::ValkeySemantic
                 | CacheType::S3
                 | CacheType::Disk
                 | CacheType::QdrantSemantic
@@ -158,11 +173,13 @@ impl NativeCacheConfig {
     }
 
     pub(super) fn service_mismatch(&self, service: &NativeResponseCache) -> Option<&'static str> {
-        if service.default_ttl()
-            != Some(match &self.backend {
-                CacheBackendConfig::Memory(config) => config.default_ttl,
-                CacheBackendConfig::Redis(config) => config.default_ttl,
-            })
+        if !matches!(self.backend, CacheBackendConfig::ValkeySemantic(_))
+            && service.default_ttl()
+                != Some(match &self.backend {
+                    CacheBackendConfig::Memory(config) => config.default_ttl,
+                    CacheBackendConfig::Redis(config) => config.default_ttl,
+                    CacheBackendConfig::ValkeySemantic(_) => Duration::ZERO,
+                })
         {
             return Some("facade and native backend default TTLs must match");
         }
@@ -185,6 +202,16 @@ impl NativeCacheConfig {
             CacheBackendConfig::Redis(config) => (service.namespace()
                 != config.namespace.as_deref())
             .then_some("facade and native backend namespaces must match"),
+            CacheBackendConfig::ValkeySemantic(config) => {
+                if service.kind() != "valkey-semantic" {
+                    return Some("facade and native backend types must match");
+                }
+                let Some((threshold, index_name)) = service.semantic_config() else {
+                    return Some("facade and native backend types must match");
+                };
+                (threshold != config.similarity_threshold || index_name != config.index_name)
+                    .then_some("facade and native semantic settings must match")
+            }
         }
     }
 }
@@ -299,6 +326,51 @@ fn project_redis(
     }))
 }
 
+#[inline(never)]
+fn project_valkey_semantic(
+    backend: &Bound<'_, PyAny>,
+) -> PyResult<Result<ValkeySemanticCacheConfig, UnsupportedCacheConfig>> {
+    let client = backend.getattr("sync_client")?;
+    let pool = client.getattr("connection_pool")?;
+    if !instance_class_is(&pool, "redis.connection", "ConnectionPool")? {
+        return Ok(Err(UnsupportedCacheConfig::RedisConnection));
+    }
+    let resolved = pool.getattr("connection_kwargs")?.cast_into::<PyDict>()?;
+    let connection_class = resolved
+        .get_item("connection_class")?
+        .unwrap_or(pool.getattr("connection_class")?);
+    if !class_is(&connection_class, "redis.connection", "Connection")?
+        && !class_is(&connection_class, "redis.connection", "SSLConnection")?
+    {
+        return Ok(Err(UnsupportedCacheConfig::RedisConnection));
+    }
+    let connection = RedisConnectionConfig {
+        host: required_string(&resolved, "host")?,
+        port: u16::try_from(required_i64(&resolved, "port")?)
+            .map_err(|_| PyValueError::new_err("invalid Redis port"))?,
+        database: optional_i64(&resolved, "db")?.unwrap_or(0),
+        username: optional_dict_string(&resolved, "username")?,
+        password: optional_dict_string(&resolved, "password")?,
+        protocol: RedisProtocol::Resp2,
+        pool_size: pool.getattr("max_connections")?.extract::<usize>()?,
+        read_timeout: None,
+        connect_timeout: None,
+        socket_keepalive: None,
+        health_check_interval: Duration::ZERO,
+        client_name: None,
+        tls: None,
+    };
+    if connection.host.is_empty() {
+        return Ok(Err(UnsupportedCacheConfig::RedisConnection));
+    }
+    Ok(Ok(ValkeySemanticCacheConfig {
+        similarity_threshold: backend.getattr("similarity_threshold")?.extract()?,
+        index_name: backend.getattr("index_name")?.extract()?,
+        embedding_model: backend.getattr("embedding_model")?.extract()?,
+        connection,
+    }))
+}
+
 #[inline(never)]
 fn project_tls(values: &Bound<'_, PyDict>) -> PyResult<RedisTlsConfig> {
     Ok(RedisTlsConfig {
diff --git a/litellm-rust/crates/python-bridge/src/cache/embedder.rs b/litellm-rust/crates/python-bridge/src/cache/embedder.rs
new file mode 100644
index 00000000000..d240d9d019e
--- /dev/null
+++ b/litellm-rust/crates/python-bridge/src/cache/embedder.rs
@@ -0,0 +1,58 @@
+use std::{future::Future, sync::Arc};
+
+use litellm_cache::Error;
+use litellm_cache_valkey_semantic::Embedder;
+use litellm_host_python::to_py;
+use pyo3::prelude::*;
+use serde_json::Value;
+
+#[derive(Clone)]
+pub(super) struct PythonEmbedder {
+    sync_embed: Arc<Py<PyAny>>,
+    async_embed: Arc<Py<PyAny>>,
+}
+
+impl PythonEmbedder {
+    pub(super) fn from_backend(backend: &Bound<'_, PyAny>) -> PyResult<Self> {
+        Ok(Self {
+            sync_embed: Arc::new(backend.getattr("_get_embedding")?.unbind()),
+            async_embed: Arc::new(backend.getattr("_get_async_embedding")?.unbind()),
+        })
+    }
+}
+
+impl Embedder for PythonEmbedder {
+    fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, Error> {
+        let result = Python::attach(|py| -> PyResult<Vec<f64>> {
+            let metadata = to_py(py, &metadata)?;
+            self.sync_embed
+                .bind(py)
+                .call1((prompt, metadata))?
+                .extract()
+        })
+        .map_err(|_| Error::Unavailable)?;
+        Ok(result.into_iter().map(|value| value as f32).collect())
+    }
+
+    fn async_embed(
+        &self,
+        prompt: &str,
+        metadata: Option<&Value>,
+    ) -> impl Future<Output = Result<Vec<f32>, Error>> + Send {
+        let callable = Arc::clone(&self.async_embed);
+        let prompt = prompt.to_owned();
+        let metadata = metadata.cloned();
+        async move {
+            let future = Python::attach(|py| -> PyResult<_> {
+                let metadata = to_py(py, &metadata)?;
+                let awaitable = callable.bind(py).call1((prompt, metadata))?;
+                pyo3_async_runtimes::tokio::into_future(awaitable)
+            })
+            .map_err(|_| Error::Unavailable)?;
+            let result = future.await.map_err(|_| Error::Unavailable)?;
+            let result = Python::attach(|py| result.bind(py).extract::<Vec<f64>>())
+                .map_err(|_| Error::Unavailable)?;
+            Ok(result.into_iter().map(|value| value as f32).collect())
+        }
+    }
+}
diff --git a/litellm-rust/crates/python-bridge/src/cache/facade.rs b/litellm-rust/crates/python-bridge/src/cache/facade.rs
index f2f86c14b37..eb76d22097d 100644
--- a/litellm-rust/crates/python-bridge/src/cache/facade.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/facade.rs
@@ -36,6 +36,7 @@ pub(super) struct FacadeGuard {
     outer: ObjectGuard,
     backend: ObjectGuard,
     redis_pool: Option<RedisPoolGuard>,
+    redis_client_name: Option<&'static str>,
 }
 
 impl ObjectGuard {
@@ -117,7 +118,9 @@ impl ObjectGuard {
                 return Ok(false);
             }
             for (name, value) in &expected.attributes {
-                if instance.contains(name)? || !attributes.get_item(name)?.is(value.bind(py)) {
+                if (instance.contains(name)? && !self.config_names.contains(&name.as_str()))
+                    || !attributes.get_item(name)?.is(value.bind(py))
+                {
                     return Ok(false);
                 }
             }
@@ -138,10 +141,8 @@ impl ObjectGuard {
 }
 
 impl RedisPoolGuard {
-    fn capture(backend: &Bound<'_, PyAny>) -> PyResult<Self> {
-        let pool = backend
-            .getattr("redis_client")?
-            .getattr("connection_pool")?;
+    fn capture(backend: &Bound<'_, PyAny>, client_name: &str) -> PyResult<Self> {
+        let pool = backend.getattr(client_name)?.getattr("connection_pool")?;
         Ok(Self {
             reference: pool.clone().unbind(),
             connection_class: pool.getattr("connection_class")?.unbind(),
@@ -153,10 +154,13 @@ impl RedisPoolGuard {
         })
     }
 
-    fn matches(&self, py: Python<'_>, backend: &Bound<'_, PyAny>) -> PyResult<bool> {
-        let pool = backend
-            .getattr("redis_client")?
-            .getattr("connection_pool")?;
+    fn matches(
+        &self,
+        py: Python<'_>,
+        backend: &Bound<'_, PyAny>,
+        client_name: &str,
+    ) -> PyResult<bool> {
+        let pool = backend.getattr(client_name)?.getattr("connection_pool")?;
         Ok(self.reference.bind(py).is(&pool)
             && self
                 .connection_class
@@ -192,6 +196,11 @@ impl FacadeGuard {
         let (module, name, cache_kind) = match kind {
             "memory" => ("litellm.caching.in_memory_cache", "InMemoryCache", "local"),
             "redis" => ("litellm.caching.redis_cache", "RedisCache", "redis"),
+            "valkey-semantic" => (
+                "litellm.caching.valkey_semantic_cache",
+                "ValkeySemanticCache",
+                "valkey-semantic",
+            ),
             _ => unreachable!(),
         };
         let backend = facade.getattr("cache")?;
@@ -235,11 +244,32 @@ impl FacadeGuard {
                     "max_size_per_item",
                     "redis_kwargs",
                     "redis_flush_size",
+                    "similarity_threshold",
+                    "embedding_model",
+                    "index_name",
+                    "embedding_max_input_tokens",
+                    "embedding_timeout",
                 ],
             )?,
-            redis_pool: (kind == "redis")
-                .then(|| RedisPoolGuard::capture(&backend))
+            redis_pool: (kind == "redis" || kind == "valkey-semantic")
+                .then(|| {
+                    RedisPoolGuard::capture(
+                        &backend,
+                        if kind == "redis" {
+                            "redis_client"
+                        } else {
+                            "sync_client"
+                        },
+                    )
+                })
                 .transpose()?,
+            redis_client_name: (kind == "redis" || kind == "valkey-semantic").then_some(
+                if kind == "redis" {
+                    "redis_client"
+                } else {
+                    "sync_client"
+                },
+            ),
         })
     }
 
@@ -252,7 +282,11 @@ impl FacadeGuard {
             return Ok(false);
         }
         match &self.redis_pool {
-            Some(guard) => guard.matches(py, &backend),
+            Some(guard) => guard.matches(
+                py,
+                &backend,
+                self.redis_client_name.unwrap_or("redis_client"),
+            ),
             None => Ok(true),
         }
     }
diff --git a/litellm-rust/crates/python-bridge/src/cache/handle.rs b/litellm-rust/crates/python-bridge/src/cache/handle.rs
index 8251b3df06c..119bd35cd25 100644
--- a/litellm-rust/crates/python-bridge/src/cache/handle.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/handle.rs
@@ -1,7 +1,10 @@
 use litellm_host_python::release_gil;
 use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*};
 
-use super::{cache_error, facade::FacadeGuard, native::NativeResponseCache, request::duration};
+use super::{
+    cache_error, embedder::PythonEmbedder, facade::FacadeGuard, native::NativeResponseCache,
+    request::duration,
+};
 
 #[pyclass(frozen, name = "_CacheTestHandle")]
 pub(crate) struct CacheTestHandle {
@@ -51,6 +54,29 @@ impl CacheTestHandle {
         })
     }
 
+    #[staticmethod]
+    #[pyo3(signature = (url, similarity_threshold, index_name, embedder))]
+    fn valkey_semantic(
+        url: String,
+        similarity_threshold: f64,
+        index_name: String,
+        embedder: &Bound<'_, PyAny>,
+    ) -> PyResult<Self> {
+        let python_embedder = PythonEmbedder::from_backend(embedder)?;
+        let service = NativeResponseCache::valkey_semantic(
+            &url,
+            similarity_threshold,
+            index_name,
+            python_embedder,
+        )
+        .map_err(cache_error)?;
+        Ok(Self {
+            service,
+            guard: None,
+            pid: std::process::id(),
+        })
+    }
+
     #[getter]
     fn backend(&self) -> &'static str {
         self.service.kind()
@@ -59,11 +85,17 @@ impl CacheTestHandle {
     fn _bind_facade(&self, py: Python<'_>, facade: &Bound<'_, PyAny>) -> PyResult<()> {
         let service = self.service()?;
         let guard = FacadeGuard::capture(py, facade, &service)?;
-        let service = service.with_redis_flush_size(
-            facade
-                .getattr("redis_flush_size")?
-                .extract::<Option<usize>>()?,
-        );
+        let service = service
+            .with_scope(
+                facade
+                    .getattr("semantic_cache_scope")?
+                    .extract::<String>()?,
+            )
+            .with_redis_flush_size(
+                facade
+                    .getattr("redis_flush_size")?
+                    .extract::<Option<usize>>()?,
+            );
         let handle = Py::new(
             py,
             Self {
diff --git a/litellm-rust/crates/python-bridge/src/cache/mod.rs b/litellm-rust/crates/python-bridge/src/cache/mod.rs
index aec08610f6e..4cc87367d91 100644
--- a/litellm-rust/crates/python-bridge/src/cache/mod.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/mod.rs
@@ -1,6 +1,7 @@
 mod binding;
 mod callback;
 mod config;
+mod embedder;
 mod facade;
 mod future;
 mod handle;
@@ -10,7 +11,7 @@ mod resolver;
 
 use litellm_cache::Error;
 use pyo3::{
-    exceptions::{PyRuntimeError, PyValueError},
+    exceptions::{PyNotImplementedError, PyRuntimeError, PyValueError},
     prelude::*,
 };
 
@@ -21,6 +22,7 @@ pub(crate) use self::{
 fn cache_error(error: Error) -> PyErr {
     match error {
         Error::InvalidEntry => PyValueError::new_err(error.to_string()),
+        Error::UnsupportedOperation => PyNotImplementedError::new_err(error.to_string()),
         _ => PyRuntimeError::new_err(error.to_string()),
     }
 }
diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs
index a9475429e45..d314cd41ac5 100644
--- a/litellm-rust/crates/python-bridge/src/cache/native.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/native.rs
@@ -1,13 +1,18 @@
 use std::{sync::Arc, time::Duration};
 
-use litellm_cache::{CacheCodec, CacheConnectionResult, Error};
+use litellm_cache::{
+    CacheCodec, CacheConnectionResult, Error, ExactCacheContext, SemanticCacheContext,
+};
 use litellm_cache_memory::InMemoryCache;
 use litellm_cache_redis::RedisCache;
 use litellm_cache_response::{
     CacheEntry, PartialHits, ResponseCache, ResponseCacheCodec, ResponseCacheRequest, WriteBuffer,
 };
+use litellm_cache_valkey_semantic::{ValkeySemanticCache, ValkeySemanticConfig};
 use serde_json::Value;
 
+use super::{embedder::PythonEmbedder, request::NativeRequest};
+
 #[derive(Clone)]
 pub(super) enum NativeResponseCache {
     Memory(Arc<ResponseCache<InMemoryCache<CacheEntry>>>),
@@ -15,6 +20,10 @@ pub(super) enum NativeResponseCache {
         cache: Arc<ResponseCache<RedisCache<ResponseCacheCodec>>>,
         buffer: Option<Arc<WriteBuffer>>,
     },
+    ValkeySemantic {
+        cache: Arc<ResponseCache<ValkeySemanticCache<PythonEmbedder, ResponseCacheCodec>>>,
+        scope: String,
+    },
 }
 
 impl NativeResponseCache {
@@ -43,41 +52,52 @@ impl NativeResponseCache {
             buffer: None,
         })
     }
-}
 
-impl NativeResponseCache {
-    pub fn kind(&self) -> &'static str {
-        match self {
-            Self::Memory(_) => "memory",
-            Self::Redis { .. } => "redis",
+    pub fn valkey_semantic(
+        url: &str,
+        similarity_threshold: f64,
+        index_name: String,
+        embedder: PythonEmbedder,
+    ) -> Result<Self, Error> {
+        let backend = ValkeySemanticCache::new(
+            url,
+            embedder,
+            ResponseCacheCodec,
+            ValkeySemanticConfig {
+                similarity_threshold,
+                index_name,
+            },
+        )?;
+        Ok(Self::ValkeySemantic {
+            cache: Arc::new(ResponseCache::new(Arc::new(backend))),
+            scope: String::from("key"),
+        })
+    }
+
+    fn exact(request: &NativeRequest) -> ResponseCacheRequest<ExactCacheContext> {
+        ResponseCacheRequest {
+            key: request.key.clone(),
+            controls: request.controls,
+            context: ExactCacheContext { ttl: request.ttl },
+            max_age: request.max_age,
         }
     }
 
-    pub fn default_ttl(&self) -> Option<Duration> {
-        match self {
-            Self::Memory(cache) => cache.default_ttl(),
-            Self::Redis { cache, .. } => cache.default_ttl(),
-        }
-    }
-
-    pub fn namespace(&self) -> Option<&str> {
-        match self {
-            Self::Memory(_) => None,
-            Self::Redis { cache, .. } => cache.backend().namespace(),
-        }
-    }
-
-    pub fn capacity(&self) -> Option<usize> {
-        match self {
-            Self::Memory(cache) => Some(cache.backend().max_size_in_memory()),
-            Self::Redis { .. } => None,
-        }
-    }
-
-    pub fn max_entry_bytes(&self) -> Option<usize> {
-        match self {
-            Self::Memory(cache) => cache.backend().max_entry_bytes(),
-            Self::Redis { .. } => None,
+    fn semantic(
+        request: &NativeRequest,
+        scope: &str,
+    ) -> ResponseCacheRequest<SemanticCacheContext> {
+        ResponseCacheRequest {
+            key: request.key.clone(),
+            controls: request.controls,
+            context: SemanticCacheContext {
+                input: request.input.clone(),
+                messages: request.messages.clone(),
+                metadata: request.metadata.clone(),
+                scope: Some(scope.to_owned()),
+                ttl: request.ttl,
+            },
+            max_age: request.max_age,
         }
     }
 
@@ -85,95 +105,206 @@ impl NativeResponseCache {
         match self {
             Self::Redis { cache, .. } => Self::Redis {
                 cache,
-                buffer: flush_size.map(|flush_size| Arc::new(WriteBuffer::new(flush_size))),
+                buffer: flush_size.map(|size| Arc::new(WriteBuffer::new(size))),
             },
-            memory => memory,
+            value => value,
         }
     }
 
-    pub fn lookup(
-        &self,
-        request: &ResponseCacheRequest,
-        now: Duration,
-    ) -> Result<Option<Value>, Error> {
+    pub fn with_scope(self, scope: String) -> Self {
         match self {
-            Self::Memory(cache) => cache.lookup(request, now),
-            Self::Redis { cache, .. } => cache.lookup(request, now),
+            Self::ValkeySemantic { cache, .. } => Self::ValkeySemantic { cache, scope },
+            value => value,
+        }
+    }
+
+    pub fn kind(&self) -> &'static str {
+        match self {
+            Self::Memory(_) => "memory",
+            Self::Redis { .. } => "redis",
+            Self::ValkeySemantic { .. } => "valkey-semantic",
+        }
+    }
+
+    pub fn default_ttl(&self) -> Option<Duration> {
+        match self {
+            Self::Memory(cache) => cache.default_ttl(),
+            Self::Redis { cache, .. } => cache.default_ttl(),
+            Self::ValkeySemantic { cache, .. } => cache.default_ttl(),
+        }
+    }
+
+    pub fn namespace(&self) -> Option<&str> {
+        match self {
+            Self::Memory(_) | Self::ValkeySemantic { .. } => None,
+            Self::Redis { cache, .. } => cache.backend().namespace(),
+        }
+    }
+
+    pub fn capacity(&self) -> Option<usize> {
+        match self {
+            Self::Memory(cache) => Some(cache.backend().max_size_in_memory()),
+            Self::Redis { .. } | Self::ValkeySemantic { .. } => None,
+        }
+    }
+
+    pub fn max_entry_bytes(&self) -> Option<usize> {
+        match self {
+            Self::Memory(cache) => cache.backend().max_entry_bytes(),
+            Self::Redis { .. } | Self::ValkeySemantic { .. } => None,
+        }
+    }
+
+    pub fn semantic_config(&self) -> Option<(f64, &str)> {
+        match self {
+            Self::ValkeySemantic { cache, .. } => Some((
+                cache.backend().similarity_threshold(),
+                cache.backend().index_name(),
+            )),
+            _ => None,
+        }
+    }
+
+    pub fn lookup(&self, request: &NativeRequest, now: Duration) -> Result<Option<Value>, Error> {
+        match self {
+            Self::Memory(cache) => cache.lookup(&Self::exact(request), now),
+            Self::Redis { cache, .. } => cache.lookup(&Self::exact(request), now),
+            Self::ValkeySemantic { cache, scope } => {
+                cache.lookup(&Self::semantic(request, scope), now)
+            }
         }
     }
 
     pub fn store(
         &self,
-        request: &ResponseCacheRequest,
+        request: &NativeRequest,
         response: Value,
         now: Duration,
     ) -> Result<(), Error> {
         match self {
-            Self::Memory(cache) => cache.store(request, response, now),
-            Self::Redis { cache, .. } => cache.store(request, response, now),
+            Self::Memory(cache) => cache.store(&Self::exact(request), response, now),
+            Self::Redis { cache, .. } => cache.store(&Self::exact(request), response, now),
+            Self::ValkeySemantic { cache, scope } => {
+                cache.store(&Self::semantic(request, scope), response, now)
+            }
         }
     }
 
     pub fn lookup_batch(
         &self,
-        requests: &[ResponseCacheRequest],
+        requests: &[NativeRequest],
         now: Duration,
     ) -> Result<PartialHits, Error> {
         match self {
-            Self::Memory(cache) => cache.lookup_batch(requests, now),
-            Self::Redis { cache, .. } => cache.lookup_batch(requests, now),
+            Self::Memory(cache) => {
+                let requests = requests.iter().map(Self::exact).collect::<Vec<_>>();
+                cache.lookup_batch(&requests, now)
+            }
+            Self::Redis { cache, .. } => {
+                let requests = requests.iter().map(Self::exact).collect::<Vec<_>>();
+                cache.lookup_batch(&requests, now)
+            }
+            Self::ValkeySemantic { .. } => Err(Error::UnsupportedOperation),
         }
     }
 
     pub async fn async_lookup(
         &self,
-        request: &ResponseCacheRequest,
+        request: &NativeRequest,
         now: Duration,
     ) -> Result<Option<Value>, Error> {
         match self {
-            Self::Memory(cache) => cache.async_lookup(request, now).await,
-            Self::Redis { cache, .. } => cache.async_lookup(request, now).await,
+            Self::Memory(cache) => cache.async_lookup(&Self::exact(request), now).await,
+            Self::Redis { cache, .. } => cache.async_lookup(&Self::exact(request), now).await,
+            Self::ValkeySemantic { cache, scope } => {
+                cache
+                    .async_lookup(&Self::semantic(request, scope), now)
+                    .await
+            }
         }
     }
 
     pub async fn async_store(
         &self,
-        request: &ResponseCacheRequest,
+        request: &NativeRequest,
         response: Value,
         now: Duration,
     ) -> Result<(), Error> {
         match self {
-            Self::Memory(cache) => cache.async_store(request, response, now).await,
+            Self::Memory(cache) => {
+                cache
+                    .async_store(&Self::exact(request), response, now)
+                    .await
+            }
             Self::Redis {
                 cache,
                 buffer: None,
-            } => cache.async_store(request, response, now).await,
+            } => {
+                cache
+                    .async_store(&Self::exact(request), response, now)
+                    .await
+            }
             Self::Redis {
                 cache,
                 buffer: Some(buffer),
-            } => buffer.async_store(cache, request, response, now).await,
+            } => {
+                buffer
+                    .async_store(cache, &Self::exact(request), response, now)
+                    .await
+            }
+            Self::ValkeySemantic { cache, scope } => {
+                cache
+                    .async_store(&Self::semantic(request, scope), response, now)
+                    .await
+            }
         }
     }
 
     pub async fn async_lookup_batch(
         &self,
-        requests: &[ResponseCacheRequest],
+        requests: &[NativeRequest],
         now: Duration,
     ) -> Result<PartialHits, Error> {
         match self {
-            Self::Memory(cache) => cache.async_lookup_batch(requests, now).await,
-            Self::Redis { cache, .. } => cache.async_lookup_batch(requests, now).await,
+            Self::Memory(cache) => {
+                let requests = requests.iter().map(Self::exact).collect::<Vec<_>>();
+                cache.async_lookup_batch(&requests, now).await
+            }
+            Self::Redis { cache, .. } => {
+                let requests = requests.iter().map(Self::exact).collect::<Vec<_>>();
+                cache.async_lookup_batch(&requests, now).await
+            }
+            Self::ValkeySemantic { .. } => Err(Error::UnsupportedOperation),
         }
     }
 
     pub async fn async_store_batch(
         &self,
-        entries: Vec<(ResponseCacheRequest, Value)>,
+        entries: Vec<(NativeRequest, Value)>,
         now: Duration,
     ) -> Result<(), Error> {
         match self {
-            Self::Memory(cache) => cache.async_store_batch(entries, now).await,
-            Self::Redis { cache, .. } => cache.async_store_batch(entries, now).await,
+            Self::Memory(cache) => {
+                let entries = entries
+                    .into_iter()
+                    .map(|(request, value)| (Self::exact(&request), value))
+                    .collect();
+                cache.async_store_batch(entries, now).await
+            }
+            Self::Redis { cache, .. } => {
+                let entries = entries
+                    .into_iter()
+                    .map(|(request, value)| (Self::exact(&request), value))
+                    .collect();
+                cache.async_store_batch(entries, now).await
+            }
+            Self::ValkeySemantic { cache, scope } => {
+                let entries = entries
+                    .into_iter()
+                    .map(|(request, value)| (Self::semantic(&request, scope), value))
+                    .collect();
+                cache.async_store_batch(entries, now).await
+            }
         }
     }
 
@@ -186,6 +317,7 @@ impl NativeResponseCache {
                 }
                 cache.async_flush().await
             }
+            Self::ValkeySemantic { .. } => Err(Error::UnsupportedOperation),
         }
     }
 
@@ -193,6 +325,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.test_connection().await,
             Self::Redis { cache, .. } => cache.test_connection().await,
+            Self::ValkeySemantic { cache, .. } => cache.test_connection().await,
         }
     }
 }
diff --git a/litellm-rust/crates/python-bridge/src/cache/request.rs b/litellm-rust/crates/python-bridge/src/cache/request.rs
index 52a5f7d9055..3e19e7fdc22 100644
--- a/litellm-rust/crates/python-bridge/src/cache/request.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/request.rs
@@ -5,6 +5,7 @@ use litellm_cache_response::{CacheControls, CacheKeyInput, ResponseCacheRequest}
 use litellm_host_python::from_py;
 use pyo3::{exceptions::PyValueError, prelude::*};
 use serde::Deserialize;
+use serde_json::Value;
 
 #[derive(Deserialize)]
 #[serde(deny_unknown_fields)]
@@ -13,24 +14,42 @@ struct RequestInput {
     controls: Option<CacheControls>,
     ttl_seconds: Option<f64>,
     max_age_seconds: Option<f64>,
+    messages: Option<Value>,
+    input: Option<Value>,
+    metadata: Option<Value>,
 }
 
-pub(super) fn request(value: &Bound<'_, PyAny>) -> PyResult<ResponseCacheRequest> {
+pub(super) struct NativeRequest {
+    pub(super) key: CacheKeyInput,
+    pub(super) controls: CacheControls,
+    pub(super) ttl: Option<Duration>,
+    pub(super) max_age: Option<Duration>,
+    pub(super) messages: Option<Value>,
+    pub(super) input: Option<Value>,
+    pub(super) metadata: Option<Value>,
+}
+
+pub(super) fn request(value: &Bound<'_, PyAny>) -> PyResult<NativeRequest> {
     let input: RequestInput = from_py(value)?;
     request_input(input)
 }
 
-fn request_input(input: RequestInput) -> PyResult<ResponseCacheRequest> {
-    let mut request = ResponseCacheRequest::<ExactCacheContext>::new(input.key);
-    if let Some(controls) = input.controls {
-        request.controls = controls;
-    }
-    request.context.ttl = input.ttl_seconds.map(duration).transpose()?;
-    request.max_age = input.max_age_seconds.map(duration).transpose()?;
-    Ok(request)
+fn request_input(input: RequestInput) -> PyResult<NativeRequest> {
+    let controls = input.controls.unwrap_or_else(|| {
+        ResponseCacheRequest::<ExactCacheContext>::new(input.key.clone()).controls
+    });
+    Ok(NativeRequest {
+        key: input.key,
+        controls,
+        ttl: input.ttl_seconds.map(duration).transpose()?,
+        max_age: input.max_age_seconds.map(duration).transpose()?,
+        messages: input.messages,
+        input: input.input,
+        metadata: input.metadata,
+    })
 }
 
-pub(super) fn requests(value: &Bound<'_, PyAny>) -> PyResult<Vec<ResponseCacheRequest>> {
+pub(super) fn requests(value: &Bound<'_, PyAny>) -> PyResult<Vec<NativeRequest>> {
     from_py::<Vec<RequestInput>>(value)?
         .into_iter()
         .map(request_input)
diff --git a/tests/test_litellm_rust/test_valkey_semantic_cache.py b/tests/test_litellm_rust/test_valkey_semantic_cache.py
new file mode 100644
index 00000000000..00037e6e29f
--- /dev/null
+++ b/tests/test_litellm_rust/test_valkey_semantic_cache.py
@@ -0,0 +1,149 @@
+import os
+from collections.abc import Generator, Mapping
+from types import SimpleNamespace
+from typing import Final, cast
+from uuid import uuid4
+
+import pytest
+import redis
+
+from litellm.caching.caching import Cache
+from litellm.caching.valkey_semantic_cache import ValkeySemanticCache
+from litellm.rust_bridge import _native
+from litellm.types.caching import LiteLLMCacheType
+
+pytestmark: Final = pytest.mark.requires_rust_extension
+
+
+@pytest.fixture
+def valkey_url() -> str:
+    url: Final = os.environ.get("LITELLM_TEST_VALKEY_URL")
+    if url is None:
+        pytest.skip("LITELLM_TEST_VALKEY_URL is not set")
+    return url
+
+
+@pytest.fixture
+def index_name(valkey_url: str) -> Generator[str]:
+    index: Final = f"litellm_test_{uuid4().hex}"
+    yield index
+    client: Final = redis.Redis.from_url(valkey_url)
+    try:
+        client.ft(index).dropindex(delete_documents=True)
+    except redis.ResponseError:
+        pass
+    finally:
+        client.close()
+
+
+def _request() -> dict[str, object]:
+    return {
+        "key": {"preset": "key"},
+        "messages": [{"role": "user", "content": "semantic cache prompt"}],
+    }
+
+
+def _backend(url: str, index_name: str) -> ValkeySemanticCache:
+    backend: Final = ValkeySemanticCache(
+        redis_url=url,
+        similarity_threshold=0.8,
+        index_name=index_name,
+    )
+    backend._get_embedding = lambda prompt, metadata=None: [1.0, 0.0]
+
+    async def async_embedding(prompt: str, metadata: dict[str, object] | None = None) -> list[float]:
+        return [1.0, 0.0]
+
+    backend._get_async_embedding = async_embedding
+    return backend
+
+
+def test_python_write_native_read(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    backend: Final = _backend(valkey_url, index_name)
+    response: Final = {"answer": "python"}
+    backend.set_cache("key", response, messages=_request()["messages"])
+    handle: Final = _native._CacheTestHandle.valkey_semantic(
+        valkey_url,
+        0.8,
+        index_name,
+        backend,
+    )
+    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    assert binding.lookup(_request()) == response
+
+
+def test_native_write_python_read(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    backend: Final = _backend(valkey_url, index_name)
+    handle: Final = _native._CacheTestHandle.valkey_semantic(
+        valkey_url,
+        0.8,
+        index_name,
+        backend,
+    )
+    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    response: Final = {"answer": "native"}
+    binding.store({**_request(), "ttl_seconds": 2.0}, response)
+    cached: Final = cast(Mapping[str, object], backend.get_cache("key", messages=_request()["messages"]))
+    assert cached["response"] == response
+
+
+async def test_async_lookup_and_store(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    backend: Final = _backend(valkey_url, index_name)
+    handle: Final = _native._CacheTestHandle.valkey_semantic(
+        valkey_url,
+        0.8,
+        index_name,
+        backend,
+    )
+    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    request: Final = {**_request(), "ttl_seconds": 2.0}
+    await binding.async_store(request, {"answer": "async"})
+    assert await binding.async_lookup(request) == {"answer": "async"}
+
+
+def test_facade_activation_and_mutation_fallback(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    facade: Final = Cache(
+        type=LiteLLMCacheType.VALKEY_SEMANTIC,
+        redis_url=valkey_url,
+        similarity_threshold=0.8,
+        valkey_semantic_cache_index_name=index_name,
+    )
+    handle: Final = _native._CacheTestHandle.valkey_semantic(
+        valkey_url,
+        0.8,
+        index_name,
+        facade.cache,
+    )
+    handle._bind_facade(facade)
+    resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade))
+    assert resolver.resolve().kind == "native"
+    facade.cache.similarity_threshold = 0.7
+    assert resolver.resolve().kind == "python_callback"
+
+
+def test_batch_lookup_is_unsupported(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    backend: Final = _backend(valkey_url, index_name)
+    handle: Final = _native._CacheTestHandle.valkey_semantic(
+        valkey_url,
+        0.8,
+        index_name,
+        backend,
+    )
+    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    with pytest.raises(NotImplementedError):
+        binding.lookup_batch([_request()])

From f6db876a3d72e143fd6638f226ac4c01a34ca088 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:34:21 +0000
Subject: [PATCH 072/160] test(cache): disambiguate Valkey semantic test module

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 ...key_semantic_cache.py => test_valkey_semantic_cache_native.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename tests/test_litellm_rust/{test_valkey_semantic_cache.py => test_valkey_semantic_cache_native.py} (100%)

diff --git a/tests/test_litellm_rust/test_valkey_semantic_cache.py b/tests/test_litellm_rust/test_valkey_semantic_cache_native.py
similarity index 100%
rename from tests/test_litellm_rust/test_valkey_semantic_cache.py
rename to tests/test_litellm_rust/test_valkey_semantic_cache_native.py

From a69ebb7ac4584e3a91497e44fd65b4bf86d52815 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:34:39 +0000
Subject: [PATCH 073/160] feat(cache): add the unsupported operation error

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/cache/src/error.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/litellm-rust/crates/cache/src/error.rs b/litellm-rust/crates/cache/src/error.rs
index ff3ff6572d4..2418ab978dc 100644
--- a/litellm-rust/crates/cache/src/error.rs
+++ b/litellm-rust/crates/cache/src/error.rs
@@ -6,4 +6,6 @@ pub enum Error {
     InvalidEntry,
     #[error("flushing Redis requires an explicit namespace")]
     UnscopedFlush,
+    #[error("cache backend does not support this operation")]
+    UnsupportedOperation,
 }

From c311073a178d295a5133d2e68300c9f5f83664bc Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:34:51 +0000
Subject: [PATCH 074/160] feat(cache-redis-semantic): expose backend accessors
 for bridge binding

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../crates/cache-redis-semantic/src/cache.rs  | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/litellm-rust/crates/cache-redis-semantic/src/cache.rs b/litellm-rust/crates/cache-redis-semantic/src/cache.rs
index 26d19d34670..5aa484356d7 100644
--- a/litellm-rust/crates/cache-redis-semantic/src/cache.rs
+++ b/litellm-rust/crates/cache-redis-semantic/src/cache.rs
@@ -41,15 +41,6 @@ pub struct RedisSemanticConfig {
     pub similarity_threshold: f32,
 }
 
-impl Default for RedisSemanticConfig {
-    fn default() -> Self {
-        Self {
-            index_name: DEFAULT_INDEX_NAME.into(),
-            similarity_threshold: 0.9,
-        }
-    }
-}
-
 struct Inner {
     index_name: String,
     distance_threshold: f64,
@@ -247,6 +238,18 @@ impl<E: Embedder, C: redis::ConnectionLike + Send + 'static> RedisSemanticCache<
         }
     }
 
+    pub fn embedder(&self) -> &E {
+        &self.embedder
+    }
+
+    pub fn index_name(&self) -> &str {
+        &self.inner.index_name
+    }
+
+    pub fn similarity_threshold(&self) -> f32 {
+        (1.0 - self.inner.distance_threshold) as f32
+    }
+
     fn tag<'a>(key: &'a str, context: &'a SemanticCacheContext) -> &'a str {
         context.scope.as_deref().unwrap_or(key)
     }

From ac1c4a399ff2dcdf82ce82b93ef95ac748d1d01c Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:34:57 +0000
Subject: [PATCH 075/160] refactor(cache-redis-semantic): drop the unused
 default index name

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/cache-redis-semantic/src/cache.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/litellm-rust/crates/cache-redis-semantic/src/cache.rs b/litellm-rust/crates/cache-redis-semantic/src/cache.rs
index 5aa484356d7..b1440e80de5 100644
--- a/litellm-rust/crates/cache-redis-semantic/src/cache.rs
+++ b/litellm-rust/crates/cache-redis-semantic/src/cache.rs
@@ -17,7 +17,6 @@ use crate::prompt::prompt_from_context;
 
 const REDIS_TIMEOUT: Duration = Duration::from_secs(5);
 const REDIS_POOL_SIZE: u32 = 16;
-const DEFAULT_INDEX_NAME: &str = "litellm_semantic_cache_index";
 const CACHE_KEY_FIELD: &str = "litellm_cache_key";
 const VECTOR_FIELD: &str = "prompt_vector";
 

From 788e24655a0e253f553af95da8ac69a51552a6cd Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:35:08 +0000
Subject: [PATCH 076/160] feat(rust): add native S3 cache backend

Mirror the Redis vertical slice for S3Cache: a litellm-cache-s3 crate built on aws-sdk-s3 with path-style custom endpoints, python-identical put_object metadata (cache-control, expires, content headers), expires-aware get_object, and no-op flush/unsupported test_connection. Wire it through python-bridge config projection, facade guards, test handle, and binding dispatch, plus an in-process S3 stub and parity tests.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.lock                       | 165 +++++++++-
 litellm-rust/Cargo.toml                       |   1 +
 litellm-rust/crates/cache-s3/Cargo.toml       |  20 ++
 litellm-rust/crates/cache-s3/src/auth.rs      |  32 ++
 litellm-rust/crates/cache-s3/src/cache.rs     | 204 +++++++++++++
 litellm-rust/crates/cache-s3/src/lib.rs       |   4 +
 litellm-rust/crates/cache-s3/tests/cache.rs   | 264 ++++++++++++++++
 litellm-rust/crates/cache/src/error.rs        |   2 +
 litellm-rust/crates/framer/Cargo.toml         |   2 +-
 litellm-rust/crates/llms/Cargo.toml           |   2 +-
 litellm-rust/crates/python-bridge/Cargo.toml  |   2 +
 .../crates/python-bridge/src/cache/config.rs  | 281 +++++++++++++++++-
 .../crates/python-bridge/src/cache/facade.rs  |  39 ++-
 .../crates/python-bridge/src/cache/handle.rs  |  38 ++-
 .../crates/python-bridge/src/cache/mod.rs     |   3 +-
 .../crates/python-bridge/src/cache/native.rs  |  43 ++-
 litellm/rust_bridge/_native.pyi               |  90 ++++++
 tests/test_litellm_rust/support/s3_stub.py    | 112 +++++++
 tests/test_litellm_rust/test_cache.py         | 182 ++++++++++++
 19 files changed, 1465 insertions(+), 21 deletions(-)
 create mode 100644 litellm-rust/crates/cache-s3/Cargo.toml
 create mode 100644 litellm-rust/crates/cache-s3/src/auth.rs
 create mode 100644 litellm-rust/crates/cache-s3/src/cache.rs
 create mode 100644 litellm-rust/crates/cache-s3/src/lib.rs
 create mode 100644 litellm-rust/crates/cache-s3/tests/cache.rs
 create mode 100644 tests/test_litellm_rust/support/s3_stub.py

diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index ed4ae4e3353..4018ee2875b 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -40,6 +40,12 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "allocator-api2"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
+
 [[package]]
 name = "android_system_properties"
 version = "0.1.6"
@@ -208,6 +214,7 @@ dependencies = [
  "aws-credential-types",
  "aws-sigv4",
  "aws-smithy-async",
+ "aws-smithy-eventstream",
  "aws-smithy-http",
  "aws-smithy-runtime",
  "aws-smithy-runtime-api",
@@ -216,7 +223,9 @@ dependencies = [
  "bytes",
  "bytes-utils",
  "fastrand",
+ "http 0.2.12",
  "http 1.4.2",
+ "http-body 0.4.6",
  "http-body 1.1.0",
  "percent-encoding",
  "pin-project-lite",
@@ -250,6 +259,43 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "aws-sdk-s3"
+version = "1.148.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a61c1db3987ab6c8740fb87f248864687d50549401115244a84f02c3047b2d5"
+dependencies = [
+ "arc-swap",
+ "aws-credential-types",
+ "aws-runtime",
+ "aws-sigv4",
+ "aws-smithy-async",
+ "aws-smithy-checksums",
+ "aws-smithy-eventstream",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-observability",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-schema",
+ "aws-smithy-types",
+ "aws-smithy-xml 0.62.1",
+ "aws-types",
+ "bytes",
+ "fastrand",
+ "hex",
+ "hmac",
+ "http 0.2.12",
+ "http 1.4.2",
+ "http-body 1.1.0",
+ "lru",
+ "percent-encoding",
+ "regex-lite",
+ "sha2 0.11.0",
+ "tracing",
+ "url",
+]
+
 [[package]]
 name = "aws-sdk-secretsmanager"
 version = "1.117.0"
@@ -294,7 +340,7 @@ dependencies = [
  "aws-smithy-runtime-api",
  "aws-smithy-schema",
  "aws-smithy-types",
- "aws-smithy-xml",
+ "aws-smithy-xml 0.61.1",
  "aws-types",
  "fastrand",
  "http 0.2.12",
@@ -310,6 +356,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "31d955e76ff96acd555bf06fa0fa6d5bf9335fa84ae7c64481b20ae61d231f70"
 dependencies = [
  "aws-credential-types",
+ "aws-smithy-eventstream",
  "aws-smithy-http",
  "aws-smithy-runtime-api",
  "aws-smithy-types",
@@ -337,10 +384,31 @@ dependencies = [
 ]
 
 [[package]]
-name = "aws-smithy-eventstream"
-version = "0.61.1"
+name = "aws-smithy-checksums"
+version = "0.65.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a9381123ab62d20c13082b151f30f962a3b112b727345394536dfa39a482944"
+checksum = "b67ecd999972b58e67cab052f5129906c08c25883bd0788ceefc55ef97d61307"
+dependencies = [
+ "aws-smithy-http",
+ "aws-smithy-types",
+ "bytes",
+ "crc-fast",
+ "hex",
+ "http 1.4.2",
+ "http-body 1.1.0",
+ "http-body-util",
+ "md-5",
+ "pin-project-lite",
+ "sha1 0.11.0",
+ "sha2 0.11.0",
+ "tracing",
+]
+
+[[package]]
+name = "aws-smithy-eventstream"
+version = "0.61.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "80c2051c2f1016fb8e6548dd07b8bc2ac9c3fe583721444b92f515e856d31609"
 dependencies = [
  "aws-smithy-types",
  "bytes",
@@ -353,6 +421,7 @@ version = "0.64.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d"
 dependencies = [
+ "aws-smithy-eventstream",
  "aws-smithy-runtime-api",
  "aws-smithy-types",
  "bytes",
@@ -532,6 +601,18 @@ dependencies = [
  "xmlparser",
 ]
 
+[[package]]
+name = "aws-smithy-xml"
+version = "0.62.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b932c8d6dc127fc980eecd78f8694ae9b9551b69a93a7def2a199c1c0033daf"
+dependencies = [
+ "aws-smithy-runtime-api",
+ "aws-smithy-schema",
+ "aws-smithy-types",
+ "xmlparser",
+]
+
 [[package]]
 name = "aws-types"
 version = "1.6.0"
@@ -927,6 +1008,16 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "crc-fast"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5"
+dependencies = [
+ "digest 0.10.7",
+ "spin",
+]
+
 [[package]]
 name = "crc32fast"
 version = "1.5.1"
@@ -1369,6 +1460,12 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foldhash"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.2.2"
@@ -1838,6 +1935,11 @@ name = "hashbrown"
 version = "0.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash",
+]
 
 [[package]]
 name = "heck"
@@ -2502,6 +2604,21 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "litellm-cache-s3"
+version = "0.1.0"
+dependencies = [
+ "aws-credential-types",
+ "aws-sdk-s3",
+ "aws-smithy-types",
+ "aws-types",
+ "litellm-auth-aws",
+ "litellm-cache",
+ "serde_json",
+ "tokio",
+ "wiremock",
+]
+
 [[package]]
 name = "litellm-callbacks-legacy-python"
 version = "0.1.0"
@@ -2664,11 +2781,13 @@ dependencies = [
  "criterion",
  "futures-util",
  "litellm-auth",
+ "litellm-auth-aws",
  "litellm-auth-gcp",
  "litellm-cache",
  "litellm-cache-memory",
  "litellm-cache-redis",
  "litellm-cache-response",
+ "litellm-cache-s3",
  "litellm-callbacks-legacy-python",
  "litellm-core",
  "litellm-core-utils",
@@ -2846,6 +2965,15 @@ version = "0.4.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
 
+[[package]]
+name = "lru"
+version = "0.18.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25"
+dependencies = [
+ "hashbrown 0.17.1",
+]
+
 [[package]]
 name = "lru-slab"
 version = "0.1.2"
@@ -2868,6 +2996,16 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c"
 
+[[package]]
+name = "md-5"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98"
+dependencies = [
+ "cfg-if",
+ "digest 0.11.3",
+]
+
 [[package]]
 name = "memchr"
 version = "2.8.3"
@@ -4297,6 +4435,17 @@ dependencies = [
  "digest 0.10.7",
 ]
 
+[[package]]
+name = "sha1"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214"
+dependencies = [
+ "cfg-if",
+ "cpufeatures 0.3.0",
+ "digest 0.11.3",
+]
+
 [[package]]
 name = "sha1_smol"
 version = "1.0.1"
@@ -4413,6 +4562,12 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "spin"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3"
+
 [[package]]
 name = "spm_precompiled"
 version = "0.1.4"
@@ -5010,7 +5165,7 @@ dependencies = [
  "rand 0.8.7",
  "rustls 0.23.42",
  "rustls-pki-types",
- "sha1",
+ "sha1 0.10.7",
  "thiserror 1.0.69",
  "utf-8",
 ]
diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
index 570d0dd3568..031cfcbbb21 100644
--- a/litellm-rust/Cargo.toml
+++ b/litellm-rust/Cargo.toml
@@ -29,6 +29,7 @@ litellm-core-utils = { path = "crates/core-utils" }
 litellm-cache = { path = "crates/cache" }
 litellm-cache-memory = { path = "crates/cache-memory" }
 litellm-cache-redis = { path = "crates/cache-redis" }
+litellm-cache-s3 = { path = "crates/cache-s3" }
 litellm-cache-response = { path = "crates/cache-response" }
 litellm-token-counter = { path = "crates/token-counter" }
 litellm-token-counter-fast = { path = "crates/token-counter-fast" }
diff --git a/litellm-rust/crates/cache-s3/Cargo.toml b/litellm-rust/crates/cache-s3/Cargo.toml
new file mode 100644
index 00000000000..c5d084207c8
--- /dev/null
+++ b/litellm-rust/crates/cache-s3/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "litellm-cache-s3"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-cache.workspace = true
+litellm-auth-aws.workspace = true
+aws-sdk-s3 = { version = "1.141.0", default-features = false, features = ["rustls", "rt-tokio"] }
+aws-credential-types = "1.3.0"
+aws-smithy-types = "1.6.0"
+aws-types = "1.6.0"
+tokio.workspace = true
+
+[dev-dependencies]
+wiremock = "0.6.5"
+serde_json.workspace = true
+tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
diff --git a/litellm-rust/crates/cache-s3/src/auth.rs b/litellm-rust/crates/cache-s3/src/auth.rs
new file mode 100644
index 00000000000..8f7eb66059d
--- /dev/null
+++ b/litellm-rust/crates/cache-s3/src/auth.rs
@@ -0,0 +1,32 @@
+use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future};
+use litellm_auth_aws::{AwsAuthConfig, resolve_credentials};
+
+#[derive(Clone)]
+pub(crate) struct Credentials {
+    config: AwsAuthConfig,
+}
+
+impl Credentials {
+    pub(crate) fn new(config: AwsAuthConfig) -> Self {
+        Self { config }
+    }
+}
+
+impl ProvideCredentials for Credentials {
+    fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a>
+    where
+        Self: 'a,
+    {
+        future::ProvideCredentials::new(async {
+            resolve_credentials(self.config.clone(), &|name| std::env::var(name).ok())
+                .await
+                .map_err(|_| CredentialsError::provider_error("S3 cache authentication failed"))
+        })
+    }
+}
+
+impl std::fmt::Debug for Credentials {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Credentials").finish_non_exhaustive()
+    }
+}
diff --git a/litellm-rust/crates/cache-s3/src/cache.rs b/litellm-rust/crates/cache-s3/src/cache.rs
new file mode 100644
index 00000000000..1aa16ecaaa0
--- /dev/null
+++ b/litellm-rust/crates/cache-s3/src/cache.rs
@@ -0,0 +1,204 @@
+use std::{
+    future::Future,
+    sync::Arc,
+    time::{Duration, SystemTime},
+};
+
+use aws_sdk_s3::{
+    config::{BehaviorVersion, Region, RequestChecksumCalculation, ResponseChecksumValidation},
+    error::SdkError,
+    primitives::ByteStream,
+};
+use aws_smithy_types::{DateTime, date_time::Format};
+use litellm_auth_aws::AwsAuthConfig;
+use litellm_cache::{
+    BaseCache, BatchCache, CacheCodec, CacheConnectionResult, Error, ExactCacheContext, FlushCache,
+};
+use tokio::runtime::Handle;
+
+use crate::auth::Credentials;
+
+pub struct S3Endpoint {
+    pub url: String,
+}
+
+pub struct S3CacheConfig {
+    pub bucket: String,
+    pub key_prefix: String,
+    pub region: String,
+    pub endpoint: Option<S3Endpoint>,
+    pub auth: AwsAuthConfig,
+}
+
+pub struct S3Cache<C: CacheCodec> {
+    client: aws_sdk_s3::Client,
+    codec: C,
+    runtime: Handle,
+    bucket: Arc<str>,
+    key_prefix: Arc<str>,
+}
+
+impl<C: CacheCodec> S3Cache<C> {
+    pub fn new(config: S3CacheConfig, codec: C, runtime: Handle) -> Self {
+        let mut builder = aws_sdk_s3::Config::builder()
+            .behavior_version(BehaviorVersion::latest())
+            .region(Region::new(config.region))
+            .credentials_provider(Credentials::new(config.auth))
+            .request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
+            .response_checksum_validation(ResponseChecksumValidation::WhenRequired);
+        if let Some(endpoint) = config.endpoint {
+            builder = builder.endpoint_url(endpoint.url).force_path_style(true);
+        }
+        Self {
+            client: aws_sdk_s3::Client::from_conf(builder.build()),
+            codec,
+            runtime,
+            bucket: config.bucket.into(),
+            key_prefix: config.key_prefix.into(),
+        }
+    }
+
+    pub fn bucket(&self) -> &str {
+        &self.bucket
+    }
+
+    pub fn key_prefix(&self) -> &str {
+        &self.key_prefix
+    }
+
+    pub fn to_s3_key(&self, key: &str) -> String {
+        format!("{}{}", self.key_prefix, key.replace(':', "/"))
+    }
+
+    fn block_on<F: Future>(&self, future: F) -> F::Output {
+        if Handle::try_current().is_ok() {
+            tokio::task::block_in_place(|| self.runtime.block_on(future))
+        } else {
+            self.runtime.block_on(future)
+        }
+    }
+
+    async fn put(
+        &self,
+        key: &str,
+        value: C::Value,
+        context: &ExactCacheContext,
+    ) -> Result<(), Error> {
+        let s3_key = self.to_s3_key(key);
+        let body = self.codec.encode(&value)?;
+        let request = self
+            .client
+            .put_object()
+            .bucket(self.bucket.as_ref())
+            .key(&s3_key)
+            .body(ByteStream::from(body))
+            .content_type("application/json")
+            .content_language("en")
+            .content_disposition(format!("inline; filename=\"{s3_key}.json\""));
+        let request = match context.ttl {
+            Some(ttl) => {
+                let seconds = ttl.as_secs_f64();
+                request
+                    .cache_control(format!("immutable, max-age={seconds}, s-maxage={seconds}"))
+                    .expires(DateTime::from(SystemTime::now() + ttl))
+            }
+            None => request.cache_control("immutable, max-age=31536000, s-maxage=31536000"),
+        };
+        request.send().await.map_err(|_| Error::Unavailable)?;
+        Ok(())
+    }
+
+    async fn get(&self, key: &str) -> Result<Option<C::Value>, Error> {
+        let output = match self
+            .client
+            .get_object()
+            .bucket(self.bucket.as_ref())
+            .key(self.to_s3_key(key))
+            .send()
+            .await
+        {
+            Ok(output) => output,
+            Err(error) => {
+                if let SdkError::ServiceError(service) = &error {
+                    let not_found = service.err().is_no_such_key()
+                        || error
+                            .raw_response()
+                            .map(|response| response.status().as_u16())
+                            == Some(404);
+                    if not_found {
+                        return Ok(None);
+                    }
+                }
+                return Err(Error::Unavailable);
+            }
+        };
+        if let Some(expires) = output.expires_string()
+            && let Ok(expires) = DateTime::from_str(expires, Format::HttpDate)
+            && expires < DateTime::from(SystemTime::now())
+        {
+            return Ok(None);
+        }
+        let bytes = output
+            .body
+            .collect()
+            .await
+            .map_err(|_| Error::Unavailable)?
+            .into_bytes();
+        self.codec.decode(&bytes).map(Some)
+    }
+}
+
+impl<C: CacheCodec> BaseCache for S3Cache<C> {
+    type Value = C::Value;
+    type Context = ExactCacheContext;
+
+    fn get_ttl(&self, _context: &Self::Context) -> Option<Duration> {
+        None
+    }
+
+    fn set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: &Self::Context,
+    ) -> Result<(), Error> {
+        self.block_on(self.put(key, value, context))
+    }
+
+    fn get_cache(&self, key: &str, _context: &Self::Context) -> Result<Option<Self::Value>, Error> {
+        self.block_on(self.get(key))
+    }
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: Self::Context,
+    ) -> Result<(), Error> {
+        self.put(key, value, &context).await
+    }
+
+    async fn async_get_cache(
+        &self,
+        key: &str,
+        _context: &Self::Context,
+    ) -> Result<Option<Self::Value>, Error> {
+        self.get(key).await
+    }
+
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        Err(Error::UnsupportedOperation)
+    }
+}
+
+impl<C: CacheCodec> BatchCache for S3Cache<C> {}
+
+impl<C: CacheCodec> FlushCache for S3Cache<C> {
+    fn flush_cache(&self) -> Result<(), Error> {
+        Ok(())
+    }
+}
diff --git a/litellm-rust/crates/cache-s3/src/lib.rs b/litellm-rust/crates/cache-s3/src/lib.rs
new file mode 100644
index 00000000000..f6126dfa908
--- /dev/null
+++ b/litellm-rust/crates/cache-s3/src/lib.rs
@@ -0,0 +1,4 @@
+mod auth;
+mod cache;
+
+pub use cache::{S3Cache, S3CacheConfig, S3Endpoint};
diff --git a/litellm-rust/crates/cache-s3/tests/cache.rs b/litellm-rust/crates/cache-s3/tests/cache.rs
new file mode 100644
index 00000000000..93e88cf7cbd
--- /dev/null
+++ b/litellm-rust/crates/cache-s3/tests/cache.rs
@@ -0,0 +1,264 @@
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+use litellm_auth_aws::AwsAuthConfig;
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, Error, ExactCacheContext, FlushCache, JsonCodec,
+};
+use litellm_cache_s3::{S3Cache, S3CacheConfig, S3Endpoint};
+use serde_json::{Value, json};
+use tokio::runtime::Handle;
+use wiremock::{
+    Mock, MockServer, ResponseTemplate,
+    matchers::{method, path},
+};
+
+fn config(endpoint: String) -> S3CacheConfig {
+    S3CacheConfig {
+        bucket: "cache-bucket".to_string(),
+        key_prefix: "team/".to_string(),
+        region: "us-east-1".to_string(),
+        endpoint: Some(S3Endpoint { url: endpoint }),
+        auth: AwsAuthConfig {
+            access_key_id: Some("key".to_string()),
+            secret_access_key: Some("secret".to_string()),
+            region_name: Some("us-east-1".to_string()),
+            ..Default::default()
+        },
+    }
+}
+
+fn cache(endpoint: &str) -> S3Cache<JsonCodec<Value>> {
+    S3Cache::new(
+        config(endpoint.to_string()),
+        JsonCodec::<Value>::new(),
+        Handle::current(),
+    )
+}
+
+async fn mock_server() -> MockServer {
+    let server = MockServer::start().await;
+    Mock::given(method("PUT"))
+        .respond_with(ResponseTemplate::new(200).insert_header("etag", "\"etag\""))
+        .mount(&server)
+        .await;
+    server
+}
+
+fn http_date_from(headers: &wiremock::http::HeaderMap, name: &str) -> Option<SystemTime> {
+    use aws_smithy_types::{DateTime, date_time::Format};
+    headers
+        .get(name)
+        .and_then(|value| DateTime::from_str(value.to_str().ok()?, Format::HttpDate).ok())
+        .map(|date| UNIX_EPOCH + Duration::new(date.secs() as u64, date.subsec_nanos()))
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn set_writes_python_metadata_with_and_without_ttl() {
+    let server = mock_server().await;
+    let cache = cache(&server.uri());
+    let context = ExactCacheContext {
+        ttl: Some(Duration::from_secs(90)),
+    };
+    cache
+        .set_cache("alpha:beta", json!({"answer": 1}), &context)
+        .unwrap();
+    cache
+        .set_cache("plain", json!({"answer": 2}), &ExactCacheContext::default())
+        .unwrap();
+
+    let requests = server.received_requests().await.unwrap();
+    let ttl_request = requests
+        .iter()
+        .find(|request| request.url.path() == "/cache-bucket/team/alpha/beta")
+        .expect("ttl write should hit the converted S3 key");
+    assert_eq!(
+        ttl_request.headers["cache-control"].to_str().unwrap(),
+        "immutable, max-age=90, s-maxage=90"
+    );
+    assert_eq!(
+        ttl_request.headers["content-type"].to_str().unwrap(),
+        "application/json"
+    );
+    assert_eq!(
+        ttl_request.headers["content-language"].to_str().unwrap(),
+        "en"
+    );
+    assert_eq!(
+        ttl_request.headers["content-disposition"].to_str().unwrap(),
+        "inline; filename=\"team/alpha/beta.json\""
+    );
+    let expires = http_date_from(&ttl_request.headers, "expires").expect("ttl write sets Expires");
+    let remaining = expires.duration_since(SystemTime::now()).unwrap();
+    assert!(remaining > Duration::from_secs(60) && remaining <= Duration::from_secs(91));
+    assert_eq!(
+        serde_json::from_slice::<Value>(&ttl_request.body).unwrap(),
+        json!({"answer": 1})
+    );
+
+    let plain = requests
+        .iter()
+        .find(|request| request.url.path() == "/cache-bucket/team/plain")
+        .expect("no-ttl write should hit the converted S3 key");
+    assert_eq!(
+        plain.headers["cache-control"].to_str().unwrap(),
+        "immutable, max-age=31536000, s-maxage=31536000"
+    );
+    assert!(plain.headers.get("expires").is_none());
+    assert_eq!(
+        plain.headers["content-disposition"].to_str().unwrap(),
+        "inline; filename=\"team/plain.json\""
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn get_hit_miss_expired_and_invalid_entries() {
+    let server = mock_server().await;
+    Mock::given(method("GET"))
+        .and(path("/cache-bucket/team/hit"))
+        .respond_with(ResponseTemplate::new(200).set_body_json(json!({"answer": 3})))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/cache-bucket/team/missing"))
+        .respond_with(
+            ResponseTemplate::new(404).set_body_string("<Error><Code>NoSuchKey</Code></Error>"),
+        )
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/cache-bucket/team/expired"))
+        .respond_with(
+            ResponseTemplate::new(200)
+                .insert_header("expires", "Thu, 01 Jan 1970 00:00:00 GMT")
+                .set_body_json(json!({"answer": 4})),
+        )
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/cache-bucket/team/malformed"))
+        .respond_with(ResponseTemplate::new(200).set_body_string("not a cache entry"))
+        .mount(&server)
+        .await;
+    let cache = cache(&server.uri());
+    let context = ExactCacheContext::default();
+
+    assert_eq!(
+        cache.get_cache("hit", &context).unwrap(),
+        Some(json!({"answer": 3}))
+    );
+    assert_eq!(cache.get_cache("missing", &context).unwrap(), None);
+    assert_eq!(cache.get_cache("expired", &context).unwrap(), None);
+    assert_eq!(
+        cache.get_cache("malformed", &context),
+        Err(Error::InvalidEntry)
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn batch_get_preserves_order_with_hits_misses_and_invalid() {
+    let server = mock_server().await;
+    for (key, status, body) in [
+        ("first", 200, "{\"answer\": 1}"),
+        ("invalid", 200, "garbage"),
+    ] {
+        Mock::given(method("GET"))
+            .and(path(format!("/cache-bucket/team/{key}")))
+            .respond_with(ResponseTemplate::new(status).set_body_string(body))
+            .mount(&server)
+            .await;
+    }
+    Mock::given(method("GET"))
+        .and(path("/cache-bucket/team/miss"))
+        .respond_with(ResponseTemplate::new(404))
+        .mount(&server)
+        .await;
+    let cache = cache(&server.uri());
+    let context = ExactCacheContext::default();
+    let keys = vec![
+        "first".to_string(),
+        "miss".to_string(),
+        "invalid".to_string(),
+    ];
+
+    let entries = cache.batch_get_cache(&keys, &context).unwrap();
+
+    assert_eq!(
+        entries,
+        vec![
+            BatchEntry::Hit(json!({"answer": 1})),
+            BatchEntry::Miss,
+            BatchEntry::Invalid,
+        ]
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn unsupported_and_noop_capabilities_match_python() {
+    let server = mock_server().await;
+    let cache = cache(&server.uri());
+
+    assert_eq!(
+        cache.test_connection().await,
+        Err(Error::UnsupportedOperation)
+    );
+    cache.flush_cache().unwrap();
+    cache.disconnect().await.unwrap();
+    assert_eq!(cache.get_ttl(&ExactCacheContext::default()), None);
+    assert!(server.received_requests().await.unwrap().is_empty());
+}
+
+#[test]
+fn key_conversion_prefixes_and_splits_colons() {
+    let runtime = tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(1)
+        .enable_all()
+        .build()
+        .unwrap();
+    let _guard = runtime.enter();
+    let cache = S3Cache::new(
+        S3CacheConfig {
+            key_prefix: "team/".to_string(),
+            ..config("http://localhost".to_string())
+        },
+        JsonCodec::<Value>::new(),
+        runtime.handle().clone(),
+    );
+
+    assert_eq!(cache.bucket(), "cache-bucket");
+    assert_eq!(cache.key_prefix(), "team/");
+    assert_eq!(cache.to_s3_key("a:b:c"), "team/a/b/c");
+    assert_eq!(cache.to_s3_key("plain"), "team/plain");
+
+    let unprefixed = S3Cache::new(
+        S3CacheConfig {
+            key_prefix: String::new(),
+            ..config("http://localhost".to_string())
+        },
+        JsonCodec::<Value>::new(),
+        runtime.handle().clone(),
+    );
+    assert_eq!(unprefixed.to_s3_key("a:b"), "a/b");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn sync_methods_block_inside_and_outside_the_runtime() {
+    let server = mock_server().await;
+    Mock::given(method("GET"))
+        .and(path("/cache-bucket/team/key"))
+        .respond_with(ResponseTemplate::new(200).set_body_json(json!({"answer": 9})))
+        .mount(&server)
+        .await;
+    let uri = server.uri();
+    let cache = tokio::task::spawn_blocking(move || {
+        let cache = cache(&uri);
+        let context = ExactCacheContext::default();
+        cache
+            .set_cache("key", json!({"answer": 9}), &context)
+            .unwrap();
+        cache.get_cache("key", &context).unwrap()
+    })
+    .await
+    .unwrap();
+
+    assert_eq!(cache, Some(json!({"answer": 9})));
+}
diff --git a/litellm-rust/crates/cache/src/error.rs b/litellm-rust/crates/cache/src/error.rs
index ff3ff6572d4..e373853e70a 100644
--- a/litellm-rust/crates/cache/src/error.rs
+++ b/litellm-rust/crates/cache/src/error.rs
@@ -6,4 +6,6 @@ pub enum Error {
     InvalidEntry,
     #[error("flushing Redis requires an explicit namespace")]
     UnscopedFlush,
+    #[error("cache operation is not supported")]
+    UnsupportedOperation,
 }
diff --git a/litellm-rust/crates/framer/Cargo.toml b/litellm-rust/crates/framer/Cargo.toml
index d22502f871a..60dfa9469d2 100644
--- a/litellm-rust/crates/framer/Cargo.toml
+++ b/litellm-rust/crates/framer/Cargo.toml
@@ -11,7 +11,7 @@ aws = ["dep:aws-smithy-eventstream", "dep:aws-smithy-types"]
 sse = ["dep:sse-stream"]
 
 [dependencies]
-aws-smithy-eventstream = { version = "=0.61.1", optional = true }
+aws-smithy-eventstream = { version = "0.61.1", optional = true }
 aws-smithy-types = { version = "1.6.1", optional = true }
 bytes = "1"
 futures-util.workspace = true
diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml
index 0cc7af1836f..8bdbb7c3b77 100644
--- a/litellm-rust/crates/llms/Cargo.toml
+++ b/litellm-rust/crates/llms/Cargo.toml
@@ -34,7 +34,7 @@ tokio = { workspace = true, features = ["sync"] }
 url.workspace = true
 
 [dev-dependencies]
-aws-smithy-eventstream = "=0.61.1"
+aws-smithy-eventstream = "0.61.1"
 aws-smithy-types = "1.6.1"
 rstest.workspace = true
 tokio.workspace = true
diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml
index 1eb2ec28036..2aea8122fa6 100644
--- a/litellm-rust/crates/python-bridge/Cargo.toml
+++ b/litellm-rust/crates/python-bridge/Cargo.toml
@@ -23,9 +23,11 @@ bytes.workspace = true
 litellm-cache.workspace = true
 litellm-cache-memory.workspace = true
 litellm-cache-redis.workspace = true
+litellm-cache-s3.workspace = true
 litellm-cache-response.workspace = true
 serde.workspace = true
 litellm-auth.workspace = true
+litellm-auth-aws.workspace = true
 litellm-callbacks-legacy-python.workspace = true
 litellm-core.workspace = true
 litellm-core-utils.workspace = true
diff --git a/litellm-rust/crates/python-bridge/src/cache/config.rs b/litellm-rust/crates/python-bridge/src/cache/config.rs
index 0e7d6aee11d..5d37ed27083 100644
--- a/litellm-rust/crates/python-bridge/src/cache/config.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/config.rs
@@ -1,10 +1,12 @@
 use std::time::Duration;
 
+use litellm_auth_aws::AwsAuthConfig;
 use litellm_cache::CacheType;
+use litellm_cache_s3::{S3CacheConfig, S3Endpoint};
 use pyo3::{
-    exceptions::{PyTypeError, PyValueError},
+    exceptions::{PyAttributeError, PyTypeError, PyValueError},
     prelude::*,
-    types::{PyAny, PyDict, PyString},
+    types::{PyAny, PyBool, PyDict, PyString},
 };
 
 use super::{native::NativeResponseCache, request::duration};
@@ -76,6 +78,7 @@ pub(super) struct RedisCacheConfig {
 pub(super) enum CacheBackendConfig {
     Memory(MemoryCacheConfig),
     Redis(Box<RedisCacheConfig>),
+    S3(Box<S3CacheConfig>),
 }
 
 #[allow(dead_code, reason = "consumed by the cache activation follow-up")]
@@ -90,6 +93,9 @@ pub(super) enum UnsupportedCacheConfig {
     RedisCredentials,
     RedisConnection,
     RedisOption,
+    S3Client,
+    S3Credentials,
+    S3Option,
 }
 
 impl UnsupportedCacheConfig {
@@ -100,6 +106,9 @@ impl UnsupportedCacheConfig {
             Self::RedisCredentials => "native Redis credentials require Python",
             Self::RedisConnection => "native Redis connection type is not implemented",
             Self::RedisOption => "native Redis configuration requires Python",
+            Self::S3Client => "native S3 client type is not implemented",
+            Self::S3Credentials => "native S3 credentials require Python",
+            Self::S3Option => "native S3 configuration requires Python",
         }
     }
 }
@@ -142,10 +151,16 @@ impl NativeCacheConfig {
                 }))),
                 Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)),
             },
+            Some(CacheType::S3) => match project_s3(&backend)? {
+                Ok(backend) => Ok(CacheConfigProjection::Native(Box::new(Self {
+                    policy,
+                    backend: CacheBackendConfig::S3(Box::new(backend)),
+                }))),
+                Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)),
+            },
             Some(
                 CacheType::RedisSemantic
                 | CacheType::ValkeySemantic
-                | CacheType::S3
                 | CacheType::Disk
                 | CacheType::QdrantSemantic
                 | CacheType::AzureBlob
@@ -159,10 +174,11 @@ impl NativeCacheConfig {
 
     pub(super) fn service_mismatch(&self, service: &NativeResponseCache) -> Option<&'static str> {
         if service.default_ttl()
-            != Some(match &self.backend {
-                CacheBackendConfig::Memory(config) => config.default_ttl,
-                CacheBackendConfig::Redis(config) => config.default_ttl,
-            })
+            != match &self.backend {
+                CacheBackendConfig::Memory(config) => Some(config.default_ttl),
+                CacheBackendConfig::Redis(config) => Some(config.default_ttl),
+                CacheBackendConfig::S3(_) => None,
+            }
         {
             return Some("facade and native backend default TTLs must match");
         }
@@ -185,6 +201,18 @@ impl NativeCacheConfig {
             CacheBackendConfig::Redis(config) => (service.namespace()
                 != config.namespace.as_deref())
             .then_some("facade and native backend namespaces must match"),
+            CacheBackendConfig::S3(_) if service.kind() != "s3" => {
+                Some("facade and native backend types must match")
+            }
+            CacheBackendConfig::S3(config) if service.bucket() != Some(config.bucket.as_str()) => {
+                Some("facade and native backend buckets must match")
+            }
+            CacheBackendConfig::S3(config)
+                if service.key_prefix() != Some(config.key_prefix.as_str()) =>
+            {
+                Some("facade and native backend key prefixes must match")
+            }
+            CacheBackendConfig::S3(_) => None,
         }
     }
 }
@@ -299,6 +327,77 @@ fn project_redis(
     }))
 }
 
+#[inline(never)]
+fn project_s3(
+    backend: &Bound<'_, PyAny>,
+) -> PyResult<Result<S3CacheConfig, UnsupportedCacheConfig>> {
+    let client = backend.getattr("s3_client")?;
+    if !instance_class_is(&client, "botocore.client", "S3")? {
+        return Ok(Err(UnsupportedCacheConfig::S3Client));
+    }
+    let meta = client.getattr("meta")?;
+    let Some(region) = optional_string(meta.getattr("region_name")?)? else {
+        return Ok(Err(UnsupportedCacheConfig::S3Option));
+    };
+    let Some(endpoint_url) = optional_string(meta.getattr("endpoint_url")?)? else {
+        return Ok(Err(UnsupportedCacheConfig::S3Option));
+    };
+    let client_config = meta.getattr("config")?;
+    for name in ["s3", "proxies", "client_cert"] {
+        if optional_attribute(&client_config, name)?.is_some_and(|value| !value.is_none()) {
+            return Ok(Err(UnsupportedCacheConfig::S3Option));
+        }
+    }
+    let signature = match optional_attribute(&client_config, "signature_version")? {
+        Some(value) => value.extract::<Option<String>>()?,
+        None => None,
+    };
+    if signature.as_deref() != Some("s3v4") {
+        return Ok(Err(UnsupportedCacheConfig::S3Option));
+    }
+    let insecure = endpoint_url.starts_with("http://");
+    let verify = optional_attribute_chain(&client, &["_endpoint", "http_session", "_verify"])?;
+    let verified = verify
+        .and_then(|value| value.cast::<PyBool>().ok().map(|value| value.is_true()))
+        .unwrap_or(false);
+    if !verified && !insecure {
+        return Ok(Err(UnsupportedCacheConfig::S3Option));
+    }
+    let credentials = optional_attribute_chain(&client, &["_request_signer", "_credentials"])?
+        .ok_or(UnsupportedCacheConfig::S3Credentials);
+    let credentials = match credentials {
+        Ok(credentials) if !credentials.is_none() => credentials,
+        _ => return Ok(Err(UnsupportedCacheConfig::S3Credentials)),
+    };
+    let auth = if credentials.getattr("method")?.extract::<String>()?.as_str() == "explicit" {
+        AwsAuthConfig {
+            access_key_id: credentials
+                .getattr("access_key")?
+                .extract::<Option<String>>()?,
+            secret_access_key: credentials
+                .getattr("secret_key")?
+                .extract::<Option<String>>()?,
+            session_token: credentials.getattr("token")?.extract::<Option<String>>()?,
+            region_name: Some(region.clone()),
+            ..Default::default()
+        }
+    } else {
+        AwsAuthConfig {
+            region_name: Some(region.clone()),
+            ..Default::default()
+        }
+    };
+    let default_endpoint = endpoint_url == format!("https://s3.{region}.amazonaws.com")
+        || (region == "us-east-1" && endpoint_url == "https://s3.amazonaws.com");
+    Ok(Ok(S3CacheConfig {
+        bucket: backend.getattr("bucket_name")?.extract::<String>()?,
+        key_prefix: backend.getattr("key_prefix")?.extract::<String>()?,
+        region,
+        endpoint: (!default_endpoint).then_some(S3Endpoint { url: endpoint_url }),
+        auth,
+    }))
+}
+
 #[inline(never)]
 fn project_tls(values: &Bound<'_, PyDict>) -> PyResult<RedisTlsConfig> {
     Ok(RedisTlsConfig {
@@ -380,6 +479,33 @@ fn optional_attribute_string(value: &Bound<'_, PyAny>, name: &str) -> PyResult<O
     }
 }
 
+#[inline(never)]
+fn optional_attribute<'py>(
+    value: &Bound<'py, PyAny>,
+    name: &str,
+) -> PyResult<Option<Bound<'py, PyAny>>> {
+    match value.getattr(name) {
+        Ok(value) => Ok(Some(value)),
+        Err(error) if error.is_instance_of::<PyAttributeError>(value.py()) => Ok(None),
+        Err(error) => Err(error),
+    }
+}
+
+#[inline(never)]
+fn optional_attribute_chain<'py>(
+    value: &Bound<'py, PyAny>,
+    names: &[&str],
+) -> PyResult<Option<Bound<'py, PyAny>>> {
+    let mut current = value.clone();
+    for name in names {
+        match optional_attribute(&current, name)? {
+            Some(next) => current = next,
+            None => return Ok(None),
+        }
+    }
+    Ok(Some(current))
+}
+
 #[inline(never)]
 fn optional_string(value: Bound<'_, PyAny>) -> PyResult<Option<String>> {
     Ok(value
@@ -591,4 +717,145 @@ mod tests {
             assert_eq!(reason.message(), "native Redis credentials require Python");
         });
     }
+
+    fn s3_facade<'py>(py: Python<'py>, body: &str) -> Bound<'py, PyAny> {
+        let locals = PyDict::new(py);
+        py.run(
+            &CString::new(format!(
+                "from types import SimpleNamespace\n\
+                 S3Client = type('S3', (), {{'__module__': 'botocore.client'}})\n\
+                 client = S3Client()\n\
+                 client.meta = SimpleNamespace(region_name='us-east-1', endpoint_url='https://example.test', config=SimpleNamespace(s3=None, proxies=None, client_cert=None, signature_version='s3v4'))\n\
+                 client._endpoint = SimpleNamespace(http_session=SimpleNamespace(_verify=True))\n\
+                 client._request_signer = SimpleNamespace(_credentials=SimpleNamespace(method='explicit', access_key='key', secret_key='secret', token='token'))\n\
+                 backend = SimpleNamespace(bucket_name='bucket', key_prefix='team/', s3_client=client)\n\
+                 facade = SimpleNamespace(type='s3', mode='default-on', ttl=None, namespace=None, supported_call_types=[], redis_flush_size=None, semantic_cache_scope='key', cache=backend)\n\
+                 {body}"
+            ))
+            .unwrap(),
+            None,
+            Some(&locals),
+        )
+        .unwrap();
+        locals.get_item("facade").unwrap().unwrap()
+    }
+
+    #[test]
+    fn projects_s3_configuration_with_explicit_credentials_and_custom_endpoint() {
+        Python::initialize();
+        Python::attach(|py| {
+            let facade = s3_facade(py, "");
+            let CacheConfigProjection::Native(config) =
+                NativeCacheConfig::project(&facade).unwrap()
+            else {
+                panic!("S3 cache should be supported");
+            };
+            let CacheBackendConfig::S3(s3) = config.backend else {
+                panic!("expected S3 configuration");
+            };
+            assert_eq!(s3.bucket, "bucket");
+            assert_eq!(s3.key_prefix, "team/");
+            assert_eq!(s3.region, "us-east-1");
+            assert_eq!(
+                s3.endpoint.map(|endpoint| endpoint.url).as_deref(),
+                Some("https://example.test")
+            );
+            assert_eq!(s3.auth.access_key_id.as_deref(), Some("key"));
+            assert_eq!(s3.auth.secret_access_key.as_deref(), Some("secret"));
+            assert_eq!(s3.auth.session_token.as_deref(), Some("token"));
+            assert_eq!(s3.auth.region_name.as_deref(), Some("us-east-1"));
+        });
+    }
+
+    #[test]
+    fn default_s3_endpoint_projects_no_custom_endpoint() {
+        Python::initialize();
+        Python::attach(|py| {
+            let facade = s3_facade(
+                py,
+                "facade.cache.s3_client.meta.endpoint_url = 'https://s3.us-east-1.amazonaws.com'",
+            );
+            let CacheConfigProjection::Native(config) =
+                NativeCacheConfig::project(&facade).unwrap()
+            else {
+                panic!("S3 cache should be supported");
+            };
+            let CacheBackendConfig::S3(s3) = config.backend else {
+                panic!("expected S3 configuration");
+            };
+            assert!(s3.endpoint.is_none());
+        });
+    }
+
+    #[test]
+    fn non_sigv4_proxies_and_disabled_verification_stay_on_python() {
+        Python::initialize();
+        Python::attach(|py| {
+            for (body, message) in [
+                (
+                    "facade.cache.s3_client.meta.config.signature_version = 's3'",
+                    "native S3 configuration requires Python",
+                ),
+                (
+                    "facade.cache.s3_client.meta.config.proxies = {'https': 'proxy'}",
+                    "native S3 configuration requires Python",
+                ),
+                (
+                    "facade.cache.s3_client._endpoint.http_session._verify = False",
+                    "native S3 configuration requires Python",
+                ),
+                (
+                    "del facade.cache.s3_client._endpoint.http_session._verify",
+                    "native S3 configuration requires Python",
+                ),
+            ] {
+                let facade = s3_facade(py, body);
+                let CacheConfigProjection::Unsupported(reason) =
+                    NativeCacheConfig::project(&facade).unwrap()
+                else {
+                    panic!("{body} must stay on Python");
+                };
+                assert_eq!(reason.message(), message);
+            }
+            let facade = s3_facade(py, "facade.cache.s3_client = SimpleNamespace()");
+            let CacheConfigProjection::Unsupported(reason) =
+                NativeCacheConfig::project(&facade).unwrap()
+            else {
+                panic!("non-botocore client must stay on Python");
+            };
+            assert_eq!(reason.message(), "native S3 client type is not implemented");
+            let facade = s3_facade(
+                py,
+                "facade.cache.s3_client._request_signer._credentials = None",
+            );
+            let CacheConfigProjection::Unsupported(reason) =
+                NativeCacheConfig::project(&facade).unwrap()
+            else {
+                panic!("missing credentials must stay on Python");
+            };
+            assert_eq!(reason.message(), "native S3 credentials require Python");
+        });
+    }
+
+    #[test]
+    fn non_explicit_s3_credentials_use_the_default_chain() {
+        Python::initialize();
+        Python::attach(|py| {
+            let facade = s3_facade(
+                py,
+                "facade.cache.s3_client._request_signer._credentials = SimpleNamespace(method='sso', access_key=None, secret_key=None, token=None)",
+            );
+            let CacheConfigProjection::Native(config) =
+                NativeCacheConfig::project(&facade).unwrap()
+            else {
+                panic!("default-chain credentials should be supported");
+            };
+            let CacheBackendConfig::S3(s3) = config.backend else {
+                panic!("expected S3 configuration");
+            };
+            assert_eq!(s3.auth.access_key_id, None);
+            assert_eq!(s3.auth.secret_access_key, None);
+            assert_eq!(s3.auth.region_name.as_deref(), Some("us-east-1"));
+        });
+    }
 }
diff --git a/litellm-rust/crates/python-bridge/src/cache/facade.rs b/litellm-rust/crates/python-bridge/src/cache/facade.rs
index f2f86c14b37..0f7b9778d3e 100644
--- a/litellm-rust/crates/python-bridge/src/cache/facade.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/facade.rs
@@ -32,10 +32,15 @@ struct RedisPoolGuard {
     max_connections: usize,
 }
 
+struct S3ClientGuard {
+    reference: Py<PyAny>,
+}
+
 pub(super) struct FacadeGuard {
     outer: ObjectGuard,
     backend: ObjectGuard,
     redis_pool: Option<RedisPoolGuard>,
+    s3_client: Option<S3ClientGuard>,
 }
 
 impl ObjectGuard {
@@ -176,6 +181,22 @@ impl RedisPoolGuard {
     }
 }
 
+impl S3ClientGuard {
+    fn capture(backend: &Bound<'_, PyAny>) -> PyResult<Self> {
+        Ok(Self {
+            reference: backend.getattr("s3_client")?.unbind(),
+        })
+    }
+
+    fn matches(&self, py: Python<'_>, backend: &Bound<'_, PyAny>) -> PyResult<bool> {
+        Ok(self.reference.bind(py).is(&backend.getattr("s3_client")?))
+    }
+
+    fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
+        visit.call(&self.reference)
+    }
+}
+
 impl FacadeGuard {
     pub(super) fn capture(
         py: Python<'_>,
@@ -192,6 +213,7 @@ impl FacadeGuard {
         let (module, name, cache_kind) = match kind {
             "memory" => ("litellm.caching.in_memory_cache", "InMemoryCache", "local"),
             "redis" => ("litellm.caching.redis_cache", "RedisCache", "redis"),
+            "s3" => ("litellm.caching.s3_cache", "S3Cache", "s3"),
             _ => unreachable!(),
         };
         let backend = facade.getattr("cache")?;
@@ -235,11 +257,16 @@ impl FacadeGuard {
                     "max_size_per_item",
                     "redis_kwargs",
                     "redis_flush_size",
+                    "bucket_name",
+                    "key_prefix",
                 ],
             )?,
             redis_pool: (kind == "redis")
                 .then(|| RedisPoolGuard::capture(&backend))
                 .transpose()?,
+            s3_client: (kind == "s3")
+                .then(|| S3ClientGuard::capture(&backend))
+                .transpose()?,
         })
     }
 
@@ -252,9 +279,14 @@ impl FacadeGuard {
             return Ok(false);
         }
         match &self.redis_pool {
-            Some(guard) => guard.matches(py, &backend),
-            None => Ok(true),
+            Some(guard) if !guard.matches(py, &backend)? => return Ok(false),
+            _ => {}
         }
+        match &self.s3_client {
+            Some(guard) if !guard.matches(py, &backend)? => return Ok(false),
+            _ => {}
+        }
+        Ok(true)
     }
 
     pub(super) fn traverse(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> {
@@ -263,6 +295,9 @@ impl FacadeGuard {
         if let Some(guard) = &self.redis_pool {
             guard.traverse(&visit)?;
         }
+        if let Some(guard) = &self.s3_client {
+            guard.traverse(&visit)?;
+        }
         Ok(())
     }
 }
diff --git a/litellm-rust/crates/python-bridge/src/cache/handle.rs b/litellm-rust/crates/python-bridge/src/cache/handle.rs
index 8251b3df06c..0d5a2ac9d8c 100644
--- a/litellm-rust/crates/python-bridge/src/cache/handle.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/handle.rs
@@ -1,4 +1,6 @@
-use litellm_host_python::release_gil;
+use litellm_auth_aws::AwsAuthConfig;
+use litellm_cache_s3::{S3CacheConfig, S3Endpoint};
+use litellm_host_python::{release_gil, run_sync_value};
 use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*};
 
 use super::{cache_error, facade::FacadeGuard, native::NativeResponseCache, request::duration};
@@ -51,6 +53,40 @@ impl CacheTestHandle {
         })
     }
 
+    #[staticmethod]
+    #[allow(clippy::too_many_arguments)]
+    #[pyo3(signature = (bucket, *, region, endpoint_url=None, key_prefix="", access_key_id=None, secret_access_key=None, session_token=None))]
+    fn s3(
+        py: Python<'_>,
+        bucket: String,
+        region: String,
+        endpoint_url: Option<String>,
+        key_prefix: &str,
+        access_key_id: Option<String>,
+        secret_access_key: Option<String>,
+        session_token: Option<String>,
+    ) -> PyResult<Self> {
+        let config = S3CacheConfig {
+            bucket,
+            key_prefix: key_prefix.to_string(),
+            region: region.clone(),
+            endpoint: endpoint_url.map(|url| S3Endpoint { url }),
+            auth: AwsAuthConfig {
+                access_key_id,
+                secret_access_key,
+                session_token,
+                region_name: Some(region),
+                ..Default::default()
+            },
+        };
+        let service = run_sync_value(py, async move { Ok(NativeResponseCache::s3(config).await) })?;
+        Ok(Self {
+            service,
+            guard: None,
+            pid: std::process::id(),
+        })
+    }
+
     #[getter]
     fn backend(&self) -> &'static str {
         self.service.kind()
diff --git a/litellm-rust/crates/python-bridge/src/cache/mod.rs b/litellm-rust/crates/python-bridge/src/cache/mod.rs
index aec08610f6e..ac4494150d9 100644
--- a/litellm-rust/crates/python-bridge/src/cache/mod.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/mod.rs
@@ -10,7 +10,7 @@ mod resolver;
 
 use litellm_cache::Error;
 use pyo3::{
-    exceptions::{PyRuntimeError, PyValueError},
+    exceptions::{PyNotImplementedError, PyRuntimeError, PyValueError},
     prelude::*,
 };
 
@@ -21,6 +21,7 @@ pub(crate) use self::{
 fn cache_error(error: Error) -> PyErr {
     match error {
         Error::InvalidEntry => PyValueError::new_err(error.to_string()),
+        Error::UnsupportedOperation => PyNotImplementedError::new_err(error.to_string()),
         _ => PyRuntimeError::new_err(error.to_string()),
     }
 }
diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs
index a9475429e45..b9ea443119b 100644
--- a/litellm-rust/crates/python-bridge/src/cache/native.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/native.rs
@@ -6,6 +6,7 @@ use litellm_cache_redis::RedisCache;
 use litellm_cache_response::{
     CacheEntry, PartialHits, ResponseCache, ResponseCacheCodec, ResponseCacheRequest, WriteBuffer,
 };
+use litellm_cache_s3::{S3Cache, S3CacheConfig};
 use serde_json::Value;
 
 #[derive(Clone)]
@@ -15,6 +16,7 @@ pub(super) enum NativeResponseCache {
         cache: Arc<ResponseCache<RedisCache<ResponseCacheCodec>>>,
         buffer: Option<Arc<WriteBuffer>>,
     },
+    S3(Arc<ResponseCache<S3Cache<ResponseCacheCodec>>>),
 }
 
 impl NativeResponseCache {
@@ -43,6 +45,15 @@ impl NativeResponseCache {
             buffer: None,
         })
     }
+
+    pub async fn s3(config: S3CacheConfig) -> Self {
+        let runtime = tokio::runtime::Handle::current();
+        Self::S3(Arc::new(ResponseCache::new(Arc::new(S3Cache::new(
+            config,
+            ResponseCacheCodec,
+            runtime,
+        )))))
+    }
 }
 
 impl NativeResponseCache {
@@ -50,6 +61,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(_) => "memory",
             Self::Redis { .. } => "redis",
+            Self::S3(_) => "s3",
         }
     }
 
@@ -57,6 +69,21 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.default_ttl(),
             Self::Redis { cache, .. } => cache.default_ttl(),
+            Self::S3(cache) => cache.default_ttl(),
+        }
+    }
+
+    pub fn bucket(&self) -> Option<&str> {
+        match self {
+            Self::S3(cache) => Some(cache.backend().bucket()),
+            _ => None,
+        }
+    }
+
+    pub fn key_prefix(&self) -> Option<&str> {
+        match self {
+            Self::S3(cache) => Some(cache.backend().key_prefix()),
+            _ => None,
         }
     }
 
@@ -64,20 +91,21 @@ impl NativeResponseCache {
         match self {
             Self::Memory(_) => None,
             Self::Redis { cache, .. } => cache.backend().namespace(),
+            Self::S3(_) => None,
         }
     }
 
     pub fn capacity(&self) -> Option<usize> {
         match self {
             Self::Memory(cache) => Some(cache.backend().max_size_in_memory()),
-            Self::Redis { .. } => None,
+            Self::Redis { .. } | Self::S3(_) => None,
         }
     }
 
     pub fn max_entry_bytes(&self) -> Option<usize> {
         match self {
             Self::Memory(cache) => cache.backend().max_entry_bytes(),
-            Self::Redis { .. } => None,
+            Self::Redis { .. } | Self::S3(_) => None,
         }
     }
 
@@ -87,7 +115,7 @@ impl NativeResponseCache {
                 cache,
                 buffer: flush_size.map(|flush_size| Arc::new(WriteBuffer::new(flush_size))),
             },
-            memory => memory,
+            cache => cache,
         }
     }
 
@@ -99,6 +127,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.lookup(request, now),
             Self::Redis { cache, .. } => cache.lookup(request, now),
+            Self::S3(cache) => cache.lookup(request, now),
         }
     }
 
@@ -111,6 +140,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.store(request, response, now),
             Self::Redis { cache, .. } => cache.store(request, response, now),
+            Self::S3(cache) => cache.store(request, response, now),
         }
     }
 
@@ -122,6 +152,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.lookup_batch(requests, now),
             Self::Redis { cache, .. } => cache.lookup_batch(requests, now),
+            Self::S3(cache) => cache.lookup_batch(requests, now),
         }
     }
 
@@ -133,6 +164,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.async_lookup(request, now).await,
             Self::Redis { cache, .. } => cache.async_lookup(request, now).await,
+            Self::S3(cache) => cache.async_lookup(request, now).await,
         }
     }
 
@@ -152,6 +184,7 @@ impl NativeResponseCache {
                 cache,
                 buffer: Some(buffer),
             } => buffer.async_store(cache, request, response, now).await,
+            Self::S3(cache) => cache.async_store(request, response, now).await,
         }
     }
 
@@ -163,6 +196,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.async_lookup_batch(requests, now).await,
             Self::Redis { cache, .. } => cache.async_lookup_batch(requests, now).await,
+            Self::S3(cache) => cache.async_lookup_batch(requests, now).await,
         }
     }
 
@@ -174,6 +208,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.async_store_batch(entries, now).await,
             Self::Redis { cache, .. } => cache.async_store_batch(entries, now).await,
+            Self::S3(cache) => cache.async_store_batch(entries, now).await,
         }
     }
 
@@ -186,6 +221,7 @@ impl NativeResponseCache {
                 }
                 cache.async_flush().await
             }
+            Self::S3(cache) => cache.async_flush().await,
         }
     }
 
@@ -193,6 +229,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.test_connection().await,
             Self::Redis { cache, .. } => cache.test_connection().await,
+            Self::S3(cache) => cache.test_connection().await,
         }
     }
 }
diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi
index 05a6df6d5af..f1b010fb26e 100644
--- a/litellm/rust_bridge/_native.pyi
+++ b/litellm/rust_bridge/_native.pyi
@@ -93,6 +93,96 @@ class ResponsesWebSocketConnection:
     def recv_text(self) -> Future[str | None]: ...
     def close(self) -> Future[None]: ...
 
+@final
+class _CacheTestBinding:
+    @property
+    def kind(self) -> str: ...
+    def lookup(
+        self,
+        request: object,
+        *,
+        callback_kwargs: Mapping[str, object] | Sequence[object] | None = None,
+    ) -> object: ...
+    def store(
+        self,
+        request: object,
+        response: object,
+        *,
+        callback_kwargs: Mapping[str, object] | None = None,
+    ) -> None: ...
+    def lookup_batch(
+        self,
+        requests: Sequence[object],
+        *,
+        callback_kwargs: Sequence[object] | None = None,
+    ) -> object: ...
+    def async_lookup(
+        self,
+        request: object,
+        *,
+        callback_kwargs: Mapping[str, object] | None = None,
+    ) -> Future[object]: ...
+    def async_store(
+        self,
+        request: object,
+        response: object,
+        *,
+        callback_kwargs: Mapping[str, object] | None = None,
+    ) -> Future[None]: ...
+    def async_lookup_batch(
+        self,
+        requests: Sequence[object],
+        *,
+        callback_kwargs: Sequence[object] | None = None,
+    ) -> Future[object]: ...
+    def async_store_batch(
+        self,
+        requests: Sequence[object],
+        responses: Sequence[object],
+        *,
+        callback_result: object = None,
+        callback_kwargs: Mapping[str, object] | None = None,
+    ) -> Future[object]: ...
+    def async_flush(self) -> Future[None]: ...
+    def ping(self) -> Future[object]: ...
+
+@final
+class _CacheTestHandle:
+    def __new__(cls, _uninstantiable: Never, /) -> Never: ...
+    @staticmethod
+    def memory(
+        *,
+        capacity: int = 200,
+        ttl_seconds: float = 600.0,
+        max_entry_bytes: int = 1048576,
+    ) -> _CacheTestHandle: ...
+    @staticmethod
+    def redis(
+        url: str,
+        *,
+        ttl_seconds: float = 60.0,
+        namespace: str | None = None,
+    ) -> _CacheTestHandle: ...
+    @staticmethod
+    def s3(
+        bucket: str,
+        *,
+        region: str,
+        endpoint_url: str | None = None,
+        key_prefix: str = "",
+        access_key_id: str | None = None,
+        secret_access_key: str | None = None,
+        session_token: str | None = None,
+    ) -> _CacheTestHandle: ...
+    @property
+    def backend(self) -> str: ...
+    def _bind_facade(self, facade: object) -> None: ...
+
+@final
+class _CacheTestResolver:
+    def __new__(cls, namespace: object) -> _CacheTestResolver: ...
+    def resolve(self) -> _CacheTestBinding: ...
+
 @final
 class TokenCounter:
     def __new__(cls, tokenizer_json: str) -> TokenCounter: ...
diff --git a/tests/test_litellm_rust/support/s3_stub.py b/tests/test_litellm_rust/support/s3_stub.py
new file mode 100644
index 00000000000..18863ad8083
--- /dev/null
+++ b/tests/test_litellm_rust/support/s3_stub.py
@@ -0,0 +1,112 @@
+"""In-process path-style S3 stub for native cache parity tests."""
+
+import threading
+from dataclasses import dataclass, field
+from email.utils import parsedate_to_datetime
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from typing import Final
+from urllib.parse import unquote, urlsplit
+
+_STORED_HEADERS: Final = (
+    "cache-control",
+    "content-type",
+    "content-language",
+    "content-disposition",
+    "expires",
+)
+
+
+@dataclass
+class S3Object:
+    body: bytes
+    headers: dict[str, str] = field(default_factory=dict)
+
+
+class S3Stub:
+    """Minimal path-style S3 endpoint serving PUT and GET object operations."""
+
+    def __init__(self) -> None:
+        self._objects: dict[str, S3Object] = {}
+        stub: Final = self
+
+        class Handler(BaseHTTPRequestHandler):
+            def _key(self) -> str:
+                parts: Final = urlsplit(self.path).path.lstrip("/").split("/", 1)
+                return unquote(parts[1]) if len(parts) == 2 else ""
+
+            def _read_body(self) -> bytes:
+                transfer: Final = self.headers.get("transfer-encoding", "")
+                if "chunked" not in transfer:
+                    return self.rfile.read(int(self.headers.get("content-length", 0)))
+                chunks: Final = bytearray()
+                while True:
+                    size = int(self.rfile.readline().split(b";")[0].strip(), 16)
+                    if size == 0:
+                        while self.rfile.readline().strip():
+                            pass
+                        return bytes(chunks)
+                    chunks.extend(self.rfile.read(size))
+                    self.rfile.readline()
+
+            def do_PUT(self) -> None:
+                body: Final = self._read_body()
+                headers: Final = {name: self.headers[name] for name in _STORED_HEADERS if name in self.headers}
+                stub._objects[self._key()] = S3Object(body=body, headers=headers)
+                self.send_response(200)
+                self.send_header("ETag", '"stub"')
+                self.send_header("Content-Length", "0")
+                self.end_headers()
+
+            def do_HEAD(self) -> None:
+                self._object(send_body=False)
+
+            def do_GET(self) -> None:
+                self._object(send_body=True)
+
+            def _object(self, send_body: bool) -> None:
+                entry: Final = stub._objects.get(self._key())
+                if entry is None:
+                    self.send_response(404)
+                    self.send_header("Content-Type", "application/xml")
+                    body: Final = b'<?xml version="1.0" encoding="UTF-8"?><Error><Code>NoSuchKey</Code></Error>'
+                    self.send_header("Content-Length", str(len(body)))
+                    self.end_headers()
+                    if send_body:
+                        self.wfile.write(body)
+                    return
+                self.send_response(200)
+                for name, value in entry.headers.items():
+                    self.send_header(name, value)
+                self.send_header("ETag", '"stub"')
+                self.send_header("Content-Length", str(len(entry.body)))
+                self.end_headers()
+                if send_body:
+                    self.wfile.write(entry.body)
+
+            def log_message(self, format: str, *args: object) -> None:  # noqa: A002
+                pass
+
+        self._server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+        self._worker: Final = threading.Thread(target=self._server.serve_forever, daemon=True)
+        self._worker.start()
+
+    @property
+    def url(self) -> str:
+        host, port = self._server.server_address[:2]
+        return f"http://{host}:{port}"
+
+    @property
+    def objects(self) -> dict[str, S3Object]:
+        return self._objects
+
+    def put_object(self, key: str, body: bytes, headers: dict[str, str] | None = None) -> None:
+        self._objects[key] = S3Object(body=body, headers=headers or {})
+
+    def expires(self, key: str) -> object:
+        header: Final = self._objects[key].headers.get("expires")
+        return parsedate_to_datetime(header) if header else None
+
+    def close(self) -> None:
+        self._server.shutdown()
+        self._server.server_close()
+        self._worker.join(timeout=5)
diff --git a/tests/test_litellm_rust/test_cache.py b/tests/test_litellm_rust/test_cache.py
index c35cb1a20fb..f18454ea8d1 100644
--- a/tests/test_litellm_rust/test_cache.py
+++ b/tests/test_litellm_rust/test_cache.py
@@ -6,10 +6,13 @@ import threading
 import time
 import weakref
 from collections.abc import Generator
+from datetime import datetime
 from types import SimpleNamespace
 from typing import Final, Protocol, cast
 from urllib.parse import urlparse
 
+import boto3
+import botocore.config
 import fakeredis
 import pytest
 import redis
@@ -17,9 +20,11 @@ import redis
 import litellm
 from litellm.caching.caching import Cache, disable_cache, enable_cache, update_cache
 from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.caching.s3_cache import S3Cache
 from litellm.rust_bridge import _native
 from litellm.types.caching import LiteLLMCacheType
 from tests.test_litellm_rust.support.isolation import rebound
+from tests.test_litellm_rust.support.s3_stub import S3Stub
 
 pytestmark: Final = pytest.mark.requires_rust_extension
 
@@ -393,3 +398,180 @@ async def test_redis_facade_buffers_native_async_writes(redis_url: str) -> None:
     assert client.get("second") is not None
     await facade.cache.disconnect()
     client.close()
+
+
+@pytest.fixture
+def s3_stub() -> Generator[S3Stub]:
+    stub: Final = S3Stub()
+    try:
+        yield stub
+    finally:
+        stub.close()
+
+
+def python_s3(url: str) -> S3Cache:
+    return S3Cache(
+        s3_bucket_name="cache-bucket",
+        s3_region_name="us-east-1",
+        s3_endpoint_url=url,
+        s3_aws_access_key_id="key",
+        s3_aws_secret_access_key="secret",
+        s3_path="team",
+    )
+
+
+async def test_s3_reads_python_entries_and_writes_with_python_metadata(s3_stub: S3Stub) -> None:
+    python_cache: Final = python_s3(s3_stub.url)
+    response: Final = {"choices": [{"text": "cached"}], "usage": {"total_tokens": 3}}
+    python_cache.set_cache("sync:key", {"timestamp": time.time(), "response": response}, ttl=90)
+    python_cache.set_cache("plain", {"timestamp": time.time(), "response": response})
+    s3_stub.put_object("team/malformed", b"not a cache entry")
+    s3_stub.put_object(
+        "team/expired",
+        json.dumps({"timestamp": time.time(), "response": response}).encode(),
+        {"expires": "Thu, 01 Jan 1970 00:00:00 GMT"},
+    )
+    binding: Final = _native._CacheTestResolver(
+        SimpleNamespace(
+            cache=_native._CacheTestHandle.s3(
+                "cache-bucket",
+                region="us-east-1",
+                endpoint_url=s3_stub.url,
+                key_prefix="team/",
+                access_key_id="key",
+                secret_access_key="secret",
+            )
+        )
+    ).resolve()
+
+    assert binding.lookup(request("sync:key")) == response
+    assert await binding.async_lookup(request("plain")) == response
+    assert binding.lookup(request("malformed")) is None
+    assert binding.lookup(request("expired")) is None
+    assert binding.lookup(request("absent")) is None
+
+    binding.store({**request("native:key"), "ttl_seconds": 90.0}, response)
+    await binding.async_store(request("no_ttl"), response)
+    stored: Final = s3_stub.objects["team/native/key"]
+    assert stored.headers["content-type"] == "application/json"
+    assert stored.headers["content-language"] == "en"
+    assert stored.headers["content-disposition"] == 'inline; filename="team/native/key.json"'
+    assert stored.headers["cache-control"] == "immutable, max-age=90, s-maxage=90"
+    expires: Final = cast(datetime, s3_stub.expires("team/native/key"))
+    remaining: Final = (expires - datetime.now(expires.tzinfo)).total_seconds()
+    assert 60 < remaining <= 91
+    no_ttl: Final = s3_stub.objects["team/no_ttl"]
+    assert no_ttl.headers["cache-control"] == "immutable, max-age=31536000, s-maxage=31536000"
+    assert "expires" not in no_ttl.headers
+    assert python_cache.get_cache("native:key")["response"] == response
+
+    partial: Final = await binding.async_lookup_batch([request("native:key"), request("absent"), request("malformed")])
+    assert partial == {"values": [response, None, None], "missing_indices": [1, 2]}
+
+
+def test_s3_facade_binds_only_exact_configuration_and_falls_back_on_mutation(s3_stub: S3Stub) -> None:
+    facade: Final = Cache(
+        type=LiteLLMCacheType.S3,
+        s3_bucket_name="cache-bucket",
+        s3_region_name="us-east-1",
+        s3_endpoint_url=s3_stub.url,
+        s3_aws_access_key_id="key",
+        s3_aws_secret_access_key="secret",
+        s3_path="team",
+    )
+    handle: Final = _native._CacheTestHandle.s3(
+        "cache-bucket",
+        region="us-east-1",
+        endpoint_url=s3_stub.url,
+        key_prefix="team/",
+        access_key_id="key",
+        secret_access_key="secret",
+    )
+    with pytest.raises(TypeError, match="buckets must match"):
+        _native._CacheTestHandle.s3("other", region="us-east-1", endpoint_url=s3_stub.url)._bind_facade(facade)
+    with pytest.raises(TypeError, match="key prefixes must match"):
+        _native._CacheTestHandle.s3(
+            "cache-bucket", region="us-east-1", endpoint_url=s3_stub.url, key_prefix="other/"
+        )._bind_facade(facade)
+    handle._bind_facade(facade)
+    resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade))
+    binding: Final = resolver.resolve()
+    assert binding.kind == "native"
+
+    calls: Final = []
+    facade.cache.s3_client.meta.events.register("before-call.s3.*", lambda **_kwargs: calls.append(1))
+    binding.store(request("native"), {"answer": 1})
+    assert binding.lookup(request("native")) == {"answer": 1}
+    assert calls == []
+    assert "team/native" in s3_stub.objects
+
+    with rebound(facade.cache, "bucket_name", "other"):
+        assert resolver.resolve().kind == "python_callback"
+    other_client: Final = boto3.client(
+        "s3",
+        region_name="us-east-1",
+        endpoint_url=s3_stub.url,
+        aws_access_key_id="key",
+        aws_secret_access_key="secret",
+    )
+    with rebound(facade.cache, "s3_client", other_client):
+        assert resolver.resolve().kind == "python_callback"
+
+    class CustomS3Cache(S3Cache):
+        pass
+
+    subclassed: Final = Cache(
+        type=LiteLLMCacheType.S3,
+        s3_bucket_name="cache-bucket",
+        s3_region_name="us-east-1",
+        s3_endpoint_url=s3_stub.url,
+        s3_aws_access_key_id="key",
+        s3_aws_secret_access_key="secret",
+        s3_path="team",
+    )
+    subclassed.cache = CustomS3Cache(
+        s3_bucket_name="cache-bucket",
+        s3_region_name="us-east-1",
+        s3_endpoint_url=s3_stub.url,
+        s3_aws_access_key_id="key",
+        s3_aws_secret_access_key="secret",
+        s3_path="team",
+    )
+    with pytest.raises(TypeError):
+        handle._bind_facade(subclassed)
+    assert _native._CacheTestResolver(SimpleNamespace(cache=subclassed)).resolve().kind == "python_callback"
+
+
+def test_s3_facade_rejects_configurations_that_require_python(s3_stub: S3Stub) -> None:
+    handle: Final = _native._CacheTestHandle.s3(
+        "cache-bucket",
+        region="us-east-1",
+        endpoint_url=s3_stub.url,
+        key_prefix="team/",
+        access_key_id="key",
+        secret_access_key="secret",
+    )
+    unverified: Final = Cache(
+        type=LiteLLMCacheType.S3,
+        s3_bucket_name="cache-bucket",
+        s3_region_name="us-east-1",
+        s3_endpoint_url="https://s3.example.test",
+        s3_aws_access_key_id="key",
+        s3_aws_secret_access_key="secret",
+        s3_path="team",
+        s3_verify=False,
+    )
+    with pytest.raises(TypeError, match="requires Python"):
+        handle._bind_facade(unverified)
+    proxied: Final = Cache(
+        type=LiteLLMCacheType.S3,
+        s3_bucket_name="cache-bucket",
+        s3_region_name="us-east-1",
+        s3_endpoint_url=s3_stub.url,
+        s3_aws_access_key_id="key",
+        s3_aws_secret_access_key="secret",
+        s3_path="team",
+        s3_config=botocore.config.Config(proxies={"https": "http://proxy.test"}),
+    )
+    with pytest.raises(TypeError, match="requires Python"):
+        handle._bind_facade(proxied)

From 94b8bf7fd6aa93afb8e8d9a84e047daf06ab01bc Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:37:26 +0000
Subject: [PATCH 077/160] feat(rust): add native disk cache backend

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.lock                       | 126 +++
 litellm-rust/Cargo.toml                       |   1 +
 litellm-rust/crates/cache-disk/Cargo.toml     |  17 +
 litellm-rust/crates/cache-disk/src/cache.rs   | 330 +++++++
 litellm-rust/crates/cache-disk/src/lib.rs     |   8 +
 litellm-rust/crates/cache-disk/src/pickle.rs  |  48 ++
 litellm-rust/crates/cache-disk/src/sqlite.rs  | 816 ++++++++++++++++++
 litellm-rust/crates/cache-disk/src/store.rs   |  33 +
 litellm-rust/crates/cache-disk/tests/cache.rs | 327 +++++++
 litellm-rust/crates/python-bridge/Cargo.toml  |   1 +
 .../crates/python-bridge/src/cache/config.rs  | 139 ++-
 .../crates/python-bridge/src/cache/facade.rs  |  38 +-
 .../crates/python-bridge/src/cache/handle.rs  |  12 +
 .../crates/python-bridge/src/cache/native.rs  |  31 +-
 tests/test_litellm_rust/test_cache.py         | 111 +++
 15 files changed, 2026 insertions(+), 12 deletions(-)
 create mode 100644 litellm-rust/crates/cache-disk/Cargo.toml
 create mode 100644 litellm-rust/crates/cache-disk/src/cache.rs
 create mode 100644 litellm-rust/crates/cache-disk/src/lib.rs
 create mode 100644 litellm-rust/crates/cache-disk/src/pickle.rs
 create mode 100644 litellm-rust/crates/cache-disk/src/sqlite.rs
 create mode 100644 litellm-rust/crates/cache-disk/src/store.rs
 create mode 100644 litellm-rust/crates/cache-disk/tests/cache.rs

diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index ed4ae4e3353..554245dbb6e 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -1318,6 +1318,18 @@ dependencies = [
  "pin-project-lite",
 ]
 
+[[package]]
+name = "fallible-iterator"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
+
+[[package]]
+name = "fallible-streaming-iterator"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
+
 [[package]]
 name = "fancy-regex"
 version = "0.17.0"
@@ -1369,6 +1381,12 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foldhash"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.2.2"
@@ -1833,11 +1851,32 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 
+[[package]]
+name = "hashbrown"
+version = "0.16.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
+dependencies = [
+ "foldhash",
+]
+
 [[package]]
 name = "hashbrown"
 version = "0.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "foldhash",
+]
+
+[[package]]
+name = "hashlink"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb"
+dependencies = [
+ "hashbrown 0.17.1",
+]
 
 [[package]]
 name = "heck"
@@ -2218,6 +2257,12 @@ version = "2.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
 
+[[package]]
+name = "iter-read"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071ed4cc1afd86650602c7b11aa2e1ce30762a1c27193201cb5cee9c6ebb1294"
+
 [[package]]
 name = "itertools"
 version = "0.13.0"
@@ -2376,6 +2421,17 @@ version = "0.2.186"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
 
+[[package]]
+name = "libsqlite3-sys"
+version = "0.38.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8"
+dependencies = [
+ "cc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.12.1"
@@ -2464,6 +2520,19 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "litellm-cache-disk"
+version = "0.1.0"
+dependencies = [
+ "litellm-cache",
+ "rand 0.8.7",
+ "rusqlite",
+ "serde-pickle",
+ "serde_json",
+ "tempfile",
+ "tokio",
+]
+
 [[package]]
 name = "litellm-cache-memory"
 version = "0.1.0"
@@ -2666,6 +2735,7 @@ dependencies = [
  "litellm-auth",
  "litellm-auth-gcp",
  "litellm-cache",
+ "litellm-cache-disk",
  "litellm-cache-memory",
  "litellm-cache-redis",
  "litellm-cache-response",
@@ -3901,6 +3971,16 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rsqlite-vfs"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c"
+dependencies = [
+ "hashbrown 0.16.1",
+ "thiserror 2.0.19",
+]
+
 [[package]]
 name = "rstest"
 version = "0.26.1"
@@ -3941,6 +4021,21 @@ dependencies = [
  "syn 2.0.119",
 ]
 
+[[package]]
+name = "rusqlite"
+version = "0.40.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3"
+dependencies = [
+ "bitflags 2.13.1",
+ "fallible-iterator",
+ "fallible-streaming-iterator",
+ "hashlink",
+ "libsqlite3-sys",
+ "smallvec",
+ "sqlite-wasm-rs",
+]
+
 [[package]]
 name = "rustc-hash"
 version = "2.1.3"
@@ -4198,6 +4293,19 @@ dependencies = [
  "serde_derive",
 ]
 
+[[package]]
+name = "serde-pickle"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b641fdc8bcf2781ee78b30c599700d64ad4f412976143e4c5d0b9df906bb4843"
+dependencies = [
+ "byteorder",
+ "iter-read",
+ "num-bigint 0.4.8",
+ "num-traits",
+ "serde",
+]
+
 [[package]]
 name = "serde_core"
 version = "1.0.229"
@@ -4425,6 +4533,18 @@ dependencies = [
  "unicode-segmentation",
 ]
 
+[[package]]
+name = "sqlite-wasm-rs"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75"
+dependencies = [
+ "cc",
+ "js-sys",
+ "rsqlite-vfs",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "sse-stream"
 version = "0.2.6"
@@ -5170,6 +5290,12 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
 
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
 [[package]]
 name = "veil"
 version = "0.3.0"
diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
index 570d0dd3568..53db97b2d01 100644
--- a/litellm-rust/Cargo.toml
+++ b/litellm-rust/Cargo.toml
@@ -29,6 +29,7 @@ litellm-core-utils = { path = "crates/core-utils" }
 litellm-cache = { path = "crates/cache" }
 litellm-cache-memory = { path = "crates/cache-memory" }
 litellm-cache-redis = { path = "crates/cache-redis" }
+litellm-cache-disk = { path = "crates/cache-disk" }
 litellm-cache-response = { path = "crates/cache-response" }
 litellm-token-counter = { path = "crates/token-counter" }
 litellm-token-counter-fast = { path = "crates/token-counter-fast" }
diff --git a/litellm-rust/crates/cache-disk/Cargo.toml b/litellm-rust/crates/cache-disk/Cargo.toml
new file mode 100644
index 00000000000..023055e391a
--- /dev/null
+++ b/litellm-rust/crates/cache-disk/Cargo.toml
@@ -0,0 +1,17 @@
+[package]
+name = "litellm-cache-disk"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-cache.workspace = true
+rand.workspace = true
+rusqlite = { version = "0.40", features = ["bundled"] }
+serde-pickle = "1.2"
+serde_json.workspace = true
+tokio.workspace = true
+
+[dev-dependencies]
+tempfile = "3.27.0"
diff --git a/litellm-rust/crates/cache-disk/src/cache.rs b/litellm-rust/crates/cache-disk/src/cache.rs
new file mode 100644
index 00000000000..bcce3a525b1
--- /dev/null
+++ b/litellm-rust/crates/cache-disk/src/cache.rs
@@ -0,0 +1,330 @@
+use std::{
+    path::Path,
+    sync::Arc,
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, CacheConnectionStatus,
+    CounterCache, DeleteCache, Error, ExactCacheContext, FlushCache,
+};
+use serde_json::Value;
+
+use crate::{DiskStore, DiskcacheSqliteStore, StoredValue, pickle};
+
+pub struct DiskCache<S, D = DiskcacheSqliteStore> {
+    store: Arc<D>,
+    codec: S,
+}
+
+impl<S: CacheCodec> DiskCache<S> {
+    pub fn open(directory: impl AsRef<Path>, codec: S) -> Result<Self, Error> {
+        Ok(Self {
+            store: Arc::new(DiskcacheSqliteStore::open(directory)?),
+            codec,
+        })
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore> DiskCache<S, D> {
+    pub fn with_store(store: D, codec: S) -> Self {
+        Self {
+            store: Arc::new(store),
+            codec,
+        }
+    }
+
+    pub fn directory(&self) -> &Path {
+        self.store.directory()
+    }
+
+    fn decode_stored(&self, value: StoredValue) -> Result<Option<S::Value>, Error> {
+        let Some(bytes) = payload(value)? else {
+            return Ok(None);
+        };
+        self.codec.decode(&bytes).map(Some)
+    }
+
+    async fn run_blocking<T, F>(store: Arc<D>, operation: F) -> Result<T, Error>
+    where
+        T: Send + 'static,
+        F: FnOnce(&D) -> Result<T, Error> + Send + 'static,
+    {
+        tokio::task::spawn_blocking(move || operation(&store))
+            .await
+            .map_err(|_| Error::Unavailable)?
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore> BaseCache for DiskCache<S, D> {
+    type Value = S::Value;
+    type Context = ExactCacheContext;
+
+    fn get_ttl(&self, context: &Self::Context) -> Option<Duration> {
+        context.ttl
+    }
+
+    fn set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: &Self::Context,
+    ) -> Result<(), Error> {
+        let value = StoredValue::Bytes(self.codec.encode(&value)?);
+        let expire_time = context.ttl.map(|ttl| unix_now() + ttl.as_secs_f64());
+        self.store.set(key, value, expire_time, unix_now())
+    }
+
+    fn get_cache(&self, key: &str, _: &Self::Context) -> Result<Option<Self::Value>, Error> {
+        self.store
+            .get(key, unix_now())?
+            .map(|value| self.decode_stored(value))
+            .transpose()
+            .map(|value| value.flatten())
+    }
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: ExactCacheContext,
+    ) -> Result<(), Error> {
+        let value = StoredValue::Bytes(self.codec.encode(&value)?);
+        let expire_time = context.ttl.map(|ttl| unix_now() + ttl.as_secs_f64());
+        let key = key.to_string();
+        Self::run_blocking(Arc::clone(&self.store), move |store| {
+            store.set(&key, value, expire_time, unix_now())
+        })
+        .await
+    }
+
+    async fn async_get_cache(
+        &self,
+        key: &str,
+        _: &ExactCacheContext,
+    ) -> Result<Option<Self::Value>, Error> {
+        let key = key.to_string();
+        let value = Self::run_blocking(Arc::clone(&self.store), move |store| {
+            store.get(&key, unix_now())
+        })
+        .await?;
+        value
+            .map(|value| self.decode_stored(value))
+            .transpose()
+            .map(|value| value.flatten())
+    }
+
+    async fn async_set_cache_pipeline(
+        &self,
+        entries: Vec<(String, Self::Value)>,
+        context: ExactCacheContext,
+    ) -> Result<(), Error> {
+        let entries = entries
+            .into_iter()
+            .map(|(key, value)| {
+                self.codec
+                    .encode(&value)
+                    .map(|value| (key, StoredValue::Bytes(value)))
+            })
+            .collect::<Result<Vec<_>, _>>()?;
+        let expire_after = context.ttl;
+        Self::run_blocking(Arc::clone(&self.store), move |store| {
+            for (key, value) in entries {
+                let expire_time = expire_after.map(|ttl| unix_now() + ttl.as_secs_f64());
+                store.set(&key, value, expire_time, unix_now())?;
+            }
+            Ok(())
+        })
+        .await
+    }
+
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        let result = Self::run_blocking(Arc::clone(&self.store), |store| {
+            store.probe().map(|_| CacheConnectionResult {
+                status: CacheConnectionStatus::Success,
+                message: "Disk cache connection test successful".into(),
+                error: None,
+            })
+        })
+        .await;
+        Ok(match result {
+            Ok(result) => result,
+            Err(error) => CacheConnectionResult {
+                status: CacheConnectionStatus::Failed,
+                message: format!("Disk cache connection failed: {error}"),
+                error: Some(error.to_string()),
+            },
+        })
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore> BatchCache for DiskCache<S, D> {
+    fn batch_get_cache(
+        &self,
+        keys: &[String],
+        context: &ExactCacheContext,
+    ) -> Result<Vec<BatchEntry<Self::Value>>, Error> {
+        keys.iter()
+            .map(|key| match self.get_cache(key, context) {
+                Ok(Some(value)) => Ok(BatchEntry::Hit(value)),
+                Ok(None) => Ok(BatchEntry::Miss),
+                Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid),
+                Err(error) => Err(error),
+            })
+            .collect()
+    }
+
+    async fn async_batch_get_cache(
+        &self,
+        keys: Vec<String>,
+        _: ExactCacheContext,
+    ) -> Result<Vec<BatchEntry<Self::Value>>, Error> {
+        let values = Self::run_blocking(Arc::clone(&self.store), move |store| {
+            keys.into_iter()
+                .map(|key| store.get(&key, unix_now()).map(|value| (key, value)))
+                .collect::<Result<Vec<_>, _>>()
+        })
+        .await?;
+        values
+            .into_iter()
+            .map(|(_, value)| match value {
+                None => Ok(BatchEntry::Miss),
+                Some(value) => match self.decode_stored(value) {
+                    Ok(Some(value)) => Ok(BatchEntry::Hit(value)),
+                    Ok(None) => Ok(BatchEntry::Miss),
+                    Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid),
+                    Err(error) => Err(error),
+                },
+            })
+            .collect()
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore> DeleteCache for DiskCache<S, D> {
+    fn delete_cache(&self, key: &str) -> Result<(), Error> {
+        self.store.pop(key, unix_now()).map(|_| ())
+    }
+
+    async fn async_delete_cache(&self, key: &str) -> Result<(), Error> {
+        let key = key.to_string();
+        Self::run_blocking(Arc::clone(&self.store), move |store| {
+            store.pop(&key, unix_now()).map(|_| ())
+        })
+        .await
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore> FlushCache for DiskCache<S, D> {
+    fn flush_cache(&self) -> Result<(), Error> {
+        self.store.clear()
+    }
+
+    async fn async_flush_cache(&self) -> Result<(), Error> {
+        Self::run_blocking(Arc::clone(&self.store), |store| store.clear()).await
+    }
+}
+
+impl<S: CacheCodec<Value = f64>, D: DiskStore> CounterCache for DiskCache<S, D> {
+    fn increment_cache(
+        &self,
+        key: &str,
+        amount: f64,
+        context: ExactCacheContext,
+    ) -> Result<f64, Error> {
+        increment(self.store.as_ref(), key, amount, context.ttl)
+    }
+
+    async fn async_increment(
+        &self,
+        key: &str,
+        amount: f64,
+        context: ExactCacheContext,
+    ) -> Result<f64, Error> {
+        let key = key.to_string();
+        Self::run_blocking(Arc::clone(&self.store), move |store| {
+            increment(store, &key, amount, context.ttl)
+        })
+        .await
+    }
+}
+
+fn increment<D: DiskStore>(
+    store: &D,
+    key: &str,
+    amount: f64,
+    ttl: Option<Duration>,
+) -> Result<f64, Error> {
+    let mut result = None;
+    let mut apply = |current: Option<StoredValue>| {
+        let initial = match current {
+            Some(StoredValue::Integer(value)) => value as f64,
+            Some(StoredValue::Pickle(value)) => match pickle::decode(&value)? {
+                Value::Number(value) => value
+                    .as_i64()
+                    .map(|value| value as f64)
+                    .or_else(|| value.as_u64().map(|value| value as f64))
+                    .unwrap_or_default(),
+                _ => 0.0,
+            },
+            _ => 0.0,
+        };
+        let value = initial + amount;
+        let stored = if value.fract() == 0.0 && value >= i64::MIN as f64 && value <= i64::MAX as f64
+        {
+            StoredValue::Integer(value as i64)
+        } else {
+            StoredValue::Float(value)
+        };
+        result = Some(value);
+        Ok((stored, ttl.map(|ttl| unix_now() + ttl.as_secs_f64())))
+    };
+    store.update(key, unix_now(), &mut apply)?;
+    result.ok_or(Error::InvalidEntry)
+}
+
+fn payload(value: StoredValue) -> Result<Option<Vec<u8>>, Error> {
+    match value {
+        StoredValue::Bytes(value) if value.is_empty() => Ok(None),
+        StoredValue::Bytes(value) => Ok(Some(value)),
+        StoredValue::Text(value) if value.is_empty() => Ok(None),
+        StoredValue::Text(value) => Ok(Some(value.into_bytes())),
+        StoredValue::Integer(0) => Ok(None),
+        StoredValue::Integer(value) => Ok(Some(value.to_string().into_bytes())),
+        StoredValue::Float(0.0) => Ok(None),
+        StoredValue::Float(value) => serde_json::to_vec(&value)
+            .map(Some)
+            .map_err(|_| Error::InvalidEntry),
+        StoredValue::Pickle(value) => {
+            let value = pickle::decode(&value)?;
+            if is_falsy(&value) {
+                Ok(None)
+            } else {
+                serde_json::to_vec(&value)
+                    .map(Some)
+                    .map_err(|_| Error::InvalidEntry)
+            }
+        }
+    }
+}
+
+fn is_falsy(value: &Value) -> bool {
+    match value {
+        Value::Null | Value::Bool(false) => true,
+        Value::Number(value) => value.as_f64().is_some_and(|value| value == 0.0),
+        Value::String(value) => value.is_empty(),
+        Value::Array(value) => value.is_empty(),
+        Value::Object(value) => value.is_empty(),
+        Value::Bool(true) => false,
+    }
+}
+
+fn unix_now() -> f64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs_f64()
+}
diff --git a/litellm-rust/crates/cache-disk/src/lib.rs b/litellm-rust/crates/cache-disk/src/lib.rs
new file mode 100644
index 00000000000..189d12ea793
--- /dev/null
+++ b/litellm-rust/crates/cache-disk/src/lib.rs
@@ -0,0 +1,8 @@
+mod cache;
+mod pickle;
+mod sqlite;
+mod store;
+
+pub use cache::DiskCache;
+pub use sqlite::DiskcacheSqliteStore;
+pub use store::{DiskStore, StoredValue};
diff --git a/litellm-rust/crates/cache-disk/src/pickle.rs b/litellm-rust/crates/cache-disk/src/pickle.rs
new file mode 100644
index 00000000000..1d31e0bbeb6
--- /dev/null
+++ b/litellm-rust/crates/cache-disk/src/pickle.rs
@@ -0,0 +1,48 @@
+use litellm_cache::Error;
+use serde_json::{Map, Number, Value};
+
+pub(crate) fn decode(bytes: &[u8]) -> Result<Value, Error> {
+    let value = serde_pickle::value_from_slice(bytes, Default::default())
+        .map_err(|_| Error::InvalidEntry)?;
+    convert(value)
+}
+
+fn convert(value: serde_pickle::Value) -> Result<Value, Error> {
+    match value {
+        serde_pickle::Value::None => Ok(Value::Null),
+        serde_pickle::Value::Bool(value) => Ok(Value::Bool(value)),
+        serde_pickle::Value::I64(value) => Ok(Value::Number(value.into())),
+        serde_pickle::Value::Int(value) => {
+            if let Ok(value) = value.to_string().parse::<i64>() {
+                Ok(Value::Number(value.into()))
+            } else if let Ok(value) = value.to_string().parse::<u64>() {
+                Ok(Value::Number(value.into()))
+            } else {
+                Err(Error::InvalidEntry)
+            }
+        }
+        serde_pickle::Value::F64(value) => Number::from_f64(value)
+            .map(Value::Number)
+            .ok_or(Error::InvalidEntry),
+        serde_pickle::Value::String(value) => Ok(Value::String(value)),
+        serde_pickle::Value::List(values) | serde_pickle::Value::Tuple(values) => values
+            .into_iter()
+            .map(convert)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::Array),
+        serde_pickle::Value::Set(values) | serde_pickle::Value::FrozenSet(values) => values
+            .into_iter()
+            .map(|value| convert(value.into_value()))
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::Array),
+        serde_pickle::Value::Dict(values) => values
+            .into_iter()
+            .map(|(key, value)| match key {
+                serde_pickle::HashableValue::String(key) => Ok((key, convert(value)?)),
+                _ => Err(Error::InvalidEntry),
+            })
+            .collect::<Result<Map<_, _>, _>>()
+            .map(Value::Object),
+        serde_pickle::Value::Bytes(_) => Err(Error::InvalidEntry),
+    }
+}
diff --git a/litellm-rust/crates/cache-disk/src/sqlite.rs b/litellm-rust/crates/cache-disk/src/sqlite.rs
new file mode 100644
index 00000000000..ff5a44b5c38
--- /dev/null
+++ b/litellm-rust/crates/cache-disk/src/sqlite.rs
@@ -0,0 +1,816 @@
+use std::{
+    collections::HashMap,
+    fs::{self, OpenOptions},
+    io::Write,
+    path::{Path, PathBuf},
+    sync::Mutex,
+};
+
+use litellm_cache::Error;
+use rand::RngCore;
+use rusqlite::{Connection, OptionalExtension, params, types::Value};
+
+use crate::{DiskStore, StoredValue};
+
+const MODE_RAW: i64 = 1;
+const MODE_BINARY: i64 = 2;
+const MODE_TEXT: i64 = 3;
+const MODE_PICKLE: i64 = 4;
+
+const DEFAULT_DISK_MIN_FILE_SIZE: i64 = 2_i64.pow(15);
+const DEFAULT_SIZE_LIMIT: i64 = 2_i64.pow(30);
+const DEFAULT_CULL_LIMIT: i64 = 10;
+
+pub struct DiskcacheSqliteStore {
+    directory: PathBuf,
+    connection: Mutex<Connection>,
+    min_file_size: usize,
+    eviction_policy: String,
+    size_limit: i64,
+    cull_limit: i64,
+    statistics: bool,
+}
+
+struct StoredColumns {
+    size: i64,
+    mode: i64,
+    filename: Option<String>,
+    value: Option<Value>,
+}
+
+struct Row {
+    rowid: i64,
+    mode: i64,
+    filename: Option<String>,
+    value: Value,
+}
+
+impl DiskcacheSqliteStore {
+    pub fn open(directory: impl AsRef<Path>) -> Result<Self, Error> {
+        let directory = directory.as_ref().to_path_buf();
+        fs::create_dir_all(&directory).map_err(|_| Error::Unavailable)?;
+        let database = directory.join("cache.db");
+        let connection = Connection::open(database).map_err(|_| Error::Unavailable)?;
+        connection
+            .busy_timeout(std::time::Duration::from_secs(60))
+            .map_err(|_| Error::Unavailable)?;
+
+        let mut settings = read_settings(&connection)?;
+        for (key, value) in default_settings() {
+            settings.entry(key).or_insert(value);
+        }
+        for (key, value) in settings
+            .iter()
+            .filter(|(key, _)| key.starts_with("sqlite_"))
+        {
+            apply_pragma(&connection, key, value)?;
+        }
+
+        connection
+            .execute_batch(
+                "CREATE TABLE IF NOT EXISTS Settings (
+                    key TEXT NOT NULL UNIQUE,
+                    value
+                )",
+            )
+            .map_err(|_| Error::Unavailable)?;
+        for (key, value) in &settings {
+            if !matches!(key.as_str(), "count" | "size" | "hits" | "misses") {
+                connection
+                    .execute(
+                        "INSERT OR REPLACE INTO Settings VALUES (?, ?)",
+                        params![key, value],
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+        }
+        for (key, value) in [
+            ("count", Value::Integer(0)),
+            ("size", Value::Integer(0)),
+            ("hits", Value::Integer(0)),
+            ("misses", Value::Integer(0)),
+        ] {
+            connection
+                .execute(
+                    "INSERT OR IGNORE INTO Settings VALUES (?, ?)",
+                    params![key, value],
+                )
+                .map_err(|_| Error::Unavailable)?;
+        }
+        connection
+            .execute_batch(
+                "CREATE TABLE IF NOT EXISTS Cache (
+                    rowid INTEGER PRIMARY KEY,
+                    key BLOB,
+                    raw INTEGER,
+                    store_time REAL,
+                    expire_time REAL,
+                    access_time REAL,
+                    access_count INTEGER DEFAULT 0,
+                    tag BLOB,
+                    size INTEGER DEFAULT 0,
+                    mode INTEGER DEFAULT 0,
+                    filename TEXT,
+                    value BLOB
+                );
+                CREATE UNIQUE INDEX IF NOT EXISTS Cache_key_raw ON Cache(key, raw);
+                CREATE INDEX IF NOT EXISTS Cache_expire_time ON Cache(expire_time);",
+            )
+            .map_err(|_| Error::Unavailable)?;
+
+        let eviction_policy = setting_string(&settings, "eviction_policy")
+            .unwrap_or_else(|| "least-recently-stored".to_string());
+        match eviction_policy.as_str() {
+            "none" => {}
+            "least-recently-stored" => {
+                connection
+                    .execute_batch(
+                        "CREATE INDEX IF NOT EXISTS Cache_store_time ON Cache(store_time)",
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+            "least-recently-used" => {
+                connection
+                    .execute_batch(
+                        "CREATE INDEX IF NOT EXISTS Cache_access_time ON Cache(access_time)",
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+            "least-frequently-used" => {
+                connection
+                    .execute_batch(
+                        "CREATE INDEX IF NOT EXISTS Cache_access_count ON Cache(access_count)",
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+            _ => return Err(Error::Unavailable),
+        }
+        connection
+            .execute_batch(
+                "CREATE TRIGGER IF NOT EXISTS Settings_count_insert
+                    AFTER INSERT ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings SET value = value + 1
+                    WHERE key = \"count\"; END;
+                 CREATE TRIGGER IF NOT EXISTS Settings_count_delete
+                    AFTER DELETE ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings SET value = value - 1
+                    WHERE key = \"count\"; END;
+                 CREATE TRIGGER IF NOT EXISTS Settings_size_insert
+                    AFTER INSERT ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings SET value = value + NEW.size
+                    WHERE key = \"size\"; END;
+                 CREATE TRIGGER IF NOT EXISTS Settings_size_update
+                    AFTER UPDATE ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings
+                    SET value = value + NEW.size - OLD.size
+                    WHERE key = \"size\"; END;
+                 CREATE TRIGGER IF NOT EXISTS Settings_size_delete
+                    AFTER DELETE ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings SET value = value - OLD.size
+                    WHERE key = \"size\"; END;",
+            )
+            .map_err(|_| Error::Unavailable)?;
+
+        let min_file_size = setting_i64(&settings, "disk_min_file_size")
+            .unwrap_or(DEFAULT_DISK_MIN_FILE_SIZE)
+            .try_into()
+            .map_err(|_| Error::Unavailable)?;
+        let size_limit = setting_i64(&settings, "size_limit").unwrap_or(DEFAULT_SIZE_LIMIT);
+        let cull_limit = setting_i64(&settings, "cull_limit").unwrap_or(DEFAULT_CULL_LIMIT);
+        let statistics = setting_i64(&settings, "statistics").unwrap_or_default() != 0;
+
+        Ok(Self {
+            directory,
+            connection: Mutex::new(connection),
+            min_file_size,
+            eviction_policy,
+            size_limit,
+            cull_limit,
+            statistics,
+        })
+    }
+
+    fn set_locked(
+        &self,
+        connection: &Connection,
+        key: &str,
+        columns: StoredColumns,
+        expire_time: Option<f64>,
+        now: f64,
+    ) -> Result<Vec<String>, Error> {
+        let mut cleanup = Vec::new();
+        if let Some(old_filename) = connection
+            .query_row(
+                "SELECT filename FROM Cache WHERE key = ? AND raw = 1",
+                params![key],
+                |row| row.get::<_, Option<String>>(0),
+            )
+            .optional()
+            .map_err(|_| Error::Unavailable)?
+            .flatten()
+        {
+            cleanup.push(old_filename);
+        }
+        let (size, mode, filename, value) =
+            (columns.size, columns.mode, columns.filename, columns.value);
+        let rowid = connection
+            .query_row(
+                "SELECT rowid FROM Cache WHERE key = ? AND raw = 1",
+                params![key],
+                |row| row.get::<_, i64>(0),
+            )
+            .optional()
+            .map_err(|_| Error::Unavailable)?;
+        if let Some(rowid) = rowid {
+            connection
+                .execute(
+                    "UPDATE Cache SET store_time = ?, expire_time = ?, access_time = ?,
+                        access_count = 0, tag = NULL, size = ?, mode = ?, filename = ?, value = ?
+                        WHERE rowid = ?",
+                    params![now, expire_time, now, size, mode, filename, value, rowid],
+                )
+                .map_err(|_| Error::Unavailable)?;
+        } else {
+            connection
+                .execute(
+                    "INSERT INTO Cache(
+                        key, raw, store_time, expire_time, access_time, access_count,
+                        tag, size, mode, filename, value
+                    ) VALUES (?, 1, ?, ?, ?, 0, NULL, ?, ?, ?, ?)",
+                    params![key, now, expire_time, now, size, mode, filename, value],
+                )
+                .map_err(|_| Error::Unavailable)?;
+        }
+        cleanup.extend(self.cull(connection, now)?);
+        Ok(cleanup)
+    }
+
+    fn cull(&self, connection: &Connection, now: f64) -> Result<Vec<String>, Error> {
+        if self.cull_limit <= 0 {
+            return Ok(Vec::new());
+        }
+        let mut cleanup = Vec::new();
+        let expired = connection
+            .prepare(
+                "SELECT rowid, filename FROM Cache
+                 WHERE expire_time IS NOT NULL AND expire_time < ?
+                 ORDER BY expire_time LIMIT ?",
+            )
+            .map_err(|_| Error::Unavailable)?
+            .query_map(params![now, self.cull_limit], |row| {
+                Ok((row.get::<_, i64>(0)?, row.get::<_, Option<String>>(1)?))
+            })
+            .map_err(|_| Error::Unavailable)?
+            .collect::<Result<Vec<_>, _>>()
+            .map_err(|_| Error::Unavailable)?;
+        for (_, filename) in &expired {
+            if let Some(filename) = filename {
+                cleanup.push(filename.clone());
+            }
+        }
+        for (rowid, _) in &expired {
+            connection
+                .execute("DELETE FROM Cache WHERE rowid = ?", params![rowid])
+                .map_err(|_| Error::Unavailable)?;
+        }
+        let remaining = self.cull_limit - i64::try_from(expired.len()).unwrap_or(self.cull_limit);
+        if remaining <= 0 || self.volume(connection)? < self.size_limit {
+            return Ok(cleanup);
+        }
+        let order = match self.eviction_policy.as_str() {
+            "none" => return Ok(cleanup),
+            "least-recently-stored" => "store_time",
+            "least-recently-used" => "access_time",
+            "least-frequently-used" => "access_count",
+            _ => return Err(Error::Unavailable),
+        };
+        let rows = connection
+            .prepare(&format!(
+                "SELECT rowid, filename FROM Cache ORDER BY {order} LIMIT ?"
+            ))
+            .map_err(|_| Error::Unavailable)?
+            .query_map(params![remaining], |row| {
+                Ok((row.get::<_, i64>(0)?, row.get::<_, Option<String>>(1)?))
+            })
+            .map_err(|_| Error::Unavailable)?
+            .collect::<Result<Vec<_>, _>>()
+            .map_err(|_| Error::Unavailable)?;
+        for (_, filename) in &rows {
+            if let Some(filename) = filename {
+                cleanup.push(filename.clone());
+            }
+        }
+        for (rowid, _) in rows {
+            connection
+                .execute("DELETE FROM Cache WHERE rowid = ?", params![rowid])
+                .map_err(|_| Error::Unavailable)?;
+        }
+        Ok(cleanup)
+    }
+
+    fn volume(&self, connection: &Connection) -> Result<i64, Error> {
+        let page_count: i64 = connection
+            .query_row("PRAGMA page_count", [], |row| row.get(0))
+            .map_err(|_| Error::Unavailable)?;
+        let page_size: i64 = connection
+            .query_row("PRAGMA page_size", [], |row| row.get(0))
+            .map_err(|_| Error::Unavailable)?;
+        let size: i64 = connection
+            .query_row("SELECT value FROM Settings WHERE key = 'size'", [], |row| {
+                row.get(0)
+            })
+            .map_err(|_| Error::Unavailable)?;
+        Ok(page_count.saturating_mul(page_size).saturating_add(size))
+    }
+}
+
+impl DiskStore for DiskcacheSqliteStore {
+    fn directory(&self) -> &Path {
+        &self.directory
+    }
+
+    fn get(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let select = "SELECT rowid, expire_time, mode, filename, value FROM Cache
+            WHERE key = ? AND raw = 1 AND (expire_time IS NULL OR expire_time > ?)";
+        let row = connection
+            .query_row(select, params![key, now], row_from_query)
+            .optional()
+            .map_err(|_| Error::Unavailable)?;
+        if !self.statistics && !has_get_update(&self.eviction_policy) {
+            return row
+                .map(|row| fetch_row(&self.directory, row))
+                .transpose()
+                .map(|value| value.flatten());
+        }
+        transactional(&connection, |connection| {
+            let row = connection
+                .query_row(select, params![key, now], row_from_query)
+                .optional()
+                .map_err(|_| Error::Unavailable)?;
+            let Some(row) = row else {
+                if self.statistics {
+                    connection
+                        .execute(
+                            "UPDATE Settings SET value = value + 1 WHERE key = 'misses'",
+                            [],
+                        )
+                        .map_err(|_| Error::Unavailable)?;
+                }
+                return Ok(None);
+            };
+            let rowid = row.rowid;
+            let value = fetch_row(&self.directory, row);
+            let hit = value.as_ref().is_ok_and(Option::is_some);
+            if hit && self.statistics {
+                connection
+                    .execute(
+                        "UPDATE Settings SET value = value + 1 WHERE key = 'hits'",
+                        [],
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            } else if !hit && self.statistics {
+                connection
+                    .execute(
+                        "UPDATE Settings SET value = value + 1 WHERE key = 'misses'",
+                        [],
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+            if has_get_update(&self.eviction_policy) && hit {
+                let update = match self.eviction_policy.as_str() {
+                    "least-recently-used" => "UPDATE Cache SET access_time = ? WHERE rowid = ?",
+                    "least-frequently-used" => {
+                        "UPDATE Cache SET access_count = access_count + 1 WHERE rowid = ?"
+                    }
+                    _ => return Err(Error::Unavailable),
+                };
+                if self.eviction_policy == "least-recently-used" {
+                    connection
+                        .execute(update, params![now, rowid])
+                        .map_err(|_| Error::Unavailable)?;
+                } else {
+                    connection
+                        .execute(update, params![rowid])
+                        .map_err(|_| Error::Unavailable)?;
+                }
+            }
+            value
+        })
+    }
+
+    fn set(
+        &self,
+        key: &str,
+        value: StoredValue,
+        expire_time: Option<f64>,
+        now: f64,
+    ) -> Result<(), Error> {
+        let columns = store_value(&self.directory, self.min_file_size, value)?;
+        let new_filename = columns.filename.clone();
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let result = transactional(&connection, |connection| {
+            self.set_locked(connection, key, columns, expire_time, now)
+        });
+        match result {
+            Ok(cleanup) => {
+                cleanup_files(&self.directory, cleanup);
+                Ok(())
+            }
+            Err(error) => {
+                if let Some(filename) = new_filename {
+                    remove_file(&self.directory, &filename);
+                }
+                Err(error)
+            }
+        }
+    }
+
+    fn pop(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let selected = transactional(&connection, |connection| {
+            let row = connection
+                .query_row(
+                    "SELECT rowid, expire_time, mode, filename, value FROM Cache
+                     WHERE key = ? AND raw = 1
+                     AND (expire_time IS NULL OR expire_time > ?)",
+                    params![key, now],
+                    row_from_query,
+                )
+                .optional()
+                .map_err(|_| Error::Unavailable)?;
+            let Some(row) = row else {
+                return Ok(None);
+            };
+            connection
+                .execute("DELETE FROM Cache WHERE rowid = ?", params![row.rowid])
+                .map_err(|_| Error::Unavailable)?;
+            Ok(Some(row))
+        })?;
+        let Some(row) = selected else {
+            return Ok(None);
+        };
+        let filename = row.filename.clone();
+        let result = fetch_row(&self.directory, row)?;
+        if let Some(filename) = filename {
+            remove_file(&self.directory, &filename);
+        }
+        Ok(result)
+    }
+
+    fn clear(&self) -> Result<(), Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let mut last_rowid = 0_i64;
+        loop {
+            let batch = transactional(&connection, |connection| {
+                let rows = connection
+                    .prepare(
+                        "SELECT rowid, filename FROM Cache
+                         WHERE rowid > ? ORDER BY rowid LIMIT 100",
+                    )
+                    .map_err(|_| Error::Unavailable)?
+                    .query_map(params![last_rowid], |row| {
+                        Ok((row.get::<_, i64>(0)?, row.get::<_, Option<String>>(1)?))
+                    })
+                    .map_err(|_| Error::Unavailable)?
+                    .collect::<Result<Vec<_>, _>>()
+                    .map_err(|_| Error::Unavailable)?;
+                if rows.is_empty() {
+                    return Ok(rows);
+                }
+                let ids = rows
+                    .iter()
+                    .map(|(rowid, _)| rowid.to_string())
+                    .collect::<Vec<_>>()
+                    .join(",");
+                connection
+                    .execute(&format!("DELETE FROM Cache WHERE rowid IN ({ids})"), [])
+                    .map_err(|_| Error::Unavailable)?;
+                Ok(rows)
+            })?;
+            if batch.is_empty() {
+                return Ok(());
+            }
+            last_rowid = batch.last().map(|(rowid, _)| *rowid).unwrap_or(last_rowid);
+            cleanup_files(
+                &self.directory,
+                batch
+                    .into_iter()
+                    .filter_map(|(_, filename)| filename)
+                    .collect(),
+            );
+        }
+    }
+
+    fn update(
+        &self,
+        key: &str,
+        now: f64,
+        apply: &mut dyn FnMut(Option<StoredValue>) -> Result<(StoredValue, Option<f64>), Error>,
+    ) -> Result<(), Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let mut created_filename = None;
+        let result = transactional(&connection, |connection| {
+            let current = connection
+                .query_row(
+                    "SELECT rowid, expire_time, mode, filename, value FROM Cache
+                     WHERE key = ? AND raw = 1
+                     AND (expire_time IS NULL OR expire_time > ?)",
+                    params![key, now],
+                    row_from_query,
+                )
+                .optional()
+                .map_err(|_| Error::Unavailable)?
+                .map(|row| fetch_row(&self.directory, row))
+                .transpose()?
+                .flatten();
+            let (value, expire_time) = apply(current)?;
+            let columns = store_value(&self.directory, self.min_file_size, value)?;
+            created_filename = columns.filename.clone();
+            let cleanup = self.set_locked(connection, key, columns, expire_time, now)?;
+            Ok(cleanup)
+        });
+        match result {
+            Ok(cleanup) => {
+                cleanup_files(&self.directory, cleanup);
+                Ok(())
+            }
+            Err(error) => {
+                if let Some(filename) = created_filename {
+                    remove_file(&self.directory, &filename);
+                }
+                Err(error)
+            }
+        }
+    }
+
+    fn probe(&self) -> Result<(), Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        connection
+            .query_row(
+                "SELECT value FROM Settings WHERE key = 'count'",
+                [],
+                |row| row.get::<_, i64>(0),
+            )
+            .map(|_| ())
+            .map_err(|_| Error::Unavailable)
+    }
+}
+
+fn default_settings() -> HashMap<String, Value> {
+    HashMap::from([
+        ("statistics".to_string(), Value::Integer(0)),
+        ("tag_index".to_string(), Value::Integer(0)),
+        (
+            "eviction_policy".to_string(),
+            Value::Text("least-recently-stored".to_string()),
+        ),
+        ("size_limit".to_string(), Value::Integer(DEFAULT_SIZE_LIMIT)),
+        ("cull_limit".to_string(), Value::Integer(DEFAULT_CULL_LIMIT)),
+        ("sqlite_auto_vacuum".to_string(), Value::Integer(1)),
+        ("sqlite_cache_size".to_string(), Value::Integer(8192)),
+        (
+            "sqlite_journal_mode".to_string(),
+            Value::Text("wal".to_string()),
+        ),
+        (
+            "sqlite_mmap_size".to_string(),
+            Value::Integer(2_i64.pow(26)),
+        ),
+        ("sqlite_synchronous".to_string(), Value::Integer(1)),
+        (
+            "disk_min_file_size".to_string(),
+            Value::Integer(DEFAULT_DISK_MIN_FILE_SIZE),
+        ),
+        ("disk_pickle_protocol".to_string(), Value::Integer(5)),
+    ])
+}
+
+fn read_settings(connection: &Connection) -> Result<HashMap<String, Value>, Error> {
+    let mut statement = match connection.prepare("SELECT key, value FROM Settings") {
+        Ok(statement) => statement,
+        Err(_) => return Ok(HashMap::new()),
+    };
+    statement
+        .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
+        .map_err(|_| Error::Unavailable)?
+        .collect::<Result<HashMap<_, _>, _>>()
+        .map_err(|_| Error::Unavailable)
+}
+
+fn apply_pragma(connection: &Connection, key: &str, value: &Value) -> Result<(), Error> {
+    let pragma = key.strip_prefix("sqlite_").ok_or(Error::Unavailable)?;
+    match value {
+        Value::Integer(value) => connection
+            .pragma_update(None, pragma, value)
+            .map_err(|_| Error::Unavailable),
+        Value::Text(value) => connection
+            .pragma_update(None, pragma, value)
+            .map_err(|_| Error::Unavailable),
+        _ => Err(Error::Unavailable),
+    }
+}
+
+fn setting_i64(settings: &HashMap<String, Value>, key: &str) -> Option<i64> {
+    match settings.get(key) {
+        Some(Value::Integer(value)) => Some(*value),
+        _ => None,
+    }
+}
+
+fn setting_string(settings: &HashMap<String, Value>, key: &str) -> Option<String> {
+    match settings.get(key) {
+        Some(Value::Text(value)) => Some(value.clone()),
+        _ => None,
+    }
+}
+
+fn has_get_update(policy: &str) -> bool {
+    matches!(policy, "least-recently-used" | "least-frequently-used")
+}
+
+fn row_from_query(row: &rusqlite::Row<'_>) -> rusqlite::Result<Row> {
+    Ok(Row {
+        rowid: row.get(0)?,
+        mode: row.get(2)?,
+        filename: row.get(3)?,
+        value: row.get(4)?,
+    })
+}
+
+fn fetch_row(directory: &Path, row: Row) -> Result<Option<StoredValue>, Error> {
+    match row.mode {
+        MODE_RAW => match row.value {
+            Value::Blob(value) => Ok(Some(StoredValue::Bytes(value))),
+            Value::Text(value) => Ok(Some(StoredValue::Text(value))),
+            Value::Integer(value) => Ok(Some(StoredValue::Integer(value))),
+            Value::Real(value) => Ok(Some(StoredValue::Float(value))),
+            Value::Null => Err(Error::InvalidEntry),
+        },
+        MODE_BINARY | MODE_PICKLE => {
+            let bytes = match row.value {
+                Value::Blob(value) => value,
+                Value::Null => {
+                    let Some(value) = read_file(directory, row.filename.as_deref())? else {
+                        return Ok(None);
+                    };
+                    value
+                }
+                _ => return Err(Error::InvalidEntry),
+            };
+            Ok(Some(if row.mode == MODE_BINARY {
+                StoredValue::Bytes(bytes)
+            } else {
+                StoredValue::Pickle(bytes)
+            }))
+        }
+        MODE_TEXT => {
+            let bytes = match row.value {
+                Value::Null => {
+                    let Some(value) = read_file(directory, row.filename.as_deref())? else {
+                        return Ok(None);
+                    };
+                    value
+                }
+                Value::Blob(value) => value,
+                Value::Text(value) => value.into_bytes(),
+                _ => return Err(Error::InvalidEntry),
+            };
+            Ok(Some(StoredValue::Text(
+                String::from_utf8(bytes).map_err(|_| Error::InvalidEntry)?,
+            )))
+        }
+        _ => Err(Error::InvalidEntry),
+    }
+}
+
+fn read_file(directory: &Path, filename: Option<&str>) -> Result<Option<Vec<u8>>, Error> {
+    let Some(filename) = filename else {
+        return Err(Error::InvalidEntry);
+    };
+    match fs::read(directory.join(filename)) {
+        Ok(value) => Ok(Some(value)),
+        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
+        Err(_) => Err(Error::Unavailable),
+    }
+}
+
+fn store_value(
+    directory: &Path,
+    min_file_size: usize,
+    value: StoredValue,
+) -> Result<StoredColumns, Error> {
+    match value {
+        StoredValue::Integer(value) => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_RAW,
+            filename: None,
+            value: Some(Value::Integer(value)),
+        }),
+        StoredValue::Float(value) => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_RAW,
+            filename: None,
+            value: Some(Value::Real(value)),
+        }),
+        StoredValue::Text(value) if value.chars().count() < min_file_size => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_RAW,
+            filename: None,
+            value: Some(Value::Text(value)),
+        }),
+        StoredValue::Text(value) => {
+            let bytes = value.into_bytes();
+            let filename = write_file(directory, &bytes)?;
+            Ok(StoredColumns {
+                size: i64::try_from(bytes.len()).map_err(|_| Error::Unavailable)?,
+                mode: MODE_TEXT,
+                filename: Some(filename),
+                value: None,
+            })
+        }
+        StoredValue::Bytes(value) if value.len() < min_file_size => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_RAW,
+            filename: None,
+            value: Some(Value::Blob(value)),
+        }),
+        StoredValue::Bytes(value) => {
+            let filename = write_file(directory, &value)?;
+            Ok(StoredColumns {
+                size: i64::try_from(value.len()).map_err(|_| Error::Unavailable)?,
+                mode: MODE_BINARY,
+                filename: Some(filename),
+                value: None,
+            })
+        }
+        StoredValue::Pickle(value) if value.len() < min_file_size => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_PICKLE,
+            filename: None,
+            value: Some(Value::Blob(value)),
+        }),
+        StoredValue::Pickle(value) => {
+            let filename = write_file(directory, &value)?;
+            Ok(StoredColumns {
+                size: i64::try_from(value.len()).map_err(|_| Error::Unavailable)?,
+                mode: MODE_PICKLE,
+                filename: Some(filename),
+                value: None,
+            })
+        }
+    }
+}
+
+fn write_file(directory: &Path, bytes: &[u8]) -> Result<String, Error> {
+    let mut random = [0_u8; 16];
+    rand::rngs::OsRng.fill_bytes(&mut random);
+    let hex = random
+        .iter()
+        .map(|byte| format!("{byte:02x}"))
+        .collect::<String>();
+    let filename = format!("{}/{}/{}.val", &hex[..2], &hex[2..4], &hex[4..]);
+    let path = directory.join(&filename);
+    if let Some(parent) = path.parent() {
+        fs::create_dir_all(parent).map_err(|_| Error::Unavailable)?;
+    }
+    let mut file = OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(path)
+        .map_err(|_| Error::Unavailable)?;
+    file.write_all(bytes).map_err(|_| Error::Unavailable)?;
+    Ok(filename)
+}
+
+fn cleanup_files(directory: &Path, filenames: Vec<String>) {
+    for filename in filenames {
+        remove_file(directory, &filename);
+    }
+}
+
+fn remove_file(directory: &Path, filename: &str) {
+    let path = directory.join(filename);
+    let _ = fs::remove_file(&path);
+}
+
+fn transactional<T>(
+    connection: &Connection,
+    operation: impl FnOnce(&Connection) -> Result<T, Error>,
+) -> Result<T, Error> {
+    connection
+        .execute_batch("BEGIN IMMEDIATE")
+        .map_err(|_| Error::Unavailable)?;
+    match operation(connection) {
+        Ok(value) => {
+            connection
+                .execute_batch("COMMIT")
+                .map_err(|_| Error::Unavailable)?;
+            Ok(value)
+        }
+        Err(error) => {
+            let _ = connection.execute_batch("ROLLBACK");
+            Err(error)
+        }
+    }
+}
diff --git a/litellm-rust/crates/cache-disk/src/store.rs b/litellm-rust/crates/cache-disk/src/store.rs
new file mode 100644
index 00000000000..ed167317cf0
--- /dev/null
+++ b/litellm-rust/crates/cache-disk/src/store.rs
@@ -0,0 +1,33 @@
+use std::path::Path;
+
+use litellm_cache::Error;
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum StoredValue {
+    Bytes(Vec<u8>),
+    Text(String),
+    Integer(i64),
+    Float(f64),
+    Pickle(Vec<u8>),
+}
+
+pub trait DiskStore: Send + Sync + 'static {
+    fn directory(&self) -> &Path;
+    fn get(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error>;
+    fn set(
+        &self,
+        key: &str,
+        value: StoredValue,
+        expire_time: Option<f64>,
+        now: f64,
+    ) -> Result<(), Error>;
+    fn pop(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error>;
+    fn clear(&self) -> Result<(), Error>;
+    fn update(
+        &self,
+        key: &str,
+        now: f64,
+        apply: &mut dyn FnMut(Option<StoredValue>) -> Result<(StoredValue, Option<f64>), Error>,
+    ) -> Result<(), Error>;
+    fn probe(&self) -> Result<(), Error>;
+}
diff --git a/litellm-rust/crates/cache-disk/tests/cache.rs b/litellm-rust/crates/cache-disk/tests/cache.rs
new file mode 100644
index 00000000000..1e3e39db0c8
--- /dev/null
+++ b/litellm-rust/crates/cache-disk/tests/cache.rs
@@ -0,0 +1,327 @@
+use std::{
+    fs,
+    path::{Path, PathBuf},
+    sync::Arc,
+    thread,
+    time::Duration,
+};
+
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CounterCache, DeleteCache, ExactCacheContext, FlushCache,
+    JsonCodec,
+};
+use litellm_cache_disk::{DiskCache, DiskStore, DiskcacheSqliteStore, StoredValue};
+use rusqlite::Connection;
+use serde_json::json;
+use tempfile::TempDir;
+
+fn store() -> (TempDir, DiskcacheSqliteStore) {
+    let directory = tempfile::tempdir().unwrap();
+    let store = DiskcacheSqliteStore::open(directory.path()).unwrap();
+    (directory, store)
+}
+
+fn cache(directory: &Path) -> DiskCache<JsonCodec<serde_json::Value>> {
+    DiskCache::open(directory, JsonCodec::new()).unwrap()
+}
+
+fn value_files(directory: &Path) -> Vec<PathBuf> {
+    fn visit(directory: &Path, files: &mut Vec<PathBuf>) {
+        for entry in fs::read_dir(directory).unwrap() {
+            let path = entry.unwrap().path();
+            if path.is_dir() {
+                visit(&path, files);
+            } else if path.extension().is_some_and(|extension| extension == "val") {
+                files.push(path);
+            }
+        }
+    }
+
+    let mut files = Vec::new();
+    visit(directory, &mut files);
+    files
+}
+
+#[test]
+fn roundtrip_persists_and_reopens() {
+    let directory = tempfile::tempdir().unwrap();
+    let context = ExactCacheContext::default();
+    let opened = cache(directory.path());
+    opened
+        .set_cache("key", json!({"answer": 42}), &context)
+        .unwrap();
+    assert_eq!(
+        opened.get_cache("key", &context).unwrap(),
+        Some(json!({"answer": 42}))
+    );
+    drop(opened);
+    let reopened = cache(directory.path());
+    assert_eq!(
+        reopened.get_cache("key", &context).unwrap(),
+        Some(json!({"answer": 42}))
+    );
+}
+
+#[test]
+fn ttl_and_expired_culling_match_cache_contract() {
+    let (directory, store) = store();
+    store
+        .set(
+            "expired",
+            StoredValue::Bytes(b"old".to_vec()),
+            Some(10.0),
+            0.0,
+        )
+        .unwrap();
+    assert_eq!(store.get("expired", 10.0).unwrap(), None);
+    store
+        .set("new", StoredValue::Bytes(b"new".to_vec()), None, 11.0)
+        .unwrap();
+    let connection = Connection::open(directory.path().join("cache.db")).unwrap();
+    assert_eq!(
+        connection
+            .query_row("SELECT COUNT(*) FROM Cache", [], |row| row.get::<_, i64>(0))
+            .unwrap(),
+        1
+    );
+    assert_eq!(
+        connection
+            .query_row(
+                "SELECT value FROM Settings WHERE key = 'count'",
+                [],
+                |row| row.get::<_, i64>(0)
+            )
+            .unwrap(),
+        1
+    );
+}
+
+#[test]
+fn batch_preserves_order_and_classifies_misses_and_invalid_values() {
+    let (directory, store) = store();
+    store
+        .set(
+            "hit",
+            StoredValue::Bytes(br#"{"ok":true}"#.to_vec()),
+            None,
+            0.0,
+        )
+        .unwrap();
+    store
+        .set(
+            "invalid",
+            StoredValue::Pickle(vec![0x80, 0x05, 0x2e]),
+            None,
+            0.0,
+        )
+        .unwrap();
+    let cache = cache(directory.path());
+    let entries = cache
+        .batch_get_cache(
+            &["hit".into(), "missing".into(), "invalid".into()],
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    assert_eq!(
+        entries,
+        vec![
+            BatchEntry::Hit(json!({"ok": true})),
+            BatchEntry::Miss,
+            BatchEntry::Invalid
+        ]
+    );
+}
+
+#[test]
+fn falsy_values_are_misses_and_protocol_five_pickle_decodes() {
+    let (directory, store) = store();
+    for (key, value) in [
+        ("empty-bytes", StoredValue::Bytes(Vec::new())),
+        ("empty-text", StoredValue::Text(String::new())),
+        ("zero-int", StoredValue::Integer(0)),
+        ("zero-float", StoredValue::Float(0.0)),
+        (
+            "empty-pickle",
+            StoredValue::Pickle(vec![0x80, 0x05, 0x7d, 0x94, 0x2e]),
+        ),
+    ] {
+        store.set(key, value, None, 0.0).unwrap();
+    }
+    store
+        .set(
+            "pickle",
+            StoredValue::Pickle(
+                b"\x80\x05\x95\x30\x00\x00\x00\x00\x00\x00\x00\x7d\x94\x28\x8c\x09timestamp\x94G\x3f\xf8\x00\x00\x00\x00\x00\x00\x8c\x08response\x94\x8c\x08{\"a\": 1}\x94u."
+                    .to_vec(),
+            ),
+            None,
+            0.0,
+        )
+        .unwrap();
+    let cache = cache(directory.path());
+    for key in [
+        "empty-bytes",
+        "empty-text",
+        "zero-int",
+        "zero-float",
+        "empty-pickle",
+    ] {
+        assert_eq!(
+            cache.get_cache(key, &ExactCacheContext::default()).unwrap(),
+            None
+        );
+    }
+    assert_eq!(
+        cache
+            .get_cache("pickle", &ExactCacheContext::default())
+            .unwrap(),
+        Some(json!({"timestamp": 1.5, "response": "{\"a\": 1}"}))
+    );
+}
+
+#[test]
+fn counters_use_atomic_native_values_and_ignore_invalid_initial_values() {
+    let (directory, store) = store();
+    store
+        .set("counter", StoredValue::Integer(2), None, 0.0)
+        .unwrap();
+    store
+        .set(
+            "invalid",
+            StoredValue::Text("not a number".into()),
+            None,
+            0.0,
+        )
+        .unwrap();
+    store
+        .set(
+            "pickle-counter",
+            StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x02, 0x2e]),
+            None,
+            0.0,
+        )
+        .unwrap();
+    let cache = DiskCache::open(directory.path(), JsonCodec::<f64>::new()).unwrap();
+    assert_eq!(
+        cache
+            .increment_cache("counter", 1.5, ExactCacheContext::default())
+            .unwrap(),
+        3.5
+    );
+    assert_eq!(
+        cache
+            .increment_cache("invalid", 2.0, ExactCacheContext::default())
+            .unwrap(),
+        2.0
+    );
+    assert_eq!(
+        cache
+            .increment_cache("pickle-counter", 1.0, ExactCacheContext::default())
+            .unwrap(),
+        3.0
+    );
+    let connection = Connection::open(directory.path().join("cache.db")).unwrap();
+    assert_eq!(
+        connection
+            .query_row(
+                "SELECT typeof(value) FROM Cache WHERE key = 'counter'",
+                [],
+                |row| row.get::<_, String>(0)
+            )
+            .unwrap(),
+        "real"
+    );
+}
+
+#[test]
+fn counters_are_atomic_across_concurrent_callers() {
+    let directory = tempfile::tempdir().unwrap();
+    let cache = Arc::new(DiskCache::open(directory.path(), JsonCodec::<f64>::new()).unwrap());
+    let workers = (0..8)
+        .map(|_| {
+            let cache = Arc::clone(&cache);
+            thread::spawn(move || {
+                for _ in 0..25 {
+                    cache
+                        .increment_cache("counter", 1.0, ExactCacheContext::default())
+                        .unwrap();
+                }
+            })
+        })
+        .collect::<Vec<_>>();
+    for worker in workers {
+        worker.join().unwrap();
+    }
+    assert_eq!(
+        cache
+            .increment_cache("counter", 0.0, ExactCacheContext::default())
+            .unwrap(),
+        200.0
+    );
+}
+
+#[test]
+fn delete_flush_and_spilled_file_replacement_clean_up_storage() {
+    let (directory, store) = store();
+    let large = vec![b'x'; 32 * 1024];
+    store
+        .set("large", StoredValue::Bytes(large.clone()), None, 0.0)
+        .unwrap();
+    assert_eq!(value_files(directory.path()).len(), 1);
+    store
+        .set(
+            "large",
+            StoredValue::Bytes(vec![b'y'; 32 * 1024]),
+            None,
+            0.0,
+        )
+        .unwrap();
+    assert_eq!(value_files(directory.path()).len(), 1);
+    store.pop("large", 0.0).unwrap();
+    assert!(value_files(directory.path()).is_empty());
+    store
+        .set("a", StoredValue::Bytes(large.clone()), None, 0.0)
+        .unwrap();
+    store
+        .set("b", StoredValue::Bytes(large), None, 0.0)
+        .unwrap();
+    store.clear().unwrap();
+    assert!(value_files(directory.path()).is_empty());
+}
+
+#[tokio::test]
+async fn async_operations_connection_and_delete_match_sync_operations() {
+    let directory = tempfile::tempdir().unwrap();
+    let cache = cache(directory.path());
+    let context = ExactCacheContext {
+        ttl: Some(Duration::from_secs(60)),
+    };
+    cache
+        .async_set_cache("a", json!(1), context.clone())
+        .await
+        .unwrap();
+    cache
+        .async_set_cache_pipeline(
+            vec![("b".into(), json!(2)), ("c".into(), json!(3))],
+            context.clone(),
+        )
+        .await
+        .unwrap();
+    assert_eq!(
+        cache.async_get_cache("a", &context).await.unwrap(),
+        Some(json!(1))
+    );
+    assert_eq!(
+        cache
+            .async_batch_get_cache(vec!["c".into(), "missing".into()], context.clone())
+            .await
+            .unwrap(),
+        vec![BatchEntry::Hit(json!(3)), BatchEntry::Miss]
+    );
+    cache.async_delete_cache("a").await.unwrap();
+    cache.async_flush_cache().await.unwrap();
+    assert_eq!(
+        cache.test_connection().await.unwrap().status,
+        litellm_cache::CacheConnectionStatus::Success
+    );
+}
diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml
index 1eb2ec28036..71846dcacbd 100644
--- a/litellm-rust/crates/python-bridge/Cargo.toml
+++ b/litellm-rust/crates/python-bridge/Cargo.toml
@@ -23,6 +23,7 @@ bytes.workspace = true
 litellm-cache.workspace = true
 litellm-cache-memory.workspace = true
 litellm-cache-redis.workspace = true
+litellm-cache-disk.workspace = true
 litellm-cache-response.workspace = true
 serde.workspace = true
 litellm-auth.workspace = true
diff --git a/litellm-rust/crates/python-bridge/src/cache/config.rs b/litellm-rust/crates/python-bridge/src/cache/config.rs
index 0e7d6aee11d..17c4cb40903 100644
--- a/litellm-rust/crates/python-bridge/src/cache/config.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/config.rs
@@ -1,4 +1,4 @@
-use std::time::Duration;
+use std::{path::PathBuf, time::Duration};
 
 use litellm_cache::CacheType;
 use pyo3::{
@@ -25,6 +25,10 @@ pub(super) struct MemoryCacheConfig {
     pub(super) max_entry_bytes: usize,
 }
 
+pub(super) struct DiskCacheConfig {
+    pub(super) directory: PathBuf,
+}
+
 #[derive(Debug, PartialEq)]
 pub(super) enum RedisProtocol {
     Resp2,
@@ -76,6 +80,7 @@ pub(super) struct RedisCacheConfig {
 pub(super) enum CacheBackendConfig {
     Memory(MemoryCacheConfig),
     Redis(Box<RedisCacheConfig>),
+    Disk(DiskCacheConfig),
 }
 
 #[allow(dead_code, reason = "consumed by the cache activation follow-up")]
@@ -90,6 +95,7 @@ pub(super) enum UnsupportedCacheConfig {
     RedisCredentials,
     RedisConnection,
     RedisOption,
+    DiskStore,
 }
 
 impl UnsupportedCacheConfig {
@@ -100,6 +106,7 @@ impl UnsupportedCacheConfig {
             Self::RedisCredentials => "native Redis credentials require Python",
             Self::RedisConnection => "native Redis connection type is not implemented",
             Self::RedisOption => "native Redis configuration requires Python",
+            Self::DiskStore => "native disk cache requires the built-in diskcache store",
         }
     }
 }
@@ -142,11 +149,17 @@ impl NativeCacheConfig {
                 }))),
                 Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)),
             },
+            Some(CacheType::Disk) => match project_disk(&backend)? {
+                Ok(backend) => Ok(CacheConfigProjection::Native(Box::new(Self {
+                    policy,
+                    backend: CacheBackendConfig::Disk(backend),
+                }))),
+                Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)),
+            },
             Some(
                 CacheType::RedisSemantic
                 | CacheType::ValkeySemantic
                 | CacheType::S3
-                | CacheType::Disk
                 | CacheType::QdrantSemantic
                 | CacheType::AzureBlob
                 | CacheType::Gcs,
@@ -158,12 +171,12 @@ impl NativeCacheConfig {
     }
 
     pub(super) fn service_mismatch(&self, service: &NativeResponseCache) -> Option<&'static str> {
-        if service.default_ttl()
-            != Some(match &self.backend {
-                CacheBackendConfig::Memory(config) => config.default_ttl,
-                CacheBackendConfig::Redis(config) => config.default_ttl,
-            })
-        {
+        let default_ttl = match &self.backend {
+            CacheBackendConfig::Memory(config) => Some(config.default_ttl),
+            CacheBackendConfig::Redis(config) => Some(config.default_ttl),
+            CacheBackendConfig::Disk(_) => None,
+        };
+        if service.default_ttl() != default_ttl {
             return Some("facade and native backend default TTLs must match");
         }
         match &self.backend {
@@ -185,6 +198,17 @@ impl NativeCacheConfig {
             CacheBackendConfig::Redis(config) => (service.namespace()
                 != config.namespace.as_deref())
             .then_some("facade and native backend namespaces must match"),
+            CacheBackendConfig::Disk(_) if service.kind() != "disk" => {
+                Some("facade and native backend types must match")
+            }
+            CacheBackendConfig::Disk(config) => {
+                let Some(directory) = service.directory() else {
+                    return Some("facade and native backend types must match");
+                };
+                let native = std::fs::canonicalize(directory).ok();
+                let facade = std::fs::canonicalize(&config.directory).ok();
+                (native != facade).then_some("facade and native backend directories must match")
+            }
         }
     }
 }
@@ -201,6 +225,21 @@ fn project_memory(backend: &Bound<'_, PyAny>) -> PyResult<MemoryCacheConfig> {
     })
 }
 
+#[inline(never)]
+fn project_disk(
+    backend: &Bound<'_, PyAny>,
+) -> PyResult<Result<DiskCacheConfig, UnsupportedCacheConfig>> {
+    let store = backend.getattr("disk_cache")?;
+    if !instance_class_is(&store, "diskcache.core", "Cache")?
+        || !instance_class_is(&store.getattr("_disk")?, "diskcache.core", "Disk")?
+    {
+        return Ok(Err(UnsupportedCacheConfig::DiskStore));
+    }
+    Ok(Ok(DiskCacheConfig {
+        directory: PathBuf::from(store.getattr("directory")?.extract::<String>()?),
+    }))
+}
+
 #[inline(never)]
 fn project_redis(
     backend: &Bound<'_, PyAny>,
@@ -468,8 +507,8 @@ mod tests {
     use pyo3::{prelude::*, types::PyDict};
 
     use super::{
-        CacheBackendConfig, CacheConfigProjection, CertificateRequirement, NativeCacheConfig,
-        RedisProtocol,
+        CacheBackendConfig, CacheConfigProjection, CachePolicy, CertificateRequirement,
+        DiskCacheConfig, NativeCacheConfig, RedisProtocol,
     };
     use crate::cache::native::NativeResponseCache;
 
@@ -591,4 +630,84 @@ mod tests {
             assert_eq!(reason.message(), "native Redis credentials require Python");
         });
     }
+
+    #[test]
+    fn projects_builtin_disk_configuration_and_rejects_custom_stores() {
+        Python::initialize();
+        Python::attach(|py| {
+            let root =
+                std::env::temp_dir().join(format!("litellm-disk-config-{}", std::process::id()));
+            let directory = root.to_string_lossy();
+            let disk_facade = facade(
+                py,
+                &format!(
+                    "Cache = type('Cache', (), {{'__module__': 'diskcache.core'}})\n\
+                     Disk = type('Disk', (), {{'__module__': 'diskcache.core'}})\n\
+                     store = Cache()\n\
+                     store._disk = Disk()\n\
+                     store.directory = {directory:?}\n\
+                     backend = SimpleNamespace(disk_cache=store)\n\
+                     facade = SimpleNamespace(type='disk', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)"
+                ),
+            );
+            let CacheConfigProjection::Native(config) =
+                NativeCacheConfig::project(&disk_facade).unwrap()
+            else {
+                panic!("disk cache should be supported");
+            };
+            let CacheBackendConfig::Disk(disk) = config.backend else {
+                panic!("expected disk configuration");
+            };
+            assert_eq!(disk.directory, root);
+            let matching = NativeResponseCache::disk(&directory).unwrap();
+            assert_eq!(
+                (NativeCacheConfig {
+                    policy: config.policy,
+                    backend: CacheBackendConfig::Disk(disk),
+                })
+                .service_mismatch(&matching),
+                None
+            );
+            let other = NativeResponseCache::disk(&root.join("other").to_string_lossy()).unwrap();
+            let mismatch = NativeCacheConfig {
+                policy: CachePolicy {
+                    mode: "default-on".into(),
+                    ttl: None,
+                    namespace: None,
+                    supported_call_types: None,
+                    redis_flush_size: None,
+                    semantic_cache_scope: "key".into(),
+                },
+                backend: CacheBackendConfig::Disk(DiskCacheConfig {
+                    directory: root.clone(),
+                }),
+            };
+            assert_eq!(
+                mismatch.service_mismatch(&other),
+                Some("facade and native backend directories must match")
+            );
+
+            let custom = facade(
+                py,
+                &format!(
+                    "CustomCache = type('CustomCache', (), {{'__module__': 'mypkg'}})\n\
+                     CustomDisk = type('CustomDisk', (), {{'__module__': 'mypkg'}})\n\
+                     store = CustomCache()\n\
+                     store._disk = CustomDisk()\n\
+                     store.directory = {directory:?}\n\
+                     backend = SimpleNamespace(disk_cache=store)\n\
+                     facade = SimpleNamespace(type='disk', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)"
+                ),
+            );
+            let CacheConfigProjection::Unsupported(reason) =
+                NativeCacheConfig::project(&custom).unwrap()
+            else {
+                panic!("custom disk store must stay on Python");
+            };
+            assert_eq!(
+                reason.message(),
+                "native disk cache requires the built-in diskcache store"
+            );
+        });
+    }
 }
diff --git a/litellm-rust/crates/python-bridge/src/cache/facade.rs b/litellm-rust/crates/python-bridge/src/cache/facade.rs
index f2f86c14b37..a22a1754169 100644
--- a/litellm-rust/crates/python-bridge/src/cache/facade.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/facade.rs
@@ -32,10 +32,16 @@ struct RedisPoolGuard {
     max_connections: usize,
 }
 
+struct DiskStoreGuard {
+    reference: Py<PyAny>,
+    directory: String,
+}
+
 pub(super) struct FacadeGuard {
     outer: ObjectGuard,
     backend: ObjectGuard,
     redis_pool: Option<RedisPoolGuard>,
+    disk_store: Option<DiskStoreGuard>,
 }
 
 impl ObjectGuard {
@@ -176,6 +182,26 @@ impl RedisPoolGuard {
     }
 }
 
+impl DiskStoreGuard {
+    fn capture(backend: &Bound<'_, PyAny>) -> PyResult<Self> {
+        let store = backend.getattr("disk_cache")?;
+        Ok(Self {
+            reference: store.clone().unbind(),
+            directory: store.getattr("directory")?.extract()?,
+        })
+    }
+
+    fn matches(&self, py: Python<'_>, backend: &Bound<'_, PyAny>) -> PyResult<bool> {
+        let store = backend.getattr("disk_cache")?;
+        Ok(self.reference.bind(py).is(&store)
+            && self.directory == store.getattr("directory")?.extract::<String>()?)
+    }
+
+    fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
+        visit.call(&self.reference)
+    }
+}
+
 impl FacadeGuard {
     pub(super) fn capture(
         py: Python<'_>,
@@ -192,6 +218,7 @@ impl FacadeGuard {
         let (module, name, cache_kind) = match kind {
             "memory" => ("litellm.caching.in_memory_cache", "InMemoryCache", "local"),
             "redis" => ("litellm.caching.redis_cache", "RedisCache", "redis"),
+            "disk" => ("litellm.caching.disk_cache", "DiskCache", "disk"),
             _ => unreachable!(),
         };
         let backend = facade.getattr("cache")?;
@@ -240,6 +267,9 @@ impl FacadeGuard {
             redis_pool: (kind == "redis")
                 .then(|| RedisPoolGuard::capture(&backend))
                 .transpose()?,
+            disk_store: (kind == "disk")
+                .then(|| DiskStoreGuard::capture(&backend))
+                .transpose()?,
         })
     }
 
@@ -253,7 +283,10 @@ impl FacadeGuard {
         }
         match &self.redis_pool {
             Some(guard) => guard.matches(py, &backend),
-            None => Ok(true),
+            None => match &self.disk_store {
+                Some(guard) => guard.matches(py, &backend),
+                None => Ok(true),
+            },
         }
     }
 
@@ -263,6 +296,9 @@ impl FacadeGuard {
         if let Some(guard) = &self.redis_pool {
             guard.traverse(&visit)?;
         }
+        if let Some(guard) = &self.disk_store {
+            guard.traverse(&visit)?;
+        }
         Ok(())
     }
 }
diff --git a/litellm-rust/crates/python-bridge/src/cache/handle.rs b/litellm-rust/crates/python-bridge/src/cache/handle.rs
index 8251b3df06c..2cb2045577c 100644
--- a/litellm-rust/crates/python-bridge/src/cache/handle.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/handle.rs
@@ -51,6 +51,18 @@ impl CacheTestHandle {
         })
     }
 
+    #[staticmethod]
+    #[pyo3(signature = (directory))]
+    fn disk(py: Python<'_>, directory: String) -> PyResult<Self> {
+        let service =
+            release_gil(py, move || NativeResponseCache::disk(&directory)).map_err(cache_error)?;
+        Ok(Self {
+            service,
+            guard: None,
+            pid: std::process::id(),
+        })
+    }
+
     #[getter]
     fn backend(&self) -> &'static str {
         self.service.kind()
diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs
index a9475429e45..14993a6e8e3 100644
--- a/litellm-rust/crates/python-bridge/src/cache/native.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/native.rs
@@ -1,6 +1,7 @@
-use std::{sync::Arc, time::Duration};
+use std::{path::Path, sync::Arc, time::Duration};
 
 use litellm_cache::{CacheCodec, CacheConnectionResult, Error};
+use litellm_cache_disk::DiskCache;
 use litellm_cache_memory::InMemoryCache;
 use litellm_cache_redis::RedisCache;
 use litellm_cache_response::{
@@ -15,6 +16,7 @@ pub(super) enum NativeResponseCache {
         cache: Arc<ResponseCache<RedisCache<ResponseCacheCodec>>>,
         buffer: Option<Arc<WriteBuffer>>,
     },
+    Disk(Arc<ResponseCache<DiskCache<ResponseCacheCodec>>>),
 }
 
 impl NativeResponseCache {
@@ -43,6 +45,11 @@ impl NativeResponseCache {
             buffer: None,
         })
     }
+
+    pub fn disk(directory: &str) -> Result<Self, Error> {
+        let cache = DiskCache::open(directory, ResponseCacheCodec)?;
+        Ok(Self::Disk(Arc::new(ResponseCache::new(Arc::new(cache)))))
+    }
 }
 
 impl NativeResponseCache {
@@ -50,6 +57,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(_) => "memory",
             Self::Redis { .. } => "redis",
+            Self::Disk(_) => "disk",
         }
     }
 
@@ -57,6 +65,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.default_ttl(),
             Self::Redis { cache, .. } => cache.default_ttl(),
+            Self::Disk(cache) => cache.default_ttl(),
         }
     }
 
@@ -64,6 +73,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(_) => None,
             Self::Redis { cache, .. } => cache.backend().namespace(),
+            Self::Disk(_) => None,
         }
     }
 
@@ -71,6 +81,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => Some(cache.backend().max_size_in_memory()),
             Self::Redis { .. } => None,
+            Self::Disk(_) => None,
         }
     }
 
@@ -78,6 +89,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.backend().max_entry_bytes(),
             Self::Redis { .. } => None,
+            Self::Disk(_) => None,
         }
     }
 
@@ -87,10 +99,18 @@ impl NativeResponseCache {
                 cache,
                 buffer: flush_size.map(|flush_size| Arc::new(WriteBuffer::new(flush_size))),
             },
+            disk @ Self::Disk(_) => disk,
             memory => memory,
         }
     }
 
+    pub fn directory(&self) -> Option<&Path> {
+        match self {
+            Self::Disk(cache) => Some(cache.backend().directory()),
+            Self::Memory(_) | Self::Redis { .. } => None,
+        }
+    }
+
     pub fn lookup(
         &self,
         request: &ResponseCacheRequest,
@@ -99,6 +119,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.lookup(request, now),
             Self::Redis { cache, .. } => cache.lookup(request, now),
+            Self::Disk(cache) => cache.lookup(request, now),
         }
     }
 
@@ -111,6 +132,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.store(request, response, now),
             Self::Redis { cache, .. } => cache.store(request, response, now),
+            Self::Disk(cache) => cache.store(request, response, now),
         }
     }
 
@@ -122,6 +144,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.lookup_batch(requests, now),
             Self::Redis { cache, .. } => cache.lookup_batch(requests, now),
+            Self::Disk(cache) => cache.lookup_batch(requests, now),
         }
     }
 
@@ -133,6 +156,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.async_lookup(request, now).await,
             Self::Redis { cache, .. } => cache.async_lookup(request, now).await,
+            Self::Disk(cache) => cache.async_lookup(request, now).await,
         }
     }
 
@@ -152,6 +176,7 @@ impl NativeResponseCache {
                 cache,
                 buffer: Some(buffer),
             } => buffer.async_store(cache, request, response, now).await,
+            Self::Disk(cache) => cache.async_store(request, response, now).await,
         }
     }
 
@@ -163,6 +188,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.async_lookup_batch(requests, now).await,
             Self::Redis { cache, .. } => cache.async_lookup_batch(requests, now).await,
+            Self::Disk(cache) => cache.async_lookup_batch(requests, now).await,
         }
     }
 
@@ -174,6 +200,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.async_store_batch(entries, now).await,
             Self::Redis { cache, .. } => cache.async_store_batch(entries, now).await,
+            Self::Disk(cache) => cache.async_store_batch(entries, now).await,
         }
     }
 
@@ -186,6 +213,7 @@ impl NativeResponseCache {
                 }
                 cache.async_flush().await
             }
+            Self::Disk(cache) => cache.async_flush().await,
         }
     }
 
@@ -193,6 +221,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.test_connection().await,
             Self::Redis { cache, .. } => cache.test_connection().await,
+            Self::Disk(cache) => cache.test_connection().await,
         }
     }
 }
diff --git a/tests/test_litellm_rust/test_cache.py b/tests/test_litellm_rust/test_cache.py
index c35cb1a20fb..13f5f686464 100644
--- a/tests/test_litellm_rust/test_cache.py
+++ b/tests/test_litellm_rust/test_cache.py
@@ -6,16 +6,19 @@ import threading
 import time
 import weakref
 from collections.abc import Generator
+from pathlib import Path
 from types import SimpleNamespace
 from typing import Final, Protocol, cast
 from urllib.parse import urlparse
 
+import diskcache
 import fakeredis
 import pytest
 import redis
 
 import litellm
 from litellm.caching.caching import Cache, disable_cache, enable_cache, update_cache
+from litellm.caching.disk_cache import DiskCache
 from litellm.caching.in_memory_cache import InMemoryCache
 from litellm.rust_bridge import _native
 from litellm.types.caching import LiteLLMCacheType
@@ -393,3 +396,111 @@ async def test_redis_facade_buffers_native_async_writes(redis_url: str) -> None:
     assert client.get("second") is not None
     await facade.cache.disconnect()
     client.close()
+
+
+async def test_disk_reads_python_entries_and_python_reads_native_entries(tmp_path: Path) -> None:
+    disk_cache: Final = DiskCache(disk_cache_dir=str(tmp_path))
+    response: Final = {"choices": [{"text": "cached"}], "usage": {"total_tokens": 3}}
+    disk_cache.disk_cache.set(
+        "sync",
+        {"timestamp": time.time(), "response": json.dumps(response)},
+    )
+    disk_cache.disk_cache.set("async", json.dumps({"timestamp": time.time(), "response": response}))
+    disk_cache.disk_cache.set("raw", json.dumps(response))
+    disk_cache.disk_cache.set("invalid", "not a cache entry")
+    disk_cache.disk_cache.set(
+        "large",
+        {"timestamp": time.time(), "response": {"text": "x" * 70_000}},
+    )
+    binding: Final = _native._CacheTestResolver(
+        SimpleNamespace(cache=_native._CacheTestHandle.disk(str(tmp_path)))
+    ).resolve()
+
+    assert binding.lookup(request("sync")) == response
+    assert await binding.async_lookup(request("async")) == response
+    assert binding.lookup(request("raw")) == response
+    assert await binding.async_lookup(request("invalid")) is None
+    assert binding.lookup(request("large")) == {"text": "x" * 70_000}
+
+    await binding.async_store({**request("native"), "ttl_seconds": 12.0}, response)
+    stored_response: Final = disk_cache.get_cache("native")
+    assert isinstance(stored_response, dict)
+    assert stored_response["response"] == response
+    stored, expire_time = disk_cache.disk_cache.get("native", expire_time=True)
+    assert stored is not None
+    assert time.time() < expire_time <= time.time() + 12.0
+    await binding.async_store(request("no-ttl"), response)
+    _, no_expiry = disk_cache.disk_cache.get("no-ttl", expire_time=True)
+    assert no_expiry is None
+
+
+async def test_disk_entries_survive_a_fresh_handle_and_expire_on_time(tmp_path: Path) -> None:
+    first: Final = _native._CacheTestResolver(
+        SimpleNamespace(cache=_native._CacheTestHandle.disk(str(tmp_path)))
+    ).resolve()
+    await first.async_store(request("persistent"), {"value": "persistent"})
+    await first.async_store({**request("expiring"), "ttl_seconds": 0.3}, {"value": "expiring"})
+    fresh: Final = _native._CacheTestResolver(
+        SimpleNamespace(cache=_native._CacheTestHandle.disk(str(tmp_path)))
+    ).resolve()
+    assert fresh.lookup(request("persistent")) == {"value": "persistent"}
+    assert fresh.lookup(request("expiring")) == {"value": "expiring"}
+    await asyncio.sleep(0.4)
+    assert fresh.lookup(request("expiring")) is None
+    assert fresh.lookup(request("persistent")) == {"value": "persistent"}
+
+
+def test_disk_facade_registers_and_store_changes_fall_back(tmp_path: Path) -> None:
+    facade: Final = Cache(type=LiteLLMCacheType.DISK, disk_cache_dir=str(tmp_path))
+    with pytest.raises(TypeError, match="directories must match"):
+        _native._CacheTestHandle.disk(str(tmp_path / "other"))._bind_facade(facade)
+    handle: Final = _native._CacheTestHandle.disk(str(tmp_path))
+    handle._bind_facade(facade)
+    resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade))
+    binding: Final = resolver.resolve()
+    assert binding.kind == "native"
+    binding.store(request("native"), {"value": "native"})
+    assert facade.get_cache(cache_key="native") == {"value": "native"}
+
+    with rebound(facade.cache, "disk_cache", diskcache.Cache(str(tmp_path))):
+        assert resolver.resolve().kind == "python_callback"
+    assert resolver.resolve().kind == "native"
+
+    class CustomDiskCache(DiskCache):
+        pass
+
+    with rebound(facade, "cache", CustomDiskCache(disk_cache_dir=str(tmp_path))):
+        assert resolver.resolve().kind == "python_callback"
+
+    class CustomStore(diskcache.Cache):
+        pass
+
+    custom_facade: Final = Cache(type=LiteLLMCacheType.DISK, disk_cache_dir=str(tmp_path))
+    custom_facade.cache.disk_cache = CustomStore(str(tmp_path))
+    with pytest.raises(TypeError, match="built-in diskcache store"):
+        _native._CacheTestHandle.disk(str(tmp_path))._bind_facade(custom_facade)
+
+
+async def test_disk_native_batch_lookup_and_store_report_partial_hits(tmp_path: Path) -> None:
+    binding: Final = _native._CacheTestResolver(
+        SimpleNamespace(cache=_native._CacheTestHandle.disk(str(tmp_path)))
+    ).resolve()
+    requests: Final = [request("hit"), request("miss"), request("disabled")]
+    requests[2]["controls"] = {
+        "supported_call_type": True,
+        "configured": True,
+        "native_backend": True,
+        "default_on": True,
+        "caching": False,
+        "no_cache": False,
+        "no_store": False,
+        "use_cache": False,
+    }
+    await binding.async_store_batch(requests, [{"value": 1}, {"value": 2}, {"value": 3}])
+
+    partial: Final = await binding.async_lookup_batch(requests)
+
+    assert partial == {
+        "values": [{"value": 1}, {"value": 2}, None],
+        "missing_indices": [2],
+    }

From a6d1497932e257a809fdd4647d85a70039376816 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:38:20 +0000
Subject: [PATCH 078/160] test(rust): add GCS native cache parity fixtures

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.lock                      |   1 -
 litellm-rust/crates/cache-gcs/Cargo.toml     |   1 -
 litellm-rust/crates/cache-gcs/src/cache.rs   |  37 +--
 litellm-rust/crates/cache-gcs/tests/cache.rs |  34 +++
 tests/test_litellm_rust/support/fake_gcs.py  | 152 +++++++++++++
 tests/test_litellm_rust/test_cache.py        | 228 +++++++++++++++++++
 6 files changed, 420 insertions(+), 33 deletions(-)
 create mode 100644 tests/test_litellm_rust/support/fake_gcs.py

diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index 99a6133b4f1..ae28553801d 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -2476,7 +2476,6 @@ dependencies = [
  "reqwest 0.12.28",
  "serde_json",
  "tokio",
- "url",
  "wiremock",
 ]
 
diff --git a/litellm-rust/crates/cache-gcs/Cargo.toml b/litellm-rust/crates/cache-gcs/Cargo.toml
index 3890e60843c..4ec60bcfa3b 100644
--- a/litellm-rust/crates/cache-gcs/Cargo.toml
+++ b/litellm-rust/crates/cache-gcs/Cargo.toml
@@ -13,7 +13,6 @@ litellm-cache.workspace = true
 percent-encoding.workspace = true
 reqwest.workspace = true
 tokio.workspace = true
-url.workspace = true
 
 [dev-dependencies]
 serde_json.workspace = true
diff --git a/litellm-rust/crates/cache-gcs/src/cache.rs b/litellm-rust/crates/cache-gcs/src/cache.rs
index 1097f3252b1..65282ac99d5 100644
--- a/litellm-rust/crates/cache-gcs/src/cache.rs
+++ b/litellm-rust/crates/cache-gcs/src/cache.rs
@@ -5,43 +5,18 @@ use litellm_cache::{
     BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, Error, ExactCacheContext,
     FlushCache,
 };
-use percent_encoding::{AsciiSet, CONTROLS, percent_encode};
+use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode};
 use reqwest::Client;
 
 use crate::{GcpTokenSource, TokenSource};
 
 pub const DEFAULT_ENDPOINT: &str = "https://storage.googleapis.com";
 
-const OBJECT_NAME_ENCODE_SET: &AsciiSet = &CONTROLS
-    .add(b' ')
-    .add(b'!')
-    .add(b'"')
-    .add(b'#')
-    .add(b'$')
-    .add(b'%')
-    .add(b'&')
-    .add(b'\'')
-    .add(b'(')
-    .add(b')')
-    .add(b'*')
-    .add(b'+')
-    .add(b',')
-    .add(b'/')
-    .add(b':')
-    .add(b';')
-    .add(b'<')
-    .add(b'=')
-    .add(b'>')
-    .add(b'?')
-    .add(b'@')
-    .add(b'[')
-    .add(b'\\')
-    .add(b']')
-    .add(b'^')
-    .add(b'`')
-    .add(b'{')
-    .add(b'|')
-    .add(b'}');
+const OBJECT_NAME_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC
+    .remove(b'-')
+    .remove(b'_')
+    .remove(b'.')
+    .remove(b'~');
 
 pub fn key_prefix(gcs_path: Option<&str>) -> String {
     match gcs_path {
diff --git a/litellm-rust/crates/cache-gcs/tests/cache.rs b/litellm-rust/crates/cache-gcs/tests/cache.rs
index 79e4ed100f8..45eecf01cec 100644
--- a/litellm-rust/crates/cache-gcs/tests/cache.rs
+++ b/litellm-rust/crates/cache-gcs/tests/cache.rs
@@ -117,6 +117,40 @@ fn key_prefix_normalizes_paths() {
     assert_eq!(key_prefix(Some("")), "");
 }
 
+#[tokio::test]
+async fn object_names_use_python_quote_encoding() {
+    let server = MockServer::start().await;
+    Mock::given(method("POST"))
+        .and(path("/upload/storage/v1/b/bucket/o"))
+        .and(query_param("uploadType", "media"))
+        .respond_with(ResponseTemplate::new(200))
+        .expect(2)
+        .mount(&server)
+        .await;
+    let cache = cache(&server, Some("p/"));
+    cache
+        .set_cache(
+            "a~b-c_d.e/f g%h",
+            json!({"value": "punctuation"}),
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    cache
+        .set_cache(
+            "ключ",
+            json!({"value": "utf8"}),
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    let requests = server.received_requests().await.unwrap();
+    let queries: Vec<_> = requests
+        .iter()
+        .filter_map(|request| request.url.query())
+        .collect();
+    assert!(queries.contains(&"uploadType=media&name=p%2Fa~b-c_d.e%2Ff%20g%25h"));
+    assert!(queries.contains(&"uploadType=media&name=p%2F%D0%BA%D0%BB%D1%8E%D1%87"));
+}
+
 #[tokio::test]
 async fn ignores_ttl_and_writes_pipeline_concurrently() {
     let server = MockServer::start().await;
diff --git a/tests/test_litellm_rust/support/fake_gcs.py b/tests/test_litellm_rust/support/fake_gcs.py
new file mode 100644
index 00000000000..67eb61798b9
--- /dev/null
+++ b/tests/test_litellm_rust/support/fake_gcs.py
@@ -0,0 +1,152 @@
+from __future__ import annotations
+
+import json
+import threading
+from collections.abc import Mapping
+from dataclasses import dataclass
+from functools import partial
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from socket import socket
+from types import MappingProxyType
+from typing import Final, cast
+from urllib.parse import unquote, urlsplit
+
+
+@dataclass(frozen=True, slots=True)
+class RecordedRequest:
+    method: str
+    path: str
+    query: str
+    headers: Mapping[str, str]
+    body: bytes
+
+
+class _FakeGcsHandler(BaseHTTPRequestHandler):
+    def __init__(
+        self,
+        request: socket | tuple[bytes, socket],
+        client_address: tuple[str, int],
+        server: ThreadingHTTPServer,
+        *,
+        fake: FakeGcs,
+    ) -> None:
+        self._fake: Final = fake
+        super().__init__(request, client_address, server)
+
+    def _handle(self) -> None:
+        parsed: Final = urlsplit(self.path)
+        content_length: Final = int(self.headers.get("Content-Length", "0"))
+        body: Final = self.rfile.read(content_length) if content_length else b""
+        headers: Final = MappingProxyType(
+            {name.title(): value for name, value in self.headers.items()}
+        )
+        self._fake.record(
+            RecordedRequest(
+                method=self.command,
+                path=parsed.path,
+                query=parsed.query,
+                headers=headers,
+                body=body,
+            )
+        )
+        if self.headers.get("Authorization") != f"Bearer {self._fake.token}":
+            self._send_json(401, {"error": "unauthorized"})
+            return
+
+        upload_prefix: Final = "/upload/storage/v1/b/"
+        download_prefix: Final = "/storage/v1/b/"
+        if parsed.path.startswith(upload_prefix) and parsed.path.endswith("/o"):
+            self._upload(parsed.path[len(upload_prefix) : -2], parsed.query, body)
+            return
+        if parsed.path.startswith(download_prefix):
+            self._download(parsed.path[len(download_prefix) :], parsed.query)
+            return
+        self._send_json(404, {"error": "not found"})
+
+    def _upload(self, path: str, query: str, body: bytes) -> None:
+        values: Final = {
+            unquote(pair.partition("=")[0]): unquote(pair.partition("=")[2])
+            for pair in query.split("&")
+            if pair
+        }
+        if not path or values.get("uploadType") != "media" or "name" not in values:
+            self._send_json(404, {"error": "not found"})
+            return
+        self._fake.put_object(path, values["name"], body)
+        self._send_json(200, {"name": values["name"], "bucket": path})
+
+    def _download(self, path: str, query: str) -> None:
+        bucket, separator, encoded_name = path.partition("/o/")
+        if not separator or query != "alt=media":
+            self._send_json(404, {"error": "not found"})
+            return
+        name: Final = unquote(encoded_name)
+        if name.endswith("/server-error") or name == "server-error":
+            self._send_json(500, {"error": "server error"})
+            return
+        body: Final = self._fake.get_object(bucket, name)
+        if body is None:
+            self._send_json(404, {"error": "not found"})
+            return
+        self._send(200, body, "application/octet-stream")
+
+    def _send_json(self, status: int, value: object) -> None:
+        payload: Final = json.dumps(value).encode()
+        self._send(status, payload, "application/json")
+
+    def _send(self, status: int, body: bytes, content_type: str) -> None:
+        self.send_response(status)
+        self.send_header("Content-Type", content_type)
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def log_message(self, format: str, *args: object) -> None:
+        pass
+
+    do_GET = _handle
+    do_POST = _handle
+
+
+class FakeGcs:
+    def __init__(self) -> None:
+        self._objects: dict[tuple[str, str], bytes] = {}  # mutable-ok: fake object store
+        self._requests: list[RecordedRequest] = []  # mutable-ok: recorded request history
+        self._server = ThreadingHTTPServer(
+            ("127.0.0.1", 0),
+            partial(_FakeGcsHandler, fake=self),
+        )
+        self._worker = threading.Thread(target=self._server.serve_forever, daemon=True)
+        self._worker.start()
+        self.token: Final = "test-token"
+
+    @property
+    def url(self) -> str:
+        address: Final = cast(tuple[str, int], self._server.server_address)
+        host, port = address
+        return f"http://{host}:{port}"
+
+    @property
+    def objects(self) -> Mapping[tuple[str, str], bytes]:
+        return MappingProxyType(self._objects)
+
+    @property
+    def requests(self) -> tuple[RecordedRequest, ...]:
+        return tuple(self._requests)
+
+    def put(self, bucket: str, name: str, body: bytes) -> None:
+        self.put_object(bucket, name, body)
+
+    def close(self) -> None:
+        self._server.shutdown()
+        self._server.server_close()
+        self._worker.join(timeout=5)
+
+    def record(self, request: RecordedRequest) -> None:
+        self._requests.append(request)
+
+    def put_object(self, bucket: str, name: str, body: bytes) -> None:
+        self._objects[(bucket, name)] = body
+
+    def get_object(self, bucket: str, name: str) -> bytes | None:
+        return self._objects.get((bucket, name))
diff --git a/tests/test_litellm_rust/test_cache.py b/tests/test_litellm_rust/test_cache.py
index c35cb1a20fb..cc409ebe49c 100644
--- a/tests/test_litellm_rust/test_cache.py
+++ b/tests/test_litellm_rust/test_cache.py
@@ -16,9 +16,11 @@ import redis
 
 import litellm
 from litellm.caching.caching import Cache, disable_cache, enable_cache, update_cache
+from litellm.caching.gcs_cache import GCSCache
 from litellm.caching.in_memory_cache import InMemoryCache
 from litellm.rust_bridge import _native
 from litellm.types.caching import LiteLLMCacheType
+from tests.test_litellm_rust.support.fake_gcs import FakeGcs
 from tests.test_litellm_rust.support.isolation import rebound
 
 pytestmark: Final = pytest.mark.requires_rust_extension
@@ -26,6 +28,7 @@ pytestmark: Final = pytest.mark.requires_rust_extension
 
 class CacheLookup(Protocol):
     def get_cache(self, **kwargs: object) -> object: ...
+    def flush_cache(self) -> object: ...
 
 
 def request(key: str = "key") -> dict[str, object]:
@@ -45,6 +48,15 @@ def redis_url() -> Generator[str]:
         worker.join(timeout=5)
 
 
+@pytest.fixture
+def fake_gcs() -> Generator[FakeGcs]:
+    server: Final = FakeGcs()
+    try:
+        yield server
+    finally:
+        server.close()
+
+
 def test_existing_constructor_and_global_are_unchanged() -> None:
     facade: Final = Cache(type=LiteLLMCacheType.LOCAL)
     assert type(facade.cache) is InMemoryCache
@@ -393,3 +405,219 @@ async def test_redis_facade_buffers_native_async_writes(redis_url: str) -> None:
     assert client.get("second") is not None
     await facade.cache.disconnect()
     client.close()
+
+
+async def test_gcs_reads_python_entries_and_writes_python_compatible_objects(
+    fake_gcs: FakeGcs, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.delenv("GCS_PATH_SERVICE_ACCOUNT", raising=False)
+    monkeypatch.delenv("GCS_BUCKET_NAME", raising=False)
+    response: Final = {"choices": [{"text": "cached"}], "usage": {"total_tokens": 3}, "flag": True, "empty": None}
+    fake_gcs.put(
+        "bucket",
+        "cache/sync",
+        json.dumps({"timestamp": time.time(), "response": json.dumps(response)}).encode(),
+    )
+    fake_gcs.put("bucket", "cache/async", json.dumps({"timestamp": time.time(), "response": response}).encode())
+    fake_gcs.put("bucket", "cache/raw", json.dumps(response).encode())
+    fake_gcs.put("bucket", "cache/invalid", b"not a cache entry")
+    binding: Final = _native._CacheTestResolver(
+        SimpleNamespace(
+            cache=_native._CacheTestHandle.gcs(
+                "bucket",
+                gcs_path="cache",
+                endpoint=fake_gcs.url,
+                token=fake_gcs.token,
+            )
+        )
+    ).resolve()
+
+    assert binding.lookup(request("sync")) == response
+    assert await binding.async_lookup(request("async")) == response
+    assert binding.lookup(request("raw")) == response
+    assert await binding.async_lookup(request("invalid")) is None
+    assert binding.lookup(request("missing")) is None
+
+    await binding.async_store({**request("native"), "ttl_seconds": 12.0}, response)
+    stored: Final = fake_gcs.objects[("bucket", "cache/native")]
+    stored_value: Final = cast(dict[str, object], json.loads(stored))
+    assert stored_value["response"] == response
+    assert isinstance(stored_value["timestamp"], float)
+    upload: Final = next(item for item in fake_gcs.requests if item.method == "POST")
+    assert upload.path == "/upload/storage/v1/b/bucket/o"
+    assert upload.query == "uploadType=media&name=cache%2Fnative"
+    assert upload.headers["Authorization"] == f"Bearer {fake_gcs.token}"
+    assert upload.headers["Content-Type"] == "application/json"
+    upload_text: Final = f"{upload.path}?{upload.query}{upload.headers}"
+    assert "ttl" not in upload_text.lower()
+    assert "expiry" not in upload_text.lower()
+    download: Final = next(item for item in fake_gcs.requests if item.path.endswith("/cache%2Fsync"))
+    assert download.path == "/storage/v1/b/bucket/o/cache%2Fsync"
+    assert download.query == "alt=media"
+
+    binding.store(request("sync2"), response)
+    assert binding.lookup(request("sync2")) == response
+    assert GCSCache(bucket_name="bucket", gcs_path="cache").key_prefix == "cache/"
+    assert GCSCache(bucket_name="bucket", gcs_path="cache/").key_prefix == "cache/"
+    assert GCSCache(bucket_name="bucket").key_prefix == ""
+
+
+async def test_gcs_batch_lookup_preserves_order_and_treats_malformed_entries_as_misses(fake_gcs: FakeGcs) -> None:
+    fake_gcs.put("bucket", "cache/hit", json.dumps({"timestamp": time.time(), "response": {"value": 1}}).encode())
+    fake_gcs.put("bucket", "cache/invalid", b"not a cache entry")
+    binding: Final = _native._CacheTestResolver(
+        SimpleNamespace(
+            cache=_native._CacheTestHandle.gcs(
+                "bucket",
+                gcs_path="cache",
+                endpoint=fake_gcs.url,
+                token=fake_gcs.token,
+            )
+        )
+    ).resolve()
+    requests: Final = [request("hit"), request("missing"), request("invalid")]
+    expected: Final = {"values": [{"value": 1}, None, None], "missing_indices": [1, 2]}
+
+    assert await binding.async_lookup_batch(requests) == expected
+    assert binding.lookup_batch(requests) == expected
+    await binding.async_store_batch([request("first"), request("second")], [{"value": 1}, {"value": 2}])
+    assert ("bucket", "cache/first") in fake_gcs.objects
+    assert ("bucket", "cache/second") in fake_gcs.objects
+
+
+async def test_gcs_facade_binds_only_exact_matching_configuration(
+    fake_gcs: FakeGcs, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.delenv("GCS_PATH_SERVICE_ACCOUNT", raising=False)
+    monkeypatch.delenv("GCS_BUCKET_NAME", raising=False)
+    monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", "/nonexistent")
+    facade: Final = Cache(type=LiteLLMCacheType.GCS, gcs_bucket_name="bucket", gcs_path="cache/")
+    assert type(facade.cache) is GCSCache
+
+    mismatched_bucket: Final = _native._CacheTestHandle.gcs(
+        "other",
+        gcs_path="cache",
+        endpoint=fake_gcs.url,
+        token=fake_gcs.token,
+    )
+    with pytest.raises(TypeError, match="buckets must match"):
+        mismatched_bucket._bind_facade(facade)
+    mismatched_prefix: Final = _native._CacheTestHandle.gcs(
+        "bucket",
+        gcs_path="x",
+        endpoint=fake_gcs.url,
+        token=fake_gcs.token,
+    )
+    with pytest.raises(TypeError, match="key prefixes must match"):
+        mismatched_prefix._bind_facade(facade)
+    mismatched_credentials: Final = _native._CacheTestHandle.gcs(
+        "bucket",
+        gcs_path="cache",
+        path_service_account="sa.json",
+        endpoint=fake_gcs.url,
+        token=fake_gcs.token,
+    )
+    with pytest.raises(TypeError, match="credentials must match"):
+        mismatched_credentials._bind_facade(facade)
+    with pytest.raises(TypeError, match="types must match"):
+        _native._CacheTestHandle.memory()._bind_facade(facade)
+
+    matching: Final = _native._CacheTestHandle.gcs(
+        "bucket",
+        gcs_path="cache",
+        endpoint=fake_gcs.url,
+        token=fake_gcs.token,
+    )
+    matching._bind_facade(facade)
+    resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade))
+    binding: Final = resolver.resolve()
+    assert binding.kind == "native"
+    await binding.async_store(request("native"), {"value": "native"})
+    assert await binding.async_lookup(request("native")) == {"value": "native"}
+    assert cast(CacheLookup, facade).get_cache(cache_key="native") is None
+
+    with rebound(facade.cache, "bucket_name", "other"):
+        assert resolver.resolve().kind == "python_callback"
+    with rebound(facade.cache, "key_prefix", "x/"):
+        assert resolver.resolve().kind == "python_callback"
+    with rebound(facade.cache, "path_service_account", "sa.json"):
+        assert resolver.resolve().kind == "python_callback"
+    def no_get_cache(*args: object, **kwargs: object) -> None:
+        return None
+
+    with rebound(facade.cache, "get_cache", no_get_cache):
+        assert resolver.resolve().kind == "python_callback"
+    with rebound(facade, "ttl", 12):
+        assert resolver.resolve().kind == "python_callback"
+
+    class CustomGcs(GCSCache):
+        pass
+
+    with rebound(facade, "cache", CustomGcs(bucket_name="bucket", gcs_path="cache/")):
+        assert resolver.resolve().kind == "python_callback"
+    custom_facade: Final = Cache(type=LiteLLMCacheType.GCS, gcs_bucket_name="bucket", gcs_path="cache/")
+    with rebound(custom_facade, "cache", CustomGcs(bucket_name="bucket", gcs_path="cache/")):
+        with pytest.raises(TypeError, match="types must match"):
+            matching._bind_facade(custom_facade)
+
+    missing_bucket: Final = Cache(type=LiteLLMCacheType.GCS)
+    with pytest.raises(TypeError, match="requires a configured bucket name"):
+        matching._bind_facade(missing_bucket)
+
+
+async def test_gcs_flush_is_a_no_op_and_ping_is_not_implemented(
+    fake_gcs: FakeGcs, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.delenv("GCS_PATH_SERVICE_ACCOUNT", raising=False)
+    monkeypatch.delenv("GCS_BUCKET_NAME", raising=False)
+    binding: Final = _native._CacheTestResolver(
+        SimpleNamespace(
+            cache=_native._CacheTestHandle.gcs(
+                "bucket",
+                gcs_path="cache",
+                endpoint=fake_gcs.url,
+                token=fake_gcs.token,
+            )
+        )
+    ).resolve()
+    await binding.async_store(request("key"), {"value": "stored"})
+    await binding.async_flush()
+    assert ("bucket", "cache/key") in fake_gcs.objects
+    assert await binding.async_lookup(request("key")) == {"value": "stored"}
+    with pytest.raises(NotImplementedError):
+        await binding.ping()
+
+    facade: Final = Cache(type=LiteLLMCacheType.GCS, gcs_bucket_name="bucket", gcs_path="cache/")
+    with pytest.raises(AttributeError):
+        await facade.ping()
+    assert cast(CacheLookup, facade.cache).flush_cache() is None
+
+
+async def test_gcs_unauthorized_and_server_errors_surface_as_runtime_errors(fake_gcs: FakeGcs) -> None:
+    wrong_token: Final = _native._CacheTestResolver(
+        SimpleNamespace(
+            cache=_native._CacheTestHandle.gcs(
+                "bucket",
+                gcs_path="cache",
+                endpoint=fake_gcs.url,
+                token="wrong-token",
+            )
+        )
+    ).resolve()
+    with pytest.raises(RuntimeError):
+        wrong_token.lookup(request("missing"))
+    assert not fake_gcs.objects
+
+    binding: Final = _native._CacheTestResolver(
+        SimpleNamespace(
+            cache=_native._CacheTestHandle.gcs(
+                "bucket",
+                gcs_path="cache",
+                endpoint=fake_gcs.url,
+                token=fake_gcs.token,
+            )
+        )
+    ).resolve()
+    with pytest.raises(RuntimeError):
+        binding.lookup(request("server-error"))
+    assert binding.lookup(request("missing")) is None

From 10b977fe29caccc1a2730568d33c21aa751cddab Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:38:33 +0000
Subject: [PATCH 079/160] feat(python-bridge): serve redis-semantic caches
 natively

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.lock                       |   1 +
 litellm-rust/crates/python-bridge/Cargo.toml  |   1 +
 .../crates/python-bridge/src/cache/config.rs  |  67 +++++++-
 .../python-bridge/src/cache/embedder.rs       |  85 +++++++++
 .../crates/python-bridge/src/cache/facade.rs  |  21 +++
 .../crates/python-bridge/src/cache/handle.rs  |  43 ++++-
 .../crates/python-bridge/src/cache/mod.rs     |   4 +-
 .../crates/python-bridge/src/cache/native.rs  | 161 ++++++++++++++----
 .../crates/python-bridge/src/cache/request.rs |  66 +++++--
 9 files changed, 399 insertions(+), 50 deletions(-)
 create mode 100644 litellm-rust/crates/python-bridge/src/cache/embedder.rs

diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index e911d0d9c45..0030018df34 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -2683,6 +2683,7 @@ dependencies = [
  "litellm-cache",
  "litellm-cache-memory",
  "litellm-cache-redis",
+ "litellm-cache-redis-semantic",
  "litellm-cache-response",
  "litellm-callbacks-legacy-python",
  "litellm-core",
diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml
index 1eb2ec28036..635c0942ceb 100644
--- a/litellm-rust/crates/python-bridge/Cargo.toml
+++ b/litellm-rust/crates/python-bridge/Cargo.toml
@@ -23,6 +23,7 @@ bytes.workspace = true
 litellm-cache.workspace = true
 litellm-cache-memory.workspace = true
 litellm-cache-redis.workspace = true
+litellm-cache-redis-semantic.workspace = true
 litellm-cache-response.workspace = true
 serde.workspace = true
 litellm-auth.workspace = true
diff --git a/litellm-rust/crates/python-bridge/src/cache/config.rs b/litellm-rust/crates/python-bridge/src/cache/config.rs
index 0e7d6aee11d..85e400d89a3 100644
--- a/litellm-rust/crates/python-bridge/src/cache/config.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/config.rs
@@ -73,9 +73,23 @@ pub(super) struct RedisCacheConfig {
     pub(super) connection: RedisConnectionConfig,
 }
 
+#[allow(
+    dead_code,
+    reason = "embedding settings are projected so drift falls back to Python"
+)]
+pub(super) struct RedisSemanticCacheConfig {
+    pub(super) redis_url: String,
+    pub(super) index_name: String,
+    pub(super) similarity_threshold: f64,
+    pub(super) embedding_model: String,
+    pub(super) embedding_max_input_tokens: Option<u64>,
+    pub(super) embedding_timeout: Option<f64>,
+}
+
 pub(super) enum CacheBackendConfig {
     Memory(MemoryCacheConfig),
     Redis(Box<RedisCacheConfig>),
+    RedisSemantic(Box<RedisSemanticCacheConfig>),
 }
 
 #[allow(dead_code, reason = "consumed by the cache activation follow-up")]
@@ -142,9 +156,14 @@ impl NativeCacheConfig {
                 }))),
                 Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)),
             },
+            Some(CacheType::RedisSemantic) => project_redis_semantic(&backend).map(|backend| {
+                CacheConfigProjection::Native(Box::new(Self {
+                    policy,
+                    backend: CacheBackendConfig::RedisSemantic(Box::new(backend)),
+                }))
+            }),
             Some(
-                CacheType::RedisSemantic
-                | CacheType::ValkeySemantic
+                CacheType::ValkeySemantic
                 | CacheType::S3
                 | CacheType::Disk
                 | CacheType::QdrantSemantic
@@ -159,10 +178,11 @@ impl NativeCacheConfig {
 
     pub(super) fn service_mismatch(&self, service: &NativeResponseCache) -> Option<&'static str> {
         if service.default_ttl()
-            != Some(match &self.backend {
-                CacheBackendConfig::Memory(config) => config.default_ttl,
-                CacheBackendConfig::Redis(config) => config.default_ttl,
-            })
+            != match &self.backend {
+                CacheBackendConfig::Memory(config) => Some(config.default_ttl),
+                CacheBackendConfig::Redis(config) => Some(config.default_ttl),
+                CacheBackendConfig::RedisSemantic(_) => None,
+            }
         {
             return Some("facade and native backend default TTLs must match");
         }
@@ -185,10 +205,45 @@ impl NativeCacheConfig {
             CacheBackendConfig::Redis(config) => (service.namespace()
                 != config.namespace.as_deref())
             .then_some("facade and native backend namespaces must match"),
+            CacheBackendConfig::RedisSemantic(_) if service.kind() != "redis_semantic" => {
+                Some("facade and native backend types must match")
+            }
+            CacheBackendConfig::RedisSemantic(config)
+                if service.index_name() != Some(config.index_name.as_str()) =>
+            {
+                Some("facade and native backend index names must match")
+            }
+            CacheBackendConfig::RedisSemantic(config)
+                if service.similarity_threshold() != Some(config.similarity_threshold as f32) =>
+            {
+                Some("facade and native backend similarity thresholds must match")
+            }
+            CacheBackendConfig::RedisSemantic(_) => None,
         }
     }
 }
 
+#[inline(never)]
+pub(super) fn project_redis_semantic(
+    backend: &Bound<'_, PyAny>,
+) -> PyResult<RedisSemanticCacheConfig> {
+    Ok(RedisSemanticCacheConfig {
+        redis_url: backend.getattr("_redis_url")?.extract::<String>()?,
+        index_name: backend
+            .getattr("_index_name")?
+            .extract::<Option<String>>()?
+            .unwrap_or_else(|| "litellm_semantic_cache_index".into()),
+        similarity_threshold: backend.getattr("similarity_threshold")?.extract::<f64>()?,
+        embedding_model: backend.getattr("embedding_model")?.extract::<String>()?,
+        embedding_max_input_tokens: backend
+            .getattr("embedding_max_input_tokens")?
+            .extract::<Option<u64>>()?,
+        embedding_timeout: backend
+            .getattr("embedding_timeout")?
+            .extract::<Option<f64>>()?,
+    })
+}
+
 #[inline(never)]
 fn project_memory(backend: &Bound<'_, PyAny>) -> PyResult<MemoryCacheConfig> {
     let max_size_kib = backend.getattr("max_size_per_item")?.extract::<usize>()?;
diff --git a/litellm-rust/crates/python-bridge/src/cache/embedder.rs b/litellm-rust/crates/python-bridge/src/cache/embedder.rs
new file mode 100644
index 00000000000..63e078cd815
--- /dev/null
+++ b/litellm-rust/crates/python-bridge/src/cache/embedder.rs
@@ -0,0 +1,85 @@
+use std::future::Future;
+
+use litellm_cache::Error;
+use litellm_cache_redis_semantic::Embedder;
+use litellm_host_python::to_py;
+use pyo3::{PyTraverseError, PyVisit, prelude::*, types::PyDict};
+use serde_json::{Map, Value};
+
+pub(super) struct PythonEmbedder(Py<PyAny>);
+
+impl PythonEmbedder {
+    pub(super) fn new(object: Py<PyAny>) -> Self {
+        Self(object)
+    }
+
+    pub(super) fn object(&self) -> &Py<PyAny> {
+        &self.0
+    }
+
+    pub(super) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
+        visit.call(&self.0)
+    }
+
+    fn metadata_kwargs<'py>(
+        py: Python<'py>,
+        metadata: &Map<String, Value>,
+    ) -> PyResult<Bound<'py, PyDict>> {
+        let kwargs = PyDict::new(py);
+        if metadata.is_empty() {
+            kwargs.set_item("metadata", py.None())?;
+        } else {
+            kwargs.set_item("metadata", to_py(py, metadata)?)?;
+        }
+        Ok(kwargs)
+    }
+
+    fn extract(vector: Bound<'_, PyAny>) -> PyResult<Vec<f32>> {
+        Ok(vector
+            .extract::<Vec<f64>>()?
+            .into_iter()
+            .map(|value| value as f32)
+            .collect())
+    }
+}
+
+impl Embedder for PythonEmbedder {
+    fn embed(&self, prompt: &str, metadata: &Map<String, Value>) -> Result<Vec<f32>, Error> {
+        Python::attach(|py| {
+            let kwargs = Self::metadata_kwargs(py, metadata)?;
+            Self::extract(self.0.bind(py).call_method(
+                "_get_embedding",
+                (prompt,),
+                Some(&kwargs),
+            )?)
+        })
+        .map_err(|_| Error::Unavailable)
+    }
+
+    fn async_embed(
+        &self,
+        prompt: &str,
+        metadata: &Map<String, Value>,
+    ) -> impl Future<Output = Result<Vec<f32>, Error>> + Send {
+        let coroutine = Python::attach(|py| {
+            let kwargs = Self::metadata_kwargs(py, metadata)?;
+            self.0
+                .bind(py)
+                .call_method("_get_async_embedding", (prompt,), Some(&kwargs))
+                .map(Bound::unbind)
+        })
+        .map_err(|_| Error::Unavailable);
+        async move {
+            let coroutine = coroutine?;
+            let awaited = Python::attach(|py| {
+                pyo3_async_runtimes::tokio::into_future(coroutine.into_bound(py))
+            })
+            .map_err(|_| Error::Unavailable)?
+            .await
+            .map_err(|_| Error::Unavailable)?;
+            let vector = Python::attach(|py| awaited.extract::<Vec<f64>>(py))
+                .map_err(|_| Error::Unavailable)?;
+            Ok(vector.into_iter().map(|value| value as f32).collect())
+        }
+    }
+}
diff --git a/litellm-rust/crates/python-bridge/src/cache/facade.rs b/litellm-rust/crates/python-bridge/src/cache/facade.rs
index f2f86c14b37..58730857d60 100644
--- a/litellm-rust/crates/python-bridge/src/cache/facade.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/facade.rs
@@ -192,6 +192,11 @@ impl FacadeGuard {
         let (module, name, cache_kind) = match kind {
             "memory" => ("litellm.caching.in_memory_cache", "InMemoryCache", "local"),
             "redis" => ("litellm.caching.redis_cache", "RedisCache", "redis"),
+            "redis_semantic" => (
+                "litellm.caching.redis_semantic_cache",
+                "RedisSemanticCache",
+                "redis-semantic",
+            ),
             _ => unreachable!(),
         };
         let backend = facade.getattr("cache")?;
@@ -211,6 +216,15 @@ impl FacadeGuard {
         if let Some(message) = config.service_mismatch(service) {
             return Err(PyTypeError::new_err(message));
         }
+        if kind == "redis_semantic"
+            && service
+                .embedder_object()
+                .is_none_or(|embedder| !backend.is(embedder.bind(py)))
+        {
+            return Err(PyTypeError::new_err(
+                "facade backend must be the native embedder",
+            ));
+        }
         Ok(Self {
             outer: ObjectGuard::capture(
                 py,
@@ -235,6 +249,13 @@ impl FacadeGuard {
                     "max_size_per_item",
                     "redis_kwargs",
                     "redis_flush_size",
+                    "similarity_threshold",
+                    "distance_threshold",
+                    "embedding_model",
+                    "embedding_max_input_tokens",
+                    "embedding_timeout",
+                    "_index_name",
+                    "_redis_url",
                 ],
             )?,
             redis_pool: (kind == "redis")
diff --git a/litellm-rust/crates/python-bridge/src/cache/handle.rs b/litellm-rust/crates/python-bridge/src/cache/handle.rs
index 8251b3df06c..b61ae59bb58 100644
--- a/litellm-rust/crates/python-bridge/src/cache/handle.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/handle.rs
@@ -1,7 +1,15 @@
+use litellm_cache_redis_semantic::RedisSemanticConfig;
 use litellm_host_python::release_gil;
-use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*};
+use pyo3::{
+    PyTraverseError, PyVisit,
+    exceptions::{PyRuntimeError, PyTypeError},
+    prelude::*,
+};
 
-use super::{cache_error, facade::FacadeGuard, native::NativeResponseCache, request::duration};
+use super::{
+    cache_error, config::project_redis_semantic, embedder::PythonEmbedder, facade::FacadeGuard,
+    native::NativeResponseCache, request::duration,
+};
 
 #[pyclass(frozen, name = "_CacheTestHandle")]
 pub(crate) struct CacheTestHandle {
@@ -51,6 +59,36 @@ impl CacheTestHandle {
         })
     }
 
+    #[staticmethod]
+    fn redis_semantic(py: Python<'_>, backend: Bound<'_, PyAny>) -> PyResult<Self> {
+        let class = py
+            .import("litellm.caching.redis_semantic_cache")?
+            .getattr("RedisSemanticCache")?;
+        if !backend.get_type().is(&class) {
+            return Err(PyTypeError::new_err(
+                "native redis-semantic handles require the built-in RedisSemanticCache",
+            ));
+        }
+        let config = project_redis_semantic(&backend)?;
+        let embedder = PythonEmbedder::new(backend.unbind());
+        let service = release_gil(py, move || {
+            NativeResponseCache::redis_semantic(
+                &config.redis_url,
+                embedder,
+                RedisSemanticConfig {
+                    index_name: config.index_name,
+                    similarity_threshold: config.similarity_threshold as f32,
+                },
+            )
+        })
+        .map_err(cache_error)?;
+        Ok(Self {
+            service,
+            guard: None,
+            pid: std::process::id(),
+        })
+    }
+
     #[getter]
     fn backend(&self) -> &'static str {
         self.service.kind()
@@ -76,6 +114,7 @@ impl CacheTestHandle {
     }
 
     fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> {
+        self.service.traverse(&visit)?;
         if let Some(guard) = &self.guard {
             guard.traverse(visit)?;
         }
diff --git a/litellm-rust/crates/python-bridge/src/cache/mod.rs b/litellm-rust/crates/python-bridge/src/cache/mod.rs
index aec08610f6e..4cc87367d91 100644
--- a/litellm-rust/crates/python-bridge/src/cache/mod.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/mod.rs
@@ -1,6 +1,7 @@
 mod binding;
 mod callback;
 mod config;
+mod embedder;
 mod facade;
 mod future;
 mod handle;
@@ -10,7 +11,7 @@ mod resolver;
 
 use litellm_cache::Error;
 use pyo3::{
-    exceptions::{PyRuntimeError, PyValueError},
+    exceptions::{PyNotImplementedError, PyRuntimeError, PyValueError},
     prelude::*,
 };
 
@@ -21,6 +22,7 @@ pub(crate) use self::{
 fn cache_error(error: Error) -> PyErr {
     match error {
         Error::InvalidEntry => PyValueError::new_err(error.to_string()),
+        Error::UnsupportedOperation => PyNotImplementedError::new_err(error.to_string()),
         _ => PyRuntimeError::new_err(error.to_string()),
     }
 }
diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs
index a9475429e45..8cd77fa8eb0 100644
--- a/litellm-rust/crates/python-bridge/src/cache/native.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/native.rs
@@ -1,13 +1,17 @@
 use std::{sync::Arc, time::Duration};
 
-use litellm_cache::{CacheCodec, CacheConnectionResult, Error};
+use litellm_cache::{CacheCodec, CacheConnectionResult, Error, ExactCacheContext};
 use litellm_cache_memory::InMemoryCache;
 use litellm_cache_redis::RedisCache;
+use litellm_cache_redis_semantic::{RedisSemanticCache, RedisSemanticConfig};
 use litellm_cache_response::{
     CacheEntry, PartialHits, ResponseCache, ResponseCacheCodec, ResponseCacheRequest, WriteBuffer,
 };
+use pyo3::{Py, PyAny, PyTraverseError, PyVisit};
 use serde_json::Value;
 
+use super::{embedder::PythonEmbedder, request::CacheRequest};
+
 #[derive(Clone)]
 pub(super) enum NativeResponseCache {
     Memory(Arc<ResponseCache<InMemoryCache<CacheEntry>>>),
@@ -15,6 +19,7 @@ pub(super) enum NativeResponseCache {
         cache: Arc<ResponseCache<RedisCache<ResponseCacheCodec>>>,
         buffer: Option<Arc<WriteBuffer>>,
     },
+    RedisSemantic(Arc<ResponseCache<RedisSemanticCache<PythonEmbedder>>>),
 }
 
 impl NativeResponseCache {
@@ -43,6 +48,17 @@ impl NativeResponseCache {
             buffer: None,
         })
     }
+
+    pub fn redis_semantic(
+        url: &str,
+        embedder: PythonEmbedder,
+        config: RedisSemanticConfig,
+    ) -> Result<Self, Error> {
+        let backend = RedisSemanticCache::new(url, embedder, config)?;
+        Ok(Self::RedisSemantic(Arc::new(ResponseCache::new(Arc::new(
+            backend,
+        )))))
+    }
 }
 
 impl NativeResponseCache {
@@ -50,6 +66,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(_) => "memory",
             Self::Redis { .. } => "redis",
+            Self::RedisSemantic(_) => "redis_semantic",
         }
     }
 
@@ -57,12 +74,13 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.default_ttl(),
             Self::Redis { cache, .. } => cache.default_ttl(),
+            Self::RedisSemantic(cache) => cache.default_ttl(),
         }
     }
 
     pub fn namespace(&self) -> Option<&str> {
         match self {
-            Self::Memory(_) => None,
+            Self::Memory(_) | Self::RedisSemantic(_) => None,
             Self::Redis { cache, .. } => cache.backend().namespace(),
         }
     }
@@ -70,110 +88,189 @@ impl NativeResponseCache {
     pub fn capacity(&self) -> Option<usize> {
         match self {
             Self::Memory(cache) => Some(cache.backend().max_size_in_memory()),
-            Self::Redis { .. } => None,
+            Self::Redis { .. } | Self::RedisSemantic(_) => None,
         }
     }
 
     pub fn max_entry_bytes(&self) -> Option<usize> {
         match self {
             Self::Memory(cache) => cache.backend().max_entry_bytes(),
-            Self::Redis { .. } => None,
+            Self::Redis { .. } | Self::RedisSemantic(_) => None,
         }
     }
 
+    pub fn index_name(&self) -> Option<&str> {
+        match self {
+            Self::RedisSemantic(cache) => Some(cache.backend().index_name()),
+            Self::Memory(_) | Self::Redis { .. } => None,
+        }
+    }
+
+    pub fn similarity_threshold(&self) -> Option<f32> {
+        match self {
+            Self::RedisSemantic(cache) => Some(cache.backend().similarity_threshold()),
+            Self::Memory(_) | Self::Redis { .. } => None,
+        }
+    }
+
+    pub fn embedder_object(&self) -> Option<&Py<PyAny>> {
+        match self {
+            Self::RedisSemantic(cache) => Some(cache.backend().embedder().object()),
+            Self::Memory(_) | Self::Redis { .. } => None,
+        }
+    }
+
+    pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
+        if let Self::RedisSemantic(cache) = self {
+            cache.backend().embedder().traverse(visit)?;
+        }
+        Ok(())
+    }
+
     pub fn with_redis_flush_size(self, flush_size: Option<usize>) -> Self {
         match self {
             Self::Redis { cache, .. } => Self::Redis {
                 cache,
                 buffer: flush_size.map(|flush_size| Arc::new(WriteBuffer::new(flush_size))),
             },
-            memory => memory,
+            other => other,
         }
     }
 
-    pub fn lookup(
-        &self,
-        request: &ResponseCacheRequest,
-        now: Duration,
-    ) -> Result<Option<Value>, Error> {
+    fn exact_requests(requests: &[CacheRequest]) -> Vec<ResponseCacheRequest<ExactCacheContext>> {
+        requests.iter().map(CacheRequest::exact).collect()
+    }
+
+    pub fn lookup(&self, request: &CacheRequest, now: Duration) -> Result<Option<Value>, Error> {
         match self {
-            Self::Memory(cache) => cache.lookup(request, now),
-            Self::Redis { cache, .. } => cache.lookup(request, now),
+            Self::Memory(cache) => cache.lookup(&request.exact(), now),
+            Self::Redis { cache, .. } => cache.lookup(&request.exact(), now),
+            Self::RedisSemantic(cache) => cache.lookup(&request.semantic(), now),
         }
     }
 
     pub fn store(
         &self,
-        request: &ResponseCacheRequest,
+        request: &CacheRequest,
         response: Value,
         now: Duration,
     ) -> Result<(), Error> {
         match self {
-            Self::Memory(cache) => cache.store(request, response, now),
-            Self::Redis { cache, .. } => cache.store(request, response, now),
+            Self::Memory(cache) => cache.store(&request.exact(), response, now),
+            Self::Redis { cache, .. } => cache.store(&request.exact(), response, now),
+            Self::RedisSemantic(cache) => cache.store(&request.semantic(), response, now),
         }
     }
 
     pub fn lookup_batch(
         &self,
-        requests: &[ResponseCacheRequest],
+        requests: &[CacheRequest],
         now: Duration,
     ) -> Result<PartialHits, Error> {
         match self {
-            Self::Memory(cache) => cache.lookup_batch(requests, now),
-            Self::Redis { cache, .. } => cache.lookup_batch(requests, now),
+            Self::Memory(cache) => cache.lookup_batch(&Self::exact_requests(requests), now),
+            Self::Redis { cache, .. } => cache.lookup_batch(&Self::exact_requests(requests), now),
+            Self::RedisSemantic(_) => Err(Error::UnsupportedOperation),
         }
     }
 
     pub async fn async_lookup(
         &self,
-        request: &ResponseCacheRequest,
+        request: &CacheRequest,
         now: Duration,
     ) -> Result<Option<Value>, Error> {
         match self {
-            Self::Memory(cache) => cache.async_lookup(request, now).await,
-            Self::Redis { cache, .. } => cache.async_lookup(request, now).await,
+            Self::Memory(cache) => cache.async_lookup(&request.exact(), now).await,
+            Self::Redis { cache, .. } => cache.async_lookup(&request.exact(), now).await,
+            Self::RedisSemantic(cache) => cache.async_lookup(&request.semantic(), now).await,
         }
     }
 
     pub async fn async_store(
         &self,
-        request: &ResponseCacheRequest,
+        request: &CacheRequest,
         response: Value,
         now: Duration,
     ) -> Result<(), Error> {
         match self {
-            Self::Memory(cache) => cache.async_store(request, response, now).await,
+            Self::Memory(cache) => cache.async_store(&request.exact(), response, now).await,
             Self::Redis {
                 cache,
                 buffer: None,
-            } => cache.async_store(request, response, now).await,
+            } => cache.async_store(&request.exact(), response, now).await,
             Self::Redis {
                 cache,
                 buffer: Some(buffer),
-            } => buffer.async_store(cache, request, response, now).await,
+            } => {
+                buffer
+                    .async_store(cache, &request.exact(), response, now)
+                    .await
+            }
+            Self::RedisSemantic(cache) => {
+                cache.async_store(&request.semantic(), response, now).await
+            }
         }
     }
 
     pub async fn async_lookup_batch(
         &self,
-        requests: &[ResponseCacheRequest],
+        requests: &[CacheRequest],
         now: Duration,
     ) -> Result<PartialHits, Error> {
         match self {
-            Self::Memory(cache) => cache.async_lookup_batch(requests, now).await,
-            Self::Redis { cache, .. } => cache.async_lookup_batch(requests, now).await,
+            Self::Memory(cache) => {
+                cache
+                    .async_lookup_batch(&Self::exact_requests(requests), now)
+                    .await
+            }
+            Self::Redis { cache, .. } => {
+                cache
+                    .async_lookup_batch(&Self::exact_requests(requests), now)
+                    .await
+            }
+            Self::RedisSemantic(_) => Err(Error::UnsupportedOperation),
         }
     }
 
     pub async fn async_store_batch(
         &self,
-        entries: Vec<(ResponseCacheRequest, Value)>,
+        entries: Vec<(CacheRequest, Value)>,
         now: Duration,
     ) -> Result<(), Error> {
         match self {
-            Self::Memory(cache) => cache.async_store_batch(entries, now).await,
-            Self::Redis { cache, .. } => cache.async_store_batch(entries, now).await,
+            Self::Memory(cache) => {
+                cache
+                    .async_store_batch(
+                        entries
+                            .into_iter()
+                            .map(|(request, value)| (request.exact(), value))
+                            .collect(),
+                        now,
+                    )
+                    .await
+            }
+            Self::Redis { cache, .. } => {
+                cache
+                    .async_store_batch(
+                        entries
+                            .into_iter()
+                            .map(|(request, value)| (request.exact(), value))
+                            .collect(),
+                        now,
+                    )
+                    .await
+            }
+            Self::RedisSemantic(cache) => {
+                cache
+                    .async_store_batch(
+                        entries
+                            .into_iter()
+                            .map(|(request, value)| (request.semantic(), value))
+                            .collect(),
+                        now,
+                    )
+                    .await
+            }
         }
     }
 
@@ -186,6 +283,7 @@ impl NativeResponseCache {
                 }
                 cache.async_flush().await
             }
+            Self::RedisSemantic(_) => Err(Error::UnsupportedOperation),
         }
     }
 
@@ -193,6 +291,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.test_connection().await,
             Self::Redis { cache, .. } => cache.test_connection().await,
+            Self::RedisSemantic(_) => Err(Error::UnsupportedOperation),
         }
     }
 }
diff --git a/litellm-rust/crates/python-bridge/src/cache/request.rs b/litellm-rust/crates/python-bridge/src/cache/request.rs
index 0c5343a63d0..26e0fe4e62c 100644
--- a/litellm-rust/crates/python-bridge/src/cache/request.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/request.rs
@@ -1,9 +1,11 @@
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
+use litellm_cache::{ExactCacheContext, SemanticCacheContext};
 use litellm_cache_response::{CacheControls, CacheKeyInput, ResponseCacheRequest};
 use litellm_host_python::from_py;
 use pyo3::{exceptions::PyValueError, prelude::*};
 use serde::Deserialize;
+use serde_json::{Map, Value};
 
 #[derive(Deserialize)]
 #[serde(deny_unknown_fields)]
@@ -12,24 +14,68 @@ struct RequestInput {
     controls: Option<CacheControls>,
     ttl_seconds: Option<f64>,
     max_age_seconds: Option<f64>,
+    input: Option<Value>,
+    messages: Option<Vec<Value>>,
+    metadata: Option<Map<String, Value>>,
+    scope: Option<String>,
 }
 
-pub(super) fn request(value: &Bound<'_, PyAny>) -> PyResult<ResponseCacheRequest> {
+pub(super) struct CacheRequest {
+    key: CacheKeyInput,
+    controls: CacheControls,
+    ttl: Option<Duration>,
+    max_age: Option<Duration>,
+    input: Option<Value>,
+    messages: Vec<Value>,
+    metadata: Map<String, Value>,
+    scope: Option<String>,
+}
+
+impl CacheRequest {
+    pub(super) fn exact(&self) -> ResponseCacheRequest<ExactCacheContext> {
+        let mut request = ResponseCacheRequest::new(self.key.clone());
+        request.controls = self.controls;
+        request.context.ttl = self.ttl;
+        request.max_age = self.max_age;
+        request
+    }
+
+    pub(super) fn semantic(&self) -> ResponseCacheRequest<SemanticCacheContext> {
+        ResponseCacheRequest {
+            key: self.key.clone(),
+            controls: self.controls,
+            context: SemanticCacheContext {
+                input: self.input.clone(),
+                messages: self.messages.clone(),
+                metadata: self.metadata.clone(),
+                scope: self.scope.clone(),
+                ttl: self.ttl,
+            },
+            max_age: self.max_age,
+        }
+    }
+}
+
+pub(super) fn request(value: &Bound<'_, PyAny>) -> PyResult<CacheRequest> {
     let input: RequestInput = from_py(value)?;
     request_input(input)
 }
 
-fn request_input(input: RequestInput) -> PyResult<ResponseCacheRequest> {
-    let mut request = ResponseCacheRequest::new(input.key);
-    if let Some(controls) = input.controls {
-        request.controls = controls;
-    }
-    request.context.ttl = input.ttl_seconds.map(duration).transpose()?;
-    request.max_age = input.max_age_seconds.map(duration).transpose()?;
-    Ok(request)
+fn request_input(input: RequestInput) -> PyResult<CacheRequest> {
+    let defaults = ResponseCacheRequest::<ExactCacheContext>::new(input.key.clone());
+    Ok(CacheRequest {
+        key: input.key,
+        controls: input.controls.unwrap_or(defaults.controls),
+        ttl: input.ttl_seconds.map(duration).transpose()?,
+        max_age: input.max_age_seconds.map(duration).transpose()?,
+        input: input.input,
+        messages: input.messages.unwrap_or_default(),
+        metadata: input.metadata.unwrap_or_default(),
+        scope: input.scope,
+    })
 }
 
-pub(super) fn requests(value: &Bound<'_, PyAny>) -> PyResult<Vec<ResponseCacheRequest>> {
+pub(super) fn requests(value: &Bound<'_, PyAny>) -> PyResult<Vec<CacheRequest>> {
     from_py::<Vec<RequestInput>>(value)?
         .into_iter()
         .map(request_input)

From 605729a194c997b8f90a5edb5494465be9cc4d95 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:39:17 +0000
Subject: [PATCH 080/160] fix(rust): pin aws-sdk-s3 without relaxing
 eventstream pins

Keep the exact eventstream pins but move them to =0.61.4 so aws-runtime 1.9.4's eventstream requirement resolves, downgrade aws-sdk-s3 to 1.146.1, and echo the context ttl from S3Cache::get_ttl to match Python

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.lock                     | 24 ++++++++++-----------
 litellm-rust/crates/cache-s3/Cargo.toml     |  2 +-
 litellm-rust/crates/cache-s3/src/cache.rs   |  4 ++--
 litellm-rust/crates/cache-s3/tests/cache.rs |  6 ++++++
 litellm-rust/crates/framer/Cargo.toml       |  2 +-
 litellm-rust/crates/llms/Cargo.toml         |  2 +-
 6 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index 4018ee2875b..e2337f41ebc 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -261,9 +261,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-s3"
-version = "1.148.0"
+version = "1.146.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a61c1db3987ab6c8740fb87f248864687d50549401115244a84f02c3047b2d5"
+checksum = "2cd651b4400d4011b8927b83a9552bf90ff11e6e5da0b9f0a7583247aceec971"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -1380,7 +1380,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
  "libc",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -2152,7 +2152,7 @@ dependencies = [
  "libc",
  "percent-encoding",
  "pin-project-lite",
- "socket2 0.6.5",
+ "socket2 0.5.10",
  "tokio",
  "tower-service",
  "tracing",
@@ -3638,7 +3638,7 @@ dependencies = [
  "quinn-udp",
  "rustc-hash",
  "rustls 0.23.42",
- "socket2 0.6.5",
+ "socket2 0.5.10",
  "thiserror 2.0.19",
  "tokio",
  "tracing",
@@ -3677,9 +3677,9 @@ dependencies = [
  "cfg_aliases",
  "libc",
  "once_cell",
- "socket2 0.6.5",
+ "socket2 0.5.10",
  "tracing",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -4104,7 +4104,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -4175,7 +4175,7 @@ dependencies = [
  "security-framework",
  "security-framework-sys",
  "webpki-root-certs",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -4699,10 +4699,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.4.3",
+ "getrandom 0.3.4",
  "once_cell",
  "rustix",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -5542,7 +5542,7 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
 dependencies = [
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
diff --git a/litellm-rust/crates/cache-s3/Cargo.toml b/litellm-rust/crates/cache-s3/Cargo.toml
index c5d084207c8..cdc17e732cb 100644
--- a/litellm-rust/crates/cache-s3/Cargo.toml
+++ b/litellm-rust/crates/cache-s3/Cargo.toml
@@ -8,7 +8,7 @@ repository.workspace = true
 [dependencies]
 litellm-cache.workspace = true
 litellm-auth-aws.workspace = true
-aws-sdk-s3 = { version = "1.141.0", default-features = false, features = ["rustls", "rt-tokio"] }
+aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] }
 aws-credential-types = "1.3.0"
 aws-smithy-types = "1.6.0"
 aws-types = "1.6.0"
diff --git a/litellm-rust/crates/cache-s3/src/cache.rs b/litellm-rust/crates/cache-s3/src/cache.rs
index 1aa16ecaaa0..a2ea33c33e4 100644
--- a/litellm-rust/crates/cache-s3/src/cache.rs
+++ b/litellm-rust/crates/cache-s3/src/cache.rs
@@ -152,8 +152,8 @@ impl<C: CacheCodec> BaseCache for S3Cache<C> {
     type Value = C::Value;
     type Context = ExactCacheContext;
 
-    fn get_ttl(&self, _context: &Self::Context) -> Option<Duration> {
-        None
+    fn get_ttl(&self, context: &Self::Context) -> Option<Duration> {
+        context.ttl
     }
 
     fn set_cache(
diff --git a/litellm-rust/crates/cache-s3/tests/cache.rs b/litellm-rust/crates/cache-s3/tests/cache.rs
index 93e88cf7cbd..6ca3eb9a758 100644
--- a/litellm-rust/crates/cache-s3/tests/cache.rs
+++ b/litellm-rust/crates/cache-s3/tests/cache.rs
@@ -204,6 +204,12 @@ async fn unsupported_and_noop_capabilities_match_python() {
     cache.flush_cache().unwrap();
     cache.disconnect().await.unwrap();
     assert_eq!(cache.get_ttl(&ExactCacheContext::default()), None);
+    assert_eq!(
+        cache.get_ttl(&ExactCacheContext {
+            ttl: Some(Duration::from_secs(45)),
+        }),
+        Some(Duration::from_secs(45))
+    );
     assert!(server.received_requests().await.unwrap().is_empty());
 }
 
diff --git a/litellm-rust/crates/framer/Cargo.toml b/litellm-rust/crates/framer/Cargo.toml
index 60dfa9469d2..62bfcc7da3d 100644
--- a/litellm-rust/crates/framer/Cargo.toml
+++ b/litellm-rust/crates/framer/Cargo.toml
@@ -11,7 +11,7 @@ aws = ["dep:aws-smithy-eventstream", "dep:aws-smithy-types"]
 sse = ["dep:sse-stream"]
 
 [dependencies]
-aws-smithy-eventstream = { version = "0.61.1", optional = true }
+aws-smithy-eventstream = { version = "=0.61.4", optional = true }
 aws-smithy-types = { version = "1.6.1", optional = true }
 bytes = "1"
 futures-util.workspace = true
diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml
index 8bdbb7c3b77..f04b78feee1 100644
--- a/litellm-rust/crates/llms/Cargo.toml
+++ b/litellm-rust/crates/llms/Cargo.toml
@@ -34,7 +34,7 @@ tokio = { workspace = true, features = ["sync"] }
 url.workspace = true
 
 [dev-dependencies]
-aws-smithy-eventstream = "0.61.1"
+aws-smithy-eventstream = "=0.61.4"
 aws-smithy-types = "1.6.1"
 rstest.workspace = true
 tokio.workspace = true

From 56237af7a9dbfcdb20b41b3959620128c8fbccd2 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:42:41 +0000
Subject: [PATCH 081/160] test(cache): add Valkey semantic contract coverage

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../crates/cache-valkey-semantic/src/lib.rs   | 659 +++++++++++++-----
 .../crates/python-bridge/src/cache/config.rs  |  84 ++-
 .../test_valkey_semantic_cache_native.py      | 140 +++-
 3 files changed, 695 insertions(+), 188 deletions(-)

diff --git a/litellm-rust/crates/cache-valkey-semantic/src/lib.rs b/litellm-rust/crates/cache-valkey-semantic/src/lib.rs
index 85c4c9af15c..ef028f0a7b2 100644
--- a/litellm-rust/crates/cache-valkey-semantic/src/lib.rs
+++ b/litellm-rust/crates/cache-valkey-semantic/src/lib.rs
@@ -62,6 +62,14 @@ enum Connections<C> {
     Fixed(Mutex<C>),
 }
 
+#[derive(Clone)]
+struct IndexState {
+    name: String,
+    prefix: String,
+    dimension: Arc<Mutex<Option<usize>>>,
+    similarity_threshold: f64,
+}
+
 struct ConnectionRef<'a>(&'a mut dyn redis::ConnectionLike);
 
 impl redis::ConnectionLike for ConnectionRef<'_> {
@@ -187,18 +195,13 @@ where
         &self.config.index_name
     }
 
-    fn key_prefix(&self) -> String {
-        format!("{}:", self.config.index_name)
-    }
-
-    fn ensure_index(&self, dimension: usize) -> Result<(), Error> {
-        ensure_index(
-            &self.connections,
-            &self.config.index_name,
-            &self.key_prefix(),
-            &self.index_dimension,
-            dimension,
-        )
+    fn index_state(&self) -> IndexState {
+        IndexState {
+            name: self.config.index_name.clone(),
+            prefix: format!("{}:", self.config.index_name),
+            dimension: Arc::clone(&self.index_dimension),
+            similarity_threshold: self.config.similarity_threshold,
+        }
     }
 }
 
@@ -225,37 +228,19 @@ where
             return Ok(());
         };
         let embedding = self.embedder.embed(&prompt, context.metadata.as_ref())?;
-        self.ensure_index(embedding.len())?;
         let scope = scope_tag(key);
-        let document = format!("{}{}:{}", self.key_prefix(), scope, Uuid::new_v4());
         let response = self.codec.encode(&value)?;
         let vector = embedding_bytes(&embedding);
-        let ttl = self.get_ttl(context);
-        self.connections.execute(|connection| {
-            let mut pipeline = redis::pipe();
-            pipeline
-                .cmd("HSET")
-                .arg(&document)
-                .arg("litellm_cache_key")
-                .arg(&scope)
-                .arg("prompt")
-                .arg(prompt)
-                .arg("response")
-                .arg(response)
-                .arg("embedding")
-                .arg(vector)
-                .ignore();
-            if let Some(ttl) = ttl {
-                pipeline
-                    .cmd("EXPIRE")
-                    .arg(&document)
-                    .arg(ttl.as_secs())
-                    .ignore();
-            }
-            pipeline
-                .query::<()>(connection)
-                .map_err(|_| Error::Unavailable)
-        })
+        let index = self.index_state();
+        write_document(
+            &self.connections,
+            &index,
+            &scope,
+            &prompt,
+            response,
+            vector,
+            self.get_ttl(context),
+        )
     }
 
     fn get_cache(&self, key: &str, context: &Self::Context) -> Result<Option<Self::Value>, Error> {
@@ -263,43 +248,14 @@ where
             return Ok(None);
         };
         let embedding = self.embedder.embed(&prompt, context.metadata.as_ref())?;
-        self.ensure_index(embedding.len())?;
         let scope = scope_tag(key);
-        let query =
-            format!("(@litellm_cache_key:{{{scope}}})=>[KNN 1 @embedding $vec AS vector_distance]");
         let vector = embedding_bytes(&embedding);
-        let response = self.connections.execute(|connection| {
-            redis::cmd("FT.SEARCH")
-                .arg(&self.config.index_name)
-                .arg(query)
-                .arg("PARAMS")
-                .arg(2)
-                .arg("vec")
-                .arg(vector)
-                .arg("RETURN")
-                .arg(2)
-                .arg("response")
-                .arg("vector_distance")
-                .arg("DIALECT")
-                .arg(2)
-                .query::<redis::Value>(connection)
-                .map_err(|_| Error::Unavailable)
-        })?;
-        let Some(fields) = search_fields(response)? else {
+        let index = self.index_state();
+        let Some(response) =
+            search_document(&self.connections, &index, &scope, vector, embedding.len())?
+        else {
             return Ok(None);
         };
-        let response = fields
-            .iter()
-            .find_map(|(name, value)| (name == "response").then(|| value.clone()))
-            .ok_or(Error::InvalidEntry)?;
-        let distance = fields
-            .iter()
-            .find_map(|(name, value)| (name == "vector_distance").then(|| value.clone()))
-            .ok_or(Error::InvalidEntry)?;
-        let distance = parse_f64(&distance)?;
-        if 1.0 - distance < self.config.similarity_threshold {
-            return Ok(None);
-        }
         self.codec.decode(&response).map(Some)
     }
 
@@ -321,47 +277,13 @@ where
                 .async_embed(&prompt, metadata.as_ref())
                 .await?;
             let connections = Arc::clone(&self.connections);
-            let config = self.config.clone();
-            let index_dimension = Arc::clone(&self.index_dimension);
+            let index = self.index_state();
             let response = self.codec.encode(&value)?;
             let vector = embedding_bytes(&embedding);
-            let prefix = format!("{}:", config.index_name);
             let scope = scope_tag(&key);
-            let document = format!("{prefix}{scope}:{}", Uuid::new_v4());
             let ttl = context.ttl;
             tokio::task::spawn_blocking(move || {
-                ensure_index(
-                    &connections,
-                    &config.index_name,
-                    &prefix,
-                    &index_dimension,
-                    embedding.len(),
-                )?;
-                connections.execute(|connection| {
-                    let mut pipeline = redis::pipe();
-                    pipeline
-                        .cmd("HSET")
-                        .arg(&document)
-                        .arg("litellm_cache_key")
-                        .arg(&scope)
-                        .arg("prompt")
-                        .arg(prompt)
-                        .arg("response")
-                        .arg(response)
-                        .arg("embedding")
-                        .arg(vector)
-                        .ignore();
-                    if let Some(ttl) = ttl {
-                        pipeline
-                            .cmd("EXPIRE")
-                            .arg(&document)
-                            .arg(ttl.as_secs())
-                            .ignore();
-                    }
-                    pipeline
-                        .query::<()>(connection)
-                        .map_err(|_| Error::Unavailable)
-                })
+                write_document(&connections, &index, &scope, &prompt, response, vector, ttl)
             })
             .await
             .map_err(|_| Error::Unavailable)?
@@ -385,56 +307,11 @@ where
                 .async_embed(&prompt, metadata.as_ref())
                 .await?;
             let connections = Arc::clone(&self.connections);
-            let config = self.config.clone();
-            let index_dimension = Arc::clone(&self.index_dimension);
-            let threshold = config.similarity_threshold;
+            let index = self.index_state();
             tokio::task::spawn_blocking(move || {
-                let prefix = format!("{}:", config.index_name);
-                ensure_index(
-                    &connections,
-                    &config.index_name,
-                    &prefix,
-                    &index_dimension,
-                    embedding.len(),
-                )?;
                 let scope = scope_tag(&key);
-                let query = format!(
-                    "(@litellm_cache_key:{{{scope}}})=>[KNN 1 @embedding $vec AS vector_distance]"
-                );
                 let vector = embedding_bytes(&embedding);
-                let response = connections.execute(|connection| {
-                    redis::cmd("FT.SEARCH")
-                        .arg(&config.index_name)
-                        .arg(query)
-                        .arg("PARAMS")
-                        .arg(2)
-                        .arg("vec")
-                        .arg(vector)
-                        .arg("RETURN")
-                        .arg(2)
-                        .arg("response")
-                        .arg("vector_distance")
-                        .arg("DIALECT")
-                        .arg(2)
-                        .query::<redis::Value>(connection)
-                        .map_err(|_| Error::Unavailable)
-                })?;
-                let Some(fields) = search_fields(response)? else {
-                    return Ok(None);
-                };
-                let response = fields
-                    .iter()
-                    .find_map(|(name, value)| (name == "response").then(|| value.clone()))
-                    .ok_or(Error::InvalidEntry)?;
-                let distance = fields
-                    .iter()
-                    .find_map(|(name, value)| (name == "vector_distance").then(|| value.clone()))
-                    .ok_or(Error::InvalidEntry)?;
-                let distance = parse_f64(&distance)?;
-                if 1.0 - distance < threshold {
-                    return Ok(None);
-                }
-                Ok(Some(response))
+                search_document(&connections, &index, &scope, vector, embedding.len())
             })
             .await
             .map_err(|_| Error::Unavailable)?
@@ -560,6 +437,108 @@ fn embedding_bytes(embedding: &[f32]) -> Vec<u8> {
         .collect()
 }
 
+fn write_document<C>(
+    connections: &Connections<C>,
+    index: &IndexState,
+    scope: &str,
+    prompt: &str,
+    response: Vec<u8>,
+    vector: Vec<u8>,
+    ttl: Option<Duration>,
+) -> Result<(), Error>
+where
+    C: redis::ConnectionLike + Send + 'static,
+{
+    let dimension = vector.len() / std::mem::size_of::<f32>();
+    ensure_index(
+        connections,
+        &index.name,
+        &index.prefix,
+        &index.dimension,
+        dimension,
+    )?;
+    let document = format!("{}{scope}:{}", index.prefix, Uuid::new_v4());
+    connections.execute(|connection| {
+        let mut pipeline = redis::pipe();
+        pipeline
+            .cmd("HSET")
+            .arg(&document)
+            .arg("litellm_cache_key")
+            .arg(scope)
+            .arg("prompt")
+            .arg(prompt)
+            .arg("response")
+            .arg(response)
+            .arg("embedding")
+            .arg(vector)
+            .ignore();
+        if let Some(ttl) = ttl {
+            pipeline
+                .cmd("EXPIRE")
+                .arg(&document)
+                .arg(ttl.as_secs())
+                .ignore();
+        }
+        pipeline
+            .query::<()>(connection)
+            .map_err(|_| Error::Unavailable)
+    })
+}
+
+fn search_document<C>(
+    connections: &Connections<C>,
+    index: &IndexState,
+    scope: &str,
+    vector: Vec<u8>,
+    dimension: usize,
+) -> Result<Option<Vec<u8>>, Error>
+where
+    C: redis::ConnectionLike + Send + 'static,
+{
+    ensure_index(
+        connections,
+        &index.name,
+        &index.prefix,
+        &index.dimension,
+        dimension,
+    )?;
+    let query =
+        format!("(@litellm_cache_key:{{{scope}}})=>[KNN 1 @embedding $vec AS vector_distance]");
+    let response = connections.execute(|connection| {
+        redis::cmd("FT.SEARCH")
+            .arg(&index.name)
+            .arg(query)
+            .arg("PARAMS")
+            .arg(2)
+            .arg("vec")
+            .arg(vector)
+            .arg("RETURN")
+            .arg(2)
+            .arg("response")
+            .arg("vector_distance")
+            .arg("DIALECT")
+            .arg(2)
+            .query::<redis::Value>(connection)
+            .map_err(|_| Error::Unavailable)
+    })?;
+    let Some(fields) = search_fields(response)? else {
+        return Ok(None);
+    };
+    let response = fields
+        .iter()
+        .find_map(|(name, value)| (name == "response").then(|| value.clone()))
+        .ok_or(Error::InvalidEntry)?;
+    let distance = fields
+        .iter()
+        .find_map(|(name, value)| (name == "vector_distance").then(|| value.clone()))
+        .ok_or(Error::InvalidEntry)?;
+    let distance = parse_f64(&distance)?;
+    if 1.0 - distance < index.similarity_threshold {
+        return Ok(None);
+    }
+    Ok(Some(response))
+}
+
 fn ensure_index<C>(
     connections: &Connections<C>,
     index_name: &str,
@@ -712,10 +691,16 @@ fn value_bytes(value: &redis::Value) -> Result<Vec<u8>, Error> {
 
 #[cfg(test)]
 mod tests {
-    use std::sync::{Arc, Mutex};
+    use std::{
+        collections::VecDeque,
+        sync::{Arc, Mutex},
+        time::Duration,
+    };
 
-    use litellm_cache::BaseCache;
-    use litellm_cache_response::ResponseCacheCodec;
+    use litellm_cache::{BaseCache, CacheCodec};
+    use litellm_cache_response::{
+        CacheEntry, CacheKeyInput, ResponseCache, ResponseCacheCodec, ResponseCacheRequest,
+    };
     use redis_test::MockRedisConnection;
     use rstest::rstest;
     use serde_json::{Value, json};
@@ -732,6 +717,9 @@ mod tests {
     }
 
     type EmbedderCalls = Arc<Mutex<Vec<(String, Option<Value>)>>>;
+    type RecordingCache =
+        ValkeySemanticCache<FixedEmbedder, ResponseCacheCodec, RecordingConnection>;
+    type RecordingSetup = (RecordingCache, Arc<Mutex<Vec<Vec<u8>>>>, EmbedderCalls);
 
     impl Embedder for FixedEmbedder {
         fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, super::Error> {
@@ -751,6 +739,61 @@ mod tests {
         }
     }
 
+    struct RecordingConnection {
+        requests: Arc<Mutex<Vec<Vec<u8>>>>,
+        replies: Mutex<VecDeque<redis::RedisResult<redis::Value>>>,
+    }
+
+    impl RecordingConnection {
+        fn new(replies: impl IntoIterator<Item = redis::RedisResult<redis::Value>>) -> Self {
+            Self {
+                requests: Arc::default(),
+                replies: Mutex::new(replies.into_iter().collect()),
+            }
+        }
+
+        fn requests(&self) -> Arc<Mutex<Vec<Vec<u8>>>> {
+            Arc::clone(&self.requests)
+        }
+
+        fn reply(&self) -> redis::RedisResult<redis::Value> {
+            self.replies
+                .lock()
+                .unwrap()
+                .pop_front()
+                .unwrap_or_else(|| Ok(redis::Value::SimpleString("OK".into())))
+        }
+    }
+
+    impl redis::ConnectionLike for RecordingConnection {
+        fn req_packed_command(&mut self, command: &[u8]) -> redis::RedisResult<redis::Value> {
+            self.requests.lock().unwrap().push(command.to_vec());
+            self.reply()
+        }
+
+        fn req_packed_commands(
+            &mut self,
+            command: &[u8],
+            _offset: usize,
+            count: usize,
+        ) -> redis::RedisResult<Vec<redis::Value>> {
+            self.requests.lock().unwrap().push(command.to_vec());
+            (0..count).map(|_| self.reply()).collect()
+        }
+
+        fn get_db(&self) -> i64 {
+            0
+        }
+
+        fn check_connection(&mut self) -> bool {
+            true
+        }
+
+        fn is_open(&self) -> bool {
+            true
+        }
+    }
+
     fn context(
         messages: Option<Value>,
         input: Option<Value>,
@@ -841,4 +884,302 @@ mod tests {
         assert_eq!(cache.get_cache("key", &context(None, None)).unwrap(), None);
         assert_eq!(cache.get_ttl(&context(None, None)), None);
     }
+
+    fn semantic_context(ttl: Option<Duration>) -> litellm_cache::SemanticCacheContext {
+        litellm_cache::SemanticCacheContext {
+            messages: Some(json!([{"role": "user", "content": "hello"}])),
+            metadata: Some(json!({"source": "test"})),
+            ttl,
+            ..Default::default()
+        }
+    }
+
+    fn cache_with_recording(
+        replies: impl IntoIterator<Item = redis::RedisResult<redis::Value>>,
+        vector: Vec<f32>,
+        threshold: f64,
+    ) -> RecordingSetup {
+        let connection = RecordingConnection::new(replies);
+        let requests = connection.requests();
+        let calls: EmbedderCalls = Arc::default();
+        let cache = ValkeySemanticCache::with_connection(
+            connection,
+            FixedEmbedder {
+                vector,
+                calls: Arc::clone(&calls),
+            },
+            ResponseCacheCodec,
+            ValkeySemanticConfig {
+                similarity_threshold: threshold,
+                index_name: "test".into(),
+            },
+        );
+        (cache, requests, calls)
+    }
+
+    fn ok() -> redis::RedisResult<redis::Value> {
+        Ok(redis::Value::SimpleString("OK".into()))
+    }
+
+    fn already_exists() -> redis::RedisResult<redis::Value> {
+        Err(redis::RedisError::from((
+            redis::ErrorKind::Io,
+            "already exists",
+        )))
+    }
+
+    fn info_dimension(dimension: usize) -> redis::Value {
+        redis::Value::Array(vec![
+            redis::Value::SimpleString("attributes".into()),
+            redis::Value::Array(vec![redis::Value::Array(vec![
+                redis::Value::SimpleString("embedding".into()),
+                redis::Value::Array(vec![
+                    redis::Value::SimpleString("dimensions".into()),
+                    redis::Value::Int(dimension as i64),
+                ]),
+            ])]),
+        ])
+    }
+
+    fn search_hit(response: Vec<u8>, distance: &str) -> redis::Value {
+        redis::Value::Array(vec![
+            redis::Value::Int(1),
+            redis::Value::BulkString(b"test:document".to_vec()),
+            redis::Value::Array(vec![
+                redis::Value::BulkString(b"response".to_vec()),
+                redis::Value::BulkString(response),
+                redis::Value::BulkString(b"vector_distance".to_vec()),
+                redis::Value::BulkString(distance.as_bytes().to_vec()),
+            ]),
+        ])
+    }
+
+    fn requests_text(requests: &Arc<Mutex<Vec<Vec<u8>>>>) -> String {
+        requests
+            .lock()
+            .unwrap()
+            .iter()
+            .map(|request| String::from_utf8_lossy(request))
+            .collect::<Vec<_>>()
+            .join("\n")
+    }
+
+    #[test]
+    fn set_without_ttl_writes_hset_without_expire() {
+        let (cache, requests, calls) = cache_with_recording([ok()], vec![1.0, 0.0], 0.8);
+        cache
+            .set_cache(
+                "key",
+                CacheEntry {
+                    timestamp: None,
+                    response: json!({"answer": "ok"}),
+                },
+                &semantic_context(None),
+            )
+            .unwrap();
+        let text = requests_text(&requests);
+        assert!(text.contains("FT.CREATE"));
+        assert!(text.contains("HSET"));
+        assert!(
+            text.contains("test:2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683:")
+        );
+        assert!(!text.contains("EXPIRE"));
+        assert_eq!(
+            *calls.lock().unwrap(),
+            vec![("hello".into(), Some(json!({"source": "test"})))]
+        );
+    }
+
+    #[test]
+    fn set_with_ttl_truncates_expire_seconds() {
+        let (cache, requests, _) = cache_with_recording([ok()], vec![1.0, 0.0], 0.8);
+        cache
+            .set_cache(
+                "key",
+                CacheEntry {
+                    timestamp: None,
+                    response: json!({"answer": "ok"}),
+                },
+                &semantic_context(Some(Duration::from_millis(1900))),
+            )
+            .unwrap();
+        let text = requests_text(&requests);
+        assert!(text.contains("EXPIRE"));
+        assert!(text.contains("\r\n$1\r\n1\r\n"));
+    }
+
+    #[test]
+    fn second_set_skips_create_after_dimension_is_cached() {
+        let (cache, requests, _) = cache_with_recording([ok()], vec![1.0, 0.0], 0.8);
+        let context = semantic_context(None);
+        let entry = CacheEntry {
+            timestamp: None,
+            response: json!({"answer": "ok"}),
+        };
+        cache.set_cache("key", entry.clone(), &context).unwrap();
+        cache.set_cache("key", entry, &context).unwrap();
+        let text = requests_text(&requests);
+        assert_eq!(text.matches("FT.CREATE").count(), 1);
+        assert_eq!(text.matches("HSET").count(), 2);
+    }
+
+    #[test]
+    fn existing_index_dimension_must_match_embedding() {
+        let (cache, _, _) = cache_with_recording(
+            [already_exists(), Ok(info_dimension(2))],
+            vec![1.0, 0.0],
+            0.8,
+        );
+        cache
+            .set_cache(
+                "key",
+                CacheEntry {
+                    timestamp: None,
+                    response: json!({"answer": "ok"}),
+                },
+                &semantic_context(None),
+            )
+            .unwrap();
+
+        let (cache, _, _) = cache_with_recording(
+            [already_exists(), Ok(info_dimension(3))],
+            vec![1.0, 0.0],
+            0.8,
+        );
+        assert_eq!(
+            cache.set_cache(
+                "key",
+                CacheEntry {
+                    timestamp: None,
+                    response: json!({"answer": "ok"}),
+                },
+                &semantic_context(None),
+            ),
+            Err(super::Error::Unavailable)
+        );
+    }
+
+    #[test]
+    fn get_applies_threshold_and_decodes_entry() {
+        let entry = CacheEntry {
+            timestamp: Some(1.0),
+            response: json!({"answer": "ok"}),
+        };
+        let encoded = ResponseCacheCodec.encode(&entry).unwrap();
+        let (cache, _, _) = cache_with_recording(
+            [ok(), Ok(search_hit(encoded.clone(), "0.1"))],
+            vec![1.0, 0.0],
+            0.8,
+        );
+        assert_eq!(
+            cache.get_cache("key", &semantic_context(None)).unwrap(),
+            Some(entry)
+        );
+
+        let (cache, _, _) =
+            cache_with_recording([ok(), Ok(search_hit(encoded, "0.5"))], vec![1.0, 0.0], 0.8);
+        assert_eq!(
+            cache.get_cache("key", &semantic_context(None)).unwrap(),
+            None
+        );
+    }
+
+    #[test]
+    fn get_zero_docs_is_a_miss() {
+        let (cache, _, _) = cache_with_recording(
+            [ok(), Ok(redis::Value::Array(vec![redis::Value::Int(0)]))],
+            vec![1.0, 0.0],
+            0.8,
+        );
+        assert_eq!(
+            cache.get_cache("key", &semantic_context(None)).unwrap(),
+            None
+        );
+    }
+
+    #[rstest]
+    #[case(redis::Value::Array(vec![
+        redis::Value::Int(1),
+        redis::Value::BulkString(b"document".to_vec()),
+        redis::Value::Array(vec![
+            redis::Value::BulkString(b"vector_distance".to_vec()),
+            redis::Value::BulkString(b"0.1".to_vec()),
+        ]),
+    ]))]
+    #[case(redis::Value::Array(vec![
+        redis::Value::Int(1),
+        redis::Value::BulkString(b"document".to_vec()),
+        redis::Value::Array(vec![
+            redis::Value::BulkString(b"response".to_vec()),
+            redis::Value::BulkString(b"not-json".to_vec()),
+            redis::Value::BulkString(b"vector_distance".to_vec()),
+            redis::Value::BulkString(b"abc".to_vec()),
+        ]),
+    ]))]
+    fn malformed_entries_are_invalid(#[case] search: redis::Value) {
+        let (cache, _, _) = cache_with_recording([ok(), Ok(search)], vec![1.0, 0.0], 0.8);
+        assert_eq!(
+            cache.get_cache("key", &semantic_context(None)),
+            Err(super::Error::InvalidEntry)
+        );
+    }
+
+    #[test]
+    fn response_cache_turns_invalid_entries_into_misses() {
+        let (cache, _, _) = cache_with_recording(
+            [
+                ok(),
+                Ok(redis::Value::Array(vec![
+                    redis::Value::Int(1),
+                    redis::Value::BulkString(b"document".to_vec()),
+                    redis::Value::Array(vec![
+                        redis::Value::BulkString(b"response".to_vec()),
+                        redis::Value::BulkString(b"not-json".to_vec()),
+                        redis::Value::BulkString(b"vector_distance".to_vec()),
+                        redis::Value::BulkString(b"0.1".to_vec()),
+                    ]),
+                ])),
+            ],
+            vec![1.0, 0.0],
+            0.8,
+        );
+        let service = ResponseCache::new(Arc::new(cache));
+        let request = ResponseCacheRequest {
+            key: CacheKeyInput {
+                preset: Some("key".into()),
+                ..Default::default()
+            },
+            context: semantic_context(None),
+            ..ResponseCacheRequest::new(CacheKeyInput::default())
+        };
+        assert_eq!(service.lookup(&request, Duration::ZERO).unwrap(), None);
+    }
+
+    #[tokio::test]
+    async fn async_set_and_get_use_shared_document_helpers() {
+        let entry = CacheEntry {
+            timestamp: Some(1.0),
+            response: json!({"answer": "ok"}),
+        };
+        let encoded = ResponseCacheCodec.encode(&entry).unwrap();
+        let (cache, requests, calls) = cache_with_recording(
+            [ok(), ok(), ok(), Ok(search_hit(encoded, "0.1"))],
+            vec![1.0, 0.0],
+            0.8,
+        );
+        let context = semantic_context(Some(Duration::from_millis(1900)));
+        cache
+            .async_set_cache("key", entry.clone(), context.clone())
+            .await
+            .unwrap();
+        assert_eq!(
+            cache.async_get_cache("key", &context).await.unwrap(),
+            Some(entry)
+        );
+        let text = requests_text(&requests);
+        assert!(text.contains("FT.CREATE"));
+        assert!(text.contains("HSET"));
+        assert!(text.contains("EXPIRE"));
+        assert_eq!(calls.lock().unwrap().len(), 2);
+    }
 }
diff --git a/litellm-rust/crates/python-bridge/src/cache/config.rs b/litellm-rust/crates/python-bridge/src/cache/config.rs
index 7218805b8df..e074c5e2f5d 100644
--- a/litellm-rust/crates/python-bridge/src/cache/config.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/config.rs
@@ -276,25 +276,15 @@ fn project_redis(
 
     let client = backend.getattr("redis_client")?;
     let pool = client.getattr("connection_pool")?;
-    if !instance_class_is(&pool, "redis.connection", "ConnectionPool")? {
+    let Ok((resolved, is_tls)) = project_connection_pool(&pool)? else {
         return Ok(Err(UnsupportedCacheConfig::RedisConnection));
-    }
-    let resolved = pool.getattr("connection_kwargs")?.cast_into::<PyDict>()?;
+    };
     for key in ["credential_provider", "redis_connect_func"] {
         if has_value(&resolved, key)? {
             return Ok(Err(UnsupportedCacheConfig::RedisCredentials));
         }
     }
-    let connection_class = resolved
-        .get_item("connection_class")?
-        .unwrap_or(pool.getattr("connection_class")?);
-    let tls = if class_is(&connection_class, "redis.connection", "Connection")? {
-        None
-    } else if class_is(&connection_class, "redis.connection", "SSLConnection")? {
-        Some(project_tls(&resolved)?)
-    } else {
-        return Ok(Err(UnsupportedCacheConfig::RedisConnection));
-    };
+    let tls = is_tls.then(|| project_tls(&resolved)).transpose()?;
 
     let protocol = match optional_i64(&resolved, "protocol")?.unwrap_or(2) {
         2 => RedisProtocol::Resp2,
@@ -332,18 +322,9 @@ fn project_valkey_semantic(
 ) -> PyResult<Result<ValkeySemanticCacheConfig, UnsupportedCacheConfig>> {
     let client = backend.getattr("sync_client")?;
     let pool = client.getattr("connection_pool")?;
-    if !instance_class_is(&pool, "redis.connection", "ConnectionPool")? {
+    let Ok((resolved, _is_tls)) = project_connection_pool(&pool)? else {
         return Ok(Err(UnsupportedCacheConfig::RedisConnection));
-    }
-    let resolved = pool.getattr("connection_kwargs")?.cast_into::<PyDict>()?;
-    let connection_class = resolved
-        .get_item("connection_class")?
-        .unwrap_or(pool.getattr("connection_class")?);
-    if !class_is(&connection_class, "redis.connection", "Connection")?
-        && !class_is(&connection_class, "redis.connection", "SSLConnection")?
-    {
-        return Ok(Err(UnsupportedCacheConfig::RedisConnection));
-    }
+    };
     let connection = RedisConnectionConfig {
         host: required_string(&resolved, "host")?,
         port: u16::try_from(required_i64(&resolved, "port")?)
@@ -371,6 +352,27 @@ fn project_valkey_semantic(
     }))
 }
 
+#[inline(never)]
+fn project_connection_pool<'py>(
+    pool: &Bound<'py, PyAny>,
+) -> PyResult<Result<(Bound<'py, PyDict>, bool), UnsupportedCacheConfig>> {
+    if !instance_class_is(pool, "redis.connection", "ConnectionPool")? {
+        return Ok(Err(UnsupportedCacheConfig::RedisConnection));
+    }
+    let resolved = pool.getattr("connection_kwargs")?.cast_into::<PyDict>()?;
+    let connection_class = resolved
+        .get_item("connection_class")?
+        .unwrap_or(pool.getattr("connection_class")?);
+    let is_tls = if class_is(&connection_class, "redis.connection", "Connection")? {
+        false
+    } else if class_is(&connection_class, "redis.connection", "SSLConnection")? {
+        true
+    } else {
+        return Ok(Err(UnsupportedCacheConfig::RedisConnection));
+    };
+    Ok(Ok((resolved, is_tls)))
+}
+
 #[inline(never)]
 fn project_tls(values: &Bound<'_, PyDict>) -> PyResult<RedisTlsConfig> {
     Ok(RedisTlsConfig {
@@ -646,6 +648,40 @@ mod tests {
         });
     }
 
+    #[test]
+    fn projects_valkey_semantic_configuration() {
+        Python::initialize();
+        Python::attach(|py| {
+            let facade = facade(
+                py,
+                "pool = ConnectionPool()\n\
+                 pool.connection_class = Connection\n\
+                 pool.max_connections = 12\n\
+                 pool.connection_kwargs = {'host': 'cache.internal', 'port': 6390, 'db': 2}\n\
+                 client = SimpleNamespace(connection_pool=pool)\n\
+                 backend = SimpleNamespace(similarity_threshold=0.85, index_name='semantic_idx', embedding_model='text-embedding-3-small', sync_client=client)\n\
+                 facade = SimpleNamespace(type='valkey-semantic', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)",
+            );
+            let CacheConfigProjection::Native(config) =
+                NativeCacheConfig::project(&facade).unwrap()
+            else {
+                panic!("Valkey semantic cache should be supported");
+            };
+            let CacheBackendConfig::ValkeySemantic(valkey) = config.backend else {
+                panic!("expected Valkey semantic configuration");
+            };
+            assert_eq!(valkey.similarity_threshold, 0.85);
+            assert_eq!(valkey.index_name, "semantic_idx");
+            assert_eq!(valkey.embedding_model, "text-embedding-3-small");
+            assert_eq!(valkey.connection.host, "cache.internal");
+            assert_eq!(valkey.connection.port, 6390);
+            assert_eq!(valkey.connection.database, 2);
+            assert_eq!(valkey.connection.pool_size, 12);
+            assert_eq!(valkey.connection.protocol, RedisProtocol::Resp2);
+            assert!(valkey.connection.tls.is_none());
+        });
+    }
+
     #[test]
     fn dynamic_redis_auth_stays_on_python() {
         Python::initialize();
diff --git a/tests/test_litellm_rust/test_valkey_semantic_cache_native.py b/tests/test_litellm_rust/test_valkey_semantic_cache_native.py
index 00037e6e29f..dc4ede8ea30 100644
--- a/tests/test_litellm_rust/test_valkey_semantic_cache_native.py
+++ b/tests/test_litellm_rust/test_valkey_semantic_cache_native.py
@@ -1,4 +1,7 @@
+import hashlib
 import os
+import struct
+import time
 from collections.abc import Generator, Mapping
 from types import SimpleNamespace
 from typing import Final, cast
@@ -36,24 +39,32 @@ def index_name(valkey_url: str) -> Generator[str]:
         client.close()
 
 
-def _request() -> dict[str, object]:
+def _request(prompt: str = "semantic cache prompt") -> dict[str, object]:
     return {
         "key": {"preset": "key"},
-        "messages": [{"role": "user", "content": "semantic cache prompt"}],
+        "messages": [{"role": "user", "content": prompt}],
     }
 
 
-def _backend(url: str, index_name: str) -> ValkeySemanticCache:
+def _backend(
+    url: str,
+    index_name: str,
+    embeddings: Mapping[str, list[float]] | None = None,
+) -> ValkeySemanticCache:
+    vectors: Final = embeddings or {"semantic cache prompt": [1.0, 0.0]}
     backend: Final = ValkeySemanticCache(
         redis_url=url,
         similarity_threshold=0.8,
         index_name=index_name,
     )
-    backend._get_embedding = lambda prompt, metadata=None: [1.0, 0.0]
+
+    def embed(prompt: str, metadata: Mapping[str, object] | None = None) -> list[float]:
+        return vectors[prompt]
 
     async def async_embedding(prompt: str, metadata: dict[str, object] | None = None) -> list[float]:
-        return [1.0, 0.0]
+        return vectors[prompt]
 
+    backend._get_embedding = embed
     backend._get_async_embedding = async_embedding
     return backend
 
@@ -147,3 +158,122 @@ def test_batch_lookup_is_unsupported(
     binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
     with pytest.raises(NotImplementedError):
         binding.lookup_batch([_request()])
+
+
+def test_ttl_expiry(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    backend: Final = _backend(valkey_url, index_name)
+    handle: Final = _native._CacheTestHandle.valkey_semantic(valkey_url, 0.8, index_name, backend)
+    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    binding.store({**_request(), "ttl_seconds": 1.0}, {"answer": "expires"})
+    client: Final = redis.Redis.from_url(valkey_url)
+    documents: Final = list(client.scan_iter(f"{index_name}:*"))
+    assert len(documents) == 1
+    assert client.ttl(documents[0]) > 0
+    time.sleep(1.5)
+    assert binding.lookup(_request()) is None
+
+
+def test_no_ttl_is_persistent_and_python_reads_native_value(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    backend: Final = _backend(valkey_url, index_name)
+    handle: Final = _native._CacheTestHandle.valkey_semantic(valkey_url, 0.8, index_name, backend)
+    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    response: Final = {"answer": "persistent"}
+    binding.store(_request(), response)
+    client: Final = redis.Redis.from_url(valkey_url)
+    documents: Final = list(client.scan_iter(f"{index_name}:*"))
+    assert len(documents) == 1
+    assert client.ttl(documents[0]) == -1
+    cached: Final = cast(Mapping[str, object], backend.get_cache("key", messages=_request()["messages"]))
+    assert cached["response"] == response
+
+
+def test_below_threshold_misses_on_native_and_python(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    backend: Final = _backend(
+        valkey_url,
+        index_name,
+        {"prompt A": [1.0, 0.0], "prompt B": [0.0, 1.0]},
+    )
+    handle: Final = _native._CacheTestHandle.valkey_semantic(valkey_url, 0.8, index_name, backend)
+    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    binding.store(_request("prompt A"), {"answer": "A"})
+    assert binding.lookup(_request("prompt B")) is None
+    assert backend.get_cache("key", messages=_request("prompt B")["messages"]) is None
+
+
+def test_malformed_entry_is_a_miss_on_native_and_python(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    backend: Final = _backend(valkey_url, index_name)
+    client: Final = redis.Redis.from_url(valkey_url)
+    scope: Final = hashlib.sha256(b"key").hexdigest()
+    document: Final = f"{index_name}:{scope}:{uuid4().hex}"
+    client.hset(
+        document,
+        mapping={
+            "litellm_cache_key": scope,
+            "prompt": "semantic cache prompt",
+            "response": "not json",
+            "embedding": struct.pack("<2f", 1.0, 0.0),
+        },
+    )
+    handle: Final = _native._CacheTestHandle.valkey_semantic(valkey_url, 0.8, index_name, backend)
+    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    assert binding.lookup(_request()) is None
+    assert backend.get_cache("key", messages=_request()["messages"]) is None
+
+
+async def test_async_store_batch_and_lookup(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    backend: Final = _backend(
+        valkey_url,
+        index_name,
+        {"prompt A": [1.0, 0.0], "prompt B": [0.0, 1.0]},
+    )
+    handle: Final = _native._CacheTestHandle.valkey_semantic(valkey_url, 0.8, index_name, backend)
+    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    requests: Final = [_request("prompt A"), _request("prompt B")]
+    responses: Final = [{"answer": "A"}, {"answer": "B"}]
+    await binding.async_store_batch(requests, responses)
+    assert await binding.async_lookup(requests[0]) == responses[0]
+    assert await binding.async_lookup(requests[1]) == responses[1]
+
+
+def test_subclass_backend_falls_back_to_python(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    class Custom(ValkeySemanticCache):
+        pass
+
+    facade: Final = Cache(
+        type=LiteLLMCacheType.VALKEY_SEMANTIC,
+        redis_url=valkey_url,
+        similarity_threshold=0.8,
+        valkey_semantic_cache_index_name=index_name,
+    )
+    facade.cache = Custom(redis_url=valkey_url, similarity_threshold=0.8, index_name=index_name)
+    resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade))
+    assert resolver.resolve().kind == "python_callback"
+
+
+async def test_ping_maps_unsupported_native_operation_to_not_implemented(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    backend: Final = _backend(valkey_url, index_name)
+    handle: Final = _native._CacheTestHandle.valkey_semantic(valkey_url, 0.8, index_name, backend)
+    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    with pytest.raises(NotImplementedError):
+        await binding.ping()

From c7458bf5fe20bcbb7902fd259fd3b113b2b9e168 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:42:49 +0000
Subject: [PATCH 082/160] refactor(rust): build vault requests without local
 reassignment

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../secrets-hashicorp/src/secret_manager.rs   | 59 +++++++++++--------
 1 file changed, 34 insertions(+), 25 deletions(-)

diff --git a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs
index e4ca8c387c3..52cc143c1e3 100644
--- a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs
+++ b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs
@@ -9,11 +9,11 @@ use litellm_secrets_types::{
     BaseSecretManager, SecretValue, async_rotate_secret, validate_secret_name,
 };
 use moka::future::Cache;
-use reqwest::Client;
+use reqwest::{Client, Identity, RequestBuilder};
 use serde_json::{Value, json};
 use tokio::sync::Mutex;
 
-use crate::{Error, HashicorpVaultConfig};
+use crate::{Error, HashicorpVaultConfig, TlsCertAuth};
 
 const CACHE_CAPACITY: u64 = 200;
 
@@ -242,10 +242,10 @@ impl HashicorpVault {
                 return Ok(token);
             }
         };
-        let mut request = self.client.post(login_url).json(&body);
-        if let Some(namespace) = self.config.login_namespace() {
-            request = request.header("X-Vault-Namespace", namespace);
-        }
+        let request: RequestBuilder = with_namespace(
+            self.client.post(login_url).json(&body),
+            self.config.login_namespace(),
+        );
         let response: reqwest::Response = request.send().await?;
         if !response.status().is_success() {
             return Err(Error::LoginStatus {
@@ -305,24 +305,33 @@ impl BaseSecretManager for HashicorpVault {
 }
 
 fn client_for_config(config: &HashicorpVaultConfig) -> Result<Client, Error> {
-    let mut builder: reqwest::ClientBuilder = Client::builder();
-    if let Some(tls) = config.tls_cert.as_ref() {
-        let cert: Vec<u8> = std::fs::read(&tls.cert_path).map_err(|source| Error::TlsIdentity {
-            path: tls.cert_path.clone(),
-            message: source.to_string(),
-        })?;
-        let key: Vec<u8> = std::fs::read(&tls.key_path).map_err(|source| Error::TlsIdentity {
-            path: tls.key_path.clone(),
-            message: source.to_string(),
-        })?;
-        let identity: reqwest::Identity = reqwest::Identity::from_pem(
-            &[cert.as_slice(), key.as_slice()].concat(),
-        )
-        .map_err(|source| Error::TlsIdentity {
-            path: tls.cert_path.clone(),
-            message: source.to_string(),
-        })?;
-        builder = builder.identity(identity);
-    }
+    let builder: reqwest::ClientBuilder = match config.tls_cert.as_ref() {
+        Some(tls) => Client::builder().identity(identity_for(tls)?),
+        None => Client::builder(),
+    };
     builder.build().map_err(Error::Request)
 }
+
+fn with_namespace(request: RequestBuilder, namespace: Option<&str>) -> RequestBuilder {
+    match namespace {
+        Some(namespace) => request.header("X-Vault-Namespace", namespace),
+        None => request,
+    }
+}
+
+fn identity_for(tls: &TlsCertAuth) -> Result<Identity, Error> {
+    let cert: Vec<u8> = std::fs::read(&tls.cert_path).map_err(|source| Error::TlsIdentity {
+        path: tls.cert_path.clone(),
+        message: source.to_string(),
+    })?;
+    let key: Vec<u8> = std::fs::read(&tls.key_path).map_err(|source| Error::TlsIdentity {
+        path: tls.key_path.clone(),
+        message: source.to_string(),
+    })?;
+    Identity::from_pem(&[cert.as_slice(), key.as_slice()].concat()).map_err(|source| {
+        Error::TlsIdentity {
+            path: tls.cert_path.clone(),
+            message: source.to_string(),
+        }
+    })
+}

From a45be4f276e2628248026731dc4f8b7b014fef1a Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Mon, 21 Sep 2026 13:48:37 -0700
Subject: [PATCH 083/160] fix(proxy): let the config file win when reporting
 nested alerting sources

_nested_setting_source returned "db" whenever the stored row held a value,
without first asking whether the config file declares the same key. For a
config-owned alerting_args field that disagrees with the database, the
endpoint reported source "db" while the proxy actually serves the file's
value and rejects any write to it.

Config ownership is now checked first, matching SettingsStore.source and
the precedence the rest of the resolver applies. The source test set grows
a field that only the database sets, a field only the file sets, and a
stored empty list, so each reported source is discriminating.
---
 litellm/proxy/proxy_server.py                 |  6 +-
 .../proxy_server/test_routes_model_metrics.py | 69 +++++++++++++------
 2 files changed, 51 insertions(+), 24 deletions(-)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 5f0295f8208..7b5086f3e8b 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -15993,12 +15993,12 @@ def _nested_setting_source(
     field_name: str,
     field_default: JsonValue,
 ) -> FieldSource:
-    db_value: Final = db_values.get(field_name)
-    if db_value is not None and not (isinstance(db_value, list) and len(db_value) == 0):
-        return "db"
     parent_value: Final = settings.config_value(parent_key)
     if isinstance(parent_value, Mapping) and field_name in parent_value:
         return "config"
+    db_value: Final = db_values.get(field_name)
+    if db_value is not None and not (isinstance(db_value, list) and len(db_value) == 0):
+        return "db"
     return "default" if field_default is not None else "unset"
 
 
diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py
index f65db69ce89..c5287db5027 100644
--- a/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py
+++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py
@@ -186,20 +186,21 @@ def test_model_settings_method_not_allowed(client, auth_as):
 def test_alerting_settings_reports_sources(client, auth_as, monkeypatch):
     from litellm.proxy.config_resolvers import SettingsStore
 
+    db_alerting_args = {
+        "daily_report_frequency": 7,
+        "outage_alert_ttl": 99,
+        "region_outage_alert_ttl": [],
+    }
+
     pc = MagicMock()
     row = MagicMock()
-    row.param_value = {
-        "alerting_args": {
-            "daily_report_frequency": 7,
-            "report_check_interval": None,
-        }
-    }
+    row.param_value = {"alerting_args": db_alerting_args}
     pc.db.litellm_config.find_first = AsyncMock(return_value=row)
     monkeypatch.setattr(proxy_server, "prisma_client", pc)
 
     logging_obj = MagicMock()
     args_model = MagicMock()
-    args_model.model_dump = MagicMock(return_value={"daily_report_frequency": 7})
+    args_model.model_dump = MagicMock(return_value={"daily_report_frequency": 3})
     logging_obj.slack_alerting_instance.alerting_args = args_model
     monkeypatch.setattr(proxy_server, "proxy_logging_obj", logging_obj)
 
@@ -207,21 +208,10 @@ def test_alerting_settings_reports_sources(client, auth_as, monkeypatch):
     store.load_yaml(
         {
             "alerting": ["slack"],
-            "alerting_args": {
-                "daily_report_frequency": 3,
-                "report_check_interval": 300,
-            },
+            "alerting_args": {"daily_report_frequency": 3, "report_check_interval": 300},
         }
     )
-    store.apply_db_row(
-        "general_settings",
-        {
-            "alerting_args": {
-                "daily_report_frequency": 7,
-                "report_check_interval": None,
-            }
-        },
-    )
+    store.apply_db_row("general_settings", {"alerting_args": db_alerting_args})
     monkeypatch.setattr(proxy_server.proxy_config, "settings", store)
     monkeypatch.setattr(proxy_server, "general_settings", store)
 
@@ -230,12 +220,48 @@ def test_alerting_settings_reports_sources(client, auth_as, monkeypatch):
 
     assert response.status_code == 200
     by_name = {entry["field_name"]: entry for entry in response.json()}
+
     assert by_name["slack_alerting"]["source"] == "config"
-    assert by_name["daily_report_frequency"]["source"] == "db"
+    assert by_name["daily_report_frequency"]["source"] == "config"
     assert by_name["report_check_interval"]["source"] == "config"
+    assert by_name["outage_alert_ttl"]["source"] == "db"
+    assert by_name["region_outage_alert_ttl"]["source"] == "default"
     assert by_name["budget_alert_ttl"]["source"] == "default"
 
 
+def test_alerting_settings_reports_config_source_when_db_disagrees(client, auth_as, monkeypatch):
+    from litellm.proxy.config_resolvers import SettingsStore
+
+    db_alerting_args = {"daily_report_frequency": 7}
+
+    pc = MagicMock()
+    row = MagicMock()
+    row.param_value = {"alerting_args": db_alerting_args}
+    pc.db.litellm_config.find_first = AsyncMock(return_value=row)
+    monkeypatch.setattr(proxy_server, "prisma_client", pc)
+
+    logging_obj = MagicMock()
+    args_model = MagicMock()
+    args_model.model_dump = MagicMock(return_value={"daily_report_frequency": 3})
+    logging_obj.slack_alerting_instance.alerting_args = args_model
+    monkeypatch.setattr(proxy_server, "proxy_logging_obj", logging_obj)
+
+    store = SettingsStore("general_settings")
+    store.load_yaml({"alerting_args": {"daily_report_frequency": 3}})
+    store.apply_db_row("general_settings", {"alerting_args": db_alerting_args})
+    monkeypatch.setattr(proxy_server.proxy_config, "settings", store)
+    monkeypatch.setattr(proxy_server, "general_settings", store)
+
+    with auth_as(LitellmUserRoles.PROXY_ADMIN):
+        response = client.get("/alerting/settings")
+
+    assert response.status_code == 200
+    by_name = {entry["field_name"]: entry for entry in response.json()}
+    assert store.source("alerting_args") == "config"
+    assert by_name["daily_report_frequency"]["field_value"] == 3
+    assert by_name["daily_report_frequency"]["source"] == "config"
+
+
 @pytest.mark.parametrize("db_alerting_args", [None, []])
 def test_alerting_settings_handles_empty_db_args(
     client: TestClient,
@@ -268,6 +294,7 @@ def test_alerting_settings_handles_empty_db_args(
     assert response.status_code == 200
     by_name = {entry["field_name"]: entry for entry in response.json()}
     assert by_name["report_check_interval"]["source"] == "config"
+    assert by_name["budget_alert_ttl"]["source"] == "default"
 
 
 def test_alerting_settings_no_db_error(client, auth_as, no_prisma):

From cf8a211048368b8fea9f161dae5eb1b8ef4b2e30 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:52:03 +0000
Subject: [PATCH 084/160] test(rust): pin disk counter restart after a
 fractional value

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/cache-disk/tests/cache.rs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/litellm-rust/crates/cache-disk/tests/cache.rs b/litellm-rust/crates/cache-disk/tests/cache.rs
index 1e3e39db0c8..c6cd6006148 100644
--- a/litellm-rust/crates/cache-disk/tests/cache.rs
+++ b/litellm-rust/crates/cache-disk/tests/cache.rs
@@ -231,6 +231,12 @@ fn counters_use_atomic_native_values_and_ignore_invalid_initial_values() {
             .unwrap(),
         "real"
     );
+    assert_eq!(
+        cache
+            .increment_cache("counter", 1.0, ExactCacheContext::default())
+            .unwrap(),
+        1.0
+    );
 }
 
 #[test]

From f998ab53d5e68d46c05238bf8ff05d711ddf7085 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Mon, 21 Sep 2026 13:52:16 -0700
Subject: [PATCH 085/160] fix(proxy): treat a config-owned alerting_args as
 shadowing the stored row

When the config file declares alerting_args at all, the resolver hands the
file's dict to every reader and the stored row never reaches one. Reporting
a nested field as "db" because the row happens to carry it told the admin a
value was in effect that the proxy does not serve: a live proxy answered
source "db" for outage_alert_ttl while serving the default.

A config-owned parent now reports the field's own default, and the DB is
consulted only when the file leaves the parent alone.
---
 litellm/proxy/proxy_server.py                 |  5 +-
 .../proxy_server/test_routes_model_metrics.py | 53 +++++++++++++------
 2 files changed, 41 insertions(+), 17 deletions(-)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 7b5086f3e8b..d162ce2914e 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -15996,10 +15996,13 @@ def _nested_setting_source(
     parent_value: Final = settings.config_value(parent_key)
     if isinstance(parent_value, Mapping) and field_name in parent_value:
         return "config"
+    unset_source: Final[FieldSource] = "default" if field_default is not None else "unset"
+    if settings.owned_by_config(parent_key):
+        return unset_source
     db_value: Final = db_values.get(field_name)
     if db_value is not None and not (isinstance(db_value, list) and len(db_value) == 0):
         return "db"
-    return "default" if field_default is not None else "unset"
+    return unset_source
 
 
 @router.get(
diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py
index c5287db5027..3fb6e6fcb45 100644
--- a/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py
+++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py
@@ -183,37 +183,39 @@ def test_model_settings_method_not_allowed(client, auth_as):
 # ---------------------------------------------------------------------------
 
 
-def test_alerting_settings_reports_sources(client, auth_as, monkeypatch):
+def _alerting_client(monkeypatch, *, yaml_values, db_row, live_args):
     from litellm.proxy.config_resolvers import SettingsStore
 
-    db_alerting_args = {
-        "daily_report_frequency": 7,
-        "outage_alert_ttl": 99,
-        "region_outage_alert_ttl": [],
-    }
-
     pc = MagicMock()
     row = MagicMock()
-    row.param_value = {"alerting_args": db_alerting_args}
+    row.param_value = db_row
     pc.db.litellm_config.find_first = AsyncMock(return_value=row)
     monkeypatch.setattr(proxy_server, "prisma_client", pc)
 
     logging_obj = MagicMock()
     args_model = MagicMock()
-    args_model.model_dump = MagicMock(return_value={"daily_report_frequency": 3})
+    args_model.model_dump = MagicMock(return_value=live_args)
     logging_obj.slack_alerting_instance.alerting_args = args_model
     monkeypatch.setattr(proxy_server, "proxy_logging_obj", logging_obj)
 
     store = SettingsStore("general_settings")
-    store.load_yaml(
-        {
-            "alerting": ["slack"],
-            "alerting_args": {"daily_report_frequency": 3, "report_check_interval": 300},
-        }
-    )
-    store.apply_db_row("general_settings", {"alerting_args": db_alerting_args})
+    store.load_yaml(yaml_values)
+    store.apply_db_row("general_settings", db_row)
     monkeypatch.setattr(proxy_server.proxy_config, "settings", store)
     monkeypatch.setattr(proxy_server, "general_settings", store)
+    return store
+
+
+def test_alerting_settings_reports_sources(client, auth_as, monkeypatch):
+    _alerting_client(
+        monkeypatch,
+        yaml_values={
+            "alerting": ["slack"],
+            "alerting_args": {"daily_report_frequency": 3, "report_check_interval": 300},
+        },
+        db_row={"alerting_args": {"daily_report_frequency": 7, "outage_alert_ttl": 4242}},
+        live_args={"daily_report_frequency": 3},
+    )
 
     with auth_as(LitellmUserRoles.PROXY_ADMIN):
         response = client.get("/alerting/settings")
@@ -224,6 +226,25 @@ def test_alerting_settings_reports_sources(client, auth_as, monkeypatch):
     assert by_name["slack_alerting"]["source"] == "config"
     assert by_name["daily_report_frequency"]["source"] == "config"
     assert by_name["report_check_interval"]["source"] == "config"
+    assert by_name["outage_alert_ttl"]["source"] == "default"
+    assert by_name["budget_alert_ttl"]["source"] == "default"
+
+
+def test_alerting_settings_reports_db_source_when_the_file_omits_alerting_args(client, auth_as, monkeypatch):
+    store = _alerting_client(
+        monkeypatch,
+        yaml_values={"alerting": ["slack"]},
+        db_row={"alerting_args": {"outage_alert_ttl": 4242, "region_outage_alert_ttl": []}},
+        live_args={"outage_alert_ttl": 4242},
+    )
+
+    with auth_as(LitellmUserRoles.PROXY_ADMIN):
+        response = client.get("/alerting/settings")
+
+    assert response.status_code == 200
+    by_name = {entry["field_name"]: entry for entry in response.json()}
+
+    assert store.owned_by_config("alerting_args") is False
     assert by_name["outage_alert_ttl"]["source"] == "db"
     assert by_name["region_outage_alert_ttl"]["source"] == "default"
     assert by_name["budget_alert_ttl"]["source"] == "default"

From 25af094e27ba4b40ceabaccbae944528a807c3cf Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:52:28 +0000
Subject: [PATCH 086/160] fix(python-bridge): allow instance attributes to
 shadow class defaults

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../crates/python-bridge/src/cache/facade.rs  |  5 +-
 litellm/rust_bridge/_native.pyi               | 64 ++++++++++++++++++-
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/litellm-rust/crates/python-bridge/src/cache/facade.rs b/litellm-rust/crates/python-bridge/src/cache/facade.rs
index 58730857d60..d7ec2052dd0 100644
--- a/litellm-rust/crates/python-bridge/src/cache/facade.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/facade.rs
@@ -117,7 +117,10 @@ impl ObjectGuard {
                 return Ok(false);
             }
             for (name, value) in &expected.attributes {
-                if instance.contains(name)? || !attributes.get_item(name)?.is(value.bind(py)) {
+                if !attributes.get_item(name)?.is(value.bind(py)) {
+                    return Ok(false);
+                }
+                if instance.contains(name)? && value.bind(py).is_callable() {
                     return Ok(false);
                 }
             }
diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi
index 05a6df6d5af..7eb266d5a09 100644
--- a/litellm/rust_bridge/_native.pyi
+++ b/litellm/rust_bridge/_native.pyi
@@ -1,6 +1,6 @@
 from asyncio import Future
 from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
-from typing import Never, final
+from typing import Literal, Never, final
 
 from litellm.llms.base_llm.ocr.transformation import OCRResponse
 from litellm.rust_bridge.messages.entrypoints import LiteLLMMessagesRequest
@@ -93,6 +93,68 @@ class ResponsesWebSocketConnection:
     def recv_text(self) -> Future[str | None]: ...
     def close(self) -> Future[None]: ...
 
+@final
+class _CacheTestHandle:
+    def __new__(cls, _uninstantiable: Never, /) -> Never: ...
+    @staticmethod
+    def memory(
+        *,
+        capacity: int = 200,
+        ttl_seconds: float = 600.0,
+        max_entry_bytes: int = 1048576,
+    ) -> _CacheTestHandle: ...
+    @staticmethod
+    def redis(
+        url: str,
+        *,
+        ttl_seconds: float = 60.0,
+        namespace: str | None = None,
+    ) -> _CacheTestHandle: ...
+    @staticmethod
+    def redis_semantic(backend: object) -> _CacheTestHandle: ...
+    @property
+    def backend(self) -> Literal["memory", "redis", "redis_semantic"]: ...
+    def _bind_facade(self, facade: object) -> None: ...
+
+@final
+class _CacheTestBinding:
+    def __new__(cls, _uninstantiable: Never, /) -> Never: ...
+    @property
+    def kind(self) -> Literal["disabled", "native", "python_callback"]: ...
+    def lookup(
+        self, request: object, *, callback_kwargs: object = None
+    ) -> object: ...
+    def store(
+        self, request: object, response: object, *, callback_kwargs: object = None
+    ) -> None: ...
+    def lookup_batch(
+        self, requests: object, *, callback_kwargs: object = None
+    ) -> object: ...
+    def async_lookup(
+        self, request: object, *, callback_kwargs: object = None
+    ) -> Future[object]: ...
+    def async_store(
+        self, request: object, response: object, *, callback_kwargs: object = None
+    ) -> Future[object]: ...
+    def async_lookup_batch(
+        self, requests: object, *, callback_kwargs: object = None
+    ) -> Future[object]: ...
+    def async_store_batch(
+        self,
+        requests: object,
+        responses: object,
+        *,
+        callback_result: object = None,
+        callback_kwargs: object = None,
+    ) -> Future[object]: ...
+    def async_flush(self) -> Future[object]: ...
+    def ping(self) -> Future[object]: ...
+
+@final
+class _CacheTestResolver:
+    def __new__(cls, namespace: object) -> _CacheTestResolver: ...
+    def resolve(self) -> _CacheTestBinding: ...
+
 @final
 class TokenCounter:
     def __new__(cls, tokenizer_json: str) -> TokenCounter: ...

From 4a0151f77fc3d8b68606a59171412fbe55b4015e Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:52:28 +0000
Subject: [PATCH 087/160] test(rust): add redis-semantic native parity fixtures

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 tests/test_litellm_rust/test_cache.py | 549 ++++++++++++++++++++++++--
 1 file changed, 518 insertions(+), 31 deletions(-)

diff --git a/tests/test_litellm_rust/test_cache.py b/tests/test_litellm_rust/test_cache.py
index c35cb1a20fb..e9f4b99a3b7 100644
--- a/tests/test_litellm_rust/test_cache.py
+++ b/tests/test_litellm_rust/test_cache.py
@@ -1,14 +1,19 @@
 import asyncio
 import contextvars
 import gc
+import hashlib
 import json
+import math
+import os
 import threading
 import time
 import weakref
-from collections.abc import Generator
+from collections.abc import Callable, Generator
+from contextlib import ExitStack
 from types import SimpleNamespace
 from typing import Final, Protocol, cast
 from urllib.parse import urlparse
+from uuid import uuid4
 
 import fakeredis
 import pytest
@@ -17,10 +22,16 @@ import redis
 import litellm
 from litellm.caching.caching import Cache, disable_cache, enable_cache, update_cache
 from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.caching.redis_semantic_cache import RedisSemanticCache
 from litellm.rust_bridge import _native
 from litellm.types.caching import LiteLLMCacheType
+from litellm.types.llms.custom_llm import CustomLLMItem
+from litellm.types.utils import EmbeddingResponse
 from tests.test_litellm_rust.support.isolation import rebound
 
+_CacheTestHandle: Final = _native._CacheTestHandle  # pyright: ignore[reportPrivateUsage]  # test-only handle has no public module name
+_CacheTestResolver: Final = _native._CacheTestResolver  # pyright: ignore[reportPrivateUsage]  # test-only resolver has no public module name
+
 pytestmark: Final = pytest.mark.requires_rust_extension
 
 
@@ -50,14 +61,14 @@ def test_existing_constructor_and_global_are_unchanged() -> None:
     assert type(facade.cache) is InMemoryCache
     assert "_native_cache_handle" not in vars(facade)
     with rebound(litellm, "cache", facade):
-        resolver: Final = _native._CacheTestResolver(litellm)
+        resolver: Final = _CacheTestResolver(litellm)
         assert resolver.resolve().kind == "python_callback"
         resolver.resolve().store(None, {"answer": 7}, callback_kwargs={"cache_key": "key"})
         assert cast(CacheLookup, facade).get_cache(cache_key="key") == {"answer": 7}
 
 
 def test_existing_global_lifecycle_remains_the_resolver_source_of_truth() -> None:
-    resolver: Final = _native._CacheTestResolver(litellm)
+    resolver: Final = _CacheTestResolver(litellm)
 
     enable_cache(type=LiteLLMCacheType.LOCAL, ttl=30)
     enabled: Final = litellm.cache
@@ -80,13 +91,13 @@ def test_existing_global_lifecycle_remains_the_resolver_source_of_truth() -> Non
 
 
 async def test_native_bindings_survive_replacement_and_capture_writes_before_dispatch() -> None:
-    namespace: Final = SimpleNamespace(cache=_native._CacheTestHandle.memory())
-    resolver: Final = _native._CacheTestResolver(namespace)
+    namespace: Final = SimpleNamespace(cache=_CacheTestHandle.memory())
+    resolver: Final = _CacheTestResolver(namespace)
     selected: Final = resolver.resolve()
     assert selected.kind == "native"
     selected.store(request(), {"answer": 1})
     assert await selected.async_lookup(request()) == {"answer": 1}
-    with rebound(namespace, "cache", _native._CacheTestHandle.memory()):
+    with rebound(namespace, "cache", _CacheTestHandle.memory()):
         replacement: Final = resolver.resolve()
         await selected.async_store(request(), {"answer": 2})
         assert replacement.lookup(request()) is None
@@ -119,7 +130,7 @@ async def test_python_callback_preserves_identity_caller_task_context_and_errors
             raise failure
 
     namespace: Final = SimpleNamespace(cache=CustomCache())
-    binding: Final = _native._CacheTestResolver(namespace).resolve()
+    binding: Final = _CacheTestResolver(namespace).resolve()
     assert binding.kind == "python_callback"
     assert await binding.async_lookup(None, callback_kwargs={"marker": sentinel}) is sentinel
     assert context.get() == "callback"
@@ -140,7 +151,7 @@ async def test_callback_cancellation_stays_in_the_callers_task() -> None:
             finally:
                 finished.set()
 
-    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=CustomCache())).resolve()
+    binding: Final = _CacheTestResolver(SimpleNamespace(cache=CustomCache())).resolve()
 
     async def lookup() -> object:
         return await binding.async_lookup(None, callback_kwargs={})
@@ -155,9 +166,9 @@ async def test_callback_cancellation_stays_in_the_callers_task() -> None:
 
 def test_registered_facade_uses_native_and_instance_overrides_fall_back() -> None:
     facade: Final = Cache(type=LiteLLMCacheType.LOCAL)
-    handle: Final = _native._CacheTestHandle.memory()
+    handle: Final = _CacheTestHandle.memory()
     handle._bind_facade(facade)
-    resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade))
+    resolver: Final = _CacheTestResolver(SimpleNamespace(cache=facade))
     native: Final = resolver.resolve()
     assert native.kind == "native"
     native.store(request(), {"source": "native"})
@@ -188,12 +199,12 @@ def test_facade_subclasses_backend_replacement_and_configuration_changes_are_not
     class CustomCache(Cache):
         pass
 
-    handle: Final = _native._CacheTestHandle.memory()
+    handle: Final = _CacheTestHandle.memory()
     with pytest.raises(TypeError):
         handle._bind_facade(CustomCache(type=LiteLLMCacheType.LOCAL))
     facade: Final = Cache(type=LiteLLMCacheType.LOCAL)
     handle._bind_facade(facade)
-    resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade))
+    resolver: Final = _CacheTestResolver(SimpleNamespace(cache=facade))
     with rebound(facade, "cache", InMemoryCache()):
         assert resolver.resolve().kind == "python_callback"
     with rebound(facade, "ttl", 12):
@@ -218,7 +229,7 @@ def test_resolver_and_callback_cycles_can_be_collected() -> None:
     def cyclic_reference() -> weakref.ReferenceType[CustomCache]:
         callback: Final = CustomCache()
         namespace: Final = SimpleNamespace(cache=callback)
-        binding: Final = _native._CacheTestResolver(namespace).resolve()
+        binding: Final = _CacheTestResolver(namespace).resolve()
         setattr(callback, "binding", binding)
         return weakref.ref(callback)
 
@@ -229,8 +240,8 @@ def test_resolver_and_callback_cycles_can_be_collected() -> None:
 
 async def test_redis_reads_python_sync_and_async_entries_and_writes_without_hidden_prefix(redis_url: str) -> None:
     client: Final = redis.Redis.from_url(redis_url)
-    namespace: Final = SimpleNamespace(cache=_native._CacheTestHandle.redis(redis_url, namespace="team"))
-    binding: Final = _native._CacheTestResolver(namespace).resolve()
+    namespace: Final = SimpleNamespace(cache=_CacheTestHandle.redis(redis_url, namespace="team"))
+    binding: Final = _CacheTestResolver(namespace).resolve()
     response: Final = {"choices": [{"text": "cached"}], "usage": {"total_tokens": 3}, "flag": True, "empty": None}
     envelope: Final = {"timestamp": time.time(), "response": json.dumps(response)}
     client.set("team:sync", str(envelope))
@@ -252,33 +263,33 @@ async def test_redis_reads_python_sync_and_async_entries_and_writes_without_hidd
 
 
 def test_invalid_duration_and_request_shape_fail_before_storage() -> None:
-    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=_native._CacheTestHandle.memory())).resolve()
+    binding: Final = _CacheTestResolver(SimpleNamespace(cache=_CacheTestHandle.memory())).resolve()
     for seconds in (-1.0, float("nan"), float("inf")):
         with pytest.raises(ValueError, match="cache durations must be finite and nonnegative"):
             binding.store({**request(), "ttl_seconds": seconds}, {"answer": 1})
     assert binding.lookup(request()) is None
     with pytest.raises(ValueError, match="cache durations must be finite and nonnegative"):
-        _native._CacheTestHandle.memory(ttl_seconds=-1)
+        _CacheTestHandle.memory(ttl_seconds=-1)
 
 
 async def test_memory_size_policy_is_applied_by_the_native_host() -> None:
-    handle: Final = _native._CacheTestHandle.memory(capacity=2, max_entry_bytes=128)
-    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    handle: Final = _CacheTestHandle.memory(capacity=2, max_entry_bytes=128)
+    binding: Final = _CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
     small: Final = {"answer": "ok"}
     binding.store(request("small"), small)
     assert await binding.async_lookup(request("small")) == small
     await binding.async_store(request("large"), {"answer": "x" * 256})
     assert binding.lookup(request("large")) is None
     assert binding.lookup(request("small")) == small
-    disabled: Final = _native._CacheTestResolver(
-        SimpleNamespace(cache=_native._CacheTestHandle.memory(capacity=0))
+    disabled: Final = _CacheTestResolver(
+        SimpleNamespace(cache=_CacheTestHandle.memory(capacity=0))
     ).resolve()
     await disabled.async_store(request(), small)
     assert await disabled.async_lookup(request()) is None
 
 
 async def test_native_batch_lookup_and_store_report_partial_hits() -> None:
-    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=_native._CacheTestHandle.memory())).resolve()
+    binding: Final = _CacheTestResolver(SimpleNamespace(cache=_CacheTestHandle.memory())).resolve()
     requests: Final = [request("hit"), request("miss"), request("disabled")]
     requests[2]["controls"] = {
         "supported_call_type": True,
@@ -316,7 +327,7 @@ async def test_python_batch_callbacks_use_the_builtin_cache_api() -> None:
         ) -> object:
             return result, kwargs
 
-    binding: Final = _native._CacheTestResolver(
+    binding: Final = _CacheTestResolver(
         SimpleNamespace(cache=CustomCache(type=LiteLLMCacheType.LOCAL))
     ).resolve()
     assert binding.kind == "python_callback"
@@ -346,7 +357,7 @@ async def test_unmodified_builtin_cache_callbacks_can_ping_and_flush() -> None:
 
     cache: Final = Cache(type=LiteLLMCacheType.LOCAL)
     cache.cache.set_cache("key", "value")
-    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=cache)).resolve()
+    binding: Final = _CacheTestResolver(SimpleNamespace(cache=cache)).resolve()
     assert binding.kind == "python_callback"
 
     setattr(cache.cache, "ping", ping)
@@ -358,7 +369,7 @@ async def test_unmodified_builtin_cache_callbacks_can_ping_and_flush() -> None:
 def test_facade_registration_rejects_mismatched_capacity() -> None:
     facade: Final = Cache(type=LiteLLMCacheType.LOCAL)
     with pytest.raises(TypeError, match="capacities must match"):
-        _native._CacheTestHandle.memory(capacity=7)._bind_facade(facade)
+        _CacheTestHandle.memory(capacity=7)._bind_facade(facade)
 
 
 async def test_redis_facade_buffers_native_async_writes(redis_url: str) -> None:
@@ -371,19 +382,19 @@ async def test_redis_facade_buffers_native_async_writes(redis_url: str) -> None:
             redis_flush_size=2,
         )
         with pytest.raises(TypeError, match="default TTLs must match"):
-            _native._CacheTestHandle.redis(redis_url, ttl_seconds=61)._bind_facade(facade)
+            _CacheTestHandle.redis(redis_url, ttl_seconds=61)._bind_facade(facade)
         with pytest.raises(TypeError, match="namespaces must match"):
-            _native._CacheTestHandle.redis(redis_url, namespace="other")._bind_facade(facade)
-        _native._CacheTestHandle.redis(redis_url, ttl_seconds=60)._bind_facade(facade)
-    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)).resolve()
+            _CacheTestHandle.redis(redis_url, namespace="other")._bind_facade(facade)
+        _CacheTestHandle.redis(redis_url, ttl_seconds=60)._bind_facade(facade)
+    binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve()
     client: Final = redis.Redis.from_url(redis_url)
 
     with rebound(facade.cache, "redis_kwargs", {**facade.cache.redis_kwargs, "ssl": True}):
-        assert _native._CacheTestResolver(SimpleNamespace(cache=facade)).resolve().kind == "python_callback"
+        assert _CacheTestResolver(SimpleNamespace(cache=facade)).resolve().kind == "python_callback"
 
     pool: Final = facade.cache.redis_client.connection_pool
     with rebound(pool, "connection_kwargs", {**pool.connection_kwargs, "db": 1}):
-        assert _native._CacheTestResolver(SimpleNamespace(cache=facade)).resolve().kind == "python_callback"
+        assert _CacheTestResolver(SimpleNamespace(cache=facade)).resolve().kind == "python_callback"
 
     await binding.async_store(request("first"), {"value": 1})
     assert client.get("first") is None
@@ -393,3 +404,479 @@ async def test_redis_facade_buffers_native_async_writes(redis_url: str) -> None:
     assert client.get("second") is not None
     await facade.cache.disconnect()
     client.close()
+
+
+PARAPHRASE_MARKER: Final = " (paraphrase)"
+SEMANTIC_EMBEDDING_MODEL: Final = "semantic-test/deterministic"
+SEMANTIC_INDEX_PREFIX: Final = "litellm_test_semantic_"
+
+
+def _normalized(vector: list[float]) -> list[float]:
+    norm: Final = math.sqrt(sum(component * component for component in vector))
+    return [component / norm for component in vector]
+
+
+def _base_embedding(prompt: str) -> list[float]:
+    digest: Final = hashlib.sha256(prompt.encode("utf-8")).digest()
+    return _normalized([float(digest[index] + 1) for index in range(8)])
+
+
+def _semantic_embedding(prompt: str) -> list[float]:
+    if PARAPHRASE_MARKER not in prompt:
+        return _base_embedding(prompt)
+    base: Final = _base_embedding(prompt.replace(PARAPHRASE_MARKER, "").strip())
+    pivot: Final = min(range(8), key=lambda index: abs(base[index]))
+    direction: Final = _normalized(
+        [
+            (1.0 - base[pivot] * base[pivot]) if index == pivot else -base[index] * base[pivot]
+            for index in range(8)
+        ]
+    )
+    # Rotating an orthogonal unit direction by 0.329 produces ~0.05 cosine distance
+    return _normalized([base[index] + 0.329 * direction[index] for index in range(8)])
+
+
+class DeterministicEmbedding(litellm.CustomLLM):
+    def __init__(self) -> None:
+        self.calls: list[dict[str, object]] = []
+
+    def _respond(
+        self,
+        model: str,
+        input: object,
+        model_response: EmbeddingResponse,
+    ) -> EmbeddingResponse:
+        texts: Final = cast(list[object], input if isinstance(input, list) else [input])
+        self.calls.append({"model": model, "input": texts})
+        model_response.model = model
+        model_response.data = [
+            {"object": "embedding", "index": index, "embedding": _semantic_embedding(str(text))}
+            for index, text in enumerate(texts)
+        ]
+        return model_response
+
+    def embedding(
+        self,
+        model: str,
+        input: list[object],
+        model_response: EmbeddingResponse,
+        print_verbose: Callable[..., object],
+        logging_obj: object,
+        optional_params: dict[str, object],
+        api_key: object = None,
+        api_base: object = None,
+        timeout: object = None,
+        litellm_params: object = None,
+    ) -> EmbeddingResponse:
+        return self._respond(model, input, model_response)
+
+    async def aembedding(
+        self,
+        model: str,
+        input: list[object],
+        model_response: EmbeddingResponse,
+        print_verbose: Callable[..., object],
+        logging_obj: object,
+        optional_params: dict[str, object],
+        api_key: object = None,
+        api_base: object = None,
+        timeout: object = None,
+        litellm_params: object = None,
+    ) -> EmbeddingResponse:
+        return self._respond(model, input, model_response)
+
+
+@pytest.fixture
+def semantic_embedding() -> Generator[DeterministicEmbedding]:
+    handler: Final = DeterministicEmbedding()
+    with ExitStack() as stack:
+        stack.enter_context(
+            rebound(
+                litellm,
+                "custom_provider_map",
+                [
+                    *litellm.custom_provider_map,
+                    cast(
+                        CustomLLMItem,
+                        {"provider": "semantic-test", "custom_handler": handler},
+                    ),
+                ],
+            )
+        )
+        stack.enter_context(
+            rebound(
+                litellm,
+                "_custom_providers",  # pyright: ignore[reportPrivateUsage]  # no public provider-registration hook
+                [*litellm._custom_providers, "semantic-test"],  # pyright: ignore[reportPrivateUsage]  # no public provider-registration hook
+            )
+        )
+        stack.enter_context(
+            rebound(litellm, "provider_list", [*litellm.provider_list, "semantic-test"])
+        )
+        yield handler
+
+
+@pytest.fixture
+def redis_stack() -> Generator[tuple[str, str]]:
+    url: Final = os.environ.get("LITELLM_REDIS_STACK_URL")
+    if url is None:
+        pytest.skip("LITELLM_REDIS_STACK_URL is not set")
+    index: Final = f"{SEMANTIC_INDEX_PREFIX}{uuid4().hex}"
+    yield url, index
+    client: Final = redis.Redis.from_url(url)
+    try:
+        client.execute_command("FT.DROPINDEX", index, "DD")  # pyright: ignore[reportUnknownMemberType]  # redis-py leaves execute_command partially unknown
+    except redis.RedisError:
+        pass
+    client.close()
+
+
+def semantic_request(key: str, prompt: str, **extra: object) -> dict[str, object]:
+    return {
+        "key": {"preset": key},
+        "messages": [{"role": "user", "content": prompt}],
+        **extra,
+    }
+
+
+def semantic_messages(prompt: str) -> list[dict[str, object]]:
+    return [{"role": "user", "content": prompt}]
+
+
+def semantic_entry_id(prompt: str, tag: str) -> str:
+    return hashlib.sha256(f"{prompt}litellm_cache_key{tag}".encode()).hexdigest()
+
+
+def semantic_facade(url: str, index: str, *, similarity_threshold: float = 0.8) -> Cache:
+    facade: Final = Cache(
+        type=LiteLLMCacheType.REDIS_SEMANTIC,
+        redis_url=url,
+        similarity_threshold=similarity_threshold,
+        redis_semantic_cache_embedding_model=SEMANTIC_EMBEDDING_MODEL,
+        redis_semantic_cache_index_name=index,
+    )
+    _CacheTestHandle.redis_semantic(facade.cache)._bind_facade(facade)
+    return facade
+
+
+def test_redis_semantic_constructor_identity_and_provenance(
+    redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding
+) -> None:
+    url, index = redis_stack
+    facade: Final = semantic_facade(url, index)
+    backend: Final = cast(RedisSemanticCache, facade.cache)
+    assert backend.__class__.__module__ == "litellm.caching.redis_semantic_cache"
+    assert type(backend) is RedisSemanticCache
+    assert backend._redis_url == url  # pyright: ignore[reportPrivateUsage]  # provenance check needs the projected config
+    assert backend._index_name == index  # pyright: ignore[reportPrivateUsage]  # provenance check needs the projected config
+    assert backend.similarity_threshold == 0.8
+    assert backend.embedding_model == SEMANTIC_EMBEDDING_MODEL
+    handle: Final = cast(object, getattr(facade, "_native_cache_handle"))
+    assert isinstance(handle, _CacheTestHandle)
+    assert handle.backend == "redis_semantic"
+    binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve()
+    assert binding.kind == "native"
+
+
+def test_redis_semantic_native_and_python_sync_entries_share_one_layout(
+    redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding
+) -> None:
+    url, index = redis_stack
+    facade: Final = semantic_facade(url, index)
+    binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve()
+    client: Final = redis.Redis.from_url(url)
+    response: Final = {"choices": [{"text": "paris"}], "usage": {"total_tokens": 2}}
+
+    binding.store(semantic_request("geo", "what is the capital of france"), response)
+
+    native_hash_key: Final = f"{index}:{semantic_entry_id('what is the capital of france', 'geo')}"
+    stored: Final = client.hgetall(native_hash_key)
+    assert set(stored) == {
+        b"entry_id",
+        b"prompt",
+        b"response",
+        b"prompt_vector",
+        b"inserted_at",
+        b"updated_at",
+        b"litellm_cache_key",
+    }, stored
+    assert stored[b"entry_id"].decode() == native_hash_key.split(":", 1)[1]
+    assert stored[b"prompt"] == b"what is the capital of france"
+    assert stored[b"litellm_cache_key"] == b"geo"
+    assert len(stored[b"prompt_vector"]) == 32
+    decoded: Final = cast(dict[str, object], json.loads(stored[b"response"]))
+    assert decoded["response"] == response
+    assert (
+        cast(RedisSemanticCache, facade.cache).get_cache(  # pyright: ignore[reportUnknownMemberType]  # **kwargs stays unknown on the backend class
+            "geo", messages=semantic_messages("what is the capital of france")
+        )
+        == decoded
+    )
+    assert semantic_embedding.calls == [
+        {"model": "deterministic", "input": ["what is the capital of france"]},
+        {"model": "deterministic", "input": ["what is the capital of france"]},
+        {"model": "deterministic", "input": ["dimension test"]},
+    ]
+
+    cast(RedisSemanticCache, facade.cache).set_cache(  # pyright: ignore[reportUnknownMemberType]  # **kwargs stays unknown on the backend class
+        "math",
+        json.dumps({"timestamp": 1700000000.0, "response": {"answer": 42}}),
+        messages=semantic_messages("what is 6 times 7"),
+    )
+    python_hash_key: Final = f"{index}:{semantic_entry_id('what is 6 times 7', 'math')}"
+    assert json.loads(cast(bytes, client.hget(python_hash_key, "response"))) == {
+        "timestamp": 1700000000.0,
+        "response": {"answer": 42},
+    }
+    assert binding.lookup(semantic_request("math", "what is 6 times 7")) == {"answer": 42}
+    client.close()
+
+
+async def test_redis_semantic_async_paths_and_store_batch_share_one_layout(
+    redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding
+) -> None:
+    url, index = redis_stack
+    facade: Final = semantic_facade(url, index)
+    binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve()
+    client: Final = redis.Redis.from_url(url)
+
+    await binding.async_store(
+        semantic_request("async", "name a primary color"), {"answer": "blue"}
+    )
+    hash_key: Final = f"{index}:{semantic_entry_id('name a primary color', 'async')}"
+    decoded: Final = cast(dict[str, object], json.loads(cast(bytes, client.hget(hash_key, "response"))))
+    python_read: Final = await cast(RedisSemanticCache, facade.cache).async_get_cache(  # pyright: ignore[reportUnknownMemberType]  # **kwargs stays unknown on the backend class
+        "async", messages=semantic_messages("name a primary color")
+    )
+    assert python_read == decoded
+
+    await binding.async_store_batch(
+        [
+            semantic_request("batch-one", "first batch prompt"),
+            semantic_request("batch-two", "second batch prompt"),
+        ],
+        [{"answer": 1}, {"answer": 2}],
+    )
+    expected: Final = {
+        key: json.loads(
+            cast(bytes, client.hget(f"{index}:{semantic_entry_id(prompt, key)}", "response"))
+        )
+        for key, prompt in (
+            ("batch-one", "first batch prompt"),
+            ("batch-two", "second batch prompt"),
+        )
+    }
+    for key, prompt in (
+        ("batch-one", "first batch prompt"),
+        ("batch-two", "second batch prompt"),
+    ):
+        assert cast(RedisSemanticCache, facade.cache).get_cache(  # pyright: ignore[reportUnknownMemberType]  # **kwargs stays unknown on the backend class
+            key, messages=semantic_messages(prompt)
+        ) == expected[key], key
+
+    cast(RedisSemanticCache, facade.cache).set_cache(  # pyright: ignore[reportUnknownMemberType]  # **kwargs stays unknown on the backend class
+        "async-python",
+        json.dumps({"timestamp": 1700000000.0, "response": {"answer": "python"}}),
+        messages=semantic_messages("python written prompt"),
+    )
+    assert await binding.async_lookup(
+        semantic_request("async-python", "python written prompt")
+    ) == {"answer": "python"}
+    client.close()
+
+
+def test_redis_semantic_similarity_tag_and_threshold_boundaries(
+    redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding
+) -> None:
+    url, index = redis_stack
+    facade: Final = semantic_facade(url, index)
+    binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve()
+
+    binding.store(semantic_request("sim", "tell me a joke"), {"answer": "haha"})
+    paraphrase: Final = f"tell me a joke{PARAPHRASE_MARKER}"
+    assert binding.lookup(semantic_request("sim", paraphrase)) == {"answer": "haha"}
+    assert binding.lookup(semantic_request("sim", "an unrelated question about spreadsheets")) is None
+    assert binding.lookup(semantic_request("other-key", "tell me a joke")) is None
+
+    strict: Final = semantic_facade(url, index, similarity_threshold=0.99)
+    strict_binding: Final = _CacheTestResolver(SimpleNamespace(cache=strict)).resolve()
+    assert strict_binding.lookup(semantic_request("sim", paraphrase)) is None
+    assert strict_binding.lookup(semantic_request("sim", "tell me a joke")) == {"answer": "haha"}
+
+
+def test_redis_semantic_ttl_is_written_only_when_requested(
+    redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding
+) -> None:
+    url, index = redis_stack
+    facade: Final = semantic_facade(url, index)
+    binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve()
+    client: Final = redis.Redis.from_url(url)
+
+    binding.store(
+        {**semantic_request("ttl", "ttl prompt"), "ttl_seconds": 12.0}, {"answer": 1}
+    )
+    expiring: Final = f"{index}:{semantic_entry_id('ttl prompt', 'ttl')}"
+    assert 0 < client.ttl(expiring) <= 12
+
+    binding.store(semantic_request("ttl-none", "untimed prompt"), {"answer": 2})
+    persistent: Final = f"{index}:{semantic_entry_id('untimed prompt', 'ttl-none')}"
+    assert client.ttl(persistent) == -1
+
+    binding.store(
+        {**semantic_request("ttl-fraction", "fractional prompt"), "ttl_seconds": 1.5},
+        {"answer": 3},
+    )
+    fractional: Final = f"{index}:{semantic_entry_id('fractional prompt', 'ttl-fraction')}"
+    assert client.ttl(fractional) == 2
+    client.close()
+
+
+def test_redis_semantic_malformed_response_is_a_miss_for_both_readers(
+    redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding
+) -> None:
+    url, index = redis_stack
+    facade: Final = semantic_facade(url, index)
+    binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve()
+    client: Final = redis.Redis.from_url(url)
+
+    binding.store(semantic_request("bad", "corrupt me"), {"answer": 1})
+    hash_key: Final = f"{index}:{semantic_entry_id('corrupt me', 'bad')}"
+    client.hset(hash_key, "response", b"{not json")
+    assert binding.lookup(semantic_request("bad", "corrupt me")) is None
+    assert (
+        cast(RedisSemanticCache, facade.cache).get_cache(  # pyright: ignore[reportUnknownMemberType]  # **kwargs stays unknown on the backend class
+            "bad", messages=semantic_messages("corrupt me")
+        )
+        is None
+    )
+    client.close()
+
+
+async def test_redis_semantic_unsupported_operations_raise_not_implemented(
+    redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding
+) -> None:
+    url, index = redis_stack
+    facade: Final = semantic_facade(url, index)
+    binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve()
+
+    with pytest.raises(NotImplementedError):
+        binding.lookup_batch([semantic_request("batch", "prompt one")])
+    with pytest.raises(NotImplementedError):
+        await binding.async_lookup_batch([semantic_request("batch", "prompt one")])
+    with pytest.raises(NotImplementedError):
+        await binding.async_flush()
+    with pytest.raises(NotImplementedError):
+        await binding.ping()
+
+
+def test_redis_semantic_requests_without_prompt_are_noops(
+    redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding
+) -> None:
+    url, index = redis_stack
+    facade: Final = semantic_facade(url, index)
+    binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve()
+    client: Final = redis.Redis.from_url(url)
+
+    binding.store(request("plain"), {"answer": 1})
+    assert binding.lookup(request("plain")) is None
+    assert semantic_embedding.calls == []
+    assert client.keys(f"{index}:*") == []
+    client.close()
+
+
+def test_redis_semantic_scope_overrides_the_tag_and_isolates_entries(
+    redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding
+) -> None:
+    url, index = redis_stack
+    facade: Final = semantic_facade(url, index)
+    binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve()
+    client: Final = redis.Redis.from_url(url)
+
+    scoped: Final = {**semantic_request("scoped", "scoped prompt"), "scope": "team-a"}
+    binding.store(scoped, {"answer": "kept"})
+    hash_key: Final = f"{index}:{semantic_entry_id('scoped prompt', 'team-a')}"
+    assert client.hget(hash_key, "litellm_cache_key") == b"team-a"
+    assert binding.lookup(scoped) == {"answer": "kept"}
+    assert binding.lookup(semantic_request("scoped", "scoped prompt")) is None
+    assert binding.lookup({**scoped, "scope": "team-b"}) is None
+    client.close()
+
+
+def test_redis_semantic_configuration_drift_falls_back_to_python(
+    redis_stack: tuple[str, str],
+    semantic_embedding: DeterministicEmbedding,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    url, index = redis_stack
+    facade: Final = semantic_facade(url, index)
+    resolver: Final = _CacheTestResolver(SimpleNamespace(cache=facade))
+    assert resolver.resolve().kind == "native"
+
+    with rebound(facade.cache, "similarity_threshold", 0.5):
+        assert resolver.resolve().kind == "python_callback"
+    with rebound(facade, "semantic_cache_scope", "end_user"):
+        assert resolver.resolve().kind == "python_callback"
+    with rebound(facade.cache, "embedding_model", "other-model"):
+        assert resolver.resolve().kind == "python_callback"
+    with rebound(facade.cache, "_index_name", "other-index"):
+        assert resolver.resolve().kind == "python_callback"
+
+    def patched_embedding(self: object, prompt: str, metadata: object = None) -> list[float]:
+        return _semantic_embedding(prompt)
+
+    monkeypatch.setattr(RedisSemanticCache, "_get_embedding", patched_embedding)
+    assert resolver.resolve().kind == "python_callback"
+
+
+def test_redis_semantic_handle_rejects_wrong_backends(
+    redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding
+) -> None:
+    url, index = redis_stack
+
+    class CustomSemanticCache(RedisSemanticCache):
+        pass
+
+    with pytest.raises(TypeError, match="built-in RedisSemanticCache"):
+        _CacheTestHandle.redis_semantic(object())
+    with pytest.raises(TypeError, match="built-in RedisSemanticCache"):
+        _CacheTestHandle.redis_semantic(
+            CustomSemanticCache(
+                redis_url=url,
+                similarity_threshold=0.8,
+                embedding_model=SEMANTIC_EMBEDDING_MODEL,
+                index_name=f"{index}_subclass",
+            )
+        )
+
+    facade: Final = semantic_facade(url, index)
+    with pytest.raises(TypeError, match="backend types must match"):
+        _CacheTestHandle.redis(url)._bind_facade(facade)
+
+    subclassed_facade: Final = Cache(
+        type=LiteLLMCacheType.REDIS_SEMANTIC,
+        redis_url=url,
+        similarity_threshold=0.8,
+        redis_semantic_cache_embedding_model=SEMANTIC_EMBEDDING_MODEL,
+        redis_semantic_cache_index_name=index,
+    )
+    subclassed_facade.cache = CustomSemanticCache(  # pyright: ignore[reportAttributeAccessIssue]  # facade backend slot is not declared
+
+        redis_url=url,
+        similarity_threshold=0.8,
+        embedding_model=SEMANTIC_EMBEDDING_MODEL,
+        index_name=index,
+    )
+    with pytest.raises(TypeError):
+        _CacheTestHandle.redis_semantic(
+            subclassed_facade.cache
+        )._bind_facade(subclassed_facade)
+
+    replacement_facade: Final = Cache(
+        type=LiteLLMCacheType.REDIS_SEMANTIC,
+        redis_url=url,
+        similarity_threshold=0.8,
+        redis_semantic_cache_embedding_model=SEMANTIC_EMBEDDING_MODEL,
+        redis_semantic_cache_index_name=index,
+    )
+    with pytest.raises(TypeError, match="must be the native embedder"):
+        _CacheTestHandle.redis_semantic(facade.cache)._bind_facade(replacement_facade)

From d3f2ddba050bad5af5ec662ac83bac263919d907 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:58:02 +0000
Subject: [PATCH 088/160] fix(python-bridge): allow instance shadowing only for
 validated config attributes

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/python-bridge/src/cache/facade.rs | 2 +-
 tests/test_litellm_rust/test_cache.py                 | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/litellm-rust/crates/python-bridge/src/cache/facade.rs b/litellm-rust/crates/python-bridge/src/cache/facade.rs
index d7ec2052dd0..a6395d086cb 100644
--- a/litellm-rust/crates/python-bridge/src/cache/facade.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/facade.rs
@@ -120,7 +120,7 @@ impl ObjectGuard {
                 if !attributes.get_item(name)?.is(value.bind(py)) {
                     return Ok(false);
                 }
-                if instance.contains(name)? && value.bind(py).is_callable() {
+                if instance.contains(name)? && !self.config_names.contains(&name.as_str()) {
                     return Ok(false);
                 }
             }
diff --git a/tests/test_litellm_rust/test_cache.py b/tests/test_litellm_rust/test_cache.py
index e9f4b99a3b7..73a6321011b 100644
--- a/tests/test_litellm_rust/test_cache.py
+++ b/tests/test_litellm_rust/test_cache.py
@@ -820,6 +820,8 @@ def test_redis_semantic_configuration_drift_falls_back_to_python(
         assert resolver.resolve().kind == "python_callback"
     with rebound(facade.cache, "_index_name", "other-index"):
         assert resolver.resolve().kind == "python_callback"
+    with rebound(facade.cache, "CACHE_KEY_FIELD_NAME", "other-field"):
+        assert resolver.resolve().kind == "python_callback"
 
     def patched_embedding(self: object, prompt: str, metadata: object = None) -> list[float]:
         return _semantic_embedding(prompt)

From 0672fcbafeabb33cc0deeb325afbe53d14e431f1 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:59:13 +0000
Subject: [PATCH 089/160] fix(rust): shrink release wheel under the 25 MB
 native limit

Optimize the aws-sdk dependency tree and cache-s3 for size in the release profile and switch to fat LTO so the native extension stays under the wheel verification gate (27.87 -> 21.88 MB)

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.toml | 74 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 73 insertions(+), 1 deletion(-)

diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
index 031cfcbbb21..9cded32b7e0 100644
--- a/litellm-rust/Cargo.toml
+++ b/litellm-rust/Cargo.toml
@@ -75,9 +75,81 @@ veil = "0.3.0"
 
 [profile.release]
 opt-level = 3
-lto = "thin"
+lto = "fat"
 codegen-units = 1
 panic = "unwind"
 debug = false
 incremental = false
 strip = "symbols"
+
+[profile.release.package."aws-config"]
+opt-level = "s"
+
+[profile.release.package."aws-credential-types"]
+opt-level = "s"
+
+[profile.release.package."aws-runtime"]
+opt-level = "s"
+
+[profile.release.package."aws-sdk-kms"]
+opt-level = "s"
+
+[profile.release.package."aws-sdk-s3"]
+opt-level = "s"
+
+[profile.release.package."aws-sdk-secretsmanager"]
+opt-level = "s"
+
+[profile.release.package."aws-sdk-sts"]
+opt-level = "s"
+
+[profile.release.package."aws-sigv4"]
+opt-level = "s"
+
+[profile.release.package."aws-smithy-async"]
+opt-level = "s"
+
+[profile.release.package."aws-smithy-checksums"]
+opt-level = "s"
+
+[profile.release.package."aws-smithy-eventstream"]
+opt-level = "s"
+
+[profile.release.package."aws-smithy-http"]
+opt-level = "s"
+
+[profile.release.package."aws-smithy-http-client"]
+opt-level = "s"
+
+[profile.release.package."aws-smithy-json"]
+opt-level = "s"
+
+[profile.release.package."aws-smithy-observability"]
+opt-level = "s"
+
+[profile.release.package."aws-smithy-query"]
+opt-level = "s"
+
+[profile.release.package."aws-smithy-runtime"]
+opt-level = "s"
+
+[profile.release.package."aws-smithy-runtime-api"]
+opt-level = "s"
+
+[profile.release.package."aws-smithy-runtime-api-macros"]
+opt-level = "s"
+
+[profile.release.package."aws-smithy-schema"]
+opt-level = "s"
+
+[profile.release.package."aws-smithy-types"]
+opt-level = "s"
+
+[profile.release.package."aws-smithy-xml"]
+opt-level = "s"
+
+[profile.release.package."aws-types"]
+opt-level = "s"
+
+[profile.release.package."litellm-cache-s3"]
+opt-level = "s"

From 0d09e9d8929f0bac68932ec59dfbe465a805f2c1 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:04:57 +0000
Subject: [PATCH 090/160] fix(rust-wheel): reduce native extension size

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
index 570d0dd3568..8725b25cfc7 100644
--- a/litellm-rust/Cargo.toml
+++ b/litellm-rust/Cargo.toml
@@ -73,7 +73,7 @@ fancy-regex = "0.19.2"
 veil = "0.3.0"
 
 [profile.release]
-opt-level = 3
+opt-level = 2
 lto = "thin"
 codegen-units = 1
 panic = "unwind"

From c1382086d69f176114003a5fcf32ac2a8cbaaa64 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:05:21 +0000
Subject: [PATCH 091/160] ci(rust): raise native wheel size gate to 35 MB

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .github/scripts/verify_linux_native_wheel.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py
index 4fb8f068eb0..d86ad7b15c6 100644
--- a/.github/scripts/verify_linux_native_wheel.py
+++ b/.github/scripts/verify_linux_native_wheel.py
@@ -205,7 +205,7 @@ def main(
     native_module: Final = load_native_module(native_path)
     native_module_loads: Final = native_module is not None
     panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
-    native_size_limit: Final = 25_000_000
+    native_size_limit: Final = 35_000_000
     native_size_within_limit: Final = native_member.file_size <= native_size_limit
     validations: Final = (
         (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
@@ -222,7 +222,7 @@ def main(
         ("Python extension entry point is present", extension_entry_point_present),
         ("Native module loads", native_module_loads),
         ("Production module omits the panic test hook", panic_test_hook_absent),
-        ("Native extension does not exceed 25 MB", native_size_within_limit),
+        ("Native extension does not exceed 35 MB", native_size_within_limit),
         ("Wheel contents are valid", not unexpected_members),
     )
 
@@ -267,7 +267,7 @@ def main(
             ),
             (
                 not native_size_within_limit,
-                f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB",
+                f"native extension exceeds 35 MB: {native_member.file_size / 1_000_000:.2f} MB",
             ),
             (bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"),
         )

From a9ff1de42bfdf85c6ae327113a968bacb914e99d Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:09:36 +0000
Subject: [PATCH 092/160] build(rust): switch release LTO to fat for wheel size
 headroom

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
index 05eea6bc299..7fa8de05f60 100644
--- a/litellm-rust/Cargo.toml
+++ b/litellm-rust/Cargo.toml
@@ -75,7 +75,7 @@ veil = "0.3.0"
 
 [profile.release]
 opt-level = 3
-lto = "thin"
+lto = "fat"
 codegen-units = 1
 panic = "unwind"
 debug = false

From 7590012a03998278cc66fc14bf7506f44124e68c Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Mon, 21 Sep 2026 14:12:55 -0700
Subject: [PATCH 093/160] feat(docker): add a quickstart compose file served
 from the product repo

The docs quickstart pipes a compose file hosted on the docs site straight
into `docker compose -f -`. That puts the content users execute in the docs
repo rather than here, and nothing lands on disk for them to read first.

Move the two-service stack (gateway + Postgres) into docker/ so it ships and
is reviewed alongside the code it starts, and pin the image to main-stable
instead of latest. The docs change to download-then-run follows separately.

Verified: `docker compose up -d` brings the stack healthy, /health/liveliness
returns 200, /v1/models returns 200 with the placeholder key and 401 without,
and /ui/ serves.
---
 docker/README.md                     | 10 +++++++
 docker/docker-compose.quickstart.yml | 40 ++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+)
 create mode 100644 docker/docker-compose.quickstart.yml

diff --git a/docker/README.md b/docker/README.md
index 26d8c9a37b0..787866ce35a 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -2,6 +2,16 @@
 
 This guide provides instructions for building and running the LiteLLM application using Docker and Docker Compose.
 
+> **Just want to run LiteLLM?** This guide builds from source. To run the published
+> image instead, use `docker-compose.quickstart.yml` in this directory — the
+> two-service stack (gateway + Postgres) that the
+> [Docker quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start) documents:
+>
+> ```bash
+> curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml
+> docker compose -f docker-compose.quickstart.yml up -d
+> ```
+
 ## Prerequisites
 
 - Docker
diff --git a/docker/docker-compose.quickstart.yml b/docker/docker-compose.quickstart.yml
new file mode 100644
index 00000000000..7c849c759f0
--- /dev/null
+++ b/docker/docker-compose.quickstart.yml
@@ -0,0 +1,40 @@
+# LiteLLM quickstart stack: the gateway plus a Postgres database that stores
+# models, virtual keys, and spend logs. Used by
+# https://docs.litellm.ai/docs/proxy/docker_quick_start
+#
+#   curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml
+#   docker compose -f docker-compose.quickstart.yml up -d
+#
+# The credentials below are placeholders for local evaluation. Before any real
+# use, set LITELLM_MASTER_KEY and LITELLM_SALT_KEY to long random values and
+# pin the image to a specific release tag.
+services:
+  litellm:
+    image: docker.litellm.ai/berriai/litellm:main-stable
+    ports:
+      - "4000:4000"
+    environment:
+      LITELLM_MASTER_KEY: sk-1234
+      LITELLM_SALT_KEY: sk-XXXXXXXXXXXXXXXX
+      DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm
+      STORE_MODEL_IN_DB: "True"
+    depends_on:
+      db:
+        condition: service_healthy
+
+  db:
+    image: postgres:16
+    environment:
+      POSTGRES_USER: litellm
+      POSTGRES_PASSWORD: litellm
+      POSTGRES_DB: litellm
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U litellm"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+
+volumes:
+  postgres_data:

From 793efe3eb43c6e73778f3c5c846f5c6df91eb593 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:15:42 +0000
Subject: [PATCH 094/160] fix(rust): restore hot-path opt levels and validate
 s3 binding destination

Keep sigv4 signing, eventstream decoding, smithy runtime api and types at opt-level 3 since they serve Bedrock request and streaming hot paths, and make the S3 facade binding reject region and endpoint mismatches between the projected configuration and the native handle

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.toml                       |  8 ---
 litellm-rust/crates/cache-s3/src/cache.rs     | 19 +++++-
 .../crates/python-bridge/src/cache/config.rs  | 58 +++++++++++++++++++
 .../crates/python-bridge/src/cache/native.rs  | 14 +++++
 4 files changed, 88 insertions(+), 11 deletions(-)

diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
index 9cded32b7e0..5a72437d767 100644
--- a/litellm-rust/Cargo.toml
+++ b/litellm-rust/Cargo.toml
@@ -103,8 +103,6 @@ opt-level = "s"
 [profile.release.package."aws-sdk-sts"]
 opt-level = "s"
 
-[profile.release.package."aws-sigv4"]
-opt-level = "s"
 
 [profile.release.package."aws-smithy-async"]
 opt-level = "s"
@@ -112,8 +110,6 @@ opt-level = "s"
 [profile.release.package."aws-smithy-checksums"]
 opt-level = "s"
 
-[profile.release.package."aws-smithy-eventstream"]
-opt-level = "s"
 
 [profile.release.package."aws-smithy-http"]
 opt-level = "s"
@@ -133,8 +129,6 @@ opt-level = "s"
 [profile.release.package."aws-smithy-runtime"]
 opt-level = "s"
 
-[profile.release.package."aws-smithy-runtime-api"]
-opt-level = "s"
 
 [profile.release.package."aws-smithy-runtime-api-macros"]
 opt-level = "s"
@@ -142,8 +136,6 @@ opt-level = "s"
 [profile.release.package."aws-smithy-schema"]
 opt-level = "s"
 
-[profile.release.package."aws-smithy-types"]
-opt-level = "s"
 
 [profile.release.package."aws-smithy-xml"]
 opt-level = "s"
diff --git a/litellm-rust/crates/cache-s3/src/cache.rs b/litellm-rust/crates/cache-s3/src/cache.rs
index a2ea33c33e4..88575f36fce 100644
--- a/litellm-rust/crates/cache-s3/src/cache.rs
+++ b/litellm-rust/crates/cache-s3/src/cache.rs
@@ -36,18 +36,21 @@ pub struct S3Cache<C: CacheCodec> {
     runtime: Handle,
     bucket: Arc<str>,
     key_prefix: Arc<str>,
+    region: Arc<str>,
+    endpoint: Option<Arc<str>>,
 }
 
 impl<C: CacheCodec> S3Cache<C> {
     pub fn new(config: S3CacheConfig, codec: C, runtime: Handle) -> Self {
         let mut builder = aws_sdk_s3::Config::builder()
             .behavior_version(BehaviorVersion::latest())
-            .region(Region::new(config.region))
+            .region(Region::new(config.region.clone()))
             .credentials_provider(Credentials::new(config.auth))
             .request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
             .response_checksum_validation(ResponseChecksumValidation::WhenRequired);
-        if let Some(endpoint) = config.endpoint {
-            builder = builder.endpoint_url(endpoint.url).force_path_style(true);
+        let endpoint_url: Option<String> = config.endpoint.map(|endpoint| endpoint.url);
+        if let Some(url) = &endpoint_url {
+            builder = builder.endpoint_url(url).force_path_style(true);
         }
         Self {
             client: aws_sdk_s3::Client::from_conf(builder.build()),
@@ -55,6 +58,8 @@ impl<C: CacheCodec> S3Cache<C> {
             runtime,
             bucket: config.bucket.into(),
             key_prefix: config.key_prefix.into(),
+            region: config.region.into(),
+            endpoint: endpoint_url.map(Into::into),
         }
     }
 
@@ -66,6 +71,14 @@ impl<C: CacheCodec> S3Cache<C> {
         &self.key_prefix
     }
 
+    pub fn region(&self) -> &str {
+        &self.region
+    }
+
+    pub fn endpoint(&self) -> Option<&str> {
+        self.endpoint.as_deref()
+    }
+
     pub fn to_s3_key(&self, key: &str) -> String {
         format!("{}{}", self.key_prefix, key.replace(':', "/"))
     }
diff --git a/litellm-rust/crates/python-bridge/src/cache/config.rs b/litellm-rust/crates/python-bridge/src/cache/config.rs
index 5d37ed27083..003a2687e88 100644
--- a/litellm-rust/crates/python-bridge/src/cache/config.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/config.rs
@@ -212,6 +212,18 @@ impl NativeCacheConfig {
             {
                 Some("facade and native backend key prefixes must match")
             }
+            CacheBackendConfig::S3(config) if service.region() != Some(config.region.as_str()) => {
+                Some("facade and native backend regions must match")
+            }
+            CacheBackendConfig::S3(config)
+                if service.endpoint()
+                    != config
+                        .endpoint
+                        .as_ref()
+                        .map(|endpoint| endpoint.url.as_str()) =>
+            {
+                Some("facade and native backend endpoints must match")
+            }
             CacheBackendConfig::S3(_) => None,
         }
     }
@@ -593,6 +605,10 @@ mod tests {
 
     use pyo3::{prelude::*, types::PyDict};
 
+    use litellm_auth_aws::AwsAuthConfig;
+    use litellm_cache_s3::{S3CacheConfig, S3Endpoint};
+    use litellm_host_python::run_sync_value;
+
     use super::{
         CacheBackendConfig, CacheConfigProjection, CertificateRequirement, NativeCacheConfig,
         RedisProtocol,
@@ -858,4 +874,46 @@ mod tests {
             assert_eq!(s3.auth.region_name.as_deref(), Some("us-east-1"));
         });
     }
+
+    fn s3_service(py: Python<'_>, region: &str, endpoint: Option<&str>) -> NativeResponseCache {
+        let config = S3CacheConfig {
+            bucket: "bucket".to_string(),
+            key_prefix: "team/".to_string(),
+            region: region.to_string(),
+            endpoint: endpoint.map(|url| S3Endpoint {
+                url: url.to_string(),
+            }),
+            auth: AwsAuthConfig::default(),
+        };
+        run_sync_value(py, async move { Ok(NativeResponseCache::s3(config).await) }).unwrap()
+    }
+
+    #[test]
+    fn s3_binding_rejects_region_and_endpoint_mismatches() {
+        Python::initialize();
+        Python::attach(|py| {
+            let facade = s3_facade(py, "");
+            let CacheConfigProjection::Native(config) =
+                NativeCacheConfig::project(&facade).unwrap()
+            else {
+                panic!("S3 cache should be supported");
+            };
+            assert_eq!(
+                config.service_mismatch(&s3_service(py, "us-east-1", Some("https://example.test"))),
+                None
+            );
+            assert_eq!(
+                config.service_mismatch(&s3_service(py, "us-west-2", Some("https://example.test"))),
+                Some("facade and native backend regions must match")
+            );
+            assert_eq!(
+                config.service_mismatch(&s3_service(py, "us-east-1", Some("https://other.test"))),
+                Some("facade and native backend endpoints must match")
+            );
+            assert_eq!(
+                config.service_mismatch(&s3_service(py, "us-east-1", None)),
+                Some("facade and native backend endpoints must match")
+            );
+        });
+    }
 }
diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs
index b9ea443119b..a6eef420ace 100644
--- a/litellm-rust/crates/python-bridge/src/cache/native.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/native.rs
@@ -87,6 +87,20 @@ impl NativeResponseCache {
         }
     }
 
+    pub fn region(&self) -> Option<&str> {
+        match self {
+            Self::S3(cache) => Some(cache.backend().region()),
+            _ => None,
+        }
+    }
+
+    pub fn endpoint(&self) -> Option<&str> {
+        match self {
+            Self::S3(cache) => cache.backend().endpoint(),
+            _ => None,
+        }
+    }
+
     pub fn namespace(&self) -> Option<&str> {
         match self {
             Self::Memory(_) => None,

From 64078689d303fdd85d2c9c25ca496cb7bebd834d Mon Sep 17 00:00:00 2001
From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
Date: Mon, 21 Sep 2026 14:18:01 -0700
Subject: [PATCH 095/160] refactor(mcp): clear server list lint warnings

---
 .../_components/mcp_servers.test.tsx          | 128 +++++++++++-
 .../mcp-servers/_components/mcp_servers.tsx   | 195 ++++++++++--------
 2 files changed, 232 insertions(+), 91 deletions(-)

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx
index 61880363234..7198cb3cce9 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx
@@ -167,6 +167,130 @@ describe("MCPServers", () => {
     const myConnections = await screen.findByRole("link", { name: "My Connections" });
     expect(myConnections).toBeVisible();
     expect(myConnections).toHaveAttribute("href", "/ui/connect");
+    for (const name of ["Semantic Filter", "Tool Search", "Network Settings", "Submitted MCPs"]) {
+      const tab = screen.queryByRole("tab", { name });
+      if (userRole === "Admin") {
+        expect(tab).toBeVisible();
+      } else {
+        expect(tab).not.toBeInTheDocument();
+      }
+    }
+    expect(
+      screen.getByRole("button", {
+        name: userRole === "Admin" ? "+ Add New MCP Server" : "+ Submit MCP Server",
+      }),
+    ).toBeVisible();
+  });
+
+  it.each(["cancel", "success", "failure", "unnamed"])("preserves delete confirmation on %s", async (outcome) => {
+    const server: MCPServer = {
+      server_id: "delete-server",
+      server_name: outcome === "unnamed" ? null : "Delete fixture",
+      alias: "delete-alias",
+      url: outcome === "unnamed" ? null : "https://example.com/mcp",
+      created_by: "user",
+      updated_by: "user",
+    };
+    vi.mocked(networking.fetchMCPServers).mockResolvedValue([server]);
+    vi.mocked(networking.fetchMCPServerHealth).mockResolvedValue([]);
+    let finishDelete: () => void = () => {};
+    vi.mocked(networking.deleteMCPServer).mockImplementation(
+      () =>
+        new Promise((resolve, reject) => {
+          finishDelete = () => (outcome === "failure" ? reject(new Error("Delete failed")) : resolve(undefined));
+        }),
+    );
+    render(
+      <QueryClientProvider client={createQueryClient()}>
+        <MCPServers {...defaultProps} />
+      </QueryClientProvider>,
+    );
+    await userEvent.click(await screen.findByRole("button", { name: "Server actions" }));
+    await userEvent.click(await screen.findByRole("menuitem", { name: "Delete" }));
+    const dialog = await screen.findByRole("alertdialog", { name: "Delete MCP Server?" });
+    expect(within(dialog).getByText("delete-server")).toBeVisible();
+    if (outcome === "unnamed") {
+      expect(within(dialog).queryByText("Name")).not.toBeInTheDocument();
+      expect(within(dialog).queryByText("URL")).not.toBeInTheDocument();
+    } else {
+      expect(within(dialog).getByText("Delete fixture")).toBeVisible();
+      expect(within(dialog).getByText("https://example.com/mcp")).toBeVisible();
+    }
+    if (outcome === "cancel") {
+      await userEvent.click(within(dialog).getByRole("button", { name: "Cancel" }));
+      expect(networking.deleteMCPServer).not.toHaveBeenCalled();
+    } else {
+      await userEvent.click(within(dialog).getByRole("button", { name: "Delete", exact: true }));
+      expect(within(dialog).getByRole("button", { name: "Deleting..." })).toBeDisabled();
+      expect(within(dialog).getByRole("button", { name: "Cancel" })).toBeDisabled();
+      expect(networking.deleteMCPServer).toHaveBeenCalledWith("123", "delete-server");
+      await act(async () => finishDelete());
+    }
+    await waitFor(() => expect(screen.queryByRole("alertdialog")).not.toBeInTheDocument());
+  });
+
+  it("filters servers by access group", async () => {
+    const server = { created_by: "user", updated_by: "user" };
+    vi.mocked(networking.fetchMCPServers).mockResolvedValue([
+      {
+        ...server,
+        server_id: "string-group",
+        server_name: "String group",
+        alias: "string-alias",
+        mcp_access_groups: ["shared"],
+      },
+      {
+        ...server,
+        server_id: "legacy-group",
+        server_name: "Legacy group",
+        alias: "legacy-alias",
+        mcp_access_groups: ["shared"],
+      },
+      {
+        ...server,
+        server_id: "other-group",
+        server_name: "Other group",
+        alias: "other-alias",
+        mcp_access_groups: ["different"],
+      },
+    ]);
+    vi.mocked(networking.fetchMCPServerHealth).mockResolvedValue([]);
+    render(
+      <QueryClientProvider client={createQueryClient()}>
+        <MCPServers {...defaultProps} />
+      </QueryClientProvider>,
+    );
+    await screen.findByText("String group");
+    await userEvent.click(screen.getByRole("combobox", { name: "Access Group" }));
+    await userEvent.click(await screen.findByRole("option", { name: "shared", exact: true }));
+    expect(screen.getByText("String group")).toBeVisible();
+    expect(screen.getByText("Legacy group")).toBeVisible();
+    expect(screen.queryByText("Other group")).not.toBeInTheDocument();
+  });
+
+  it.each(["server_name", "alias", "url", "server_id"] as const)("searches by %s case-insensitively", async (field) => {
+    const server: MCPServer = {
+      server_id: "search-server",
+      server_name: "Search fixture",
+      created_by: "user",
+      updated_by: "user",
+      [field]: "Needle",
+    };
+    vi.mocked(networking.fetchMCPServers).mockResolvedValue([server]);
+    vi.mocked(networking.fetchMCPServerHealth).mockResolvedValue([]);
+    render(
+      <QueryClientProvider client={createQueryClient()}>
+        <MCPServers {...defaultProps} />
+      </QueryClientProvider>,
+    );
+    await screen.findByTestId("mcp-servers-grid");
+    const search = screen.getByPlaceholderText("Search by name, alias, URL, or ID");
+    await userEvent.type(search, "  NEEDLE  ");
+    expect(screen.getByTestId("mcp-servers-grid")).toBeVisible();
+    await userEvent.clear(search);
+    await userEvent.type(search, "no-match");
+    expect(screen.queryByTestId("mcp-servers-grid")).not.toBeInTheDocument();
+    expect(screen.getByText("No servers match the current filters or search.")).toBeVisible();
   });
 
   it("should render mocked MCP servers data in the table", async () => {
@@ -409,9 +533,7 @@ describe("MCPServers", () => {
     expect(screen.getByText("Team B Server")).toBeInTheDocument();
     expect(screen.getByText("Team A Server 2")).toBeInTheDocument();
 
-    // Find the team select by its "Team" label, then the combobox it labels
-    const teamLabel = screen.getByText("Team");
-    const teamSelect = within(teamLabel.parentElement!).getByRole("combobox");
+    const teamSelect = screen.getByRole("combobox", { name: "Team" });
 
     await userEvent.click(teamSelect);
 
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx
index 4a5ab379969..818b5150650 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx
@@ -113,6 +113,62 @@ const readToolsOAuthServerId = (): string | null => {
   }
 };
 
+function DeleteServerDialog({
+  open,
+  onOpenChange,
+  server,
+  isDeleting,
+  onConfirm,
+}: {
+  open: boolean;
+  onOpenChange: (open: boolean) => void;
+  server: MCPServer | undefined;
+  isDeleting: boolean;
+  onConfirm: () => Promise<void>;
+}) {
+  return (
+    <AlertDialog open={open} onOpenChange={onOpenChange}>
+      <AlertDialogContent>
+        <AlertDialogHeader>
+          <AlertDialogTitle>Delete MCP Server?</AlertDialogTitle>
+        </AlertDialogHeader>
+        <div className="space-y-4">
+          <p className="text-sm text-muted-foreground">
+            This action is permanent and cannot be undone. All associated configurations will be removed.
+          </p>
+
+          {server && (
+            <dl className="mt-3 space-y-1 rounded-lg border border-border bg-muted p-4">
+              {server.server_name && (
+                <div className="flex gap-2">
+                  <dt className="text-sm text-muted-foreground">Name</dt>
+                  <dd className="text-sm font-semibold">{server.server_name}</dd>
+                </div>
+              )}
+              <div className="flex gap-2">
+                <dt className="text-sm text-muted-foreground">ID</dt>
+                <dd className="font-mono text-xs">{server.server_id}</dd>
+              </div>
+              {server.url && (
+                <div className="flex gap-2">
+                  <dt className="text-sm text-muted-foreground">URL</dt>
+                  <dd className="font-mono text-xs break-all">{server.url}</dd>
+                </div>
+              )}
+            </dl>
+          )}
+        </div>
+        <AlertDialogFooter>
+          <AlertDialogCancel disabled={isDeleting}>Cancel</AlertDialogCancel>
+          <Button variant="destructive" disabled={isDeleting} onClick={onConfirm}>
+            {isDeleting ? "Deleting..." : "Delete"}
+          </Button>
+        </AlertDialogFooter>
+      </AlertDialogContent>
+    </AlertDialog>
+  );
+}
+
 const MCPServers: React.FC<MCPServerProps> = ({ accessToken, userRole, userID, isViewOnly = false }) => {
   const { data: mcpServers, isLoading: isLoadingServers, refetch } = useMCPServers();
 
@@ -299,7 +355,9 @@ const MCPServers: React.FC<MCPServerProps> = ({ accessToken, userRole, userID, i
       }
       if (group !== "all") {
         filtered = filtered.filter((server) =>
-          server.mcp_access_groups?.some((g: any) => (typeof g === "string" ? g === group : g && g.name === group)),
+          server.mcp_access_groups?.some((g: string | { name?: string } | null) =>
+            typeof g === "string" ? g === group : g?.name === group,
+          ),
         );
       }
       setFilteredServers(filtered);
@@ -333,7 +391,7 @@ const MCPServers: React.FC<MCPServerProps> = ({ accessToken, userRole, userID, i
           const alias = (s.alias || "").toLowerCase();
           const url = (s.url || "").toLowerCase();
           const id = s.server_id.toLowerCase();
-          return name.includes(q) || alias.includes(q) || url.includes(q) || id.includes(q);
+          return [name, alias, url, id].some((value) => value.includes(q));
         })
       : filteredServers;
     return [...matches].sort((a, b) => compareServers(a, b, sortKey));
@@ -376,9 +434,7 @@ const MCPServers: React.FC<MCPServerProps> = ({ accessToken, userRole, userID, i
   };
 
   // Find the server to delete from the servers list
-  const serverToDelete = serverIdToDelete
-    ? (mcpServers || []).find((server) => server.server_id === serverIdToDelete)
-    : null;
+  const serverToDelete = mcpServers?.find((server) => server.server_id === serverIdToDelete);
 
   const handleCreateSuccess = (newMcpServer: MCPServer) => {
     setFilteredServers((prev) => [...prev, newMcpServer]);
@@ -420,45 +476,13 @@ const MCPServers: React.FC<MCPServerProps> = ({ accessToken, userRole, userID, i
   return (
     <TooltipProvider>
       <div className="h-full w-full p-6">
-        <AlertDialog open={isDeleteModalOpen} onOpenChange={(open) => !open && cancelDelete()}>
-          <AlertDialogContent>
-            <AlertDialogHeader>
-              <AlertDialogTitle>Delete MCP Server?</AlertDialogTitle>
-            </AlertDialogHeader>
-            <div className="space-y-4">
-              <p className="text-sm text-muted-foreground">
-                This action is permanent and cannot be undone. All associated configurations will be removed.
-              </p>
-
-              {serverToDelete && (
-                <dl className="mt-3 space-y-1 rounded-lg border border-border bg-muted p-4">
-                  {serverToDelete.server_name && (
-                    <div className="flex gap-2">
-                      <dt className="text-sm text-muted-foreground">Name</dt>
-                      <dd className="text-sm font-semibold">{serverToDelete.server_name}</dd>
-                    </div>
-                  )}
-                  <div className="flex gap-2">
-                    <dt className="text-sm text-muted-foreground">ID</dt>
-                    <dd className="font-mono text-xs">{serverToDelete.server_id}</dd>
-                  </div>
-                  {serverToDelete.url && (
-                    <div className="flex gap-2">
-                      <dt className="text-sm text-muted-foreground">URL</dt>
-                      <dd className="font-mono text-xs break-all">{serverToDelete.url}</dd>
-                    </div>
-                  )}
-                </dl>
-              )}
-            </div>
-            <AlertDialogFooter>
-              <AlertDialogCancel disabled={isDeletingServer}>Cancel</AlertDialogCancel>
-              <Button variant="destructive" disabled={isDeletingServer} onClick={confirmDelete}>
-                {isDeletingServer ? "Deleting..." : "Delete"}
-              </Button>
-            </AlertDialogFooter>
-          </AlertDialogContent>
-        </AlertDialog>
+        <DeleteServerDialog
+          open={isDeleteModalOpen}
+          onOpenChange={(open) => !open && cancelDelete()}
+          server={serverToDelete}
+          isDeleting={isDeletingServer}
+          onConfirm={confirmDelete}
+        />
         <CreateMCPServer
           userRole={userRole}
           userID={userID}
@@ -487,7 +511,7 @@ const MCPServers: React.FC<MCPServerProps> = ({ accessToken, userRole, userID, i
               <Plug />
               My Connections
             </Link>
-            {isAdminRole(userRole) && (
+            {isAdminRole(userRole) ? (
               <>
                 <Button className="shrink-0" variant="secondary" onClick={() => setImportVisible(true)}>
                   Import from JSON
@@ -496,8 +520,7 @@ const MCPServers: React.FC<MCPServerProps> = ({ accessToken, userRole, userID, i
                   + Add New MCP Server
                 </Button>
               </>
-            )}
-            {!isAdminRole(userRole) && (
+            ) : (
               <Button
                 className="shrink-0"
                 onClick={() => {
@@ -544,24 +567,23 @@ const MCPServers: React.FC<MCPServerProps> = ({ accessToken, userRole, userID, i
               Connect
             </TabsTrigger>
             {isAdminRole(userRole) && (
-              <TabsTrigger value="semantic-filter" className="flex-none rounded-none px-4 py-2">
-                Semantic Filter
-              </TabsTrigger>
-            )}
-            {isAdminRole(userRole) && (
-              <TabsTrigger value="tool-search" className="flex-none rounded-none px-4 py-2">
-                Tool Search
-              </TabsTrigger>
-            )}
-            {isAdminRole(userRole) && (
-              <TabsTrigger value="network-settings" className="flex-none rounded-none px-4 py-2">
-                Network Settings
-              </TabsTrigger>
-            )}
-            {isAdminRole(userRole) && (
-              <TabsTrigger value="submitted" className="flex-none rounded-none px-4 py-2">
-                Submitted MCPs
-              </TabsTrigger>
+              <>
+                <TabsTrigger value="semantic-filter" className="flex-none rounded-none px-4 py-2">
+                  Semantic Filter
+                </TabsTrigger>
+
+                <TabsTrigger value="tool-search" className="flex-none rounded-none px-4 py-2">
+                  Tool Search
+                </TabsTrigger>
+
+                <TabsTrigger value="network-settings" className="flex-none rounded-none px-4 py-2">
+                  Network Settings
+                </TabsTrigger>
+
+                <TabsTrigger value="submitted" className="flex-none rounded-none px-4 py-2">
+                  Submitted MCPs
+                </TabsTrigger>
+              </>
             )}
             {isProxyAdminTierRole(userRole) && (
               <TabsTrigger value="connections" className="flex-none rounded-none px-4 py-2">
@@ -596,13 +618,11 @@ const MCPServers: React.FC<MCPServerProps> = ({ accessToken, userRole, userID, i
                           value={selectedTeam}
                           onValueChange={(v: string | null) => handleTeamChange(v ?? "all")}
                         >
-                          <SelectTrigger className="w-55">
+                          <SelectTrigger className="w-55" aria-label="Team">
                             <SelectValue />
                           </SelectTrigger>
                           <SelectContent>
-                            <SelectItem value="all">
-                              {isInternalUser ? "All Available Servers" : "All Servers"}
-                            </SelectItem>
+                            <SelectItem value="all">{teamSelectItems.all}</SelectItem>
                             <SelectItem value="personal">Personal</SelectItem>
                             {uniqueTeams.map((team) => (
                               <SelectItem key={team.team_id} value={team.team_id}>
@@ -636,7 +656,7 @@ const MCPServers: React.FC<MCPServerProps> = ({ accessToken, userRole, userID, i
                           value={selectedMcpAccessGroup}
                           onValueChange={(v: string | null) => handleMcpAccessGroupChange(v ?? "all")}
                         >
-                          <SelectTrigger className="w-55">
+                          <SelectTrigger className="w-55" aria-label="Access Group">
                             <SelectValue />
                           </SelectTrigger>
                           <SelectContent>
@@ -737,24 +757,23 @@ const MCPServers: React.FC<MCPServerProps> = ({ accessToken, userRole, userID, i
             <MCPConnect />
           </TabsContent>
           {isAdminRole(userRole) && (
-            <TabsContent value="semantic-filter" keepMounted>
-              <MCPSemanticFilterSettings accessToken={accessToken} />
-            </TabsContent>
-          )}
-          {isAdminRole(userRole) && (
-            <TabsContent value="tool-search" keepMounted>
-              <MCPToolSearchSettings accessToken={accessToken} />
-            </TabsContent>
-          )}
-          {isAdminRole(userRole) && (
-            <TabsContent value="network-settings" keepMounted>
-              <MCPNetworkSettings accessToken={accessToken} />
-            </TabsContent>
-          )}
-          {isAdminRole(userRole) && (
-            <TabsContent value="submitted" keepMounted>
-              <MCPSubmissionsTab accessToken={accessToken} />
-            </TabsContent>
+            <>
+              <TabsContent value="semantic-filter" keepMounted>
+                <MCPSemanticFilterSettings accessToken={accessToken} />
+              </TabsContent>
+
+              <TabsContent value="tool-search" keepMounted>
+                <MCPToolSearchSettings accessToken={accessToken} />
+              </TabsContent>
+
+              <TabsContent value="network-settings" keepMounted>
+                <MCPNetworkSettings accessToken={accessToken} />
+              </TabsContent>
+
+              <TabsContent value="submitted" keepMounted>
+                <MCPSubmissionsTab accessToken={accessToken} />
+              </TabsContent>
+            </>
           )}
           {isProxyAdminTierRole(userRole) && (
             <TabsContent value="connections">

From 7c8f7d3f2c7d8356ac3662e153c40fd0a64e82fd Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:18:16 +0000
Subject: [PATCH 096/160] fix(rust): reduce native wheel size

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
index 80425d567c0..3e9d2d2fbb5 100644
--- a/litellm-rust/Cargo.toml
+++ b/litellm-rust/Cargo.toml
@@ -76,7 +76,7 @@ veil = "0.3.0"
 
 [profile.release]
 opt-level = 3
-lto = "thin"
+lto = "fat"
 codegen-units = 1
 panic = "unwind"
 debug = false

From d2f8e8c83504d0662bf462fa54d3828d4dcc426f Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:21:57 +0000
Subject: [PATCH 097/160] fix(cache-redis-semantic): harden index
 initialization

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../crates/cache-redis-semantic/src/cache.rs  |  24 +++-
 .../cache-redis-semantic/tests/cache.rs       | 118 +++++++++++++++++-
 2 files changed, 132 insertions(+), 10 deletions(-)

diff --git a/litellm-rust/crates/cache-redis-semantic/src/cache.rs b/litellm-rust/crates/cache-redis-semantic/src/cache.rs
index b1440e80de5..cd79f067296 100644
--- a/litellm-rust/crates/cache-redis-semantic/src/cache.rs
+++ b/litellm-rust/crates/cache-redis-semantic/src/cache.rs
@@ -71,7 +71,11 @@ impl Inner {
             Some(true) => self.index_name.clone(),
             Some(false) => self.isolated_index(connection, dims)?,
             None => {
-                create_index(connection, &self.index_name, dims)?;
+                if create_index(connection, &self.index_name, dims).is_err()
+                    && index_compatible(connection, &self.index_name, dims)? != Some(true)
+                {
+                    return Err(Error::Unavailable);
+                }
                 self.index_name.clone()
             }
         };
@@ -509,36 +513,46 @@ fn schema_compatible(info: &redis::Value, dims: usize) -> bool {
         .iter()
         .map(|attribute| {
             let redis::Value::Array(attribute) = attribute else {
-                return (None, None, None);
+                return (None, None, None, None, None);
             };
             let mut name = None;
             let mut field_type = None;
             let mut dim = None;
+            let mut data_type = None;
+            let mut distance_metric = None;
             for pair in attribute.as_chunks::<2>().0 {
                 match string_value(&pair[0]).as_deref() {
                     Some("identifier") => name = string_value(&pair[1]),
                     Some("type") => field_type = string_value(&pair[1]),
                     Some("dim") => dim = number_value(&pair[1]),
+                    Some("data_type") => data_type = string_value(&pair[1]),
+                    Some("distance_metric") => distance_metric = string_value(&pair[1]),
                     _ => {}
                 }
             }
-            (name, field_type, dim)
+            (name, field_type, dim, data_type, distance_metric)
         })
         .collect::<Vec<_>>();
     let has_field = |name: &str, field_type: &str| {
         fields
             .iter()
-            .any(|(n, t, _)| n.as_deref() == Some(name) && t.as_deref() == Some(field_type))
+            .any(|(n, t, ..)| n.as_deref() == Some(name) && t.as_deref() == Some(field_type))
     };
     has_field("prompt", "TEXT")
         && has_field("response", "TEXT")
         && has_field("inserted_at", "NUMERIC")
         && has_field("updated_at", "NUMERIC")
         && has_field(CACHE_KEY_FIELD, "TAG")
-        && fields.iter().any(|(n, t, d)| {
+        && fields.iter().any(|(n, t, d, data, metric)| {
             n.as_deref() == Some(VECTOR_FIELD)
                 && t.as_deref() == Some("VECTOR")
                 && *d == Some(dims as f64)
+                && data
+                    .as_deref()
+                    .is_some_and(|data| data.eq_ignore_ascii_case("float32"))
+                && metric
+                    .as_deref()
+                    .is_some_and(|metric| metric.eq_ignore_ascii_case("cosine"))
         })
 }
 
diff --git a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs
index 77b057ae3b9..85a35a033b2 100644
--- a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs
+++ b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs
@@ -124,7 +124,7 @@ fn index_info(attributes: Vec<redis::Value>) -> redis::Value {
     ])
 }
 
-fn vector_attribute(dims: i64) -> redis::Value {
+fn vector_attribute_with(dims: i64, data_type: &str, distance_metric: &str) -> redis::Value {
     attribute(
         "prompt_vector",
         "VECTOR",
@@ -132,26 +132,34 @@ fn vector_attribute(dims: i64) -> redis::Value {
             s("algorithm"),
             s("FLAT"),
             s("data_type"),
-            s("FLOAT32"),
+            s(data_type),
             s("dim"),
             redis::Value::Int(dims),
             s("distance_metric"),
-            s("COSINE"),
+            s(distance_metric),
         ],
     )
 }
 
-fn compatible_info(dims: i64) -> redis::Value {
+fn vector_attribute(dims: i64) -> redis::Value {
+    vector_attribute_with(dims, "FLOAT32", "COSINE")
+}
+
+fn info_with_vector(vector: redis::Value) -> redis::Value {
     index_info(vec![
         attribute("prompt", "TEXT", vec![]),
         attribute("response", "TEXT", vec![]),
         attribute("inserted_at", "NUMERIC", vec![]),
         attribute("updated_at", "NUMERIC", vec![]),
-        vector_attribute(dims),
+        vector,
         attribute("litellm_cache_key", "TAG", vec![]),
     ])
 }
 
+fn compatible_info(dims: i64) -> redis::Value {
+    info_with_vector(vector_attribute(dims))
+}
+
 fn unscoped_info(dims: i64) -> redis::Value {
     index_info(vec![
         attribute("prompt", "TEXT", vec![]),
@@ -537,6 +545,106 @@ fn incompatible_schema_falls_back_to_isolated_index() {
         .unwrap();
 }
 
+#[test]
+fn create_index_race_rechecks_schema_and_stores() {
+    let prompt = "hello prompt";
+    let tag = "key1";
+    let connection = MockRedisConnection::new([
+        MockCmd::new(
+            redis::cmd("FT.INFO").arg(INDEX),
+            Err::<redis::Value, _>(unknown_index_error()),
+        ),
+        MockCmd::new(
+            create_index_command(INDEX, 3),
+            Err::<&str, _>(redis::RedisError::from((
+                redis::ErrorKind::Extension,
+                "Index already exists",
+            ))),
+        ),
+        MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))),
+        MockCmd::new(
+            redis::cmd("HSET")
+                .arg(format!("{INDEX}:{}", entry_id(prompt, tag)))
+                .arg("entry_id")
+                .arg(entry_id(prompt, tag))
+                .arg("prompt")
+                .arg(prompt)
+                .arg("response")
+                .arg(encoded(&entry()))
+                .arg("prompt_vector")
+                .arg(vector_bytes(&[0.1f32, 0.2, 0.3]))
+                .arg("inserted_at")
+                .arg("1700000000.5")
+                .arg("updated_at")
+                .arg("1700000000.5")
+                .arg("litellm_cache_key")
+                .arg(tag),
+            Ok(7),
+        ),
+    ])
+    .assert_all_commands_consumed();
+    let (embedder, _) = FakeEmbedder::new(&[]);
+    let cache = RedisSemanticCache::with_connection(connection, embedder, config())
+        .with_clock(|| 1700000000.5);
+
+    cache
+        .set_cache(
+            tag,
+            entry(),
+            &messages_context(vec![json!({"role": "user", "content": prompt})]),
+        )
+        .unwrap();
+}
+
+#[test]
+fn wrong_distance_metric_falls_back_to_isolated_index() {
+    let prompt = "hello prompt";
+    let tag = "key1";
+    let isolated = format!("{INDEX}_isolated");
+    let connection = MockRedisConnection::new([
+        MockCmd::new(
+            redis::cmd("FT.INFO").arg(INDEX),
+            Ok(info_with_vector(vector_attribute_with(3, "FLOAT32", "L2"))),
+        ),
+        MockCmd::new(
+            redis::cmd("FT.INFO").arg(&isolated),
+            Err::<redis::Value, _>(unknown_index_error()),
+        ),
+        MockCmd::new(create_index_command(&isolated, 3), Ok("OK")),
+        MockCmd::new(
+            redis::cmd("HSET")
+                .arg(format!("{isolated}:{}", entry_id(prompt, tag)))
+                .arg("entry_id")
+                .arg(entry_id(prompt, tag))
+                .arg("prompt")
+                .arg(prompt)
+                .arg("response")
+                .arg(encoded(&entry()))
+                .arg("prompt_vector")
+                .arg(vector_bytes(&[0.1f32, 0.2, 0.3]))
+                .arg("inserted_at")
+                .arg("1700000000.5")
+                .arg("updated_at")
+                .arg("1700000000.5")
+                .arg("litellm_cache_key")
+                .arg(tag),
+            Ok(7),
+        ),
+    ])
+    .assert_all_commands_consumed();
+    let (embedder, _) = FakeEmbedder::new(&[]);
+    let cache = RedisSemanticCache::with_connection(connection, embedder, config())
+        .with_clock(|| 1700000000.5);
+
+    cache
+        .set_cache(
+            tag,
+            entry(),
+            &messages_context(vec![json!({"role": "user", "content": prompt})]),
+        )
+        .unwrap();
+}
+
 #[test]
 fn tag_special_characters_are_escaped_in_search_filter() {
     let vector = vec![0.1f32, 0.2, 0.3];

From 4509eb991453a89ba45c13b500bc5533d4801c9a Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:22:01 +0000
Subject: [PATCH 098/160] fix(cache): align native semantic cache scope keys

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.lock                       |   1 +
 litellm-rust/crates/python-bridge/Cargo.toml  |   1 +
 .../crates/python-bridge/src/cache/config.rs  |  57 ++++++++-
 .../crates/python-bridge/src/cache/native.rs  | 115 +++++++++++++++++-
 .../test_valkey_semantic_cache_native.py      | 115 ++++++++++++++++++
 5 files changed, 286 insertions(+), 3 deletions(-)

diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index e4c9c385f1f..6ca4ff69648 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -2700,6 +2700,7 @@ dependencies = [
  "rstest",
  "serde",
  "serde_json",
+ "sha2 0.10.9",
  "tokio",
  "tokio-tungstenite",
 ]
diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml
index ce2405f33c4..afc4f8833b5 100644
--- a/litellm-rust/crates/python-bridge/Cargo.toml
+++ b/litellm-rust/crates/python-bridge/Cargo.toml
@@ -46,6 +46,7 @@ tokio = { workspace = true, features = ["sync"] }
 criterion.workspace = true
 futures-util.workspace = true
 rstest.workspace = true
+sha2.workspace = true
 tokio-tungstenite.workspace = true
 
 [[bench]]
diff --git a/litellm-rust/crates/python-bridge/src/cache/config.rs b/litellm-rust/crates/python-bridge/src/cache/config.rs
index e074c5e2f5d..169bb5accda 100644
--- a/litellm-rust/crates/python-bridge/src/cache/config.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/config.rs
@@ -322,9 +322,17 @@ fn project_valkey_semantic(
 ) -> PyResult<Result<ValkeySemanticCacheConfig, UnsupportedCacheConfig>> {
     let client = backend.getattr("sync_client")?;
     let pool = client.getattr("connection_pool")?;
-    let Ok((resolved, _is_tls)) = project_connection_pool(&pool)? else {
+    let Ok((resolved, is_tls)) = project_connection_pool(&pool)? else {
         return Ok(Err(UnsupportedCacheConfig::RedisConnection));
     };
+    for key in ["credential_provider", "redis_connect_func"] {
+        if has_value(&resolved, key)? {
+            return Ok(Err(UnsupportedCacheConfig::RedisCredentials));
+        }
+    }
+    if is_tls {
+        return Ok(Err(UnsupportedCacheConfig::RedisConnection));
+    }
     let connection = RedisConnectionConfig {
         host: required_string(&resolved, "host")?,
         port: u16::try_from(required_i64(&resolved, "port")?)
@@ -682,6 +690,53 @@ mod tests {
         });
     }
 
+    #[test]
+    fn valkey_semantic_tls_stays_on_python() {
+        Python::initialize();
+        Python::attach(|py| {
+            let facade = facade(
+                py,
+                "pool = ConnectionPool()\n\
+                 pool.connection_class = SSLConnection\n\
+                 pool.connection_kwargs = {'host': 'cache.internal', 'port': 6390}\n\
+                 client = SimpleNamespace(connection_pool=pool)\n\
+                 backend = SimpleNamespace(similarity_threshold=0.85, index_name='semantic_idx', embedding_model='text-embedding-3-small', sync_client=client)\n\
+                 facade = SimpleNamespace(type='valkey-semantic', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)",
+            );
+            let CacheConfigProjection::Unsupported(reason) =
+                NativeCacheConfig::project(&facade).unwrap()
+            else {
+                panic!("TLS Valkey semantic cache should stay on Python");
+            };
+            assert_eq!(
+                reason.message(),
+                "native Redis connection type is not implemented"
+            );
+        });
+    }
+
+    #[test]
+    fn valkey_semantic_dynamic_auth_stays_on_python() {
+        Python::initialize();
+        Python::attach(|py| {
+            let facade = facade(
+                py,
+                "pool = ConnectionPool()\n\
+                 pool.connection_class = Connection\n\
+                 pool.connection_kwargs = {'host': 'cache.internal', 'port': 6390, 'credential_provider': object()}\n\
+                 client = SimpleNamespace(connection_pool=pool)\n\
+                 backend = SimpleNamespace(similarity_threshold=0.85, index_name='semantic_idx', embedding_model='text-embedding-3-small', sync_client=client)\n\
+                 facade = SimpleNamespace(type='valkey-semantic', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)",
+            );
+            let CacheConfigProjection::Unsupported(reason) =
+                NativeCacheConfig::project(&facade).unwrap()
+            else {
+                panic!("dynamic Valkey authentication must stay on Python");
+            };
+            assert_eq!(reason.message(), "native Redis credentials require Python");
+        });
+    }
+
     #[test]
     fn dynamic_redis_auth_stays_on_python() {
         Python::initialize();
diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs
index d314cd41ac5..e7c875579fd 100644
--- a/litellm-rust/crates/python-bridge/src/cache/native.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/native.rs
@@ -6,13 +6,50 @@ use litellm_cache::{
 use litellm_cache_memory::InMemoryCache;
 use litellm_cache_redis::RedisCache;
 use litellm_cache_response::{
-    CacheEntry, PartialHits, ResponseCache, ResponseCacheCodec, ResponseCacheRequest, WriteBuffer,
+    CacheEntry, CacheKeyField, PartialHits, ResponseCache, ResponseCacheCodec,
+    ResponseCacheRequest, WriteBuffer,
 };
 use litellm_cache_valkey_semantic::{ValkeySemanticCache, ValkeySemanticConfig};
 use serde_json::Value;
 
 use super::{embedder::PythonEmbedder, request::NativeRequest};
 
+fn semantic_key(request: &NativeRequest, scope: &str) -> litellm_cache_response::CacheKeyInput {
+    let mut key = request.key.clone();
+    if key.preset.is_some() {
+        return key;
+    }
+    key.fields
+        .retain(|field| !matches!(field.name.as_str(), "messages" | "prompt" | "input"));
+    const TENANT: [&str; 3] = [
+        "user_api_key",
+        "user_api_key_team_id",
+        "user_api_key_org_id",
+    ];
+    let end_user = (scope == "end_user").then_some("user_api_key_end_user_id");
+    for name in TENANT.into_iter().chain(end_user) {
+        let Some(value) = request
+            .metadata
+            .as_ref()
+            .and_then(|metadata| metadata.get(name))
+        else {
+            continue;
+        };
+        let value = match value {
+            Value::Null => continue,
+            Value::String(text) => text.clone(),
+            other => other.to_string(),
+        };
+        key.fields.push(CacheKeyField {
+            name: name.to_owned(),
+            value: Some(value),
+            api_parameter: true,
+            internal_parameter: false,
+        });
+    }
+    key
+}
+
 #[derive(Clone)]
 pub(super) enum NativeResponseCache {
     Memory(Arc<ResponseCache<InMemoryCache<CacheEntry>>>),
@@ -88,7 +125,7 @@ impl NativeResponseCache {
         scope: &str,
     ) -> ResponseCacheRequest<SemanticCacheContext> {
         ResponseCacheRequest {
-            key: request.key.clone(),
+            key: semantic_key(request, scope),
             controls: request.controls,
             context: SemanticCacheContext {
                 input: request.input.clone(),
@@ -329,3 +366,77 @@ impl NativeResponseCache {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use litellm_cache_response::{CacheControls, CacheKeyInput, cache_key};
+    use serde_json::json;
+    use sha2::{Digest, Sha256};
+
+    use super::*;
+
+    fn native_request(key: CacheKeyInput, metadata: Value) -> NativeRequest {
+        NativeRequest {
+            key,
+            controls: CacheControls::default(),
+            ttl: None,
+            max_age: None,
+            messages: Some(json!([{"role": "user", "content": "prompt"}])),
+            input: None,
+            metadata: Some(metadata),
+        }
+    }
+
+    #[test]
+    fn semantic_key_matches_python_scope_material() {
+        let key = CacheKeyInput {
+            fields: vec![
+                CacheKeyField {
+                    name: "model".to_owned(),
+                    value: Some("gpt-4.1".to_owned()),
+                    api_parameter: true,
+                    internal_parameter: false,
+                },
+                CacheKeyField {
+                    name: "messages".to_owned(),
+                    value: Some("prompt".to_owned()),
+                    api_parameter: true,
+                    internal_parameter: false,
+                },
+            ],
+            ..Default::default()
+        };
+        let request = native_request(
+            key,
+            json!({"user_api_key": "k1", "user_api_key_team_id": null}),
+        );
+        let expected = format!("{:x}", Sha256::digest(b"model: gpt-4.1user_api_key: k1"));
+        assert_eq!(cache_key(&semantic_key(&request, "key")), expected);
+
+        let end_user_request = native_request(
+            request.key.clone(),
+            json!({"user_api_key": "k1", "user_api_key_end_user_id": "u1"}),
+        );
+        let expected = format!(
+            "{:x}",
+            Sha256::digest(b"model: gpt-4.1user_api_key: k1user_api_key_end_user_id: u1")
+        );
+        assert_eq!(
+            cache_key(&semantic_key(&end_user_request, "end_user")),
+            expected
+        );
+
+        let preset_request = native_request(
+            CacheKeyInput {
+                preset: Some("preset-key".to_owned()),
+                ..Default::default()
+            },
+            json!({"user_api_key": "k1"}),
+        );
+        assert_eq!(
+            semantic_key(&preset_request, "end_user").preset.as_deref(),
+            Some("preset-key")
+        );
+        assert!(semantic_key(&preset_request, "end_user").fields.is_empty());
+    }
+}
diff --git a/tests/test_litellm_rust/test_valkey_semantic_cache_native.py b/tests/test_litellm_rust/test_valkey_semantic_cache_native.py
index dc4ede8ea30..81fcf00ffc0 100644
--- a/tests/test_litellm_rust/test_valkey_semantic_cache_native.py
+++ b/tests/test_litellm_rust/test_valkey_semantic_cache_native.py
@@ -46,6 +46,56 @@ def _request(prompt: str = "semantic cache prompt") -> dict[str, object]:
     }
 
 
+def _field_request(
+    prompt: str,
+    metadata: Mapping[str, object],
+) -> dict[str, object]:
+    return {
+        "key": {
+            "fields": [
+                {
+                    "name": "model",
+                    "value": "gpt-4.1",
+                    "api_parameter": True,
+                    "internal_parameter": False,
+                },
+                {
+                    "name": "messages",
+                    "value": prompt,
+                    "api_parameter": True,
+                    "internal_parameter": False,
+                },
+            ]
+        },
+        "messages": [{"role": "user", "content": prompt}],
+        "metadata": dict(metadata),
+    }
+
+
+def _facade(
+    url: str,
+    index_name: str,
+    embeddings: Mapping[str, list[float]],
+) -> Cache:
+    facade: Final = Cache(
+        type=LiteLLMCacheType.VALKEY_SEMANTIC,
+        redis_url=url,
+        similarity_threshold=0.8,
+        valkey_semantic_cache_index_name=index_name,
+    )
+    vectors: Final = embeddings
+
+    def embed(prompt: str, metadata: Mapping[str, object] | None = None) -> list[float]:
+        return vectors[prompt]
+
+    async def async_embedding(prompt: str, metadata: dict[str, object] | None = None) -> list[float]:
+        return vectors[prompt]
+
+    facade.cache._get_embedding = embed
+    facade.cache._get_async_embedding = async_embedding
+    return facade
+
+
 def _backend(
     url: str,
     index_name: str,
@@ -268,6 +318,71 @@ def test_subclass_backend_falls_back_to_python(
     assert resolver.resolve().kind == "python_callback"
 
 
+def test_field_key_matches_python_semantic_scope(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    facade: Final = _facade(valkey_url, index_name, {"semantic cache prompt": [1.0, 0.0]})
+    metadata: Final = {"user_api_key": "k1"}
+    expected: Final = facade.get_cache_key(
+        model="gpt-4.1",
+        messages=[{"role": "user", "content": "semantic cache prompt"}],
+        metadata=metadata,
+    )
+    handle: Final = _native._CacheTestHandle.valkey_semantic(
+        valkey_url,
+        0.8,
+        index_name,
+        facade.cache,
+    )
+    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    binding.store(_field_request("semantic cache prompt", metadata), {"answer": "scoped"})
+    client: Final = redis.Redis.from_url(valkey_url)
+    documents: Final = list(client.scan_iter(f"{index_name}:*"))
+    assert len(documents) == 1
+    document_parts: Final = documents[0].decode().split(":")
+    assert document_parts[1] == hashlib.sha256(expected.encode()).hexdigest()
+    client.close()
+
+
+def test_field_key_isolates_tenant_scope(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    facade: Final = _facade(valkey_url, index_name, {"semantic cache prompt": [1.0, 0.0]})
+    handle: Final = _native._CacheTestHandle.valkey_semantic(
+        valkey_url,
+        0.8,
+        index_name,
+        facade.cache,
+    )
+    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    binding.store(
+        _field_request("semantic cache prompt", {"user_api_key": "k1"}),
+        {"answer": "tenant one"},
+    )
+    assert (
+        binding.lookup(_field_request("semantic cache prompt", {"user_api_key": "k2"}))
+        is None
+    )
+    assert binding.lookup(_field_request("semantic cache prompt", {"user_api_key": "k1"})) == {
+        "answer": "tenant one"
+    }
+
+
+def test_tls_valkey_facade_falls_back_to_python(
+    index_name: str,
+) -> None:
+    facade: Final = Cache(
+        type=LiteLLMCacheType.VALKEY_SEMANTIC,
+        redis_url="rediss://127.0.0.1:6390/0",
+        similarity_threshold=0.8,
+        valkey_semantic_cache_index_name=index_name,
+    )
+    resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade))
+    assert resolver.resolve().kind == "python_callback"
+
+
 async def test_ping_maps_unsupported_native_operation_to_not_implemented(
     valkey_url: str,
     index_name: str,

From 974d9f97ff13b4deb7afa19b2ab43387baf597a0 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:22:09 +0000
Subject: [PATCH 099/160] revert(rust): restore thin LTO in the release profile

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
index 7fa8de05f60..05eea6bc299 100644
--- a/litellm-rust/Cargo.toml
+++ b/litellm-rust/Cargo.toml
@@ -75,7 +75,7 @@ veil = "0.3.0"
 
 [profile.release]
 opt-level = 3
-lto = "fat"
+lto = "thin"
 codegen-units = 1
 panic = "unwind"
 debug = false

From 769917a7e8d9ffccca42f084dddfab94f299e246 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:22:15 +0000
Subject: [PATCH 100/160] revert(rust-wheel): restore release optimization

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
index 8725b25cfc7..570d0dd3568 100644
--- a/litellm-rust/Cargo.toml
+++ b/litellm-rust/Cargo.toml
@@ -73,7 +73,7 @@ fancy-regex = "0.19.2"
 veil = "0.3.0"
 
 [profile.release]
-opt-level = 2
+opt-level = 3
 lto = "thin"
 codegen-units = 1
 panic = "unwind"

From 3ce436af5ed72281214df3449ebf31323e8d3874 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:22:22 +0000
Subject: [PATCH 101/160] refactor(cache-disk): isolate python compatibility
 behind a value adapter

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.lock                       |   2 +
 litellm-rust/crates/cache-disk/Cargo.toml     |   2 +
 litellm-rust/crates/cache-disk/src/adapter.rs |  10 +
 litellm-rust/crates/cache-disk/src/cache.rs   | 112 ++----
 litellm-rust/crates/cache-disk/src/lib.rs     |   5 +-
 litellm-rust/crates/cache-disk/src/pickle.rs  |  48 ---
 .../crates/cache-disk/src/python/mod.rs       |  81 ++++
 .../crates/cache-disk/src/python/value.rs     | 173 ++++++++
 litellm-rust/crates/cache-disk/tests/cache.rs | 369 +++++++++++-------
 .../crates/cache-disk/tests/python_compat.rs  | 105 +++++
 10 files changed, 643 insertions(+), 264 deletions(-)
 create mode 100644 litellm-rust/crates/cache-disk/src/adapter.rs
 delete mode 100644 litellm-rust/crates/cache-disk/src/pickle.rs
 create mode 100644 litellm-rust/crates/cache-disk/src/python/mod.rs
 create mode 100644 litellm-rust/crates/cache-disk/src/python/value.rs
 create mode 100644 litellm-rust/crates/cache-disk/tests/python_compat.rs

diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index 554245dbb6e..0a1208787eb 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -2525,7 +2525,9 @@ name = "litellm-cache-disk"
 version = "0.1.0"
 dependencies = [
  "litellm-cache",
+ "py_literal",
  "rand 0.8.7",
+ "rstest",
  "rusqlite",
  "serde-pickle",
  "serde_json",
diff --git a/litellm-rust/crates/cache-disk/Cargo.toml b/litellm-rust/crates/cache-disk/Cargo.toml
index 023055e391a..b96994b3b55 100644
--- a/litellm-rust/crates/cache-disk/Cargo.toml
+++ b/litellm-rust/crates/cache-disk/Cargo.toml
@@ -7,6 +7,7 @@ repository.workspace = true
 
 [dependencies]
 litellm-cache.workspace = true
+py_literal = "0.4.0"
 rand.workspace = true
 rusqlite = { version = "0.40", features = ["bundled"] }
 serde-pickle = "1.2"
@@ -14,4 +15,5 @@ serde_json.workspace = true
 tokio.workspace = true
 
 [dev-dependencies]
+rstest.workspace = true
 tempfile = "3.27.0"
diff --git a/litellm-rust/crates/cache-disk/src/adapter.rs b/litellm-rust/crates/cache-disk/src/adapter.rs
new file mode 100644
index 00000000000..b6d318d5509
--- /dev/null
+++ b/litellm-rust/crates/cache-disk/src/adapter.rs
@@ -0,0 +1,10 @@
+use litellm_cache::Error;
+
+use crate::StoredValue;
+
+pub trait ValueAdapter: Send + Sync + 'static {
+    fn read(&self, value: StoredValue) -> Result<Option<Vec<u8>>, Error>;
+    fn write(&self, payload: Vec<u8>) -> StoredValue;
+    fn counter_seed(&self, value: Option<StoredValue>) -> Result<f64, Error>;
+    fn counter_value(&self, value: f64) -> StoredValue;
+}
diff --git a/litellm-rust/crates/cache-disk/src/cache.rs b/litellm-rust/crates/cache-disk/src/cache.rs
index bcce3a525b1..98182d93968 100644
--- a/litellm-rust/crates/cache-disk/src/cache.rs
+++ b/litellm-rust/crates/cache-disk/src/cache.rs
@@ -8,28 +8,42 @@ use litellm_cache::{
     BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, CacheConnectionStatus,
     CounterCache, DeleteCache, Error, ExactCacheContext, FlushCache,
 };
-use serde_json::Value;
 
-use crate::{DiskStore, DiskcacheSqliteStore, StoredValue, pickle};
+use crate::{DiskStore, DiskcacheSqliteStore, PythonDiskCacheAdapter, StoredValue, ValueAdapter};
 
-pub struct DiskCache<S, D = DiskcacheSqliteStore> {
+pub struct DiskCache<S, D = DiskcacheSqliteStore, A = PythonDiskCacheAdapter> {
     store: Arc<D>,
+    adapter: Arc<A>,
     codec: S,
 }
 
 impl<S: CacheCodec> DiskCache<S> {
+    #[allow(clippy::default_constructed_unit_structs)]
     pub fn open(directory: impl AsRef<Path>, codec: S) -> Result<Self, Error> {
         Ok(Self {
             store: Arc::new(DiskcacheSqliteStore::open(directory)?),
+            adapter: Arc::new(PythonDiskCacheAdapter::default()),
             codec,
         })
     }
 }
 
-impl<S: CacheCodec, D: DiskStore> DiskCache<S, D> {
+impl<S: CacheCodec, D: DiskStore> DiskCache<S, D, PythonDiskCacheAdapter> {
+    #[allow(clippy::default_constructed_unit_structs)]
     pub fn with_store(store: D, codec: S) -> Self {
         Self {
             store: Arc::new(store),
+            adapter: Arc::new(PythonDiskCacheAdapter::default()),
+            codec,
+        }
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> DiskCache<S, D, A> {
+    pub fn with_adapter(store: D, adapter: A, codec: S) -> Self {
+        Self {
+            store: Arc::new(store),
+            adapter: Arc::new(adapter),
             codec,
         }
     }
@@ -39,7 +53,7 @@ impl<S: CacheCodec, D: DiskStore> DiskCache<S, D> {
     }
 
     fn decode_stored(&self, value: StoredValue) -> Result<Option<S::Value>, Error> {
-        let Some(bytes) = payload(value)? else {
+        let Some(bytes) = self.adapter.read(value)? else {
             return Ok(None);
         };
         self.codec.decode(&bytes).map(Some)
@@ -56,7 +70,7 @@ impl<S: CacheCodec, D: DiskStore> DiskCache<S, D> {
     }
 }
 
-impl<S: CacheCodec, D: DiskStore> BaseCache for DiskCache<S, D> {
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> BaseCache for DiskCache<S, D, A> {
     type Value = S::Value;
     type Context = ExactCacheContext;
 
@@ -70,7 +84,7 @@ impl<S: CacheCodec, D: DiskStore> BaseCache for DiskCache<S, D> {
         value: Self::Value,
         context: &Self::Context,
     ) -> Result<(), Error> {
-        let value = StoredValue::Bytes(self.codec.encode(&value)?);
+        let value = self.adapter.write(self.codec.encode(&value)?);
         let expire_time = context.ttl.map(|ttl| unix_now() + ttl.as_secs_f64());
         self.store.set(key, value, expire_time, unix_now())
     }
@@ -89,7 +103,7 @@ impl<S: CacheCodec, D: DiskStore> BaseCache for DiskCache<S, D> {
         value: Self::Value,
         context: ExactCacheContext,
     ) -> Result<(), Error> {
-        let value = StoredValue::Bytes(self.codec.encode(&value)?);
+        let value = self.adapter.write(self.codec.encode(&value)?);
         let expire_time = context.ttl.map(|ttl| unix_now() + ttl.as_secs_f64());
         let key = key.to_string();
         Self::run_blocking(Arc::clone(&self.store), move |store| {
@@ -124,7 +138,7 @@ impl<S: CacheCodec, D: DiskStore> BaseCache for DiskCache<S, D> {
             .map(|(key, value)| {
                 self.codec
                     .encode(&value)
-                    .map(|value| (key, StoredValue::Bytes(value)))
+                    .map(|value| (key, self.adapter.write(value)))
             })
             .collect::<Result<Vec<_>, _>>()?;
         let expire_after = context.ttl;
@@ -162,7 +176,7 @@ impl<S: CacheCodec, D: DiskStore> BaseCache for DiskCache<S, D> {
     }
 }
 
-impl<S: CacheCodec, D: DiskStore> BatchCache for DiskCache<S, D> {
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> BatchCache for DiskCache<S, D, A> {
     fn batch_get_cache(
         &self,
         keys: &[String],
@@ -204,7 +218,7 @@ impl<S: CacheCodec, D: DiskStore> BatchCache for DiskCache<S, D> {
     }
 }
 
-impl<S: CacheCodec, D: DiskStore> DeleteCache for DiskCache<S, D> {
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> DeleteCache for DiskCache<S, D, A> {
     fn delete_cache(&self, key: &str) -> Result<(), Error> {
         self.store.pop(key, unix_now()).map(|_| ())
     }
@@ -218,7 +232,7 @@ impl<S: CacheCodec, D: DiskStore> DeleteCache for DiskCache<S, D> {
     }
 }
 
-impl<S: CacheCodec, D: DiskStore> FlushCache for DiskCache<S, D> {
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> FlushCache for DiskCache<S, D, A> {
     fn flush_cache(&self) -> Result<(), Error> {
         self.store.clear()
     }
@@ -228,14 +242,22 @@ impl<S: CacheCodec, D: DiskStore> FlushCache for DiskCache<S, D> {
     }
 }
 
-impl<S: CacheCodec<Value = f64>, D: DiskStore> CounterCache for DiskCache<S, D> {
+impl<S: CacheCodec<Value = f64>, D: DiskStore, A: ValueAdapter> CounterCache
+    for DiskCache<S, D, A>
+{
     fn increment_cache(
         &self,
         key: &str,
         amount: f64,
         context: ExactCacheContext,
     ) -> Result<f64, Error> {
-        increment(self.store.as_ref(), key, amount, context.ttl)
+        increment(
+            self.adapter.as_ref(),
+            self.store.as_ref(),
+            key,
+            amount,
+            context.ttl,
+        )
     }
 
     async fn async_increment(
@@ -245,14 +267,16 @@ impl<S: CacheCodec<Value = f64>, D: DiskStore> CounterCache for DiskCache<S, D>
         context: ExactCacheContext,
     ) -> Result<f64, Error> {
         let key = key.to_string();
+        let adapter = Arc::clone(&self.adapter);
         Self::run_blocking(Arc::clone(&self.store), move |store| {
-            increment(store, &key, amount, context.ttl)
+            increment(adapter.as_ref(), store, &key, amount, context.ttl)
         })
         .await
     }
 }
 
-fn increment<D: DiskStore>(
+fn increment<A: ValueAdapter, D: DiskStore>(
+    adapter: &A,
     store: &D,
     key: &str,
     amount: f64,
@@ -260,25 +284,9 @@ fn increment<D: DiskStore>(
 ) -> Result<f64, Error> {
     let mut result = None;
     let mut apply = |current: Option<StoredValue>| {
-        let initial = match current {
-            Some(StoredValue::Integer(value)) => value as f64,
-            Some(StoredValue::Pickle(value)) => match pickle::decode(&value)? {
-                Value::Number(value) => value
-                    .as_i64()
-                    .map(|value| value as f64)
-                    .or_else(|| value.as_u64().map(|value| value as f64))
-                    .unwrap_or_default(),
-                _ => 0.0,
-            },
-            _ => 0.0,
-        };
+        let initial = adapter.counter_seed(current)?;
         let value = initial + amount;
-        let stored = if value.fract() == 0.0 && value >= i64::MIN as f64 && value <= i64::MAX as f64
-        {
-            StoredValue::Integer(value as i64)
-        } else {
-            StoredValue::Float(value)
-        };
+        let stored = adapter.counter_value(value);
         result = Some(value);
         Ok((stored, ttl.map(|ttl| unix_now() + ttl.as_secs_f64())))
     };
@@ -286,42 +294,6 @@ fn increment<D: DiskStore>(
     result.ok_or(Error::InvalidEntry)
 }
 
-fn payload(value: StoredValue) -> Result<Option<Vec<u8>>, Error> {
-    match value {
-        StoredValue::Bytes(value) if value.is_empty() => Ok(None),
-        StoredValue::Bytes(value) => Ok(Some(value)),
-        StoredValue::Text(value) if value.is_empty() => Ok(None),
-        StoredValue::Text(value) => Ok(Some(value.into_bytes())),
-        StoredValue::Integer(0) => Ok(None),
-        StoredValue::Integer(value) => Ok(Some(value.to_string().into_bytes())),
-        StoredValue::Float(0.0) => Ok(None),
-        StoredValue::Float(value) => serde_json::to_vec(&value)
-            .map(Some)
-            .map_err(|_| Error::InvalidEntry),
-        StoredValue::Pickle(value) => {
-            let value = pickle::decode(&value)?;
-            if is_falsy(&value) {
-                Ok(None)
-            } else {
-                serde_json::to_vec(&value)
-                    .map(Some)
-                    .map_err(|_| Error::InvalidEntry)
-            }
-        }
-    }
-}
-
-fn is_falsy(value: &Value) -> bool {
-    match value {
-        Value::Null | Value::Bool(false) => true,
-        Value::Number(value) => value.as_f64().is_some_and(|value| value == 0.0),
-        Value::String(value) => value.is_empty(),
-        Value::Array(value) => value.is_empty(),
-        Value::Object(value) => value.is_empty(),
-        Value::Bool(true) => false,
-    }
-}
-
 fn unix_now() -> f64 {
     SystemTime::now()
         .duration_since(UNIX_EPOCH)
diff --git a/litellm-rust/crates/cache-disk/src/lib.rs b/litellm-rust/crates/cache-disk/src/lib.rs
index 189d12ea793..9b2ffc24915 100644
--- a/litellm-rust/crates/cache-disk/src/lib.rs
+++ b/litellm-rust/crates/cache-disk/src/lib.rs
@@ -1,8 +1,11 @@
+mod adapter;
 mod cache;
-mod pickle;
+mod python;
 mod sqlite;
 mod store;
 
+pub use adapter::ValueAdapter;
 pub use cache::DiskCache;
+pub use python::PythonDiskCacheAdapter;
 pub use sqlite::DiskcacheSqliteStore;
 pub use store::{DiskStore, StoredValue};
diff --git a/litellm-rust/crates/cache-disk/src/pickle.rs b/litellm-rust/crates/cache-disk/src/pickle.rs
deleted file mode 100644
index 1d31e0bbeb6..00000000000
--- a/litellm-rust/crates/cache-disk/src/pickle.rs
+++ /dev/null
@@ -1,48 +0,0 @@
-use litellm_cache::Error;
-use serde_json::{Map, Number, Value};
-
-pub(crate) fn decode(bytes: &[u8]) -> Result<Value, Error> {
-    let value = serde_pickle::value_from_slice(bytes, Default::default())
-        .map_err(|_| Error::InvalidEntry)?;
-    convert(value)
-}
-
-fn convert(value: serde_pickle::Value) -> Result<Value, Error> {
-    match value {
-        serde_pickle::Value::None => Ok(Value::Null),
-        serde_pickle::Value::Bool(value) => Ok(Value::Bool(value)),
-        serde_pickle::Value::I64(value) => Ok(Value::Number(value.into())),
-        serde_pickle::Value::Int(value) => {
-            if let Ok(value) = value.to_string().parse::<i64>() {
-                Ok(Value::Number(value.into()))
-            } else if let Ok(value) = value.to_string().parse::<u64>() {
-                Ok(Value::Number(value.into()))
-            } else {
-                Err(Error::InvalidEntry)
-            }
-        }
-        serde_pickle::Value::F64(value) => Number::from_f64(value)
-            .map(Value::Number)
-            .ok_or(Error::InvalidEntry),
-        serde_pickle::Value::String(value) => Ok(Value::String(value)),
-        serde_pickle::Value::List(values) | serde_pickle::Value::Tuple(values) => values
-            .into_iter()
-            .map(convert)
-            .collect::<Result<Vec<_>, _>>()
-            .map(Value::Array),
-        serde_pickle::Value::Set(values) | serde_pickle::Value::FrozenSet(values) => values
-            .into_iter()
-            .map(|value| convert(value.into_value()))
-            .collect::<Result<Vec<_>, _>>()
-            .map(Value::Array),
-        serde_pickle::Value::Dict(values) => values
-            .into_iter()
-            .map(|(key, value)| match key {
-                serde_pickle::HashableValue::String(key) => Ok((key, convert(value)?)),
-                _ => Err(Error::InvalidEntry),
-            })
-            .collect::<Result<Map<_, _>, _>>()
-            .map(Value::Object),
-        serde_pickle::Value::Bytes(_) => Err(Error::InvalidEntry),
-    }
-}
diff --git a/litellm-rust/crates/cache-disk/src/python/mod.rs b/litellm-rust/crates/cache-disk/src/python/mod.rs
new file mode 100644
index 00000000000..5c9b57de000
--- /dev/null
+++ b/litellm-rust/crates/cache-disk/src/python/mod.rs
@@ -0,0 +1,81 @@
+mod value;
+
+use litellm_cache::Error;
+use py_literal::Value;
+
+use crate::{StoredValue, ValueAdapter};
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct PythonDiskCacheAdapter;
+
+impl PythonDiskCacheAdapter {
+    fn python_get_cache(value: StoredValue) -> Result<Option<Value>, Error> {
+        let value = match value {
+            StoredValue::Bytes(value) => Value::Bytes(value),
+            StoredValue::Text(value) => Value::String(value),
+            StoredValue::Integer(value) => {
+                value::from_json(serde_json::Value::Number(value.into()))
+            }
+            StoredValue::Float(value) => Value::Float(value),
+            StoredValue::Pickle(value) => value::from_pickle(&value)?,
+        };
+        if !value::is_truthy(&value) {
+            return Ok(None);
+        }
+        match value {
+            Value::String(text) => Ok(Some(
+                value::from_json_text(&text).unwrap_or(Value::String(text)),
+            )),
+            Value::Bytes(bytes) => match std::str::from_utf8(&bytes) {
+                Ok(text) => Ok(Some(
+                    value::from_json_text(text).unwrap_or(Value::Bytes(bytes)),
+                )),
+                Err(_) => Ok(Some(Value::Bytes(bytes))),
+            },
+            value => Ok(Some(value)),
+        }
+    }
+}
+
+impl ValueAdapter for PythonDiskCacheAdapter {
+    fn read(&self, value: StoredValue) -> Result<Option<Vec<u8>>, Error> {
+        let raw = match &value {
+            StoredValue::Text(value) => Some(value.as_bytes().to_vec()),
+            StoredValue::Bytes(value) => Some(value.clone()),
+            StoredValue::Integer(_) | StoredValue::Float(_) | StoredValue::Pickle(_) => None,
+        };
+        let Some(value) = Self::python_get_cache(value)? else {
+            return Ok(None);
+        };
+        if let Some(raw) = raw {
+            return Ok(Some(raw));
+        }
+        value::to_json(&value).map(Some)
+    }
+
+    fn write(&self, payload: Vec<u8>) -> StoredValue {
+        StoredValue::Bytes(payload)
+    }
+
+    fn counter_seed(&self, value: Option<StoredValue>) -> Result<f64, Error> {
+        let Some(value) = value else {
+            return Ok(0.0);
+        };
+        let Some(value) = Self::python_get_cache(value)? else {
+            return Ok(0.0);
+        };
+        Ok(if value::is_int(&value) {
+            value::to_f64(&value).unwrap_or(0.0)
+        } else {
+            0.0
+        })
+    }
+
+    fn counter_value(&self, value: f64) -> StoredValue {
+        if value.fract() == 0.0 && value >= i64::MIN as f64 && value <= i64::MAX as f64 {
+            StoredValue::Integer(value as i64)
+        } else {
+            StoredValue::Float(value)
+        }
+    }
+}
diff --git a/litellm-rust/crates/cache-disk/src/python/value.rs b/litellm-rust/crates/cache-disk/src/python/value.rs
new file mode 100644
index 00000000000..645eba7b757
--- /dev/null
+++ b/litellm-rust/crates/cache-disk/src/python/value.rs
@@ -0,0 +1,173 @@
+use litellm_cache::Error;
+use py_literal::Value;
+use serde_json::{Map, Number};
+
+pub(crate) fn from_pickle(bytes: &[u8]) -> Result<Value, Error> {
+    let value = serde_pickle::value_from_slice(bytes, Default::default())
+        .map_err(|_| Error::InvalidEntry)?;
+    from_pickle_value(value)
+}
+
+fn from_pickle_value(value: serde_pickle::Value) -> Result<Value, Error> {
+    match value {
+        serde_pickle::Value::None => Ok(Value::None),
+        serde_pickle::Value::Bool(value) => Ok(Value::Boolean(value)),
+        serde_pickle::Value::I64(value) => integer(value.to_string()),
+        serde_pickle::Value::Int(value) => integer(value.to_string()),
+        serde_pickle::Value::F64(value) => Ok(Value::Float(value)),
+        serde_pickle::Value::String(value) => Ok(Value::String(value)),
+        serde_pickle::Value::Bytes(value) => Ok(Value::Bytes(value)),
+        serde_pickle::Value::List(values) => values
+            .into_iter()
+            .map(from_pickle_value)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::List),
+        serde_pickle::Value::Tuple(values) => values
+            .into_iter()
+            .map(from_pickle_value)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::Tuple),
+        serde_pickle::Value::Set(values) => values
+            .into_iter()
+            .map(from_pickle_hashable)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::Set),
+        serde_pickle::Value::FrozenSet(values) => values
+            .into_iter()
+            .map(from_pickle_hashable)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::Set),
+        serde_pickle::Value::Dict(values) => values
+            .into_iter()
+            .map(|(key, value)| Ok((from_pickle_hashable(key)?, from_pickle_value(value)?)))
+            .collect::<Result<Vec<_>, Error>>()
+            .map(Value::Dict),
+    }
+}
+
+fn from_pickle_hashable(value: serde_pickle::HashableValue) -> Result<Value, Error> {
+    Ok(match value {
+        serde_pickle::HashableValue::None => Value::None,
+        serde_pickle::HashableValue::Bool(value) => Value::Boolean(value),
+        serde_pickle::HashableValue::I64(value) => integer(value.to_string())?,
+        serde_pickle::HashableValue::Int(value) => integer(value.to_string())?,
+        serde_pickle::HashableValue::F64(value) => Value::Float(value),
+        serde_pickle::HashableValue::Bytes(value) => Value::Bytes(value),
+        serde_pickle::HashableValue::String(value) => Value::String(value),
+        serde_pickle::HashableValue::Tuple(values) => Value::Tuple(
+            values
+                .into_iter()
+                .map(from_pickle_hashable)
+                .collect::<Result<Vec<_>, _>>()?,
+        ),
+        serde_pickle::HashableValue::FrozenSet(values) => Value::Set(
+            values
+                .into_iter()
+                .map(from_pickle_hashable)
+                .collect::<Result<Vec<_>, _>>()?,
+        ),
+    })
+}
+
+fn integer(value: String) -> Result<Value, Error> {
+    value.parse().map_err(|_| Error::InvalidEntry)
+}
+
+pub(crate) fn from_json(value: serde_json::Value) -> Value {
+    match value {
+        serde_json::Value::Null => Value::None,
+        serde_json::Value::Bool(value) => Value::Boolean(value),
+        serde_json::Value::Number(value) => {
+            if value.is_i64() || value.is_u64() {
+                integer(value.to_string())
+                    .unwrap_or(Value::Float(value.as_f64().unwrap_or(f64::NAN)))
+            } else {
+                Value::Float(value.as_f64().unwrap_or(f64::NAN))
+            }
+        }
+        serde_json::Value::String(value) => Value::String(value),
+        serde_json::Value::Array(values) => {
+            Value::List(values.into_iter().map(from_json).collect())
+        }
+        serde_json::Value::Object(values) => Value::Dict(
+            values
+                .into_iter()
+                .map(|(key, value)| (Value::String(key), from_json(value)))
+                .collect(),
+        ),
+    }
+}
+
+pub(crate) fn from_json_text(value: &str) -> Result<Value, Error> {
+    serde_json::from_str(value)
+        .map(from_json)
+        .map_err(|_| Error::InvalidEntry)
+}
+
+pub(crate) fn is_truthy(value: &Value) -> bool {
+    match value {
+        Value::None => false,
+        Value::Boolean(value) => *value,
+        Value::Integer(value) => value.to_string() != "0",
+        Value::Float(value) => *value != 0.0,
+        Value::Complex(value) => value.re != 0.0 || value.im != 0.0,
+        Value::String(value) => !value.is_empty(),
+        Value::Bytes(value) => !value.is_empty(),
+        Value::Tuple(value) | Value::List(value) | Value::Set(value) => !value.is_empty(),
+        Value::Dict(value) => !value.is_empty(),
+    }
+}
+
+pub(crate) fn is_int(value: &Value) -> bool {
+    matches!(value, Value::Integer(_) | Value::Boolean(_))
+}
+
+pub(crate) fn to_f64(value: &Value) -> Option<f64> {
+    match value {
+        Value::Integer(value) => value.to_string().parse().ok(),
+        Value::Boolean(value) => Some(if *value { 1.0 } else { 0.0 }),
+        _ => None,
+    }
+}
+
+pub(crate) fn to_json(value: &Value) -> Result<Vec<u8>, Error> {
+    serde_json::to_vec(&to_json_value(value)?).map_err(|_| Error::InvalidEntry)
+}
+
+fn to_json_value(value: &Value) -> Result<serde_json::Value, Error> {
+    Ok(match value {
+        Value::None => serde_json::Value::Null,
+        Value::Boolean(value) => serde_json::Value::Bool(*value),
+        Value::Integer(value) => serde_json::Value::Number(
+            value
+                .to_string()
+                .parse::<Number>()
+                .map_err(|_| Error::InvalidEntry)?,
+        ),
+        Value::Float(value) => {
+            serde_json::Value::Number(Number::from_f64(*value).ok_or(Error::InvalidEntry)?)
+        }
+        Value::Complex(_) | Value::Bytes(_) => return Err(Error::InvalidEntry),
+        Value::String(value) => serde_json::Value::String(value.clone()),
+        Value::Tuple(values) | Value::List(values) | Value::Set(values) => {
+            serde_json::Value::Array(
+                values
+                    .iter()
+                    .map(to_json_value)
+                    .collect::<Result<Vec<_>, _>>()?,
+            )
+        }
+        Value::Dict(values) => {
+            let values = values
+                .iter()
+                .map(|(key, value)| {
+                    let Value::String(key) = key else {
+                        return Err(Error::InvalidEntry);
+                    };
+                    Ok((key.clone(), to_json_value(value)?))
+                })
+                .collect::<Result<Map<String, serde_json::Value>, _>>()?;
+            serde_json::Value::Object(values)
+        }
+    })
+}
diff --git a/litellm-rust/crates/cache-disk/tests/cache.rs b/litellm-rust/crates/cache-disk/tests/cache.rs
index c6cd6006148..4fb77d233ec 100644
--- a/litellm-rust/crates/cache-disk/tests/cache.rs
+++ b/litellm-rust/crates/cache-disk/tests/cache.rs
@@ -7,46 +7,92 @@ use std::{
 };
 
 use litellm_cache::{
-    BaseCache, BatchCache, BatchEntry, CounterCache, DeleteCache, ExactCacheContext, FlushCache,
-    JsonCodec,
+    BaseCache, BatchCache, BatchEntry, CacheCodec, CounterCache, DeleteCache, ExactCacheContext,
+    FlushCache, JsonCodec,
 };
-use litellm_cache_disk::{DiskCache, DiskStore, DiskcacheSqliteStore, StoredValue};
+use litellm_cache_disk::{DiskCache, DiskStore, DiskcacheSqliteStore, StoredValue, ValueAdapter};
+use rstest::{fixture, rstest};
 use rusqlite::Connection;
-use serde_json::json;
+use serde_json::{Value, json};
 use tempfile::TempDir;
 
-fn store() -> (TempDir, DiskcacheSqliteStore) {
-    let directory = tempfile::tempdir().unwrap();
-    let store = DiskcacheSqliteStore::open(directory.path()).unwrap();
-    (directory, store)
+struct Sandbox {
+    directory: TempDir,
 }
 
-fn cache(directory: &Path) -> DiskCache<JsonCodec<serde_json::Value>> {
-    DiskCache::open(directory, JsonCodec::new()).unwrap()
+#[fixture]
+fn sandbox() -> Sandbox {
+    Sandbox {
+        directory: tempfile::tempdir().unwrap(),
+    }
 }
 
-fn value_files(directory: &Path) -> Vec<PathBuf> {
-    fn visit(directory: &Path, files: &mut Vec<PathBuf>) {
-        for entry in fs::read_dir(directory).unwrap() {
-            let path = entry.unwrap().path();
-            if path.is_dir() {
-                visit(&path, files);
-            } else if path.extension().is_some_and(|extension| extension == "val") {
-                files.push(path);
+impl Sandbox {
+    fn store(&self) -> DiskcacheSqliteStore {
+        DiskcacheSqliteStore::open(self.directory.path()).unwrap()
+    }
+
+    fn cache<V>(&self) -> DiskCache<JsonCodec<V>>
+    where
+        JsonCodec<V>: CacheCodec,
+    {
+        DiskCache::open(self.directory.path(), JsonCodec::new()).unwrap()
+    }
+
+    fn db(&self) -> Connection {
+        Connection::open(self.directory.path().join("cache.db")).unwrap()
+    }
+
+    fn value_files(&self) -> Vec<PathBuf> {
+        fn visit(directory: &Path, files: &mut Vec<PathBuf>) {
+            for entry in fs::read_dir(directory).unwrap() {
+                let path = entry.unwrap().path();
+                if path.is_dir() {
+                    visit(&path, files);
+                } else if path.extension().is_some_and(|extension| extension == "val") {
+                    files.push(path);
+                }
             }
         }
+
+        let mut files = Vec::new();
+        visit(self.directory.path(), &mut files);
+        files
+    }
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+struct TextAdapter;
+
+impl ValueAdapter for TextAdapter {
+    fn read(&self, value: StoredValue) -> Result<Option<Vec<u8>>, litellm_cache::Error> {
+        match value {
+            StoredValue::Text(value) => Ok(Some(value.into_bytes())),
+            _ => Ok(None),
+        }
     }
 
-    let mut files = Vec::new();
-    visit(directory, &mut files);
-    files
+    fn write(&self, payload: Vec<u8>) -> StoredValue {
+        StoredValue::Text(String::from_utf8(payload).unwrap())
+    }
+
+    fn counter_seed(&self, _: Option<StoredValue>) -> Result<f64, litellm_cache::Error> {
+        Ok(0.0)
+    }
+
+    fn counter_value(&self, value: f64) -> StoredValue {
+        if value.fract() == 0.0 {
+            StoredValue::Integer(value as i64)
+        } else {
+            StoredValue::Float(value)
+        }
+    }
 }
 
-#[test]
-fn roundtrip_persists_and_reopens() {
-    let directory = tempfile::tempdir().unwrap();
+#[rstest]
+fn roundtrip_persists_and_reopens(sandbox: Sandbox) {
     let context = ExactCacheContext::default();
-    let opened = cache(directory.path());
+    let opened = sandbox.cache::<Value>();
     opened
         .set_cache("key", json!({"answer": 42}), &context)
         .unwrap();
@@ -55,16 +101,16 @@ fn roundtrip_persists_and_reopens() {
         Some(json!({"answer": 42}))
     );
     drop(opened);
-    let reopened = cache(directory.path());
+    let reopened = sandbox.cache::<Value>();
     assert_eq!(
         reopened.get_cache("key", &context).unwrap(),
         Some(json!({"answer": 42}))
     );
 }
 
-#[test]
-fn ttl_and_expired_culling_match_cache_contract() {
-    let (directory, store) = store();
+#[rstest]
+fn ttl_and_expired_culling_match_cache_contract(sandbox: Sandbox) {
+    let store = sandbox.store();
     store
         .set(
             "expired",
@@ -77,15 +123,16 @@ fn ttl_and_expired_culling_match_cache_contract() {
     store
         .set("new", StoredValue::Bytes(b"new".to_vec()), None, 11.0)
         .unwrap();
-    let connection = Connection::open(directory.path().join("cache.db")).unwrap();
     assert_eq!(
-        connection
+        sandbox
+            .db()
             .query_row("SELECT COUNT(*) FROM Cache", [], |row| row.get::<_, i64>(0))
             .unwrap(),
         1
     );
     assert_eq!(
-        connection
+        sandbox
+            .db()
             .query_row(
                 "SELECT value FROM Settings WHERE key = 'count'",
                 [],
@@ -96,9 +143,9 @@ fn ttl_and_expired_culling_match_cache_contract() {
     );
 }
 
-#[test]
-fn batch_preserves_order_and_classifies_misses_and_invalid_values() {
-    let (directory, store) = store();
+#[rstest]
+fn batch_preserves_order_and_classifies_misses_and_invalid_values(sandbox: Sandbox) {
+    let store = sandbox.store();
     store
         .set(
             "hit",
@@ -115,8 +162,8 @@ fn batch_preserves_order_and_classifies_misses_and_invalid_values() {
             0.0,
         )
         .unwrap();
-    let cache = cache(directory.path());
-    let entries = cache
+    let entries = sandbox
+        .cache::<Value>()
         .batch_get_cache(
             &["hit".into(), "missing".into(), "invalid".into()],
             &ExactCacheContext::default(),
@@ -132,117 +179,74 @@ fn batch_preserves_order_and_classifies_misses_and_invalid_values() {
     );
 }
 
-#[test]
-fn falsy_values_are_misses_and_protocol_five_pickle_decodes() {
-    let (directory, store) = store();
-    for (key, value) in [
-        ("empty-bytes", StoredValue::Bytes(Vec::new())),
-        ("empty-text", StoredValue::Text(String::new())),
-        ("zero-int", StoredValue::Integer(0)),
-        ("zero-float", StoredValue::Float(0.0)),
-        (
-            "empty-pickle",
-            StoredValue::Pickle(vec![0x80, 0x05, 0x7d, 0x94, 0x2e]),
-        ),
-    ] {
-        store.set(key, value, None, 0.0).unwrap();
-    }
-    store
-        .set(
-            "pickle",
-            StoredValue::Pickle(
-                b"\x80\x05\x95\x30\x00\x00\x00\x00\x00\x00\x00\x7d\x94\x28\x8c\x09timestamp\x94G\x3f\xf8\x00\x00\x00\x00\x00\x00\x8c\x08response\x94\x8c\x08{\"a\": 1}\x94u."
-                    .to_vec(),
-            ),
-            None,
-            0.0,
-        )
-        .unwrap();
-    let cache = cache(directory.path());
-    for key in [
-        "empty-bytes",
-        "empty-text",
-        "zero-int",
-        "zero-float",
-        "empty-pickle",
-    ] {
-        assert_eq!(
-            cache.get_cache(key, &ExactCacheContext::default()).unwrap(),
-            None
-        );
-    }
+#[rstest]
+#[case(StoredValue::Bytes(Vec::new()))]
+#[case(StoredValue::Text(String::new()))]
+#[case(StoredValue::Integer(0))]
+#[case(StoredValue::Float(0.0))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x89, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x00, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x94, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x5d, 0x94, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x7d, 0x94, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x29, 0x2e]))]
+fn falsy_values_are_misses(sandbox: Sandbox, #[case] value: StoredValue) {
+    sandbox.store().set("key", value, None, 0.0).unwrap();
     assert_eq!(
-        cache
-            .get_cache("pickle", &ExactCacheContext::default())
+        sandbox
+            .cache::<Value>()
+            .get_cache("key", &ExactCacheContext::default())
             .unwrap(),
-        Some(json!({"timestamp": 1.5, "response": "{\"a\": 1}"}))
+        None
     );
 }
 
-#[test]
-fn counters_use_atomic_native_values_and_ignore_invalid_initial_values() {
-    let (directory, store) = store();
-    store
-        .set("counter", StoredValue::Integer(2), None, 0.0)
-        .unwrap();
-    store
-        .set(
-            "invalid",
-            StoredValue::Text("not a number".into()),
-            None,
-            0.0,
-        )
-        .unwrap();
-    store
-        .set(
-            "pickle-counter",
-            StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x02, 0x2e]),
-            None,
-            0.0,
-        )
-        .unwrap();
-    let cache = DiskCache::open(directory.path(), JsonCodec::<f64>::new()).unwrap();
+#[rstest]
+#[case(Some(StoredValue::Integer(2)), 1.5, 3.5, "real")]
+#[case(Some(StoredValue::Integer(2)), 1.0, 3.0, "integer")]
+#[case(Some(StoredValue::Float(3.5)), 1.0, 1.0, "integer")]
+#[case(Some(StoredValue::Text("not a number".into())), 2.0, 2.0, "integer")]
+#[case(Some(StoredValue::Text("5".into())), 2.0, 7.0, "integer")]
+#[case(Some(StoredValue::Text("3.5".into())), 2.0, 2.0, "integer")]
+#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e])), 1.0, 2.0, "integer")]
+#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x02, 0x2e])), 1.0, 3.0, "integer")]
+#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e])), 1.0, 1.0, "integer")]
+#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e])), 4.0, 4.0, "integer")]
+fn counters_follow_python_initialization(
+    sandbox: Sandbox,
+    #[case] initial: Option<StoredValue>,
+    #[case] amount: f64,
+    #[case] expected: f64,
+    #[case] sqlite_type: &str,
+) {
+    if let Some(initial) = initial {
+        sandbox.store().set("counter", initial, None, 0.0).unwrap();
+    }
+    let cache = sandbox.cache::<f64>();
     assert_eq!(
         cache
-            .increment_cache("counter", 1.5, ExactCacheContext::default())
+            .increment_cache("counter", amount, ExactCacheContext::default())
             .unwrap(),
-        3.5
+        expected
     );
     assert_eq!(
-        cache
-            .increment_cache("invalid", 2.0, ExactCacheContext::default())
-            .unwrap(),
-        2.0
-    );
-    assert_eq!(
-        cache
-            .increment_cache("pickle-counter", 1.0, ExactCacheContext::default())
-            .unwrap(),
-        3.0
-    );
-    let connection = Connection::open(directory.path().join("cache.db")).unwrap();
-    assert_eq!(
-        connection
+        sandbox
+            .db()
             .query_row(
                 "SELECT typeof(value) FROM Cache WHERE key = 'counter'",
                 [],
                 |row| row.get::<_, String>(0)
             )
             .unwrap(),
-        "real"
-    );
-    assert_eq!(
-        cache
-            .increment_cache("counter", 1.0, ExactCacheContext::default())
-            .unwrap(),
-        1.0
+        sqlite_type
     );
 }
 
-#[test]
-fn counters_are_atomic_across_concurrent_callers() {
-    let directory = tempfile::tempdir().unwrap();
-    let cache = Arc::new(DiskCache::open(directory.path(), JsonCodec::<f64>::new()).unwrap());
+#[rstest]
+fn counters_are_atomic_across_concurrent_callers(sandbox: Sandbox) {
+    let cache = Arc::new(sandbox.cache::<f64>());
     let workers = (0..8)
         .map(|_| {
             let cache = Arc::clone(&cache);
@@ -266,15 +270,88 @@ fn counters_are_atomic_across_concurrent_callers() {
     );
 }
 
-#[test]
-fn delete_flush_and_spilled_file_replacement_clean_up_storage() {
-    let (directory, store) = store();
+#[rstest]
+fn fractional_then_integer_increment_follows_python_behavior(sandbox: Sandbox) {
+    let cache = sandbox.cache::<f64>();
+    assert_eq!(
+        cache
+            .increment_cache("counter", 3.5, ExactCacheContext::default())
+            .unwrap(),
+        3.5
+    );
+    assert_eq!(
+        cache
+            .increment_cache("counter", 1.0, ExactCacheContext::default())
+            .unwrap(),
+        1.0
+    );
+}
+
+#[rstest]
+fn increment_ttl_replacement_clears_expiry_without_ttl(sandbox: Sandbox) {
+    let cache = sandbox.cache::<f64>();
+    cache
+        .increment_cache(
+            "counter",
+            1.0,
+            ExactCacheContext {
+                ttl: Some(Duration::from_secs(60)),
+            },
+        )
+        .unwrap();
+    assert!(
+        sandbox
+            .db()
+            .query_row(
+                "SELECT expire_time IS NOT NULL FROM Cache WHERE key = 'counter'",
+                [],
+                |row| row.get::<_, bool>(0)
+            )
+            .unwrap()
+    );
+    cache
+        .increment_cache("counter", 1.0, ExactCacheContext::default())
+        .unwrap();
+    assert!(
+        !sandbox
+            .db()
+            .query_row(
+                "SELECT expire_time IS NOT NULL FROM Cache WHERE key = 'counter'",
+                [],
+                |row| row.get::<_, bool>(0)
+            )
+            .unwrap()
+    );
+}
+
+#[rstest]
+fn custom_adapter_controls_storage_and_reads(sandbox: Sandbox) {
+    let cache = DiskCache::with_adapter(sandbox.store(), TextAdapter, JsonCodec::<Value>::new());
+    cache
+        .set_cache("key", json!({"answer": 42}), &ExactCacheContext::default())
+        .unwrap();
+    assert!(matches!(
+        sandbox.store().get("key", 0.0).unwrap(),
+        Some(StoredValue::Text(_))
+    ));
+    assert_eq!(
+        cache
+            .get_cache("key", &ExactCacheContext::default())
+            .unwrap(),
+        Some(json!({"answer": 42}))
+    );
+}
+
+#[rstest]
+fn delete_flush_and_spilled_file_replacement_clean_up_storage(sandbox: Sandbox) {
     let large = vec![b'x'; 32 * 1024];
-    store
+    sandbox
+        .store()
         .set("large", StoredValue::Bytes(large.clone()), None, 0.0)
         .unwrap();
-    assert_eq!(value_files(directory.path()).len(), 1);
-    store
+    assert_eq!(sandbox.value_files().len(), 1);
+    sandbox
+        .store()
         .set(
             "large",
             StoredValue::Bytes(vec![b'y'; 32 * 1024]),
@@ -282,23 +359,25 @@ fn delete_flush_and_spilled_file_replacement_clean_up_storage() {
             0.0,
         )
         .unwrap();
-    assert_eq!(value_files(directory.path()).len(), 1);
-    store.pop("large", 0.0).unwrap();
-    assert!(value_files(directory.path()).is_empty());
-    store
+    assert_eq!(sandbox.value_files().len(), 1);
+    sandbox.store().pop("large", 0.0).unwrap();
+    assert!(sandbox.value_files().is_empty());
+    sandbox
+        .store()
         .set("a", StoredValue::Bytes(large.clone()), None, 0.0)
         .unwrap();
-    store
+    sandbox
+        .store()
         .set("b", StoredValue::Bytes(large), None, 0.0)
         .unwrap();
-    store.clear().unwrap();
-    assert!(value_files(directory.path()).is_empty());
+    sandbox.store().clear().unwrap();
+    assert!(sandbox.value_files().is_empty());
 }
 
+#[rstest]
 #[tokio::test]
-async fn async_operations_connection_and_delete_match_sync_operations() {
-    let directory = tempfile::tempdir().unwrap();
-    let cache = cache(directory.path());
+async fn async_operations_connection_and_delete_match_sync_operations(sandbox: Sandbox) {
+    let cache = sandbox.cache::<Value>();
     let context = ExactCacheContext {
         ttl: Some(Duration::from_secs(60)),
     };
diff --git a/litellm-rust/crates/cache-disk/tests/python_compat.rs b/litellm-rust/crates/cache-disk/tests/python_compat.rs
new file mode 100644
index 00000000000..f9e12af6424
--- /dev/null
+++ b/litellm-rust/crates/cache-disk/tests/python_compat.rs
@@ -0,0 +1,105 @@
+use litellm_cache::Error;
+use litellm_cache_disk::{PythonDiskCacheAdapter, StoredValue, ValueAdapter};
+use rstest::rstest;
+
+enum ReadExpectation {
+    Bytes(&'static [u8]),
+    Miss,
+    Invalid,
+}
+
+#[rstest]
+#[case::pickled_dictionary_with_string_keys(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e]),
+    ReadExpectation::Bytes(br#"{"a":1}"#)
+)]
+#[case::pickled_list_of_integers(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x94, 0x28, 0x4b, 0x01, 0x4b, 0x02, 0x65, 0x2e]),
+    ReadExpectation::Bytes(br#"[1,2]"#)
+)]
+#[case::pickled_tuple_of_integers(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x01, 0x4b, 0x02, 0x86, 0x94, 0x2e]),
+    ReadExpectation::Bytes(br#"[1,2]"#)
+)]
+#[case::pickled_set_of_integers(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x94, 0x28, 0x4b, 0x01, 0x4b, 0x02, 0x90, 0x2e]),
+    ReadExpectation::Bytes(br#"[1,2]"#)
+)]
+#[case::pickled_response_envelope(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x28, 0x8c, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x94, 0x47, 0x3f, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x94, 0x8c, 0x08, 0x7b, 0x22, 0x61, 0x22, 0x3a, 0x20, 0x31, 0x7d, 0x94, 0x75, 0x2e]),
+    ReadExpectation::Bytes(br#"{"response":"{\"a\": 1}","timestamp":1.5}"#)
+)]
+#[case::non_json_text(
+    StoredValue::Text("not json".into()),
+    ReadExpectation::Bytes(b"not json")
+)]
+#[case::json_text(
+    StoredValue::Text("{\"a\": 1}".into()),
+    ReadExpectation::Bytes(br#"{"a": 1}"#)
+)]
+#[case::non_utf8_bytes(
+    StoredValue::Bytes(vec![0xff, 0xfe]),
+    ReadExpectation::Bytes(&[0xff, 0xfe])
+)]
+#[case::integer_seven(StoredValue::Integer(7), ReadExpectation::Bytes(b"7"))]
+#[case::float_one_point_five(StoredValue::Float(1.5), ReadExpectation::Bytes(b"1.5"))]
+#[case::pickled_bytes(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x05, 0x62, 0x79, 0x74, 0x65, 0x73, 0x94, 0x2e]),
+    ReadExpectation::Invalid
+)]
+#[case::pickled_dictionary_with_integer_key(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x4b, 0x01, 0x8c, 0x01, 0x61, 0x94, 0x73, 0x2e]),
+    ReadExpectation::Invalid
+)]
+#[case::pickled_complex(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x2e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x08, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x73, 0x94, 0x8c, 0x07, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x94, 0x93, 0x94, 0x47, 0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0x94, 0x52, 0x94, 0x2e]),
+    ReadExpectation::Invalid
+)]
+#[case::truncated_pickle(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x2e]),
+    ReadExpectation::Invalid
+)]
+#[case::empty_bytes(StoredValue::Bytes(Vec::new()), ReadExpectation::Miss)]
+#[case::empty_text(StoredValue::Text(String::new()), ReadExpectation::Miss)]
+#[case::zero_integer(StoredValue::Integer(0), ReadExpectation::Miss)]
+#[case::zero_float(StoredValue::Float(0.0), ReadExpectation::Miss)]
+#[case::pickled_none(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_false(StoredValue::Pickle(vec![0x80, 0x05, 0x89, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_zero(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x00, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_zero_float(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_empty_string(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x94, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_empty_list(StoredValue::Pickle(vec![0x80, 0x05, 0x5d, 0x94, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_empty_dictionary(StoredValue::Pickle(vec![0x80, 0x05, 0x7d, 0x94, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_empty_tuple(StoredValue::Pickle(vec![0x80, 0x05, 0x29, 0x2e]), ReadExpectation::Miss)]
+fn python_read_cases(#[case] row: StoredValue, #[case] expected: ReadExpectation) {
+    let result = PythonDiskCacheAdapter.read(row);
+    match expected {
+        ReadExpectation::Bytes(expected) => assert_eq!(result.unwrap().unwrap(), expected),
+        ReadExpectation::Miss => assert_eq!(result.unwrap(), None),
+        ReadExpectation::Invalid => assert!(matches!(result, Err(Error::InvalidEntry))),
+    }
+}
+
+#[rstest]
+#[case::integer_two(Some(StoredValue::Integer(2)), 2.0)]
+#[case::float_three_point_five(Some(StoredValue::Float(3.5)), 0.0)]
+#[case::text_not_a_number(Some(StoredValue::Text("not a number".into())), 0.0)]
+#[case::text_five(Some(StoredValue::Text("5".into())), 5.0)]
+#[case::text_three_point_five(Some(StoredValue::Text("3.5".into())), 0.0)]
+#[case::pickled_true(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e])), 1.0)]
+#[case::pickled_two(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x02, 0x2e])), 2.0)]
+#[case::pickled_dictionary(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e])), 0.0)]
+#[case::missing(None, 0.0)]
+#[case::pickled_none(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e])), 0.0)]
+fn python_counter_seed_cases(#[case] row: Option<StoredValue>, #[case] expected: f64) {
+    assert_eq!(PythonDiskCacheAdapter.counter_seed(row).unwrap(), expected);
+}
+
+#[rstest]
+#[case::integer_three(3.0, StoredValue::Integer(3))]
+#[case::fractional_three_point_five(3.5, StoredValue::Float(3.5))]
+#[case::negative_zero(-0.0, StoredValue::Integer(0))]
+#[case::large_float(1e300, StoredValue::Float(1e300))]
+fn python_counter_value_cases(#[case] value: f64, #[case] expected: StoredValue) {
+    assert_eq!(PythonDiskCacheAdapter.counter_value(value), expected);
+}

From 59675bc0c948d124affe549f5ffc277bd8a31585 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:22:26 +0000
Subject: [PATCH 102/160] Revert "ci(rust): raise native wheel size gate to 35
 MB"

This reverts commit c1382086d69f176114003a5fcf32ac2a8cbaaa64.
---
 .github/scripts/verify_linux_native_wheel.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py
index d86ad7b15c6..4fb8f068eb0 100644
--- a/.github/scripts/verify_linux_native_wheel.py
+++ b/.github/scripts/verify_linux_native_wheel.py
@@ -205,7 +205,7 @@ def main(
     native_module: Final = load_native_module(native_path)
     native_module_loads: Final = native_module is not None
     panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
-    native_size_limit: Final = 35_000_000
+    native_size_limit: Final = 25_000_000
     native_size_within_limit: Final = native_member.file_size <= native_size_limit
     validations: Final = (
         (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
@@ -222,7 +222,7 @@ def main(
         ("Python extension entry point is present", extension_entry_point_present),
         ("Native module loads", native_module_loads),
         ("Production module omits the panic test hook", panic_test_hook_absent),
-        ("Native extension does not exceed 35 MB", native_size_within_limit),
+        ("Native extension does not exceed 25 MB", native_size_within_limit),
         ("Wheel contents are valid", not unexpected_members),
     )
 
@@ -267,7 +267,7 @@ def main(
             ),
             (
                 not native_size_within_limit,
-                f"native extension exceeds 35 MB: {native_member.file_size / 1_000_000:.2f} MB",
+                f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB",
             ),
             (bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"),
         )

From e729820301f1a3e11bfe95daca2e132407402eb2 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:22:27 +0000
Subject: [PATCH 103/160] revert(rust): drop wheel-size release profile tweaks,
 gate moves in #42300

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.toml | 66 +----------------------------------------
 1 file changed, 1 insertion(+), 65 deletions(-)

diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
index 5a72437d767..031cfcbbb21 100644
--- a/litellm-rust/Cargo.toml
+++ b/litellm-rust/Cargo.toml
@@ -75,73 +75,9 @@ veil = "0.3.0"
 
 [profile.release]
 opt-level = 3
-lto = "fat"
+lto = "thin"
 codegen-units = 1
 panic = "unwind"
 debug = false
 incremental = false
 strip = "symbols"
-
-[profile.release.package."aws-config"]
-opt-level = "s"
-
-[profile.release.package."aws-credential-types"]
-opt-level = "s"
-
-[profile.release.package."aws-runtime"]
-opt-level = "s"
-
-[profile.release.package."aws-sdk-kms"]
-opt-level = "s"
-
-[profile.release.package."aws-sdk-s3"]
-opt-level = "s"
-
-[profile.release.package."aws-sdk-secretsmanager"]
-opt-level = "s"
-
-[profile.release.package."aws-sdk-sts"]
-opt-level = "s"
-
-
-[profile.release.package."aws-smithy-async"]
-opt-level = "s"
-
-[profile.release.package."aws-smithy-checksums"]
-opt-level = "s"
-
-
-[profile.release.package."aws-smithy-http"]
-opt-level = "s"
-
-[profile.release.package."aws-smithy-http-client"]
-opt-level = "s"
-
-[profile.release.package."aws-smithy-json"]
-opt-level = "s"
-
-[profile.release.package."aws-smithy-observability"]
-opt-level = "s"
-
-[profile.release.package."aws-smithy-query"]
-opt-level = "s"
-
-[profile.release.package."aws-smithy-runtime"]
-opt-level = "s"
-
-
-[profile.release.package."aws-smithy-runtime-api-macros"]
-opt-level = "s"
-
-[profile.release.package."aws-smithy-schema"]
-opt-level = "s"
-
-
-[profile.release.package."aws-smithy-xml"]
-opt-level = "s"
-
-[profile.release.package."aws-types"]
-opt-level = "s"
-
-[profile.release.package."litellm-cache-s3"]
-opt-level = "s"

From 6237dd51cb8711df528b360100e080d5fc73d6c2 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:22:59 +0000
Subject: [PATCH 104/160] fix(cache-redis-semantic): isolate on an incompatible
 index after a lost create race

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../crates/cache-redis-semantic/src/cache.rs     | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/litellm-rust/crates/cache-redis-semantic/src/cache.rs b/litellm-rust/crates/cache-redis-semantic/src/cache.rs
index cd79f067296..4181ce719e3 100644
--- a/litellm-rust/crates/cache-redis-semantic/src/cache.rs
+++ b/litellm-rust/crates/cache-redis-semantic/src/cache.rs
@@ -70,14 +70,14 @@ impl Inner {
         let name = match index_compatible(connection, &self.index_name, dims)? {
             Some(true) => self.index_name.clone(),
             Some(false) => self.isolated_index(connection, dims)?,
-            None => {
-                if create_index(connection, &self.index_name, dims).is_err()
-                    && index_compatible(connection, &self.index_name, dims)? != Some(true)
-                {
-                    return Err(Error::Unavailable);
-                }
-                self.index_name.clone()
-            }
+            None => match create_index(connection, &self.index_name, dims) {
+                Ok(()) => self.index_name.clone(),
+                Err(_) => match index_compatible(connection, &self.index_name, dims)? {
+                    Some(true) => self.index_name.clone(),
+                    Some(false) => self.isolated_index(connection, dims)?,
+                    None => return Err(Error::Unavailable),
+                },
+            },
         };
         let _ = self.resolved_index.set(name.clone());
         Ok(name)

From 2b257bd9a3f09faedecb700cffa00a175df46d49 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 20:24:23 +0000
Subject: [PATCH 105/160] feat(cache-redis): expose the pooled connection
 handling for reuse

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/cache-redis/src/cache.rs  | 93 +++++++++++--------
 .../cache-redis/src/cache/operations.rs       | 28 +++---
 litellm-rust/crates/cache-redis/src/lib.rs    |  4 +
 3 files changed, 72 insertions(+), 53 deletions(-)

diff --git a/litellm-rust/crates/cache-redis/src/cache.rs b/litellm-rust/crates/cache-redis/src/cache.rs
index a960c383bf4..6388448accc 100644
--- a/litellm-rust/crates/cache-redis/src/cache.rs
+++ b/litellm-rust/crates/cache-redis/src/cache.rs
@@ -19,7 +19,7 @@ const DEFAULT_TTL: Duration = Duration::from_secs(600);
 const REDIS_TIMEOUT: Duration = Duration::from_secs(5);
 const REDIS_POOL_SIZE: u32 = 16;
 
-struct PooledConnection {
+pub struct PooledConnection {
     connection: redis::Connection,
     failed: bool,
 }
@@ -27,16 +27,19 @@ struct PooledConnection {
 /// Pools connections without a checkout PING, which would double every operation's round trips.
 /// A timed-out command leaves its reply on the socket while redis still reports the connection
 /// open, so any connection whose operation failed is discarded instead of being reused.
-struct ConnectionManager(redis::Client);
+pub struct ConnectionManager {
+    client: redis::Client,
+    timeout: Duration,
+}
 
 impl r2d2::ManageConnection for ConnectionManager {
     type Connection = PooledConnection;
     type Error = redis::RedisError;
 
     fn connect(&self) -> Result<PooledConnection, redis::RedisError> {
-        let connection = self.0.get_connection()?;
-        connection.set_read_timeout(Some(REDIS_TIMEOUT))?;
-        connection.set_write_timeout(Some(REDIS_TIMEOUT))?;
+        let connection = self.client.get_connection()?;
+        connection.set_read_timeout(Some(self.timeout))?;
+        connection.set_write_timeout(Some(self.timeout))?;
         Ok(PooledConnection {
             connection,
             failed: false,
@@ -68,12 +71,12 @@ const CLAIM_SCRIPT: &str = concat!(
 );
 const CLAIM_ATTEMPTS: usize = 8;
 
-enum Connections<C> {
+pub enum Connections<C> {
     Pool(r2d2::Pool<ConnectionManager>),
     Fixed(Mutex<C>),
 }
 
-struct ConnectionRef<'a>(&'a mut dyn redis::ConnectionLike);
+pub struct ConnectionRef<'a>(&'a mut dyn redis::ConnectionLike);
 
 impl redis::ConnectionLike for ConnectionRef<'_> {
     fn req_packed_command(&mut self, cmd: &[u8]) -> redis::RedisResult<redis::Value> {
@@ -110,7 +113,23 @@ impl<C> Connections<C>
 where
     C: redis::ConnectionLike + Send + 'static,
 {
-    fn execute<T>(
+    pub fn pooled(url: &str, timeout: Duration, pool_size: u32) -> Result<Self, Error> {
+        let client = redis::Client::open(url).map_err(|_| Error::Unavailable)?;
+        let pool = r2d2::Pool::builder()
+            .max_size(pool_size)
+            .min_idle(Some(0))
+            .connection_timeout(timeout)
+            .test_on_check_out(false)
+            .build(ConnectionManager { client, timeout })
+            .map_err(|_| Error::Unavailable)?;
+        Ok(Self::Pool(pool))
+    }
+
+    pub fn fixed(connection: C) -> Self {
+        Self::Fixed(Mutex::new(connection))
+    }
+
+    pub fn execute<T>(
         &self,
         operation: impl FnOnce(&mut ConnectionRef<'_>) -> Result<T, Error>,
     ) -> Result<T, Error> {
@@ -127,6 +146,16 @@ where
             }
         }
     }
+
+    pub async fn run_blocking<T, F>(connections: Arc<Self>, operation: F) -> Result<T, Error>
+    where
+        T: Send + 'static,
+        F: FnOnce(&mut ConnectionRef<'_>) -> Result<T, Error> + Send + 'static,
+    {
+        tokio::task::spawn_blocking(move || connections.execute(operation))
+            .await
+            .map_err(|_| Error::Unavailable)?
+    }
 }
 
 pub struct RedisCache<S, C = redis::Connection> {
@@ -138,16 +167,8 @@ pub struct RedisCache<S, C = redis::Connection> {
 
 impl<S: CacheCodec> RedisCache<S> {
     pub fn new(url: &str, default_ttl: Option<Duration>, codec: S) -> Result<Self, Error> {
-        let client = redis::Client::open(url).map_err(|_| Error::Unavailable)?;
-        let pool = r2d2::Pool::builder()
-            .max_size(REDIS_POOL_SIZE)
-            .min_idle(Some(0))
-            .connection_timeout(REDIS_TIMEOUT)
-            .test_on_check_out(false)
-            .build(ConnectionManager(client))
-            .map_err(|_| Error::Unavailable)?;
         Ok(Self {
-            connections: Arc::new(Connections::Pool(pool)),
+            connections: Arc::new(Connections::pooled(url, REDIS_TIMEOUT, REDIS_POOL_SIZE)?),
             default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
             codec,
             namespace: None,
@@ -162,7 +183,7 @@ where
 {
     pub fn with_connection(connection: C, default_ttl: Option<Duration>, codec: S) -> Self {
         Self {
-            connections: Arc::new(Connections::Fixed(Mutex::new(connection))),
+            connections: Arc::new(Connections::fixed(connection)),
             default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
             codec,
             namespace: None,
@@ -241,20 +262,14 @@ where
     }
 
     fn ttl_seconds(ttl: Duration) -> u64 {
-        ttl.as_secs()
-            .saturating_add(u64::from(ttl.subsec_nanos() > 0))
-            .max(1)
+        ttl_seconds(ttl)
     }
+}
 
-    async fn run_blocking<T, F>(connections: Arc<Connections<C>>, operation: F) -> Result<T, Error>
-    where
-        T: Send + 'static,
-        F: FnOnce(&mut ConnectionRef<'_>) -> Result<T, Error> + Send + 'static,
-    {
-        tokio::task::spawn_blocking(move || connections.execute(operation))
-            .await
-            .map_err(|_| Error::Unavailable)?
-    }
+pub fn ttl_seconds(ttl: Duration) -> u64 {
+    ttl.as_secs()
+        .saturating_add(u64::from(ttl.subsec_nanos() > 0))
+        .max(1)
 }
 
 fn namespaced_key(namespace: Option<&str>, key: &str) -> String {
@@ -313,7 +328,7 @@ where
         let payload = self.codec.encode(&value)?;
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             connection
                 .set_ex::<_, _, ()>(key, payload, ttl)
                 .map_err(|_| Error::Unavailable)
@@ -327,7 +342,7 @@ where
         _: &ExactCacheContext,
     ) -> Result<Option<Self::Value>, Error> {
         let key = self.namespaced_key(key);
-        let value = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let value = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             connection
                 .get::<_, redis::Value>(key)
                 .map_err(|_| Error::Unavailable)
@@ -350,7 +365,7 @@ where
             })
             .collect::<Result<Vec<_>, _>>()?;
         let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut pipeline = redis::pipe();
             for (key, payload) in entries {
                 pipeline
@@ -372,7 +387,7 @@ where
     }
 
     async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
-        match Self::run_blocking(Arc::clone(&self.connections), |connection| {
+        match Connections::run_blocking(Arc::clone(&self.connections), |connection| {
             Ok(match redis::cmd("PING").query::<String>(connection) {
                 Ok(_) => CacheConnectionResult {
                     status: CacheConnectionStatus::Success,
@@ -433,7 +448,7 @@ where
             .iter()
             .map(|key| self.namespaced_key(key))
             .collect::<Vec<_>>();
-        let values = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("MGET")
                 .arg(keys)
                 .query::<Vec<redis::Value>>(connection)
@@ -460,7 +475,7 @@ where
 
     async fn async_delete_cache(&self, key: &str) -> Result<(), Error> {
         let key = self.namespaced_key(key);
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             connection.del::<_, ()>(key).map_err(|_| Error::Unavailable)
         })
         .await
@@ -480,7 +495,7 @@ where
 
     async fn async_flush_cache(&self) -> Result<(), Error> {
         let pattern = self.namespaced_pattern()?;
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             Self::flush_matching(connection, &pattern)
         })
         .await
@@ -512,7 +527,7 @@ where
     ) -> Result<f64, Error> {
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             increment(connection, key, amount, ttl)
         })
         .await
@@ -623,7 +638,7 @@ where
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
         let codec = self.codec.clone();
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             claim(connection, &codec, &key, candidate, &eligible, ttl)
         })
         .await
diff --git a/litellm-rust/crates/cache-redis/src/cache/operations.rs b/litellm-rust/crates/cache-redis/src/cache/operations.rs
index d8d9ae24c4c..f27a7802bab 100644
--- a/litellm-rust/crates/cache-redis/src/cache/operations.rs
+++ b/litellm-rust/crates/cache-redis/src/cache/operations.rs
@@ -144,7 +144,7 @@ where
             .into_iter()
             .map(|key| self.namespaced_key(&key))
             .collect::<Vec<_>>();
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             connection.del(keys).map_err(|_| Error::Unavailable)
         })
         .await
@@ -172,7 +172,7 @@ where
             .iter()
             .map(|key| self.namespaced_key(key))
             .collect::<Vec<_>>();
-        let values = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("MGET")
                 .arg(keys)
                 .query::<Vec<redis::Value>>(connection)
@@ -192,7 +192,7 @@ where
     }
 
     pub async fn ping(&self) -> Result<bool, Error> {
-        Self::run_blocking(Arc::clone(&self.connections), |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), |connection| {
             redis::cmd("PING")
                 .query::<String>(connection)
                 .map(|response| response == "PONG")
@@ -203,7 +203,7 @@ where
 
     pub async fn async_get_ttl(&self, key: &str) -> Result<Option<i64>, Error> {
         let key = self.namespaced_key(key);
-        let ttl = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let ttl = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("TTL")
                 .arg(key)
                 .query::<i64>(connection)
@@ -215,7 +215,7 @@ where
 
     pub async fn async_scan_iter(&self, pattern: &str, count: usize) -> Result<Vec<String>, Error> {
         let pattern = format!("{}*", self.namespaced_key(pattern));
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut cursor = 0u64;
             let mut matches = Vec::new();
             loop {
@@ -249,7 +249,7 @@ where
         }
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(ttl.unwrap_or(self.default_ttl));
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut pipeline = redis::pipe();
             pipeline.cmd("SADD").arg(&key).arg(values);
             pipeline.cmd("EXPIRE").arg(&key).arg(ttl).ignore();
@@ -266,7 +266,7 @@ where
             return Err(Error::InvalidEntry);
         }
         let key = self.namespaced_key(key);
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("RPUSH")
                 .arg(key)
                 .arg(values)
@@ -292,7 +292,7 @@ where
         if operations.is_empty() {
             return Ok(Vec::new());
         }
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut pipeline = redis::pipe();
             for (key, values) in operations {
                 pipeline.cmd("RPUSH").arg(key).arg(values);
@@ -309,7 +309,7 @@ where
     ) -> Result<RedisLpopResult, Error> {
         let key = self.namespaced_key(key);
         let multiple = count.is_some();
-        let value = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let value = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut command = redis::cmd("LPOP");
             command.arg(key);
             if let Some(count) = count {
@@ -338,7 +338,7 @@ where
             .iter()
             .map(|(_, count)| count.is_some())
             .collect::<Vec<_>>();
-        let values = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut pipeline = redis::pipe();
             for (key, count) in operations {
                 let command = pipeline.cmd("LPOP").arg(key);
@@ -368,7 +368,7 @@ where
             .into_iter()
             .map(|key| self.namespaced_key(&key))
             .collect::<Vec<_>>();
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("EVAL")
                 .arg(script)
                 .arg(keys.len())
@@ -440,7 +440,7 @@ where
         if operations.is_empty() {
             return Ok(Vec::new());
         }
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut pipeline = redis::pipe();
             for (key, amount, ttl) in operations {
                 pipeline.cmd("INCRBYFLOAT").arg(&key).arg(amount);
@@ -461,7 +461,7 @@ where
     ) -> Result<i64, Error> {
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(ttl);
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             increment_with_floor(connection, key, amount, ttl)
         })
         .await
@@ -475,7 +475,7 @@ where
     ) -> Result<f64, Error> {
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(ttl.unwrap_or(self.default_ttl));
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("EVAL")
                 .arg(SET_MAX_SCRIPT)
                 .arg(1)
diff --git a/litellm-rust/crates/cache-redis/src/lib.rs b/litellm-rust/crates/cache-redis/src/lib.rs
index 98f6bfd8ce5..ea75906e9c9 100644
--- a/litellm-rust/crates/cache-redis/src/lib.rs
+++ b/litellm-rust/crates/cache-redis/src/lib.rs
@@ -1,6 +1,10 @@
 mod cache;
 mod topology;
 
+pub mod connection {
+    pub use crate::cache::{ConnectionRef, Connections, ttl_seconds};
+}
+
 pub use cache::{
     RedisArg, RedisCache, RedisLpopOperation, RedisLpopResult, RedisRpushOperation, RedisScript,
 };

From 4a8826e03b08311901cd8cb8186b9c1192206811 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:23:56 +0000
Subject: [PATCH 106/160] refactor(cache-disk): tidy python adapter

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/cache-disk/src/cache.rs   |  6 ++---
 .../crates/cache-disk/src/python/mod.rs       | 24 ++++++++-----------
 .../crates/cache-disk/tests/python_compat.rs  |  8 +++++++
 3 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/litellm-rust/crates/cache-disk/src/cache.rs b/litellm-rust/crates/cache-disk/src/cache.rs
index 98182d93968..adff078a89b 100644
--- a/litellm-rust/crates/cache-disk/src/cache.rs
+++ b/litellm-rust/crates/cache-disk/src/cache.rs
@@ -18,22 +18,20 @@ pub struct DiskCache<S, D = DiskcacheSqliteStore, A = PythonDiskCacheAdapter> {
 }
 
 impl<S: CacheCodec> DiskCache<S> {
-    #[allow(clippy::default_constructed_unit_structs)]
     pub fn open(directory: impl AsRef<Path>, codec: S) -> Result<Self, Error> {
         Ok(Self {
             store: Arc::new(DiskcacheSqliteStore::open(directory)?),
-            adapter: Arc::new(PythonDiskCacheAdapter::default()),
+            adapter: Arc::new(PythonDiskCacheAdapter),
             codec,
         })
     }
 }
 
 impl<S: CacheCodec, D: DiskStore> DiskCache<S, D, PythonDiskCacheAdapter> {
-    #[allow(clippy::default_constructed_unit_structs)]
     pub fn with_store(store: D, codec: S) -> Self {
         Self {
             store: Arc::new(store),
-            adapter: Arc::new(PythonDiskCacheAdapter::default()),
+            adapter: Arc::new(PythonDiskCacheAdapter),
             codec,
         }
     }
diff --git a/litellm-rust/crates/cache-disk/src/python/mod.rs b/litellm-rust/crates/cache-disk/src/python/mod.rs
index 5c9b57de000..7a370db357c 100644
--- a/litellm-rust/crates/cache-disk/src/python/mod.rs
+++ b/litellm-rust/crates/cache-disk/src/python/mod.rs
@@ -13,9 +13,7 @@ impl PythonDiskCacheAdapter {
         let value = match value {
             StoredValue::Bytes(value) => Value::Bytes(value),
             StoredValue::Text(value) => Value::String(value),
-            StoredValue::Integer(value) => {
-                value::from_json(serde_json::Value::Number(value.into()))
-            }
+            StoredValue::Integer(value) => Value::Integer(value.into()),
             StoredValue::Float(value) => Value::Float(value),
             StoredValue::Pickle(value) => value::from_pickle(&value)?,
         };
@@ -39,18 +37,16 @@ impl PythonDiskCacheAdapter {
 
 impl ValueAdapter for PythonDiskCacheAdapter {
     fn read(&self, value: StoredValue) -> Result<Option<Vec<u8>>, Error> {
-        let raw = match &value {
-            StoredValue::Text(value) => Some(value.as_bytes().to_vec()),
-            StoredValue::Bytes(value) => Some(value.clone()),
-            StoredValue::Integer(_) | StoredValue::Float(_) | StoredValue::Pickle(_) => None,
-        };
-        let Some(value) = Self::python_get_cache(value)? else {
-            return Ok(None);
-        };
-        if let Some(raw) = raw {
-            return Ok(Some(raw));
+        match value {
+            StoredValue::Text(value) => Ok((!value.is_empty()).then(|| value.into_bytes())),
+            StoredValue::Bytes(value) => Ok((!value.is_empty()).then_some(value)),
+            value => {
+                let Some(value) = Self::python_get_cache(value)? else {
+                    return Ok(None);
+                };
+                value::to_json(&value).map(Some)
+            }
         }
-        value::to_json(&value).map(Some)
     }
 
     fn write(&self, payload: Vec<u8>) -> StoredValue {
diff --git a/litellm-rust/crates/cache-disk/tests/python_compat.rs b/litellm-rust/crates/cache-disk/tests/python_compat.rs
index f9e12af6424..9cbef8573bd 100644
--- a/litellm-rust/crates/cache-disk/tests/python_compat.rs
+++ b/litellm-rust/crates/cache-disk/tests/python_compat.rs
@@ -43,6 +43,14 @@ enum ReadExpectation {
 )]
 #[case::integer_seven(StoredValue::Integer(7), ReadExpectation::Bytes(b"7"))]
 #[case::float_one_point_five(StoredValue::Float(1.5), ReadExpectation::Bytes(b"1.5"))]
+#[case::pickled_true(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e]),
+    ReadExpectation::Bytes(b"true")
+)]
+#[case::pickled_negative_integer(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0xfd, 0xff, 0xff, 0xff, 0x2e]),
+    ReadExpectation::Bytes(b"-3")
+)]
 #[case::pickled_bytes(
     StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x05, 0x62, 0x79, 0x74, 0x65, 0x73, 0x94, 0x2e]),
     ReadExpectation::Invalid

From 46023769774e018d6fe75f4b5102e01dbd507146 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Mon, 21 Sep 2026 14:24:32 -0700
Subject: [PATCH 107/160] fix(proxy): report a stored alerting value as db even
 when it is null

A stored null or empty list for a nested alerting field is still the value
the proxy serves when the config file leaves alerting_args alone, so the
source is db. Keying off the value rather than its presence reported those
fields as default and hid a stored setting that is genuinely in effect.

Presence in the stored row now decides, with the config file still checked
first so a config-owned key keeps reporting config. Test helpers are typed
and the router test injects a stub rather than patching a class attribute.
---
 litellm/proxy/proxy_server.py                 |  7 +--
 .../test_router_settings_endpoints.py         | 21 +++----
 .../proxy_server/test_routes_model_metrics.py | 59 +++++++++++++++----
 .../test_proxy_setting_endpoints.py           |  6 +-
 4 files changed, 64 insertions(+), 29 deletions(-)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index d162ce2914e..2ee8d1627c0 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -15993,16 +15993,13 @@ def _nested_setting_source(
     field_name: str,
     field_default: JsonValue,
 ) -> FieldSource:
+    unset_source: Final[FieldSource] = "default" if field_default is not None else "unset"
     parent_value: Final = settings.config_value(parent_key)
     if isinstance(parent_value, Mapping) and field_name in parent_value:
         return "config"
-    unset_source: Final[FieldSource] = "default" if field_default is not None else "unset"
     if settings.owned_by_config(parent_key):
         return unset_source
-    db_value: Final = db_values.get(field_name)
-    if db_value is not None and not (isinstance(db_value, list) and len(db_value) == 0):
-        return "db"
-    return unset_source
+    return "db" if field_name in db_values else unset_source
 
 
 @router.get(
diff --git a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py
index 51c8679e89e..889bed13099 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py
@@ -4,6 +4,8 @@ Tests for router settings management endpoints.
 Tests the GET endpoints for router settings and router fields.
 """
 
+from collections.abc import Mapping
+from typing import Any, Final
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
@@ -22,15 +24,14 @@ from litellm.router import Router
 client = TestClient(app)
 
 
-def _stub_proxy_config(router_settings, config_router_settings):
-    class _StubProxyConfig:
-        def __init__(self):
-            self.router_settings = router_settings
+class _StubProxyConfig:
+    def __init__(self, router_settings: SettingsStore, config_router_settings: Mapping[str, Any]) -> None:
+        self.router_settings: Final = router_settings
+        self._config_router_settings: Final = dict(config_router_settings)
 
-        async def get_config(self, config_file_path=None):
-            return {"router_settings": dict(config_router_settings)}
-
-    return _StubProxyConfig()
+    async def get_config(self, config_file_path: str | None = None) -> dict[str, Any]:
+        del config_file_path
+        return {"router_settings": dict(self._config_router_settings)}
 
 
 class TestRouterSettingsEndpoints:
@@ -95,7 +96,7 @@ class TestRouterSettingsEndpoints:
         monkeypatch.setattr(
             proxy_server,
             "proxy_config",
-            _stub_proxy_config(
+            _StubProxyConfig(
                 store,
                 {"routing_strategy": "simple-shuffle", "num_retries": 3},
             ),
@@ -140,7 +141,7 @@ class TestRouterSettingsEndpoints:
         monkeypatch.setattr(
             proxy_server,
             "proxy_config",
-            _stub_proxy_config(SettingsStore("router_settings"), {}),
+            _StubProxyConfig(SettingsStore("router_settings"), {}),
         )
 
         admin_user = UserAPIKeyAuth(
diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py
index 3fb6e6fcb45..b5536b7618c 100644
--- a/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py
+++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py
@@ -11,7 +11,7 @@ Pins (PR2):
 
 from __future__ import annotations
 
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from contextlib import AbstractContextManager
 from unittest.mock import AsyncMock, MagicMock
 
@@ -22,6 +22,7 @@ import litellm
 from litellm.proxy import proxy_server
 from litellm.proxy._types import LitellmUserRoles
 from litellm.proxy.config_resolvers.settings_rules import JsonValue
+from litellm.proxy.config_resolvers.settings_store import SettingsStore
 
 from .conftest import normalize  # type: ignore[import-not-found]
 
@@ -183,9 +184,13 @@ def test_model_settings_method_not_allowed(client, auth_as):
 # ---------------------------------------------------------------------------
 
 
-def _alerting_client(monkeypatch, *, yaml_values, db_row, live_args):
-    from litellm.proxy.config_resolvers import SettingsStore
-
+def _alerting_client(
+    monkeypatch: pytest.MonkeyPatch,
+    *,
+    yaml_values: Mapping[str, JsonValue],
+    db_row: Mapping[str, JsonValue],
+    live_args: Mapping[str, JsonValue],
+) -> "SettingsStore":
     pc = MagicMock()
     row = MagicMock()
     row.param_value = db_row
@@ -206,7 +211,11 @@ def _alerting_client(monkeypatch, *, yaml_values, db_row, live_args):
     return store
 
 
-def test_alerting_settings_reports_sources(client, auth_as, monkeypatch):
+def test_alerting_settings_reports_sources(
+    client: TestClient,
+    auth_as: Callable[..., AbstractContextManager[None]],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
     _alerting_client(
         monkeypatch,
         yaml_values={
@@ -230,11 +239,21 @@ def test_alerting_settings_reports_sources(client, auth_as, monkeypatch):
     assert by_name["budget_alert_ttl"]["source"] == "default"
 
 
-def test_alerting_settings_reports_db_source_when_the_file_omits_alerting_args(client, auth_as, monkeypatch):
+def test_alerting_settings_reports_db_source_when_the_file_omits_alerting_args(
+    client: TestClient,
+    auth_as: Callable[..., AbstractContextManager[None]],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
     store = _alerting_client(
         monkeypatch,
         yaml_values={"alerting": ["slack"]},
-        db_row={"alerting_args": {"outage_alert_ttl": 4242, "region_outage_alert_ttl": []}},
+        db_row={
+            "alerting_args": {
+                "outage_alert_ttl": 4242,
+                "region_outage_alert_ttl": [],
+                "report_check_interval": None,
+            }
+        },
         live_args={"outage_alert_ttl": 4242},
     )
 
@@ -244,13 +263,18 @@ def test_alerting_settings_reports_db_source_when_the_file_omits_alerting_args(c
     assert response.status_code == 200
     by_name = {entry["field_name"]: entry for entry in response.json()}
 
-    assert store.owned_by_config("alerting_args") is False
+    assert store.source("alerting_args") == "db"
     assert by_name["outage_alert_ttl"]["source"] == "db"
-    assert by_name["region_outage_alert_ttl"]["source"] == "default"
+    assert by_name["region_outage_alert_ttl"]["source"] == "db"
+    assert by_name["report_check_interval"]["source"] == "db"
     assert by_name["budget_alert_ttl"]["source"] == "default"
 
 
-def test_alerting_settings_reports_config_source_when_db_disagrees(client, auth_as, monkeypatch):
+def test_alerting_settings_reports_config_source_when_db_disagrees(
+    client: TestClient,
+    auth_as: Callable[..., AbstractContextManager[None]],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
     from litellm.proxy.config_resolvers import SettingsStore
 
     db_alerting_args = {"daily_report_frequency": 7}
@@ -289,7 +313,7 @@ def test_alerting_settings_handles_empty_db_args(
     auth_as: Callable[..., AbstractContextManager[None]],
     monkeypatch: pytest.MonkeyPatch,
     db_alerting_args: JsonValue,
-):
+) -> None:
     from litellm.proxy.config_resolvers import SettingsStore
 
     pc = MagicMock()
@@ -318,6 +342,19 @@ def test_alerting_settings_handles_empty_db_args(
     assert by_name["budget_alert_ttl"]["source"] == "default"
 
 
+@pytest.mark.parametrize(
+    ("field_default", "expected"),
+    [(43200, "default"), (None, "unset")],
+)
+def test_nested_setting_source_without_a_config_or_db_value(field_default: JsonValue, expected: str) -> None:
+    store = SettingsStore("general_settings")
+    store.load_yaml({})
+
+    assert (
+        proxy_server._nested_setting_source(store, {}, "alerting_args", "budget_alert_ttl", field_default) == expected
+    )
+
+
 def test_alerting_settings_no_db_error(client, auth_as, no_prisma):
     """Pins ``GET /alerting/settings`` (error: db not connected)."""
     with auth_as(LitellmUserRoles.PROXY_ADMIN):
diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
index 524ab647099..75feb746bd7 100644
--- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
+++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
@@ -1342,7 +1342,7 @@ class TestProxySettingEndpoints:
             where={"id": "ui_settings"}
         )
 
-    def test_get_ui_settings_reports_sources(self, monkeypatch):
+    def test_get_ui_settings_reports_sources(self, monkeypatch: pytest.MonkeyPatch) -> None:
         from unittest.mock import AsyncMock, MagicMock
 
         from litellm.proxy import proxy_server
@@ -3532,7 +3532,7 @@ class TestPtuCostAttributionUISetting:
 
     def test_reported_config_when_secret_manager_enables_the_flag(
         self, mock_auth: None, monkeypatch: pytest.MonkeyPatch
-    ):
+    ) -> None:
         from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_ENV_VAR
 
         monkeypatch.delenv(PTU_COST_ATTRIBUTION_ENV_VAR, raising=False)
@@ -3550,7 +3550,7 @@ class TestPtuCostAttributionUISetting:
 
     def test_reported_config_when_secret_manager_disables_the_flag(
         self, mock_auth: None, monkeypatch: pytest.MonkeyPatch
-    ):
+    ) -> None:
         from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_ENV_VAR
 
         monkeypatch.delenv(PTU_COST_ATTRIBUTION_ENV_VAR, raising=False)

From a5571333fb95d4989d81f7e0ab55b69d0260db22 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:25:22 +0000
Subject: [PATCH 108/160] refactor(cache-valkey-semantic): reuse cache-redis
 connection layer

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.lock                       |   2 +-
 .../crates/cache-valkey-semantic/Cargo.toml   |   2 +-
 .../crates/cache-valkey-semantic/src/lib.rs   | 303 ++++++------------
 3 files changed, 99 insertions(+), 208 deletions(-)

diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index 6ca4ff69648..425509c6c1a 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -2507,8 +2507,8 @@ name = "litellm-cache-valkey-semantic"
 version = "0.1.0"
 dependencies = [
  "litellm-cache",
+ "litellm-cache-redis",
  "litellm-cache-response",
- "r2d2",
  "redis",
  "redis-test",
  "rstest",
diff --git a/litellm-rust/crates/cache-valkey-semantic/Cargo.toml b/litellm-rust/crates/cache-valkey-semantic/Cargo.toml
index 9a0a566ca3b..f98bb5a5fa8 100644
--- a/litellm-rust/crates/cache-valkey-semantic/Cargo.toml
+++ b/litellm-rust/crates/cache-valkey-semantic/Cargo.toml
@@ -7,8 +7,8 @@ repository.workspace = true
 
 [dependencies]
 litellm-cache.workspace = true
+litellm-cache-redis.workspace = true
 litellm-cache-response.workspace = true
-r2d2 = "0.8.10"
 redis = { version = "1.7.0", features = ["tls-rustls"] }
 serde_json.workspace = true
 sha2.workspace = true
diff --git a/litellm-rust/crates/cache-valkey-semantic/src/lib.rs b/litellm-rust/crates/cache-valkey-semantic/src/lib.rs
index ef028f0a7b2..dfc6c82596c 100644
--- a/litellm-rust/crates/cache-valkey-semantic/src/lib.rs
+++ b/litellm-rust/crates/cache-valkey-semantic/src/lib.rs
@@ -5,6 +5,7 @@ use std::{
 };
 
 use litellm_cache::{BaseCache, CacheCodec, CacheConnectionResult, Error, SemanticCacheContext};
+use litellm_cache_redis::connection::{ConnectionRef, Connections};
 use litellm_cache_response::CacheEntry;
 use serde_json::Value;
 use sha2::{Digest, Sha256};
@@ -28,40 +29,6 @@ pub struct ValkeySemanticConfig {
 
 pub const DEFAULT_INDEX_NAME: &str = "litellm_semantic_cache_index";
 
-struct PooledConnection {
-    connection: redis::Connection,
-    failed: bool,
-}
-
-struct ConnectionManager(redis::Client);
-
-impl r2d2::ManageConnection for ConnectionManager {
-    type Connection = PooledConnection;
-    type Error = redis::RedisError;
-
-    fn connect(&self) -> Result<Self::Connection, Self::Error> {
-        let connection = self.0.get_connection()?;
-        Ok(PooledConnection {
-            connection,
-            failed: false,
-        })
-    }
-
-    fn is_valid(&self, connection: &mut Self::Connection) -> Result<(), Self::Error> {
-        redis::cmd("PING").query::<String>(&mut connection.connection)?;
-        Ok(())
-    }
-
-    fn has_broken(&self, connection: &mut Self::Connection) -> bool {
-        connection.failed || !redis::ConnectionLike::is_open(&connection.connection)
-    }
-}
-
-enum Connections<C> {
-    Pool(r2d2::Pool<ConnectionManager>),
-    Fixed(Mutex<C>),
-}
-
 #[derive(Clone)]
 struct IndexState {
     name: String,
@@ -70,61 +37,8 @@ struct IndexState {
     similarity_threshold: f64,
 }
 
-struct ConnectionRef<'a>(&'a mut dyn redis::ConnectionLike);
-
-impl redis::ConnectionLike for ConnectionRef<'_> {
-    fn req_packed_command(&mut self, cmd: &[u8]) -> redis::RedisResult<redis::Value> {
-        self.0.req_packed_command(cmd)
-    }
-
-    fn req_packed_commands(
-        &mut self,
-        cmd: &[u8],
-        offset: usize,
-        count: usize,
-    ) -> redis::RedisResult<Vec<redis::Value>> {
-        self.0.req_packed_commands(cmd, offset, count)
-    }
-
-    fn get_db(&self) -> i64 {
-        self.0.get_db()
-    }
-
-    fn supports_pipelining(&self) -> bool {
-        self.0.supports_pipelining()
-    }
-
-    fn check_connection(&mut self) -> bool {
-        self.0.check_connection()
-    }
-
-    fn is_open(&self) -> bool {
-        self.0.is_open()
-    }
-}
-
-impl<C> Connections<C>
-where
-    C: redis::ConnectionLike + Send + 'static,
-{
-    fn execute<T>(
-        &self,
-        operation: impl FnOnce(&mut ConnectionRef<'_>) -> Result<T, Error>,
-    ) -> Result<T, Error> {
-        match self {
-            Self::Pool(pool) => {
-                let mut pooled = pool.get().map_err(|_| Error::Unavailable)?;
-                let result = operation(&mut ConnectionRef(&mut pooled.connection));
-                pooled.failed = matches!(result, Err(Error::Unavailable));
-                result
-            }
-            Self::Fixed(connection) => {
-                let mut connection = connection.lock().map_err(|_| Error::Unavailable)?;
-                operation(&mut ConnectionRef(&mut *connection))
-            }
-        }
-    }
-}
+const REDIS_TIMEOUT: Duration = Duration::from_secs(5);
+const REDIS_POOL_SIZE: u32 = 16;
 
 pub struct ValkeySemanticCache<
     E: Embedder,
@@ -149,15 +63,8 @@ where
         codec: S,
         config: ValkeySemanticConfig,
     ) -> Result<Self, Error> {
-        let client = redis::Client::open(url).map_err(|_| Error::Unavailable)?;
-        let pool = r2d2::Pool::builder()
-            .max_size(16)
-            .min_idle(Some(0))
-            .test_on_check_out(false)
-            .build(ConnectionManager(client))
-            .map_err(|_| Error::Unavailable)?;
         Ok(Self {
-            connections: Arc::new(Connections::Pool(pool)),
+            connections: Arc::new(Connections::pooled(url, REDIS_TIMEOUT, REDIS_POOL_SIZE)?),
             embedder,
             codec,
             config,
@@ -179,7 +86,7 @@ where
         config: ValkeySemanticConfig,
     ) -> Self {
         Self {
-            connections: Arc::new(Connections::Fixed(Mutex::new(connection))),
+            connections: Arc::new(Connections::fixed(connection)),
             embedder,
             codec,
             config,
@@ -232,15 +139,17 @@ where
         let response = self.codec.encode(&value)?;
         let vector = embedding_bytes(&embedding);
         let index = self.index_state();
-        write_document(
-            &self.connections,
-            &index,
-            &scope,
-            &prompt,
-            response,
-            vector,
-            self.get_ttl(context),
-        )
+        self.connections.execute(|connection| {
+            write_document(
+                connection,
+                &index,
+                &scope,
+                &prompt,
+                response,
+                vector,
+                self.get_ttl(context),
+            )
+        })
     }
 
     fn get_cache(&self, key: &str, context: &Self::Context) -> Result<Option<Self::Value>, Error> {
@@ -251,9 +160,10 @@ where
         let scope = scope_tag(key);
         let vector = embedding_bytes(&embedding);
         let index = self.index_state();
-        let Some(response) =
-            search_document(&self.connections, &index, &scope, vector, embedding.len())?
-        else {
+        let response = self.connections.execute(|connection| {
+            search_document(connection, &index, &scope, vector, embedding.len())
+        })?;
+        let Some(response) = response else {
             return Ok(None);
         };
         self.codec.decode(&response).map(Some)
@@ -282,11 +192,10 @@ where
             let vector = embedding_bytes(&embedding);
             let scope = scope_tag(&key);
             let ttl = context.ttl;
-            tokio::task::spawn_blocking(move || {
-                write_document(&connections, &index, &scope, &prompt, response, vector, ttl)
+            Connections::run_blocking(connections, move |connection| {
+                write_document(connection, &index, &scope, &prompt, response, vector, ttl)
             })
             .await
-            .map_err(|_| Error::Unavailable)?
         }
     }
 
@@ -308,13 +217,12 @@ where
                 .await?;
             let connections = Arc::clone(&self.connections);
             let index = self.index_state();
-            tokio::task::spawn_blocking(move || {
+            Connections::run_blocking(connections, move |connection| {
                 let scope = scope_tag(&key);
                 let vector = embedding_bytes(&embedding);
-                search_document(&connections, &index, &scope, vector, embedding.len())
+                search_document(connection, &index, &scope, vector, embedding.len())
             })
             .await
-            .map_err(|_| Error::Unavailable)?
             .and_then(|response| response.map(|bytes| self.codec.decode(&bytes)).transpose())
         }
     }
@@ -437,66 +345,58 @@ fn embedding_bytes(embedding: &[f32]) -> Vec<u8> {
         .collect()
 }
 
-fn write_document<C>(
-    connections: &Connections<C>,
+fn write_document(
+    connection: &mut ConnectionRef<'_>,
     index: &IndexState,
     scope: &str,
     prompt: &str,
     response: Vec<u8>,
     vector: Vec<u8>,
     ttl: Option<Duration>,
-) -> Result<(), Error>
-where
-    C: redis::ConnectionLike + Send + 'static,
-{
+) -> Result<(), Error> {
     let dimension = vector.len() / std::mem::size_of::<f32>();
     ensure_index(
-        connections,
+        connection,
         &index.name,
         &index.prefix,
         &index.dimension,
         dimension,
     )?;
     let document = format!("{}{scope}:{}", index.prefix, Uuid::new_v4());
-    connections.execute(|connection| {
-        let mut pipeline = redis::pipe();
+    let mut pipeline = redis::pipe();
+    pipeline
+        .cmd("HSET")
+        .arg(&document)
+        .arg("litellm_cache_key")
+        .arg(scope)
+        .arg("prompt")
+        .arg(prompt)
+        .arg("response")
+        .arg(response)
+        .arg("embedding")
+        .arg(vector)
+        .ignore();
+    if let Some(ttl) = ttl {
         pipeline
-            .cmd("HSET")
+            .cmd("EXPIRE")
             .arg(&document)
-            .arg("litellm_cache_key")
-            .arg(scope)
-            .arg("prompt")
-            .arg(prompt)
-            .arg("response")
-            .arg(response)
-            .arg("embedding")
-            .arg(vector)
+            .arg(ttl.as_secs())
             .ignore();
-        if let Some(ttl) = ttl {
-            pipeline
-                .cmd("EXPIRE")
-                .arg(&document)
-                .arg(ttl.as_secs())
-                .ignore();
-        }
-        pipeline
-            .query::<()>(connection)
-            .map_err(|_| Error::Unavailable)
-    })
+    }
+    pipeline
+        .query::<()>(connection)
+        .map_err(|_| Error::Unavailable)
 }
 
-fn search_document<C>(
-    connections: &Connections<C>,
+fn search_document(
+    connection: &mut ConnectionRef<'_>,
     index: &IndexState,
     scope: &str,
     vector: Vec<u8>,
     dimension: usize,
-) -> Result<Option<Vec<u8>>, Error>
-where
-    C: redis::ConnectionLike + Send + 'static,
-{
+) -> Result<Option<Vec<u8>>, Error> {
     ensure_index(
-        connections,
+        connection,
         &index.name,
         &index.prefix,
         &index.dimension,
@@ -504,23 +404,21 @@ where
     )?;
     let query =
         format!("(@litellm_cache_key:{{{scope}}})=>[KNN 1 @embedding $vec AS vector_distance]");
-    let response = connections.execute(|connection| {
-        redis::cmd("FT.SEARCH")
-            .arg(&index.name)
-            .arg(query)
-            .arg("PARAMS")
-            .arg(2)
-            .arg("vec")
-            .arg(vector)
-            .arg("RETURN")
-            .arg(2)
-            .arg("response")
-            .arg("vector_distance")
-            .arg("DIALECT")
-            .arg(2)
-            .query::<redis::Value>(connection)
-            .map_err(|_| Error::Unavailable)
-    })?;
+    let response = redis::cmd("FT.SEARCH")
+        .arg(&index.name)
+        .arg(query)
+        .arg("PARAMS")
+        .arg(2)
+        .arg("vec")
+        .arg(vector)
+        .arg("RETURN")
+        .arg(2)
+        .arg("response")
+        .arg("vector_distance")
+        .arg("DIALECT")
+        .arg(2)
+        .query::<redis::Value>(connection)
+        .map_err(|_| Error::Unavailable)?;
     let Some(fields) = search_fields(response)? else {
         return Ok(None);
     };
@@ -539,16 +437,13 @@ where
     Ok(Some(response))
 }
 
-fn ensure_index<C>(
-    connections: &Connections<C>,
+fn ensure_index(
+    connection: &mut ConnectionRef<'_>,
     index_name: &str,
     prefix: &str,
     index_dimension: &Mutex<Option<usize>>,
     dimension: usize,
-) -> Result<(), Error>
-where
-    C: redis::ConnectionLike + Send + 'static,
-{
+) -> Result<(), Error> {
     if index_dimension
         .lock()
         .map_err(|_| Error::Unavailable)?
@@ -556,41 +451,37 @@ where
     {
         return Ok(());
     }
-    let create = connections.execute(|connection| {
-        Ok(redis::cmd("FT.CREATE")
-            .arg(index_name)
-            .arg("ON")
-            .arg("HASH")
-            .arg("PREFIX")
-            .arg(1)
-            .arg(prefix)
-            .arg("SCHEMA")
-            .arg("litellm_cache_key")
-            .arg("TAG")
-            .arg("embedding")
-            .arg("VECTOR")
-            .arg("HNSW")
-            .arg(6)
-            .arg("TYPE")
-            .arg("FLOAT32")
-            .arg("DIM")
-            .arg(dimension)
-            .arg("DISTANCE_METRIC")
-            .arg("COSINE")
-            .query::<String>(connection)
-            .map(|_| ())
-            .map_err(|error| error.to_string()))
-    })?;
+    let create = redis::cmd("FT.CREATE")
+        .arg(index_name)
+        .arg("ON")
+        .arg("HASH")
+        .arg("PREFIX")
+        .arg(1)
+        .arg(prefix)
+        .arg("SCHEMA")
+        .arg("litellm_cache_key")
+        .arg("TAG")
+        .arg("embedding")
+        .arg("VECTOR")
+        .arg("HNSW")
+        .arg(6)
+        .arg("TYPE")
+        .arg("FLOAT32")
+        .arg("DIM")
+        .arg(dimension)
+        .arg("DISTANCE_METRIC")
+        .arg("COSINE")
+        .query::<String>(connection)
+        .map(|_| ())
+        .map_err(|error| error.to_string());
     if let Err(message) = create {
         if !message.to_ascii_lowercase().contains("already exists") {
             return Err(Error::Unavailable);
         }
-        let info = connections.execute(|connection| {
-            redis::cmd("FT.INFO")
-                .arg(index_name)
-                .query::<redis::Value>(connection)
-                .map_err(|_| Error::Unavailable)
-        })?;
+        let info = redis::cmd("FT.INFO")
+            .arg(index_name)
+            .query::<redis::Value>(connection)
+            .map_err(|_| Error::Unavailable)?;
         let existing = index_dimension_from_info(&info).ok_or(Error::Unavailable)?;
         if existing != dimension {
             return Err(Error::Unavailable);

From d1b160310f680f6c19e63dd8cd492847c66ce684 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Mon, 21 Sep 2026 14:27:29 -0700
Subject: [PATCH 109/160] fix(docker): require a generated master key in the
 quickstart stack

The committed sk-1234 placeholder is in PUBLICLY_KNOWN_MASTER_KEYS, so once an
image ships fe480533e8 the proxy refuses to boot and the quickstart stops
working. It also meant the documented stack came up on port 4000 with a
credential anyone could guess.

Both keys now come from .env and compose refuses to render without them. The
salt key is generated alongside so it stays stable across restarts, which
keeps stored credentials readable.

Verified: no .env -> compose fails closed naming the missing variable; with a
generated .env the stack is healthy, /v1/models returns 200 for the generated
key, 401 for sk-1234 and 401 unauthenticated, /ui/ serves, and the key still
works after a restart.
---
 docker/README.md                     |  1 +
 docker/docker-compose.quickstart.yml | 11 ++++++-----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/docker/README.md b/docker/README.md
index 787866ce35a..376dc7b2d97 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -9,6 +9,7 @@ This guide provides instructions for building and running the LiteLLM applicatio
 >
 > ```bash
 > curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml
+> printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env
 > docker compose -f docker-compose.quickstart.yml up -d
 > ```
 
diff --git a/docker/docker-compose.quickstart.yml b/docker/docker-compose.quickstart.yml
index 7c849c759f0..11631603a72 100644
--- a/docker/docker-compose.quickstart.yml
+++ b/docker/docker-compose.quickstart.yml
@@ -3,19 +3,20 @@
 # https://docs.litellm.ai/docs/proxy/docker_quick_start
 #
 #   curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml
+#   printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env
 #   docker compose -f docker-compose.quickstart.yml up -d
 #
-# The credentials below are placeholders for local evaluation. Before any real
-# use, set LITELLM_MASTER_KEY and LITELLM_SALT_KEY to long random values and
-# pin the image to a specific release tag.
+# Compose reads .env from this directory. Keep it: regenerating LITELLM_SALT_KEY
+# makes credentials already stored in the database unreadable. For anything
+# beyond local evaluation, pin the image to a specific release tag.
 services:
   litellm:
     image: docker.litellm.ai/berriai/litellm:main-stable
     ports:
       - "4000:4000"
     environment:
-      LITELLM_MASTER_KEY: sk-1234
-      LITELLM_SALT_KEY: sk-XXXXXXXXXXXXXXXX
+      LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?set it in .env - see the header of this file}
+      LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:?set it in .env - see the header of this file}
       DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm
       STORE_MODEL_IN_DB: "True"
     depends_on:

From 4c1a6309c2d677036fbb86e5acb670775dc43f2c Mon Sep 17 00:00:00 2001
From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
Date: Mon, 21 Sep 2026 14:31:28 -0700
Subject: [PATCH 110/160] test(mcp): type server fixtures and role queries

---
 .../mcp-servers/_components/mcp_servers.test.tsx          | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx
index 7198cb3cce9..1217d878489 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx
@@ -184,6 +184,8 @@ describe("MCPServers", () => {
 
   it.each(["cancel", "success", "failure", "unnamed"])("preserves delete confirmation on %s", async (outcome) => {
     const server: MCPServer = {
+      created_at: "",
+      updated_at: "",
       server_id: "delete-server",
       server_name: outcome === "unnamed" ? null : "Delete fixture",
       alias: "delete-alias",
@@ -220,7 +222,7 @@ describe("MCPServers", () => {
       await userEvent.click(within(dialog).getByRole("button", { name: "Cancel" }));
       expect(networking.deleteMCPServer).not.toHaveBeenCalled();
     } else {
-      await userEvent.click(within(dialog).getByRole("button", { name: "Delete", exact: true }));
+      await userEvent.click(within(dialog).getByRole("button", { name: "Delete" }));
       expect(within(dialog).getByRole("button", { name: "Deleting..." })).toBeDisabled();
       expect(within(dialog).getByRole("button", { name: "Cancel" })).toBeDisabled();
       expect(networking.deleteMCPServer).toHaveBeenCalledWith("123", "delete-server");
@@ -262,7 +264,7 @@ describe("MCPServers", () => {
     );
     await screen.findByText("String group");
     await userEvent.click(screen.getByRole("combobox", { name: "Access Group" }));
-    await userEvent.click(await screen.findByRole("option", { name: "shared", exact: true }));
+    await userEvent.click(await screen.findByRole("option", { name: "shared" }));
     expect(screen.getByText("String group")).toBeVisible();
     expect(screen.getByText("Legacy group")).toBeVisible();
     expect(screen.queryByText("Other group")).not.toBeInTheDocument();
@@ -270,6 +272,8 @@ describe("MCPServers", () => {
 
   it.each(["server_name", "alias", "url", "server_id"] as const)("searches by %s case-insensitively", async (field) => {
     const server: MCPServer = {
+      created_at: "",
+      updated_at: "",
       server_id: "search-server",
       server_name: "Search fixture",
       created_by: "user",

From 2a5112d146bf83f8fe70473d75bc11b259e139f7 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:32:40 +0000
Subject: [PATCH 111/160] feat(cache-redis): expose the pooled connection
 handling for reuse

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/cache-redis/src/cache.rs  | 67 +++++++++++--------
 .../cache-redis/src/cache/connection.rs       |  2 +-
 .../cache-redis/src/cache/operations.rs       | 28 ++++----
 litellm-rust/crates/cache-redis/src/lib.rs    |  4 ++
 4 files changed, 57 insertions(+), 44 deletions(-)

diff --git a/litellm-rust/crates/cache-redis/src/cache.rs b/litellm-rust/crates/cache-redis/src/cache.rs
index e2e2656fcbb..24399c9b2f9 100644
--- a/litellm-rust/crates/cache-redis/src/cache.rs
+++ b/litellm-rust/crates/cache-redis/src/cache.rs
@@ -14,7 +14,7 @@ use crate::topology::RedisTopology;
 mod connection;
 mod operations;
 
-pub(crate) use connection::ConnectionRef;
+pub use connection::ConnectionRef;
 use connection::{ClusterConnectionManager, ConnectionManager};
 
 pub use operations::{
@@ -40,7 +40,8 @@ const CLAIM_SCRIPT: &str = concat!(
 );
 const CLAIM_ATTEMPTS: usize = 8;
 
-enum Connections<C> {
+#[allow(private_interfaces)]
+pub enum Connections<C> {
     Pool(r2d2::Pool<ConnectionManager>),
     Cluster(r2d2::Pool<ClusterConnectionManager>),
     Fixed(Mutex<C>),
@@ -50,7 +51,7 @@ impl<C> Connections<C>
 where
     C: redis::ConnectionLike + Send + 'static,
 {
-    fn execute<T>(
+    pub fn execute<T>(
         &self,
         operation: impl FnOnce(&mut ConnectionRef<'_>) -> Result<T, Error>,
     ) -> Result<T, Error> {
@@ -73,6 +74,29 @@ where
             }
         }
     }
+
+    pub async fn run_blocking<T, F>(connections: Arc<Self>, operation: F) -> Result<T, Error>
+    where
+        T: Send + 'static,
+        F: FnOnce(&mut ConnectionRef<'_>) -> Result<T, Error> + Send + 'static,
+    {
+        tokio::task::spawn_blocking(move || connections.execute(operation))
+            .await
+            .map_err(|_| Error::Unavailable)?
+    }
+
+    pub fn fixed(connection: C) -> Self {
+        Self::Fixed(Mutex::new(connection))
+    }
+
+    pub fn open(url: &str, topology: &RedisTopology) -> Result<Self, Error> {
+        match topology {
+            RedisTopology::Standalone => Ok(Self::Pool(pool(ConnectionManager::open(url)?)?)),
+            RedisTopology::Cluster { startup_nodes } => Ok(Self::Cluster(pool(
+                ClusterConnectionManager::open(url, startup_nodes)?,
+            )?)),
+        }
+    }
 }
 
 pub struct RedisCache<S, C = redis::Connection> {
@@ -94,12 +118,7 @@ impl<S: CacheCodec> RedisCache<S> {
         default_ttl: Option<Duration>,
         codec: S,
     ) -> Result<Self, Error> {
-        let connections = match topology {
-            RedisTopology::Standalone => Connections::Pool(pool(ConnectionManager::open(url)?)?),
-            RedisTopology::Cluster { startup_nodes } => {
-                Connections::Cluster(pool(ClusterConnectionManager::open(url, startup_nodes)?)?)
-            }
-        };
+        let connections = Connections::open(url, topology)?;
         Ok(Self {
             connections: Arc::new(connections),
             default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
@@ -127,7 +146,7 @@ where
 {
     pub fn with_connection(connection: C, default_ttl: Option<Duration>, codec: S) -> Self {
         Self {
-            connections: Arc::new(Connections::Fixed(Mutex::new(connection))),
+            connections: Arc::new(Connections::fixed(connection)),
             default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
             codec,
             namespace: None,
@@ -203,16 +222,6 @@ where
             .saturating_add(u64::from(ttl.subsec_nanos() > 0))
             .max(1)
     }
-
-    async fn run_blocking<T, F>(connections: Arc<Connections<C>>, operation: F) -> Result<T, Error>
-    where
-        T: Send + 'static,
-        F: FnOnce(&mut ConnectionRef<'_>) -> Result<T, Error> + Send + 'static,
-    {
-        tokio::task::spawn_blocking(move || connections.execute(operation))
-            .await
-            .map_err(|_| Error::Unavailable)?
-    }
 }
 
 fn namespaced_key(namespace: Option<&str>, key: &str) -> String {
@@ -271,7 +280,7 @@ where
         let payload = self.codec.encode(&value)?;
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             connection
                 .set_ex::<_, _, ()>(key, payload, ttl)
                 .map_err(|_| Error::Unavailable)
@@ -285,7 +294,7 @@ where
         _: &ExactCacheContext,
     ) -> Result<Option<Self::Value>, Error> {
         let key = self.namespaced_key(key);
-        let value = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let value = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             connection
                 .get::<_, redis::Value>(key)
                 .map_err(|_| Error::Unavailable)
@@ -311,7 +320,7 @@ where
         if entries.is_empty() {
             return Ok(());
         }
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let commands = entries
                 .into_iter()
                 .map(|(key, payload)| {
@@ -330,7 +339,7 @@ where
     }
 
     async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
-        match Self::run_blocking(Arc::clone(&self.connections), |connection| {
+        match Connections::run_blocking(Arc::clone(&self.connections), |connection| {
             Ok(match connection.ping() {
                 Ok(_) => CacheConnectionResult {
                     status: CacheConnectionStatus::Success,
@@ -391,7 +400,7 @@ where
             .iter()
             .map(|key| self.namespaced_key(key))
             .collect::<Vec<_>>();
-        let values = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("MGET")
                 .arg(keys)
                 .query::<Vec<redis::Value>>(connection)
@@ -418,7 +427,7 @@ where
 
     async fn async_delete_cache(&self, key: &str) -> Result<(), Error> {
         let key = self.namespaced_key(key);
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             connection.del::<_, ()>(key).map_err(|_| Error::Unavailable)
         })
         .await
@@ -438,7 +447,7 @@ where
 
     async fn async_flush_cache(&self) -> Result<(), Error> {
         let pattern = self.namespaced_pattern()?;
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             Self::flush_matching(connection, &pattern)
         })
         .await
@@ -470,7 +479,7 @@ where
     ) -> Result<f64, Error> {
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             increment(connection, key, amount, ttl)
         })
         .await
@@ -581,7 +590,7 @@ where
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
         let codec = self.codec.clone();
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             claim(connection, &codec, &key, candidate, &eligible, ttl)
         })
         .await
diff --git a/litellm-rust/crates/cache-redis/src/cache/connection.rs b/litellm-rust/crates/cache-redis/src/cache/connection.rs
index 1834f1d94e5..06364296992 100644
--- a/litellm-rust/crates/cache-redis/src/cache/connection.rs
+++ b/litellm-rust/crates/cache-redis/src/cache/connection.rs
@@ -117,7 +117,7 @@ impl r2d2::ManageConnection for ClusterConnectionManager {
     }
 }
 
-pub(crate) enum ConnectionRef<'a> {
+pub enum ConnectionRef<'a> {
     Node(&'a mut dyn redis::ConnectionLike),
     Cluster(&'a mut ClusterConnection),
 }
diff --git a/litellm-rust/crates/cache-redis/src/cache/operations.rs b/litellm-rust/crates/cache-redis/src/cache/operations.rs
index 9a7023338bf..4345ee879b3 100644
--- a/litellm-rust/crates/cache-redis/src/cache/operations.rs
+++ b/litellm-rust/crates/cache-redis/src/cache/operations.rs
@@ -144,7 +144,7 @@ where
             .into_iter()
             .map(|key| self.namespaced_key(&key))
             .collect::<Vec<_>>();
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             connection.del(keys).map_err(|_| Error::Unavailable)
         })
         .await
@@ -172,7 +172,7 @@ where
             .iter()
             .map(|key| self.namespaced_key(key))
             .collect::<Vec<_>>();
-        let values = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("MGET")
                 .arg(keys)
                 .query::<Vec<redis::Value>>(connection)
@@ -188,7 +188,7 @@ where
     }
 
     pub async fn ping(&self) -> Result<bool, Error> {
-        Self::run_blocking(Arc::clone(&self.connections), |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), |connection| {
             connection.ping().map_err(|_| Error::Unavailable)
         })
         .await
@@ -196,7 +196,7 @@ where
 
     pub async fn async_get_ttl(&self, key: &str) -> Result<Option<i64>, Error> {
         let key = self.namespaced_key(key);
-        let ttl = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let ttl = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("TTL")
                 .arg(key)
                 .query::<i64>(connection)
@@ -208,7 +208,7 @@ where
 
     pub async fn async_scan_iter(&self, pattern: &str, count: usize) -> Result<Vec<String>, Error> {
         let pattern = format!("{}*", self.namespaced_key(pattern));
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut matches = Vec::new();
             connection.scan(&pattern, count, |_, keys| {
                 matches.extend(keys);
@@ -231,7 +231,7 @@ where
         }
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(ttl.unwrap_or(self.default_ttl));
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut sadd = redis::cmd("SADD");
             sadd.arg(&key).arg(values);
             let mut expire = redis::cmd("EXPIRE");
@@ -253,7 +253,7 @@ where
             return Err(Error::InvalidEntry);
         }
         let key = self.namespaced_key(key);
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("RPUSH")
                 .arg(key)
                 .arg(values)
@@ -279,7 +279,7 @@ where
         if operations.is_empty() {
             return Ok(Vec::new());
         }
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let commands = operations
                 .into_iter()
                 .map(|(key, values)| {
@@ -304,7 +304,7 @@ where
     ) -> Result<RedisLpopResult, Error> {
         let key = self.namespaced_key(key);
         let multiple = count.is_some();
-        let value = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let value = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut command = redis::cmd("LPOP");
             command.arg(key);
             if let Some(count) = count {
@@ -333,7 +333,7 @@ where
             .iter()
             .map(|(_, count)| count.is_some())
             .collect::<Vec<_>>();
-        let values = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let commands = operations
                 .into_iter()
                 .map(|(key, count)| {
@@ -365,7 +365,7 @@ where
             .into_iter()
             .map(|key| self.namespaced_key(&key))
             .collect::<Vec<_>>();
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("EVAL")
                 .arg(script)
                 .arg(keys.len())
@@ -426,7 +426,7 @@ where
         if operations.is_empty() {
             return Ok(Vec::new());
         }
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             let mut commands = Vec::with_capacity(operations.len() * 2);
             let mut increments = Vec::with_capacity(operations.len());
             for (key, amount, ttl) in operations {
@@ -460,7 +460,7 @@ where
     ) -> Result<i64, Error> {
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(ttl);
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             increment_with_floor(connection, key, amount, ttl)
         })
         .await
@@ -474,7 +474,7 @@ where
     ) -> Result<f64, Error> {
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(ttl.unwrap_or(self.default_ttl));
-        Self::run_blocking(Arc::clone(&self.connections), move |connection| {
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             redis::cmd("EVAL")
                 .arg(SET_MAX_SCRIPT)
                 .arg(1)
diff --git a/litellm-rust/crates/cache-redis/src/lib.rs b/litellm-rust/crates/cache-redis/src/lib.rs
index 98f6bfd8ce5..efb0db931ac 100644
--- a/litellm-rust/crates/cache-redis/src/lib.rs
+++ b/litellm-rust/crates/cache-redis/src/lib.rs
@@ -1,6 +1,10 @@
 mod cache;
 mod topology;
 
+pub mod connection {
+    pub use crate::cache::{ConnectionRef, Connections};
+}
+
 pub use cache::{
     RedisArg, RedisCache, RedisLpopOperation, RedisLpopResult, RedisRpushOperation, RedisScript,
 };

From b6b0e58ba399d5e8a01c4fc13de4f4c2c6af7fd6 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:32:42 +0000
Subject: [PATCH 112/160] refactor(cache-valkey-semantic): reuse cache-redis
 connection layer

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/cache-valkey-semantic/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/litellm-rust/crates/cache-valkey-semantic/src/lib.rs b/litellm-rust/crates/cache-valkey-semantic/src/lib.rs
index dfc6c82596c..2bd4bd71bce 100644
--- a/litellm-rust/crates/cache-valkey-semantic/src/lib.rs
+++ b/litellm-rust/crates/cache-valkey-semantic/src/lib.rs
@@ -5,7 +5,10 @@ use std::{
 };
 
 use litellm_cache::{BaseCache, CacheCodec, CacheConnectionResult, Error, SemanticCacheContext};
-use litellm_cache_redis::connection::{ConnectionRef, Connections};
+use litellm_cache_redis::{
+    RedisTopology,
+    connection::{ConnectionRef, Connections},
+};
 use litellm_cache_response::CacheEntry;
 use serde_json::Value;
 use sha2::{Digest, Sha256};
@@ -37,9 +40,6 @@ struct IndexState {
     similarity_threshold: f64,
 }
 
-const REDIS_TIMEOUT: Duration = Duration::from_secs(5);
-const REDIS_POOL_SIZE: u32 = 16;
-
 pub struct ValkeySemanticCache<
     E: Embedder,
     S: CacheCodec<Value = CacheEntry>,
@@ -64,7 +64,7 @@ where
         config: ValkeySemanticConfig,
     ) -> Result<Self, Error> {
         Ok(Self {
-            connections: Arc::new(Connections::pooled(url, REDIS_TIMEOUT, REDIS_POOL_SIZE)?),
+            connections: Arc::new(Connections::open(url, &RedisTopology::Standalone)?),
             embedder,
             codec,
             config,

From ef2ab74c7aecab4f970827ec21d740ed3aaa430b Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:33:38 +0000
Subject: [PATCH 113/160] refactor(rust): back the vault secret manager with
 vaultrs

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.lock                       |  91 +++-
 litellm-rust/Cargo.toml                       |   3 +
 .../crates/secrets-hashicorp/Cargo.toml       |   4 +-
 .../secrets-hashicorp/src/cert_login.rs       |  25 ++
 .../crates/secrets-hashicorp/src/error.rs     |   8 +-
 .../crates/secrets-hashicorp/src/lib.rs       |   3 +-
 .../secrets-hashicorp/src/secret_manager.rs   | 406 +++++++++---------
 .../secrets-hashicorp/tests/secret_manager.rs | 293 +++++++++----
 litellm-rust/crates/secrets/tests/handler.rs  |  39 +-
 9 files changed, 586 insertions(+), 286 deletions(-)
 create mode 100644 litellm-rust/crates/secrets-hashicorp/src/cert_login.rs

diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index 6f05bba9417..7b59533ad06 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -2763,13 +2763,15 @@ dependencies = [
  "litellm-core-utils",
  "litellm-secrets-types",
  "moka",
- "reqwest 0.12.28",
  "rstest",
+ "rustify",
+ "rustify_derive",
  "serde",
  "serde_json",
  "tempfile",
  "thiserror 2.0.19",
  "tokio",
+ "vaultrs",
  "veil",
  "wiremock",
 ]
@@ -3975,6 +3977,40 @@ dependencies = [
  "semver",
 ]
 
+[[package]]
+name = "rustify"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4800ce4c1cc2fec12c559dae2ddbf0e17fcee7569b796e6d75898efef443368b"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "bytes",
+ "http 1.4.2",
+ "reqwest 0.13.5",
+ "rustify_derive",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "thiserror 1.0.69",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "rustify_derive"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78ea7fda74240f7410d0198b603a8a2f662acc7d76b6667a49f9b162cd8d9b4f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "regex",
+ "serde_urlencoded",
+ "syn 1.0.109",
+ "synstructure 0.12.6",
+]
+
 [[package]]
 name = "rustix"
 version = "1.1.5"
@@ -4502,6 +4538,17 @@ version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "syn"
 version = "2.0.119"
@@ -4533,6 +4580,18 @@ dependencies = [
  "futures-core",
 ]
 
+[[package]]
+name = "synstructure"
+version = "0.12.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "unicode-xid",
+]
+
 [[package]]
 name = "synstructure"
 version = "0.13.2"
@@ -4947,6 +5006,7 @@ version = "0.1.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
 dependencies = [
+ "log",
  "pin-project-lite",
  "tracing-attributes",
  "tracing-core",
@@ -5130,6 +5190,12 @@ version = "1.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
 
+[[package]]
+name = "unicode-xid"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
+
 [[package]]
 name = "unicode_categories"
 version = "0.1.1"
@@ -5189,6 +5255,25 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
 
+[[package]]
+name = "vaultrs"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30ffcc0e81025065dda612ec1e26a3d81bb16ef3062354873d17a35965d68522"
+dependencies = [
+ "async-trait",
+ "derive_builder",
+ "http 1.4.2",
+ "reqwest 0.13.5",
+ "rustify",
+ "rustify_derive",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.19",
+ "tracing",
+ "url",
+]
+
 [[package]]
 name = "veil"
 version = "0.3.0"
@@ -5632,7 +5717,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "syn 2.0.119",
- "synstructure",
+ "synstructure 0.13.2",
 ]
 
 [[package]]
@@ -5673,7 +5758,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "syn 2.0.119",
- "synstructure",
+ "synstructure 0.13.2",
 ]
 
 [[package]]
diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
index 5030a94b140..edc9f5a91f6 100644
--- a/litellm-rust/Cargo.toml
+++ b/litellm-rust/Cargo.toml
@@ -51,6 +51,9 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "mul
 rstest = "0.26.1"
 rstest_reuse = "0.7.0"
 rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] }
+rustify = "=0.7.0"
+rustify_derive = "=0.5.5"
+vaultrs = { version = "=0.8.0", default-features = false, features = ["rustls"] }
 rustls-native-certs = "0.8"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["float_roundtrip"] }
diff --git a/litellm-rust/crates/secrets-hashicorp/Cargo.toml b/litellm-rust/crates/secrets-hashicorp/Cargo.toml
index 0656980a033..c646e02ef09 100644
--- a/litellm-rust/crates/secrets-hashicorp/Cargo.toml
+++ b/litellm-rust/crates/secrets-hashicorp/Cargo.toml
@@ -9,11 +9,13 @@ repository.workspace = true
 litellm-core-utils.workspace = true
 litellm-secrets-types.workspace = true
 moka.workspace = true
-reqwest.workspace = true
+rustify.workspace = true
+rustify_derive.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 thiserror.workspace = true
 tokio.workspace = true
+vaultrs.workspace = true
 veil.workspace = true
 
 [dev-dependencies]
diff --git a/litellm-rust/crates/secrets-hashicorp/src/cert_login.rs b/litellm-rust/crates/secrets-hashicorp/src/cert_login.rs
new file mode 100644
index 00000000000..1eb09a89715
--- /dev/null
+++ b/litellm-rust/crates/secrets-hashicorp/src/cert_login.rs
@@ -0,0 +1,25 @@
+#[derive(Debug, rustify_derive::Endpoint)]
+#[endpoint(path = "/auth/{self.mount}/login", method = "POST")]
+pub struct CertLoginRequest {
+    #[endpoint(skip)]
+    pub mount: String,
+    #[endpoint(skip)]
+    #[allow(dead_code)]
+    pub name: Option<String>,
+    #[endpoint(raw)]
+    body: Vec<u8>,
+}
+
+impl CertLoginRequest {
+    pub fn new(name: Option<String>) -> Self {
+        let body: Vec<u8> = match name.as_deref() {
+            Some(name) => serde_json::to_vec(&serde_json::json!({ "name": name })).unwrap(),
+            None => b"{}".to_vec(),
+        };
+        Self {
+            mount: "cert".to_owned(),
+            name,
+            body,
+        }
+    }
+}
diff --git a/litellm-rust/crates/secrets-hashicorp/src/error.rs b/litellm-rust/crates/secrets-hashicorp/src/error.rs
index 3f6a5b66475..e26033af085 100644
--- a/litellm-rust/crates/secrets-hashicorp/src/error.rs
+++ b/litellm-rust/crates/secrets-hashicorp/src/error.rs
@@ -4,12 +4,14 @@ pub enum Error {
     EnterpriseRequired,
     #[error("invalid secret name")]
     InvalidSecretName(#[from] litellm_secrets_types::Error),
-    #[error("HashiCorp Vault request failed")]
-    Request(
+    #[error("HashiCorp Vault client failed")]
+    Client(
         #[from]
         #[redact]
-        reqwest::Error,
+        vaultrs::error::ClientError,
     ),
+    #[error("HashiCorp Vault client settings are invalid: {message}")]
+    ClientSettings { message: String },
     #[error("HashiCorp Vault TLS identity could not be configured for {path}: {message}")]
     TlsIdentity {
         path: std::path::PathBuf,
diff --git a/litellm-rust/crates/secrets-hashicorp/src/lib.rs b/litellm-rust/crates/secrets-hashicorp/src/lib.rs
index a561a58b59e..0c2b05647f8 100644
--- a/litellm-rust/crates/secrets-hashicorp/src/lib.rs
+++ b/litellm-rust/crates/secrets-hashicorp/src/lib.rs
@@ -1,9 +1,10 @@
 #![forbid(unsafe_code)]
 
+mod cert_login;
 mod config;
 mod error;
 pub mod secret_manager;
 
 pub use config::{AppRoleAuth, HashicorpVaultConfig, TlsCertAuth};
 pub use error::Error;
-pub use secret_manager::HashicorpVault;
+pub use secret_manager::{HashicorpVault, SecretLocation};
diff --git a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs
index 52cc143c1e3..7937fa93f8f 100644
--- a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs
+++ b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs
@@ -1,4 +1,5 @@
 use std::{
+    collections::HashMap,
     fmt,
     sync::Arc,
     time::{Duration, Instant},
@@ -9,26 +10,39 @@ use litellm_secrets_types::{
     BaseSecretManager, SecretValue, async_rotate_secret, validate_secret_name,
 };
 use moka::future::Cache;
-use reqwest::{Client, Identity, RequestBuilder};
-use serde_json::{Value, json};
+use rustify::errors::ClientError as RustifyClientError;
+use serde_json::Value;
 use tokio::sync::Mutex;
+use vaultrs::{
+    api,
+    auth::approle,
+    client::{Identity, VaultClient, VaultClientSettingsBuilder},
+    error::ClientError,
+    kv2,
+};
 
-use crate::{Error, HashicorpVaultConfig, TlsCertAuth};
+use crate::{Error, HashicorpVaultConfig, TlsCertAuth, cert_login::CertLoginRequest};
 
 const CACHE_CAPACITY: u64 = 200;
 
 #[derive(Clone)]
-struct CachedToken {
-    token: SecretValue,
+struct CachedClient {
+    client: Arc<VaultClient>,
     expires_at: Option<Instant>,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct SecretLocation {
+    pub namespace: Option<String>,
+    pub mount: String,
+    pub path: String,
+}
+
 #[derive(Clone)]
 pub struct HashicorpVault {
-    client: Arc<Client>,
     config: HashicorpVaultConfig,
     cache: Cache<String, SecretValue>,
-    auth_token: Arc<Mutex<Option<CachedToken>>>,
+    auth_client: Arc<Mutex<Option<CachedClient>>>,
 }
 
 impl fmt::Debug for HashicorpVault {
@@ -45,17 +59,12 @@ impl HashicorpVault {
         environment: Arc<dyn Lookup + Send + Sync>,
         enterprise_enabled: bool,
     ) -> Result<Self, Error> {
-        if !enterprise_enabled {
-            return Err(Error::EnterpriseRequired);
-        }
         let config: HashicorpVaultConfig =
             HashicorpVaultConfig::from_environment(environment.as_ref())?;
-        let client: Client = client_for_config(&config)?;
-        Self::with_client(client, config, enterprise_enabled)
+        Self::from_config(config, enterprise_enabled)
     }
 
-    pub fn with_client(
-        client: Client,
+    pub fn from_config(
         config: HashicorpVaultConfig,
         enterprise_enabled: bool,
     ) -> Result<Self, Error> {
@@ -67,47 +76,27 @@ impl HashicorpVault {
             .time_to_live(config.refresh_interval)
             .build();
         Ok(Self {
-            client: Arc::new(client),
             config,
             cache,
-            auth_token: Arc::new(Mutex::new(None)),
+            auth_client: Arc::new(Mutex::new(None)),
         })
     }
 
-    pub fn secret_url(&self, secret_name: &str) -> Result<String, Error> {
+    pub fn secret_location(&self, secret_name: &str) -> Result<SecretLocation, Error> {
         validate_secret_name(secret_name).map_err(Error::InvalidSecretName)?;
-        let namespace: String = self
-            .config
-            .secret_namespace()
-            .map(|value| format!("{value}/"))
-            .unwrap_or_default();
-        let path_prefix: String = self
-            .config
-            .path_prefix
-            .as_deref()
-            .map(|value| format!("{value}/"))
-            .unwrap_or_default();
-        Ok(format!(
-            "{}/v1/{}{}/data/{}{}",
-            self.config.address, namespace, self.config.mount, path_prefix, secret_name
-        ))
-    }
-
-    pub fn login_url(&self) -> Option<String> {
-        self.config.approle.as_ref().map_or_else(
-            || {
-                self.config
-                    .tls_cert
-                    .as_ref()
-                    .map(|_| format!("{}/v1/auth/cert/login", self.config.address))
-            },
-            |approle| {
-                Some(format!(
-                    "{}/v1/auth/{}/login",
-                    self.config.address, approle.mount_path
-                ))
-            },
-        )
+        let path: String = [
+            self.config.path_prefix.clone(),
+            Some(secret_name.to_owned()),
+        ]
+        .into_iter()
+        .flatten()
+        .collect::<Vec<String>>()
+        .join("/");
+        Ok(SecretLocation {
+            namespace: self.config.secret_namespace().map(str::to_owned),
+            mount: self.config.mount.clone(),
+            path,
+        })
     }
 
     pub fn config(&self) -> &HashicorpVaultConfig {
@@ -115,40 +104,24 @@ impl HashicorpVault {
     }
 
     pub async fn async_read_secret(&self, secret_name: &str) -> Result<Option<SecretValue>, Error> {
-        let url: String = self.secret_url(secret_name)?;
-        if let Some(value) = self.cache.get(&url).await {
+        let location: SecretLocation = self.secret_location(secret_name)?;
+        let cache_key: String = cache_key(&location);
+        if let Some(value) = self.cache.get(&cache_key).await {
             return Ok(Some(value));
         }
-        let token: SecretValue = self.vault_token().await?;
-        let response: reqwest::Response = self
-            .client
-            .get(&url)
-            .header("X-Vault-Token", token.expose())
-            .send()
-            .await?;
-        if response.status() == reqwest::StatusCode::NOT_FOUND {
-            return Ok(None);
-        }
-        if !response.status().is_success() {
-            return Err(Error::Status {
-                status: response.status().as_u16(),
-            });
-        }
-        let body = response.bytes().await?;
-        let body: Value = serde_json::from_slice(&body).map_err(|_| Error::MalformedPayload)?;
-        let data: &Value = body
-            .get("data")
-            .and_then(Value::as_object)
-            .and_then(|value| value.get("data"))
-            .ok_or(Error::MalformedPayload)?;
-        let data: &serde_json::Map<String, Value> =
-            data.as_object().ok_or(Error::MalformedPayload)?;
+        let client: Arc<VaultClient> = self.vault_client().await?;
+        let data: HashMap<String, Value> =
+            match kv2::read(client.as_ref(), &location.mount, &location.path).await {
+                Ok(data) => data,
+                Err(error) if api_status(&error) == Some(404) => return Ok(None),
+                Err(error) => return Err(map_api_error(error, ErrorContext::Read)),
+            };
         let Some(value) = data.get("key") else {
             return Ok(None);
         };
         let value: &str = value.as_str().ok_or(Error::NonStringValue)?;
         let value: SecretValue = SecretValue::new(value);
-        self.cache.insert(url, value.clone()).await;
+        self.cache.insert(cache_key, value.clone()).await;
         Ok(Some(value))
     }
 
@@ -158,47 +131,39 @@ impl HashicorpVault {
         value: SecretValue,
         description: Option<&str>,
     ) -> Result<Value, Error> {
-        let url: String = self.secret_url(secret_name)?;
-        let data: Value = match description {
-            Some(description) => json!({"key": value.expose(), "description": description}),
-            None => json!({"key": value.expose()}),
+        let location: SecretLocation = self.secret_location(secret_name)?;
+        let cache_key: String = cache_key(&location);
+        let data: HashMap<String, Value> = match description {
+            Some(description) => [
+                ("key".to_owned(), Value::String(value.expose().to_owned())),
+                (
+                    "description".to_owned(),
+                    Value::String(description.to_owned()),
+                ),
+            ]
+            .into_iter()
+            .collect(),
+            None => [("key".to_owned(), Value::String(value.expose().to_owned()))]
+                .into_iter()
+                .collect(),
         };
-        let token: SecretValue = self.vault_token().await?;
-        let response: reqwest::Response = self
-            .client
-            .post(&url)
-            .header("X-Vault-Token", token.expose())
-            .json(&json!({"data": data}))
-            .send()
-            .await?;
-        if !response.status().is_success() {
-            return Err(Error::Status {
-                status: response.status().as_u16(),
-            });
-        }
-        self.cache.invalidate(&url).await;
-        let body = response.bytes().await?;
-        if body.is_empty() {
-            return Ok(Value::Null);
-        }
-        serde_json::from_slice(&body).map_err(|_| Error::MalformedPayload)
+        let client: Arc<VaultClient> = self.vault_client().await?;
+        let metadata = kv2::set(client.as_ref(), &location.mount, &location.path, &data)
+            .await
+            .map_err(|error| map_api_error(error, ErrorContext::Secret))?;
+        self.cache.invalidate(&cache_key).await;
+        serde_json::to_value(metadata)
+            .map_err(|source| Error::Client(ClientError::JsonParseError { source }))
     }
 
     pub async fn async_delete_secret(&self, secret_name: &str) -> Result<(), Error> {
-        let url: String = self.secret_url(secret_name)?;
-        let token: SecretValue = self.vault_token().await?;
-        let response: reqwest::Response = self
-            .client
-            .delete(&url)
-            .header("X-Vault-Token", token.expose())
-            .send()
-            .await?;
-        if !response.status().is_success() {
-            return Err(Error::Status {
-                status: response.status().as_u16(),
-            });
-        }
-        self.cache.invalidate(&url).await;
+        let location: SecretLocation = self.secret_location(secret_name)?;
+        let cache_key: String = cache_key(&location);
+        let client: Arc<VaultClient> = self.vault_client().await?;
+        kv2::delete_latest(client.as_ref(), &location.mount, &location.path)
+            .await
+            .map_err(|error| map_api_error(error, ErrorContext::Secret))?;
+        self.cache.invalidate(&cache_key).await;
         Ok(())
     }
 
@@ -211,69 +176,76 @@ impl HashicorpVault {
         async_rotate_secret(self, current_name, new_name, value).await
     }
 
-    async fn vault_token(&self) -> Result<SecretValue, Error> {
-        let mut cached: tokio::sync::MutexGuard<'_, Option<CachedToken>> =
-            self.auth_token.lock().await;
+    async fn vault_client(&self) -> Result<Arc<VaultClient>, Error> {
+        let mut cached = self.auth_client.lock().await;
         if let Some(entry) = cached.as_ref()
             && entry
                 .expires_at
                 .is_none_or(|expires_at| expires_at > Instant::now())
         {
-            return Ok(entry.token.clone());
+            return Ok(entry.client.clone());
         }
-        let Some(login_url) = self.login_url() else {
-            let Some(token) = self.config.token.clone() else {
-                return Err(Error::NoAuthConfigured);
+
+        let (client, expires_at): (VaultClient, Option<Instant>) =
+            match (self.config.approle.as_ref(), self.config.tls_cert.as_ref()) {
+                (Some(approle), _) => {
+                    let login_client: VaultClient =
+                        self.build_client(self.config.login_namespace(), "")?;
+                    let auth = approle::login(
+                        &login_client,
+                        &approle.mount_path,
+                        &approle.role_id,
+                        approle.secret_id.expose(),
+                    )
+                    .await
+                    .map_err(|error| map_api_error(error, ErrorContext::Login))?;
+                    (
+                        self.build_client(self.config.secret_namespace(), &auth.client_token)?,
+                        token_expiry(auth.lease_duration),
+                    )
+                }
+                (None, Some(tls)) => {
+                    let login_client: VaultClient =
+                        self.build_client(self.config.login_namespace(), "")?;
+                    let endpoint: CertLoginRequest = CertLoginRequest::new(tls.role.clone());
+                    let auth = api::auth(&login_client, endpoint)
+                        .await
+                        .map_err(|error| map_api_error(error, ErrorContext::Login))?;
+                    (
+                        self.build_client(self.config.secret_namespace(), &auth.client_token)?,
+                        token_expiry(auth.lease_duration),
+                    )
+                }
+                (None, None) => {
+                    let token: SecretValue =
+                        self.config.token.clone().ok_or(Error::NoAuthConfigured)?;
+                    (
+                        self.build_client(self.config.secret_namespace(), token.expose())?,
+                        None,
+                    )
+                }
             };
-            return Ok(token);
-        };
-        let body: Value = match (self.config.approle.as_ref(), self.config.tls_cert.as_ref()) {
-            (Some(approle), _) => {
-                json!({"role_id": approle.role_id, "secret_id": approle.secret_id.expose()})
-            }
-            (None, Some(tls)) => tls
-                .role
-                .as_deref()
-                .map_or_else(|| json!({}), |role| json!({"name": role})),
-            (None, None) => {
-                let Some(token) = self.config.token.clone() else {
-                    return Err(Error::NoAuthConfigured);
-                };
-                return Ok(token);
-            }
-        };
-        let request: RequestBuilder = with_namespace(
-            self.client.post(login_url).json(&body),
-            self.config.login_namespace(),
-        );
-        let response: reqwest::Response = request.send().await?;
-        if !response.status().is_success() {
-            return Err(Error::LoginStatus {
-                status: response.status().as_u16(),
-            });
-        }
-        let body = response.bytes().await?;
-        let payload: Value = serde_json::from_slice(&body).map_err(|_| Error::MalformedLogin)?;
-        let auth: &serde_json::Map<String, Value> = payload
-            .get("auth")
-            .and_then(Value::as_object)
-            .ok_or(Error::MalformedLogin)?;
-        let token: SecretValue = SecretValue::new(
-            auth.get("client_token")
-                .and_then(Value::as_str)
-                .ok_or(Error::MalformedLogin)?,
-        );
-        let lease_duration: u64 = auth
-            .get("lease_duration")
-            .and_then(Value::as_u64)
-            .ok_or(Error::MalformedLogin)?;
-        let expires_at: Option<Instant> =
-            (lease_duration > 0).then(|| Instant::now() + Duration::from_secs(lease_duration));
-        *cached = Some(CachedToken {
-            token: token.clone(),
+        let client: Arc<VaultClient> = Arc::new(client);
+        *cached = Some(CachedClient {
+            client: client.clone(),
             expires_at,
         });
-        Ok(token)
+        Ok(client)
+    }
+
+    fn build_client(&self, namespace: Option<&str>, token: &str) -> Result<VaultClient, Error> {
+        let settings = VaultClientSettingsBuilder::default()
+            .address(&self.config.address)
+            .token(token.to_owned())
+            .namespace(namespace.map(str::to_owned))
+            .identity(identity_for(self.config.tls_cert.as_ref())?)
+            .ca_certs(Vec::new())
+            .verify(true)
+            .build()
+            .map_err(|message| Error::ClientSettings {
+                message: message.to_string(),
+            })?;
+        VaultClient::new(settings).map_err(Error::Client)
     }
 }
 
@@ -304,34 +276,84 @@ impl BaseSecretManager for HashicorpVault {
     }
 }
 
-fn client_for_config(config: &HashicorpVaultConfig) -> Result<Client, Error> {
-    let builder: reqwest::ClientBuilder = match config.tls_cert.as_ref() {
-        Some(tls) => Client::builder().identity(identity_for(tls)?),
-        None => Client::builder(),
-    };
-    builder.build().map_err(Error::Request)
+#[derive(Clone, Copy)]
+enum ErrorContext {
+    Login,
+    Read,
+    Secret,
 }
 
-fn with_namespace(request: RequestBuilder, namespace: Option<&str>) -> RequestBuilder {
-    match namespace {
-        Some(namespace) => request.header("X-Vault-Namespace", namespace),
-        None => request,
+fn cache_key(location: &SecretLocation) -> String {
+    format!(
+        "{:?}/{}/{}",
+        location.namespace, location.mount, location.path
+    )
+}
+
+fn identity_for(tls: Option<&TlsCertAuth>) -> Result<Option<Identity>, Error> {
+    tls.map(|tls| {
+        let cert: Vec<u8> = std::fs::read(&tls.cert_path).map_err(|source| Error::TlsIdentity {
+            path: tls.cert_path.clone(),
+            message: source.to_string(),
+        })?;
+        let key: Vec<u8> = std::fs::read(&tls.key_path).map_err(|source| Error::TlsIdentity {
+            path: tls.key_path.clone(),
+            message: source.to_string(),
+        })?;
+        Identity::from_pem(&[cert.as_slice(), key.as_slice()].concat()).map_err(|source| {
+            Error::TlsIdentity {
+                path: tls.cert_path.clone(),
+                message: source.to_string(),
+            }
+        })
+    })
+    .transpose()
+}
+
+fn map_api_error(error: ClientError, context: ErrorContext) -> Error {
+    match error {
+        ClientError::APIError { code, .. } => match context {
+            ErrorContext::Login => Error::LoginStatus { status: code },
+            ErrorContext::Read | ErrorContext::Secret => Error::Status { status: code },
+        },
+        ClientError::JsonParseError { source } => match context {
+            ErrorContext::Login => Error::MalformedLogin,
+            ErrorContext::Read => Error::MalformedPayload,
+            ErrorContext::Secret => Error::Client(ClientError::JsonParseError { source }),
+        },
+        ClientError::ResponseEmptyError | ClientError::ResponseDataEmptyError => {
+            malformed_response(context)
+        }
+        ClientError::RestClientError { source } => match source {
+            RustifyClientError::ServerResponseError { code, .. } => match context {
+                ErrorContext::Login => Error::LoginStatus { status: code },
+                ErrorContext::Read | ErrorContext::Secret => Error::Status { status: code },
+            },
+            RustifyClientError::ResponseParseError { .. } => malformed_response(context),
+            source => Error::Client(ClientError::RestClientError { source }),
+        },
+        error => Error::Client(error),
     }
 }
 
-fn identity_for(tls: &TlsCertAuth) -> Result<Identity, Error> {
-    let cert: Vec<u8> = std::fs::read(&tls.cert_path).map_err(|source| Error::TlsIdentity {
-        path: tls.cert_path.clone(),
-        message: source.to_string(),
-    })?;
-    let key: Vec<u8> = std::fs::read(&tls.key_path).map_err(|source| Error::TlsIdentity {
-        path: tls.key_path.clone(),
-        message: source.to_string(),
-    })?;
-    Identity::from_pem(&[cert.as_slice(), key.as_slice()].concat()).map_err(|source| {
-        Error::TlsIdentity {
-            path: tls.cert_path.clone(),
-            message: source.to_string(),
-        }
-    })
+fn api_status(error: &ClientError) -> Option<u16> {
+    match error {
+        ClientError::APIError { code, .. } => Some(*code),
+        ClientError::RestClientError {
+            source: RustifyClientError::ServerResponseError { code, .. },
+        } => Some(*code),
+        _ => None,
+    }
+}
+
+fn malformed_response(context: ErrorContext) -> Error {
+    match context {
+        ErrorContext::Login => Error::MalformedLogin,
+        ErrorContext::Read => Error::MalformedPayload,
+        ErrorContext::Secret => Error::Client(ClientError::ResponseDataEmptyError),
+    }
+}
+
+fn token_expiry(lease_duration: u64) -> Option<Instant> {
+    (lease_duration > 0).then(|| Instant::now() + Duration::from_secs(lease_duration))
 }
diff --git a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs
index b7a81f49ad4..c52db46e41e 100644
--- a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs
+++ b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs
@@ -22,7 +22,51 @@ fn config(server: &MockServer, values: &[(&str, &str)]) -> HashicorpVaultConfig
 }
 
 fn manager(server: &MockServer, values: &[(&str, &str)]) -> HashicorpVault {
-    HashicorpVault::with_client(reqwest::Client::new(), config(server, values), true).unwrap()
+    HashicorpVault::from_config(config(server, values), true).unwrap()
+}
+
+fn auth_response(token: &str, lease_duration: u64) -> serde_json::Value {
+    json!({
+        "auth": {
+            "client_token": token,
+            "accessor": "",
+            "policies": [],
+            "token_policies": [],
+            "metadata": null,
+            "lease_duration": lease_duration,
+            "renewable": false,
+            "entity_id": "",
+            "token_type": "service",
+            "orphan": false
+        },
+        "lease_id": "",
+        "lease_duration": lease_duration,
+        "renewable": false,
+        "request_id": "",
+        "warnings": null,
+        "wrap_info": null
+    })
+}
+
+fn read_response(data: serde_json::Value) -> serde_json::Value {
+    json!({
+        "data": {
+            "data": data,
+            "metadata": {
+                "created_time": "",
+                "deletion_time": "",
+                "custom_metadata": null,
+                "destroyed": false,
+                "version": 1
+            }
+        },
+        "lease_id": "",
+        "lease_duration": 0,
+        "renewable": false,
+        "request_id": "",
+        "warnings": null,
+        "wrap_info": null
+    })
 }
 
 #[tokio::test]
@@ -32,7 +76,7 @@ async fn token_reads_use_vault_headers_and_cache_values() {
         .and(path("/v1/secret/data/name"))
         .and(header("X-Vault-Token", "token"))
         .respond_with(
-            ResponseTemplate::new(200).set_body_json(json!({"data": {"data": {"key": "value"}}})),
+            ResponseTemplate::new(200).set_body_json(read_response(json!({"key": "value"}))),
         )
         .expect(1)
         .mount(&server)
@@ -48,6 +92,12 @@ async fn token_reads_use_vault_headers_and_cache_values() {
             .expose(),
         "value"
     );
+    let requests = server.received_requests().await.unwrap();
+    assert!(
+        requests
+            .iter()
+            .all(|request| !request.headers.contains_key("X-Vault-Namespace"))
+    );
     assert_eq!(
         manager
             .async_read_secret("name")
@@ -63,9 +113,10 @@ async fn token_reads_use_vault_headers_and_cache_values() {
 async fn namespace_mount_and_prefix_are_sanitized_in_the_url() {
     let server: MockServer = MockServer::start().await;
     Mock::given(method("GET"))
-        .and(path("/v1/team-a/kv-prod/data/virtual-keys/name"))
+        .and(path("/v1/kv-prod/data/virtual-keys/name"))
+        .and(header("X-Vault-Namespace", "team-a"))
         .respond_with(
-            ResponseTemplate::new(200).set_body_json(json!({"data": {"data": {"key": "value"}}})),
+            ResponseTemplate::new(200).set_body_json(read_response(json!({"key": "value"}))),
         )
         .expect(1)
         .mount(&server)
@@ -80,10 +131,10 @@ async fn namespace_mount_and_prefix_are_sanitized_in_the_url() {
         ],
     );
 
-    assert_eq!(
-        manager.secret_url("name").unwrap(),
-        format!("{}/v1/team-a/kv-prod/data/virtual-keys/name", server.uri())
-    );
+    let location = manager.secret_location("name").unwrap();
+    assert_eq!(location.namespace.as_deref(), Some("team-a"));
+    assert_eq!(location.mount, "kv-prod");
+    assert_eq!(location.path, "virtual-keys/name");
     assert!(manager.async_read_secret("name").await.unwrap().is_some());
 }
 
@@ -96,12 +147,15 @@ fn trailing_address_slashes_are_removed() {
     });
     let config: HashicorpVaultConfig =
         HashicorpVaultConfig::from_environment(environment.as_ref()).unwrap();
-    let manager: HashicorpVault =
-        HashicorpVault::with_client(reqwest::Client::new(), config, true).unwrap();
+    let manager: HashicorpVault = HashicorpVault::from_config(config, true).unwrap();
 
     assert_eq!(
-        manager.secret_url("name").unwrap(),
-        "http://vault.test:8200/v1/secret/data/name"
+        manager.secret_location("name").unwrap(),
+        litellm_secrets_hashicorp::SecretLocation {
+            namespace: None,
+            mount: "secret".to_owned(),
+            path: "name".to_owned(),
+        }
     );
 }
 
@@ -127,21 +181,26 @@ async fn approle_login_uses_namespace_and_reuses_the_token() {
         .and(path("/v1/auth/custom-approle/login"))
         .and(header("X-Vault-Namespace", "login-root"))
         .and(body_json(json!({"role_id": "role", "secret_id": "secret"})))
-        .respond_with(ResponseTemplate::new(200).set_body_json(
-            json!({"auth": {"client_token": "login-token", "lease_duration": 3600}}),
-        ))
+        .respond_with(ResponseTemplate::new(200).set_body_json(auth_response("login-token", 3600)))
         .expect(1)
         .mount(&server)
         .await;
     Mock::given(method("GET"))
-        .and(path("/v1/secret-root/secret/data/name"))
+        .and(path("/v1/secret/data/name"))
         .and(header("X-Vault-Token", "login-token"))
+        .and(header("X-Vault-Namespace", "secret-root"))
         .respond_with(
-            ResponseTemplate::new(200).set_body_json(json!({"data": {"data": {"key": "value"}}})),
+            ResponseTemplate::new(200).set_body_json(read_response(json!({"key": "value"}))),
         )
         .expect(1)
         .mount(&server)
         .await;
+    Mock::given(method("GET"))
+        .and(path("/v1/secret/data/name-2"))
+        .respond_with(ResponseTemplate::new(404).set_body_json(json!({"errors": ["missing"]})))
+        .expect(1)
+        .mount(&server)
+        .await;
     let manager: HashicorpVault = manager(
         &server,
         &[
@@ -162,17 +221,13 @@ async fn approle_tokens_expire_after_the_vault_lease() {
     let server: MockServer = MockServer::start().await;
     Mock::given(method("POST"))
         .and(path("/v1/auth/approle/login"))
-        .respond_with(
-            ResponseTemplate::new(200).set_body_json(
-                json!({"auth": {"client_token": "login-token", "lease_duration": 1}}),
-            ),
-        )
+        .respond_with(ResponseTemplate::new(200).set_body_json(auth_response("login-token", 1)))
         .expect(2)
         .mount(&server)
         .await;
     Mock::given(method("GET"))
         .respond_with(
-            ResponseTemplate::new(200).set_body_json(json!({"data": {"data": {"key": "value"}}})),
+            ResponseTemplate::new(200).set_body_json(read_response(json!({"key": "value"}))),
         )
         .expect(2)
         .mount(&server)
@@ -201,54 +256,98 @@ async fn tls_login_posts_the_role_and_uses_the_client_identity() {
     std::fs::write(&key_path, TEST_PRIVATE_KEY).unwrap();
     Mock::given(method("POST"))
         .and(path("/v1/auth/cert/login"))
-        .and(body_json(json!({"name": "vault-role"})))
-        .respond_with(
-            ResponseTemplate::new(200).set_body_json(
-                json!({"auth": {"client_token": "cert-token", "lease_duration": 0}}),
-            ),
-        )
-        .expect(1)
+        .and(header("X-Vault-Namespace", "login-ns"))
+        .respond_with(ResponseTemplate::new(200).set_body_json(auth_response("cert-token", 0)))
+        .expect(2)
         .mount(&server)
         .await;
     Mock::given(method("GET"))
         .and(path("/v1/secret/data/name"))
+        .and(header("X-Vault-Token", "cert-token"))
+        .and(header("X-Vault-Namespace", "secret-ns"))
         .respond_with(
-            ResponseTemplate::new(200).set_body_json(json!({"data": {"data": {"key": "value"}}})),
+            ResponseTemplate::new(200).set_body_json(read_response(json!({"key": "value"}))),
         )
+        .expect(2)
         .mount(&server)
         .await;
-    let manager: HashicorpVault = HashicorpVault::new(
-        {
-            let environment_values: HashMap<String, String> = HashMap::from([
-                ("HCP_VAULT_ADDR".to_owned(), server.uri()),
-                (
-                    "HCP_VAULT_CLIENT_CERT".to_owned(),
-                    cert_path.to_str().unwrap().to_owned(),
-                ),
-                (
-                    "HCP_VAULT_CLIENT_KEY".to_owned(),
-                    key_path.to_str().unwrap().to_owned(),
-                ),
-                ("HCP_VAULT_CERT_ROLE".to_owned(), "vault-role".to_owned()),
-            ]);
-            Arc::new(move |name: &str| environment_values.get(name).cloned())
-        },
-        true,
-    )
-    .unwrap();
-
-    assert!(manager.async_read_secret("name").await.unwrap().is_some());
-    assert_eq!(
-        manager.login_url().as_deref(),
-        Some(format!("{}/v1/auth/cert/login", server.uri()).as_str())
+    let role_values: HashMap<String, String> = HashMap::from([
+        ("HCP_VAULT_ADDR".to_owned(), server.uri()),
+        (
+            "HCP_VAULT_CLIENT_CERT".to_owned(),
+            cert_path.to_str().unwrap().to_owned(),
+        ),
+        (
+            "HCP_VAULT_CLIENT_KEY".to_owned(),
+            key_path.to_str().unwrap().to_owned(),
+        ),
+        ("HCP_VAULT_CERT_ROLE".to_owned(), "vault-role".to_owned()),
+        (
+            "HCP_VAULT_LOGIN_NAMESPACE".to_owned(),
+            "login-ns".to_owned(),
+        ),
+        (
+            "HCP_VAULT_SECRET_NAMESPACE".to_owned(),
+            "secret-ns".to_owned(),
+        ),
+    ]);
+    let role_environment: Arc<dyn Lookup + Send + Sync> =
+        Arc::new(move |name: &str| role_values.get(name).cloned());
+    let role_manager: HashicorpVault = HashicorpVault::new(role_environment, true).unwrap();
+    assert!(
+        role_manager
+            .async_read_secret("name")
+            .await
+            .unwrap()
+            .is_some()
     );
+
+    let no_role_values: HashMap<String, String> = HashMap::from([
+        ("HCP_VAULT_ADDR".to_owned(), server.uri()),
+        (
+            "HCP_VAULT_CLIENT_CERT".to_owned(),
+            cert_path.to_str().unwrap().to_owned(),
+        ),
+        (
+            "HCP_VAULT_CLIENT_KEY".to_owned(),
+            key_path.to_str().unwrap().to_owned(),
+        ),
+        (
+            "HCP_VAULT_LOGIN_NAMESPACE".to_owned(),
+            "login-ns".to_owned(),
+        ),
+        (
+            "HCP_VAULT_SECRET_NAMESPACE".to_owned(),
+            "secret-ns".to_owned(),
+        ),
+    ]);
+    let no_role_environment: Arc<dyn Lookup + Send + Sync> =
+        Arc::new(move |name: &str| no_role_values.get(name).cloned());
+    let no_role_manager: HashicorpVault = HashicorpVault::new(no_role_environment, true).unwrap();
+    assert!(
+        no_role_manager
+            .async_read_secret("name")
+            .await
+            .unwrap()
+            .is_some()
+    );
+    let login_bodies: Vec<serde_json::Value> = server
+        .received_requests()
+        .await
+        .unwrap()
+        .iter()
+        .filter(|request| request.method.as_str() == "POST")
+        .map(|request| serde_json::from_slice(&request.body).unwrap())
+        .collect();
+    assert!(login_bodies.contains(&json!({"name": "vault-role"})));
+    assert!(login_bodies.contains(&json!({})));
 }
 
 #[rstest::rstest]
-#[case::missing(404, json!({}), 0)]
-#[case::malformed(200, json!({}), 1)]
-#[case::missing_key(200, json!({"data": {"data": {}}}), 0)]
-#[case::non_string(200, json!({"data": {"data": {"key": 1}}}), 2)]
+#[case::missing(404, json!({"errors": ["missing"]}), 0)]
+#[case::malformed(200, json!({"data": "invalid"}), 1)]
+#[case::missing_key(200, json!({}), 0)]
+#[case::non_string(200, json!({"key": 1}), 2)]
 #[tokio::test]
 async fn read_responses_distinguish_absence_and_malformed_payloads(
     #[case] status: u16,
@@ -257,7 +356,13 @@ async fn read_responses_distinguish_absence_and_malformed_payloads(
 ) {
     let server: MockServer = MockServer::start().await;
     Mock::given(method("GET"))
-        .respond_with(ResponseTemplate::new(status).set_body_json(body))
+        .respond_with(ResponseTemplate::new(status).set_body_json(
+            if status == 200 && expected != 1 {
+                read_response(body)
+            } else {
+                body
+            },
+        ))
         .expect(1)
         .mount(&server)
         .await;
@@ -279,7 +384,7 @@ async fn write_and_delete_invalidate_the_read_cache() {
     Mock::given(method("GET"))
         .and(path("/v1/secret/data/name"))
         .respond_with(
-            ResponseTemplate::new(200).set_body_json(json!({"data": {"data": {"key": "value"}}})),
+            ResponseTemplate::new(200).set_body_json(read_response(json!({"key": "value"}))),
         )
         .expect(2)
         .mount(&server)
@@ -289,7 +394,21 @@ async fn write_and_delete_invalidate_the_read_cache() {
         .and(body_json(
             json!({"data": {"key": "updated", "description": "description"}}),
         ))
-        .respond_with(ResponseTemplate::new(200).set_body_json(json!({"data": {"version": 2}})))
+        .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+            "data": {
+                "created_time": "",
+                "deletion_time": "",
+                "custom_metadata": null,
+                "destroyed": false,
+                "version": 2
+            },
+            "lease_id": "",
+            "lease_duration": 0,
+            "renewable": false,
+            "request_id": "",
+            "warnings": null,
+            "wrap_info": null
+        })))
         .expect(1)
         .mount(&server)
         .await;
@@ -331,12 +450,9 @@ async fn no_auth_and_invalid_names_fail_without_requests() {
 #[tokio::test]
 async fn debug_output_redacts_authentication_values() {
     let server: MockServer = MockServer::start().await;
-    let manager: HashicorpVault = HashicorpVault::with_client(
-        reqwest::Client::new(),
-        config(&server, &[("HCP_VAULT_TOKEN", "token-value")]),
-        true,
-    )
-    .unwrap();
+    let manager: HashicorpVault =
+        HashicorpVault::from_config(config(&server, &[("HCP_VAULT_TOKEN", "token-value")]), true)
+            .unwrap();
     let debug: String = format!("{manager:?}");
     assert!(!debug.contains("token-value"));
     assert!(!debug.contains("secret-id"));
@@ -365,13 +481,35 @@ fn configuration_matches_python_parity_fixture() {
             Arc::new(move |name: &str| values.get(name).cloned());
         let config: HashicorpVaultConfig =
             HashicorpVaultConfig::from_environment(environment.as_ref()).unwrap();
-        let manager: HashicorpVault =
-            HashicorpVault::with_client(reqwest::Client::new(), config, true).unwrap();
+        let manager: HashicorpVault = HashicorpVault::from_config(config.clone(), true).unwrap();
+        let location = manager.secret_location(&case.secret_name).unwrap();
+        let namespace = location
+            .namespace
+            .as_deref()
+            .map(|namespace| format!("{namespace}/"))
+            .unwrap_or_default();
         assert_eq!(
-            manager.secret_url(&case.secret_name).unwrap(),
+            format!(
+                "{}/v1/{}{}/data/{}",
+                config.address, namespace, location.mount, location.path
+            ),
             case.expected_secret_url
         );
-        assert_eq!(manager.login_url(), case.expected_login_url);
+        let login_url = config.approle.as_ref().map_or_else(
+            || {
+                config
+                    .tls_cert
+                    .as_ref()
+                    .map(|_| format!("{}/v1/auth/cert/login", config.address))
+            },
+            |approle| {
+                Some(format!(
+                    "{}/v1/auth/{}/login",
+                    config.address, approle.mount_path
+                ))
+            },
+        );
+        assert_eq!(login_url, case.expected_login_url);
         assert_eq!(
             manager.config().login_namespace(),
             case.expected_login_namespace.as_deref()
@@ -391,8 +529,15 @@ async fn live_vault_round_trip() {
     let manager: HashicorpVault = HashicorpVault::new(environment, true).unwrap();
     let name: String = std::env::var("LITELLM_VAULT_LIVE_SECRET_NAME").unwrap();
     let value: SecretValue = SecretValue::new("native-live-value");
-    let url: String = manager.secret_url(&name).unwrap();
-    println!("native provenance: {} {}", module_path!(), url);
+    let location = manager.secret_location(&name).unwrap();
+    println!(
+        "native provenance: {} vaultrs {} {:?} {} {}",
+        module_path!(),
+        manager.config().address,
+        location.namespace,
+        location.mount,
+        location.path
+    );
     manager
         .async_write_secret(&name, value.clone(), None)
         .await
diff --git a/litellm-rust/crates/secrets/tests/handler.rs b/litellm-rust/crates/secrets/tests/handler.rs
index 58941171fd9..c169d411f8f 100644
--- a/litellm-rust/crates/secrets/tests/handler.rs
+++ b/litellm-rust/crates/secrets/tests/handler.rs
@@ -124,10 +124,24 @@ async fn hashicorp_handler_resolves_found_missing_and_failed_values() {
     let found_server = MockServer::start().await;
     Mock::given(method("GET"))
         .and(path("/v1/secret/data/KEY"))
-        .respond_with(
-            ResponseTemplate::new(200)
-                .set_body_json(serde_json::json!({"data": {"data": {"key": "remote"}}})),
-        )
+        .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
+            "data": {
+                "data": {"key": "remote"},
+                "metadata": {
+                    "created_time": "",
+                    "deletion_time": "",
+                    "custom_metadata": null,
+                    "destroyed": false,
+                    "version": 1
+                }
+            },
+            "lease_id": "",
+            "lease_duration": 0,
+            "renewable": false,
+            "request_id": "",
+            "warnings": null,
+            "wrap_info": null
+        })))
         .mount(&found_server)
         .await;
     let found_environment: Arc<dyn Lookup + Send + Sync> = Arc::new({
@@ -139,8 +153,7 @@ async fn hashicorp_handler_resolves_found_missing_and_failed_values() {
         }
     });
     let found_config = HashicorpVaultConfig::from_environment(found_environment.as_ref()).unwrap();
-    let found_manager =
-        HashicorpVault::with_client(reqwest::Client::new(), found_config, true).unwrap();
+    let found_manager = HashicorpVault::from_config(found_config, true).unwrap();
     let found_resolver = SecretResolver::new(
         Arc::new(SecretManagerState::new(
             SecretManager::HashicorpVault(found_manager),
@@ -164,7 +177,9 @@ async fn hashicorp_handler_resolves_found_missing_and_failed_values() {
 
     let missing_server = MockServer::start().await;
     Mock::given(method("GET"))
-        .respond_with(ResponseTemplate::new(404))
+        .respond_with(
+            ResponseTemplate::new(404).set_body_json(serde_json::json!({"errors": ["missing"]})),
+        )
         .mount(&missing_server)
         .await;
     let missing_environment: Arc<dyn Lookup + Send + Sync> = Arc::new({
@@ -177,8 +192,7 @@ async fn hashicorp_handler_resolves_found_missing_and_failed_values() {
     });
     let missing_config =
         HashicorpVaultConfig::from_environment(missing_environment.as_ref()).unwrap();
-    let missing_manager =
-        HashicorpVault::with_client(reqwest::Client::new(), missing_config, true).unwrap();
+    let missing_manager = HashicorpVault::from_config(missing_config, true).unwrap();
     let missing_state = SecretManagerState::new(
         SecretManager::HashicorpVault(missing_manager),
         KeyManagementSettings {
@@ -198,7 +212,9 @@ async fn hashicorp_handler_resolves_found_missing_and_failed_values() {
 
     let failed_server = MockServer::start().await;
     Mock::given(method("GET"))
-        .respond_with(ResponseTemplate::new(500))
+        .respond_with(
+            ResponseTemplate::new(500).set_body_json(serde_json::json!({"errors": ["failed"]})),
+        )
         .mount(&failed_server)
         .await;
     let failed_environment: Arc<dyn Lookup + Send + Sync> = Arc::new({
@@ -211,8 +227,7 @@ async fn hashicorp_handler_resolves_found_missing_and_failed_values() {
     });
     let failed_config =
         HashicorpVaultConfig::from_environment(failed_environment.as_ref()).unwrap();
-    let failed_manager =
-        HashicorpVault::with_client(reqwest::Client::new(), failed_config, true).unwrap();
+    let failed_manager = HashicorpVault::from_config(failed_config, true).unwrap();
     let failed_state = SecretManagerState::new(
         SecretManager::HashicorpVault(failed_manager),
         KeyManagementSettings {

From d90ff4a30b67b41199e171ee997d93cdea4b3a57 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:35:39 +0000
Subject: [PATCH 114/160] refactor(rust): trim vault cert login endpoint

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../crates/secrets-hashicorp/src/cert_login.rs        | 11 ++++-------
 .../crates/secrets-hashicorp/src/secret_manager.rs    |  2 +-
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/litellm-rust/crates/secrets-hashicorp/src/cert_login.rs b/litellm-rust/crates/secrets-hashicorp/src/cert_login.rs
index 1eb09a89715..f99df62db12 100644
--- a/litellm-rust/crates/secrets-hashicorp/src/cert_login.rs
+++ b/litellm-rust/crates/secrets-hashicorp/src/cert_login.rs
@@ -3,22 +3,19 @@
 pub struct CertLoginRequest {
     #[endpoint(skip)]
     pub mount: String,
-    #[endpoint(skip)]
-    #[allow(dead_code)]
-    pub name: Option<String>,
     #[endpoint(raw)]
     body: Vec<u8>,
 }
 
 impl CertLoginRequest {
-    pub fn new(name: Option<String>) -> Self {
-        let body: Vec<u8> = match name.as_deref() {
-            Some(name) => serde_json::to_vec(&serde_json::json!({ "name": name })).unwrap(),
+    pub fn new(name: Option<&str>) -> Self {
+        let body: Vec<u8> = match name {
+            Some(name) => serde_json::to_vec(&serde_json::json!({ "name": name }))
+                .expect("json object serialization is infallible"),
             None => b"{}".to_vec(),
         };
         Self {
             mount: "cert".to_owned(),
-            name,
             body,
         }
     }
diff --git a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs
index 7937fa93f8f..3ad9a549438 100644
--- a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs
+++ b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs
@@ -207,7 +207,7 @@ impl HashicorpVault {
                 (None, Some(tls)) => {
                     let login_client: VaultClient =
                         self.build_client(self.config.login_namespace(), "")?;
-                    let endpoint: CertLoginRequest = CertLoginRequest::new(tls.role.clone());
+                    let endpoint: CertLoginRequest = CertLoginRequest::new(tls.role.as_deref());
                     let auth = api::auth(&login_client, endpoint)
                         .await
                         .map_err(|error| map_api_error(error, ErrorContext::Login))?;

From f43f9012f1a0d58402aaec5393937225c1636869 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:44:24 +0000
Subject: [PATCH 115/160] fix(rust): handle disk cache topology

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/python-bridge/src/cache/native.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs
index e764fde8b39..50e02338574 100644
--- a/litellm-rust/crates/python-bridge/src/cache/native.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/native.rs
@@ -83,6 +83,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(_) => None,
             Self::Redis { cache, .. } => Some(cache.backend().topology()),
+            Self::Disk(_) => None,
         }
     }
 

From 4845beddc067cd852c79a45e676c7128cd3bc23d Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:44:26 +0000
Subject: [PATCH 116/160] ci(rust): raise native wheel size gate to 40 MB

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .github/scripts/verify_linux_native_wheel.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py
index 0adbc015ad0..7723c022351 100644
--- a/.github/scripts/verify_linux_native_wheel.py
+++ b/.github/scripts/verify_linux_native_wheel.py
@@ -205,7 +205,7 @@ def main(
     native_module: Final = load_native_module(native_path)
     native_module_loads: Final = native_module is not None
     panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
-    native_size_limit: Final = 30_000_000
+    native_size_limit: Final = 40_000_000
     native_size_within_limit: Final = native_member.file_size <= native_size_limit
     validations: Final = (
         (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
@@ -222,7 +222,7 @@ def main(
         ("Python extension entry point is present", extension_entry_point_present),
         ("Native module loads", native_module_loads),
         ("Production module omits the panic test hook", panic_test_hook_absent),
-        ("Native extension does not exceed 30 MB", native_size_within_limit),
+        ("Native extension does not exceed 40 MB", native_size_within_limit),
         ("Wheel contents are valid", not unexpected_members),
     )
 
@@ -267,7 +267,7 @@ def main(
             ),
             (
                 not native_size_within_limit,
-                f"native extension exceeds 30 MB: {native_member.file_size / 1_000_000:.2f} MB",
+                f"native extension exceeds 40 MB: {native_member.file_size / 1_000_000:.2f} MB",
             ),
             (bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"),
         )

From cf7234c6f43ea220be426503bc1a1e69becbb62b Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:48:40 +0000
Subject: [PATCH 117/160] chore(rust): refresh workspace lockfile

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.lock | 1 +
 1 file changed, 1 insertion(+)

diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index 7bd166f48b6..097884c8ecc 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -2707,6 +2707,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_with",
+ "sha2 0.10.9",
  "tokio",
  "tokio-tungstenite",
 ]

From 220b981ab4c390fdfce6d8baaad557a2bebb7812 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 21:53:18 +0000
Subject: [PATCH 118/160] test(rust): deduplicate the merged os import

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 tests/test_litellm_rust/test_cache.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_litellm_rust/test_cache.py b/tests/test_litellm_rust/test_cache.py
index 87e9d76ccb4..a60feb9973a 100644
--- a/tests/test_litellm_rust/test_cache.py
+++ b/tests/test_litellm_rust/test_cache.py
@@ -3,7 +3,6 @@ import contextvars
 import gc
 import hashlib
 import json
-import os
 import math
 import os
 import threading

From 58ffa8e4cea27e34dd38e56e58bfa5c1180181b2 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 22:00:09 +0000
Subject: [PATCH 119/160] refactor(rust): drop variable rebinding in s3 cache
 and bridge config

Also adds the startup_nodes parameter to the _CacheTestHandle.redis stub to match the merged runtime signature

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/cache-s3/src/cache.rs          | 11 ++++++-----
 .../crates/python-bridge/src/cache/config.rs       | 14 ++++++--------
 litellm/rust_bridge/_native.pyi                    |  1 +
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/litellm-rust/crates/cache-s3/src/cache.rs b/litellm-rust/crates/cache-s3/src/cache.rs
index 88575f36fce..78e5f39dbcc 100644
--- a/litellm-rust/crates/cache-s3/src/cache.rs
+++ b/litellm-rust/crates/cache-s3/src/cache.rs
@@ -42,16 +42,17 @@ pub struct S3Cache<C: CacheCodec> {
 
 impl<C: CacheCodec> S3Cache<C> {
     pub fn new(config: S3CacheConfig, codec: C, runtime: Handle) -> Self {
-        let mut builder = aws_sdk_s3::Config::builder()
+        let endpoint_url: Option<String> = config.endpoint.map(|endpoint| endpoint.url);
+        let base = aws_sdk_s3::Config::builder()
             .behavior_version(BehaviorVersion::latest())
             .region(Region::new(config.region.clone()))
             .credentials_provider(Credentials::new(config.auth))
             .request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
             .response_checksum_validation(ResponseChecksumValidation::WhenRequired);
-        let endpoint_url: Option<String> = config.endpoint.map(|endpoint| endpoint.url);
-        if let Some(url) = &endpoint_url {
-            builder = builder.endpoint_url(url).force_path_style(true);
-        }
+        let builder = match &endpoint_url {
+            Some(url) => base.endpoint_url(url).force_path_style(true),
+            None => base,
+        };
         Self {
             client: aws_sdk_s3::Client::from_conf(builder.build()),
             codec,
diff --git a/litellm-rust/crates/python-bridge/src/cache/config.rs b/litellm-rust/crates/python-bridge/src/cache/config.rs
index 70d6f7c1841..269ddedfc0e 100644
--- a/litellm-rust/crates/python-bridge/src/cache/config.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/config.rs
@@ -642,14 +642,12 @@ fn optional_attribute_chain<'py>(
     value: &Bound<'py, PyAny>,
     names: &[&str],
 ) -> PyResult<Option<Bound<'py, PyAny>>> {
-    let mut current = value.clone();
-    for name in names {
-        match optional_attribute(&current, name)? {
-            Some(next) => current = next,
-            None => return Ok(None),
-        }
-    }
-    Ok(Some(current))
+    names
+        .iter()
+        .try_fold(Some(value.clone()), |current, name| match current {
+            Some(current) => optional_attribute(&current, name),
+            None => Ok(None),
+        })
 }
 
 #[inline(never)]
diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi
index f1b010fb26e..6ad0e6be26e 100644
--- a/litellm/rust_bridge/_native.pyi
+++ b/litellm/rust_bridge/_native.pyi
@@ -162,6 +162,7 @@ class _CacheTestHandle:
         *,
         ttl_seconds: float = 60.0,
         namespace: str | None = None,
+        startup_nodes: Sequence[tuple[str, int]] | None = None,
     ) -> _CacheTestHandle: ...
     @staticmethod
     def s3(

From 1cc38f05f43c1e2108fc3a35f1e81b8498e35805 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Mon, 21 Sep 2026 15:00:24 -0700
Subject: [PATCH 120/160] test(proxy): type the router settings source test
 parameters

---
 .../management_endpoints/test_router_settings_endpoints.py    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py
index 889bed13099..3fcda310435 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py
@@ -89,7 +89,9 @@ class TestRouterSettingsEndpoints:
         assert len(routing_strategy_field["options"]) > 0
 
     @pytest.mark.asyncio
-    async def test_get_router_settings_reports_sources(self, monkeypatch):
+    async def test_get_router_settings_reports_sources(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
         store = SettingsStore("router_settings")
         store.load_yaml({"routing_strategy": "simple-shuffle"})
         store.apply_db_row("router_settings", {"num_retries": 3})

From 5c6854510400ca53a8718401adf0ca911f65e62a Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Mon, 21 Sep 2026 15:00:54 -0700
Subject: [PATCH 121/160] test: fix stale budget-status and bad-database-url
 assertions
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two CI checks were asserting behaviour the proxy no longer has. Neither
was catching anything; both now fail for the right reason.

budget_exceeded (tests/otel_tests/test_e2e_budgeting.py)

bf804f5188 made 422 the default for budget refusals and added
budget_exceeded_status_code to restore 429 for callers that need it. The
e2e budget tests still asserted 429, so all six have been failing on a
status change that was deliberate. Assert 422, the documented default,
rather than reading litellm.budget_exceeded_status_code back — a test
that asks the code what it does would have passed straight through this
change and through the next one.

The helpers also caught bare Exception, so a connection error reached
`e.body` and surfaced as an AttributeError instead of a failed
assertion. Narrow both to openai.APIStatusError, which is what a refusal
actually raises (UnprocessableEntityError for 422, RateLimitError for
429), and let anything else propagate as itself.

test_bad_database_url (.circleci/config.yml)

The check required "Database setup failed after multiple retries", which
only the v1 resolver emits, OR uvicorn's "Application startup failed.
Exiting.". With v2 the default, the first branch is dead and the whole
assertion rests on an incidental uvicorn line that HEAD's run did not
emit at all.

Assert the behaviour instead of the wording: the container exits
non-zero, the log names the unreachable server (P1001), it never reaches
"Application startup complete", and it is not left running. The exit
code was previously discarded by `|| true`, so the one thing the job
most needed to check was never checked.

Verified by running the bad-DATABASE_URL container: exit 3, P1001
present, no startup-complete line, container stopped — the new check
passes and the old one passed only by the uvicorn line's accident.
---
 .circleci/config.yml                   | 23 ++++++++++++-----------
 tests/otel_tests/test_e2e_budgeting.py | 13 +++++++------
 2 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index cc9aa7fe1c4..c69ed6a3510 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -3033,28 +3033,29 @@ jobs:
       - run:
           name: Run Docker container with bad DATABASE_URL
           command: |
+            set +e
             docker run --name my-app \
               -p 4000:4000 \
               -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
               -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
               -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \
               myapp:latest \
-              --port 4000 > docker_output.log 2>&1 || true
+              --port 4000 > docker_output.log 2>&1
+            echo "$?" > docker_exit_code
+            set -e
       - run:
           name: Display Docker logs
           command: cat docker_output.log
       - run:
-          name: Check for expected error
+          name: Proxy must refuse to serve on an unreachable database
           command: |
-            if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \
-               (grep -q "Database setup failed after multiple retries" docker_output.log || \
-                grep -q "ERROR:    Application startup failed. Exiting." docker_output.log); then
-              echo "Expected error found. Test passed."
-            else
-              echo "Expected error not found. Test failed."
-              cat docker_output.log
-              exit 1
-            fi
+            fail() { echo "FAILED: $1"; cat docker_output.log; exit 1; }
+            exit_code="$(cat docker_exit_code)"
+            [ "$exit_code" -ne 0 ] || fail "proxy exited 0 with an unreachable database"
+            grep -q "P1001" docker_output.log || fail "log does not name the unreachable database server"
+            ! grep -q "Application startup complete" docker_output.log || fail "proxy reached serving state"
+            ! docker exec my-app true 2>/dev/null || fail "container is still running"
+            echo "Proxy refused to serve (exit $exit_code) and never reached startup. Test passed."
 
   provider_replay_harness:
     docker:
diff --git a/tests/otel_tests/test_e2e_budgeting.py b/tests/otel_tests/test_e2e_budgeting.py
index ca5058818e4..ae8f0ddc3ec 100644
--- a/tests/otel_tests/test_e2e_budgeting.py
+++ b/tests/otel_tests/test_e2e_budgeting.py
@@ -5,6 +5,7 @@ import uuid
 from typing import Any, Optional
 
 import aiohttp
+import openai
 import pytest
 from httpx import AsyncClient
 
@@ -23,7 +24,7 @@ async def make_calls_until_budget_exceeded(session, key: str, call_function, **k
             call_count += 1
             await asyncio.sleep(0.1)  # allow spend tracking to catch up
         pytest.fail(f"Budget was not exceeded after {MAX_CALLS} calls")
-    except Exception as e:
+    except openai.APIStatusError as e:
         print("vars: ", vars(e))
         print("e.body: ", e.body)
 
@@ -32,8 +33,8 @@ async def make_calls_until_budget_exceeded(session, key: str, call_function, **k
 
         # Check error structure and values that should be consistent
         assert (
-            error_dict["code"] == "429"
-        ), f"Expected error code 429, got: {error_dict['code']}"
+            error_dict["code"] == "422"
+        ), f"Expected error code 422, got: {error_dict['code']}"
         assert (
             error_dict["type"] == "budget_exceeded"
         ), f"Expected error type budget_exceeded, got: {error_dict['type']}"
@@ -506,9 +507,9 @@ async def make_calls_until_team_budget_exceeded_cli_sso(
             call_count += 1
             await asyncio.sleep(0.1)
         pytest.fail(f"Budget was not exceeded after {MAX_CALLS} calls")
-    except Exception as e:
+    except openai.APIStatusError as e:
         error_dict = e.body
-        assert error_dict["code"] == "429"
+        assert error_dict["code"] == "422"
         assert error_dict["type"] == "budget_exceeded"
         message = error_dict["message"]
         assert "Budget has been exceeded!" in message
@@ -556,7 +557,7 @@ async def test_team_budget_enforcement_cli_sso_token():
     1. Create team with a tiny max_budget and a user on that team
     2. Obtain a CLI SSO JWT (HTTP poll flow when Redis is shared, else mint)
     3. Make chat completion calls until the team budget is exceeded
-    4. Verify HTTP 429 budget_exceeded names the team
+    4. Verify HTTP 422 budget_exceeded names the team
     """
     user_id = f"cli-budget-user-{uuid.uuid4().hex[:8]}"
     user_email = f"{user_id}@example.com"

From 847f732f5d6e4c55d1b925bd50f480af4f858440 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 22:01:13 +0000
Subject: [PATCH 122/160] fix(cache): await valkey semantic embeddings inline
 on the caller loop

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../crates/cache-response/src/response.rs     |   4 +
 .../crates/cache-valkey-semantic/src/lib.rs   |  78 ++++++++-
 .../crates/python-bridge/src/cache/binding.rs |  14 +-
 .../python-bridge/src/cache/embedder.rs       |  45 ++---
 .../crates/python-bridge/src/cache/mod.rs     |   1 +
 .../crates/python-bridge/src/cache/native.rs  | 103 ++++++++++--
 .../python-bridge/src/cache/semantic_step.rs  | 154 ++++++++++++++++++
 .../test_valkey_semantic_cache_native.py      |  41 +++++
 8 files changed, 393 insertions(+), 47 deletions(-)
 create mode 100644 litellm-rust/crates/python-bridge/src/cache/semantic_step.rs

diff --git a/litellm-rust/crates/cache-response/src/response.rs b/litellm-rust/crates/cache-response/src/response.rs
index e70e07a5d26..2f949d511de 100644
--- a/litellm-rust/crates/cache-response/src/response.rs
+++ b/litellm-rust/crates/cache-response/src/response.rs
@@ -52,6 +52,10 @@ where
         &self.backend
     }
 
+    pub fn backend_arc(&self) -> &Arc<B> {
+        &self.backend
+    }
+
     pub fn default_ttl(&self) -> Option<Duration> {
         self.backend.get_ttl(&B::Context::default())
     }
diff --git a/litellm-rust/crates/cache-valkey-semantic/src/lib.rs b/litellm-rust/crates/cache-valkey-semantic/src/lib.rs
index 2bd4bd71bce..e3a3e6094c4 100644
--- a/litellm-rust/crates/cache-valkey-semantic/src/lib.rs
+++ b/litellm-rust/crates/cache-valkey-semantic/src/lib.rs
@@ -24,6 +24,22 @@ pub trait Embedder: Send + Sync + 'static {
     ) -> impl Future<Output = Result<Vec<f32>, Error>> + Send;
 }
 
+pub struct PreparedEmbedding(pub Vec<f32>);
+
+impl Embedder for PreparedEmbedding {
+    fn embed(&self, _prompt: &str, _metadata: Option<&Value>) -> Result<Vec<f32>, Error> {
+        Ok(self.0.clone())
+    }
+
+    async fn async_embed(
+        &self,
+        _prompt: &str,
+        _metadata: Option<&Value>,
+    ) -> Result<Vec<f32>, Error> {
+        Ok(self.0.clone())
+    }
+}
+
 #[derive(Clone, Debug, PartialEq)]
 pub struct ValkeySemanticConfig {
     pub similarity_threshold: f64,
@@ -112,6 +128,23 @@ where
     }
 }
 
+impl<E, S, C> ValkeySemanticCache<E, S, C>
+where
+    E: Embedder,
+    S: CacheCodec<Value = CacheEntry> + Clone,
+    C: redis::ConnectionLike + Send + 'static,
+{
+    pub fn with_embedder<E2: Embedder>(&self, embedder: E2) -> ValkeySemanticCache<E2, S, C> {
+        ValkeySemanticCache {
+            connections: Arc::clone(&self.connections),
+            embedder,
+            codec: self.codec.clone(),
+            config: self.config.clone(),
+            index_dimension: Arc::clone(&self.index_dimension),
+        }
+    }
+}
+
 impl<E, S, C> BaseCache for ValkeySemanticCache<E, S, C>
 where
     E: Embedder,
@@ -597,8 +630,8 @@ mod tests {
     use serde_json::{Value, json};
 
     use super::{
-        Embedder, ValkeySemanticCache, ValkeySemanticConfig, index_dimension_from_info,
-        prompt_from_context, scope_tag,
+        Embedder, PreparedEmbedding, ValkeySemanticCache, ValkeySemanticConfig,
+        index_dimension_from_info, prompt_from_context, scope_tag,
     };
 
     #[derive(Clone)]
@@ -758,6 +791,47 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn prepared_embedding_returns_its_vector_for_any_prompt() {
+        let embedding = PreparedEmbedding(vec![1.0, 2.0]);
+        assert_eq!(
+            embedding
+                .async_embed("different prompt", None)
+                .await
+                .unwrap(),
+            vec![1.0, 2.0]
+        );
+    }
+
+    #[test]
+    fn with_embedder_shares_index_state_and_connections() {
+        let entry = CacheEntry {
+            timestamp: Some(1.0),
+            response: json!({"answer": "ok"}),
+        };
+        let encoded = ResponseCacheCodec.encode(&entry).unwrap();
+        let cache = ValkeySemanticCache::with_connection(
+            RecordingConnection::new([ok(), ok(), Ok(search_hit(encoded, "0.1"))]),
+            FixedEmbedder {
+                vector: vec![1.0, 0.0],
+                calls: Arc::default(),
+            },
+            ResponseCacheCodec,
+            ValkeySemanticConfig {
+                similarity_threshold: 0.8,
+                index_name: "test".into(),
+            },
+        );
+        cache
+            .set_cache("key", entry.clone(), &semantic_context(None))
+            .unwrap();
+        let prepared = cache.with_embedder(PreparedEmbedding(vec![1.0, 0.0]));
+        assert_eq!(
+            prepared.get_cache("key", &semantic_context(None)).unwrap(),
+            Some(entry)
+        );
+    }
+
     #[test]
     fn missing_prompt_does_not_touch_redis() {
         let cache = ValkeySemanticCache::with_connection(
diff --git a/litellm-rust/crates/python-bridge/src/cache/binding.rs b/litellm-rust/crates/python-bridge/src/cache/binding.rs
index ad64b24d3c1..a8881f19e45 100644
--- a/litellm-rust/crates/python-bridge/src/cache/binding.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/binding.rs
@@ -56,12 +56,7 @@ impl ResolvedCache {
             CacheBinding::Disabled => ready_none(py)?,
             CacheBinding::Native(service) => {
                 let request = request(input)?;
-                let service = service.clone();
-                run_async(
-                    py,
-                    async move { service.async_lookup(&request, now()).await },
-                    cache_error,
-                )?
+                service.async_lookup_py(py, request)?
             }
             CacheBinding::PythonCallback(callback) => callback.async_lookup(py, kwargs)?,
         };
@@ -179,12 +174,7 @@ impl ResolvedCache {
             CacheBinding::Native(service) => {
                 let request = self::request(request)?;
                 let response: Value = from_py(response)?;
-                let service = service.clone();
-                run_async(
-                    py,
-                    async move { service.async_store(&request, response, now()).await },
-                    cache_error,
-                )
+                service.async_store_py(py, request, response)
             }
             CacheBinding::PythonCallback(callback) => {
                 callback.async_store(py, response, callback_kwargs)
diff --git a/litellm-rust/crates/python-bridge/src/cache/embedder.rs b/litellm-rust/crates/python-bridge/src/cache/embedder.rs
index d240d9d019e..3de0ceb3b67 100644
--- a/litellm-rust/crates/python-bridge/src/cache/embedder.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/embedder.rs
@@ -3,22 +3,37 @@ use std::{future::Future, sync::Arc};
 use litellm_cache::Error;
 use litellm_cache_valkey_semantic::Embedder;
 use litellm_host_python::to_py;
-use pyo3::prelude::*;
+use pyo3::{PyTraverseError, PyVisit, prelude::*};
 use serde_json::Value;
 
 #[derive(Clone)]
 pub(super) struct PythonEmbedder {
     sync_embed: Arc<Py<PyAny>>,
-    async_embed: Arc<Py<PyAny>>,
+    async_embed_callable: Arc<Py<PyAny>>,
 }
 
 impl PythonEmbedder {
     pub(super) fn from_backend(backend: &Bound<'_, PyAny>) -> PyResult<Self> {
         Ok(Self {
             sync_embed: Arc::new(backend.getattr("_get_embedding")?.unbind()),
-            async_embed: Arc::new(backend.getattr("_get_async_embedding")?.unbind()),
+            async_embed_callable: Arc::new(backend.getattr("_get_async_embedding")?.unbind()),
         })
     }
+
+    pub(super) fn async_embed_awaitable<'py>(
+        &self,
+        py: Python<'py>,
+        prompt: &str,
+        metadata: &Option<Value>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let metadata = to_py(py, metadata)?;
+        self.async_embed_callable.bind(py).call1((prompt, metadata))
+    }
+
+    pub(super) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
+        visit.call(&*self.sync_embed)?;
+        visit.call(&*self.async_embed_callable)
+    }
 }
 
 impl Embedder for PythonEmbedder {
@@ -34,25 +49,15 @@ impl Embedder for PythonEmbedder {
         Ok(result.into_iter().map(|value| value as f32).collect())
     }
 
+    #[expect(
+        clippy::manual_async_fn,
+        reason = "the shared Embedder trait uses an impl Future return"
+    )]
     fn async_embed(
         &self,
-        prompt: &str,
-        metadata: Option<&Value>,
+        _prompt: &str,
+        _metadata: Option<&Value>,
     ) -> impl Future<Output = Result<Vec<f32>, Error>> + Send {
-        let callable = Arc::clone(&self.async_embed);
-        let prompt = prompt.to_owned();
-        let metadata = metadata.cloned();
-        async move {
-            let future = Python::attach(|py| -> PyResult<_> {
-                let metadata = to_py(py, &metadata)?;
-                let awaitable = callable.bind(py).call1((prompt, metadata))?;
-                pyo3_async_runtimes::tokio::into_future(awaitable)
-            })
-            .map_err(|_| Error::Unavailable)?;
-            let result = future.await.map_err(|_| Error::Unavailable)?;
-            let result = Python::attach(|py| result.bind(py).extract::<Vec<f64>>())
-                .map_err(|_| Error::Unavailable)?;
-            Ok(result.into_iter().map(|value| value as f32).collect())
-        }
+        async { Err(Error::Unavailable) }
     }
 }
diff --git a/litellm-rust/crates/python-bridge/src/cache/mod.rs b/litellm-rust/crates/python-bridge/src/cache/mod.rs
index 4cc87367d91..278d3da1ff9 100644
--- a/litellm-rust/crates/python-bridge/src/cache/mod.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/mod.rs
@@ -8,6 +8,7 @@ mod handle;
 mod native;
 mod request;
 mod resolver;
+mod semantic_step;
 
 use litellm_cache::Error;
 use pyo3::{
diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs
index 3380a914c41..7340ef6cce0 100644
--- a/litellm-rust/crates/python-bridge/src/cache/native.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/native.rs
@@ -10,9 +10,14 @@ use litellm_cache_response::{
     ResponseCacheRequest, WriteBuffer,
 };
 use litellm_cache_valkey_semantic::{ValkeySemanticCache, ValkeySemanticConfig};
+use pyo3::prelude::*;
 use serde_json::Value;
 
-use super::{embedder::PythonEmbedder, request::NativeRequest};
+use super::{
+    embedder::PythonEmbedder,
+    request::NativeRequest,
+    semantic_step::{SemanticEmbedExecution, drive_semantic},
+};
 
 fn semantic_key(request: &NativeRequest, scope: &str) -> litellm_cache_response::CacheKeyInput {
     let mut key = request.key.clone();
@@ -59,6 +64,7 @@ pub(super) enum NativeResponseCache {
     },
     ValkeySemantic {
         cache: Arc<ResponseCache<ValkeySemanticCache<PythonEmbedder, ResponseCacheCodec>>>,
+        embedder: PythonEmbedder,
         scope: String,
     },
 }
@@ -100,7 +106,7 @@ impl NativeResponseCache {
     ) -> Result<Self, Error> {
         let backend = ValkeySemanticCache::new(
             url,
-            embedder,
+            embedder.clone(),
             ResponseCacheCodec,
             ValkeySemanticConfig {
                 similarity_threshold,
@@ -109,6 +115,7 @@ impl NativeResponseCache {
         )?;
         Ok(Self::ValkeySemantic {
             cache: Arc::new(ResponseCache::new(Arc::new(backend))),
+            embedder,
             scope: String::from("key"),
         })
     }
@@ -152,7 +159,13 @@ impl NativeResponseCache {
 
     pub fn with_scope(self, scope: String) -> Self {
         match self {
-            Self::ValkeySemantic { cache, .. } => Self::ValkeySemantic { cache, scope },
+            Self::ValkeySemantic {
+                cache, embedder, ..
+            } => Self::ValkeySemantic {
+                cache,
+                embedder,
+                scope,
+            },
             value => value,
         }
     }
@@ -215,7 +228,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.lookup(&Self::exact(request), now),
             Self::Redis { cache, .. } => cache.lookup(&Self::exact(request), now),
-            Self::ValkeySemantic { cache, scope } => {
+            Self::ValkeySemantic { cache, scope, .. } => {
                 cache.lookup(&Self::semantic(request, scope), now)
             }
         }
@@ -230,7 +243,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.store(&Self::exact(request), response, now),
             Self::Redis { cache, .. } => cache.store(&Self::exact(request), response, now),
-            Self::ValkeySemantic { cache, scope } => {
+            Self::ValkeySemantic { cache, scope, .. } => {
                 cache.store(&Self::semantic(request, scope), response, now)
             }
         }
@@ -262,7 +275,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.async_lookup(&Self::exact(request), now).await,
             Self::Redis { cache, .. } => cache.async_lookup(&Self::exact(request), now).await,
-            Self::ValkeySemantic { cache, scope } => {
+            Self::ValkeySemantic { cache, scope, .. } => {
                 cache
                     .async_lookup(&Self::semantic(request, scope), now)
                     .await
@@ -270,6 +283,36 @@ impl NativeResponseCache {
         }
     }
 
+    pub(super) fn async_lookup_py<'py>(
+        &self,
+        py: Python<'py>,
+        request: NativeRequest,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        match self {
+            Self::Memory(_) | Self::Redis { .. } => {
+                let service = self.clone();
+                litellm_host_python::run_async(
+                    py,
+                    async move { service.async_lookup(&request, super::request::now()).await },
+                    super::cache_error,
+                )
+            }
+            Self::ValkeySemantic {
+                cache,
+                embedder,
+                scope,
+            } => drive_semantic(
+                py,
+                SemanticEmbedExecution::lookup(
+                    Arc::clone(cache.backend_arc()),
+                    embedder.clone(),
+                    Self::semantic(&request, scope),
+                    super::request::now(),
+                ),
+            ),
+        }
+    }
+
     pub async fn async_store(
         &self,
         request: &NativeRequest,
@@ -298,7 +341,7 @@ impl NativeResponseCache {
                     .async_store(cache, &Self::exact(request), response, now)
                     .await
             }
-            Self::ValkeySemantic { cache, scope } => {
+            Self::ValkeySemantic { cache, scope, .. } => {
                 cache
                     .async_store(&Self::semantic(request, scope), response, now)
                     .await
@@ -306,6 +349,42 @@ impl NativeResponseCache {
         }
     }
 
+    pub(super) fn async_store_py<'py>(
+        &self,
+        py: Python<'py>,
+        request: NativeRequest,
+        response: Value,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        match self {
+            Self::Memory(_) | Self::Redis { .. } => {
+                let service = self.clone();
+                litellm_host_python::run_async(
+                    py,
+                    async move {
+                        service
+                            .async_store(&request, response, super::request::now())
+                            .await
+                    },
+                    super::cache_error,
+                )
+            }
+            Self::ValkeySemantic {
+                cache,
+                embedder,
+                scope,
+            } => drive_semantic(
+                py,
+                SemanticEmbedExecution::store(
+                    Arc::clone(cache.backend_arc()),
+                    embedder.clone(),
+                    Self::semantic(&request, scope),
+                    response,
+                    super::request::now(),
+                ),
+            ),
+        }
+    }
+
     pub async fn async_lookup_batch(
         &self,
         requests: &[NativeRequest],
@@ -344,12 +423,10 @@ impl NativeResponseCache {
                     .collect();
                 cache.async_store_batch(entries, now).await
             }
-            Self::ValkeySemantic { cache, scope } => {
-                let entries = entries
-                    .into_iter()
-                    .map(|(request, value)| (Self::semantic(&request, scope), value))
-                    .collect();
-                cache.async_store_batch(entries, now).await
+            Self::ValkeySemantic { cache, scope, .. } => {
+                entries.into_iter().try_for_each(|(request, value)| {
+                    cache.store(&Self::semantic(&request, scope), value, now)
+                })
             }
         }
     }
diff --git a/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs b/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs
new file mode 100644
index 00000000000..c62cdb1d9a6
--- /dev/null
+++ b/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs
@@ -0,0 +1,154 @@
+use std::{sync::Arc, time::Duration};
+
+use litellm_cache::SemanticCacheContext;
+use litellm_cache_response::{ResponseCache, ResponseCacheCodec, ResponseCacheRequest};
+use litellm_cache_valkey_semantic::{
+    Embedder, PreparedEmbedding, ValkeySemanticCache, prompt_from_context,
+};
+use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async};
+use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*};
+use serde_json::Value;
+
+use super::{cache_error, embedder::PythonEmbedder};
+
+pub(super) enum Op {
+    Lookup,
+    Store(Value),
+}
+
+#[derive(Clone, Copy)]
+enum State {
+    Start,
+    AwaitingEmbedding,
+    AwaitingStorage,
+    Done,
+}
+
+pub(super) struct SemanticEmbedExecution {
+    backend: Arc<ValkeySemanticCache<PythonEmbedder, ResponseCacheCodec>>,
+    embedder: PythonEmbedder,
+    request: ResponseCacheRequest<SemanticCacheContext>,
+    op: Op,
+    now: Duration,
+    state: State,
+}
+
+impl SemanticEmbedExecution {
+    pub(super) fn lookup(
+        backend: Arc<ValkeySemanticCache<PythonEmbedder, ResponseCacheCodec>>,
+        embedder: PythonEmbedder,
+        request: ResponseCacheRequest<SemanticCacheContext>,
+        now: Duration,
+    ) -> Self {
+        Self {
+            backend,
+            embedder,
+            request,
+            op: Op::Lookup,
+            now,
+            state: State::Start,
+        }
+    }
+
+    pub(super) fn store(
+        backend: Arc<ValkeySemanticCache<PythonEmbedder, ResponseCacheCodec>>,
+        embedder: PythonEmbedder,
+        request: ResponseCacheRequest<SemanticCacheContext>,
+        response: Value,
+        now: Duration,
+    ) -> Self {
+        Self {
+            backend,
+            embedder,
+            request,
+            op: Op::Store(response),
+            now,
+            state: State::Start,
+        }
+    }
+
+    fn start(&mut self, py: Python<'_>) -> PyResult<ExecutionStep> {
+        let Some(prompt) = prompt_from_context(&self.request.context) else {
+            let cache = Arc::new(ResponseCache::new(Arc::clone(&self.backend)));
+            self.state = State::AwaitingStorage;
+            return storage_step(py, cache, self.request.clone(), &self.op, self.now);
+        };
+        let awaitable =
+            self.embedder
+                .async_embed_awaitable(py, &prompt, &self.request.context.metadata)?;
+        self.state = State::AwaitingEmbedding;
+        Ok(ExecutionStep::Await(awaitable.unbind()))
+    }
+
+    fn resume_py(
+        &mut self,
+        py: Python<'_>,
+        result: Option<PyResult<Py<PyAny>>>,
+    ) -> PyResult<ExecutionStep> {
+        match (self.state, result) {
+            (State::Start, None) => self.start(py),
+            (State::AwaitingEmbedding, Some(Ok(value))) => {
+                let values = value.bind(py).extract::<Vec<f64>>()?;
+                let backend = self.backend.with_embedder(PreparedEmbedding(
+                    values.into_iter().map(|value| value as f32).collect(),
+                ));
+                let cache = Arc::new(ResponseCache::new(Arc::new(backend)));
+                self.state = State::AwaitingStorage;
+                storage_step(py, cache, self.request.clone(), &self.op, self.now)
+            }
+            (State::AwaitingStorage, Some(Ok(value))) => {
+                self.state = State::Done;
+                Ok(ExecutionStep::Return(value))
+            }
+            (_, Some(Err(error))) => Err(error),
+            _ => Err(PyRuntimeError::new_err(
+                "invalid semantic cache execution state",
+            )),
+        }
+    }
+}
+
+impl ExecutionBody for SemanticEmbedExecution {
+    fn resume(&mut self, result: Option<PyResult<Py<PyAny>>>) -> PyResult<ExecutionStep> {
+        Python::attach(|py| self.resume_py(py, result))
+    }
+
+    fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
+        self.embedder.traverse(visit)
+    }
+}
+
+fn storage_step<E: Embedder>(
+    py: Python<'_>,
+    cache: Arc<ResponseCache<ValkeySemanticCache<E, ResponseCacheCodec>>>,
+    request: ResponseCacheRequest<SemanticCacheContext>,
+    op: &Op,
+    now: Duration,
+) -> PyResult<ExecutionStep> {
+    let awaitable = match op {
+        Op::Lookup => run_async(
+            py,
+            async move { cache.async_lookup(&request, now).await },
+            cache_error,
+        )?,
+        Op::Store(response) => {
+            let response = response.clone();
+            run_async(
+                py,
+                async move { cache.async_store(&request, response, now).await },
+                cache_error,
+            )?
+        }
+    };
+    Ok(ExecutionStep::Await(awaitable.unbind()))
+}
+
+pub(super) fn drive_semantic<'py>(
+    py: Python<'py>,
+    body: SemanticEmbedExecution,
+) -> PyResult<Bound<'py, PyAny>> {
+    let execution = Py::new(py, Execution::new(body))?;
+    py.import("litellm.rust_bridge.lifecycle")?
+        .getattr("drive")?
+        .call1((execution,))
+}
diff --git a/tests/test_litellm_rust/test_valkey_semantic_cache_native.py b/tests/test_litellm_rust/test_valkey_semantic_cache_native.py
index 81fcf00ffc0..e2bd3dcb10a 100644
--- a/tests/test_litellm_rust/test_valkey_semantic_cache_native.py
+++ b/tests/test_litellm_rust/test_valkey_semantic_cache_native.py
@@ -1,6 +1,9 @@
+import asyncio
+import contextvars
 import hashlib
 import os
 import struct
+import threading
 import time
 from collections.abc import Generator, Mapping
 from types import SimpleNamespace
@@ -16,6 +19,7 @@ from litellm.rust_bridge import _native
 from litellm.types.caching import LiteLLMCacheType
 
 pytestmark: Final = pytest.mark.requires_rust_extension
+embedding_context: Final = contextvars.ContextVar("embedding_context")
 
 
 @pytest.fixture
@@ -171,6 +175,43 @@ async def test_async_lookup_and_store(
     assert await binding.async_lookup(request) == {"answer": "async"}
 
 
+async def test_async_embedding_runs_inline_in_caller_task(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    backend: Final = _backend(valkey_url, index_name)
+    observed: dict[str, object] = {}
+
+    async def async_embedding(prompt: str, metadata: dict[str, object] | None = None) -> list[float]:
+        observed["context"] = embedding_context.get("missing")
+        observed["task"] = asyncio.current_task()
+        observed["thread"] = threading.get_ident()
+        embedding_context.set("embedder")
+        return [1.0, 0.0]
+
+    backend._get_async_embedding = async_embedding
+    handle: Final = _native._CacheTestHandle.valkey_semantic(
+        valkey_url,
+        0.8,
+        index_name,
+        backend,
+    )
+    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    request: Final = {**_request(), "ttl_seconds": 2.0}
+    caller_task: Final = asyncio.current_task()
+    caller_thread: Final = threading.get_ident()
+    token: Final = embedding_context.set("caller")
+    try:
+        await binding.async_store(request, {"answer": "inline"})
+        assert observed["context"] == "caller"
+        assert observed["task"] is caller_task
+        assert observed["thread"] == caller_thread
+        assert embedding_context.get() == "embedder"
+        assert await binding.async_lookup(request) == {"answer": "inline"}
+    finally:
+        embedding_context.reset(token)
+
+
 def test_facade_activation_and_mutation_fallback(
     valkey_url: str,
     index_name: str,

From 28f70c75d17815cc9b88963c0825b2c31e1a1853 Mon Sep 17 00:00:00 2001
From: ryan <ryan@berri.ai>
Date: Mon, 21 Sep 2026 22:02:05 +0000
Subject: [PATCH 123/160] fix(auth): reject unchanged password on
 /user/password/change

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../password_endpoints.py                     | 17 +++++++++----
 .../test_password_endpoints.py                | 25 +++++++++++++++++++
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/litellm/proxy/management_endpoints/password_endpoints.py b/litellm/proxy/management_endpoints/password_endpoints.py
index a2466c83ef9..c94399ba25d 100644
--- a/litellm/proxy/management_endpoints/password_endpoints.py
+++ b/litellm/proxy/management_endpoints/password_endpoints.py
@@ -63,11 +63,12 @@ async def change_password(
     """
     Change the calling user's own password.
 
-    Requires the current password. The new password must satisfy the
-    configured password policy (`general_settings.password_policy_*`: minimum
-    length, character classes, and, when enabled, breached-password screening
-    via haveibeenpwned.com). A successful change lifts any pending forced
-    password reset (`password_reset_required`) on the account.
+    Requires the current password. The new password must differ from the
+    current one and satisfy the configured password policy
+    (`general_settings.password_policy_*`: minimum length, character classes,
+    and, when enabled, breached-password screening via haveibeenpwned.com).
+    A successful change lifts any pending forced password reset
+    (`password_reset_required`) on the account.
 
     Parameters:
     - current_password: str - The user's current password.
@@ -103,6 +104,12 @@ async def change_password(
     if not verify_password(data.current_password, stored_password):
         raise HTTPException(status_code=400, detail=_error_detail("Current password is incorrect."))
 
+    if data.new_password == data.current_password:
+        raise HTTPException(
+            status_code=400,
+            detail=_error_detail("New password must be different from the current password."),
+        )
+
     validate_password_policy(data.new_password, general_settings)
     await validate_password_not_breached(data.new_password, general_settings)
 
diff --git a/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py
index bd154ebab41..581a2ef8df8 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py
@@ -105,6 +105,31 @@ async def test_change_password_rejects_wrong_current_password():
     prisma.db.litellm_usertable.update.assert_not_called()
 
 
+@pytest.mark.asyncio
+async def test_change_password_rejects_unchanged_password():
+    from litellm.proxy._types import ChangePasswordRequest
+
+    prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
+
+    with (
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", prisma
+        ),
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
+        ),
+    ):
+        with pytest.raises(HTTPException) as exc_info:
+            await change_password(
+                data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=CURRENT_PASSWORD),
+                user_api_key_dict=_caller(),
+            )
+
+    assert exc_info.value.status_code == 400
+    assert "must be different from the current password" in exc_info.value.detail["error"]
+    prisma.db.litellm_usertable.update.assert_not_called()
+
+
 @pytest.mark.asyncio
 async def test_change_password_rejects_session_without_user():
     from litellm.proxy._types import ChangePasswordRequest

From 8c8250596473aaef9ba6b1e685eeee3ead42b8a1 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 22:02:52 +0000
Subject: [PATCH 124/160] fix(python-bridge): await semantic embeddings inline
 in the caller's task

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../crates/cache-redis-semantic/src/lib.rs    |   1 +
 litellm-rust/crates/python-bridge/Cargo.toml  |   2 +-
 .../crates/python-bridge/src/cache/binding.rs |  22 ++-
 .../python-bridge/src/cache/embedder.rs       |  78 ++++++---
 .../crates/python-bridge/src/cache/mod.rs     |   1 +
 .../crates/python-bridge/src/cache/native.rs  |   7 +
 .../crates/python-bridge/src/cache/request.rs |   1 +
 .../python-bridge/src/cache/semantic.rs       | 165 ++++++++++++++++++
 tests/test_litellm_rust/test_cache.py         |  56 ++++++
 9 files changed, 308 insertions(+), 25 deletions(-)
 create mode 100644 litellm-rust/crates/python-bridge/src/cache/semantic.rs

diff --git a/litellm-rust/crates/cache-redis-semantic/src/lib.rs b/litellm-rust/crates/cache-redis-semantic/src/lib.rs
index a34603cd18f..51d0b4ba5f3 100644
--- a/litellm-rust/crates/cache-redis-semantic/src/lib.rs
+++ b/litellm-rust/crates/cache-redis-semantic/src/lib.rs
@@ -2,3 +2,4 @@ mod cache;
 mod prompt;
 
 pub use cache::{Embedder, RedisSemanticCache, RedisSemanticConfig};
+pub use prompt::prompt_from_context;
diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml
index b28ddc50181..93ce5828489 100644
--- a/litellm-rust/crates/python-bridge/Cargo.toml
+++ b/litellm-rust/crates/python-bridge/Cargo.toml
@@ -39,7 +39,7 @@ litellm-token-counter = { path = "../token-counter", default-features = false }
 pyo3.workspace = true
 pyo3-async-runtimes.workspace = true
 serde_json.workspace = true
-tokio = { workspace = true, features = ["sync"] }
+tokio = { workspace = true, features = ["rt", "sync"] }
 
 [dev-dependencies]
 serde.workspace = true
diff --git a/litellm-rust/crates/python-bridge/src/cache/binding.rs b/litellm-rust/crates/python-bridge/src/cache/binding.rs
index ad64b24d3c1..0b90e8151ea 100644
--- a/litellm-rust/crates/python-bridge/src/cache/binding.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/binding.rs
@@ -14,6 +14,7 @@ use super::{
     future::{ready_none, ready_value},
     native::NativeResponseCache,
     request::{now, request, requests},
+    semantic::{SemanticOperation, drive},
 };
 
 pub(super) enum CacheBinding {
@@ -56,6 +57,11 @@ impl ResolvedCache {
             CacheBinding::Disabled => ready_none(py)?,
             CacheBinding::Native(service) => {
                 let request = request(input)?;
+                if service.semantic_embedder().is_some() {
+                    return Ok(ExecutionStep::Await(
+                        drive(py, service.clone(), SemanticOperation::Lookup(request))?.unbind(),
+                    ));
+                }
                 let service = service.clone();
                 run_async(
                     py,
@@ -179,6 +185,13 @@ impl ResolvedCache {
             CacheBinding::Native(service) => {
                 let request = self::request(request)?;
                 let response: Value = from_py(response)?;
+                if service.semantic_embedder().is_some() {
+                    return drive(
+                        py,
+                        service.clone(),
+                        SemanticOperation::Store(request, response),
+                    );
+                }
                 let service = service.clone();
                 run_async(
                     py,
@@ -240,7 +253,14 @@ impl ResolvedCache {
                         "batch cache requests and responses must have equal lengths",
                     ));
                 }
-                let entries = requests.into_iter().zip(responses).collect();
+                let entries = requests.into_iter().zip(responses).collect::<Vec<_>>();
+                if service.semantic_embedder().is_some() {
+                    return drive(
+                        py,
+                        service.clone(),
+                        SemanticOperation::StoreBatch(entries.into()),
+                    );
+                }
                 let service = service.clone();
                 run_async(
                     py,
diff --git a/litellm-rust/crates/python-bridge/src/cache/embedder.rs b/litellm-rust/crates/python-bridge/src/cache/embedder.rs
index 63e078cd815..26edb26f428 100644
--- a/litellm-rust/crates/python-bridge/src/cache/embedder.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/embedder.rs
@@ -6,6 +6,17 @@ use litellm_host_python::to_py;
 use pyo3::{PyTraverseError, PyVisit, prelude::*, types::PyDict};
 use serde_json::{Map, Value};
 
+tokio::task_local! {
+    static PREPARED_EMBEDDING: Result<Vec<f32>, Error>;
+}
+
+pub(super) fn with_prepared_embedding<F: Future>(
+    vector: Result<Vec<f32>, Error>,
+    future: F,
+) -> impl Future<Output = F::Output> {
+    PREPARED_EMBEDDING.scope(vector, future)
+}
+
 pub(super) struct PythonEmbedder(Py<PyAny>);
 
 impl PythonEmbedder {
@@ -34,7 +45,20 @@ impl PythonEmbedder {
         Ok(kwargs)
     }
 
-    fn extract(vector: Bound<'_, PyAny>) -> PyResult<Vec<f32>> {
+    pub(super) fn async_embedding_coroutine(
+        &self,
+        py: Python<'_>,
+        prompt: &str,
+        metadata: &Map<String, Value>,
+    ) -> PyResult<Py<PyAny>> {
+        let kwargs = Self::metadata_kwargs(py, metadata)?;
+        self.0
+            .bind(py)
+            .call_method("_get_async_embedding", (prompt,), Some(&kwargs))
+            .map(Bound::unbind)
+    }
+
+    pub(super) fn extract(vector: Bound<'_, PyAny>) -> PyResult<Vec<f32>> {
         Ok(vector
             .extract::<Vec<f64>>()?
             .into_iter()
@@ -58,28 +82,36 @@ impl Embedder for PythonEmbedder {
 
     fn async_embed(
         &self,
-        prompt: &str,
-        metadata: &Map<String, Value>,
+        _prompt: &str,
+        _metadata: &Map<String, Value>,
     ) -> impl Future<Output = Result<Vec<f32>, Error>> + Send {
-        let coroutine = Python::attach(|py| {
-            let kwargs = Self::metadata_kwargs(py, metadata)?;
-            self.0
-                .bind(py)
-                .call_method("_get_async_embedding", (prompt,), Some(&kwargs))
-                .map(Bound::unbind)
-        })
-        .map_err(|_| Error::Unavailable);
-        async move {
-            let coroutine = coroutine?;
-            let awaited = Python::attach(|py| {
-                pyo3_async_runtimes::tokio::into_future(coroutine.into_bound(py))
-            })
-            .map_err(|_| Error::Unavailable)?
-            .await
-            .map_err(|_| Error::Unavailable)?;
-            let vector = Python::attach(|py| awaited.extract::<Vec<f64>>(py))
-                .map_err(|_| Error::Unavailable)?;
-            Ok(vector.into_iter().map(|value| value as f32).collect())
-        }
+        let seeded = PREPARED_EMBEDDING
+            .try_with(Clone::clone)
+            .unwrap_or(Err(Error::Unavailable));
+        std::future::ready(seeded)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn async_embed_returns_the_seeded_vector_or_unavailable() {
+        let embedder = Python::attach(|py| PythonEmbedder::new(py.None()));
+        let metadata = Map::new();
+        let embedder_ref = &embedder;
+        let metadata_ref = &metadata;
+        assert_eq!(
+            with_prepared_embedding(Ok(vec![0.5f32, 0.25]), async move {
+                embedder_ref.async_embed("prompt", metadata_ref).await
+            })
+            .await,
+            Ok(vec![0.5, 0.25])
+        );
+        assert_eq!(
+            embedder.async_embed("prompt", &metadata).await,
+            Err(Error::Unavailable)
+        );
     }
 }
diff --git a/litellm-rust/crates/python-bridge/src/cache/mod.rs b/litellm-rust/crates/python-bridge/src/cache/mod.rs
index 4cc87367d91..cd772d571cb 100644
--- a/litellm-rust/crates/python-bridge/src/cache/mod.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/mod.rs
@@ -8,6 +8,7 @@ mod handle;
 mod native;
 mod request;
 mod resolver;
+mod semantic;
 
 use litellm_cache::Error;
 use pyo3::{
diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs
index 182010fab02..de9c4afa236 100644
--- a/litellm-rust/crates/python-bridge/src/cache/native.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/native.rs
@@ -122,6 +122,13 @@ impl NativeResponseCache {
         }
     }
 
+    pub fn semantic_embedder(&self) -> Option<&PythonEmbedder> {
+        match self {
+            Self::RedisSemantic(cache) => Some(cache.backend().embedder()),
+            Self::Memory(_) | Self::Redis { .. } => None,
+        }
+    }
+
     pub fn embedder_object(&self) -> Option<&Py<PyAny>> {
         match self {
             Self::RedisSemantic(cache) => Some(cache.backend().embedder().object()),
diff --git a/litellm-rust/crates/python-bridge/src/cache/request.rs b/litellm-rust/crates/python-bridge/src/cache/request.rs
index 26e0fe4e62c..b06087bcc83 100644
--- a/litellm-rust/crates/python-bridge/src/cache/request.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/request.rs
@@ -20,6 +20,7 @@ struct RequestInput {
     scope: Option<String>,
 }
 
+#[derive(Clone)]
 pub(super) struct CacheRequest {
     key: CacheKeyInput,
     controls: CacheControls,
diff --git a/litellm-rust/crates/python-bridge/src/cache/semantic.rs b/litellm-rust/crates/python-bridge/src/cache/semantic.rs
new file mode 100644
index 00000000000..eb38b8b9c67
--- /dev/null
+++ b/litellm-rust/crates/python-bridge/src/cache/semantic.rs
@@ -0,0 +1,165 @@
+use std::collections::VecDeque;
+
+use litellm_cache::Error;
+use litellm_cache_redis_semantic::prompt_from_context;
+use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async};
+use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*};
+use serde_json::Value;
+
+use super::{
+    cache_error,
+    embedder::{PythonEmbedder, with_prepared_embedding},
+    native::NativeResponseCache,
+    request::{CacheRequest, now},
+};
+
+pub(super) enum SemanticOperation {
+    Lookup(CacheRequest),
+    Store(CacheRequest, Value),
+    StoreBatch(VecDeque<(CacheRequest, Value)>),
+}
+
+enum Phase {
+    Start,
+    AwaitingEmbedding,
+    AwaitingBackend,
+}
+
+pub(super) struct SemanticBody {
+    service: NativeResponseCache,
+    operation: SemanticOperation,
+    pending: Option<(CacheRequest, Option<Value>)>,
+    phase: Phase,
+}
+
+impl SemanticBody {
+    pub(super) fn new(service: NativeResponseCache, operation: SemanticOperation) -> Self {
+        Self {
+            service,
+            operation,
+            pending: None,
+            phase: Phase::Start,
+        }
+    }
+
+    fn backend_step(
+        &mut self,
+        py: Python<'_>,
+        seed: Result<Vec<f32>, Error>,
+    ) -> PyResult<ExecutionStep> {
+        self.phase = Phase::AwaitingBackend;
+        let (request, response) = self.pending.take().ok_or_else(|| {
+            PyRuntimeError::new_err("semantic execution resumed without a pending operation")
+        })?;
+        let service = self.service.clone();
+        let future = async move {
+            match response {
+                None => service.async_lookup(&request, now()).await,
+                Some(response) => service
+                    .async_store(&request, response, now())
+                    .await
+                    .map(|_| None),
+            }
+        };
+        let awaitable = run_async(py, with_prepared_embedding(seed, future), cache_error)?;
+        Ok(ExecutionStep::Await(awaitable.unbind()))
+    }
+}
+
+impl ExecutionBody for SemanticBody {
+    fn resume(&mut self, mut result: Option<PyResult<Py<PyAny>>>) -> PyResult<ExecutionStep> {
+        Python::attach(|py| {
+            loop {
+                match self.phase {
+                    Phase::Start => {
+                        if result.is_some() {
+                            return Err(PyRuntimeError::new_err(
+                                "semantic execution received a result before starting",
+                            ));
+                        }
+                        if self.pending.is_none() {
+                            match &mut self.operation {
+                                SemanticOperation::Lookup(request) => {
+                                    self.pending = Some((request.clone(), None));
+                                }
+                                SemanticOperation::Store(request, response) => {
+                                    let response = std::mem::replace(response, Value::Null);
+                                    self.pending = Some((request.clone(), Some(response)));
+                                }
+                                SemanticOperation::StoreBatch(queue) => {
+                                    let Some((request, response)) = queue.pop_front() else {
+                                        return Ok(ExecutionStep::Return(py.None()));
+                                    };
+                                    self.pending = Some((request, Some(response)));
+                                }
+                            }
+                        }
+                        let (request, _) = self.pending.as_ref().ok_or_else(|| {
+                            PyRuntimeError::new_err("semantic execution has no pending operation")
+                        })?;
+                        let semantic = request.semantic();
+                        let Some(prompt) = prompt_from_context(&semantic.context) else {
+                            return self.backend_step(py, Err(Error::Unavailable));
+                        };
+                        let embedder = self.service.semantic_embedder().ok_or_else(|| {
+                            PyRuntimeError::new_err(
+                                "semantic execution requires a redis-semantic backend",
+                            )
+                        })?;
+                        let coroutine = embedder.async_embedding_coroutine(
+                            py,
+                            &prompt,
+                            &semantic.context.metadata,
+                        )?;
+                        self.phase = Phase::AwaitingEmbedding;
+                        return Ok(ExecutionStep::Await(coroutine));
+                    }
+                    Phase::AwaitingEmbedding => {
+                        let result = result.take().ok_or_else(|| {
+                            PyRuntimeError::new_err(
+                                "semantic execution expected an embedding result",
+                            )
+                        })?;
+                        let seed = result
+                            .and_then(|value| PythonEmbedder::extract(value.into_bound(py)))
+                            .map_err(|_| Error::Unavailable);
+                        return self.backend_step(py, seed);
+                    }
+                    Phase::AwaitingBackend => {
+                        let result = result.take().ok_or_else(|| {
+                            PyRuntimeError::new_err("semantic execution expected a backend result")
+                        })?;
+                        let value = match result {
+                            Ok(value) => value,
+                            Err(error) => return Err(error),
+                        };
+                        let more = matches!(
+                            &self.operation,
+                            SemanticOperation::StoreBatch(queue) if !queue.is_empty()
+                        );
+                        if more {
+                            self.phase = Phase::Start;
+                            continue;
+                        }
+                        return Ok(ExecutionStep::Return(value));
+                    }
+                }
+            }
+        })
+    }
+
+    fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
+        self.service.traverse(visit)
+    }
+}
+
+pub(super) fn drive(
+    py: Python<'_>,
+    service: NativeResponseCache,
+    operation: SemanticOperation,
+) -> PyResult<Bound<'_, PyAny>> {
+    let execution = Py::new(py, Execution::new(SemanticBody::new(service, operation)))?;
+    py.import("litellm.rust_bridge.lifecycle")?
+        .getattr("drive")?
+        .call1((execution,))
+}
diff --git a/tests/test_litellm_rust/test_cache.py b/tests/test_litellm_rust/test_cache.py
index a60feb9973a..9312fdde075 100644
--- a/tests/test_litellm_rust/test_cache.py
+++ b/tests/test_litellm_rust/test_cache.py
@@ -479,6 +479,7 @@ async def test_redis_cluster_facade_serves_multi_slot_batches_and_scoped_flush_n
 PARAPHRASE_MARKER: Final = " (paraphrase)"
 SEMANTIC_EMBEDDING_MODEL: Final = "semantic-test/deterministic"
 SEMANTIC_INDEX_PREFIX: Final = "litellm_test_semantic_"
+SEMANTIC_CONTEXT: Final = contextvars.ContextVar("semantic_test_context", default="unset")
 
 
 def _normalized(vector: list[float]) -> list[float]:
@@ -509,6 +510,7 @@ def _semantic_embedding(prompt: str) -> list[float]:
 class DeterministicEmbedding(litellm.CustomLLM):
     def __init__(self) -> None:
         self.calls: list[dict[str, object]] = []
+        self.async_calls: list[dict[str, object]] = []
 
     def _respond(
         self,
@@ -553,6 +555,16 @@ class DeterministicEmbedding(litellm.CustomLLM):
         timeout: object = None,
         litellm_params: object = None,
     ) -> EmbeddingResponse:
+        texts: Final = cast(list[object], input if isinstance(input, list) else [input])
+        self.async_calls.append(
+            {
+                "model": model,
+                "input": texts,
+                "task": asyncio.current_task(),
+                "context": SEMANTIC_CONTEXT.get(),
+            }
+        )
+        SEMANTIC_CONTEXT.set("written-in-aembedding")
         return self._respond(model, input, model_response)
 
 
@@ -755,6 +767,50 @@ async def test_redis_semantic_async_paths_and_store_batch_share_one_layout(
     client.close()
 
 
+async def test_native_semantic_async_embedding_runs_inline_in_the_callers_task(
+    redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding
+) -> None:
+    url, index = redis_stack
+    facade: Final = semantic_facade(url, index)
+    binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve()
+    assert binding.kind == "native"
+    caller: Final = asyncio.current_task()
+    SEMANTIC_CONTEXT.set("caller-sentinel")
+    response: Final = {"choices": [{"text": "paris"}]}
+
+    await binding.async_store(
+        semantic_request("inline", "what is the capital of france"), response
+    )
+    assert (
+        await binding.async_lookup(
+            semantic_request("inline", f"what is the capital of france{PARAPHRASE_MARKER}")
+        )
+        == response
+    )
+    assert await binding.async_lookup(semantic_request("inline", "python written prompt")) is None
+    assert SEMANTIC_CONTEXT.get() == "written-in-aembedding"
+    assert semantic_embedding.async_calls == [
+        {
+            "model": "deterministic",
+            "input": ["what is the capital of france"],
+            "task": caller,
+            "context": "caller-sentinel",
+        },
+        {
+            "model": "deterministic",
+            "input": [f"what is the capital of france{PARAPHRASE_MARKER}"],
+            "task": caller,
+            "context": "written-in-aembedding",
+        },
+        {
+            "model": "deterministic",
+            "input": ["python written prompt"],
+            "task": caller,
+            "context": "written-in-aembedding",
+        },
+    ], semantic_embedding.async_calls
+
+
 def test_redis_semantic_similarity_tag_and_threshold_boundaries(
     redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding
 ) -> None:

From c233377d9c073fd9e2262c58ea14b2ca75aadf0a Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 22:04:46 +0000
Subject: [PATCH 125/160] fix(cache): await valkey semantic batch embeddings
 inline

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../crates/python-bridge/src/cache/binding.rs |   7 +-
 .../crates/python-bridge/src/cache/native.rs  |  49 ++++-
 .../python-bridge/src/cache/semantic_step.rs  | 175 +++++++++++++-----
 .../test_valkey_semantic_cache_native.py      |  20 ++
 4 files changed, 197 insertions(+), 54 deletions(-)

diff --git a/litellm-rust/crates/python-bridge/src/cache/binding.rs b/litellm-rust/crates/python-bridge/src/cache/binding.rs
index a8881f19e45..2ff73238202 100644
--- a/litellm-rust/crates/python-bridge/src/cache/binding.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/binding.rs
@@ -231,12 +231,7 @@ impl ResolvedCache {
                     ));
                 }
                 let entries = requests.into_iter().zip(responses).collect();
-                let service = service.clone();
-                run_async(
-                    py,
-                    async move { service.async_store_batch(entries, now()).await },
-                    cache_error,
-                )
+                service.async_store_batch_py(py, entries)
             }
             CacheBinding::PythonCallback(callback) => {
                 callback.async_store_batch(py, callback_result, callback_kwargs)
diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs
index 7340ef6cce0..260dc8f349e 100644
--- a/litellm-rust/crates/python-bridge/src/cache/native.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/native.rs
@@ -424,9 +424,52 @@ impl NativeResponseCache {
                 cache.async_store_batch(entries, now).await
             }
             Self::ValkeySemantic { cache, scope, .. } => {
-                entries.into_iter().try_for_each(|(request, value)| {
-                    cache.store(&Self::semantic(&request, scope), value, now)
-                })
+                let entries = entries
+                    .into_iter()
+                    .map(|(request, value)| (Self::semantic(&request, scope), value))
+                    .collect();
+                cache.async_store_batch(entries, now).await
+            }
+        }
+    }
+
+    pub(super) fn async_store_batch_py<'py>(
+        &self,
+        py: Python<'py>,
+        entries: Vec<(NativeRequest, Value)>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        match self {
+            Self::Memory(_) | Self::Redis { .. } => {
+                let service = self.clone();
+                litellm_host_python::run_async(
+                    py,
+                    async move {
+                        service
+                            .async_store_batch(entries, super::request::now())
+                            .await
+                    },
+                    super::cache_error,
+                )
+            }
+            Self::ValkeySemantic {
+                cache,
+                embedder,
+                scope,
+            } => {
+                let (requests, responses): (Vec<_>, Vec<_>) = entries
+                    .into_iter()
+                    .map(|(request, response)| (Self::semantic(&request, scope), response))
+                    .unzip();
+                drive_semantic(
+                    py,
+                    SemanticEmbedExecution::store_batch(
+                        Arc::clone(cache.backend_arc()),
+                        embedder.clone(),
+                        requests,
+                        responses,
+                        super::request::now(),
+                    ),
+                )
             }
         }
     }
diff --git a/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs b/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs
index c62cdb1d9a6..d8f3ab297f8 100644
--- a/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs
@@ -2,9 +2,7 @@ use std::{sync::Arc, time::Duration};
 
 use litellm_cache::SemanticCacheContext;
 use litellm_cache_response::{ResponseCache, ResponseCacheCodec, ResponseCacheRequest};
-use litellm_cache_valkey_semantic::{
-    Embedder, PreparedEmbedding, ValkeySemanticCache, prompt_from_context,
-};
+use litellm_cache_valkey_semantic::{PreparedEmbedding, ValkeySemanticCache, prompt_from_context};
 use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async};
 use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*};
 use serde_json::Value;
@@ -14,6 +12,7 @@ use super::{cache_error, embedder::PythonEmbedder};
 pub(super) enum Op {
     Lookup,
     Store(Value),
+    StoreBatch(Vec<Value>),
 }
 
 #[derive(Clone, Copy)]
@@ -27,9 +26,11 @@ enum State {
 pub(super) struct SemanticEmbedExecution {
     backend: Arc<ValkeySemanticCache<PythonEmbedder, ResponseCacheCodec>>,
     embedder: PythonEmbedder,
-    request: ResponseCacheRequest<SemanticCacheContext>,
+    requests: Vec<ResponseCacheRequest<SemanticCacheContext>>,
     op: Op,
     now: Duration,
+    prepared: Vec<Option<Vec<f32>>>,
+    index: usize,
     state: State,
 }
 
@@ -43,9 +44,11 @@ impl SemanticEmbedExecution {
         Self {
             backend,
             embedder,
-            request,
+            requests: vec![request],
             op: Op::Lookup,
             now,
+            prepared: vec![None],
+            index: 0,
             state: State::Start,
         }
     }
@@ -60,23 +63,132 @@ impl SemanticEmbedExecution {
         Self {
             backend,
             embedder,
-            request,
+            requests: vec![request],
             op: Op::Store(response),
             now,
+            prepared: vec![None],
+            index: 0,
+            state: State::Start,
+        }
+    }
+
+    pub(super) fn store_batch(
+        backend: Arc<ValkeySemanticCache<PythonEmbedder, ResponseCacheCodec>>,
+        embedder: PythonEmbedder,
+        requests: Vec<ResponseCacheRequest<SemanticCacheContext>>,
+        responses: Vec<Value>,
+        now: Duration,
+    ) -> Self {
+        Self {
+            backend,
+            embedder,
+            prepared: vec![None; requests.len()],
+            requests,
+            op: Op::StoreBatch(responses),
+            now,
+            index: 0,
             state: State::Start,
         }
     }
 
     fn start(&mut self, py: Python<'_>) -> PyResult<ExecutionStep> {
-        let Some(prompt) = prompt_from_context(&self.request.context) else {
-            let cache = Arc::new(ResponseCache::new(Arc::clone(&self.backend)));
-            self.state = State::AwaitingStorage;
-            return storage_step(py, cache, self.request.clone(), &self.op, self.now);
+        while self.index < self.requests.len() {
+            let request = &self.requests[self.index];
+            let Some(prompt) = prompt_from_context(&request.context) else {
+                self.index += 1;
+                continue;
+            };
+            let metadata = request.context.metadata.clone();
+            let awaitable = self
+                .embedder
+                .async_embed_awaitable(py, &prompt, &metadata)?;
+            self.state = State::AwaitingEmbedding;
+            return Ok(ExecutionStep::Await(awaitable.unbind()));
+        }
+        self.state = State::AwaitingStorage;
+        self.storage_step(py)
+    }
+
+    fn storage_step(&self, py: Python<'_>) -> PyResult<ExecutionStep> {
+        let requests = self.requests.clone();
+        let prepared = self.prepared.clone();
+        let backend = Arc::clone(&self.backend);
+        let now = self.now;
+        let awaitable = match &self.op {
+            Op::Lookup => {
+                let Some(request) = requests.into_iter().next() else {
+                    return Err(PyRuntimeError::new_err(
+                        "semantic lookup requires one request",
+                    ));
+                };
+                match prepared.into_iter().next().flatten() {
+                    Some(values) => {
+                        let backend = backend.with_embedder(PreparedEmbedding(values));
+                        let cache = Arc::new(ResponseCache::new(Arc::new(backend)));
+                        run_async(
+                            py,
+                            async move { cache.async_lookup(&request, now).await },
+                            cache_error,
+                        )?
+                    }
+                    None => {
+                        let cache = Arc::new(ResponseCache::new(backend));
+                        run_async(
+                            py,
+                            async move { cache.async_lookup(&request, now).await },
+                            cache_error,
+                        )?
+                    }
+                }
+            }
+            Op::Store(response) => {
+                let Some(request) = requests.into_iter().next() else {
+                    return Err(PyRuntimeError::new_err(
+                        "semantic store requires one request",
+                    ));
+                };
+                let response = response.clone();
+                match prepared.into_iter().next().flatten() {
+                    Some(values) => {
+                        let backend = backend.with_embedder(PreparedEmbedding(values));
+                        let cache = Arc::new(ResponseCache::new(Arc::new(backend)));
+                        run_async(
+                            py,
+                            async move { cache.async_store(&request, response, now).await },
+                            cache_error,
+                        )?
+                    }
+                    None => {
+                        let cache = Arc::new(ResponseCache::new(backend));
+                        run_async(
+                            py,
+                            async move { cache.async_store(&request, response, now).await },
+                            cache_error,
+                        )?
+                    }
+                }
+            }
+            Op::StoreBatch(responses) => {
+                let responses = responses.clone();
+                run_async(
+                    py,
+                    async move {
+                        for ((request, response), prepared) in
+                            requests.into_iter().zip(responses).zip(prepared)
+                        {
+                            let Some(values) = prepared else {
+                                continue;
+                            };
+                            let backend = backend.with_embedder(PreparedEmbedding(values));
+                            let cache = ResponseCache::new(Arc::new(backend));
+                            cache.async_store(&request, response, now).await?;
+                        }
+                        Ok(())
+                    },
+                    cache_error,
+                )?
+            }
         };
-        let awaitable =
-            self.embedder
-                .async_embed_awaitable(py, &prompt, &self.request.context.metadata)?;
-        self.state = State::AwaitingEmbedding;
         Ok(ExecutionStep::Await(awaitable.unbind()))
     }
 
@@ -89,12 +201,10 @@ impl SemanticEmbedExecution {
             (State::Start, None) => self.start(py),
             (State::AwaitingEmbedding, Some(Ok(value))) => {
                 let values = value.bind(py).extract::<Vec<f64>>()?;
-                let backend = self.backend.with_embedder(PreparedEmbedding(
-                    values.into_iter().map(|value| value as f32).collect(),
-                ));
-                let cache = Arc::new(ResponseCache::new(Arc::new(backend)));
-                self.state = State::AwaitingStorage;
-                storage_step(py, cache, self.request.clone(), &self.op, self.now)
+                self.prepared[self.index] =
+                    Some(values.into_iter().map(|value| value as f32).collect());
+                self.index += 1;
+                self.start(py)
             }
             (State::AwaitingStorage, Some(Ok(value))) => {
                 self.state = State::Done;
@@ -118,31 +228,6 @@ impl ExecutionBody for SemanticEmbedExecution {
     }
 }
 
-fn storage_step<E: Embedder>(
-    py: Python<'_>,
-    cache: Arc<ResponseCache<ValkeySemanticCache<E, ResponseCacheCodec>>>,
-    request: ResponseCacheRequest<SemanticCacheContext>,
-    op: &Op,
-    now: Duration,
-) -> PyResult<ExecutionStep> {
-    let awaitable = match op {
-        Op::Lookup => run_async(
-            py,
-            async move { cache.async_lookup(&request, now).await },
-            cache_error,
-        )?,
-        Op::Store(response) => {
-            let response = response.clone();
-            run_async(
-                py,
-                async move { cache.async_store(&request, response, now).await },
-                cache_error,
-            )?
-        }
-    };
-    Ok(ExecutionStep::Await(awaitable.unbind()))
-}
-
 pub(super) fn drive_semantic<'py>(
     py: Python<'py>,
     body: SemanticEmbedExecution,
diff --git a/tests/test_litellm_rust/test_valkey_semantic_cache_native.py b/tests/test_litellm_rust/test_valkey_semantic_cache_native.py
index e2bd3dcb10a..fb5c5965333 100644
--- a/tests/test_litellm_rust/test_valkey_semantic_cache_native.py
+++ b/tests/test_litellm_rust/test_valkey_semantic_cache_native.py
@@ -332,11 +332,31 @@ async def test_async_store_batch_and_lookup(
         index_name,
         {"prompt A": [1.0, 0.0], "prompt B": [0.0, 1.0]},
     )
+    sync_calls: Final = []
+    async_tasks: Final = []
+
+    def sync_embedding(prompt: str, metadata: Mapping[str, object] | None = None) -> list[float]:
+        sync_calls.append(prompt)
+        return {"prompt A": [1.0, 0.0], "prompt B": [0.0, 1.0]}[prompt]
+
+    async def async_embedding(
+        prompt: str,
+        metadata: dict[str, object] | None = None,
+    ) -> list[float]:
+        async_tasks.append(asyncio.current_task())
+        return {"prompt A": [1.0, 0.0], "prompt B": [0.0, 1.0]}[prompt]
+
+    backend._get_embedding = sync_embedding
+    backend._get_async_embedding = async_embedding
     handle: Final = _native._CacheTestHandle.valkey_semantic(valkey_url, 0.8, index_name, backend)
     binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
     requests: Final = [_request("prompt A"), _request("prompt B")]
     responses: Final = [{"answer": "A"}, {"answer": "B"}]
+    caller_task: Final = asyncio.current_task()
     await binding.async_store_batch(requests, responses)
+    assert sync_calls == []
+    assert async_tasks
+    assert all(task is caller_task for task in async_tasks)
     assert await binding.async_lookup(requests[0]) == responses[0]
     assert await binding.async_lookup(requests[1]) == responses[1]
 

From 5da590a3c535e0c351ff70e8ee1fb4afb9dc5c28 Mon Sep 17 00:00:00 2001
From: ryan <ryan@berri.ai>
Date: Mon, 21 Sep 2026 22:05:44 +0000
Subject: [PATCH 126/160] chore(ui): regenerate schema.d.ts for change_password
 docstring

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 ui/litellm-dashboard/src/lib/http/schema.d.ts | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index ad8e2623d3a..93753d29993 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -17386,11 +17386,12 @@ export interface paths {
          * Change Password
          * @description Change the calling user's own password.
          *
-         *     Requires the current password. The new password must satisfy the
-         *     configured password policy (`general_settings.password_policy_*`: minimum
-         *     length, character classes, and, when enabled, breached-password screening
-         *     via haveibeenpwned.com). A successful change lifts any pending forced
-         *     password reset (`password_reset_required`) on the account.
+         *     Requires the current password. The new password must differ from the
+         *     current one and satisfy the configured password policy
+         *     (`general_settings.password_policy_*`: minimum length, character classes,
+         *     and, when enabled, breached-password screening via haveibeenpwned.com).
+         *     A successful change lifts any pending forced password reset
+         *     (`password_reset_required`) on the account.
          *
          *     Parameters:
          *     - current_password: str - The user's current password.

From 1827be1302b912ace13a56de2760a2f487025083 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 22:06:55 +0000
Subject: [PATCH 127/160] fix(cache-disk): absolutize store directory and stamp
 async expiry at write time

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/cache-disk/src/cache.rs   |  3 ++-
 litellm-rust/crates/cache-disk/src/sqlite.rs  |  1 +
 litellm-rust/crates/cache-disk/tests/cache.rs | 19 +++++++++++++++++++
 3 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/litellm-rust/crates/cache-disk/src/cache.rs b/litellm-rust/crates/cache-disk/src/cache.rs
index adff078a89b..8e1223309b4 100644
--- a/litellm-rust/crates/cache-disk/src/cache.rs
+++ b/litellm-rust/crates/cache-disk/src/cache.rs
@@ -102,9 +102,10 @@ impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> BaseCache for DiskCache<S, D,
         context: ExactCacheContext,
     ) -> Result<(), Error> {
         let value = self.adapter.write(self.codec.encode(&value)?);
-        let expire_time = context.ttl.map(|ttl| unix_now() + ttl.as_secs_f64());
+        let ttl = context.ttl;
         let key = key.to_string();
         Self::run_blocking(Arc::clone(&self.store), move |store| {
+            let expire_time = ttl.map(|ttl| unix_now() + ttl.as_secs_f64());
             store.set(&key, value, expire_time, unix_now())
         })
         .await
diff --git a/litellm-rust/crates/cache-disk/src/sqlite.rs b/litellm-rust/crates/cache-disk/src/sqlite.rs
index ff5a44b5c38..9a36f8af6ad 100644
--- a/litellm-rust/crates/cache-disk/src/sqlite.rs
+++ b/litellm-rust/crates/cache-disk/src/sqlite.rs
@@ -49,6 +49,7 @@ impl DiskcacheSqliteStore {
     pub fn open(directory: impl AsRef<Path>) -> Result<Self, Error> {
         let directory = directory.as_ref().to_path_buf();
         fs::create_dir_all(&directory).map_err(|_| Error::Unavailable)?;
+        let directory = std::path::absolute(&directory).map_err(|_| Error::Unavailable)?;
         let database = directory.join("cache.db");
         let connection = Connection::open(database).map_err(|_| Error::Unavailable)?;
         connection
diff --git a/litellm-rust/crates/cache-disk/tests/cache.rs b/litellm-rust/crates/cache-disk/tests/cache.rs
index 4fb77d233ec..dd1f2b1f04e 100644
--- a/litellm-rust/crates/cache-disk/tests/cache.rs
+++ b/litellm-rust/crates/cache-disk/tests/cache.rs
@@ -61,6 +61,25 @@ impl Sandbox {
     }
 }
 
+#[rstest]
+fn relative_store_directory_is_absolutized(sandbox: Sandbox) {
+    let relative = PathBuf::from(format!(
+        ".litellm-cache-disk-{}",
+        sandbox
+            .directory
+            .path()
+            .file_name()
+            .unwrap()
+            .to_string_lossy()
+    ));
+    let store = DiskcacheSqliteStore::open(&relative).unwrap();
+    assert!(store.directory().is_absolute());
+    assert!(store.directory().ends_with(&relative));
+    let directory = store.directory().to_path_buf();
+    drop(store);
+    fs::remove_dir_all(directory).unwrap();
+}
+
 #[derive(Clone, Copy, Debug, Default)]
 struct TextAdapter;
 

From eeaf4f36e49d507cdfe0614c9bb374d5338b571d Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 22:07:15 +0000
Subject: [PATCH 128/160] test(python-bridge): initialize the interpreter in
 the embedder seed test

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/python-bridge/src/cache/embedder.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/litellm-rust/crates/python-bridge/src/cache/embedder.rs b/litellm-rust/crates/python-bridge/src/cache/embedder.rs
index 26edb26f428..99c3de34e05 100644
--- a/litellm-rust/crates/python-bridge/src/cache/embedder.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/embedder.rs
@@ -98,6 +98,7 @@ mod tests {
 
     #[tokio::test]
     async fn async_embed_returns_the_seeded_vector_or_unavailable() {
+        Python::initialize();
         let embedder = Python::attach(|py| PythonEmbedder::new(py.None()));
         let metadata = Map::new();
         let embedder_ref = &embedder;

From a8003102b21cc9a14b62ca16726e2fc7f4974ec0 Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Mon, 21 Sep 2026 15:11:48 -0700
Subject: [PATCH 129/160] fix(auth): let jwt team_allowed_routes paths grant
 auth=true passthrough

Explicit paths and trailing-wildcard prefixes in litellm_jwtauth.team_allowed_routes
passed the JWT route check but were then denied by the auth-enforced passthrough
gates, which only read allowed_passthrough_routes from key or team metadata. Both
gates now also accept an explicit team_allowed_routes entry for tokens built by JWT
auth. Named route groups still never grant, and virtual keys, including JWT-mapped
ones, stay key-scoped
---
 litellm/proxy/auth/handle_jwt.py              |  11 +-
 litellm/proxy/auth/route_checks.py            |  33 +++-
 .../proxy/auth/test_handle_jwt.py             | 184 ++++++++++++++++++
 .../proxy/auth/test_route_checks.py           | 106 ++++++++++
 4 files changed, 330 insertions(+), 4 deletions(-)

diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py
index 996911cdfaa..2ae21bd0785 100644
--- a/litellm/proxy/auth/handle_jwt.py
+++ b/litellm/proxy/auth/handle_jwt.py
@@ -14,7 +14,7 @@ import hashlib
 import os
 import re
 import time
-from collections.abc import Awaitable, Callable, Mapping, Sequence
+from collections.abc import Awaitable, Callable, Collection, Mapping, Sequence
 from dataclasses import dataclass
 from typing import Any, Final, Literal, NoReturn, Protocol, TypeVar, cast
 
@@ -1602,6 +1602,7 @@ class JWTAuthManager:
         team_object: LiteLLM_TeamTable | None,
         route: str,
         request_method: str | None = None,
+        team_allowed_routes: Collection[str] = (),
     ) -> bool:
         normalized_request_method: Final = request_method.upper() if isinstance(request_method, str) else None
         if not RouteChecks.is_auth_enforced_pass_through_route(
@@ -1610,8 +1611,11 @@ class JWTAuthManager:
         ):
             return True
 
+        if RouteChecks.jwt_team_routes_grant_pass_through(route=route, team_allowed_routes=team_allowed_routes):
+            return True
+
         # JWT team selection is team-scoped; key metadata is not available here,
-        # so passthrough access is granted only by the selected team's metadata.
+        # so beyond the JWT config grant above, only the selected team's metadata grants access.
         return RouteChecks.check_passthrough_route_access(
             route=route,
             user_api_key_dict=UserAPIKeyAuth(team_metadata=(team_object.metadata or {}) if team_object else {}),
@@ -1689,6 +1693,7 @@ class JWTAuthManager:
                             team_object=team_object,
                             route=route,
                             request_method=request_method,
+                            team_allowed_routes=jwt_handler.litellm_jwtauth.team_allowed_routes or (),
                         ):
                             is_allowed = False
                             denied_auth_enforced_pass_through_route = True
@@ -2584,6 +2589,7 @@ class JWTAuthManager:
             team_object=team_object,
             route=route,
             request_method=request_method,
+            team_allowed_routes=handler.litellm_jwtauth.team_allowed_routes or (),
         ):
             JWTAuthManager._raise_team_passthrough_route_denial(route=route)
 
@@ -2653,6 +2659,7 @@ class JWTAuthManager:
                 team_object=team_object,
                 route=route,
                 request_method=request_method,
+                team_allowed_routes=handler.litellm_jwtauth.team_allowed_routes or (),
             ):
                 JWTAuthManager._raise_team_passthrough_route_denial(route=route)
         elif team_id is None:
diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py
index 1b9fd7c42bf..009ea986aad 100644
--- a/litellm/proxy/auth/route_checks.py
+++ b/litellm/proxy/auth/route_checks.py
@@ -268,7 +268,11 @@ class RouteChecks:
             route=route,
             method=RouteChecks._get_request_method(request=request),
         ):
-            RouteChecks._require_auth_pass_through_access(route=route, valid_token=valid_token)
+            RouteChecks._require_auth_pass_through_access(
+                route=route,
+                valid_token=valid_token,
+                jwt_team_allowed_routes=RouteChecks._jwt_team_allowed_routes(valid_token=valid_token),
+            )
         elif RouteChecks.is_llm_api_route(route=route):
             pass
         elif RouteChecks.is_info_route(route=route):
@@ -679,16 +683,41 @@ class RouteChecks:
             ),
         )
 
+    @staticmethod
+    def jwt_team_routes_grant_pass_through(route: str, team_allowed_routes: Collection[str]) -> bool:
+        """
+        Explicit paths and trailing-wildcard prefixes grant auth=true pass-through. Entries are only ever compared
+        as paths, so a named route group like ``openai_routes`` never grants.
+        """
+        return any(
+            RouteChecks.route_matches_wildcard_pattern(route=route, pattern=allowed_route)
+            for allowed_route in team_allowed_routes
+        )
+
+    @staticmethod
+    def _jwt_team_allowed_routes(valid_token: UserAPIKeyAuth) -> Collection[str]:
+        """``team_allowed_routes`` for tokens built by JWT auth; JWT-mapped virtual keys stay key-scoped."""
+        if valid_token.jwt_claims is None or valid_token.token is not None:
+            return ()
+
+        from litellm.proxy.proxy_server import jwt_handler
+
+        return jwt_handler.litellm_jwtauth.team_allowed_routes or ()
+
     @staticmethod
     def _require_auth_pass_through_access(
         route: str,
         valid_token: UserAPIKeyAuth,
+        jwt_team_allowed_routes: Collection[str] = (),
     ) -> None:
         """
-        Require an explicit ``allowed_passthrough_routes`` match for auth=true pass-through.
+        Require an explicit grant for auth=true pass-through: ``allowed_passthrough_routes`` on the
+        key or team, or an explicit JWT ``team_allowed_routes`` entry.
         """
         if RouteChecks.check_passthrough_route_access(route=route, user_api_key_dict=valid_token):
             return
+        if RouteChecks.jwt_team_routes_grant_pass_through(route=route, team_allowed_routes=jwt_team_allowed_routes):
+            return
         raise RouteChecks._auth_pass_through_denied_exception(route=route)
 
     @staticmethod
diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py
index e768139f04a..0969a913605 100644
--- a/tests/test_litellm/proxy/auth/test_handle_jwt.py
+++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py
@@ -291,6 +291,106 @@ async def test_find_team_with_model_access_uses_request_method_for_passthrough_a
     assert "allowed_passthrough_routes" in exc_info.value.detail
 
 
+_AUTH_ENFORCED_MODEL_HOST_ROUTES: Final = {
+    "test-uuid-1:subpath:/model-host/v1/extractor:GET,POST": {
+        "endpoint_id": "test-uuid-1",
+        "path": "/model-host/v1/extractor",
+        "type": "subpath",
+        "auth": True,
+    },
+}
+
+
+@pytest.mark.asyncio
+async def test_find_team_with_model_access_team_allowed_routes_wildcard_grants_auth_passthrough():
+    jwt_handler = JWTHandler()
+    jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(team_allowed_routes=["openai_routes", "/model-host/*"])
+    team_without_passthrough_allowlist = LiteLLM_TeamTable(team_id="team-a", models=["all-proxy-models"], metadata={})
+
+    with (
+        patch(
+            "litellm.proxy.auth.handle_jwt.get_team_object",
+            new_callable=AsyncMock,
+            return_value=team_without_passthrough_allowlist,
+        ),
+        patch(
+            "litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes",
+            _AUTH_ENFORCED_MODEL_HOST_ROUTES,
+        ),
+        patch("litellm.proxy.utils.get_server_root_path", return_value="/"),
+    ):
+        team_id, team_obj = await JWTAuthManager.find_team_with_model_access(
+            team_ids={"team-a"},
+            requested_model=None,
+            route="/model-host/v1/extractor/predict",
+            jwt_handler=jwt_handler,
+            prisma_client=None,
+            user_api_key_cache=MagicMock(),
+            parent_otel_span=None,
+            proxy_logging_obj=MagicMock(),
+            request_method="POST",
+        )
+
+    assert team_id == "team-a"
+    assert team_obj == team_without_passthrough_allowlist
+
+
+@pytest.mark.asyncio
+async def test_auth_builder_header_team_allows_auth_passthrough_for_team_allowed_routes_wildcard():
+    from litellm.proxy.utils import ProxyLogging
+
+    jwt_handler = JWTHandler()
+    user_api_key_cache = DualCache()
+    jwt_handler.update_environment(
+        prisma_client=None,
+        user_api_key_cache=user_api_key_cache,
+        litellm_jwtauth=LiteLLM_JWTAuth(
+            team_ids_jwt_field="groups",
+            user_id_jwt_field="sub",
+            team_allowed_routes=["openai_routes", "/model-host/*"],
+        ),
+    )
+
+    with (
+        patch.object(jwt_handler, "auth_jwt", new_callable=AsyncMock) as mock_auth_jwt,
+        patch.object(JWTAuthManager, "check_rbac_role", new_callable=AsyncMock),
+        patch.object(JWTAuthManager, "check_admin_access", new_callable=AsyncMock, return_value=None),
+        patch(
+            "litellm.proxy.auth.handle_jwt.get_team_object",
+            new_callable=AsyncMock,
+            return_value=LiteLLM_TeamTable(team_id="team-2", metadata={}),
+        ),
+        patch.object(
+            JWTAuthManager,
+            "get_objects",
+            new_callable=AsyncMock,
+            return_value=(None, None, None, None, "user-1"),
+        ),
+        patch(
+            "litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes",
+            _AUTH_ENFORCED_MODEL_HOST_ROUTES,
+        ),
+        patch("litellm.proxy.utils.get_server_root_path", return_value="/"),
+    ):
+        mock_auth_jwt.return_value = {"sub": "user-1", "scope": "", "groups": ["team-1", "team-2"]}
+
+        result = await JWTAuthManager.auth_builder(
+            api_key="jwt-token",
+            jwt_handler=jwt_handler,
+            request_data={},
+            general_settings={},
+            route="/model-host/v1/extractor/predict",
+            prisma_client=None,
+            user_api_key_cache=user_api_key_cache,
+            parent_otel_span=None,
+            proxy_logging_obj=ProxyLogging(user_api_key_cache=user_api_key_cache),
+            request_headers={"x-litellm-team-id": "team-2"},
+            request_method="POST",
+        )
+
+    assert result["team_id"] == "team-2"
+
+
 @pytest.mark.asyncio
 async def test_auth_builder_proxy_admin_user_role():
     """Test that is_proxy_admin is True when user_object.user_role is PROXY_ADMIN"""
@@ -6463,6 +6563,90 @@ async def test_auth_builder_db_fallback_enforces_passthrough_route_access():
     assert "passthrough route" in exc_info.value.detail
 
 
+async def _auth_builder_via_db_team_fallback(team_allowed_routes: list[str]):
+    user_id = "u_passthrough"
+    user_object = LiteLLM_UserTable(
+        user_id=user_id,
+        user_role=LitellmUserRoles.INTERNAL_USER,
+        teams=["team_no_passthrough"],
+    )
+    jwt_handler = JWTHandler()
+    jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(fallback_to_db_teams=True, team_allowed_routes=team_allowed_routes)
+
+    async def fake_get_team(team_id, **kwargs):
+        return LiteLLM_TeamTable(team_id=team_id, metadata={})
+
+    with (
+        patch.object(jwt_handler, "auth_jwt", new_callable=AsyncMock, return_value={"sub": user_id, "scope": ""}),
+        patch.object(JWTAuthManager, "check_rbac_role", new_callable=AsyncMock),
+        patch.object(jwt_handler, "get_rbac_role", return_value=None),
+        patch.object(jwt_handler, "get_scopes", return_value=[]),
+        patch.object(jwt_handler, "get_object_id", return_value=None),
+        patch.object(
+            JWTAuthManager,
+            "get_user_info",
+            new_callable=AsyncMock,
+            return_value=(user_id, "u@example.com", True),
+        ),
+        patch.object(jwt_handler, "get_org_id", return_value=None),
+        patch.object(jwt_handler, "get_end_user_id", return_value=None),
+        patch.object(JWTAuthManager, "check_admin_access", new_callable=AsyncMock, return_value=None),
+        patch.object(
+            JWTAuthManager,
+            "get_objects",
+            new_callable=AsyncMock,
+            return_value=(user_object, None, None, None, user_id),
+        ),
+        patch.object(JWTAuthManager, "map_user_to_teams", new_callable=AsyncMock),
+        patch.object(JWTAuthManager, "validate_object_id", return_value=True),
+        patch.object(JWTAuthManager, "sync_user_role_and_teams", new_callable=AsyncMock),
+        patch(
+            "litellm.proxy.auth.handle_jwt.get_team_object",
+            new_callable=AsyncMock,
+            side_effect=fake_get_team,
+        ),
+        patch(
+            "litellm.proxy.auth.handle_jwt.get_team_membership",
+            new_callable=AsyncMock,
+            return_value=None,
+        ),
+        patch(
+            "litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes",
+            _AUTH_ENFORCED_MODEL_HOST_ROUTES,
+        ),
+        patch("litellm.proxy.utils.get_server_root_path", return_value="/"),
+    ):
+        return await JWTAuthManager.auth_builder(
+            api_key="test_jwt_token",
+            jwt_handler=jwt_handler,
+            request_data={},
+            general_settings={"enforce_rbac": False},
+            route="/model-host/v1/extractor/predict",
+            prisma_client=None,
+            user_api_key_cache=None,
+            parent_otel_span=None,
+            proxy_logging_obj=None,
+            request_headers=None,
+            request_method="POST",
+        )
+
+
+@pytest.mark.asyncio
+async def test_auth_builder_db_fallback_team_allowed_routes_wildcard_grants_auth_passthrough():
+    result = await _auth_builder_via_db_team_fallback(team_allowed_routes=["openai_routes", "/model-host/*"])
+
+    assert result["team_id"] == "team_no_passthrough"
+
+
+@pytest.mark.asyncio
+async def test_auth_builder_db_fallback_route_groups_alone_do_not_grant_auth_passthrough():
+    with pytest.raises(HTTPException) as exc_info:
+        await _auth_builder_via_db_team_fallback(team_allowed_routes=["openai_routes", "mapped_pass_through_routes"])
+
+    assert exc_info.value.status_code == 403, exc_info.value.detail
+    assert "allowed_passthrough_routes" in exc_info.value.detail
+
+
 @pytest.mark.asyncio
 async def test_sync_user_role_and_teams_singular_claim_reconciles_memberships():
     """When fallback_to_db_teams is on but the JWT carries a singular team claim
diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py
index 87bf4595af5..2a92f3d7fbe 100644
--- a/tests/test_litellm/proxy/auth/test_route_checks.py
+++ b/tests/test_litellm/proxy/auth/test_route_checks.py
@@ -1263,6 +1263,112 @@ def test_non_proxy_admin_allows_auth_pass_through_with_team_allowlist():
         )
 
 
+@pytest.mark.parametrize(
+    "route, team_allowed_routes, expected",
+    [
+        ("/model-host/v1/extractor/predict", ["/model-host/*"], True),
+        ("/model-host", ["/model-host/*"], False),
+        ("/model-host/v1/extractor", ["/model-host/v1/extractor"], True),
+        ("/model-host/v1/extractor/predict", ["/model-host/v1/extractor"], False),
+        ("/other/v1/extractor", ["/model-host/*"], False),
+        ("/model-host/v1/extractor", ["openai_routes", "llm_api_routes", "mapped_pass_through_routes"], False),
+        ("/model-host/v1/extractor", [], False),
+    ],
+)
+def test_jwt_team_routes_grant_pass_through_only_for_explicit_paths(route, team_allowed_routes, expected):
+    assert (
+        RouteChecks.jwt_team_routes_grant_pass_through(route=route, team_allowed_routes=team_allowed_routes)
+        is expected
+    )
+
+
+_AUTH_ENFORCED_MODEL_HOST_ROUTES: Final = {
+    "test-uuid-1:subpath:/model-host/v1/extractor:GET,POST": {
+        "endpoint_id": "test-uuid-1",
+        "path": "/model-host/v1/extractor",
+        "type": "subpath",
+        "auth": True,
+    },
+}
+
+
+def _jwt_handler_with_team_allowed_routes(team_allowed_routes: list[str]):
+    from litellm.proxy._types import LiteLLM_JWTAuth
+    from litellm.proxy.auth.handle_jwt import JWTHandler
+
+    jwt_handler: Final = JWTHandler()
+    jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(team_allowed_routes=team_allowed_routes)
+    return jwt_handler
+
+
+def _check_model_host_route_as(valid_token: UserAPIKeyAuth, team_allowed_routes: list[str]) -> None:
+    with (
+        patch(
+            "litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes",
+            _AUTH_ENFORCED_MODEL_HOST_ROUTES,
+        ),
+        patch("litellm.proxy.utils.get_server_root_path", return_value="/"),
+        patch(
+            "litellm.proxy.proxy_server.jwt_handler",
+            _jwt_handler_with_team_allowed_routes(team_allowed_routes),
+        ),
+    ):
+        RouteChecks.non_proxy_admin_allowed_routes_check(
+            user_obj=None,
+            _user_role=LitellmUserRoles.INTERNAL_USER.value,
+            route="/model-host/v1/extractor/predict",
+            request=MagicMock(spec=Request),
+            valid_token=valid_token,
+            request_data={},
+        )
+
+
+def test_non_proxy_admin_allows_auth_pass_through_for_jwt_team_allowed_routes_wildcard():
+    jwt_token: Final = UserAPIKeyAuth(
+        user_id="test_user",
+        user_role=LitellmUserRoles.INTERNAL_USER.value,
+        team_id="team-a",
+        jwt_claims={"sub": "test_user"},
+    )
+
+    _check_model_host_route_as(jwt_token, team_allowed_routes=["openai_routes", "/model-host/*"])
+
+
+def test_non_proxy_admin_denies_auth_pass_through_for_jwt_when_only_route_groups_configured():
+    jwt_token: Final = UserAPIKeyAuth(
+        user_id="test_user",
+        user_role=LitellmUserRoles.INTERNAL_USER.value,
+        team_id="team-a",
+        jwt_claims={"sub": "test_user"},
+    )
+
+    with pytest.raises(HTTPException) as exc_info:
+        _check_model_host_route_as(jwt_token, team_allowed_routes=["openai_routes", "mapped_pass_through_routes"])
+
+    assert exc_info.value.status_code == 403, exc_info.value.detail
+    assert "allowed_passthrough_routes" in exc_info.value.detail
+
+
+@pytest.mark.parametrize(
+    "jwt_claims",
+    [None, {"sub": "test_user"}],
+    ids=["plain_virtual_key", "jwt_mapped_virtual_key"],
+)
+def test_non_proxy_admin_jwt_team_allowed_routes_never_grant_pass_through_to_virtual_keys(jwt_claims):
+    virtual_key: Final = UserAPIKeyAuth(
+        api_key="sk-test-key",
+        user_id="test_user",
+        user_role=LitellmUserRoles.INTERNAL_USER.value,
+        jwt_claims=jwt_claims,
+    )
+
+    with pytest.raises(HTTPException) as exc_info:
+        _check_model_host_route_as(virtual_key, team_allowed_routes=["openai_routes", "/model-host/*"])
+
+    assert exc_info.value.status_code == 403, exc_info.value.detail
+    assert "allowed_passthrough_routes" in exc_info.value.detail
+
+
 def test_virtual_key_without_llm_api_routes_cannot_access_pass_through():
     """
     Test that virtual keys without llm_api_routes permission cannot access registered pass-through endpoints.

From 2a3ae253d49f0a5147ff2f571d9a8b601f21de38 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 22:14:22 +0000
Subject: [PATCH 130/160] refactor(rust): fold disk arms into shared match
 patterns

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../crates/python-bridge/src/cache/native.rs      | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs
index bbd72e58825..80789cc279a 100644
--- a/litellm-rust/crates/python-bridge/src/cache/native.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/native.rs
@@ -99,35 +99,29 @@ impl NativeResponseCache {
 
     pub fn namespace(&self) -> Option<&str> {
         match self {
-            Self::Memory(_) | Self::AzureBlob(_) => None,
+            Self::Memory(_) | Self::Disk(_) | Self::AzureBlob(_) => None,
             Self::Redis { cache, .. } => cache.backend().namespace(),
-            Self::Disk(_) => None,
         }
     }
 
     pub fn topology(&self) -> Option<&RedisTopology> {
         match self {
-            Self::Memory(_) | Self::AzureBlob(_) => None,
+            Self::Memory(_) | Self::Disk(_) | Self::AzureBlob(_) => None,
             Self::Redis { cache, .. } => Some(cache.backend().topology()),
-            Self::Disk(_) => None,
         }
     }
 
     pub fn capacity(&self) -> Option<usize> {
         match self {
             Self::Memory(cache) => Some(cache.backend().max_size_in_memory()),
-            Self::Redis { .. } => None,
-            Self::Disk(_) => None,
-            Self::AzureBlob(_) => None,
+            Self::Redis { .. } | Self::Disk(_) | Self::AzureBlob(_) => None,
         }
     }
 
     pub fn max_entry_bytes(&self) -> Option<usize> {
         match self {
             Self::Memory(cache) => cache.backend().max_entry_bytes(),
-            Self::Redis { .. } => None,
-            Self::Disk(_) => None,
-            Self::AzureBlob(_) => None,
+            Self::Redis { .. } | Self::Disk(_) | Self::AzureBlob(_) => None,
         }
     }
 
@@ -137,7 +131,6 @@ impl NativeResponseCache {
                 cache,
                 buffer: flush_size.map(|flush_size| Arc::new(WriteBuffer::new(flush_size))),
             },
-            disk @ Self::Disk(_) => disk,
             other => other,
         }
     }

From a286106f4fa6a3c76a58cab869931cafc9592273 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 22:17:11 +0000
Subject: [PATCH 131/160] fix(rust): isolate explicit s3 keys from env tokens
 and treat 403 misses

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/cache-s3/src/auth.rs    | 75 ++++++++++++++++++++-
 litellm-rust/crates/cache-s3/src/cache.rs   | 10 +--
 litellm-rust/crates/cache-s3/tests/cache.rs |  8 +++
 litellm/rust_bridge/_native.pyi             |  2 +
 tests/test_litellm_rust/support/s3_stub.py  |  8 ++-
 tests/test_litellm_rust/test_cache.py       | 30 ++++++---
 6 files changed, 113 insertions(+), 20 deletions(-)

diff --git a/litellm-rust/crates/cache-s3/src/auth.rs b/litellm-rust/crates/cache-s3/src/auth.rs
index 8f7eb66059d..b7ca722cea3 100644
--- a/litellm-rust/crates/cache-s3/src/auth.rs
+++ b/litellm-rust/crates/cache-s3/src/auth.rs
@@ -1,14 +1,22 @@
-use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future};
+use aws_credential_types::{
+    Credentials as AwsCredentials,
+    provider::{ProvideCredentials, error::CredentialsError, future},
+};
 use litellm_auth_aws::{AwsAuthConfig, resolve_credentials};
 
 #[derive(Clone)]
 pub(crate) struct Credentials {
     config: AwsAuthConfig,
+    env: fn(&str) -> Option<String>,
 }
 
 impl Credentials {
     pub(crate) fn new(config: AwsAuthConfig) -> Self {
-        Self { config }
+        Self::with_env(config, |name| std::env::var(name).ok())
+    }
+
+    pub(crate) fn with_env(config: AwsAuthConfig, env: fn(&str) -> Option<String>) -> Self {
+        Self { config, env }
     }
 }
 
@@ -18,7 +26,19 @@ impl ProvideCredentials for Credentials {
         Self: 'a,
     {
         future::ProvideCredentials::new(async {
-            resolve_credentials(self.config.clone(), &|name| std::env::var(name).ok())
+            if let (Some(access_key_id), Some(secret_access_key)) = (
+                self.config.access_key_id.clone(),
+                self.config.secret_access_key.clone(),
+            ) {
+                return Ok(AwsCredentials::new(
+                    access_key_id,
+                    secret_access_key,
+                    self.config.session_token.clone(),
+                    None,
+                    "litellm-s3-cache",
+                ));
+            }
+            resolve_credentials(self.config.clone(), &self.env)
                 .await
                 .map_err(|_| CredentialsError::provider_error("S3 cache authentication failed"))
         })
@@ -30,3 +50,52 @@ impl std::fmt::Debug for Credentials {
         f.debug_struct("Credentials").finish_non_exhaustive()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn explicit_keys_ignore_an_ambient_session_token() {
+        let provider = Credentials::with_env(
+            AwsAuthConfig {
+                access_key_id: Some("key".to_string()),
+                secret_access_key: Some("secret".to_string()),
+                region_name: Some("us-east-1".to_string()),
+                ..Default::default()
+            },
+            |name| (name == "AWS_SESSION_TOKEN").then(|| "ambient".to_string()),
+        );
+        let credentials = provider.provide_credentials().await.unwrap();
+        assert_eq!(credentials.access_key_id(), "key");
+        assert_eq!(credentials.secret_access_key(), "secret");
+        assert_eq!(credentials.session_token(), None);
+    }
+
+    #[tokio::test]
+    async fn explicit_keys_keep_their_session_token() {
+        let provider = Credentials::new(AwsAuthConfig {
+            access_key_id: Some("key".to_string()),
+            secret_access_key: Some("secret".to_string()),
+            session_token: Some("t".to_string()),
+            region_name: Some("us-east-1".to_string()),
+            ..Default::default()
+        });
+        let credentials = provider.provide_credentials().await.unwrap();
+        assert_eq!(credentials.session_token(), Some("t"));
+    }
+
+    #[tokio::test]
+    async fn environment_keys_resolve_with_their_session_token() {
+        let provider = Credentials::with_env(AwsAuthConfig::default(), |name| match name {
+            "AWS_ACCESS_KEY_ID" => Some("env-key".to_string()),
+            "AWS_SECRET_ACCESS_KEY" => Some("env-secret".to_string()),
+            "AWS_SESSION_TOKEN" => Some("env-token".to_string()),
+            _ => None,
+        });
+        let credentials = provider.provide_credentials().await.unwrap();
+        assert_eq!(credentials.access_key_id(), "env-key");
+        assert_eq!(credentials.secret_access_key(), "env-secret");
+        assert_eq!(credentials.session_token(), Some("env-token"));
+    }
+}
diff --git a/litellm-rust/crates/cache-s3/src/cache.rs b/litellm-rust/crates/cache-s3/src/cache.rs
index 78e5f39dbcc..9c791f42c3b 100644
--- a/litellm-rust/crates/cache-s3/src/cache.rs
+++ b/litellm-rust/crates/cache-s3/src/cache.rs
@@ -134,11 +134,13 @@ impl<C: CacheCodec> S3Cache<C> {
             Ok(output) => output,
             Err(error) => {
                 if let SdkError::ServiceError(service) = &error {
+                    let status = error
+                        .raw_response()
+                        .map(|response| response.status().as_u16());
                     let not_found = service.err().is_no_such_key()
-                        || error
-                            .raw_response()
-                            .map(|response| response.status().as_u16())
-                            == Some(404);
+                        || service.err().meta().code() == Some("AccessDenied")
+                        || status == Some(404)
+                        || status == Some(403);
                     if not_found {
                         return Ok(None);
                     }
diff --git a/litellm-rust/crates/cache-s3/tests/cache.rs b/litellm-rust/crates/cache-s3/tests/cache.rs
index 6ca3eb9a758..9a71656286b 100644
--- a/litellm-rust/crates/cache-s3/tests/cache.rs
+++ b/litellm-rust/crates/cache-s3/tests/cache.rs
@@ -125,6 +125,13 @@ async fn get_hit_miss_expired_and_invalid_entries() {
         )
         .mount(&server)
         .await;
+    Mock::given(method("GET"))
+        .and(path("/cache-bucket/team/denied"))
+        .respond_with(
+            ResponseTemplate::new(403).set_body_string("<Error><Code>AccessDenied</Code></Error>"),
+        )
+        .mount(&server)
+        .await;
     Mock::given(method("GET"))
         .and(path("/cache-bucket/team/expired"))
         .respond_with(
@@ -147,6 +154,7 @@ async fn get_hit_miss_expired_and_invalid_entries() {
         Some(json!({"answer": 3}))
     );
     assert_eq!(cache.get_cache("missing", &context).unwrap(), None);
+    assert_eq!(cache.get_cache("denied", &context).unwrap(), None);
     assert_eq!(cache.get_cache("expired", &context).unwrap(), None);
     assert_eq!(
         cache.get_cache("malformed", &context),
diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi
index 6ad0e6be26e..3f747dadc56 100644
--- a/litellm/rust_bridge/_native.pyi
+++ b/litellm/rust_bridge/_native.pyi
@@ -165,6 +165,8 @@ class _CacheTestHandle:
         startup_nodes: Sequence[tuple[str, int]] | None = None,
     ) -> _CacheTestHandle: ...
     @staticmethod
+    def azure_blob(account_url: str, container: str) -> _CacheTestHandle: ...
+    @staticmethod
     def s3(
         bucket: str,
         *,
diff --git a/tests/test_litellm_rust/support/s3_stub.py b/tests/test_litellm_rust/support/s3_stub.py
index 18863ad8083..7e5d068b975 100644
--- a/tests/test_litellm_rust/support/s3_stub.py
+++ b/tests/test_litellm_rust/support/s3_stub.py
@@ -51,7 +51,7 @@ class S3Stub:
             def do_PUT(self) -> None:
                 body: Final = self._read_body()
                 headers: Final = {name: self.headers[name] for name in _STORED_HEADERS if name in self.headers}
-                stub._objects[self._key()] = S3Object(body=body, headers=headers)
+                stub._objects = {**stub._objects, self._key(): S3Object(body=body, headers=headers)}
                 self.send_response(200)
                 self.send_header("ETag", '"stub"')
                 self.send_header("Content-Length", "0")
@@ -83,7 +83,9 @@ class S3Stub:
                 if send_body:
                     self.wfile.write(entry.body)
 
-            def log_message(self, format: str, *args: object) -> None:  # noqa: A002
+            def log_message(
+                self, format: str, *args: object
+            ) -> None:  # BaseHTTPRequestHandler.log_message names this parameter format
                 pass
 
         self._server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
@@ -100,7 +102,7 @@ class S3Stub:
         return self._objects
 
     def put_object(self, key: str, body: bytes, headers: dict[str, str] | None = None) -> None:
-        self._objects[key] = S3Object(body=body, headers=headers or {})
+        self._objects = {**self._objects, key: S3Object(body=body, headers=headers or {})}
 
     def expires(self, key: str) -> object:
         header: Final = self._objects[key].headers.get("expires")
diff --git a/tests/test_litellm_rust/test_cache.py b/tests/test_litellm_rust/test_cache.py
index cfa8e95703c..2c98f0ca6d3 100644
--- a/tests/test_litellm_rust/test_cache.py
+++ b/tests/test_litellm_rust/test_cache.py
@@ -11,6 +11,7 @@ from collections.abc import Generator
 from datetime import datetime
 from types import SimpleNamespace
 from typing import Final, Protocol, cast
+from unittest.mock import Mock
 from urllib.parse import urlparse
 
 import boto3
@@ -24,8 +25,8 @@ import litellm
 from litellm.caching.azure_blob_cache import AzureBlobCache
 from litellm.caching.caching import Cache, disable_cache, enable_cache, update_cache
 from litellm.caching.in_memory_cache import InMemoryCache
-from litellm.caching.s3_cache import S3Cache
 from litellm.caching.redis_cluster_cache import RedisClusterCache
+from litellm.caching.s3_cache import S3Cache
 from litellm.rust_bridge import _native
 from litellm.types.caching import LiteLLMCacheType
 from tests.test_litellm_rust.support.isolation import rebound
@@ -416,15 +417,20 @@ def test_azure_blob_facade_serves_natively_and_python_reads_the_same_blobs(azure
     assert handle.backend == "azure-blob"
     account_url: Final = backend.container_client.url.removesuffix(f"/{backend.container_client.container_name}")
     with pytest.raises(TypeError, match="containers must match"):
-        _native._CacheTestHandle.azure_blob(account_url, f"{backend.container_client.container_name}-other")._bind_facade(
-            azure_blob_facade
-        )
+        _native._CacheTestHandle.azure_blob(
+            account_url, f"{backend.container_client.container_name}-other"
+        )._bind_facade(azure_blob_facade)
     handle._bind_facade(azure_blob_facade)
     resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=azure_blob_facade))
     native: Final = resolver.resolve()
     assert native.kind == "native"
 
-    response: Final = {"choices": [{"text": "caf\u00e9 \u2603"}], "usage": {"total_tokens": 3}, "flag": True, "empty": None}
+    response: Final = {
+        "choices": [{"text": "caf\u00e9 \u2603"}],
+        "usage": {"total_tokens": 3},
+        "flag": True,
+        "empty": None,
+    }
     native.store({**request("sync"), "ttl_seconds": 0.001}, response)
     native.store(request("sync"), {"choices": [{"text": "second"}]})
     time.sleep(0.01)
@@ -479,7 +485,9 @@ async def test_azure_blob_native_async_writes_overwrite_batch_and_flush_like_pyt
     await binding.async_store({**request("async"), "ttl_seconds": 0.001}, {"value": 2})
     time.sleep(0.01)
     assert await binding.async_lookup(request("async")) == {"value": 2}
-    assert await backend.async_get_cache("async") == json.loads(backend.container_client.download_blob("async").readall())
+    assert await backend.async_get_cache("async") == json.loads(
+        backend.container_client.download_blob("async").readall()
+    )
     assert cast(CacheLookup, azure_blob_facade).get_cache(cache_key="async") == {"value": 2}
 
     await binding.async_store_batch([request("first"), request("second")], [{"value": 3}, {"value": 4}])
@@ -624,11 +632,11 @@ def test_s3_facade_binds_only_exact_configuration_and_falls_back_on_mutation(s3_
     binding: Final = resolver.resolve()
     assert binding.kind == "native"
 
-    calls: Final = []
-    facade.cache.s3_client.meta.events.register("before-call.s3.*", lambda **_kwargs: calls.append(1))
+    handler: Final = Mock()
+    facade.cache.s3_client.meta.events.register("before-call.s3.*", handler)
     binding.store(request("native"), {"answer": 1})
     assert binding.lookup(request("native")) == {"answer": 1}
-    assert calls == []
+    assert handler.call_count == 0
     assert "team/native" in s3_stub.objects
 
     with rebound(facade.cache, "bucket_name", "other"):
@@ -756,7 +764,9 @@ async def test_redis_cluster_facade_serves_multi_slot_batches_and_scoped_flush_n
 
     await binding.async_flush()
 
-    remaining: Final = tuple(sorted(key for node in client.get_primaries() for key in client.keys("parity:*", target_nodes=node)))
+    remaining: Final = tuple(
+        sorted(key for node in client.get_primaries() for key in client.keys("parity:*", target_nodes=node))
+    )
     assert remaining == (), remaining
     assert client.get("unscoped") == b"stays"
     client.delete("unscoped")

From 02cc4da25260890c16a4ea60e4c3424601db41d7 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 22:18:18 +0000
Subject: [PATCH 132/160] test(rust): keep s3 parity test edits scoped to the
 mock handler change

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 tests/test_litellm_rust/support/s3_stub.py |  4 +---
 tests/test_litellm_rust/test_cache.py      | 23 +++++++---------------
 2 files changed, 8 insertions(+), 19 deletions(-)

diff --git a/tests/test_litellm_rust/support/s3_stub.py b/tests/test_litellm_rust/support/s3_stub.py
index 7e5d068b975..5a683fb78f3 100644
--- a/tests/test_litellm_rust/support/s3_stub.py
+++ b/tests/test_litellm_rust/support/s3_stub.py
@@ -83,9 +83,7 @@ class S3Stub:
                 if send_body:
                     self.wfile.write(entry.body)
 
-            def log_message(
-                self, format: str, *args: object
-            ) -> None:  # BaseHTTPRequestHandler.log_message names this parameter format
+            def log_message(self, format: str, *args: object) -> None:
                 pass
 
         self._server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
diff --git a/tests/test_litellm_rust/test_cache.py b/tests/test_litellm_rust/test_cache.py
index 2c98f0ca6d3..04bca2d5dd3 100644
--- a/tests/test_litellm_rust/test_cache.py
+++ b/tests/test_litellm_rust/test_cache.py
@@ -25,8 +25,8 @@ import litellm
 from litellm.caching.azure_blob_cache import AzureBlobCache
 from litellm.caching.caching import Cache, disable_cache, enable_cache, update_cache
 from litellm.caching.in_memory_cache import InMemoryCache
-from litellm.caching.redis_cluster_cache import RedisClusterCache
 from litellm.caching.s3_cache import S3Cache
+from litellm.caching.redis_cluster_cache import RedisClusterCache
 from litellm.rust_bridge import _native
 from litellm.types.caching import LiteLLMCacheType
 from tests.test_litellm_rust.support.isolation import rebound
@@ -417,20 +417,15 @@ def test_azure_blob_facade_serves_natively_and_python_reads_the_same_blobs(azure
     assert handle.backend == "azure-blob"
     account_url: Final = backend.container_client.url.removesuffix(f"/{backend.container_client.container_name}")
     with pytest.raises(TypeError, match="containers must match"):
-        _native._CacheTestHandle.azure_blob(
-            account_url, f"{backend.container_client.container_name}-other"
-        )._bind_facade(azure_blob_facade)
+        _native._CacheTestHandle.azure_blob(account_url, f"{backend.container_client.container_name}-other")._bind_facade(
+            azure_blob_facade
+        )
     handle._bind_facade(azure_blob_facade)
     resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=azure_blob_facade))
     native: Final = resolver.resolve()
     assert native.kind == "native"
 
-    response: Final = {
-        "choices": [{"text": "caf\u00e9 \u2603"}],
-        "usage": {"total_tokens": 3},
-        "flag": True,
-        "empty": None,
-    }
+    response: Final = {"choices": [{"text": "caf\u00e9 \u2603"}], "usage": {"total_tokens": 3}, "flag": True, "empty": None}
     native.store({**request("sync"), "ttl_seconds": 0.001}, response)
     native.store(request("sync"), {"choices": [{"text": "second"}]})
     time.sleep(0.01)
@@ -485,9 +480,7 @@ async def test_azure_blob_native_async_writes_overwrite_batch_and_flush_like_pyt
     await binding.async_store({**request("async"), "ttl_seconds": 0.001}, {"value": 2})
     time.sleep(0.01)
     assert await binding.async_lookup(request("async")) == {"value": 2}
-    assert await backend.async_get_cache("async") == json.loads(
-        backend.container_client.download_blob("async").readall()
-    )
+    assert await backend.async_get_cache("async") == json.loads(backend.container_client.download_blob("async").readall())
     assert cast(CacheLookup, azure_blob_facade).get_cache(cache_key="async") == {"value": 2}
 
     await binding.async_store_batch([request("first"), request("second")], [{"value": 3}, {"value": 4}])
@@ -764,9 +757,7 @@ async def test_redis_cluster_facade_serves_multi_slot_batches_and_scoped_flush_n
 
     await binding.async_flush()
 
-    remaining: Final = tuple(
-        sorted(key for node in client.get_primaries() for key in client.keys("parity:*", target_nodes=node))
-    )
+    remaining: Final = tuple(sorted(key for node in client.get_primaries() for key in client.keys("parity:*", target_nodes=node)))
     assert remaining == (), remaining
     assert client.get("unscoped") == b"stays"
     client.delete("unscoped")

From 8a1c6e49fb97d44528a451b2cf0c266468ac256d Mon Sep 17 00:00:00 2001
From: ryan <ryan@berri.ai>
Date: Mon, 21 Sep 2026 22:22:12 +0000
Subject: [PATCH 133/160] feat(auth): only allow password-login dashboard
 sessions to call /user/password/change

Password login now stamps login_method=username_password into the UI session key
metadata, and change_password rejects any caller that is not a litellm-dashboard
key carrying that marker with 403 before the user row is read. SSO sessions and
user-associated virtual keys can no longer use the endpoint as a current_password
guessing oracle. The forced-reset session is still minted by the password login
path, so it keeps access to the endpoint

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/auth/login_utils.py             |  6 ++-
 .../password_endpoints.py                     | 23 ++++++++-
 .../proxy/auth/test_login_utils.py            |  6 +--
 .../test_password_endpoints.py                | 49 ++++++++++++++++++-
 ui/litellm-dashboard/src/lib/http/schema.d.ts |  4 +-
 5 files changed, 80 insertions(+), 8 deletions(-)

diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py
index 0dddb16b531..4c2b5d3d0fe 100644
--- a/litellm/proxy/auth/login_utils.py
+++ b/litellm/proxy/auth/login_utils.py
@@ -58,6 +58,7 @@ if TYPE_CHECKING:
 
 BREACH_RECHECK_INTERVAL: Final = timedelta(hours=24)
 PASSWORD_RESET_ALLOWED_ROUTES: Final = ("/user/password/change",)
+PASSWORD_SESSION_METADATA: Final = MappingProxyType({"login_method": "username_password"})
 
 
 def _breach_recheck_due(last_breach_check_at: datetime | None) -> bool:
@@ -431,7 +432,10 @@ async def _sign_in(
                     user_id=user_id,
                     team_id="litellm-dashboard",
                     allowed_routes=list(PASSWORD_RESET_ALLOWED_ROUTES) if password_reset_required else None,
-                    metadata={"password_reset_required": True} if password_reset_required else {},
+                    metadata={
+                        **PASSWORD_SESSION_METADATA,
+                        **({"password_reset_required": True} if password_reset_required else {}),
+                    },
                 )
             else:
                 raise ProxyException(
diff --git a/litellm/proxy/management_endpoints/password_endpoints.py b/litellm/proxy/management_endpoints/password_endpoints.py
index c94399ba25d..03a8b4c4010 100644
--- a/litellm/proxy/management_endpoints/password_endpoints.py
+++ b/litellm/proxy/management_endpoints/password_endpoints.py
@@ -11,9 +11,11 @@ signal is emitted by hand below, with field names only, never values.
 from typing import TYPE_CHECKING, Annotated, Final
 
 from fastapi import APIRouter, Depends, HTTPException
+from pydantic import TypeAdapter
 
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import (
+    UI_TEAM_ID,
     ChangePasswordRequest,
     ChangePasswordResponse,
     CommonProxyErrors,
@@ -21,6 +23,7 @@ from litellm.proxy._types import (
     LitellmTableNames,
     UserAPIKeyAuth,
 )
+from litellm.proxy.auth.login_utils import PASSWORD_SESSION_METADATA
 from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.management_helpers.audit_logs import create_object_audit_log
@@ -37,6 +40,7 @@ if TYPE_CHECKING:
 router: Final = APIRouter()
 
 _PASSWORD_CHANGED_AUDIT_VALUES: Final = '{"fields_changed": ["password"]}'
+_KEY_METADATA: Final = TypeAdapter(dict[str, object])
 
 
 def _error_detail(message: str) -> HTTPExceptionErrorDetail:
@@ -44,6 +48,13 @@ def _error_detail(message: str) -> HTTPExceptionErrorDetail:
     return detail
 
 
+def _is_password_login_session(user_api_key_dict: UserAPIKeyAuth) -> bool:
+    if user_api_key_dict.team_id != UI_TEAM_ID:
+        return False
+    key_metadata: Final = _KEY_METADATA.validate_python(user_api_key_dict.metadata)
+    return all(key_metadata.get(k) == v for k, v in PASSWORD_SESSION_METADATA.items())
+
+
 def _user_table(
     prisma_client: "PrismaClient | None",
 ) -> "TableActions[prisma_models.LiteLLM_UserTable]":
@@ -63,7 +74,9 @@ async def change_password(
     """
     Change the calling user's own password.
 
-    Requires the current password. The new password must differ from the
+    Only callable with the dashboard session issued by a username/password
+    login; SSO sessions and virtual keys are rejected with 403. Requires the
+    current password. The new password must differ from the
     current one and satisfy the configured password policy
     (`general_settings.password_policy_*`: minimum length, character classes,
     and, when enabled, breached-password screening via haveibeenpwned.com).
@@ -82,6 +95,14 @@ async def change_password(
             detail=_error_detail(CommonProxyErrors.db_not_connected_error.value),
         )
 
+    if not _is_password_login_session(user_api_key_dict):
+        raise HTTPException(
+            status_code=403,
+            detail=_error_detail(
+                "Passwords can only be changed from a dashboard session created by logging in with a password."
+            ),
+        )
+
     user_id: Final = user_api_key_dict.user_id
     if user_id is None:
         raise HTTPException(
diff --git a/tests/test_litellm/proxy/auth/test_login_utils.py b/tests/test_litellm/proxy/auth/test_login_utils.py
index ada231a3e95..3786169c320 100644
--- a/tests/test_litellm/proxy/auth/test_login_utils.py
+++ b/tests/test_litellm/proxy/auth/test_login_utils.py
@@ -2138,7 +2138,7 @@ class TestPasswordResetRequiredSessionMinting:
         result, key_kwargs = await self._login(_prisma_with_user(row))
 
         assert key_kwargs["allowed_routes"] == ["/user/password/change"]
-        assert key_kwargs["metadata"] == {"password_reset_required": True}
+        assert key_kwargs["metadata"] == {"login_method": "username_password", "password_reset_required": True}
         assert result.password_reset_required is True
 
     @pytest.mark.asyncio
@@ -2147,7 +2147,7 @@ class TestPasswordResetRequiredSessionMinting:
         result, key_kwargs = await self._login(_prisma_with_user(row))
 
         assert key_kwargs["allowed_routes"] is None
-        assert not key_kwargs["metadata"]
+        assert key_kwargs["metadata"] == {"login_method": "username_password"}
         assert result.password_reset_required is False
 
     async def _login_with_screen_result(self, mock_prisma_client, breached: bool) -> tuple[LoginResult, dict, dict]:
@@ -2199,7 +2199,7 @@ class TestPasswordResetRequiredSessionMinting:
         result, key_kwargs, _ = await self._login_with_screen_result(mock_prisma_client, breached=True)
 
         assert key_kwargs["allowed_routes"] == ["/user/password/change"]
-        assert key_kwargs["metadata"] == {"password_reset_required": True}
+        assert key_kwargs["metadata"] == {"login_method": "username_password", "password_reset_required": True}
         assert result.password_reset_required is True
 
 
diff --git a/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py
index 581a2ef8df8..c04353fec99 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py
@@ -12,7 +12,8 @@ import pytest
 import respx
 from fastapi import HTTPException
 
-from litellm.proxy._types import LitellmTableNames, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
+from litellm.proxy._types import UI_TEAM_ID, LitellmTableNames, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
+from litellm.proxy.auth.login_utils import PASSWORD_SESSION_METADATA
 from litellm.proxy.management_endpoints.password_endpoints import change_password
 from litellm.proxy.utils import hash_password, verify_password
 
@@ -37,7 +38,15 @@ def _make_prisma(user: MagicMock | None) -> MagicMock:
 
 
 def _caller(user_id: str | None = "user-123") -> UserAPIKeyAuth:
-    return UserAPIKeyAuth(user_id=user_id)
+    return UserAPIKeyAuth(user_id=user_id, team_id=UI_TEAM_ID, metadata=dict(PASSWORD_SESSION_METADATA))
+
+
+def _sso_session_caller() -> UserAPIKeyAuth:
+    return UserAPIKeyAuth(user_id="user-123", team_id=UI_TEAM_ID, metadata={})
+
+
+def _virtual_key_caller() -> UserAPIKeyAuth:
+    return UserAPIKeyAuth(user_id="user-123", team_id="team-abc", metadata=dict(PASSWORD_SESSION_METADATA))
 
 
 def _hibp_url_for(password: str) -> str:
@@ -130,6 +139,42 @@ async def test_change_password_rejects_unchanged_password():
     prisma.db.litellm_usertable.update.assert_not_called()
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "caller",
+    [
+        pytest.param(_sso_session_caller(), id="sso_dashboard_session"),
+        pytest.param(_virtual_key_caller(), id="virtual_key_with_forged_metadata"),
+    ],
+)
+async def test_change_password_rejects_non_password_login_session(caller: UserAPIKeyAuth):
+    """Only the session minted by a password login may change the password, so a
+    stolen virtual key or an SSO session cannot use the endpoint as a
+    current_password guessing oracle."""
+    from litellm.proxy._types import ChangePasswordRequest
+
+    prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
+
+    with (
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", prisma
+        ),
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
+        ),
+    ):
+        with pytest.raises(HTTPException) as exc_info:
+            await change_password(
+                data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD),
+                user_api_key_dict=caller,
+            )
+
+    assert exc_info.value.status_code == 403
+    assert "logging in with a password" in exc_info.value.detail["error"]
+    prisma.db.litellm_usertable.find_first.assert_not_called()
+    prisma.db.litellm_usertable.update.assert_not_called()
+
+
 @pytest.mark.asyncio
 async def test_change_password_rejects_session_without_user():
     from litellm.proxy._types import ChangePasswordRequest
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 93753d29993..84e3270c48b 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -17386,7 +17386,9 @@ export interface paths {
          * Change Password
          * @description Change the calling user's own password.
          *
-         *     Requires the current password. The new password must differ from the
+         *     Only callable with the dashboard session issued by a username/password
+         *     login; SSO sessions and virtual keys are rejected with 403. Requires the
+         *     current password. The new password must differ from the
          *     current one and satisfy the configured password policy
          *     (`general_settings.password_policy_*`: minimum length, character classes,
          *     and, when enabled, breached-password screening via haveibeenpwned.com).

From dac88cc6d0b3aa1fd869da8c0fd29d706a1ea594 Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Mon, 21 Sep 2026 15:28:25 -0700
Subject: [PATCH 134/160] fix(auth): keep blanket wildcards and team-less jwts
 out of the passthrough grant

A team_allowed_routes entry that names no path segment, such as * or /*, is a blanket grant like a named route group, so it no longer opens auth=true passthroughs. The grant in the shared route check now also requires a team on the JWT token, because team_allowed_routes should not apply to a JWT that resolved no team
---
 litellm/proxy/auth/handle_jwt.py              |  6 ++---
 litellm/proxy/auth/route_checks.py            | 12 ++++++----
 .../proxy/auth/test_route_checks.py           | 24 +++++++++++++------
 3 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py
index 2ae21bd0785..010a1b4536e 100644
--- a/litellm/proxy/auth/handle_jwt.py
+++ b/litellm/proxy/auth/handle_jwt.py
@@ -1693,7 +1693,7 @@ class JWTAuthManager:
                             team_object=team_object,
                             route=route,
                             request_method=request_method,
-                            team_allowed_routes=jwt_handler.litellm_jwtauth.team_allowed_routes or (),
+                            team_allowed_routes=jwt_handler.litellm_jwtauth.team_allowed_routes,
                         ):
                             is_allowed = False
                             denied_auth_enforced_pass_through_route = True
@@ -2589,7 +2589,7 @@ class JWTAuthManager:
             team_object=team_object,
             route=route,
             request_method=request_method,
-            team_allowed_routes=handler.litellm_jwtauth.team_allowed_routes or (),
+            team_allowed_routes=handler.litellm_jwtauth.team_allowed_routes,
         ):
             JWTAuthManager._raise_team_passthrough_route_denial(route=route)
 
@@ -2659,7 +2659,7 @@ class JWTAuthManager:
                 team_object=team_object,
                 route=route,
                 request_method=request_method,
-                team_allowed_routes=handler.litellm_jwtauth.team_allowed_routes or (),
+                team_allowed_routes=handler.litellm_jwtauth.team_allowed_routes,
             ):
                 JWTAuthManager._raise_team_passthrough_route_denial(route=route)
         elif team_id is None:
diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py
index 009ea986aad..7b39b68633d 100644
--- a/litellm/proxy/auth/route_checks.py
+++ b/litellm/proxy/auth/route_checks.py
@@ -686,23 +686,25 @@ class RouteChecks:
     @staticmethod
     def jwt_team_routes_grant_pass_through(route: str, team_allowed_routes: Collection[str]) -> bool:
         """
-        Explicit paths and trailing-wildcard prefixes grant auth=true pass-through. Entries are only ever compared
-        as paths, so a named route group like ``openai_routes`` never grants.
+        Explicit paths and trailing-wildcard prefixes grant auth=true pass-through. Blanket grants never do:
+        a named route group like ``openai_routes`` is only ever compared as a path, and an entry that names
+        no path segment (``*``, ``/*``) is skipped.
         """
         return any(
             RouteChecks.route_matches_wildcard_pattern(route=route, pattern=allowed_route)
             for allowed_route in team_allowed_routes
+            if allowed_route.rstrip("*").strip("/")
         )
 
     @staticmethod
     def _jwt_team_allowed_routes(valid_token: UserAPIKeyAuth) -> Collection[str]:
-        """``team_allowed_routes`` for tokens built by JWT auth; JWT-mapped virtual keys stay key-scoped."""
-        if valid_token.jwt_claims is None or valid_token.token is not None:
+        """``team_allowed_routes`` for team tokens built by JWT auth; JWT-mapped virtual keys stay key-scoped."""
+        if valid_token.jwt_claims is None or valid_token.token is not None or valid_token.team_id is None:
             return ()
 
         from litellm.proxy.proxy_server import jwt_handler
 
-        return jwt_handler.litellm_jwtauth.team_allowed_routes or ()
+        return jwt_handler.litellm_jwtauth.team_allowed_routes
 
     @staticmethod
     def _require_auth_pass_through_access(
diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py
index 2a92f3d7fbe..92b52f0e8e5 100644
--- a/tests/test_litellm/proxy/auth/test_route_checks.py
+++ b/tests/test_litellm/proxy/auth/test_route_checks.py
@@ -1272,6 +1272,8 @@ def test_non_proxy_admin_allows_auth_pass_through_with_team_allowlist():
         ("/model-host/v1/extractor/predict", ["/model-host/v1/extractor"], False),
         ("/other/v1/extractor", ["/model-host/*"], False),
         ("/model-host/v1/extractor", ["openai_routes", "llm_api_routes", "mapped_pass_through_routes"], False),
+        ("/model-host/v1/extractor", ["*"], False),
+        ("/model-host/v1/extractor", ["/*"], False),
         ("/model-host/v1/extractor", [], False),
     ],
 )
@@ -1350,20 +1352,28 @@ def test_non_proxy_admin_denies_auth_pass_through_for_jwt_when_only_route_groups
 
 
 @pytest.mark.parametrize(
-    "jwt_claims",
-    [None, {"sub": "test_user"}],
-    ids=["plain_virtual_key", "jwt_mapped_virtual_key"],
+    "api_key, team_id, jwt_claims",
+    [
+        ("sk-test-key", "team-a", None),
+        ("sk-test-key", "team-a", {"sub": "test_user"}),
+        (None, "team-a", None),
+        (None, None, {"sub": "test_user"}),
+    ],
+    ids=["plain_virtual_key", "jwt_mapped_virtual_key", "keyless_non_jwt_caller", "jwt_without_team"],
 )
-def test_non_proxy_admin_jwt_team_allowed_routes_never_grant_pass_through_to_virtual_keys(jwt_claims):
-    virtual_key: Final = UserAPIKeyAuth(
-        api_key="sk-test-key",
+def test_non_proxy_admin_jwt_team_allowed_routes_grant_pass_through_only_to_jwt_team_callers(
+    api_key, team_id, jwt_claims
+):
+    caller: Final = UserAPIKeyAuth(
+        api_key=api_key,
         user_id="test_user",
         user_role=LitellmUserRoles.INTERNAL_USER.value,
+        team_id=team_id,
         jwt_claims=jwt_claims,
     )
 
     with pytest.raises(HTTPException) as exc_info:
-        _check_model_host_route_as(virtual_key, team_allowed_routes=["openai_routes", "/model-host/*"])
+        _check_model_host_route_as(caller, team_allowed_routes=["openai_routes", "/model-host/*"])
 
     assert exc_info.value.status_code == 403, exc_info.value.detail
     assert "allowed_passthrough_routes" in exc_info.value.detail

From 5c0589207cb1c78600b0407d003f0e1bc3822be7 Mon Sep 17 00:00:00 2001
From: kerry <kerry@berri.ai>
Date: Mon, 21 Sep 2026 22:30:13 +0000
Subject: [PATCH 135/160] fix(xai): accept max_completion_tokens as a supported
 param

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/llms/xai/chat/transformation.py                |  1 +
 .../llms/xai/test_xai_chat_transformation.py           | 10 ++++++++++
 2 files changed, 11 insertions(+)

diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py
index 91bf697487d..33ee727dfab 100644
--- a/litellm/llms/xai/chat/transformation.py
+++ b/litellm/llms/xai/chat/transformation.py
@@ -118,6 +118,7 @@ class XAIChatConfig(OpenAIGPTConfig):
         base_openai_params: Final = [
             "logit_bias",
             "logprobs",
+            "max_completion_tokens",
             "max_tokens",
             "n",
             "parallel_tool_calls",
diff --git a/tests/test_litellm/llms/xai/test_xai_chat_transformation.py b/tests/test_litellm/llms/xai/test_xai_chat_transformation.py
index 290cd3dcb3a..3fd666e4f50 100644
--- a/tests/test_litellm/llms/xai/test_xai_chat_transformation.py
+++ b/tests/test_litellm/llms/xai/test_xai_chat_transformation.py
@@ -90,6 +90,16 @@ class TestXAIReasoningTokenFolding:
         assert response.usage.total_tokens == 999
 
 
+def test_max_completion_tokens_is_accepted_and_mapped_to_max_tokens() -> None:
+    optional_params = litellm.get_optional_params(
+        model="grok-4.20",
+        custom_llm_provider="xai",
+        max_completion_tokens=64,
+    )
+    assert optional_params["max_tokens"] == 64, optional_params
+    assert "max_completion_tokens" not in optional_params, optional_params
+
+
 class TestXAIParallelToolCalls:
     """Test suite for XAI parallel tool calls functionality."""
 

From 2ab4b255883b660bbc88a94e682c5ecd9e4e9ccf Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 22:30:29 +0000
Subject: [PATCH 136/160] fix(python-bridge): update cache test handle stubs
 for merged backends

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/rust_bridge/_native.pyi | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi
index 7eb266d5a09..baac21bb4bd 100644
--- a/litellm/rust_bridge/_native.pyi
+++ b/litellm/rust_bridge/_native.pyi
@@ -109,11 +109,14 @@ class _CacheTestHandle:
         *,
         ttl_seconds: float = 60.0,
         namespace: str | None = None,
+        startup_nodes: list[tuple[str, int]] | None = None,
     ) -> _CacheTestHandle: ...
     @staticmethod
+    def azure_blob(account_url: str, container: str) -> _CacheTestHandle: ...
+    @staticmethod
     def redis_semantic(backend: object) -> _CacheTestHandle: ...
     @property
-    def backend(self) -> Literal["memory", "redis", "redis_semantic"]: ...
+    def backend(self) -> Literal["memory", "redis", "azure-blob", "redis_semantic"]: ...
     def _bind_facade(self, facade: object) -> None: ...
 
 @final

From cf7988a2df65f6e775426bde8656cf4840f09f56 Mon Sep 17 00:00:00 2001
From: "berriai-litellm-provider-info-sync[bot]"
 <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com>
Date: Mon, 21 Sep 2026 22:31:01 +0000
Subject: [PATCH 137/160] chore(prices): sync OpenRouter prices: 3 models

openrouter/~z-ai/glm-flash-latest: max_tokens, max_output_tokens, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost
openrouter/deepseek/deepseek-v4-pro: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost
openrouter/z-ai/glm-5.3-flash: max_tokens, max_output_tokens, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost
---
 ...odel_prices_and_context_window_backup.json | 26 +++++++++----------
 model_prices_and_context_window.json          | 26 +++++++++----------
 2 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 6b49b1d47a5..beb90c2fb8c 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -43037,21 +43037,21 @@
         "supports_web_search": false
     },
     "openrouter/deepseek/deepseek-v4-pro": {
-        "input_cost_per_token": 8.95578e-07,
+        "input_cost_per_token": 8.92272e-07,
         "input_cost_per_token_cache_hit": 4.4e-08,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 384000,
         "max_tokens": 384000,
         "mode": "chat",
-        "output_cost_per_token": 1.791156e-06,
+        "output_cost_per_token": 1.784544e-06,
         "source": "https://openrouter.ai/api/v1/models",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "cache_read_input_token_cost": 7.46315e-08,
+        "cache_read_input_token_cost": 7.4356e-08,
         "supports_audio_input": false,
         "supports_pdf_input": false,
         "supports_vision": false,
@@ -68212,13 +68212,13 @@
         "supports_web_search": false
     },
     "openrouter/z-ai/glm-5.3-flash": {
-        "input_cost_per_token": 7.5e-08,
-        "output_cost_per_token": 2.5e-07,
-        "cache_read_input_token_cost": 2e-08,
+        "input_cost_per_token": 1.5e-07,
+        "output_cost_per_token": 5e-07,
+        "cache_read_input_token_cost": 5e-08,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1310720,
-        "max_output_tokens": 102400,
-        "max_tokens": 102400,
+        "max_output_tokens": 943718,
+        "max_tokens": 943718,
         "mode": "chat",
         "source": "https://openrouter.ai/api/v1/models",
         "supports_audio_input": false,
@@ -73252,14 +73252,14 @@
         "supports_web_search": true
     },
     "openrouter/~z-ai/glm-flash-latest": {
-        "cache_read_input_token_cost": 2e-08,
-        "input_cost_per_token": 7.5e-08,
+        "cache_read_input_token_cost": 5e-08,
+        "input_cost_per_token": 1.5e-07,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1310720,
-        "max_output_tokens": 102400,
-        "max_tokens": 102400,
+        "max_output_tokens": 943718,
+        "max_tokens": 943718,
         "mode": "chat",
-        "output_cost_per_token": 2.5e-07,
+        "output_cost_per_token": 5e-07,
         "source": "https://openrouter.ai/api/v1/models",
         "supports_audio_input": false,
         "supports_function_calling": true,
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 6b49b1d47a5..beb90c2fb8c 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -43037,21 +43037,21 @@
         "supports_web_search": false
     },
     "openrouter/deepseek/deepseek-v4-pro": {
-        "input_cost_per_token": 8.95578e-07,
+        "input_cost_per_token": 8.92272e-07,
         "input_cost_per_token_cache_hit": 4.4e-08,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 384000,
         "max_tokens": 384000,
         "mode": "chat",
-        "output_cost_per_token": 1.791156e-06,
+        "output_cost_per_token": 1.784544e-06,
         "source": "https://openrouter.ai/api/v1/models",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "cache_read_input_token_cost": 7.46315e-08,
+        "cache_read_input_token_cost": 7.4356e-08,
         "supports_audio_input": false,
         "supports_pdf_input": false,
         "supports_vision": false,
@@ -68212,13 +68212,13 @@
         "supports_web_search": false
     },
     "openrouter/z-ai/glm-5.3-flash": {
-        "input_cost_per_token": 7.5e-08,
-        "output_cost_per_token": 2.5e-07,
-        "cache_read_input_token_cost": 2e-08,
+        "input_cost_per_token": 1.5e-07,
+        "output_cost_per_token": 5e-07,
+        "cache_read_input_token_cost": 5e-08,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1310720,
-        "max_output_tokens": 102400,
-        "max_tokens": 102400,
+        "max_output_tokens": 943718,
+        "max_tokens": 943718,
         "mode": "chat",
         "source": "https://openrouter.ai/api/v1/models",
         "supports_audio_input": false,
@@ -73252,14 +73252,14 @@
         "supports_web_search": true
     },
     "openrouter/~z-ai/glm-flash-latest": {
-        "cache_read_input_token_cost": 2e-08,
-        "input_cost_per_token": 7.5e-08,
+        "cache_read_input_token_cost": 5e-08,
+        "input_cost_per_token": 1.5e-07,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1310720,
-        "max_output_tokens": 102400,
-        "max_tokens": 102400,
+        "max_output_tokens": 943718,
+        "max_tokens": 943718,
         "mode": "chat",
-        "output_cost_per_token": 2.5e-07,
+        "output_cost_per_token": 5e-07,
         "source": "https://openrouter.ai/api/v1/models",
         "supports_audio_input": false,
         "supports_function_calling": true,

From 82eef2fcca5e706914a1e81a3d15094f51436208 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Mon, 21 Sep 2026 22:32:32 +0000
Subject: [PATCH 138/160] fix(proxy): scope agent permissions to invoking
 caller

An agent key that echoes the x-litellm-user-id / x-litellm-team-id headers
forwarded by /a2a is capped at that user's and team's models, MCP servers
and agents, on top of its own grants and access group ceiling. The echoed
ids only narrow, and nested A2A hops forward the original human caller

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../mcp_server/auth/user_api_key_auth_mcp.py  |  34 ++++-
 litellm/proxy/_types.py                       |  11 ++
 .../proxy/agent_endpoints/a2a_endpoints.py    |   9 +-
 .../agent_endpoints/auth/agent_caller.py      |  87 +++++++++++++
 .../auth/agent_permission_handler.py          |  19 ++-
 litellm/proxy/auth/auth_checks.py             |  62 ++++++++-
 litellm/proxy/auth/user_api_key_auth.py       |   4 +
 litellm/types/agents.py                       |  16 ++-
 .../auth/test_user_api_key_auth_mcp.py        |  84 +++++++++++++
 .../agent_endpoints/auth/test_agent_caller.py |  57 +++++++++
 .../auth/test_agent_permission_handler.py     |  56 ++++++++-
 .../agent_endpoints/test_a2a_endpoints.py     |  19 +++
 .../proxy/auth/test_auth_checks.py            | 118 ++++++++++++++++++
 13 files changed, 565 insertions(+), 11 deletions(-)
 create mode 100644 litellm/proxy/agent_endpoints/auth/agent_caller.py
 create mode 100644 tests/test_litellm/proxy/agent_endpoints/auth/test_agent_caller.py

diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
index 4bca15190cf..f1a0de09162 100644
--- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
+++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
@@ -48,6 +48,7 @@ from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
     CeilingResolver,
     resolve_agent_access_group_ceiling,
 )
+from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_auth
 from litellm.proxy.auth.ip_address_utils import IPAddressUtils
 from litellm.proxy.auth.user_api_key_auth import (
     _get_bearer_token_or_received_api_key,  # pyright: ignore[reportPrivateUsage]  # shared x-litellm-api-key parser lives with user_api_key_auth
@@ -1577,14 +1578,21 @@ class MCPRequestHandler:
                         "Applied agent intersection filter. Final allowed servers: %s", allowed_mcp_servers
                     )
 
+            #########################################################
+            # Cap an agent key at what the user and team that invoked the agent may reach
+            #########################################################
+            caller_capped, caller_restricts = await MCPRequestHandler._apply_agent_caller_ceiling(
+                allowed_mcp_servers, user_api_key_auth
+            )
+
             #########################################################
             # Apply the internal user's own ceiling (the entitlement attached to the human)
             #########################################################
             capped, user_restricts = await MCPRequestHandler._apply_user_server_ceiling(
-                allowed_mcp_servers, user_api_key_auth, keyless_source=keyless_source
+                caller_capped, user_api_key_auth, keyless_source=keyless_source
             )
             allowed_mcp_servers = list(capped)
-            has_lower_level_mcp_restrictions = has_lower_level_mcp_restrictions or user_restricts
+            has_lower_level_mcp_restrictions = has_lower_level_mcp_restrictions or caller_restricts or user_restricts
 
             #########################################################
             # Apply org-level ceiling if org_id is set
@@ -2927,6 +2935,28 @@ class MCPRequestHandler:
         verbose_logger.debug("Applied user ceiling filter. Final allowed servers: %s", capped)
         return capped, True
 
+    @staticmethod
+    async def _apply_agent_caller_ceiling(
+        allowed_mcp_servers: Sequence[str],
+        user_api_key_auth: UserAPIKeyAuth | None = None,
+    ) -> tuple[tuple[str, ...], bool]:
+        """Narrow an agent key's servers to those the invoking user and team (echoed back by the agent
+        as ``x-litellm-user-id`` / ``x-litellm-team-id``) may reach: the echoed team's grants when it
+        names any, then the echoed user's own entitlement. Raises like the user ceiling when that
+        entitlement is known but unreadable, so the resolver denies rather than widens."""
+        caller_auth: Final = agent_caller_auth(user_api_key_auth) if user_api_key_auth else None
+        if caller_auth is None:
+            return tuple(allowed_mcp_servers), False
+        team_servers: Final = frozenset(await MCPRequestHandler._get_allowed_mcp_servers_for_team(caller_auth))
+        team_capped: Final = (
+            tuple(server for server in allowed_mcp_servers if server in team_servers)
+            if team_servers
+            else tuple(allowed_mcp_servers)
+        )
+        user_capped, user_restricts = await MCPRequestHandler._apply_user_server_ceiling(team_capped, caller_auth)
+        verbose_logger.debug("Applied agent caller ceiling. Final allowed servers: %s", user_capped)
+        return user_capped, bool(team_servers) or user_restricts
+
     @staticmethod
     async def _user_places_mcp_ceiling(user_api_key_auth: UserAPIKeyAuth | None = None) -> bool:
         """Whether this human's own entitlement bounds their MCP access at all.
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index c28858b48d9..50ddf52fc3f 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -27,6 +27,7 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
     validate_langfuse_span_scope_value,
     validate_no_callback_env_reference,
 )
+from litellm.types.agents import AgentCaller
 from litellm.types.integrations.compression_interception import (
     CompressionSavingsMetadata,
 )
@@ -3247,6 +3248,15 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView):  # the expected response ob
             "user id."
         ),
     )
+    agent_caller: AgentCaller | None = Field(
+        default=None,
+        exclude=True,
+        description=(
+            "Set per request from the x-litellm-user-id / x-litellm-team-id headers an agent echoes back on "
+            "calls made with its own key. Every check treats it as a ceiling, so a forged value can only "
+            "narrow the agent's access."
+        ),
+    )
     budget_reservation: dict[str, Any] | None = Field(default=None, exclude=True)
     team_budget_snapshot: TeamBudgetSnapshot | None = Field(default=None, exclude=True)
     user_budget_snapshot: UserBudgetSnapshot | None = Field(default=None, exclude=True)
@@ -3278,6 +3288,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView):  # the expected response ob
         values.pop("mcp_source_team_rpm_limits", None)
         values.pop("mcp_session_resource_server_id", None)
         values.pop("via_virtual_key", None)
+        values.pop("agent_caller", None)
         if values.get("api_key") is not None:
             values.update({"token": cls._safe_hash_litellm_api_key(values.get("api_key"))})
             if isinstance(values.get("api_key"), str):
diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py
index 834c16ba6dc..2a189a76545 100644
--- a/litellm/proxy/agent_endpoints/a2a_endpoints.py
+++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py
@@ -146,12 +146,17 @@ def _validate_push_notification_url(url: str) -> None:
 
 
 def _caller_identity_headers(user_api_key_dict: UserAPIKeyAuth) -> Mapping[str, str]:
+    """The human behind this call. An agent key acting for an invoking user forwards that user, not
+    itself, so a chain of agents stays capped at what the original caller may reach."""
+    caller: Final = user_api_key_dict.agent_caller
+    user_id: Final = caller.user_id if caller is not None else user_api_key_dict.user_id
+    team_id: Final = caller.team_id if caller is not None else user_api_key_dict.team_id
     return MappingProxyType(
         {
             name: value
             for name, value in (
-                ("X-LiteLLM-User-Id", user_api_key_dict.user_id),
-                ("X-LiteLLM-Team-Id", user_api_key_dict.team_id),
+                ("X-LiteLLM-User-Id", user_id),
+                ("X-LiteLLM-Team-Id", team_id),
             )
             if value
         }
diff --git a/litellm/proxy/agent_endpoints/auth/agent_caller.py b/litellm/proxy/agent_endpoints/auth/agent_caller.py
new file mode 100644
index 00000000000..47d43e8f71b
--- /dev/null
+++ b/litellm/proxy/agent_endpoints/auth/agent_caller.py
@@ -0,0 +1,87 @@
+"""The human behind an agent's own proxy calls.
+
+``/a2a/{agent}`` forwards the invoking key's ``X-LiteLLM-User-Id`` / ``X-LiteLLM-Team-Id`` to the
+agent backend. When the agent echoes them back on requests made with its own key, the proxy caps
+that key at what the invoking user and team may reach. The cap is intersected with, never
+substituted for, the agent key's own grants and the agent's access group ceiling, so the headers
+can only narrow access and need no trust.
+"""
+
+from collections.abc import Mapping
+from typing import Final
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy._types import LiteLLM_TeamTable, LiteLLM_UserTable, UserAPIKeyAuth
+from litellm.types.agents import (
+    AGENT_CALLER_TEAM_ID_HEADER,
+    AGENT_CALLER_USER_ID_HEADER,
+    AgentCaller,
+)
+
+
+def _header(headers: Mapping[str, str], name: str) -> str | None:
+    value: Final = next((raw for key, raw in headers.items() if key.lower() == name), None)
+    return value.strip() or None if value is not None else None
+
+
+def agent_caller_from_headers(headers: Mapping[str, str], user_api_key_auth: UserAPIKeyAuth) -> AgentCaller | None:
+    """The caller an agent key is acting for, or ``None`` when the key is not an agent's or no id was echoed."""
+    if not user_api_key_auth.agent_id:
+        return None
+    user_id: Final = _header(headers, AGENT_CALLER_USER_ID_HEADER)
+    team_id: Final = _header(headers, AGENT_CALLER_TEAM_ID_HEADER)
+    if user_id is None and team_id is None:
+        return None
+    return AgentCaller(user_id=user_id, team_id=team_id)
+
+
+def agent_caller_auth(user_api_key_auth: UserAPIKeyAuth) -> UserAPIKeyAuth | None:
+    """A minimal auth context standing for the invoking user and team, so the shared key/team/user
+    resolvers can be reused unchanged to compute what the caller may reach."""
+    caller: Final = user_api_key_auth.agent_caller
+    if caller is None:
+        return None
+    return UserAPIKeyAuth(
+        user_id=caller.user_id,
+        team_id=caller.team_id,
+        parent_otel_span=user_api_key_auth.parent_otel_span,
+    )
+
+
+async def load_agent_caller_team(user_api_key_auth: UserAPIKeyAuth) -> LiteLLM_TeamTable | None:
+    """The invoking team's row, or ``None`` when no team id was echoed. Raises when the id names a team
+    that cannot be loaded, since a caller we cannot resolve must not be treated as unrestricted."""
+    from litellm.proxy.auth.auth_checks import get_team_object
+    from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
+
+    caller: Final = user_api_key_auth.agent_caller
+    if caller is None or caller.team_id is None:
+        return None
+    return await get_team_object(
+        team_id=caller.team_id,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        parent_otel_span=user_api_key_auth.parent_otel_span,
+        proxy_logging_obj=proxy_logging_obj,
+    )
+
+
+async def load_agent_caller_user(user_api_key_auth: UserAPIKeyAuth) -> LiteLLM_UserTable | None:
+    """The invoking user's row, or ``None`` when no user id was echoed or the row does not exist."""
+    from litellm.proxy.auth.auth_checks import get_user_object
+    from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
+
+    caller: Final = user_api_key_auth.agent_caller
+    if caller is None or caller.user_id is None:
+        return None
+    user_object: Final = await get_user_object(
+        user_id=caller.user_id,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        user_id_upsert=False,
+        parent_otel_span=user_api_key_auth.parent_otel_span,
+        proxy_logging_obj=proxy_logging_obj,
+    )
+    if user_object is None:
+        verbose_proxy_logger.debug("agent caller user %r not found; no user ceiling applied", caller.user_id)
+    return user_object
diff --git a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
index b7b7638e478..9fe74bfee3f 100644
--- a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
+++ b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
@@ -23,6 +23,7 @@ from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
     CeilingResolver,
     resolve_agent_access_group_ceiling,
 )
+from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_auth
 from litellm.repositories.table_repositories import AgentsRepository
 from litellm.types.agents import AgentResponse
 
@@ -83,14 +84,24 @@ class AgentRequestHandler:
         user_api_key_auth: UserAPIKeyAuth | None = None,
         resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling,
     ) -> AgentAccess:
-        """Agents the key may reach: key and team grants intersected with the agent's access group ceiling."""
+        """Agents the key may reach: key and team grants, intersected with the agent's access group ceiling
+        and, for an agent key acting on behalf of an invoking user, with that user's team grants."""
         key_team_access: Final = await AgentRequestHandler._resolve_key_team_agent_access(user_api_key_auth)
+        caller_access: Final = await AgentRequestHandler._agent_caller_access(user_api_key_auth)
+        own_access: Final = _intersect_agent_access(key_team_access, caller_access)
         agent_ceiling: Final = await AgentRequestHandler._agent_access_group_ceiling(user_api_key_auth, resolve_ceiling)
         if agent_ceiling is None:
-            return key_team_access
-        if isinstance(key_team_access, UnrestrictedAgentAccess):
+            return own_access
+        if isinstance(own_access, UnrestrictedAgentAccess):
             return RestrictedAgentAccess(agent_ceiling)
-        return RestrictedAgentAccess(key_team_access.agent_ids & agent_ceiling)
+        return RestrictedAgentAccess(own_access.agent_ids & agent_ceiling)
+
+    @staticmethod
+    async def _agent_caller_access(user_api_key_auth: UserAPIKeyAuth | None) -> AgentAccess:
+        caller_auth: Final = agent_caller_auth(user_api_key_auth) if user_api_key_auth else None
+        if caller_auth is None:
+            return UnrestrictedAgentAccess()
+        return await AgentRequestHandler._get_allowed_agents_for_team(caller_auth)
 
     @staticmethod
     async def _resolve_key_team_agent_access(
diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py
index b5e7ef73d36..a477eecba38 100644
--- a/litellm/proxy/auth/auth_checks.py
+++ b/litellm/proxy/auth/auth_checks.py
@@ -15,7 +15,7 @@ import re
 import time
 from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol
+from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, TypeAlias
 
 from fastapi import HTTPException, Request, status
 from pydantic import BaseModel, TypeAdapter
@@ -72,6 +72,11 @@ from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
     CeilingResolver,
     resolve_agent_access_group_ceiling,
 )
+from litellm.proxy.agent_endpoints.auth.agent_caller import (
+    agent_caller_auth,
+    load_agent_caller_team,
+    load_agent_caller_user,
+)
 from litellm.proxy.auth.budget_throttle import (
     budget_throttle_percentage,
     should_throttle_budget_exceeded,
@@ -1010,6 +1015,14 @@ async def common_checks(
                     )
 
     await _check_agent_access_group_model_access(model=_model, valid_token=valid_token, llm_router=llm_router)
+    await _check_agent_caller_model_access(
+        model=_model,
+        valid_token=valid_token,
+        llm_router=llm_router,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        proxy_logging_obj=proxy_logging_obj,
+    )
 
     ## 2.1 If user can call model (if personal key)
     if _model and team_object is None and user_object is not None:
@@ -4355,6 +4368,53 @@ async def _check_agent_access_group_model_access(
     )
 
 
+LoadedCallerTeam: TypeAlias = LiteLLM_TeamTable | None
+LoadedCallerUser: TypeAlias = LiteLLM_UserTable | None
+CallerTeamLoader: TypeAlias = Callable[[UserAPIKeyAuth], Awaitable[LoadedCallerTeam]]  # mutable-ok: Callable params
+CallerUserLoader: TypeAlias = Callable[[UserAPIKeyAuth], Awaitable[LoadedCallerUser]]  # mutable-ok: Callable params
+
+
+async def _check_agent_caller_model_access(
+    model: str | list[str] | None,  # mutable-ok: the model checks it delegates to take list[str]
+    valid_token: UserAPIKeyAuth | None,
+    llm_router: Router | None,
+    prisma_client: Optional["PrismaClient"],
+    user_api_key_cache: UserApiKeyCache,
+    proxy_logging_obj: ProxyLogging,
+    load_team: CallerTeamLoader = load_agent_caller_team,
+    load_user: CallerUserLoader = load_agent_caller_user,
+) -> None:
+    """An agent key acting for an invoking user may call only what that user's own key could: the
+    invoking team's models (and per-member scope) when a team was echoed, else the user's models."""
+    if not model or valid_token is None:
+        return
+    caller_auth: Final = agent_caller_auth(valid_token)
+    if caller_auth is None:
+        return
+    caller_team: Final = await load_team(valid_token)
+    if caller_team is not None:
+        await can_team_access_model(
+            model=model,
+            team_object=caller_team,
+            llm_router=llm_router,
+            prisma_client=prisma_client,
+        )
+        await _check_team_member_model_access(
+            model=model,
+            team_object=caller_team,
+            valid_token=caller_auth,
+            llm_router=llm_router,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+        return
+    caller_user: Final = await load_user(valid_token)
+    if caller_user is None:
+        return
+    await can_user_call_model(model=model, llm_router=llm_router, user_object=caller_user)
+
+
 def _model_in_team_aliases(model: str, team_model_aliases: dict[str, str] | None = None) -> bool:
     """
     Returns True if `model` being accessed is an alias of a team model
diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py
index de0131772bc..b2a71a27090 100644
--- a/litellm/proxy/auth/user_api_key_auth.py
+++ b/litellm/proxy/auth/user_api_key_auth.py
@@ -39,6 +39,7 @@ from litellm.integrations.otel.runtime import phase_span, seed_request_identity
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value
 from litellm.proxy._types import *
+from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_from_headers
 from litellm.proxy.auth.auth_checks import (
     ExperimentalUIJWTToken,
     TeamNotFoundError,
@@ -3320,6 +3321,9 @@ async def user_api_key_auth(
                 raise body_parse_exception
             raise
         user_api_key_auth_obj.budget_reservation = None
+        user_api_key_auth_obj.agent_caller = agent_caller_from_headers(
+            _safe_get_request_headers(request), user_api_key_auth_obj
+        )
         _seed_request_destinations(user_api_key_auth_obj, request)
 
         # A body that never parsed is authenticated (so the trace carries identity
diff --git a/litellm/types/agents.py b/litellm/types/agents.py
index 12e60352a97..7f8d8c6af66 100644
--- a/litellm/types/agents.py
+++ b/litellm/types/agents.py
@@ -2,7 +2,7 @@ from collections.abc import Mapping, Sequence
 from datetime import datetime
 from typing import TYPE_CHECKING, Any, Final, Literal
 
-from pydantic import BaseModel, PrivateAttr, StrictInt
+from pydantic import BaseModel, ConfigDict, PrivateAttr, StrictInt
 from typing_extensions import ReadOnly, Required, TypedDict
 
 from litellm.types.llms.base import LiteLLMPydanticObjectBase
@@ -206,6 +206,20 @@ class PatchAgentRequest(TypedDict, total=False):
     access_group_ids: ReadOnly[Sequence[str] | None]
 
 
+AGENT_CALLER_USER_ID_HEADER: Final = "x-litellm-user-id"
+AGENT_CALLER_TEAM_ID_HEADER: Final = "x-litellm-team-id"
+
+
+class AgentCaller(BaseModel):
+    """The user and team that invoked an agent, echoed back by the agent on its own proxy calls.
+    Only ever narrows what the agent's key may do."""
+
+    model_config = ConfigDict(frozen=True)
+
+    user_id: str | None = None
+    team_id: str | None = None
+
+
 # Request/Response models for CRUD endpoints
 
 
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py
index 032e0f69a46..91c87cbe146 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py
@@ -22,6 +22,7 @@ from litellm.proxy._types import (
     SpecialMCPServerNames,
     UserAPIKeyAuth,
 )
+from litellm.types.agents import AgentCaller
 
 
 @pytest.mark.asyncio
@@ -4195,6 +4196,89 @@ def test_agent_capped_servers_without_agent_restrictions_is_uncapped():
 class TestAgentMCPPermissions:
     """Test agent-level MCP server and tool permission intersection."""
 
+    @staticmethod
+    def _agent_key_acting_for(user_id: str, team_id: str | None) -> UserAPIKeyAuth:
+        agent_key = UserAPIKeyAuth(api_key="agent-key", user_id="agent-owner", team_id="agent-team", agent_id="agent-1")
+        agent_key.agent_caller = AgentCaller(user_id=user_id, team_id=team_id)
+        return agent_key
+
+    @staticmethod
+    def _team_servers(grants: dict[str, list[str]]) -> AsyncMock:
+        async def by_team(user_api_key_auth: UserAPIKeyAuth | None = None) -> list[str]:
+            assert user_api_key_auth is not None
+            return grants.get(user_api_key_auth.team_id or "", [])
+
+        return AsyncMock(side_effect=by_team)
+
+    @staticmethod
+    def _user_servers(grants: dict[str, list[str] | None]) -> AsyncMock:
+        async def by_user(user_api_key_auth: UserAPIKeyAuth | None = None) -> list[str] | None:
+            assert user_api_key_auth is not None
+            return grants.get(user_api_key_auth.user_id or "", [])
+
+        return AsyncMock(side_effect=by_user)
+
+    async def test_agent_key_acting_for_a_user_is_capped_at_the_invoking_teams_servers(self):
+        """LIT-8014: the agent's own key reaches server_1 and server_2, but the human who invoked it
+        belongs to a team granted only server_2, so on their behalf the agent reaches only server_2."""
+        agent_key = self._agent_key_acting_for(user_id="alice", team_id="callers")
+
+        with (
+            patch.object(  # test-quality-ok: the level resolvers read proxy_server globals with no injection seam
+                MCPRequestHandler, "_get_allowed_mcp_servers_for_key", AsyncMock(return_value=["server_1", "server_2"])
+            ),
+            patch.object(  # test-quality-ok: same seam, keyed by which team is being asked about
+                MCPRequestHandler,
+                "_get_allowed_mcp_servers_for_team",
+                self._team_servers({"callers": ["server_2", "server_3"]}),
+            ),
+            patch.object(  # test-quality-ok: agent object_permission lookup hits the DB, not under test here
+                MCPRequestHandler, "_get_allowed_mcp_servers_for_agent", AsyncMock(return_value=[])
+            ),
+            patch.object(  # test-quality-ok: neither the agent's owner nor the caller has a personal grant
+                MCPRequestHandler, "_get_allowed_mcp_servers_for_user", self._user_servers({})
+            ),
+        ):
+            assert await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth=agent_key) == ["server_2"]
+
+    async def test_agent_key_acting_for_a_teamless_user_is_capped_at_that_users_servers(self):
+        agent_key = self._agent_key_acting_for(user_id="alice", team_id=None)
+
+        with (
+            patch.object(  # test-quality-ok: the level resolvers read proxy_server globals with no injection seam
+                MCPRequestHandler, "_get_allowed_mcp_servers_for_key", AsyncMock(return_value=["server_1", "server_2"])
+            ),
+            patch.object(  # test-quality-ok: same seam
+                MCPRequestHandler, "_get_allowed_mcp_servers_for_team", self._team_servers({})
+            ),
+            patch.object(  # test-quality-ok: agent object_permission lookup hits the DB, not under test here
+                MCPRequestHandler, "_get_allowed_mcp_servers_for_agent", AsyncMock(return_value=[])
+            ),
+            patch.object(  # test-quality-ok: same seam, keyed by which user is being asked about
+                MCPRequestHandler, "_get_allowed_mcp_servers_for_user", self._user_servers({"alice": ["server_1"]})
+            ),
+        ):
+            assert await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth=agent_key) == ["server_1"]
+
+    async def test_agent_key_acting_for_a_caller_whose_entitlement_is_unreadable_reaches_nothing(self):
+        agent_key = self._agent_key_acting_for(user_id="alice", team_id=None)
+
+        with (
+            patch.object(  # test-quality-ok: the level resolvers read proxy_server globals with no injection seam
+                MCPRequestHandler, "_get_allowed_mcp_servers_for_key", AsyncMock(return_value=["server_1"])
+            ),
+            patch.object(  # test-quality-ok: same seam
+                MCPRequestHandler, "_get_allowed_mcp_servers_for_team", self._team_servers({})
+            ),
+            patch.object(  # test-quality-ok: agent object_permission lookup hits the DB, not under test here
+                MCPRequestHandler, "_get_allowed_mcp_servers_for_agent", AsyncMock(return_value=[])
+            ),
+            patch.object(  # test-quality-ok: None is the resolver's own "entitlement unresolvable" signal
+                MCPRequestHandler, "_get_allowed_mcp_servers_for_user", self._user_servers({"alice": None})
+            ),
+        ):
+            assert await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth=agent_key) == []
+
     async def test_get_allowed_mcp_servers_agent_intersection(self):
         """Key/team allow [server_1, server_2]; agent allows [server_1]. Result = [server_1]."""
         user_api_key_auth = UserAPIKeyAuth(
diff --git a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_caller.py b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_caller.py
new file mode 100644
index 00000000000..b08964503c8
--- /dev/null
+++ b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_caller.py
@@ -0,0 +1,57 @@
+from typing import Final
+
+import pytest
+
+from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_auth, agent_caller_from_headers
+from litellm.types.agents import AgentCaller
+
+_AGENT_KEY: Final = UserAPIKeyAuth(api_key="agent-key", user_id="agent-owner", team_id="agent-team", agent_id="agent-1")
+
+
+def test_agent_key_echoing_both_ids_acts_for_that_user_and_team() -> None:
+    headers: Final = {"X-LiteLLM-User-Id": " alice ", "x-litellm-team-id": "callers"}
+
+    assert agent_caller_from_headers(headers, _AGENT_KEY) == AgentCaller(user_id="alice", team_id="callers")
+
+
+def test_agent_key_echoing_only_a_user_id_acts_for_a_teamless_user() -> None:
+    assert agent_caller_from_headers({"x-litellm-user-id": "alice"}, _AGENT_KEY) == AgentCaller(user_id="alice")
+
+
+@pytest.mark.parametrize("headers", [{}, {"x-litellm-user-id": "  ", "x-litellm-team-id": ""}])
+def test_agent_key_echoing_no_caller_acts_for_itself(headers: dict[str, str]) -> None:
+    assert agent_caller_from_headers(headers, _AGENT_KEY) is None
+
+
+def test_caller_headers_on_a_key_without_an_agent_are_ignored() -> None:
+    plain_key: Final = UserAPIKeyAuth(api_key="plain-key", user_id="bob")
+
+    assert agent_caller_from_headers({"x-litellm-user-id": "alice", "x-litellm-team-id": "callers"}, plain_key) is None
+
+
+def test_caller_auth_stands_for_the_invoking_user_not_the_agent() -> None:
+    agent_key: Final = UserAPIKeyAuth(
+        api_key="agent-key", user_id="agent-owner", team_id="agent-team", agent_id="agent-1"
+    )
+    agent_key.agent_caller = AgentCaller(user_id="alice", team_id="callers")
+
+    caller_auth: Final = agent_caller_auth(agent_key)
+
+    assert caller_auth is not None
+    assert (caller_auth.user_id, caller_auth.team_id, caller_auth.agent_id, caller_auth.api_key) == (
+        "alice",
+        "callers",
+        None,
+        None,
+    )
+    assert agent_caller_auth(_AGENT_KEY) is None
+
+
+def test_agent_caller_cannot_be_set_from_a_request_payload() -> None:
+    forged: Final = UserAPIKeyAuth.model_validate(
+        {"api_key": "agent-key", "agent_id": "agent-1", "agent_caller": {"user_id": "alice", "team_id": "callers"}}
+    )
+
+    assert forged.agent_caller is None
+    assert "agent_caller" not in forged.model_dump()
diff --git a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py
index 2a98e6e4feb..a87716375e8 100644
--- a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py
+++ b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py
@@ -9,7 +9,6 @@ from unittest.mock import AsyncMock, patch
 
 import pytest
 
-
 from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
 from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LitellmUserRoles, UserAPIKeyAuth
 from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry
@@ -21,6 +20,7 @@ from litellm.proxy.agent_endpoints.auth.agent_permission_handler import (
     UnrestrictedAgentAccess,
     accessible_agents,
 )
+from litellm.types.agents import AgentCaller
 
 
 def _registry_with(*agent_names: str) -> AgentRegistry:
@@ -196,6 +196,60 @@ class TestAgentRequestHandler:
         assert await AgentRequestHandler.is_agent_allowed("agent-alpha", agent_key, resolve) is False
         assert asked == ["caller-agent"] * 3
 
+    @staticmethod
+    def _team_grants(grants: dict[str, AgentAccess]) -> AsyncMock:
+        async def by_team(user_api_key_auth: UserAPIKeyAuth | None = None) -> AgentAccess:
+            assert user_api_key_auth is not None
+            return grants.get(user_api_key_auth.team_id or "", UnrestrictedAgentAccess())
+
+        return AsyncMock(side_effect=by_team)
+
+    async def test_agent_key_acting_for_a_user_is_capped_at_the_invoking_teams_agents(self):
+        """LIT-8014: the agent's key and access groups reach alpha and beta, but the human who
+        invoked it belongs to a team granted only beta, so on their behalf the agent reaches only beta."""
+        agent_key: Final = self._key_granting(["agent-alpha", "agent-beta"], agent_id="caller-agent")
+        agent_key.agent_caller = AgentCaller(user_id="alice", team_id="callers")
+        resolve, _ = self._ceiling_resolver(frozenset({"agent-alpha", "agent-beta", "agent-gamma"}))
+
+        with patch.object(  # test-quality-ok: the team resolver reads proxy_server globals with no injection seam
+            AgentRequestHandler,
+            "_get_allowed_agents_for_team",
+            self._team_grants({"callers": RestrictedAgentAccess(frozenset({"agent-beta", "agent-gamma"}))}),
+        ) as mock_team:
+            assert await AgentRequestHandler.resolve_agent_access(agent_key, resolve) == RestrictedAgentAccess(
+                frozenset({"agent-beta"})
+            )
+            assert await AgentRequestHandler.is_agent_allowed("agent-alpha", agent_key, resolve) is False
+
+        assert {call.args[0].team_id for call in mock_team.call_args_list} == {None, "callers"}
+
+    async def test_agent_key_acting_for_a_user_whose_team_grants_no_agent_reaches_none(self):
+        agent_key: Final = UserAPIKeyAuth(api_key="test-key", user_id="test-user", agent_id="caller-agent")
+        agent_key.agent_caller = AgentCaller(user_id="alice", team_id="callers")
+        resolve, _ = self._ceiling_resolver(None)
+
+        with patch.object(  # test-quality-ok: the team resolver reads proxy_server globals with no injection seam
+            AgentRequestHandler,
+            "_get_allowed_agents_for_team",
+            self._team_grants({"callers": RestrictedAgentAccess(frozenset())}),
+        ):
+            assert await AgentRequestHandler.resolve_agent_access(agent_key, resolve) == RestrictedAgentAccess(
+                frozenset()
+            )
+
+    async def test_agent_key_acting_for_an_ungranted_caller_keeps_its_own_agents(self):
+        agent_key: Final = self._key_granting(["agent-alpha"], agent_id="caller-agent")
+        agent_key.agent_caller = AgentCaller(user_id="alice", team_id="callers")
+        resolve, _ = self._ceiling_resolver(None)
+
+        with patch.object(  # test-quality-ok: the team resolver reads proxy_server globals with no injection seam
+            AgentRequestHandler, "_get_allowed_agents_for_team", self._team_grants({})
+        ):
+            assert await AgentRequestHandler.resolve_agent_access(agent_key, resolve) == RestrictedAgentAccess(
+                frozenset({"agent-alpha"})
+            )
+
+
     async def test_agent_access_groups_intersect_with_key_grants(self):
         agent_key: Final = self._key_granting(["agent-alpha", "agent-beta"], agent_id="caller-agent")
         resolve, _ = self._ceiling_resolver(frozenset({"agent-beta", "agent-gamma"}))
diff --git a/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py b/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py
index 441e9640ef9..b9a260f5b14 100644
--- a/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py
+++ b/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py
@@ -16,6 +16,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
 import pytest
 
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.types.agents import AgentCaller
 
 AddLiteLLMData = Callable[..., Awaitable[dict[str, object]]]
 
@@ -511,6 +512,24 @@ async def test_message_methods_forward_caller_identity_headers(method: str):
     assert forwarded_headers.get("X-LiteLLM-Team-Id") == "team-xyz"
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("method", ["message/send", "message/stream"])
+async def test_agent_calling_another_agent_forwards_the_human_who_invoked_it(method: str):
+    """LIT-8014: an agent acting for alice calls a second agent through the proxy. That hop must
+    carry alice, not the first agent's owner, so the chain stays capped at what alice may reach."""
+    mock_request = _make_request_mock(method, _HELLO_MESSAGE_PARAMS)
+    agent_key = UserAPIKeyAuth(api_key="sk-agent", user_id="agent-owner", team_id="agent-team", agent_id="agent-1")
+    agent_key.agent_caller = AgentCaller(user_id="alice", team_id="callers")
+
+    captured = await _invoke_message_method(method, mock_request, agent_key)
+
+    forwarded_headers = captured.agent_extra_headers or {}
+    assert (forwarded_headers.get("X-LiteLLM-User-Id"), forwarded_headers.get("X-LiteLLM-Team-Id")) == (
+        "alice",
+        "callers",
+    )
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("method", ["message/send", "message/stream"])
 async def test_message_methods_send_the_entra_bearer_for_azure_agents(method: str):
diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py
index a1179617718..7d5cf7dad9f 100644
--- a/tests/test_litellm/proxy/auth/test_auth_checks.py
+++ b/tests/test_litellm/proxy/auth/test_auth_checks.py
@@ -35,6 +35,7 @@ from litellm.proxy._types import (
     WebhookEvent,
 )
 from litellm.proxy.agent_endpoints.auth.agent_access_groups import AgentAccessGroupCeiling, CeilingResolver
+from litellm.types.agents import AgentCaller
 from litellm.proxy.auth.auth_checks import (
     ExperimentalUIJWTToken,
     _cache_management_object,
@@ -45,11 +46,14 @@ from litellm.proxy.auth.auth_checks import (
     _check_team_member_budget,
     _fetch_key_object_from_db_with_reconnect,
     _get_fuzzy_user_object,
+    CallerTeamLoader,
+    CallerUserLoader,
     _get_team_db_check,
     _log_budget_lookup_failure,
     _tag_max_budget_check,
     _team_max_budget_check,
     _virtual_key_max_budget_alert_check,
+    _check_agent_caller_model_access,
     _virtual_key_max_budget_check,
     _virtual_key_soft_budget_check,
     get_key_object,
@@ -9119,3 +9123,117 @@ async def test_team_member_budget_check_adds_temp_increase_to_live_team_default(
                 proxy_logging_obj=ProxyLogging(user_api_key_cache=None),
             )
     assert exc_info.value.max_budget == expected_cap
+
+
+def _agent_key_acting_for(user_id: str | None, team_id: str | None) -> UserAPIKeyAuth:
+    agent_key: Final = UserAPIKeyAuth(token="agent-token", agent_id="agent-1", models=["gpt-5", "claude-sonnet"])
+    agent_key.agent_caller = AgentCaller(user_id=user_id, team_id=team_id)
+    return agent_key
+
+
+def _caller_loaders(
+    team: LiteLLM_TeamTable | None,
+    user: LiteLLM_UserTable | None,
+) -> tuple[CallerTeamLoader, CallerUserLoader, list[str]]:
+    """Loaders that hand back fixed caller rows and record the agent_caller they were asked about."""
+    asked: Final[list[str]] = []
+
+    async def load_team(valid_token: UserAPIKeyAuth) -> LiteLLM_TeamTable | None:
+        asked.append(f"team:{valid_token.agent_caller.team_id if valid_token.agent_caller else None}")
+        return team
+
+    async def load_user(valid_token: UserAPIKeyAuth) -> LiteLLM_UserTable | None:
+        asked.append(f"user:{valid_token.agent_caller.user_id if valid_token.agent_caller else None}")
+        return user
+
+    return load_team, load_user, asked
+
+
+async def _cache_with_membership(user_id: str, team_id: str, allowed_models: list[str] | None) -> UserApiKeyCache:
+    from litellm.proxy._types import LiteLLM_TeamMembership
+    from litellm.proxy.common_utils.user_api_key_cache import team_membership_reservation_cache_key
+
+    cache: Final = UserApiKeyCache()
+    await cache.async_set_cache(
+        key=team_membership_reservation_cache_key(user_id=user_id, team_id=team_id),
+        value=LiteLLM_TeamMembership(
+            user_id=user_id,
+            team_id=team_id,
+            litellm_budget_table=LiteLLM_BudgetTable(allowed_models=allowed_models) if allowed_models else None,
+        ),
+        model_type=LiteLLM_TeamMembership,
+    )
+    return cache
+
+
+async def _check_caller_models(
+    agent_key: UserAPIKeyAuth,
+    model: str,
+    load_team: CallerTeamLoader,
+    load_user: CallerUserLoader,
+    cache: UserApiKeyCache | None = None,
+) -> None:
+    await _check_agent_caller_model_access(
+        model=model,
+        valid_token=agent_key,
+        llm_router=None,
+        prisma_client=None,
+        user_api_key_cache=cache or UserApiKeyCache(),
+        proxy_logging_obj=MagicMock(),
+        load_team=load_team,
+        load_user=load_user,
+    )
+
+
+@pytest.mark.asyncio
+async def test_agent_key_acting_for_a_team_is_capped_at_that_teams_models():
+    """LIT-8014: the invoking team may only call gpt-5, so the agent's own claude grant does not help."""
+    agent_key: Final = _agent_key_acting_for(user_id="alice", team_id="team-a")
+    load_team, load_user, asked = _caller_loaders(LiteLLM_TeamTable(team_id="team-a", models=["gpt-5"]), None)
+    cache: Final = await _cache_with_membership("alice", "team-a", allowed_models=None)
+
+    await _check_caller_models(agent_key, "gpt-5", load_team, load_user, cache)
+    with pytest.raises(ProxyException) as exc_info:
+        await _check_caller_models(agent_key, "claude-sonnet", load_team, load_user, cache)
+
+    assert exc_info.value.type == ProxyErrorTypes.team_model_access_denied
+    assert exc_info.value.code == str(status.HTTP_403_FORBIDDEN)
+    assert asked == ["team:team-a", "team:team-a"]
+
+
+@pytest.mark.asyncio
+async def test_agent_key_acting_for_a_team_member_is_capped_at_the_members_scope():
+    agent_key: Final = _agent_key_acting_for(user_id="alice", team_id="team-a")
+    load_team, load_user, _ = _caller_loaders(
+        LiteLLM_TeamTable(team_id="team-a", models=["gpt-5", "claude-sonnet"]), None
+    )
+    cache: Final = await _cache_with_membership("alice", "team-a", allowed_models=["gpt-5"])
+
+    await _check_caller_models(agent_key, "gpt-5", load_team, load_user, cache)
+    with pytest.raises(ProxyException) as exc_info:
+        await _check_caller_models(agent_key, "claude-sonnet", load_team, load_user, cache)
+
+    assert "User=alice, Team=team-a" in exc_info.value.internal_message
+
+
+@pytest.mark.asyncio
+async def test_agent_key_acting_for_a_teamless_user_is_capped_at_that_users_models():
+    agent_key: Final = _agent_key_acting_for(user_id="alice", team_id=None)
+    load_team, load_user, asked = _caller_loaders(None, LiteLLM_UserTable(user_id="alice", models=["gpt-5"]))
+
+    await _check_caller_models(agent_key, "gpt-5", load_team, load_user)
+    with pytest.raises(ProxyException) as exc_info:
+        await _check_caller_models(agent_key, "claude-sonnet", load_team, load_user)
+
+    assert exc_info.value.type == ProxyErrorTypes.user_model_access_denied
+    assert asked == ["team:None", "user:alice", "team:None", "user:alice"]
+
+
+@pytest.mark.asyncio
+async def test_agent_key_without_an_echoed_caller_keeps_its_own_models():
+    agent_key: Final = UserAPIKeyAuth(token="agent-token", agent_id="agent-1", models=["gpt-5", "claude-sonnet"])
+    load_team, load_user, asked = _caller_loaders(LiteLLM_TeamTable(team_id="team-a", models=[]), None)
+
+    await _check_caller_models(agent_key, "claude-sonnet", load_team, load_user)
+
+    assert asked == []

From 21d1604e64eb77a430bb95c4f80d3e44f94b1e1f Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 22:33:11 +0000
Subject: [PATCH 139/160] fix(cache): honor controls and tenant metadata in
 valkey semantic bridge

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../crates/cache-valkey-semantic/src/lib.rs   |  35 ++--
 .../crates/python-bridge/src/cache/native.rs  |  28 ++-
 .../crates/python-bridge/src/cache/request.rs |   6 +
 .../python-bridge/src/cache/semantic_step.rs  |  26 ++-
 .../test_valkey_semantic_cache_native.py      | 162 +++++++++++++++++-
 5 files changed, 216 insertions(+), 41 deletions(-)

diff --git a/litellm-rust/crates/cache-valkey-semantic/src/lib.rs b/litellm-rust/crates/cache-valkey-semantic/src/lib.rs
index e3a3e6094c4..6062ccc842c 100644
--- a/litellm-rust/crates/cache-valkey-semantic/src/lib.rs
+++ b/litellm-rust/crates/cache-valkey-semantic/src/lib.rs
@@ -273,13 +273,11 @@ pub fn prompt_from_context(context: &SemanticCacheContext) -> Option<String> {
     if let Some(Value::Array(messages)) = context.messages.as_ref()
         && !messages.is_empty()
     {
-        return Some(
-            messages
-                .iter()
-                .filter_map(Value::as_object)
-                .map(message_text)
-                .collect(),
-        );
+        return messages
+            .iter()
+            .filter_map(Value::as_object)
+            .map(message_text)
+            .collect();
     }
     let input = context.input.as_ref()?;
     let mut parts = Vec::new();
@@ -288,21 +286,25 @@ pub fn prompt_from_context(context: &SemanticCacheContext) -> Option<String> {
     (!prompt.is_empty()).then_some(prompt)
 }
 
-fn message_text(message: &serde_json::Map<String, Value>) -> String {
+fn message_text(message: &serde_json::Map<String, Value>) -> Option<String> {
     let content = match message.get("content") {
         Some(Value::String(value)) => value.clone(),
-        Some(Value::Array(parts)) => parts
-            .iter()
-            .filter_map(Value::as_object)
-            .filter_map(|part| part.get("text").and_then(Value::as_str))
-            .filter(|text| !text.is_empty())
-            .collect(),
+        Some(Value::Array(parts)) => {
+            let mut content = String::new();
+            for part in parts {
+                let part = part.as_object()?;
+                if let Some(text) = part.get("text").and_then(Value::as_str) {
+                    content.push_str(text);
+                }
+            }
+            content
+        }
         _ => String::new(),
     };
-    format!(
+    Some(format!(
         "{content}{}",
         search_results_text(message.get("search_results"))
-    )
+    ))
 }
 
 fn search_results_text(value: Option<&Value>) -> String {
@@ -732,6 +734,7 @@ mod tests {
     #[rstest]
     #[case(json!([{"content": "hello"}]), None, Some("hello"))]
     #[case(json!([{"content": [{"text": "hello"}, {"text": " world"}]}]), None, Some("hello world"))]
+    #[case(json!([{"content": ["raw", {"text": "hello"}]}]), None, None)]
     #[case(json!([{"search_results": [{"source": "s", "title": "t", "content": [{"text": "c"}], "citations": ["x"]}]}]), None, Some(r#"stc["x"]"#))]
     #[case(Value::Array(vec![]), Some(json!(" hello ")), Some("hello"))]
     #[case(Value::Array(vec![]), Some(json!([{"content": "first"}, {"text": "second"}])), Some("first\nsecond"))]
diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs
index fefc09c1321..d8f7a693108 100644
--- a/litellm-rust/crates/python-bridge/src/cache/native.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/native.rs
@@ -34,11 +34,24 @@ fn semantic_key(request: &NativeRequest, scope: &str) -> litellm_cache_response:
     ];
     let end_user = (scope == "end_user").then_some("user_api_key_end_user_id");
     for name in TENANT.into_iter().chain(end_user) {
-        let Some(value) = request
-            .metadata
-            .as_ref()
-            .and_then(|metadata| metadata.get(name))
-        else {
+        let sources = [
+            request.metadata.as_ref(),
+            request.litellm_metadata.as_ref(),
+            request
+                .litellm_params
+                .as_ref()
+                .and_then(|params| params.get("metadata")),
+            request
+                .litellm_params
+                .as_ref()
+                .and_then(|params| params.get("litellm_metadata")),
+        ];
+        let Some(value) = sources.into_iter().flatten().find_map(|source| {
+            source
+                .as_object()
+                .and_then(|values| values.get(name))
+                .filter(|value| !value.is_null())
+        }) else {
             continue;
         };
         let value = match value {
@@ -340,7 +353,6 @@ impl NativeResponseCache {
                     Arc::clone(cache.backend_arc()),
                     embedder.clone(),
                     Self::semantic(&request, scope),
-                    super::request::now(),
                 ),
             ),
         }
@@ -417,7 +429,6 @@ impl NativeResponseCache {
                     embedder.clone(),
                     Self::semantic(&request, scope),
                     response,
-                    super::request::now(),
                 ),
             ),
         }
@@ -517,7 +528,6 @@ impl NativeResponseCache {
                         embedder.clone(),
                         requests,
                         responses,
-                        super::request::now(),
                     ),
                 )
             }
@@ -565,6 +575,8 @@ mod tests {
             messages: Some(json!([{"role": "user", "content": "prompt"}])),
             input: None,
             metadata: Some(metadata),
+            litellm_metadata: None,
+            litellm_params: None,
         }
     }
 
diff --git a/litellm-rust/crates/python-bridge/src/cache/request.rs b/litellm-rust/crates/python-bridge/src/cache/request.rs
index 3e19e7fdc22..036951891a1 100644
--- a/litellm-rust/crates/python-bridge/src/cache/request.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/request.rs
@@ -17,6 +17,8 @@ struct RequestInput {
     messages: Option<Value>,
     input: Option<Value>,
     metadata: Option<Value>,
+    litellm_metadata: Option<Value>,
+    litellm_params: Option<Value>,
 }
 
 pub(super) struct NativeRequest {
@@ -27,6 +29,8 @@ pub(super) struct NativeRequest {
     pub(super) messages: Option<Value>,
     pub(super) input: Option<Value>,
     pub(super) metadata: Option<Value>,
+    pub(super) litellm_metadata: Option<Value>,
+    pub(super) litellm_params: Option<Value>,
 }
 
 pub(super) fn request(value: &Bound<'_, PyAny>) -> PyResult<NativeRequest> {
@@ -46,6 +50,8 @@ fn request_input(input: RequestInput) -> PyResult<NativeRequest> {
         messages: input.messages,
         input: input.input,
         metadata: input.metadata,
+        litellm_metadata: input.litellm_metadata,
+        litellm_params: input.litellm_params,
     })
 }
 
diff --git a/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs b/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs
index d8f3ab297f8..24caf3374d6 100644
--- a/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs
@@ -28,7 +28,7 @@ pub(super) struct SemanticEmbedExecution {
     embedder: PythonEmbedder,
     requests: Vec<ResponseCacheRequest<SemanticCacheContext>>,
     op: Op,
-    now: Duration,
+    now: Option<Duration>,
     prepared: Vec<Option<Vec<f32>>>,
     index: usize,
     state: State,
@@ -39,14 +39,13 @@ impl SemanticEmbedExecution {
         backend: Arc<ValkeySemanticCache<PythonEmbedder, ResponseCacheCodec>>,
         embedder: PythonEmbedder,
         request: ResponseCacheRequest<SemanticCacheContext>,
-        now: Duration,
     ) -> Self {
         Self {
             backend,
             embedder,
             requests: vec![request],
             op: Op::Lookup,
-            now,
+            now: None,
             prepared: vec![None],
             index: 0,
             state: State::Start,
@@ -58,14 +57,13 @@ impl SemanticEmbedExecution {
         embedder: PythonEmbedder,
         request: ResponseCacheRequest<SemanticCacheContext>,
         response: Value,
-        now: Duration,
     ) -> Self {
         Self {
             backend,
             embedder,
             requests: vec![request],
             op: Op::Store(response),
-            now,
+            now: None,
             prepared: vec![None],
             index: 0,
             state: State::Start,
@@ -77,7 +75,6 @@ impl SemanticEmbedExecution {
         embedder: PythonEmbedder,
         requests: Vec<ResponseCacheRequest<SemanticCacheContext>>,
         responses: Vec<Value>,
-        now: Duration,
     ) -> Self {
         Self {
             backend,
@@ -85,15 +82,26 @@ impl SemanticEmbedExecution {
             prepared: vec![None; requests.len()],
             requests,
             op: Op::StoreBatch(responses),
-            now,
+            now: None,
             index: 0,
             state: State::Start,
         }
     }
 
     fn start(&mut self, py: Python<'_>) -> PyResult<ExecutionStep> {
+        if self.now.is_none() {
+            self.now = Some(super::request::now());
+        }
         while self.index < self.requests.len() {
             let request = &self.requests[self.index];
+            let enabled = match &self.op {
+                Op::Lookup => request.controls.reads(),
+                Op::Store(_) | Op::StoreBatch(_) => request.controls.writes(),
+            };
+            if !enabled {
+                self.index += 1;
+                continue;
+            }
             let Some(prompt) = prompt_from_context(&request.context) else {
                 self.index += 1;
                 continue;
@@ -113,7 +121,9 @@ impl SemanticEmbedExecution {
         let requests = self.requests.clone();
         let prepared = self.prepared.clone();
         let backend = Arc::clone(&self.backend);
-        let now = self.now;
+        let now = self
+            .now
+            .ok_or_else(|| PyRuntimeError::new_err("semantic cache timestamp is unavailable"))?;
         let awaitable = match &self.op {
             Op::Lookup => {
                 let Some(request) = requests.into_iter().next() else {
diff --git a/tests/test_litellm_rust/test_valkey_semantic_cache_native.py b/tests/test_litellm_rust/test_valkey_semantic_cache_native.py
index fb5c5965333..c87a9f86a80 100644
--- a/tests/test_litellm_rust/test_valkey_semantic_cache_native.py
+++ b/tests/test_litellm_rust/test_valkey_semantic_cache_native.py
@@ -53,8 +53,12 @@ def _request(prompt: str = "semantic cache prompt") -> dict[str, object]:
 def _field_request(
     prompt: str,
     metadata: Mapping[str, object],
+    *,
+    namespace: str | None = None,
+    litellm_metadata: Mapping[str, object] | None = None,
+    litellm_params: Mapping[str, object] | None = None,
 ) -> dict[str, object]:
-    return {
+    request: Final = {
         "key": {
             "fields": [
                 {
@@ -69,23 +73,32 @@ def _field_request(
                     "api_parameter": True,
                     "internal_parameter": False,
                 },
-            ]
+            ],
+            "namespace": namespace,
         },
         "messages": [{"role": "user", "content": prompt}],
         "metadata": dict(metadata),
     }
+    if litellm_metadata is not None:
+        request["litellm_metadata"] = dict(litellm_metadata)
+    if litellm_params is not None:
+        request["litellm_params"] = dict(litellm_params)
+    return request
 
 
 def _facade(
     url: str,
     index_name: str,
     embeddings: Mapping[str, list[float]],
+    *,
+    namespace: str | None = None,
 ) -> Cache:
     facade: Final = Cache(
         type=LiteLLMCacheType.VALKEY_SEMANTIC,
         redis_url=url,
         similarity_threshold=0.8,
         valkey_semantic_cache_index_name=index_name,
+        namespace=namespace,
     )
     vectors: Final = embeddings
 
@@ -175,6 +188,45 @@ async def test_async_lookup_and_store(
     assert await binding.async_lookup(request) == {"answer": "async"}
 
 
+async def test_disabled_cache_controls_skip_async_embedding(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    backend: Final = _backend(valkey_url, index_name)
+    calls: Final = []
+
+    async def fail_embedding(prompt: str, metadata: dict[str, object] | None = None) -> list[float]:
+        calls.append(prompt)
+        raise AssertionError("embedding must not run")
+
+    backend._get_async_embedding = fail_embedding
+    handle: Final = _native._CacheTestHandle.valkey_semantic(
+        valkey_url,
+        0.8,
+        index_name,
+        backend,
+    )
+    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    controls: Final = {
+        "supported_call_type": True,
+        "configured": True,
+        "native_backend": True,
+        "default_on": True,
+        "caching": True,
+        "no_cache": False,
+        "no_store": False,
+        "use_cache": True,
+    }
+    no_read_request: Final = {**_request(), "controls": {**controls, "no_cache": True}}
+    assert await binding.async_lookup(no_read_request) is None
+    no_write_request: Final = {**_request(), "controls": {**controls, "no_store": True}}
+    await binding.async_store(no_write_request, {"answer": "blocked"})
+    assert calls == []
+    client: Final = redis.Redis.from_url(valkey_url)
+    assert list(client.scan_iter(f"{index_name}:*")) == []
+    client.close()
+
+
 async def test_async_embedding_runs_inline_in_caller_task(
     valkey_url: str,
     index_name: str,
@@ -323,6 +375,25 @@ def test_malformed_entry_is_a_miss_on_native_and_python(
     assert backend.get_cache("key", messages=_request()["messages"]) is None
 
 
+def test_mixed_content_parts_match_python_semantic_behavior(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    backend: Final = _backend(valkey_url, index_name)
+    messages: Final = [{"role": "user", "content": ["raw", {"text": "hello"}]}]
+    backend.set_cache("key", {"answer": "mixed"}, messages=messages)
+    assert backend.get_cache("key", messages=messages) is None
+
+    handle: Final = _native._CacheTestHandle.valkey_semantic(valkey_url, 0.8, index_name, backend)
+    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    request: Final = {**_request(), "messages": messages}
+    binding.store(request, {"answer": "mixed"})
+    assert binding.lookup(request) is None
+    client: Final = redis.Redis.from_url(valkey_url)
+    assert list(client.scan_iter(f"{index_name}:*")) == []
+    client.close()
+
+
 async def test_async_store_batch_and_lookup(
     valkey_url: str,
     index_name: str,
@@ -406,6 +477,84 @@ def test_field_key_matches_python_semantic_scope(
     client.close()
 
 
+def test_field_key_reads_all_python_tenant_metadata_sources(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    facade: Final = _facade(valkey_url, index_name, {"semantic cache prompt": [1.0, 0.0]})
+    params_metadata: Final = {"user_api_key_team_id": "team-from-params"}
+    expected: Final = facade.get_cache_key(
+        model="gpt-4.1",
+        messages=[{"role": "user", "content": "semantic cache prompt"}],
+        metadata={},
+        litellm_params={"metadata": params_metadata},
+    )
+    handle: Final = _native._CacheTestHandle.valkey_semantic(
+        valkey_url,
+        0.8,
+        index_name,
+        facade.cache,
+    )
+    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    binding.store(
+        _field_request(
+            "semantic cache prompt",
+            {},
+            litellm_params={"metadata": params_metadata},
+        ),
+        {"answer": "params"},
+    )
+    client: Final = redis.Redis.from_url(valkey_url)
+    documents: Final = list(client.scan_iter(f"{index_name}:*"))
+    assert len(documents) == 1
+    document_parts: Final = documents[0].decode().split(":")
+    assert document_parts[1] == hashlib.sha256(expected.encode()).hexdigest()
+    client.close()
+
+    assert (
+        binding.lookup(
+            _field_request(
+                "semantic cache prompt",
+                {},
+                litellm_metadata={"user_api_key_team_id": "team-from-litellm"},
+            )
+        )
+        is None
+    )
+
+
+def test_namespace_isolates_semantic_entries(
+    valkey_url: str,
+    index_name: str,
+) -> None:
+    facade: Final = _facade(
+        valkey_url,
+        index_name,
+        {"semantic cache prompt": [1.0, 0.0]},
+        namespace="team-a",
+    )
+    handle: Final = _native._CacheTestHandle.valkey_semantic(
+        valkey_url,
+        0.8,
+        index_name,
+        facade.cache,
+    )
+    binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve()
+    team_a: Final = _field_request("semantic cache prompt", {}, namespace="team-a")
+    team_b: Final = _field_request("semantic cache prompt", {}, namespace="team-b")
+    binding.store(team_a, {"answer": "team-a"})
+    assert binding.lookup(team_b) is None
+    assert binding.lookup(team_a) == {"answer": "team-a"}
+    cached: Final = cast(
+        Mapping[str, object],
+        facade.get_cache(
+            model="gpt-4.1",
+            messages=[{"role": "user", "content": "semantic cache prompt"}],
+        ),
+    )
+    assert cached == {"answer": "team-a"}
+
+
 def test_field_key_isolates_tenant_scope(
     valkey_url: str,
     index_name: str,
@@ -422,13 +571,8 @@ def test_field_key_isolates_tenant_scope(
         _field_request("semantic cache prompt", {"user_api_key": "k1"}),
         {"answer": "tenant one"},
     )
-    assert (
-        binding.lookup(_field_request("semantic cache prompt", {"user_api_key": "k2"}))
-        is None
-    )
-    assert binding.lookup(_field_request("semantic cache prompt", {"user_api_key": "k1"})) == {
-        "answer": "tenant one"
-    }
+    assert binding.lookup(_field_request("semantic cache prompt", {"user_api_key": "k2"})) is None
+    assert binding.lookup(_field_request("semantic cache prompt", {"user_api_key": "k1"})) == {"answer": "tenant one"}
 
 
 def test_tls_valkey_facade_falls_back_to_python(

From 4ff251e6b1ae6e2bf89424af8c06a453ded64e9b Mon Sep 17 00:00:00 2001
From: kerry <kerry@berri.ai>
Date: Mon, 21 Sep 2026 22:37:13 +0000
Subject: [PATCH 140/160] fix(xai): accept max_completion_tokens as a supported
 param

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../llms/xai/test_xai_chat_transformation.py        | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/test_litellm/llms/xai/test_xai_chat_transformation.py b/tests/test_litellm/llms/xai/test_xai_chat_transformation.py
index 3fd666e4f50..0a9e3234f12 100644
--- a/tests/test_litellm/llms/xai/test_xai_chat_transformation.py
+++ b/tests/test_litellm/llms/xai/test_xai_chat_transformation.py
@@ -13,6 +13,7 @@ from litellm.types.utils import (
     ModelResponse,
     Usage,
 )
+from litellm.utils import get_optional_params
 
 
 class TestXAIReasoningTokenFolding:
@@ -309,3 +310,15 @@ class TestXAIReportedCost:
             0.0,
             0.0037756,
         )
+
+
+class TestXAIMaxCompletionTokens:
+    def test_max_completion_tokens_is_accepted_and_mapped_to_max_tokens(self):
+        result = get_optional_params(
+            model="grok-4.20-beta",
+            custom_llm_provider="xai",
+            max_completion_tokens=64,
+        )
+
+        assert result["max_tokens"] == 64
+        assert "max_completion_tokens" not in result

From 32c63e332eaca89c0c740d097cc7b7727a3d628c Mon Sep 17 00:00:00 2001
From: kerry <kerry@berri.ai>
Date: Mon, 21 Sep 2026 22:38:29 +0000
Subject: [PATCH 141/160] Revert "fix(xai): accept max_completion_tokens as a
 supported param"

This reverts commit 4ff251e6b1ae6e2bf89424af8c06a453ded64e9b.
---
 .../llms/xai/test_xai_chat_transformation.py        | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/tests/test_litellm/llms/xai/test_xai_chat_transformation.py b/tests/test_litellm/llms/xai/test_xai_chat_transformation.py
index 0a9e3234f12..3fd666e4f50 100644
--- a/tests/test_litellm/llms/xai/test_xai_chat_transformation.py
+++ b/tests/test_litellm/llms/xai/test_xai_chat_transformation.py
@@ -13,7 +13,6 @@ from litellm.types.utils import (
     ModelResponse,
     Usage,
 )
-from litellm.utils import get_optional_params
 
 
 class TestXAIReasoningTokenFolding:
@@ -310,15 +309,3 @@ class TestXAIReportedCost:
             0.0,
             0.0037756,
         )
-
-
-class TestXAIMaxCompletionTokens:
-    def test_max_completion_tokens_is_accepted_and_mapped_to_max_tokens(self):
-        result = get_optional_params(
-            model="grok-4.20-beta",
-            custom_llm_provider="xai",
-            max_completion_tokens=64,
-        )
-
-        assert result["max_tokens"] == 64
-        assert "max_completion_tokens" not in result

From 212ab630b81957610b9708af7fb85b827538979e Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 21 Sep 2026 15:44:26 -0700
Subject: [PATCH 142/160] fix(logging_worker): make flush() survive an event
 loop change

flush() awaited join() on whatever queue the worker held, even one bound to
an event loop that has since closed. Its unfinished counter is never
decremented on the new loop, so the first flush() after a loop change hung
until pytest-timeout killed it and every later one raised "is bound to a
different event loop" from the queue's Event. The CircleCI unit job has
been red on every branch since the first tests that flush without
enqueueing landed, and an SDK script that flushes from a second
asyncio.run() hangs the same way.

flush() now goes through start() first, which carries the tasks stranded
on the previous loop onto the current one and guarantees a worker there to
drain them, the same loop-change handling every other entry point already
had.
---
 litellm/litellm_core_utils/logging_worker.py  |  6 ++++
 .../litellm_core_utils/test_logging_worker.py | 34 +++++++++++++++++++
 2 files changed, 40 insertions(+)

diff --git a/litellm/litellm_core_utils/logging_worker.py b/litellm/litellm_core_utils/logging_worker.py
index 5ccc5632646..cb3d8bf4fe5 100644
--- a/litellm/litellm_core_utils/logging_worker.py
+++ b/litellm/litellm_core_utils/logging_worker.py
@@ -484,9 +484,15 @@ class LoggingWorker:
         so it correctly handles items that have been dequeued but whose
         callback hasn't finished yet — ``queue.empty()`` would return True in
         that window and cause us to skip the wait.
+
+        ``start()`` runs first so that, after an event loop change, the tasks
+        still on the previous loop's queue move onto this loop and a worker
+        here drains them; joining the old queue directly would wait on a
+        counter nothing on this loop ever decrements.
         """
         if self._queue is None:
             return
+        self.start()
         await self._queue.join()
 
     async def clear_queue(self):
diff --git a/tests/test_litellm/litellm_core_utils/test_logging_worker.py b/tests/test_litellm/litellm_core_utils/test_logging_worker.py
index 1553e788472..891bd0686b2 100644
--- a/tests/test_litellm/litellm_core_utils/test_logging_worker.py
+++ b/tests/test_litellm/litellm_core_utils/test_logging_worker.py
@@ -180,6 +180,40 @@ class TestLoggingWorker:
 
         assert sorted(fired) == ["first", "second"]
 
+    @pytest.mark.parametrize("stranded", ["still_queued", "dequeued_never_started"])
+    def test_flush_on_new_loop_drains_tasks_stranded_on_previous_loop(self, stranded):
+        """
+        Regression: ``flush()`` from a new event loop used to ``join()`` the queue bound to the
+        previous loop, whose unfinished counter nothing on the new loop ever decrements. The first
+        such flush hung until pytest-timeout killed it and every later one raised
+        ``RuntimeError: ... is bound to a different event loop`` from the queue's Event.
+        """
+        worker = LoggingWorker(timeout=1.0, max_queue_size=10)
+        fired = []
+
+        async def marker():
+            fired.append(True)
+
+        async def enqueue_on_first_loop():
+            if stranded == "still_queued":
+                worker._ensure_queue()
+                worker.enqueue(marker())
+                return
+            worker.ensure_initialized_and_enqueue(marker())
+
+        asyncio.run(enqueue_on_first_loop())
+        assert worker._queue is not None
+        expected_shape = (1, 0) if stranded == "still_queued" else (0, 1)
+        assert (worker._queue.qsize(), len(worker._unstarted_dequeued_tasks())) == expected_shape
+        assert fired == [], "precondition: the callback never ran before the first loop closed"
+
+        async def flush_on_second_loop():
+            await asyncio.wait_for(worker.flush(), timeout=5)
+
+        asyncio.run(flush_on_second_loop())
+
+        assert fired == [True]
+
     def test_flush_on_exit_swallows_cancellation_and_drains_remaining(self):
         """A callback raising CancelledError must not abort the atexit flush of later events."""
         worker = LoggingWorker(timeout=1.0, max_queue_size=10)

From a5cce1b85966debd1468575d83fddb6910750a73 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Mon, 21 Sep 2026 15:49:09 -0700
Subject: [PATCH 143/160] fix(proxy): let the config file win over the stored
 row in ui settings

---
 .../proxy/ui_crud_endpoints/proxy_setting_endpoints.py    | 8 ++++----
 .../ui_crud_endpoints/test_proxy_setting_endpoints.py     | 7 +++++--
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
index ed626bdb624..f4d4ccf5851 100644
--- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
+++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
@@ -1763,8 +1763,8 @@ async def get_ui_settings():
 
     effective_ui_settings: Final[Mapping[str, object]] = MappingProxyType(
         {
-            **{key: proxy_config.settings[key] for key in ALLOWED_UI_SETTINGS_FIELDS if key in proxy_config.settings},
             **ui_settings,
+            **{key: proxy_config.settings[key] for key in ALLOWED_UI_SETTINGS_FIELDS if key in proxy_config.settings},
         }
     )
     config: Final[Mapping[str, object]] = MappingProxyType(
@@ -1787,9 +1787,9 @@ async def get_ui_settings():
     source: Final[Mapping[str, FieldSource]] = MappingProxyType(
         {
             key: (
-                "db"
-                if key in ui_settings
-                else _ui_setting_source(key, values[key], proxy_config.settings, settings_class)
+                _ui_setting_source(key, values[key], proxy_config.settings, settings_class)
+                if key in proxy_config.settings or key not in ui_settings
+                else "db"
             )
             for key in values
         }
diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
index 75feb746bd7..eecee2fd0f1 100644
--- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
+++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
@@ -1352,6 +1352,7 @@ class TestProxySettingEndpoints:
         mock_db_record = MagicMock()
         mock_db_record.ui_settings = {
             "disable_model_add_for_internal_users": True,
+            "require_auth_for_public_ai_hub": True,
         }
         mock_prisma.db.litellm_uisettings.find_unique = AsyncMock(
             return_value=mock_db_record
@@ -1376,10 +1377,12 @@ class TestProxySettingEndpoints:
 
         assert response.status_code == 200
         data = response.json()
-        assert data["values"]["disable_model_add_for_internal_users"] is True
+        assert data["values"]["disable_model_add_for_internal_users"] is False
         assert data["values"]["forward_client_headers_to_llm_api"] is True
-        assert data["source"]["disable_model_add_for_internal_users"] == "db"
+        assert data["values"]["require_auth_for_public_ai_hub"] is True
+        assert data["source"]["disable_model_add_for_internal_users"] == "config"
         assert data["source"]["forward_client_headers_to_llm_api"] == "config"
+        assert data["source"]["require_auth_for_public_ai_hub"] == "db"
 
     def test_get_ui_settings_schema_description_preserved_with_extensions(
         self, mock_auth, monkeypatch

From 636a60d3bbc9fbfc59fa594b128520eb3e43f433 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 22:51:14 +0000
Subject: [PATCH 144/160] chore(rust): update merged workspace lockfile

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/Cargo.lock | 108 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 106 insertions(+), 2 deletions(-)

diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index 1a4eb51af08..5acfa22e824 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -2850,6 +2850,7 @@ dependencies = [
  "litellm-secrets-azure",
  "litellm-secrets-cyberark",
  "litellm-secrets-google",
+ "litellm-secrets-hashicorp",
  "litellm-secrets-types",
  "moka",
  "reqwest 0.12.28",
@@ -2947,6 +2948,26 @@ dependencies = [
  "wiremock",
 ]
 
+[[package]]
+name = "litellm-secrets-hashicorp"
+version = "0.1.0"
+dependencies = [
+ "litellm-core-utils",
+ "litellm-secrets-types",
+ "moka",
+ "rstest",
+ "rustify",
+ "rustify_derive",
+ "serde",
+ "serde_json",
+ "tempfile",
+ "thiserror 2.0.19",
+ "tokio",
+ "vaultrs",
+ "veil",
+ "wiremock",
+]
+
 [[package]]
 name = "litellm-secrets-types"
 version = "0.1.0"
@@ -4185,6 +4206,40 @@ dependencies = [
  "semver",
 ]
 
+[[package]]
+name = "rustify"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4800ce4c1cc2fec12c559dae2ddbf0e17fcee7569b796e6d75898efef443368b"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "bytes",
+ "http 1.4.2",
+ "reqwest 0.13.5",
+ "rustify_derive",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "thiserror 1.0.69",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "rustify_derive"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78ea7fda74240f7410d0198b603a8a2f662acc7d76b6667a49f9b162cd8d9b4f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "regex",
+ "serde_urlencoded",
+ "syn 1.0.109",
+ "synstructure 0.12.6",
+]
+
 [[package]]
 name = "rustix"
 version = "1.1.5"
@@ -4737,6 +4792,17 @@ version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "syn"
 version = "2.0.119"
@@ -4768,6 +4834,18 @@ dependencies = [
  "futures-core",
 ]
 
+[[package]]
+name = "synstructure"
+version = "0.12.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "unicode-xid",
+]
+
 [[package]]
 name = "synstructure"
 version = "0.13.2"
@@ -5182,6 +5260,7 @@ version = "0.1.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
 dependencies = [
+ "log",
  "pin-project-lite",
  "tracing-attributes",
  "tracing-core",
@@ -5366,6 +5445,12 @@ version = "1.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
 
+[[package]]
+name = "unicode-xid"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
+
 [[package]]
 name = "unicode_categories"
 version = "0.1.1"
@@ -5425,6 +5510,25 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
 
+[[package]]
+name = "vaultrs"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30ffcc0e81025065dda612ec1e26a3d81bb16ef3062354873d17a35965d68522"
+dependencies = [
+ "async-trait",
+ "derive_builder",
+ "http 1.4.2",
+ "reqwest 0.13.5",
+ "rustify",
+ "rustify_derive",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.19",
+ "tracing",
+ "url",
+]
+
 [[package]]
 name = "vcpkg"
 version = "0.2.15"
@@ -5874,7 +5978,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "syn 2.0.119",
- "synstructure",
+ "synstructure 0.13.2",
 ]
 
 [[package]]
@@ -5915,7 +6019,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "syn 2.0.119",
- "synstructure",
+ "synstructure 0.13.2",
 ]
 
 [[package]]

From 467d13ebac68dee4e8da5365c5e139eedd62008f Mon Sep 17 00:00:00 2001
From: Tin Chi Lo <tin@berri.ai>
Date: Mon, 21 Sep 2026 15:54:41 -0700
Subject: [PATCH 145/160] fix(cli): preserve newer installed status lines
 during setup

---
 litellm/proxy/_lazy_openapi_snapshot.json     |  2 +-
 .../client/cli/commands/claude_settings.py    | 35 +++++++++-
 .../client/cli/commands/statusline_script.py  |  2 +-
 pyproject.toml                                |  1 +
 .../proxy/client/cli/test_claude_settings.py  | 68 +++++++++++++++++--
 uv.lock                                       |  2 +
 6 files changed, 101 insertions(+), 9 deletions(-)

diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json
index 6f9a2d8c96d..8bce873aa24 100644
--- a/litellm/proxy/_lazy_openapi_snapshot.json
+++ b/litellm/proxy/_lazy_openapi_snapshot.json
@@ -19632,7 +19632,7 @@
                   }
                 }
               },
-              "description": "\n    Unified rate-limit error.\n\n    Every rate-limit condition surfaced by litellm \u2014 whether it originated from\n    an upstream LLM provider, a vendor batch endpoint, or one of litellm's own\n    proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\n    max-iterations, etc.) \u2014 is raised as an instance of this class.\n\n    The :attr:`category` attribute lets callers distinguish the source. See\n    :class:`RateLimitErrorCategory` for the available values.\n    "
+              "description": "\nUnified rate-limit error.\n\nEvery rate-limit condition surfaced by litellm \u2014 whether it originated from\nan upstream LLM provider, a vendor batch endpoint, or one of litellm's own\nproxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\nmax-iterations, etc.) \u2014 is raised as an instance of this class.\n\nThe :attr:`category` attribute lets callers distinguish the source. See\n:class:`RateLimitErrorCategory` for the available values.\n"
             },
             "500": {
               "content": {
diff --git a/litellm/proxy/client/cli/commands/claude_settings.py b/litellm/proxy/client/cli/commands/claude_settings.py
index f4bebc4a4cb..ffc26d22e84 100644
--- a/litellm/proxy/client/cli/commands/claude_settings.py
+++ b/litellm/proxy/client/cli/commands/claude_settings.py
@@ -22,8 +22,11 @@ from pathlib import Path
 from types import MappingProxyType
 from typing import Final, TypeAlias
 
+import click
+from packaging.version import InvalidVersion, Version
 from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError
 
+from litellm._version import version as litellm_version
 from litellm.litellm_core_utils.private_json import (
     commit_staged_json,
     discard_staged_json,
@@ -75,6 +78,7 @@ BACKUP_PATH: Final = Path.home() / ".litellm" / "claude_settings_backup.json"
 AUTOROUTE_BACKUP_PATH: Final = Path.home() / ".litellm" / "autorouter" / "claude_settings_backup.json"
 CONFIGURE_STATE_PATH: Final = Path.home() / ".litellm" / "claude_configure_state.json"
 STATUSLINE_SCRIPT_PATH: Final = Path.home() / ".litellm" / "statusline.py"
+STATUSLINE_VERSION_PREFIX: Final = b"# litellm-statusline-version: "
 
 
 @dataclass(frozen=True, slots=True)
@@ -305,12 +309,37 @@ def statusline_command(script_path: Path, platform: str = sys.platform) -> str:
     return " ".join(quote(token) for token in (sys.executable, str(script_path)))
 
 
-def install_statusline_script(script_path: Path | None = None) -> str:
+def _installed_statusline_version(target: Path) -> Version | None:
+    try:
+        with target.open("rb") as script:
+            header: Final = script.readline(256)
+    except FileNotFoundError:
+        return None
+    if not header.startswith(STATUSLINE_VERSION_PREFIX):
+        return None
+    try:
+        return Version(header.removeprefix(STATUSLINE_VERSION_PREFIX).decode("ascii").strip())
+    except (InvalidVersion, UnicodeDecodeError):
+        return None
+
+
+def install_statusline_script(script_path: Path | None = None, *, package_version: str = litellm_version) -> str:
     target: Final = script_path or STATUSLINE_SCRIPT_PATH
     try:
         ensure_private_dir(target.parent)
-        write_private_bytes(str(target), Path(statusline_script.__file__).read_bytes())
-    except OSError as e:
+        bundled_version: Final = Version(package_version)
+        installed_version: Final = _installed_statusline_version(target)
+        if installed_version is not None and installed_version > bundled_version:
+            click.echo(
+                f"Keeping the status line from LiteLLM {installed_version}; this CLI is {bundled_version}. "
+                "Upgrade the CLI to refresh it.",
+                err=True,
+            )
+            return statusline_command(target)
+        source: Final = Path(statusline_script.__file__).read_bytes()
+        header: Final = STATUSLINE_VERSION_PREFIX + str(bundled_version).encode("ascii") + b"\n"
+        write_private_bytes(str(target), header + source)
+    except (OSError, InvalidVersion) as e:
         raise ClaudeSettingsError(f"Could not install the status line script at {target}: {e}") from e
     return statusline_command(target)
 
diff --git a/litellm/proxy/client/cli/commands/statusline_script.py b/litellm/proxy/client/cli/commands/statusline_script.py
index 09dd062c888..6264acb39d1 100644
--- a/litellm/proxy/client/cli/commands/statusline_script.py
+++ b/litellm/proxy/client/cli/commands/statusline_script.py
@@ -1,6 +1,6 @@
 """Claude Code status line and Codex Stop hook for auto-routed sessions.
 
-`lite` copies this file verbatim to ~/.litellm/statusline.py and registers it as Claude
+`lite` copies this file with a CLI version header to ~/.litellm/statusline.py and registers it as Claude
 Code's `statusLine` command and as Codex's `[[hooks.Stop]]` command, so it must stay
 standard-library only and must never import litellm. Claude Code re-runs it on every
 status refresh (about every 300ms while typing), so the proxy is asked at most once per
diff --git a/pyproject.toml b/pyproject.toml
index 95da93df41e..8a7d2981c12 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -21,6 +21,7 @@ dependencies = [
     "tiktoken>=0.8.0,<1.0; python_version < '3.14'",
     "tiktoken>=0.12.0,<1.0; python_version >= '3.14'",
     "importlib-metadata>=8.0.0,<9.0",
+    "packaging>=24.0",
     "tokenizers>=0.21.0,<1.0",
     "click>=8.0.0,<9.0",
     "jinja2>=3.1.6,<4.0",
diff --git a/tests/test_litellm/proxy/client/cli/test_claude_settings.py b/tests/test_litellm/proxy/client/cli/test_claude_settings.py
index a48c64eb4a0..f596e1e5d6c 100644
--- a/tests/test_litellm/proxy/client/cli/test_claude_settings.py
+++ b/tests/test_litellm/proxy/client/cli/test_claude_settings.py
@@ -6,6 +6,7 @@ import stat
 import sys
 import time
 from pathlib import Path
+from typing import Final
 from unittest.mock import patch
 
 import pytest
@@ -773,7 +774,7 @@ class TestStatusLine:
 
         script = tmp_path / "lite" / "statusline.py"
         command = install_statusline_script(script)
-        assert script.read_bytes() == pathlib.Path(statusline_script.__file__).read_bytes()
+        assert script.read_bytes().split(b"\n", 1)[1] == pathlib.Path(statusline_script.__file__).read_bytes()
         assert shlex.split(command) == [sys.executable, str(script)]
         assert command == statusline_command(script)
         assert stat.S_IMODE(script.stat().st_mode) == 0o600
@@ -783,11 +784,9 @@ class TestStatusLine:
     def test_a_reinstall_replaces_the_script_in_one_step_and_a_refused_one_leaves_the_old_script_whole(self, tmp_path):
         # Claude Code may be running the script at the moment `lite` reinstalls it; the file it has open
         # must stay complete, and a reinstall that cannot land must not leave a truncated script behind.
-        from litellm.proxy.client.cli.commands import statusline_script
-
         script = tmp_path / "lite" / "statusline.py"
         install_statusline_script(script)
-        bundled = pathlib.Path(statusline_script.__file__).read_bytes()
+        bundled = script.read_bytes()
         with script.open("rb") as running:
             install_statusline_script(script)
             assert running.read() == bundled
@@ -802,6 +801,67 @@ class TestStatusLine:
                 script.parent.chmod(0o700)
             assert script.read_bytes() == bundled
 
+    @pytest.mark.parametrize(
+        ("installed_version", "older_version"),
+        (("2.10.0", "2.9.0"), ("2.1.0", "2.1.0rc1"), ("2.1.0rc1", "2.1.0.dev2"), ("2.1.0.post1", "2.1.0")),
+    )
+    def test_an_older_cli_preserves_the_newer_footer(
+        self, tmp_path: Path, capsys: pytest.CaptureFixture[str], installed_version: str, older_version: str
+    ) -> None:
+        script: Final = tmp_path / "statusline.py"
+        command: Final = install_statusline_script(script, package_version=installed_version)
+        installed: Final = script.read_bytes()
+        modified: Final = script.stat().st_mtime_ns
+
+        assert install_statusline_script(script, package_version=older_version) == command
+
+        assert script.read_bytes() == installed
+        assert script.stat().st_mtime_ns == modified
+        assert f"Keeping the status line from LiteLLM {installed_version}" in capsys.readouterr().err
+
+    @pytest.mark.parametrize(
+        "old_header", (b"", b"# litellm-statusline-version: invalid\n", b"# litellm-statusline-version: \xff\n")
+    )
+    def test_a_legacy_or_damaged_version_marker_is_repaired(self, tmp_path: Path, old_header: bytes) -> None:
+        from litellm.proxy.client.cli.commands import statusline_script
+
+        script: Final = tmp_path / "statusline.py"
+        script.write_bytes(old_header + b"print('old footer')\n")
+
+        install_statusline_script(script, package_version="2.1.0")
+
+        assert script.read_bytes() == (
+            b"# litellm-statusline-version: 2.1.0\n" + Path(statusline_script.__file__).read_bytes()
+        )
+
+    @pytest.mark.parametrize("next_version", ("2.1.0", "2.2.0"))
+    def test_an_equal_or_newer_cli_refreshes_the_footer(self, tmp_path: Path, next_version: str) -> None:
+        from litellm.proxy.client.cli.commands import statusline_script
+
+        script: Final = tmp_path / "statusline.py"
+        script.write_bytes(b"# litellm-statusline-version: 2.1.0\nprint('old footer')\n")
+
+        install_statusline_script(script, package_version=next_version)
+
+        assert script.read_bytes() == (
+            f"# litellm-statusline-version: {next_version}\n".encode() + Path(statusline_script.__file__).read_bytes()
+        )
+
+    def test_configure_keeps_a_newer_footer_while_updating_settings(
+        self, tmp_path: Path, capsys: pytest.CaptureFixture[str]
+    ) -> None:
+        script: Final = tmp_path / "statusline.py"
+        script.write_bytes(b"# litellm-statusline-version: 999999.0.0\nprint('newer footer')\n")
+        installed: Final = script.read_bytes()
+        rig: Final = _Rig(tmp_path, {"theme": "dark"})
+
+        rig.configure(script_path=script)
+
+        assert script.read_bytes() == installed
+        assert rig.read()["statusLine"]["command"] == statusline_command(script)
+        assert rig.read()["env"]["ANTHROPIC_BASE_URL"] == PROXY
+        assert "Keeping the status line" in capsys.readouterr().err
+
     def test_configure_installs_it_and_unconfigure_removes_only_ours(self, tmp_path):
         rig = _Rig(tmp_path, {"theme": "dark"})
         script = tmp_path / "statusline.py"
diff --git a/uv.lock b/uv.lock
index 543581cbc23..e1d4ab791c6 100644
--- a/uv.lock
+++ b/uv.lock
@@ -4521,6 +4521,7 @@ dependencies = [
     { name = "jinja2" },
     { name = "jsonschema" },
     { name = "openai" },
+    { name = "packaging" },
     { name = "pydantic" },
     { name = "pydantic-settings" },
     { name = "python-dotenv" },
@@ -4802,6 +4803,7 @@ requires-dist = [
     { name = "numpydoc", marker = "extra == 'utils'", specifier = ">=1.8.0,<2.0" },
     { name = "nvidia-riva-client", marker = "extra == 'stt-nvidia-riva'", specifier = ">=2.15.0" },
     { name = "openai", specifier = ">=2.20.0,<3.0.0" },
+    { name = "packaging", specifier = ">=24.0" },
     { name = "opentelemetry-api", marker = "extra == 'proxy-runtime'", specifier = "==1.28.0" },
     { name = "opentelemetry-exporter-otlp", marker = "extra == 'proxy-runtime'", specifier = "==1.28.0" },
     { name = "opentelemetry-instrumentation-fastapi", marker = "extra == 'proxy-runtime'", specifier = "==0.49b0" },

From e86ba8bbebfad37f558b91bb67e5fe55a71493f2 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 21 Sep 2026 15:56:09 -0700
Subject: [PATCH 146/160] test(logging_worker): cover a same-loop flush and a
 repeated flush after a loop change

---
 litellm/litellm_core_utils/logging_worker.py  |  6 ++---
 .../litellm_core_utils/test_logging_worker.py | 23 +++++++++++++++++--
 2 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/litellm/litellm_core_utils/logging_worker.py b/litellm/litellm_core_utils/logging_worker.py
index cb3d8bf4fe5..2f8e7bdccea 100644
--- a/litellm/litellm_core_utils/logging_worker.py
+++ b/litellm/litellm_core_utils/logging_worker.py
@@ -485,10 +485,8 @@ class LoggingWorker:
         callback hasn't finished yet — ``queue.empty()`` would return True in
         that window and cause us to skip the wait.
 
-        ``start()`` runs first so that, after an event loop change, the tasks
-        still on the previous loop's queue move onto this loop and a worker
-        here drains them; joining the old queue directly would wait on a
-        counter nothing on this loop ever decrements.
+        ``start()`` runs first so a queue left behind by a previous event loop
+        is carried onto this one and drained here instead of joined forever.
         """
         if self._queue is None:
             return
diff --git a/tests/test_litellm/litellm_core_utils/test_logging_worker.py b/tests/test_litellm/litellm_core_utils/test_logging_worker.py
index 891bd0686b2..336067976fa 100644
--- a/tests/test_litellm/litellm_core_utils/test_logging_worker.py
+++ b/tests/test_litellm/litellm_core_utils/test_logging_worker.py
@@ -207,10 +207,29 @@ class TestLoggingWorker:
         assert (worker._queue.qsize(), len(worker._unstarted_dequeued_tasks())) == expected_shape
         assert fired == [], "precondition: the callback never ran before the first loop closed"
 
-        async def flush_on_second_loop():
+        async def flush_twice_on_second_loop():
+            await asyncio.wait_for(worker.flush(), timeout=5)
             await asyncio.wait_for(worker.flush(), timeout=5)
 
-        asyncio.run(flush_on_second_loop())
+        asyncio.run(flush_twice_on_second_loop())
+
+        assert fired == [True]
+
+    def test_flush_starts_a_worker_when_the_queue_has_none(self):
+        """``flush()`` must drain a queue that exists on the current loop without a running worker."""
+        worker = LoggingWorker(timeout=1.0, max_queue_size=10)
+        fired = []
+
+        async def marker():
+            fired.append(True)
+
+        async def enqueue_then_flush():
+            worker._ensure_queue()
+            worker.enqueue(marker())
+            assert worker._worker_task is None, "precondition: nothing is draining the queue yet"
+            await asyncio.wait_for(worker.flush(), timeout=3)
+
+        asyncio.run(enqueue_then_flush())
 
         assert fired == [True]
 

From c371617786752c20727936704e0257ffa2f55479 Mon Sep 17 00:00:00 2001
From: "berriai-litellm-provider-info-sync[bot]"
 <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com>
Date: Mon, 21 Sep 2026 23:01:01 +0000
Subject: [PATCH 147/160] chore(prices): sync OpenRouter prices: 4 models, 3
 deprecated

openrouter/bytedance-seed/seed-1.6: deprecation_date
openrouter/bytedance-seed/seed-1.6-flash: deprecation_date
openrouter/bytedance-seed/seed-2.0-code: deprecation_date
openrouter/deepseek/deepseek-v4-pro: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost
---
 litellm/model_prices_and_context_window_backup.json | 9 ++++++---
 model_prices_and_context_window.json                | 9 ++++++---
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index beb90c2fb8c..6a414a0908e 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -43037,21 +43037,21 @@
         "supports_web_search": false
     },
     "openrouter/deepseek/deepseek-v4-pro": {
-        "input_cost_per_token": 8.92272e-07,
+        "input_cost_per_token": 8.87226e-07,
         "input_cost_per_token_cache_hit": 4.4e-08,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 384000,
         "max_tokens": 384000,
         "mode": "chat",
-        "output_cost_per_token": 1.784544e-06,
+        "output_cost_per_token": 1.774452e-06,
         "source": "https://openrouter.ai/api/v1/models",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "cache_read_input_token_cost": 7.4356e-08,
+        "cache_read_input_token_cost": 7.39355e-08,
         "supports_audio_input": false,
         "supports_pdf_input": false,
         "supports_vision": false,
@@ -73794,6 +73794,7 @@
         "supports_web_search": false
     },
     "openrouter/bytedance-seed/seed-1.6": {
+        "deprecation_date": "2026-11-11",
         "input_cost_per_token": 2.5e-07,
         "input_cost_per_token_above_128k_tokens": 5e-07,
         "litellm_provider": "openrouter",
@@ -73815,6 +73816,7 @@
         "supports_web_search": false
     },
     "openrouter/bytedance-seed/seed-1.6-flash": {
+        "deprecation_date": "2026-11-11",
         "input_cost_per_token": 7.5e-08,
         "input_cost_per_token_above_128k_tokens": 1e-07,
         "litellm_provider": "openrouter",
@@ -73855,6 +73857,7 @@
         "supports_web_search": false
     },
     "openrouter/bytedance-seed/seed-2.0-code": {
+        "deprecation_date": "2026-11-11",
         "input_cost_per_token": 5e-07,
         "input_cost_per_token_above_128k_tokens": 1e-06,
         "litellm_provider": "openrouter",
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index beb90c2fb8c..6a414a0908e 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -43037,21 +43037,21 @@
         "supports_web_search": false
     },
     "openrouter/deepseek/deepseek-v4-pro": {
-        "input_cost_per_token": 8.92272e-07,
+        "input_cost_per_token": 8.87226e-07,
         "input_cost_per_token_cache_hit": 4.4e-08,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 384000,
         "max_tokens": 384000,
         "mode": "chat",
-        "output_cost_per_token": 1.784544e-06,
+        "output_cost_per_token": 1.774452e-06,
         "source": "https://openrouter.ai/api/v1/models",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "cache_read_input_token_cost": 7.4356e-08,
+        "cache_read_input_token_cost": 7.39355e-08,
         "supports_audio_input": false,
         "supports_pdf_input": false,
         "supports_vision": false,
@@ -73794,6 +73794,7 @@
         "supports_web_search": false
     },
     "openrouter/bytedance-seed/seed-1.6": {
+        "deprecation_date": "2026-11-11",
         "input_cost_per_token": 2.5e-07,
         "input_cost_per_token_above_128k_tokens": 5e-07,
         "litellm_provider": "openrouter",
@@ -73815,6 +73816,7 @@
         "supports_web_search": false
     },
     "openrouter/bytedance-seed/seed-1.6-flash": {
+        "deprecation_date": "2026-11-11",
         "input_cost_per_token": 7.5e-08,
         "input_cost_per_token_above_128k_tokens": 1e-07,
         "litellm_provider": "openrouter",
@@ -73855,6 +73857,7 @@
         "supports_web_search": false
     },
     "openrouter/bytedance-seed/seed-2.0-code": {
+        "deprecation_date": "2026-11-11",
         "input_cost_per_token": 5e-07,
         "input_cost_per_token_above_128k_tokens": 1e-06,
         "litellm_provider": "openrouter",

From d38514dfc6be3ad591d3462ee57b1d7987a3c1fe Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 23:02:23 +0000
Subject: [PATCH 148/160] test(cache-redis-semantic): pin shared-index behavior
 across embedding dimensions

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../cache-redis-semantic/tests/cache.rs       | 148 ++++++++++++++++++
 1 file changed, 148 insertions(+)

diff --git a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs
index 85a35a033b2..fc37cf9f97f 100644
--- a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs
+++ b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs
@@ -817,6 +817,154 @@ async fn async_paths_embed_then_run_blocking_redis_work() {
     );
 }
 
+#[test]
+fn shared_base_index_across_dimensions_replaces_the_isolated_index() {
+    // Pins parity with Python's `_isolated` + overwrite=True flow.
+    let prompt = "shared prompt";
+    let tag = "key1";
+    let isolated = format!("{INDEX}_isolated");
+    let value = entry();
+    let context = || messages_context(vec![json!({"role": "user", "content": prompt})]);
+    let store_hash = |index: &str, vector: &[f32]| {
+        MockCmd::new(
+            redis::cmd("HSET")
+                .arg(format!("{index}:{}", entry_id(prompt, tag)))
+                .arg("entry_id")
+                .arg(entry_id(prompt, tag))
+                .arg("prompt")
+                .arg(prompt)
+                .arg("response")
+                .arg(encoded(&value))
+                .arg("prompt_vector")
+                .arg(vector_bytes(vector))
+                .arg("inserted_at")
+                .arg("1700000000.5")
+                .arg("updated_at")
+                .arg("1700000000.5")
+                .arg("litellm_cache_key")
+                .arg(tag),
+            Ok(7),
+        )
+    };
+
+    let vector_a = vec![0.1f32; 8];
+    let connection_a = MockRedisConnection::new([
+        MockCmd::new(
+            redis::cmd("FT.INFO").arg(INDEX),
+            Err::<redis::Value, _>(unknown_index_error()),
+        ),
+        MockCmd::new(create_index_command(INDEX, 8), Ok("OK")),
+        store_hash(INDEX, &vector_a),
+    ])
+    .assert_all_commands_consumed();
+    let (embedder_a, _) = FakeEmbedder::new(&[(prompt, &vector_a)]);
+    let worker_a = RedisSemanticCache::with_connection(connection_a, embedder_a, config())
+        .with_clock(|| 1700000000.5);
+    worker_a.set_cache(tag, value.clone(), &context()).unwrap();
+
+    let vector_b = vec![0.2f32; 4];
+    let connection_b = MockRedisConnection::new([
+        MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(8))),
+        MockCmd::new(
+            redis::cmd("FT.INFO").arg(&isolated),
+            Err::<redis::Value, _>(unknown_index_error()),
+        ),
+        MockCmd::new(create_index_command(&isolated, 4), Ok("OK")),
+        store_hash(&isolated, &vector_b),
+        MockCmd::new(
+            search_command(&isolated, tag, &vector_b),
+            Ok(search_result(hit_fields(tag, "0.0", encoded(&value)))),
+        ),
+        MockCmd::new(
+            search_command(&isolated, tag, &vector_b),
+            Err::<redis::Value, _>(redis::RedisError::from((
+                redis::ErrorKind::Extension,
+                "Vector dimension mismatch",
+            ))),
+        ),
+    ])
+    .assert_all_commands_consumed();
+    let (embedder_b, _) = FakeEmbedder::new(&[(prompt, &vector_b)]);
+    let worker_b = RedisSemanticCache::with_connection(connection_b, embedder_b, config())
+        .with_clock(|| 1700000000.5);
+    worker_b.set_cache(tag, value.clone(), &context()).unwrap();
+    assert_eq!(
+        worker_b.get_cache(tag, &context()).unwrap(),
+        Some(value.clone())
+    );
+
+    let vector_c = vec![0.3f32; 16];
+    let connection_c = MockRedisConnection::new([
+        MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(8))),
+        MockCmd::new(redis::cmd("FT.INFO").arg(&isolated), Ok(compatible_info(4))),
+        MockCmd::new(redis::cmd("FT.DROPINDEX").arg(&isolated), Ok("OK")),
+        MockCmd::new(create_index_command(&isolated, 16), Ok("OK")),
+        store_hash(&isolated, &vector_c),
+    ])
+    .assert_all_commands_consumed();
+    let (embedder_c, _) = FakeEmbedder::new(&[(prompt, &vector_c)]);
+    let worker_c = RedisSemanticCache::with_connection(connection_c, embedder_c, config())
+        .with_clock(|| 1700000000.5);
+    worker_c.set_cache(tag, value.clone(), &context()).unwrap();
+
+    assert_eq!(
+        worker_b.get_cache(tag, &context()).unwrap_err(),
+        Error::Unavailable
+    );
+}
+
+#[test]
+fn live_shared_index_is_replaced_across_dimensions() {
+    let Ok(url) = std::env::var("LITELLM_REDIS_STACK_URL") else {
+        return;
+    };
+    // Pins parity with Python's `_isolated` + overwrite=True flow.
+    let base = format!("rust_semantic_shared_{}", std::process::id());
+    let isolated = format!("{base}_isolated");
+    let prompt = "shared live prompt";
+    let tag = "key1";
+    let context = || messages_context(vec![json!({"role": "user", "content": prompt})]);
+    let value = entry();
+    let worker = |vector: Vec<f32>| {
+        let (embedder, _) = FakeEmbedder::new(&[(prompt, vector.as_slice())]);
+        RedisSemanticCache::new(
+            &url,
+            embedder,
+            RedisSemanticConfig {
+                index_name: base.clone(),
+                similarity_threshold: 0.9,
+            },
+        )
+        .unwrap()
+    };
+
+    let worker_a = worker(vec![0.1f32; 8]);
+    worker_a.set_cache(tag, value.clone(), &context()).unwrap();
+
+    let worker_b = worker(vec![0.2f32; 4]);
+    worker_b.set_cache(tag, value.clone(), &context()).unwrap();
+    assert_eq!(
+        worker_b.get_cache(tag, &context()).unwrap(),
+        Some(value.clone())
+    );
+
+    let worker_c = worker(vec![0.3f32; 16]);
+    worker_c.set_cache(tag, value.clone(), &context()).unwrap();
+
+    assert_eq!(
+        worker_b.get_cache(tag, &context()).unwrap_err(),
+        Error::Unavailable
+    );
+
+    let mut connection = redis::Client::open(url).unwrap().get_connection().unwrap();
+    for index in [&base, &isolated] {
+        let _: Result<(), _> = redis::cmd("FT.DROPINDEX")
+            .arg(index)
+            .arg("DD")
+            .query(&mut connection);
+    }
+}
+
 #[test]
 fn live_store_lookup_and_ttl_against_redis_stack() {
     let Ok(url) = std::env::var("LITELLM_REDIS_STACK_URL") else {

From 7282494c30e009ba655e6f1453e8c28f6e1e5f3c Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 23:04:44 +0000
Subject: [PATCH 149/160] fix(python-bridge): propagate cancellation from
 semantic embedding awaits

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../python-bridge/src/cache/semantic.rs       | 19 +++++++---
 tests/test_litellm_rust/test_cache.py         | 35 +++++++++++++++++++
 2 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/litellm-rust/crates/python-bridge/src/cache/semantic.rs b/litellm-rust/crates/python-bridge/src/cache/semantic.rs
index eb38b8b9c67..f0f75de1edf 100644
--- a/litellm-rust/crates/python-bridge/src/cache/semantic.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/semantic.rs
@@ -3,7 +3,11 @@ use std::collections::VecDeque;
 use litellm_cache::Error;
 use litellm_cache_redis_semantic::prompt_from_context;
 use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async};
-use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*};
+use pyo3::{
+    PyTraverseError, PyVisit,
+    exceptions::{PyException, PyRuntimeError},
+    prelude::*,
+};
 use serde_json::Value;
 
 use super::{
@@ -120,9 +124,16 @@ impl ExecutionBody for SemanticBody {
                                 "semantic execution expected an embedding result",
                             )
                         })?;
-                        let seed = result
-                            .and_then(|value| PythonEmbedder::extract(value.into_bound(py)))
-                            .map_err(|_| Error::Unavailable);
+                        let seed = match result {
+                            Ok(value) => PythonEmbedder::extract(value.into_bound(py))
+                                .map_err(|_| Error::Unavailable),
+                            Err(error) => {
+                                if !error.is_instance_of::<PyException>(py) {
+                                    return Err(error);
+                                }
+                                Err(Error::Unavailable)
+                            }
+                        };
                         return self.backend_step(py, seed);
                     }
                     Phase::AwaitingBackend => {
diff --git a/tests/test_litellm_rust/test_cache.py b/tests/test_litellm_rust/test_cache.py
index 3f5449a1aa9..8a8c83cd82b 100644
--- a/tests/test_litellm_rust/test_cache.py
+++ b/tests/test_litellm_rust/test_cache.py
@@ -736,6 +736,8 @@ class DeterministicEmbedding(litellm.CustomLLM):
     def __init__(self) -> None:
         self.calls: list[dict[str, object]] = []
         self.async_calls: list[dict[str, object]] = []
+        self.entered = asyncio.Event()
+        self.gate: asyncio.Event | None = None
 
     def _respond(
         self,
@@ -790,6 +792,9 @@ class DeterministicEmbedding(litellm.CustomLLM):
             }
         )
         SEMANTIC_CONTEXT.set("written-in-aembedding")
+        self.entered.set()
+        if self.gate is not None:
+            await self.gate.wait()
         return self._respond(model, input, model_response)
 
 
@@ -1036,6 +1041,36 @@ async def test_native_semantic_async_embedding_runs_inline_in_the_callers_task(
     ], semantic_embedding.async_calls
 
 
+async def test_native_semantic_cancellation_during_embedding_skips_the_backend(
+    redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding
+) -> None:
+    url, index = redis_stack
+    facade: Final = semantic_facade(url, index)
+    binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve()
+    assert binding.kind == "native"
+    semantic_embedding.gate = asyncio.Event()
+
+    async def lookup() -> object:
+        return await binding.async_lookup(
+            semantic_request("cancel", "cancelled prompt")
+        )
+
+    task: Final = asyncio.create_task(lookup())
+    await semantic_embedding.entered.wait()
+    task.cancel()
+    with pytest.raises(asyncio.CancelledError):
+        await task
+    semantic_embedding.gate.set()
+
+    assert len(semantic_embedding.async_calls) == 1
+    assert (
+        await cast(RedisSemanticCache, facade.cache).async_get_cache(  # pyright: ignore[reportUnknownMemberType]  # **kwargs stays unknown on the backend class
+            "cancel", messages=semantic_messages("cancelled prompt")
+        )
+        is None
+    )
+
+
 def test_redis_semantic_similarity_tag_and_threshold_boundaries(
     redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding
 ) -> None:

From 9fa1683ed97f29f4593b28259c0250a9f771f2c8 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 23:09:03 +0000
Subject: [PATCH 150/160] chore(cache): keep main's UnsupportedOperation
 message

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/cache/src/error.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/litellm-rust/crates/cache/src/error.rs b/litellm-rust/crates/cache/src/error.rs
index 51e4fe2d66a..1a381d0afd8 100644
--- a/litellm-rust/crates/cache/src/error.rs
+++ b/litellm-rust/crates/cache/src/error.rs
@@ -6,6 +6,6 @@ pub enum Error {
     InvalidEntry,
     #[error("flushing Redis requires an explicit namespace")]
     UnscopedFlush,
-    #[error("cache operation is not supported by this backend")]
+    #[error("operation is not supported by this cache")]
     UnsupportedOperation,
 }

From 47a1053065116e54f78f287ba50c0d7756a195bc Mon Sep 17 00:00:00 2001
From: kerry <kerry@berri.ai>
Date: Mon, 21 Sep 2026 23:12:57 +0000
Subject: [PATCH 151/160] feat(xiaomi_mimo): add mimo-v2.6-pro and
 mimo-v2.6-flash cost map rows with live e2e coverage

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 ...odel_prices_and_context_window_backup.json |  46 ++++
 model_prices_and_context_window.json          |  46 ++++
 .../coverage_registry/llm_conversational.yaml |   3 +
 tests/e2e/coverage_registry/schema.py         |   1 +
 .../llm_translation/test_xiaomi_mimo_e2e.py   | 214 ++++++++++++++++++
 5 files changed, 310 insertions(+)
 create mode 100644 tests/e2e/llm_translation/test_xiaomi_mimo_e2e.py

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index beb90c2fb8c..2d8be7f1e87 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -76975,5 +76975,51 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": false
+    },
+    "xiaomi_mimo/mimo-v2.6-pro": {
+        "cache_read_input_token_cost": 3.6e-09,
+        "input_cost_per_token": 4.35e-07,
+        "litellm_provider": "xiaomi_mimo",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 8.7e-07,
+        "source": "https://platform.xiaomimimo.com/static/docs/price/pay-as-you-go.md",
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_audio_input": true,
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_video_input": true,
+        "supports_vision": true
+    },
+    "xiaomi_mimo/mimo-v2.6-flash": {
+        "cache_read_input_token_cost": 2.8e-09,
+        "input_cost_per_token": 1.4e-07,
+        "litellm_provider": "xiaomi_mimo",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 2.8e-07,
+        "source": "https://platform.xiaomimimo.com/static/docs/price/pay-as-you-go.md",
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_audio_input": true,
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_video_input": true,
+        "supports_vision": true
     }
 }
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index beb90c2fb8c..2d8be7f1e87 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -76975,5 +76975,51 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": false
+    },
+    "xiaomi_mimo/mimo-v2.6-pro": {
+        "cache_read_input_token_cost": 3.6e-09,
+        "input_cost_per_token": 4.35e-07,
+        "litellm_provider": "xiaomi_mimo",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 8.7e-07,
+        "source": "https://platform.xiaomimimo.com/static/docs/price/pay-as-you-go.md",
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_audio_input": true,
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_video_input": true,
+        "supports_vision": true
+    },
+    "xiaomi_mimo/mimo-v2.6-flash": {
+        "cache_read_input_token_cost": 2.8e-09,
+        "input_cost_per_token": 1.4e-07,
+        "litellm_provider": "xiaomi_mimo",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 2.8e-07,
+        "source": "https://platform.xiaomimimo.com/static/docs/price/pay-as-you-go.md",
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_audio_input": true,
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_video_input": true,
+        "supports_vision": true
     }
 }
diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml
index 49d4d92ff0b..64335aa560c 100644
--- a/tests/e2e/coverage_registry/llm_conversational.yaml
+++ b/tests/e2e/coverage_registry/llm_conversational.yaml
@@ -89,6 +89,9 @@
 - {id: llm.chat_completions.together_ai.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool result round trip"}
 - {id: llm.chat_completions.together_ai.basic.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together cost header and spend row match the registry price"}
 - {id: llm.chat_completions.together_ai.thinking.nonstream.effort_none_disables, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: thinking, streaming: nonstream, assertions: [effort_none_disables], source: "llm_translation/test_together_ai_e2e.py", rationale: "reasoning_effort=none maps to Together's reasoning disable toggle on hybrid models"}
+- {id: llm.chat_completions.xiaomi_mimo.basic.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: xiaomi_mimo, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_xiaomi_mimo_e2e.py", rationale: "Native MiMo v2.6 rows price the cost header and spend row from the cost map"}
+- {id: llm.chat_completions.xiaomi_mimo.thinking.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: xiaomi_mimo, capability: thinking, streaming: stream, assertions: [works], source: "llm_translation/test_xiaomi_mimo_e2e.py", rationale: "MiMo reasoning deltas stream as reasoning_content"}
+- {id: llm.chat_completions.xiaomi_mimo.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: xiaomi_mimo, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_xiaomi_mimo_e2e.py", rationale: "MiMo tool calls are not dropped"}
 - {id: llm.chat_completions.together_ai.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: structured_output, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "response_format json_schema reaches Together and constrains the reply"}
 - {id: llm.chat_completions.together_ai.prompt_cache_5m.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: prompt_cache_5m, streaming: nonstream, assertions: [cache_hit, cost_logged], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together prefix-cache reads bill at cache_read_input_token_cost, not full input price"}
 - {id: llm.messages.together_ai.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together over /v1/messages streaming"}
diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py
index 8b0d38a083c..d9c20d5c588 100644
--- a/tests/e2e/coverage_registry/schema.py
+++ b/tests/e2e/coverage_registry/schema.py
@@ -58,6 +58,7 @@ LlmRoute = Literal[
     "openai",
     "together_ai",
     "vertex",
+    "xiaomi_mimo",
 ]
 
 LlmCapability = Literal[
diff --git a/tests/e2e/llm_translation/test_xiaomi_mimo_e2e.py b/tests/e2e/llm_translation/test_xiaomi_mimo_e2e.py
new file mode 100644
index 00000000000..efca216634b
--- /dev/null
+++ b/tests/e2e/llm_translation/test_xiaomi_mimo_e2e.py
@@ -0,0 +1,214 @@
+"""Live e2e: Xiaomi MiMo v2.6 through the gateway on /chat/completions.
+
+Both native ``xiaomi_mimo/`` v2.6 rows (pro and flash) are registered via
+``/model/new`` and driven against Xiaomi's own endpoint. What the gateway owes
+us is that the reasoning chain surfaces as ``reasoning_content``, tool calls
+survive translation, and the cost header plus spend row follow the proxy's own
+cost-map price for the row (read back from ``/model/info``, never pinned here).
+Requires XIAOMI_MIMO_API_KEY on the proxy; no skip gate.
+"""
+
+from __future__ import annotations
+
+from typing import Final
+
+import pytest
+from e2e_config import unique_marker
+from e2e_http import StreamingResponse, require_successful_call, unwrap
+from lifecycle import ResourceManager
+from models import (
+    ChatBody,
+    ChatMessage,
+    ChatResponse,
+    ChatTool,
+    ChatToolFunction,
+    CostMapEntry,
+    LiteLLMParamsBody,
+    OutMessage,
+    SpendLogRow,
+)
+from passthrough_client import PassthroughClient
+from pydantic import BaseModel
+
+pytestmark = pytest.mark.e2e
+
+BACKENDS: Final = ("xiaomi_mimo/mimo-v2.6-pro", "xiaomi_mimo/mimo-v2.6-flash")
+ARITHMETIC_PROMPT = "What is 17 + 26? Answer with just the number."
+WEATHER_PROMPT = "What is the weather in Paris? Use the tool."
+COUNTING_PROMPT = "Count from 1 to 50, one number per line."
+
+WEATHER_TOOL = ChatTool(
+    function=ChatToolFunction(
+        name="get_weather",
+        description="Get the current weather for a location.",
+        parameters={
+            "type": "object",
+            "properties": {"location": {"type": "string"}},
+            "required": ["location"],
+        },
+    )
+)
+
+
+class _WeatherArgs(BaseModel):
+    location: str
+
+
+class _StreamDelta(BaseModel):
+    content: str | None = None
+    reasoning_content: str | None = None
+
+
+class _StreamChoice(BaseModel):
+    delta: _StreamDelta | None = None
+
+
+class _StreamChunk(BaseModel):
+    choices: list[_StreamChoice] = []
+
+
+def _approx_equal(actual: float, expected: float) -> bool:
+    return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2)
+
+
+@pytest.fixture(scope="module")
+def registry(client: PassthroughClient) -> dict[str, CostMapEntry]:
+    return client.proxy.model_cost_map()
+
+
+def _register(client: PassthroughClient, resources: ResourceManager, backend: str) -> tuple[str, str]:
+    model = f"e2e-xiaomi-{unique_marker()}"
+    model_id = client.proxy.create_model(
+        model, LiteLLMParamsBody(model=backend, api_key="os.environ/XIAOMI_MIMO_API_KEY")
+    )
+    resources.defer(lambda: client.proxy.delete_model(model_id))
+    return model, resources.key()
+
+
+def _message(response: ChatResponse) -> OutMessage:
+    assert response.choices, f"Xiaomi returned no choices: {response}"
+    message = response.choices[0].message
+    assert message is not None, f"Xiaomi choice has no message: {response}"
+    return message
+
+
+def _deltas(result: StreamingResponse) -> list[_StreamDelta]:
+    require_successful_call(result)
+    assert result.is_streaming, f"response was not streamed: {result.headers}"
+    assert not result.stream_error, f"stream errored: {result.stream_error}"
+    assert result.stream_done, f"stream never reached [DONE]: {result.stream_events[-3:]}"
+    return [
+        choice.delta
+        for event in result.stream_events
+        for choice in _StreamChunk.model_validate_json(event).choices
+        if choice.delta is not None
+    ]
+
+
+@pytest.mark.parametrize("backend", BACKENDS)
+class TestXiaomiMimoChatCompletions:
+    @pytest.mark.covers("llm.chat_completions.xiaomi_mimo.basic.nonstream.cost_logged")
+    def test_cost_header_and_spend_row_match_the_registry_price(
+        self,
+        client: PassthroughClient,
+        resources: ResourceManager,
+        registry: dict[str, CostMapEntry],
+        backend: str,
+    ) -> None:
+        price = registry.get(backend)
+        assert price is not None, f"{backend} has no row in the proxy's cost map, so native calls would bill $0"
+        assert price.litellm_provider == "xiaomi_mimo", f"{backend} is filed under the wrong provider: {price}"
+        assert price.input_cost_per_token and price.output_cost_per_token, f"{backend} carries no price: {price}"
+        model, key = _register(client, resources, backend)
+
+        result = client.proxy.transport.send(
+            "/chat/completions",
+            headers=client.proxy.transport.bearer(key),
+            json=ChatBody(
+                model=model,
+                messages=[ChatMessage(role="user", content=f"{ARITHMETIC_PROMPT} {unique_marker()}")],
+                max_tokens=1024,
+            ),
+        )
+        require_successful_call(result)
+        response = ChatResponse.model_validate_json(result.body)
+        message = _message(response)
+        assert message.content and "43" in message.content, f"answer lost: {message}"
+        assert message.reasoning_content, f"{backend} reasons, but no reasoning_content came back: {message}"
+
+        usage = response.usage
+        assert usage is not None and usage.prompt_tokens and usage.completion_tokens, (
+            f"response carries no usage, so the cost cannot be real: {result.body[:300]}"
+        )
+        header_cost = result.response_cost
+        assert header_cost is not None and header_cost > 0, (
+            f"x-litellm-response-cost header missing or non-positive: {result.headers}"
+        )
+        cached = (usage.prompt_tokens_details.cached_tokens or 0) if usage.prompt_tokens_details else 0
+        expected = (
+            (usage.prompt_tokens - cached) * price.input_cost_per_token
+            + cached * (price.cache_read_input_token_cost or 0.0)
+            + usage.completion_tokens * price.output_cost_per_token
+        )
+        assert _approx_equal(header_cost, expected), (
+            f"header cost {header_cost} disagrees with the registry price for {backend} at {usage}: expected {expected}"
+        )
+
+        def _priced(rows: list[SpendLogRow]) -> bool:
+            return any(row.spend is not None and row.spend > 0 for row in rows)
+
+        rows = client.proxy.poll_logs_for_key(key, predicate=_priced)
+        priced = [row for row in rows if row.spend is not None and row.spend > 0]
+        assert priced, f"no priced spend row landed for key {key}; got {rows}"
+        row = priced[0]
+        assert row.custom_llm_provider == "xiaomi_mimo", f"spend row misattributed: {row}"
+        assert row.spend is not None and _approx_equal(row.spend, header_cost), (
+            f"logged spend {row.spend} disagrees with the x-litellm-response-cost header {header_cost}"
+        )
+
+    @pytest.mark.covers("llm.chat_completions.xiaomi_mimo.thinking.stream.works")
+    def test_reasoning_and_answer_stream_as_deltas(
+        self, client: PassthroughClient, resources: ResourceManager, backend: str
+    ) -> None:
+        model, key = _register(client, resources, backend)
+
+        deltas = _deltas(
+            client.proxy.chat_stream(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[ChatMessage(role="user", content=COUNTING_PROMPT)],
+                    max_tokens=2048,
+                    stream=True,
+                ),
+            )
+        )
+        reasoning = "".join(delta.reasoning_content or "" for delta in deltas)
+        content = "".join(delta.content or "" for delta in deltas)
+        assert reasoning, f"stream carried no reasoning_content deltas: {deltas[:5]}"
+        assert "50" in content, f"streamed answer lost: {content[:300]!r}"
+
+    @pytest.mark.covers("llm.chat_completions.xiaomi_mimo.tool_use.nonstream.works")
+    def test_tool_call_is_returned(self, client: PassthroughClient, resources: ResourceManager, backend: str) -> None:
+        model, key = _register(client, resources, backend)
+
+        message = _message(
+            unwrap(
+                client.proxy.chat(
+                    key,
+                    ChatBody(
+                        model=model,
+                        messages=[ChatMessage(role="user", content=WEATHER_PROMPT)],
+                        tools=[WEATHER_TOOL],
+                        max_tokens=1024,
+                    ),
+                )
+            )
+        )
+        assert message.tool_calls, f"{backend} dropped the tool call: {message}"
+        call = message.tool_calls[0]
+        assert call.id, f"tool call carries no id, so a tool result cannot answer it: {call}"
+        assert call.function.name == "get_weather", f"wrong tool called: {call}"
+        assert call.function.arguments, f"tool call carries no arguments: {call}"
+        args = _WeatherArgs.model_validate_json(call.function.arguments)
+        assert "paris" in args.location.lower(), f"tool arguments lost the location: {args}"

From 9388602f467b4c1d3ce2f137191c4a8638a8a5d1 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 21 Sep 2026 16:13:09 -0700
Subject: [PATCH 152/160] test(logging_worker): track callback runs with
 AsyncMock instead of a mutated list

---
 .../litellm_core_utils/test_logging_worker.py | 22 +++++++------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/tests/test_litellm/litellm_core_utils/test_logging_worker.py b/tests/test_litellm/litellm_core_utils/test_logging_worker.py
index 336067976fa..2bb93a58531 100644
--- a/tests/test_litellm/litellm_core_utils/test_logging_worker.py
+++ b/tests/test_litellm/litellm_core_utils/test_logging_worker.py
@@ -189,23 +189,20 @@ class TestLoggingWorker:
         ``RuntimeError: ... is bound to a different event loop`` from the queue's Event.
         """
         worker = LoggingWorker(timeout=1.0, max_queue_size=10)
-        fired = []
-
-        async def marker():
-            fired.append(True)
+        callback = AsyncMock()
 
         async def enqueue_on_first_loop():
             if stranded == "still_queued":
                 worker._ensure_queue()
-                worker.enqueue(marker())
+                worker.enqueue(callback())
                 return
-            worker.ensure_initialized_and_enqueue(marker())
+            worker.ensure_initialized_and_enqueue(callback())
 
         asyncio.run(enqueue_on_first_loop())
         assert worker._queue is not None
         expected_shape = (1, 0) if stranded == "still_queued" else (0, 1)
         assert (worker._queue.qsize(), len(worker._unstarted_dequeued_tasks())) == expected_shape
-        assert fired == [], "precondition: the callback never ran before the first loop closed"
+        assert callback.await_count == 0, "precondition: the callback never ran before the first loop closed"
 
         async def flush_twice_on_second_loop():
             await asyncio.wait_for(worker.flush(), timeout=5)
@@ -213,25 +210,22 @@ class TestLoggingWorker:
 
         asyncio.run(flush_twice_on_second_loop())
 
-        assert fired == [True]
+        assert callback.await_count == 1
 
     def test_flush_starts_a_worker_when_the_queue_has_none(self):
         """``flush()`` must drain a queue that exists on the current loop without a running worker."""
         worker = LoggingWorker(timeout=1.0, max_queue_size=10)
-        fired = []
-
-        async def marker():
-            fired.append(True)
+        callback = AsyncMock()
 
         async def enqueue_then_flush():
             worker._ensure_queue()
-            worker.enqueue(marker())
+            worker.enqueue(callback())
             assert worker._worker_task is None, "precondition: nothing is draining the queue yet"
             await asyncio.wait_for(worker.flush(), timeout=3)
 
         asyncio.run(enqueue_then_flush())
 
-        assert fired == [True]
+        assert callback.await_count == 1
 
     def test_flush_on_exit_swallows_cancellation_and_drains_remaining(self):
         """A callback raising CancelledError must not abort the atexit flush of later events."""

From 9af3363c2fd646bc5fc8945c22258c9e8fb368dd Mon Sep 17 00:00:00 2001
From: kerry <kerry@berri.ai>
Date: Mon, 21 Sep 2026 23:18:18 +0000
Subject: [PATCH 153/160] fix(bedrock): add bare moonshotai.kimi-k3 cost map
 entry mirroring the global inference profile

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 model_prices_and_context_window.json | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 6a414a0908e..b9b47fd45b5 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -76879,6 +76879,26 @@
         "supports_vision": true,
         "supports_web_search": true
     },
+    "moonshotai.kimi-k3": {
+        "cache_creation_input_token_cost": 3.75e-06,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "supports_audio_input": false,
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "global.moonshotai.kimi-k3": {
         "cache_creation_input_token_cost": 3.75e-06,
         "cache_read_input_token_cost": 3e-07,

From 2915272f2f870f11e5bad9b1ca3738a88d424c06 Mon Sep 17 00:00:00 2001
From: Tin Chi Lo <tin@berri.ai>
Date: Mon, 21 Sep 2026 16:20:16 -0700
Subject: [PATCH 154/160] chore: retain the CI-generated API snapshot

---
 litellm/proxy/_lazy_openapi_snapshot.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json
index 8bce873aa24..6f9a2d8c96d 100644
--- a/litellm/proxy/_lazy_openapi_snapshot.json
+++ b/litellm/proxy/_lazy_openapi_snapshot.json
@@ -19632,7 +19632,7 @@
                   }
                 }
               },
-              "description": "\nUnified rate-limit error.\n\nEvery rate-limit condition surfaced by litellm \u2014 whether it originated from\nan upstream LLM provider, a vendor batch endpoint, or one of litellm's own\nproxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\nmax-iterations, etc.) \u2014 is raised as an instance of this class.\n\nThe :attr:`category` attribute lets callers distinguish the source. See\n:class:`RateLimitErrorCategory` for the available values.\n"
+              "description": "\n    Unified rate-limit error.\n\n    Every rate-limit condition surfaced by litellm \u2014 whether it originated from\n    an upstream LLM provider, a vendor batch endpoint, or one of litellm's own\n    proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\n    max-iterations, etc.) \u2014 is raised as an instance of this class.\n\n    The :attr:`category` attribute lets callers distinguish the source. See\n    :class:`RateLimitErrorCategory` for the available values.\n    "
             },
             "500": {
               "content": {

From cbed8cd0d81c351cb368ab90cb72d432a046322d Mon Sep 17 00:00:00 2001
From: kerry <kerry@berri.ai>
Date: Mon, 21 Sep 2026 23:20:26 +0000
Subject: [PATCH 155/160] fix(bedrock): sync moonshotai.kimi-k3 entry into cost
 map backup file

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 ...odel_prices_and_context_window_backup.json | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 6a414a0908e..b9b47fd45b5 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -76879,6 +76879,26 @@
         "supports_vision": true,
         "supports_web_search": true
     },
+    "moonshotai.kimi-k3": {
+        "cache_creation_input_token_cost": 3.75e-06,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "supports_audio_input": false,
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "global.moonshotai.kimi-k3": {
         "cache_creation_input_token_cost": 3.75e-06,
         "cache_read_input_token_cost": 3e-07,

From e350455a668ff18b2e19d3682039ea9b7348119a Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 23:30:14 +0000
Subject: [PATCH 156/160] fix(rust): declare _CacheTestHandle.valkey_semantic
 in native stub

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/rust_bridge/_native.pyi | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi
index 90d876b71e6..978e770b4be 100644
--- a/litellm/rust_bridge/_native.pyi
+++ b/litellm/rust_bridge/_native.pyi
@@ -169,6 +169,13 @@ class _CacheTestHandle:
     @staticmethod
     def azure_blob(account_url: str, container: str) -> _CacheTestHandle: ...
     @staticmethod
+    def valkey_semantic(
+        url: str,
+        similarity_threshold: float,
+        index_name: str,
+        embedder: object,
+    ) -> _CacheTestHandle: ...
+    @staticmethod
     def gcs(
         bucket_name: str,
         *,

From 10ac81cfb5ae5a4c21f17dfdc336fd67ec1e37c3 Mon Sep 17 00:00:00 2001
From: "berriai-litellm-provider-info-sync[bot]"
 <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com>
Date: Mon, 21 Sep 2026 23:30:58 +0000
Subject: [PATCH 157/160] chore(prices): sync OpenRouter prices: 2 models

openrouter/deepseek/deepseek-v4-pro: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost
openrouter/meta-llama/llama-4-maverick: input_cost_per_token, output_cost_per_token
---
 litellm/model_prices_and_context_window_backup.json | 10 +++++-----
 model_prices_and_context_window.json                | 10 +++++-----
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 5cf5ffc102e..3ca9958ad0d 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -43037,21 +43037,21 @@
         "supports_web_search": false
     },
     "openrouter/deepseek/deepseek-v4-pro": {
-        "input_cost_per_token": 8.87226e-07,
+        "input_cost_per_token": 8.83746e-07,
         "input_cost_per_token_cache_hit": 4.4e-08,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 384000,
         "max_tokens": 384000,
         "mode": "chat",
-        "output_cost_per_token": 1.774452e-06,
+        "output_cost_per_token": 1.767492e-06,
         "source": "https://openrouter.ai/api/v1/models",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "cache_read_input_token_cost": 7.39355e-08,
+        "cache_read_input_token_cost": 7.36455e-08,
         "supports_audio_input": false,
         "supports_pdf_input": false,
         "supports_vision": false,
@@ -70299,8 +70299,8 @@
         "supports_web_search": true
     },
     "openrouter/meta-llama/llama-4-maverick": {
-        "input_cost_per_token": 2e-07,
-        "output_cost_per_token": 8e-07,
+        "input_cost_per_token": 1.875e-07,
+        "output_cost_per_token": 6.525e-07,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 16384,
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 5cf5ffc102e..3ca9958ad0d 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -43037,21 +43037,21 @@
         "supports_web_search": false
     },
     "openrouter/deepseek/deepseek-v4-pro": {
-        "input_cost_per_token": 8.87226e-07,
+        "input_cost_per_token": 8.83746e-07,
         "input_cost_per_token_cache_hit": 4.4e-08,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 384000,
         "max_tokens": 384000,
         "mode": "chat",
-        "output_cost_per_token": 1.774452e-06,
+        "output_cost_per_token": 1.767492e-06,
         "source": "https://openrouter.ai/api/v1/models",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "cache_read_input_token_cost": 7.39355e-08,
+        "cache_read_input_token_cost": 7.36455e-08,
         "supports_audio_input": false,
         "supports_pdf_input": false,
         "supports_vision": false,
@@ -70299,8 +70299,8 @@
         "supports_web_search": true
     },
     "openrouter/meta-llama/llama-4-maverick": {
-        "input_cost_per_token": 2e-07,
-        "output_cost_per_token": 8e-07,
+        "input_cost_per_token": 1.875e-07,
+        "output_cost_per_token": 6.525e-07,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 16384,

From 0cfc4bc7825ad880617a8c767854971881f00e05 Mon Sep 17 00:00:00 2001
From: Tin Chi Lo <tin@berri.ai>
Date: Mon, 21 Sep 2026 16:48:08 -0700
Subject: [PATCH 158/160] fix(cli): serialize footer installs and tolerate
 unknown versions

---
 .../client/cli/commands/claude_settings.py    | 49 ++++++++----
 .../client/cli/commands/statusline_script.py  |  4 +-
 pyproject.toml                                |  1 +
 .../proxy/client/cli/test_claude_settings.py  | 80 ++++++++++++++++++-
 uv.lock                                       |  2 +
 5 files changed, 117 insertions(+), 19 deletions(-)

diff --git a/litellm/proxy/client/cli/commands/claude_settings.py b/litellm/proxy/client/cli/commands/claude_settings.py
index ffc26d22e84..b3fdc4695cb 100644
--- a/litellm/proxy/client/cli/commands/claude_settings.py
+++ b/litellm/proxy/client/cli/commands/claude_settings.py
@@ -23,6 +23,7 @@ from types import MappingProxyType
 from typing import Final, TypeAlias
 
 import click
+from filelock import FileLock
 from packaging.version import InvalidVersion, Version
 from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError
 
@@ -309,6 +310,13 @@ def statusline_command(script_path: Path, platform: str = sys.platform) -> str:
     return " ".join(quote(token) for token in (sys.executable, str(script_path)))
 
 
+def _statusline_version(value: str) -> Version | None:
+    try:
+        return Version(value)
+    except InvalidVersion:
+        return None
+
+
 def _installed_statusline_version(target: Path) -> Version | None:
     try:
         with target.open("rb") as script:
@@ -318,28 +326,39 @@ def _installed_statusline_version(target: Path) -> Version | None:
     if not header.startswith(STATUSLINE_VERSION_PREFIX):
         return None
     try:
-        return Version(header.removeprefix(STATUSLINE_VERSION_PREFIX).decode("ascii").strip())
-    except (InvalidVersion, UnicodeDecodeError):
+        return _statusline_version(header.removeprefix(STATUSLINE_VERSION_PREFIX).decode("ascii").strip())
+    except UnicodeDecodeError:
         return None
 
 
-def install_statusline_script(script_path: Path | None = None, *, package_version: str = litellm_version) -> str:
+def install_statusline_script(
+    script_path: Path | None = None,
+    *,
+    package_version: str = litellm_version,
+    write: Callable[[str, bytes], None] = write_private_bytes,
+) -> str:
     target: Final = script_path or STATUSLINE_SCRIPT_PATH
     try:
         ensure_private_dir(target.parent)
-        bundled_version: Final = Version(package_version)
-        installed_version: Final = _installed_statusline_version(target)
-        if installed_version is not None and installed_version > bundled_version:
-            click.echo(
-                f"Keeping the status line from LiteLLM {installed_version}; this CLI is {bundled_version}. "
-                "Upgrade the CLI to refresh it.",
-                err=True,
+        bundled_version: Final = _statusline_version(package_version)
+        with FileLock(str(target) + ".lock", timeout=10, mode=0o600):
+            installed_version: Final = _installed_statusline_version(target)
+            if installed_version is not None and (bundled_version is None or installed_version > bundled_version):
+                cli_version: Final = str(bundled_version) if bundled_version is not None else "unknown"
+                click.echo(
+                    f"Keeping the status line from LiteLLM {installed_version}; this CLI is {cli_version}. "
+                    "Upgrade the CLI to refresh it.",
+                    err=True,
+                )
+                return statusline_command(target)
+            source: Final = Path(statusline_script.__file__).read_bytes()
+            header: Final = (
+                STATUSLINE_VERSION_PREFIX + str(bundled_version).encode("ascii") + b"\n"
+                if bundled_version is not None
+                else b""
             )
-            return statusline_command(target)
-        source: Final = Path(statusline_script.__file__).read_bytes()
-        header: Final = STATUSLINE_VERSION_PREFIX + str(bundled_version).encode("ascii") + b"\n"
-        write_private_bytes(str(target), header + source)
-    except (OSError, InvalidVersion) as e:
+            write(str(target), header + source)
+    except OSError as e:
         raise ClaudeSettingsError(f"Could not install the status line script at {target}: {e}") from e
     return statusline_command(target)
 
diff --git a/litellm/proxy/client/cli/commands/statusline_script.py b/litellm/proxy/client/cli/commands/statusline_script.py
index 6264acb39d1..d16160b1ab8 100644
--- a/litellm/proxy/client/cli/commands/statusline_script.py
+++ b/litellm/proxy/client/cli/commands/statusline_script.py
@@ -1,7 +1,7 @@
 """Claude Code status line and Codex Stop hook for auto-routed sessions.
 
-`lite` copies this file with a CLI version header to ~/.litellm/statusline.py and registers it as Claude
-Code's `statusLine` command and as Codex's `[[hooks.Stop]]` command, so it must stay
+`lite` copies this file to ~/.litellm/statusline.py with a CLI version header when known and registers
+it as Claude Code's `statusLine` command and as Codex's `[[hooks.Stop]]` command, so it must stay
 standard-library only and must never import litellm. Claude Code re-runs it on every
 status refresh (about every 300ms while typing), so the proxy is asked at most once per
 TTL per session and every other refresh is served from a small on-disk cache that holds
diff --git a/pyproject.toml b/pyproject.toml
index 8a7d2981c12..a1b276e4e8c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -15,6 +15,7 @@ dependencies = [
     # When changing a floor, verify it installs + imports on every supported
     # Python with: `uv pip install --resolution=lowest-direct .`
     "fastuuid>=0.14.0,<1.0",
+    "filelock>=3.16.1,<4.0",
     "httpx[http2]>=0.28.0,<1.0",
     "openai>=2.20.0,<3.0.0",
     "python-dotenv>=1.0.0,<2.0",
diff --git a/tests/test_litellm/proxy/client/cli/test_claude_settings.py b/tests/test_litellm/proxy/client/cli/test_claude_settings.py
index f596e1e5d6c..9353c149d15 100644
--- a/tests/test_litellm/proxy/client/cli/test_claude_settings.py
+++ b/tests/test_litellm/proxy/client/cli/test_claude_settings.py
@@ -5,14 +5,16 @@ import shlex
 import stat
 import sys
 import time
+from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeoutError
 from pathlib import Path
+from threading import Event
 from typing import Final
 from unittest.mock import patch
 
 import pytest
 from click.testing import CliRunner
 
-from litellm.litellm_core_utils.private_json import commit_staged_json
+from litellm.litellm_core_utils.private_json import commit_staged_json, write_private_bytes
 from litellm.proxy.client.cli.commands.claude_settings import (
     ANTHROPIC_DEFAULT_MODEL_ENV_KEYS,
     AUTOROUTE_BACKUP_PATH,
@@ -790,7 +792,7 @@ class TestStatusLine:
         with script.open("rb") as running:
             install_statusline_script(script)
             assert running.read() == bundled
-        assert [child.name for child in script.parent.iterdir()] == ["statusline.py"]
+        assert {child.name for child in script.parent.iterdir()} <= {"statusline.py", "statusline.py.lock"}
 
         if os.geteuid() != 0:
             script.parent.chmod(0o500)
@@ -862,6 +864,80 @@ class TestStatusLine:
         assert rig.read()["env"]["ANTHROPIC_BASE_URL"] == PROXY
         assert "Keeping the status line" in capsys.readouterr().err
 
+    @pytest.mark.parametrize("package_version", ("unknown", "", "invalid-version"))
+    @pytest.mark.parametrize("existing", (None, b"print('legacy footer')\n", b"# litellm-statusline-version: invalid\n"))
+    def test_an_unknown_cli_version_can_install_and_refresh_an_unversioned_footer(
+        self, tmp_path: Path, package_version: str, existing: bytes | None
+    ) -> None:
+        from litellm.proxy.client.cli.commands import statusline_script
+
+        script: Final = tmp_path / "statusline.py"
+        if existing is not None:
+            script.write_bytes(existing)
+
+        assert install_statusline_script(script, package_version=package_version) == statusline_command(script)
+        assert script.read_bytes() == Path(statusline_script.__file__).read_bytes()
+
+    def test_an_unknown_cli_version_preserves_a_versioned_footer(
+        self, tmp_path: Path, capsys: pytest.CaptureFixture[str]
+    ) -> None:
+        script: Final = tmp_path / "statusline.py"
+        command: Final = install_statusline_script(script, package_version="2.1.0")
+        installed: Final = script.read_bytes()
+
+        assert install_statusline_script(script, package_version="unknown") == command
+        assert script.read_bytes() == installed
+        assert "Keeping the status line from LiteLLM 2.1.0" in capsys.readouterr().err
+
+    @pytest.mark.parametrize(("first_version", "second_version"), (("2.0", "3.0"), ("3.0", "2.0")))
+    def test_overlapping_installs_keep_the_newest_footer(
+        self, tmp_path: Path, first_version: str, second_version: str
+    ) -> None:
+        from litellm.proxy.client.cli.commands import statusline_script
+
+        script: Final = tmp_path / "statusline.py"
+        first_writing: Final = Event()
+        release_first: Final = Event()
+        second_started: Final = Event()
+
+        def paused_write(path: str, data: bytes) -> None:
+            first_writing.set()
+            assert release_first.wait(5), "First installer was never released"
+            write_private_bytes(path, data)
+
+        def second_install() -> str:
+            second_started.set()
+            return install_statusline_script(script, package_version=second_version)
+
+        with ThreadPoolExecutor(max_workers=2) as pool:
+            first: Final = pool.submit(install_statusline_script, script, package_version=first_version, write=paused_write)
+            try:
+                assert first_writing.wait(5), "First installer did not reach the write"
+                second: Final = pool.submit(second_install)
+                assert second_started.wait(5), "Second installer did not start"
+                with pytest.raises(FutureTimeoutError):
+                    second.result(timeout=0.5)
+            finally:
+                release_first.set()
+            assert first.result(timeout=5) == statusline_command(script)
+            assert second.result(timeout=5) == statusline_command(script)
+
+        assert script.read_bytes() == b"# litellm-statusline-version: 3.0\n" + Path(statusline_script.__file__).read_bytes()
+
+    def test_a_failed_install_keeps_the_footer_and_releases_the_lock(self, tmp_path: Path) -> None:
+        script: Final = tmp_path / "statusline.py"
+        install_statusline_script(script, package_version="2.0")
+        installed: Final = script.read_bytes()
+
+        def failed_write(path: str, data: bytes) -> None:
+            raise OSError("disk full")
+
+        with pytest.raises(ClaudeSettingsError, match="disk full"):
+            install_statusline_script(script, package_version="3.0", write=failed_write)
+        assert script.read_bytes() == installed
+        assert install_statusline_script(script, package_version="3.0") == statusline_command(script)
+        assert script.read_bytes().startswith(b"# litellm-statusline-version: 3.0\n")
+
     def test_configure_installs_it_and_unconfigure_removes_only_ours(self, tmp_path):
         rig = _Rig(tmp_path, {"theme": "dark"})
         script = tmp_path / "statusline.py"
diff --git a/uv.lock b/uv.lock
index e1d4ab791c6..18d57aa991b 100644
--- a/uv.lock
+++ b/uv.lock
@@ -4516,6 +4516,7 @@ dependencies = [
     { name = "boto3" },
     { name = "click" },
     { name = "fastuuid" },
+    { name = "filelock" },
     { name = "httpx", extra = ["http2"] },
     { name = "importlib-metadata" },
     { name = "jinja2" },
@@ -4770,6 +4771,7 @@ requires-dist = [
     { name = "fastapi", marker = "extra == 'proxy'", specifier = ">=0.136.3,<1.0" },
     { name = "fastapi-sso", marker = "extra == 'proxy'", specifier = ">=0.19.0,<1.0" },
     { name = "fastuuid", specifier = ">=0.14.0,<1.0" },
+    { name = "filelock", specifier = ">=3.16.1,<4.0" },
     { name = "google-cloud-aiplatform", marker = "extra == 'google'", specifier = ">=1.133.0,<2.0" },
     { name = "google-cloud-aiplatform", marker = "extra == 'proxy-runtime'", specifier = ">=1.133.0,<2.0" },
     { name = "google-cloud-iam", marker = "extra == 'extra-proxy'", specifier = ">=2.19.1,<3.0" },

From 69b7224aef7e64e7a38eb624d1075c8e141335d7 Mon Sep 17 00:00:00 2001
From: Yujong Lee <yujong@berri.ai>
Date: Mon, 21 Sep 2026 23:56:40 +0000
Subject: [PATCH 159/160] test(python-bridge): initialize the interpreter in
 the embedder seed test

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-rust/crates/python-bridge/src/cache/embedder.rs | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/litellm-rust/crates/python-bridge/src/cache/embedder.rs b/litellm-rust/crates/python-bridge/src/cache/embedder.rs
index 10ccf396510..9398e5a862b 100644
--- a/litellm-rust/crates/python-bridge/src/cache/embedder.rs
+++ b/litellm-rust/crates/python-bridge/src/cache/embedder.rs
@@ -140,10 +140,8 @@ mod tests {
 
     #[tokio::test]
     async fn async_embed_returns_the_seeded_vector_or_unavailable() {
-        let object = Python::attach(|py| {
-            Python::initialize();
-            py.None()
-        });
+        Python::initialize();
+        let object = Python::attach(|py| py.None());
         let embedder = PythonEmbedder::new(object);
         let scoped_embedder = embedder.clone();
         let scoped = with_prepared_embedding(Ok(vec![0.25]), async move {

From c8252a50f5e913ea1f683ab1f5bd33163ffe3541 Mon Sep 17 00:00:00 2001
From: "berriai-litellm-provider-info-sync[bot]"
 <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com>
Date: Tue, 22 Sep 2026 00:01:01 +0000
Subject: [PATCH 160/160] chore(prices): sync OpenRouter prices: 4 models

openrouter/~deepseek/deepseek-pro-latest: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost, off_peak_pricing
openrouter/deepseek/deepseek-v4-flash: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost
openrouter/deepseek/deepseek-v4-pro: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost
openrouter/deepseek/deepseek-v4-pro-0813: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost, off_peak_pricing
---
 ...odel_prices_and_context_window_backup.json | 28 +++++++++----------
 model_prices_and_context_window.json          | 28 +++++++++----------
 2 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 3ca9958ad0d..80bb2bf9bd2 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -43037,21 +43037,21 @@
         "supports_web_search": false
     },
     "openrouter/deepseek/deepseek-v4-pro": {
-        "input_cost_per_token": 8.83746e-07,
+        "input_cost_per_token": 9.5526e-07,
         "input_cost_per_token_cache_hit": 4.4e-08,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 384000,
         "max_tokens": 384000,
         "mode": "chat",
-        "output_cost_per_token": 1.767492e-06,
+        "output_cost_per_token": 1.91052e-06,
         "source": "https://openrouter.ai/api/v1/models",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "cache_read_input_token_cost": 7.36455e-08,
+        "cache_read_input_token_cost": 7.9605e-08,
         "supports_audio_input": false,
         "supports_pdf_input": false,
         "supports_vision": false,
@@ -43079,22 +43079,22 @@
         "supports_web_search": false
     },
     "openrouter/deepseek/deepseek-v4-pro-0813": {
-        "input_cost_per_token": 1.32e-06,
+        "input_cost_per_token": 5.58624e-07,
         "input_cost_per_token_cache_hit": 1.9272e-08,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 384000,
         "max_tokens": 384000,
         "mode": "chat",
-        "output_cost_per_token": 3.96e-06,
+        "output_cost_per_token": 1.675872e-06,
         "source": "https://openrouter.ai/api/v1/models",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "cache_read_input_token_cost": 4.4e-08,
-        "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":6.6e-7,"output_cost_per_token":0.00000198,"cache_read_input_token_cost":2.2e-8},
+        "cache_read_input_token_cost": 1.86208e-08,
+        "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.58624e-7,"output_cost_per_token":0.000001675872,"cache_read_input_token_cost":1.86208e-8},
         "supports_audio_input": false,
         "supports_pdf_input": false,
         "supports_vision": false,
@@ -68941,9 +68941,9 @@
         "supports_web_search": true
     },
     "openrouter/deepseek/deepseek-v4-flash": {
-        "input_cost_per_token": 5.544e-08,
-        "output_cost_per_token": 1.1088e-07,
-        "cache_read_input_token_cost": 1.1088e-08,
+        "input_cost_per_token": 8.8606e-08,
+        "output_cost_per_token": 1.77212e-07,
+        "cache_read_input_token_cost": 1.77212e-08,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 384000,
@@ -72999,15 +72999,15 @@
         "supports_web_search": false
     },
     "openrouter/~deepseek/deepseek-pro-latest": {
-        "cache_read_input_token_cost": 4.4e-08,
-        "input_cost_per_token": 1.32e-06,
+        "cache_read_input_token_cost": 1.86208e-08,
+        "input_cost_per_token": 5.58624e-07,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 384000,
         "max_tokens": 384000,
         "mode": "chat",
-        "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":6.6e-7,"output_cost_per_token":0.00000198,"cache_read_input_token_cost":2.2e-8},
-        "output_cost_per_token": 3.96e-06,
+        "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.58624e-7,"output_cost_per_token":0.000001675872,"cache_read_input_token_cost":1.86208e-8},
+        "output_cost_per_token": 1.675872e-06,
         "source": "https://openrouter.ai/api/v1/models",
         "supports_audio_input": false,
         "supports_function_calling": true,
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 3ca9958ad0d..80bb2bf9bd2 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -43037,21 +43037,21 @@
         "supports_web_search": false
     },
     "openrouter/deepseek/deepseek-v4-pro": {
-        "input_cost_per_token": 8.83746e-07,
+        "input_cost_per_token": 9.5526e-07,
         "input_cost_per_token_cache_hit": 4.4e-08,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 384000,
         "max_tokens": 384000,
         "mode": "chat",
-        "output_cost_per_token": 1.767492e-06,
+        "output_cost_per_token": 1.91052e-06,
         "source": "https://openrouter.ai/api/v1/models",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "cache_read_input_token_cost": 7.36455e-08,
+        "cache_read_input_token_cost": 7.9605e-08,
         "supports_audio_input": false,
         "supports_pdf_input": false,
         "supports_vision": false,
@@ -43079,22 +43079,22 @@
         "supports_web_search": false
     },
     "openrouter/deepseek/deepseek-v4-pro-0813": {
-        "input_cost_per_token": 1.32e-06,
+        "input_cost_per_token": 5.58624e-07,
         "input_cost_per_token_cache_hit": 1.9272e-08,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 384000,
         "max_tokens": 384000,
         "mode": "chat",
-        "output_cost_per_token": 3.96e-06,
+        "output_cost_per_token": 1.675872e-06,
         "source": "https://openrouter.ai/api/v1/models",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "cache_read_input_token_cost": 4.4e-08,
-        "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":6.6e-7,"output_cost_per_token":0.00000198,"cache_read_input_token_cost":2.2e-8},
+        "cache_read_input_token_cost": 1.86208e-08,
+        "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.58624e-7,"output_cost_per_token":0.000001675872,"cache_read_input_token_cost":1.86208e-8},
         "supports_audio_input": false,
         "supports_pdf_input": false,
         "supports_vision": false,
@@ -68941,9 +68941,9 @@
         "supports_web_search": true
     },
     "openrouter/deepseek/deepseek-v4-flash": {
-        "input_cost_per_token": 5.544e-08,
-        "output_cost_per_token": 1.1088e-07,
-        "cache_read_input_token_cost": 1.1088e-08,
+        "input_cost_per_token": 8.8606e-08,
+        "output_cost_per_token": 1.77212e-07,
+        "cache_read_input_token_cost": 1.77212e-08,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 384000,
@@ -72999,15 +72999,15 @@
         "supports_web_search": false
     },
     "openrouter/~deepseek/deepseek-pro-latest": {
-        "cache_read_input_token_cost": 4.4e-08,
-        "input_cost_per_token": 1.32e-06,
+        "cache_read_input_token_cost": 1.86208e-08,
+        "input_cost_per_token": 5.58624e-07,
         "litellm_provider": "openrouter",
         "max_input_tokens": 1048576,
         "max_output_tokens": 384000,
         "max_tokens": 384000,
         "mode": "chat",
-        "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":6.6e-7,"output_cost_per_token":0.00000198,"cache_read_input_token_cost":2.2e-8},
-        "output_cost_per_token": 3.96e-06,
+        "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.58624e-7,"output_cost_per_token":0.000001675872,"cache_read_input_token_cost":1.86208e-8},
+        "output_cost_per_token": 1.675872e-06,
         "source": "https://openrouter.ai/api/v1/models",
         "supports_audio_input": false,
         "supports_function_calling": true,