From 0669d1b3cb7152c3ebc58618dd766a41705503c0 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Thu, 25 Jun 2026 17:31:13 -0700
Subject: [PATCH 01/51] =?UTF-8?q?bump:=20version=200.1.43=20=E2=86=92=200.?=
 =?UTF-8?q?1.44?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 enterprise/pyproject.toml | 4 ++--
 pyproject.toml            | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml
index b032942427c..66f6aeb7abc 100644
--- a/enterprise/pyproject.toml
+++ b/enterprise/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.43"
+version = "0.1.44"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.43"
+version = "0.1.44"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",
diff --git a/pyproject.toml b/pyproject.toml
index 6a6a47f540e..6e99d81f8f3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -63,7 +63,7 @@ proxy = [
     "azure-storage-blob>=12.28.0,<13.0",
     "mcp>=1.26.0,<2.0",
     "litellm-proxy-extras==0.4.74",
-    "litellm-enterprise==0.1.43",
+    "litellm-enterprise==0.1.44",
     "RestrictedPython>=8.1,<9.0",
     "rich>=13.9.4,<14.0",
     "polars>=1.38.1,<2.0",

From 2c8cd6ad4dc65371ac3b7f3dc58206c6120773ac Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Thu, 25 Jun 2026 17:35:56 -0700
Subject: [PATCH 02/51] uv lock

---
 uv.lock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/uv.lock b/uv.lock
index 917dff39e38..da44ad25715 100644
--- a/uv.lock
+++ b/uv.lock
@@ -9,7 +9,7 @@ resolution-markers = [
 ]
 
 [options]
-exclude-newer = "2026-06-22T19:05:55.080417Z"
+exclude-newer = "2026-06-23T00:31:52.495979Z"
 exclude-newer-span = "P3D"
 
 [manifest]
@@ -3597,7 +3597,7 @@ proxy-dev = [
 
 [[package]]
 name = "litellm-enterprise"
-version = "0.1.43"
+version = "0.1.44"
 source = { editable = "enterprise" }
 
 [[package]]

From 4a6f0dbd8c4af8eb56d0ee6a60eaf7139d051931 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Fri, 26 Jun 2026 00:10:14 -0700
Subject: [PATCH 03/51] fix(ui): size Request Logs table columns so it scrolls
 instead of overflowing

Tremor's Table forwards className to a wrapper div rather than the inner table element, so the table-fixed class never reached the table and it stayed table-layout: auto. Across 16 whitespace-nowrap columns that expanded the table far past the viewport

Give each spend-logs column an explicit pixel size and drive the table width from getCenterTotalSize(), matching the Virtual Keys table. The shared DataTable applies this only when columns declare sizes, so the other consumers keep their existing fluid layout
---
 .../src/components/view_logs/columns.tsx      | 16 +++++++
 .../src/components/view_logs/table.test.tsx   | 44 +++++++++++++++++++
 .../src/components/view_logs/table.tsx        | 15 ++++++-
 3 files changed, 73 insertions(+), 2 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/components/view_logs/table.test.tsx

diff --git a/ui/litellm-dashboard/src/components/view_logs/columns.tsx b/ui/litellm-dashboard/src/components/view_logs/columns.tsx
index 1265b8449de..aeeb31f78f3 100644
--- a/ui/litellm-dashboard/src/components/view_logs/columns.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/columns.tsx
@@ -119,11 +119,13 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "Time",
     accessorKey: "startTime",
+    size: 200,
     cell: (info: any) => <TimeCell utcTime={info.getValue()} />,
   },
   {
     header: "Type",
     id: "type",
+    size: 90,
     cell: (info: any) => {
       const row = info.row.original;
       const sessionCount = row.session_total_count || 1;
@@ -168,6 +170,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Status",
     accessorKey: "metadata.status",
+    size: 100,
     cell: (info: any) => {
       const status = info.getValue() || "Success";
       const isSuccess = status.toLowerCase() !== "failure";
@@ -186,6 +189,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Session ID",
     accessorKey: "session_id",
+    size: 160,
     cell: (info: any) => {
       const value = String(info.getValue() || "");
       const onSessionClick = info.row.original.onSessionClick;
@@ -207,6 +211,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Request ID",
     accessorKey: "request_id",
+    size: 160,
     cell: (info: any) => (
       <Tooltip title={String(info.getValue() || "")}>
         <span className="font-mono text-xs max-w-[15ch] truncate block">{String(info.getValue() || "")}</span>
@@ -226,6 +231,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "Cost",
     accessorKey: "spend",
+    size: 110,
     cell: (info: any) => {
       const row = info.row.original;
       const mcpCount = row.mcp_tool_call_count || 0;
@@ -258,6 +264,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "Duration (s)",
     accessorKey: "request_duration_ms",
+    size: 120,
     cell: (info: any) => {
       const ms = info.getValue();
       if (ms == null) return <span>-</span>;
@@ -282,6 +289,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "TTFT (s)",
     accessorKey: "completionStartTime",
+    size: 110,
     cell: (info: any) => {
       const row = info.row.original;
       const completionStartTime = info.getValue();
@@ -301,6 +309,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Team Name",
     accessorKey: "metadata.user_api_key_team_alias",
+    size: 150,
     cell: (info: any) => (
       <Tooltip title={String(info.getValue() || "-")}>
         <span className="max-w-[15ch] truncate block">{String(info.getValue() || "-")}</span>
@@ -310,6 +319,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Key Hash",
     accessorKey: "metadata.user_api_key",
+    size: 160,
     cell: (info: any) => {
       const value = String(info.getValue() || "-");
       const onKeyHashClick = info.row.original.onKeyHashClick;
@@ -329,6 +339,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Key Alias",
     accessorKey: "metadata.user_api_key_alias",
+    size: 150,
     cell: (info: any) => (
       <Tooltip title={String(info.getValue() || "-")}>
         <span className="max-w-[15ch] truncate block">{String(info.getValue() || "-")}</span>
@@ -348,6 +359,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "Model",
     accessorKey: "model",
+    size: 200,
     cell: (info: any) => {
       const row = info.row.original;
       const provider = row.custom_llm_provider;
@@ -385,6 +397,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "Tokens",
     accessorKey: "total_tokens",
+    size: 140,
     cell: (info: any) => {
       const row = info.row.original;
       return (
@@ -400,6 +413,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Internal User",
     accessorKey: "user",
+    size: 150,
     cell: (info: any) => (
       <Tooltip title={String(info.getValue() || "-")}>
         <span className="max-w-[15ch] truncate block">{String(info.getValue() || "-")}</span>
@@ -409,6 +423,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "End User",
     accessorKey: "end_user",
+    size: 140,
     cell: (info: any) => (
       <Tooltip title={String(info.getValue() || "-")}>
         <span className="max-w-[15ch] truncate block">{String(info.getValue() || "-")}</span>
@@ -419,6 +434,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Tags",
     accessorKey: "request_tags",
+    size: 150,
     cell: (info: any) => {
       const tags = info.getValue();
       if (!tags || Object.keys(tags).length === 0) return "-";
diff --git a/ui/litellm-dashboard/src/components/view_logs/table.test.tsx b/ui/litellm-dashboard/src/components/view_logs/table.test.tsx
new file mode 100644
index 00000000000..f88e9bd75c8
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/table.test.tsx
@@ -0,0 +1,44 @@
+import type { ColumnDef } from "@tanstack/react-table";
+import { render, screen } from "@testing-library/react";
+import { describe, expect, it } from "vitest";
+import { DataTable } from "./table";
+
+type Row = { request_id: string; a: string; b: string };
+
+const data: Row[] = [{ request_id: "r1", a: "alpha", b: "beta" }];
+
+const sizedColumns: ColumnDef<Row>[] = [
+  { header: "A", accessorKey: "a", size: 120 },
+  { header: "B", accessorKey: "b", size: 80 },
+];
+
+const unsizedColumns: ColumnDef<Row>[] = [
+  { header: "A", accessorKey: "a" },
+  { header: "B", accessorKey: "b" },
+];
+
+describe("DataTable column sizing", () => {
+  it("widths the table and every cell from column sizes when columns declare them", () => {
+    render(<DataTable data={data} columns={sizedColumns} />);
+
+    expect(screen.getByRole("table").style.width).toBe("200px");
+
+    const headers = screen.getAllByRole("columnheader");
+    expect(headers.map((h) => h.style.width)).toEqual(["120px", "80px"]);
+
+    const cells = screen.getAllByRole("cell");
+    expect(cells.map((c) => c.style.width)).toEqual(["120px", "80px"]);
+  });
+
+  it("leaves cells unsized and keeps the fluid table when no column declares a size", () => {
+    render(<DataTable data={data} columns={unsizedColumns} />);
+
+    const table = screen.getByRole("table");
+    expect(table.style.width).toBe("");
+    expect(table.style.minWidth).toBe("400px");
+
+    for (const cell of [...screen.getAllByRole("columnheader"), ...screen.getAllByRole("cell")]) {
+      expect(cell.style.width).toBe("");
+    }
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/view_logs/table.tsx b/ui/litellm-dashboard/src/components/view_logs/table.tsx
index 6aa349513d5..a47bf5a8e5c 100644
--- a/ui/litellm-dashboard/src/components/view_logs/table.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/table.tsx
@@ -41,6 +41,7 @@ export function DataTable<TData, TValue>({
   enableSorting = false,
 }: DataTableProps<TData, TValue>) {
   const supportsExpansion = !!(renderSubComponent || renderChildRows) && !!getRowCanExpand;
+  const hasExplicitColumnSizes = columns.some((column) => column.size !== undefined);
   const [sorting, setSorting] = useState<SortingState>([]);
 
   const table = useReactTable<TData>({
@@ -63,9 +64,14 @@ export function DataTable<TData, TValue>({
     ...(supportsExpansion && { getExpandedRowModel: getExpandedRowModel() }),
   });
 
+  const tableClassName = hasExplicitColumnSizes
+    ? "[&_td]:py-0.5 [&_th]:py-1 [&_table]:table-fixed"
+    : "[&_td]:py-0.5 [&_th]:py-1 table-fixed w-full box-border";
+  const tableStyle = hasExplicitColumnSizes ? { width: table.getCenterTotalSize() } : { minWidth: "400px" };
+
   return (
     <div className="rounded-lg custom-border overflow-x-auto w-full max-w-full box-border">
-      <Table className="[&_td]:py-0.5 [&_th]:py-1 table-fixed w-full box-border" style={{ minWidth: "400px" }}>
+      <Table className={tableClassName} style={tableStyle}>
         <TableHead>
           {table.getHeaderGroups().map((headerGroup) => (
             <TableRow key={headerGroup.id}>
@@ -77,6 +83,7 @@ export function DataTable<TData, TValue>({
                   <TableHeaderCell
                     key={header.id}
                     className={`py-1 h-8 ${canSort ? "cursor-pointer select-none hover:bg-gray-50" : ""}`}
+                    style={hasExplicitColumnSizes ? { width: header.getSize() } : undefined}
                     onClick={canSort ? header.column.getToggleSortingHandler() : undefined}
                   >
                     {header.isPlaceholder ? null : (
@@ -112,7 +119,11 @@ export function DataTable<TData, TValue>({
                   onClick={() => onRowClick?.(row.original)}
                 >
                   {row.getVisibleCells().map((cell) => (
-                    <TableCell key={cell.id} className="py-0.5 max-h-8 overflow-hidden text-ellipsis whitespace-nowrap">
+                    <TableCell
+                      key={cell.id}
+                      className="py-0.5 max-h-8 overflow-hidden text-ellipsis whitespace-nowrap"
+                      style={hasExplicitColumnSizes ? { width: cell.column.getSize() } : undefined}
+                    >
                       {flexRender(cell.column.columnDef.cell, cell.getContext())}
                     </TableCell>
                   ))}

From 014754be947e60cbefa1a3ecaae0a68a05a7443f Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Fri, 26 Jun 2026 18:34:07 -0700
Subject: [PATCH 04/51] fix(ui): let dashboard main pane shrink so wide tables
 scroll instead of overflowing

The Request Logs page pushed the whole page past the viewport horizontally. The cause was the app shell flex layout: <main className="flex-1"> is a flex item, and flex items default to min-width: auto, so they refuse to shrink below their content's intrinsic width. The logs table is intrinsically ~2300px across its 16 nowrap columns, so main grew to that width and dragged the page with it; the table's own overflow-x-auto wrapper never got the chance to scroll

Add min-w-0 to main so it can shrink to the available width, at which point the existing overflow-x-auto wrapper engages and the table scrolls inside its card. This applies to every dashboard page, not just logs

Also drop the dead max-w-screen class on the logs container (not a real Tailwind utility, so it was a no-op), and revert the earlier column-sizing attempt which targeted table-layout rather than the actual containment problem
---
 .../src/app/(dashboard)/layout.tsx            |  2 +-
 .../src/components/view_logs/columns.tsx      | 16 -------
 .../src/components/view_logs/index.tsx        |  2 +-
 .../src/components/view_logs/table.test.tsx   | 44 -------------------
 .../src/components/view_logs/table.tsx        | 15 +------
 5 files changed, 4 insertions(+), 75 deletions(-)
 delete mode 100644 ui/litellm-dashboard/src/components/view_logs/table.test.tsx

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx
index a5e83436888..09951dc1923 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx
@@ -126,7 +126,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) {
             <div className="mt-2">
               <SidebarProvider setPage={navigateToPage} defaultSelectedKey={page} sidebarCollapsed={sidebarCollapsed} />
             </div>
-            <main className="flex-1">{children}</main>
+            <main className="flex-1 min-w-0">{children}</main>
           </>
         )}
       </div>
diff --git a/ui/litellm-dashboard/src/components/view_logs/columns.tsx b/ui/litellm-dashboard/src/components/view_logs/columns.tsx
index aeeb31f78f3..1265b8449de 100644
--- a/ui/litellm-dashboard/src/components/view_logs/columns.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/columns.tsx
@@ -119,13 +119,11 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "Time",
     accessorKey: "startTime",
-    size: 200,
     cell: (info: any) => <TimeCell utcTime={info.getValue()} />,
   },
   {
     header: "Type",
     id: "type",
-    size: 90,
     cell: (info: any) => {
       const row = info.row.original;
       const sessionCount = row.session_total_count || 1;
@@ -170,7 +168,6 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Status",
     accessorKey: "metadata.status",
-    size: 100,
     cell: (info: any) => {
       const status = info.getValue() || "Success";
       const isSuccess = status.toLowerCase() !== "failure";
@@ -189,7 +186,6 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Session ID",
     accessorKey: "session_id",
-    size: 160,
     cell: (info: any) => {
       const value = String(info.getValue() || "");
       const onSessionClick = info.row.original.onSessionClick;
@@ -211,7 +207,6 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Request ID",
     accessorKey: "request_id",
-    size: 160,
     cell: (info: any) => (
       <Tooltip title={String(info.getValue() || "")}>
         <span className="font-mono text-xs max-w-[15ch] truncate block">{String(info.getValue() || "")}</span>
@@ -231,7 +226,6 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "Cost",
     accessorKey: "spend",
-    size: 110,
     cell: (info: any) => {
       const row = info.row.original;
       const mcpCount = row.mcp_tool_call_count || 0;
@@ -264,7 +258,6 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "Duration (s)",
     accessorKey: "request_duration_ms",
-    size: 120,
     cell: (info: any) => {
       const ms = info.getValue();
       if (ms == null) return <span>-</span>;
@@ -289,7 +282,6 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "TTFT (s)",
     accessorKey: "completionStartTime",
-    size: 110,
     cell: (info: any) => {
       const row = info.row.original;
       const completionStartTime = info.getValue();
@@ -309,7 +301,6 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Team Name",
     accessorKey: "metadata.user_api_key_team_alias",
-    size: 150,
     cell: (info: any) => (
       <Tooltip title={String(info.getValue() || "-")}>
         <span className="max-w-[15ch] truncate block">{String(info.getValue() || "-")}</span>
@@ -319,7 +310,6 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Key Hash",
     accessorKey: "metadata.user_api_key",
-    size: 160,
     cell: (info: any) => {
       const value = String(info.getValue() || "-");
       const onKeyHashClick = info.row.original.onKeyHashClick;
@@ -339,7 +329,6 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Key Alias",
     accessorKey: "metadata.user_api_key_alias",
-    size: 150,
     cell: (info: any) => (
       <Tooltip title={String(info.getValue() || "-")}>
         <span className="max-w-[15ch] truncate block">{String(info.getValue() || "-")}</span>
@@ -359,7 +348,6 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "Model",
     accessorKey: "model",
-    size: 200,
     cell: (info: any) => {
       const row = info.row.original;
       const provider = row.custom_llm_provider;
@@ -397,7 +385,6 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "Tokens",
     accessorKey: "total_tokens",
-    size: 140,
     cell: (info: any) => {
       const row = info.row.original;
       return (
@@ -413,7 +400,6 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Internal User",
     accessorKey: "user",
-    size: 150,
     cell: (info: any) => (
       <Tooltip title={String(info.getValue() || "-")}>
         <span className="max-w-[15ch] truncate block">{String(info.getValue() || "-")}</span>
@@ -423,7 +409,6 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "End User",
     accessorKey: "end_user",
-    size: 140,
     cell: (info: any) => (
       <Tooltip title={String(info.getValue() || "-")}>
         <span className="max-w-[15ch] truncate block">{String(info.getValue() || "-")}</span>
@@ -434,7 +419,6 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Tags",
     accessorKey: "request_tags",
-    size: 150,
     cell: (info: any) => {
       const tags = info.getValue();
       if (!tags || Object.keys(tags).length === 0) return "-";
diff --git a/ui/litellm-dashboard/src/components/view_logs/index.tsx b/ui/litellm-dashboard/src/components/view_logs/index.tsx
index 6c5fd03f0a0..cfe1bd6025a 100644
--- a/ui/litellm-dashboard/src/components/view_logs/index.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/index.tsx
@@ -234,7 +234,7 @@ export default function SpendLogsTable({ accessToken, token, userRole, userID, p
   };
 
   return (
-    <div className="w-full max-w-screen p-6 overflow-x-hidden box-border">
+    <div className="w-full p-6 overflow-x-hidden box-border">
       <TabGroup defaultIndex={0} onIndexChange={(index) => setActiveTab(index === 0 ? "request logs" : "audit logs")}>
         <TabList>
           <Tab>Request Logs</Tab>
diff --git a/ui/litellm-dashboard/src/components/view_logs/table.test.tsx b/ui/litellm-dashboard/src/components/view_logs/table.test.tsx
deleted file mode 100644
index f88e9bd75c8..00000000000
--- a/ui/litellm-dashboard/src/components/view_logs/table.test.tsx
+++ /dev/null
@@ -1,44 +0,0 @@
-import type { ColumnDef } from "@tanstack/react-table";
-import { render, screen } from "@testing-library/react";
-import { describe, expect, it } from "vitest";
-import { DataTable } from "./table";
-
-type Row = { request_id: string; a: string; b: string };
-
-const data: Row[] = [{ request_id: "r1", a: "alpha", b: "beta" }];
-
-const sizedColumns: ColumnDef<Row>[] = [
-  { header: "A", accessorKey: "a", size: 120 },
-  { header: "B", accessorKey: "b", size: 80 },
-];
-
-const unsizedColumns: ColumnDef<Row>[] = [
-  { header: "A", accessorKey: "a" },
-  { header: "B", accessorKey: "b" },
-];
-
-describe("DataTable column sizing", () => {
-  it("widths the table and every cell from column sizes when columns declare them", () => {
-    render(<DataTable data={data} columns={sizedColumns} />);
-
-    expect(screen.getByRole("table").style.width).toBe("200px");
-
-    const headers = screen.getAllByRole("columnheader");
-    expect(headers.map((h) => h.style.width)).toEqual(["120px", "80px"]);
-
-    const cells = screen.getAllByRole("cell");
-    expect(cells.map((c) => c.style.width)).toEqual(["120px", "80px"]);
-  });
-
-  it("leaves cells unsized and keeps the fluid table when no column declares a size", () => {
-    render(<DataTable data={data} columns={unsizedColumns} />);
-
-    const table = screen.getByRole("table");
-    expect(table.style.width).toBe("");
-    expect(table.style.minWidth).toBe("400px");
-
-    for (const cell of [...screen.getAllByRole("columnheader"), ...screen.getAllByRole("cell")]) {
-      expect(cell.style.width).toBe("");
-    }
-  });
-});
diff --git a/ui/litellm-dashboard/src/components/view_logs/table.tsx b/ui/litellm-dashboard/src/components/view_logs/table.tsx
index a47bf5a8e5c..6aa349513d5 100644
--- a/ui/litellm-dashboard/src/components/view_logs/table.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/table.tsx
@@ -41,7 +41,6 @@ export function DataTable<TData, TValue>({
   enableSorting = false,
 }: DataTableProps<TData, TValue>) {
   const supportsExpansion = !!(renderSubComponent || renderChildRows) && !!getRowCanExpand;
-  const hasExplicitColumnSizes = columns.some((column) => column.size !== undefined);
   const [sorting, setSorting] = useState<SortingState>([]);
 
   const table = useReactTable<TData>({
@@ -64,14 +63,9 @@ export function DataTable<TData, TValue>({
     ...(supportsExpansion && { getExpandedRowModel: getExpandedRowModel() }),
   });
 
-  const tableClassName = hasExplicitColumnSizes
-    ? "[&_td]:py-0.5 [&_th]:py-1 [&_table]:table-fixed"
-    : "[&_td]:py-0.5 [&_th]:py-1 table-fixed w-full box-border";
-  const tableStyle = hasExplicitColumnSizes ? { width: table.getCenterTotalSize() } : { minWidth: "400px" };
-
   return (
     <div className="rounded-lg custom-border overflow-x-auto w-full max-w-full box-border">
-      <Table className={tableClassName} style={tableStyle}>
+      <Table className="[&_td]:py-0.5 [&_th]:py-1 table-fixed w-full box-border" style={{ minWidth: "400px" }}>
         <TableHead>
           {table.getHeaderGroups().map((headerGroup) => (
             <TableRow key={headerGroup.id}>
@@ -83,7 +77,6 @@ export function DataTable<TData, TValue>({
                   <TableHeaderCell
                     key={header.id}
                     className={`py-1 h-8 ${canSort ? "cursor-pointer select-none hover:bg-gray-50" : ""}`}
-                    style={hasExplicitColumnSizes ? { width: header.getSize() } : undefined}
                     onClick={canSort ? header.column.getToggleSortingHandler() : undefined}
                   >
                     {header.isPlaceholder ? null : (
@@ -119,11 +112,7 @@ export function DataTable<TData, TValue>({
                   onClick={() => onRowClick?.(row.original)}
                 >
                   {row.getVisibleCells().map((cell) => (
-                    <TableCell
-                      key={cell.id}
-                      className="py-0.5 max-h-8 overflow-hidden text-ellipsis whitespace-nowrap"
-                      style={hasExplicitColumnSizes ? { width: cell.column.getSize() } : undefined}
-                    >
+                    <TableCell key={cell.id} className="py-0.5 max-h-8 overflow-hidden text-ellipsis whitespace-nowrap">
                       {flexRender(cell.column.columnDef.cell, cell.getContext())}
                     </TableCell>
                   ))}

From 76be4461cad2dfe89fb2930d42eb99a65546d004 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Fri, 26 Jun 2026 19:05:33 -0700
Subject: [PATCH 05/51] feat(ui): give Request Logs columns explicit widths and
 tighten the dense ones

Now that the page-overflow bug is fixed by letting the main pane shrink, bring back per-column sizing purely to control widths. Columns declare explicit pixel sizes and the table derives its min-width from getCenterTotalSize(), so it stretches to fill a wide card but scrolls once the columns no longer fit. The shared DataTable applies this only when columns declare sizes, leaving the other consumers on their existing fluid layout

Trim the columns that were eating horizontal space without earning it: Request ID and Key Hash drop ~30% (Key Hash now narrower than Key Alias, which is the more useful of the two), and Duration and TTFT shrink to fit their short numeric values
---
 .../src/components/view_logs/columns.tsx      | 16 +++++++
 .../src/components/view_logs/table.test.tsx   | 46 +++++++++++++++++++
 .../src/components/view_logs/table.tsx        | 15 +++++-
 3 files changed, 75 insertions(+), 2 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/components/view_logs/table.test.tsx

diff --git a/ui/litellm-dashboard/src/components/view_logs/columns.tsx b/ui/litellm-dashboard/src/components/view_logs/columns.tsx
index 1265b8449de..310316d205e 100644
--- a/ui/litellm-dashboard/src/components/view_logs/columns.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/columns.tsx
@@ -119,11 +119,13 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "Time",
     accessorKey: "startTime",
+    size: 200,
     cell: (info: any) => <TimeCell utcTime={info.getValue()} />,
   },
   {
     header: "Type",
     id: "type",
+    size: 90,
     cell: (info: any) => {
       const row = info.row.original;
       const sessionCount = row.session_total_count || 1;
@@ -168,6 +170,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Status",
     accessorKey: "metadata.status",
+    size: 100,
     cell: (info: any) => {
       const status = info.getValue() || "Success";
       const isSuccess = status.toLowerCase() !== "failure";
@@ -186,6 +189,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Session ID",
     accessorKey: "session_id",
+    size: 160,
     cell: (info: any) => {
       const value = String(info.getValue() || "");
       const onSessionClick = info.row.original.onSessionClick;
@@ -207,6 +211,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Request ID",
     accessorKey: "request_id",
+    size: 110,
     cell: (info: any) => (
       <Tooltip title={String(info.getValue() || "")}>
         <span className="font-mono text-xs max-w-[15ch] truncate block">{String(info.getValue() || "")}</span>
@@ -226,6 +231,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "Cost",
     accessorKey: "spend",
+    size: 110,
     cell: (info: any) => {
       const row = info.row.original;
       const mcpCount = row.mcp_tool_call_count || 0;
@@ -258,6 +264,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "Duration (s)",
     accessorKey: "request_duration_ms",
+    size: 90,
     cell: (info: any) => {
       const ms = info.getValue();
       if (ms == null) return <span>-</span>;
@@ -282,6 +289,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "TTFT (s)",
     accessorKey: "completionStartTime",
+    size: 80,
     cell: (info: any) => {
       const row = info.row.original;
       const completionStartTime = info.getValue();
@@ -301,6 +309,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Team Name",
     accessorKey: "metadata.user_api_key_team_alias",
+    size: 150,
     cell: (info: any) => (
       <Tooltip title={String(info.getValue() || "-")}>
         <span className="max-w-[15ch] truncate block">{String(info.getValue() || "-")}</span>
@@ -310,6 +319,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Key Hash",
     accessorKey: "metadata.user_api_key",
+    size: 110,
     cell: (info: any) => {
       const value = String(info.getValue() || "-");
       const onKeyHashClick = info.row.original.onKeyHashClick;
@@ -329,6 +339,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Key Alias",
     accessorKey: "metadata.user_api_key_alias",
+    size: 150,
     cell: (info: any) => (
       <Tooltip title={String(info.getValue() || "-")}>
         <span className="max-w-[15ch] truncate block">{String(info.getValue() || "-")}</span>
@@ -348,6 +359,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "Model",
     accessorKey: "model",
+    size: 200,
     cell: (info: any) => {
       const row = info.row.original;
       const provider = row.custom_llm_provider;
@@ -385,6 +397,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "Tokens",
     accessorKey: "total_tokens",
+    size: 140,
     cell: (info: any) => {
       const row = info.row.original;
       return (
@@ -400,6 +413,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Internal User",
     accessorKey: "user",
+    size: 150,
     cell: (info: any) => (
       <Tooltip title={String(info.getValue() || "-")}>
         <span className="max-w-[15ch] truncate block">{String(info.getValue() || "-")}</span>
@@ -409,6 +423,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "End User",
     accessorKey: "end_user",
+    size: 140,
     cell: (info: any) => (
       <Tooltip title={String(info.getValue() || "-")}>
         <span className="max-w-[15ch] truncate block">{String(info.getValue() || "-")}</span>
@@ -419,6 +434,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Tags",
     accessorKey: "request_tags",
+    size: 150,
     cell: (info: any) => {
       const tags = info.getValue();
       if (!tags || Object.keys(tags).length === 0) return "-";
diff --git a/ui/litellm-dashboard/src/components/view_logs/table.test.tsx b/ui/litellm-dashboard/src/components/view_logs/table.test.tsx
new file mode 100644
index 00000000000..da9bcef1455
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/table.test.tsx
@@ -0,0 +1,46 @@
+import type { ColumnDef } from "@tanstack/react-table";
+import { render, screen } from "@testing-library/react";
+import { describe, expect, it } from "vitest";
+import { DataTable } from "./table";
+
+type Row = { request_id: string; a: string; b: string };
+
+const data: Row[] = [{ request_id: "r1", a: "alpha", b: "beta" }];
+
+const sizedColumns: ColumnDef<Row>[] = [
+  { header: "A", accessorKey: "a", size: 120 },
+  { header: "B", accessorKey: "b", size: 80 },
+];
+
+const unsizedColumns: ColumnDef<Row>[] = [
+  { header: "A", accessorKey: "a" },
+  { header: "B", accessorKey: "b" },
+];
+
+describe("DataTable column sizing", () => {
+  it("min-widths the table to the column total and sizes every cell when columns declare sizes", () => {
+    render(<DataTable data={data} columns={sizedColumns} />);
+
+    const table = screen.getByRole("table");
+    expect(table.style.minWidth).toBe("200px");
+    expect(table.style.width).toBe("");
+
+    const headers = screen.getAllByRole("columnheader");
+    expect(headers.map((h) => h.style.width)).toEqual(["120px", "80px"]);
+
+    const cells = screen.getAllByRole("cell");
+    expect(cells.map((c) => c.style.width)).toEqual(["120px", "80px"]);
+  });
+
+  it("leaves cells unsized and keeps the fluid table when no column declares a size", () => {
+    render(<DataTable data={data} columns={unsizedColumns} />);
+
+    const table = screen.getByRole("table");
+    expect(table.style.width).toBe("");
+    expect(table.style.minWidth).toBe("400px");
+
+    for (const cell of [...screen.getAllByRole("columnheader"), ...screen.getAllByRole("cell")]) {
+      expect(cell.style.width).toBe("");
+    }
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/view_logs/table.tsx b/ui/litellm-dashboard/src/components/view_logs/table.tsx
index 6aa349513d5..4510cc9a1f0 100644
--- a/ui/litellm-dashboard/src/components/view_logs/table.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/table.tsx
@@ -41,6 +41,7 @@ export function DataTable<TData, TValue>({
   enableSorting = false,
 }: DataTableProps<TData, TValue>) {
   const supportsExpansion = !!(renderSubComponent || renderChildRows) && !!getRowCanExpand;
+  const hasExplicitColumnSizes = columns.some((column) => column.size !== undefined);
   const [sorting, setSorting] = useState<SortingState>([]);
 
   const table = useReactTable<TData>({
@@ -63,9 +64,14 @@ export function DataTable<TData, TValue>({
     ...(supportsExpansion && { getExpandedRowModel: getExpandedRowModel() }),
   });
 
+  const tableClassName = hasExplicitColumnSizes
+    ? "[&_td]:py-0.5 [&_th]:py-1 [&_table]:table-fixed"
+    : "[&_td]:py-0.5 [&_th]:py-1 table-fixed w-full box-border";
+  const tableStyle = hasExplicitColumnSizes ? { minWidth: table.getCenterTotalSize() } : { minWidth: "400px" };
+
   return (
     <div className="rounded-lg custom-border overflow-x-auto w-full max-w-full box-border">
-      <Table className="[&_td]:py-0.5 [&_th]:py-1 table-fixed w-full box-border" style={{ minWidth: "400px" }}>
+      <Table className={tableClassName} style={tableStyle}>
         <TableHead>
           {table.getHeaderGroups().map((headerGroup) => (
             <TableRow key={headerGroup.id}>
@@ -77,6 +83,7 @@ export function DataTable<TData, TValue>({
                   <TableHeaderCell
                     key={header.id}
                     className={`py-1 h-8 ${canSort ? "cursor-pointer select-none hover:bg-gray-50" : ""}`}
+                    style={hasExplicitColumnSizes ? { width: header.getSize() } : undefined}
                     onClick={canSort ? header.column.getToggleSortingHandler() : undefined}
                   >
                     {header.isPlaceholder ? null : (
@@ -112,7 +119,11 @@ export function DataTable<TData, TValue>({
                   onClick={() => onRowClick?.(row.original)}
                 >
                   {row.getVisibleCells().map((cell) => (
-                    <TableCell key={cell.id} className="py-0.5 max-h-8 overflow-hidden text-ellipsis whitespace-nowrap">
+                    <TableCell
+                      key={cell.id}
+                      className="py-0.5 max-h-8 overflow-hidden text-ellipsis whitespace-nowrap"
+                      style={hasExplicitColumnSizes ? { width: cell.column.getSize() } : undefined}
+                    >
                       {flexRender(cell.column.columnDef.cell, cell.getContext())}
                     </TableCell>
                   ))}

From 84d7a320201edc57c4ab6a293a08375909bbb3fd Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Mon, 29 Jun 2026 21:08:40 -0700
Subject: [PATCH 06/51] fix(ui): revert Duration and TTFT column widths to
 default

The explicit 90px/80px sizes were too narrow for the Duration (s) and
TTFT (s) headers once the sort arrows were factored in, cramping the
header labels. Dropping the size lets these two columns fall back to the
default width like before
---
 ui/litellm-dashboard/src/components/view_logs/columns.tsx | 2 --
 1 file changed, 2 deletions(-)

diff --git a/ui/litellm-dashboard/src/components/view_logs/columns.tsx b/ui/litellm-dashboard/src/components/view_logs/columns.tsx
index 310316d205e..73eecf9c01e 100644
--- a/ui/litellm-dashboard/src/components/view_logs/columns.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/columns.tsx
@@ -264,7 +264,6 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "Duration (s)",
     accessorKey: "request_duration_ms",
-    size: 90,
     cell: (info: any) => {
       const ms = info.getValue();
       if (ms == null) return <span>-</span>;
@@ -289,7 +288,6 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
         )
       : "TTFT (s)",
     accessorKey: "completionStartTime",
-    size: 80,
     cell: (info: any) => {
       const row = info.row.original;
       const completionStartTime = info.getValue();

From 256b5aadfbf5168facfd1add1cfc956dce44773b Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Mon, 29 Jun 2026 21:16:59 -0700
Subject: [PATCH 07/51] fix(ui): revert Request ID width to default, tighten
 Session ID

Drop the explicit size on Request ID so it falls back to the default
width like the other reverted columns. Narrow Session ID from 160px to
120px since its truncated value needs less room
---
 ui/litellm-dashboard/src/components/view_logs/columns.tsx | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ui/litellm-dashboard/src/components/view_logs/columns.tsx b/ui/litellm-dashboard/src/components/view_logs/columns.tsx
index 73eecf9c01e..8de67e6ae19 100644
--- a/ui/litellm-dashboard/src/components/view_logs/columns.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/columns.tsx
@@ -189,7 +189,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Session ID",
     accessorKey: "session_id",
-    size: 160,
+    size: 120,
     cell: (info: any) => {
       const value = String(info.getValue() || "");
       const onSessionClick = info.row.original.onSessionClick;
@@ -211,7 +211,6 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef<LogEntry>[]
   {
     header: "Request ID",
     accessorKey: "request_id",
-    size: 110,
     cell: (info: any) => (
       <Tooltip title={String(info.getValue() || "")}>
         <span className="font-mono text-xs max-w-[15ch] truncate block">{String(info.getValue() || "")}</span>

From 3dce3daff644863178a7d59b53630d0519a6417b Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Tue, 30 Jun 2026 09:58:01 -0700
Subject: [PATCH 08/51] feat(proxy): type Customer Management response_model
 for OpenAPI coverage (#31043)

* feat(proxy): type Customer Management response_model for OpenAPI coverage

Add response_model to the five remaining untyped /customer operations
(block, unblock, new, update, delete) so the generated OpenAPI schema
documents a concrete response body. new/update reuse the canonical
LiteLLM_EndUserTable (matching info/list); block, unblock, and delete
get small dedicated models in
litellm/types/proxy/management_endpoints/customer_endpoints.py.

Together with the already-typed info/list/daily-activity routes this
brings the Customer Management group to full response_model coverage.
Regression tests assert each public /customer/* route declares the
expected response_model and that /customer/new surfaces a typed schema
in app.openapi(), so dropping a response_model fails CI.

* fix(proxy): keep budget_id in typed customer responses

Address review feedback on the Customer Management response_model typing.
Greptile flagged that response_model=LiteLLM_EndUserTable on /customer/new
and /customer/update silently drops fields the raw Prisma model_dump()
echoed. Checking the schema, budget_id is the only such scalar column that
was missing from the Pydantic model (created_at/updated_at/tpm_limit do not
exist on litellm_endusertable), so add budget_id to LiteLLM_EndUserTable.

This restores budget_id on new/update and also fixes the pre-existing gap
where /customer/info and /customer/list (already typed) dropped it, which
the UI Customer type expects. A regression test pins budget_id surviving the
response_model filter on /customer/update.

Also document UnblockUsersResponse.blocked_users via a Field description: it
holds the users that remain blocked after the call. The key name predates
this PR and is kept to avoid a backwards-incompatible rename on a beta route.

* fix(proxy): keep nested budget fields in customer responses

response_model=LiteLLM_EndUserTable nests the budget as the narrow write
allowlist LiteLLM_BudgetTable, which silently drops the server-managed
fields the customer endpoints used to return (budget_reset_at, created_at).

Introduce CustomerResponse, a thin response model that nests
LiteLLM_BudgetTableFull (the repo's budget response model), and apply it on
/customer/new, /customer/update, /customer/info and /customer/list. list
also builds CustomerResponse so its budget isn't narrowed at construction
time. created_by/updated_at/updated_by remain omitted, matching how budgets
are returned elsewhere.

The shared LiteLLM_EndUserTable is left untouched: it's constructed in many
places that pass narrow budget instances, and pydantic v2 won't coerce a
budget instance into a wider nested model. Typing only at the response
boundary (where the handler hands FastAPI a dict) sidesteps that. A
regression test pins budget_reset_at + created_at through the filter and
asserts the internal audit fields stay out.

* test(proxy): add golden-master characterization tests for customer responses

Lock the exact JSON body each customer-object endpoint (info/list/new/update)
and delete return today, so the upcoming type-safety refactor of the handlers
is only allowed to land if it reproduces these byte for byte. Pins null-field
inclusion, the nested budget shape (server fields kept, audit fields dropped),
and object_permission reverse-relation stripping. Green against current code.

* refactor(proxy): make the customer response flow type-safe

Replace the untyped dict + bolt-on response_model pattern on the customer
object endpoints with explicit typed construction. A single mapper,
_to_customer_response, validates a DB row into CustomerResponse at one
Any -> typed seam; new/update/info/list now return it (or a list of it) and
carry real -> CustomerResponse / -> List[CustomerResponse] return
annotations, and delete returns DeleteCustomersResponse. basedpyright now
verifies the handlers' return shapes instead of a runtime filter doing it
silently.

This also deletes the four copy-pasted object_permission reverse-relation
cleanup loops: pydantic's extra=ignore drops those undeclared fields during
validation, so the loops were dead code (proven by the golden-master tests,
which stay byte-for-byte green). basedpyright errors on the file drop from
140 to 116, all from removed dict plumbing.

CustomerResponse stays a thin subclass of LiteLLM_EndUserTable so it inherits
the existing validators/config unchanged (behavior preservation); only the
nested budget type is widened.

* refactor(proxy): annotate customer response mapper param as BaseModel

Address review nit: the mapper's untyped `record` added an ANN001 violation.
The incoming rows are pydantic v2 models, so type the param as BaseModel
rather than object (object has no model_dump, which would just move the
problem to basedpyright). This clears the ANN001 and also drops three
basedpyright unknown-type violations the untyped param was adding.

* style(test): ruff format customer endpoint tests

* test(proxy): give customer budget test update mocks a valid model_dump

The type-safe response refactor validates the update result via
_to_customer_response (CustomerResponse.model_validate(record.model_dump())).
These budget tests mocked the end-user update to return a bare MagicMock,
so model_dump() yielded a MagicMock that fails validation. Give each update
mock a minimal valid dict; the tests assert on the prisma calls, not the body.

* chore(ui): regenerate API types from proxy OpenAPI spec

* fix(ui): make generated API types stable across Python versions

Python 3.13 strips a docstring's common leading indentation at compile
time while 3.12 keeps it, so app.openapi() emits differently-indented
description strings depending on the interpreter. The dashboard type
generator ran locally on 3.13 and in CI on 3.12, so schema.d.ts drifted
and the "Verify schema.d.ts matches the proxy OpenAPI spec" check failed

Normalize every description through inspect.cleandoc in the spec dump so
the output is identical regardless of interpreter, then regenerate
---
 litellm/models/end_user.py                    |   1 +
 .../customer_endpoints.py                     | 105 ++---
 .../customer_endpoints.py                     |  30 ++
 .../test_customer_budget.py                   |  13 +-
 .../test_customer_endpoints.py                | 380 +++++++++++++++---
 .../scripts/gen-api-types.mjs                 |  15 +-
 ui/litellm-dashboard/src/lib/http/schema.d.ts | 101 +++--
 7 files changed, 495 insertions(+), 150 deletions(-)
 create mode 100644 litellm/types/proxy/management_endpoints/customer_endpoints.py

diff --git a/litellm/models/end_user.py b/litellm/models/end_user.py
index 15fd03ec2ca..9bf895b9447 100644
--- a/litellm/models/end_user.py
+++ b/litellm/models/end_user.py
@@ -21,6 +21,7 @@ class LiteLLM_EndUserTable(LiteLLMPydanticObjectBase):
     spend: float = 0.0
     allowed_model_region: Optional[Literal["eu", "us"]] = None
     default_model: Optional[str] = None
+    budget_id: Optional[str] = None
     litellm_budget_table: Optional[LiteLLM_BudgetTable] = None
     object_permission_id: Optional[str] = None
     object_permission: Optional[LiteLLM_ObjectPermissionTable] = None
diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py
index 7c8a9b88191..84f67bdc3bc 100644
--- a/litellm/proxy/management_endpoints/customer_endpoints.py
+++ b/litellm/proxy/management_endpoints/customer_endpoints.py
@@ -15,6 +15,7 @@ from typing import List, Optional
 
 import fastapi
 from fastapi import APIRouter, Depends, HTTPException, Request
+from pydantic import BaseModel
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -32,10 +33,26 @@ from litellm.repositories.table_repositories import EndUserRepository
 from litellm.types.proxy.management_endpoints.common_daily_activity import (
     SpendAnalyticsPaginatedResponse,
 )
+from litellm.types.proxy.management_endpoints.customer_endpoints import (
+    BlockUsersResponse,
+    CustomerResponse,
+    DeleteCustomersResponse,
+    UnblockUsersResponse,
+)
 
 router = APIRouter()
 
 
+def _to_customer_response(record: BaseModel) -> CustomerResponse:
+    """Validate a raw end-user DB row into the typed customer response.
+
+    object_permission reverse relations and the budget's audit fields are
+    dropped here by the response model's field set, so callers need no manual
+    cleanup.
+    """
+    return CustomerResponse.model_validate(record.model_dump())
+
+
 @router.post(
     "/end_user/block",
     tags=["Customer Management"],
@@ -46,6 +63,7 @@ router = APIRouter()
     "/customer/block",
     tags=["Customer Management"],
     dependencies=[Depends(user_api_key_auth)],
+    response_model=BlockUsersResponse,
 )
 async def block_user(data: BlockUsers):
     """
@@ -100,6 +118,7 @@ async def block_user(data: BlockUsers):
     "/customer/unblock",
     tags=["Customer Management"],
     dependencies=[Depends(user_api_key_auth)],
+    response_model=UnblockUsersResponse,
 )
 async def unblock_user(data: BlockUsers):
     """
@@ -213,11 +232,12 @@ async def _handle_customer_object_permission_update(
     "/customer/new",
     tags=["Customer Management"],
     dependencies=[Depends(user_api_key_auth)],
+    response_model=CustomerResponse,
 )
 async def new_end_user(
     data: NewCustomerRequest,
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
-):
+) -> CustomerResponse:
     """
     Allow creating a new Customer 
 
@@ -370,20 +390,7 @@ async def new_end_user(
             include={"litellm_budget_table": True, "object_permission": True},
         )
 
-        # Convert to dict and clean up recursive fields
-        response_dict = end_user_record.model_dump()
-        if response_dict.get("object_permission"):
-            # Remove reverse relations from object_permission
-            for field in [
-                "teams",
-                "verification_tokens",
-                "organizations",
-                "users",
-                "end_users",
-            ]:
-                response_dict["object_permission"].pop(field, None)
-
-        return response_dict
+        return _to_customer_response(end_user_record)
     except Exception as e:
         verbose_proxy_logger.exception(
             "litellm.proxy.management_endpoints.customer_endpoints.new_end_user(): Exception occured - {}".format(
@@ -404,7 +411,7 @@ async def new_end_user(
     "/customer/info",
     tags=["Customer Management"],
     dependencies=[Depends(user_api_key_auth)],
-    response_model=LiteLLM_EndUserTable,
+    response_model=CustomerResponse,
 )
 @router.get(
     "/end_user/info",
@@ -414,7 +421,7 @@ async def new_end_user(
 )
 async def end_user_info(
     end_user_id: str = fastapi.Query(description="End User ID in the request parameters"),
-):
+) -> CustomerResponse:
     """
     Get information about an end-user. An `end_user` is a customer (external user) of the proxy.
 
@@ -449,20 +456,7 @@ async def end_user_info(
                 param="end_user_id",
             )
 
-        # Convert to dict and clean up recursive fields
-        response_dict = user_info.model_dump(exclude_none=True)
-        if response_dict.get("object_permission"):
-            # Remove reverse relations from object_permission
-            for field in [
-                "teams",
-                "verification_tokens",
-                "organizations",
-                "users",
-                "end_users",
-            ]:
-                response_dict["object_permission"].pop(field, None)
-
-        return response_dict
+        return _to_customer_response(user_info)
 
     except Exception as e:
         verbose_proxy_logger.exception(
@@ -477,6 +471,7 @@ async def end_user_info(
     "/customer/update",
     tags=["Customer Management"],
     dependencies=[Depends(user_api_key_auth)],
+    response_model=CustomerResponse,
 )
 @router.post(
     "/end_user/update",
@@ -487,7 +482,7 @@ async def end_user_info(
 async def update_end_user(
     data: UpdateCustomerRequest,
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
-):
+) -> CustomerResponse:
     """
     Example curl 
 
@@ -641,20 +636,7 @@ async def update_end_user(
                 raise ValueError(f"Failed updating customer data. User ID does not exist passed user_id={data.user_id}")
             verbose_proxy_logger.debug(f"received response from updating prisma client. response={response}")
 
-            # Convert to dict and clean up recursive fields
-            response_dict = response.model_dump()
-            if response_dict.get("object_permission"):
-                # Remove reverse relations from object_permission
-                for field in [
-                    "teams",
-                    "verification_tokens",
-                    "organizations",
-                    "users",
-                    "end_users",
-                ]:
-                    response_dict["object_permission"].pop(field, None)
-
-            return response_dict
+            return _to_customer_response(response)
         else:
             raise ValueError(f"user_id is required, passed user_id = {data.user_id}")
 
@@ -671,6 +653,7 @@ async def update_end_user(
     "/customer/delete",
     tags=["Customer Management"],
     dependencies=[Depends(user_api_key_auth)],
+    response_model=DeleteCustomersResponse,
 )
 @router.post(
     "/end_user/delete",
@@ -681,7 +664,7 @@ async def update_end_user(
 async def delete_end_user(
     data: DeleteCustomerRequest,
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
-):
+) -> DeleteCustomersResponse:
     """
     Delete multiple end-users.
 
@@ -728,10 +711,10 @@ async def delete_end_user(
                 where={"user_id": {"in": data.user_ids}}
             )
             verbose_proxy_logger.debug(f"received response from updating prisma client. response={response}")
-            return {
-                "deleted_customers": response,
-                "message": "Successfully deleted customers with ids: " + str(data.user_ids),
-            }
+            return DeleteCustomersResponse(
+                deleted_customers=response,
+                message="Successfully deleted customers with ids: " + str(data.user_ids),
+            )
         else:
             raise ValueError(f"user_id is required, passed user_id = {data.user_ids}")
 
@@ -747,7 +730,7 @@ async def delete_end_user(
     "/customer/list",
     tags=["Customer Management"],
     dependencies=[Depends(user_api_key_auth)],
-    response_model=List[LiteLLM_EndUserTable],
+    response_model=List[CustomerResponse],
 )
 @router.get(
     "/end_user/list",
@@ -758,7 +741,7 @@ async def delete_end_user(
 async def list_end_user(
     http_request: Request,
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
-):
+) -> List[CustomerResponse]:
     """
     [Admin-only] List all available customers
 
@@ -791,21 +774,7 @@ async def list_end_user(
             include={"litellm_budget_table": True, "object_permission": True}
         )
 
-        returned_response: List[LiteLLM_EndUserTable] = []
-        for item in response:
-            item_dict = item.model_dump()
-            # Remove reverse relations from object_permission
-            if item_dict.get("object_permission"):
-                for field in [
-                    "teams",
-                    "verification_tokens",
-                    "organizations",
-                    "users",
-                    "end_users",
-                ]:
-                    item_dict["object_permission"].pop(field, None)
-            returned_response.append(LiteLLM_EndUserTable(**item_dict))
-        return returned_response
+        return [_to_customer_response(item) for item in response]
 
     except Exception as e:
         verbose_proxy_logger.exception(
diff --git a/litellm/types/proxy/management_endpoints/customer_endpoints.py b/litellm/types/proxy/management_endpoints/customer_endpoints.py
new file mode 100644
index 00000000000..e7653360d63
--- /dev/null
+++ b/litellm/types/proxy/management_endpoints/customer_endpoints.py
@@ -0,0 +1,30 @@
+from typing import List, Optional
+
+from pydantic import BaseModel, Field
+
+from litellm.models.budget import LiteLLM_BudgetTableFull
+from litellm.models.end_user import LiteLLM_EndUserTable
+
+
+class CustomerResponse(LiteLLM_EndUserTable):
+    """Customer object returned by the /customer read+write endpoints.
+
+    Nests the full budget response model so server-managed budget fields
+    (budget_reset_at, created_at) survive response_model filtering, rather than
+    the narrow write-allowlist shape LiteLLM_EndUserTable carries for internal use.
+    """
+
+    litellm_budget_table: Optional[LiteLLM_BudgetTableFull] = None  # pyright: ignore
+
+
+class BlockUsersResponse(BaseModel):
+    blocked_users: List[LiteLLM_EndUserTable]
+
+
+class UnblockUsersResponse(BaseModel):
+    blocked_users: List[str] = Field(description="User IDs that remain blocked after this unblock call")
+
+
+class DeleteCustomersResponse(BaseModel):
+    deleted_customers: int
+    message: str
diff --git a/tests/test_litellm/proxy/management_endpoints/test_customer_budget.py b/tests/test_litellm/proxy/management_endpoints/test_customer_budget.py
index 41f43c75f7d..0beca0c15e8 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_customer_budget.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_customer_budget.py
@@ -134,8 +134,10 @@ async def test_update_customer_creates_budget_with_proper_relations(
     )
 
     # Mock end user update
+    mock_updated_user = MagicMock()
+    mock_updated_user.model_dump.return_value = {"user_id": "test-user", "blocked": False}
     mock_prisma_client.db.litellm_endusertable.update = AsyncMock(
-        return_value=MagicMock()
+        return_value=mock_updated_user
     )
 
     # Create update request with budget creation fields (not just budget_id)
@@ -190,8 +192,10 @@ async def test_update_customer_creates_budget_with_required_fields(
     )
 
     # Mock end user update
+    mock_updated_user = MagicMock()
+    mock_updated_user.model_dump.return_value = {"user_id": "test-user", "blocked": False}
     mock_prisma_client.db.litellm_endusertable.update = AsyncMock(
-        return_value=MagicMock()
+        return_value=mock_updated_user
     )
 
     # Create update request with budget creation fields
@@ -253,8 +257,10 @@ async def test_update_customer_budget_creation_with_fallback_admin(
     )
 
     # Mock end user update
+    mock_updated_user = MagicMock()
+    mock_updated_user.model_dump.return_value = {"user_id": "test-user", "blocked": False}
     mock_prisma_client.db.litellm_endusertable.update = AsyncMock(
-        return_value=MagicMock()
+        return_value=mock_updated_user
     )
 
     # Create update request with budget creation fields
@@ -309,6 +315,7 @@ async def test_update_customer_with_budget_id_and_creation_fields(
 
     # Mock end user update
     mock_updated_user = MagicMock()
+    mock_updated_user.model_dump.return_value = {"user_id": "test-user", "blocked": False}
     mock_prisma_client.db.litellm_endusertable.update = AsyncMock(
         return_value=mock_updated_user
     )
diff --git a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py
index 6c5ccd3562f..d4089b23e81 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py
@@ -1,18 +1,28 @@
+from typing import List
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 from fastapi import FastAPI, HTTPException, Request, status
 from fastapi.responses import JSONResponse
+from fastapi.routing import APIRoute
 from fastapi.testclient import TestClient
 
 from litellm.proxy._types import (
-    LiteLLM_BudgetTable,
     LiteLLM_EndUserTable,
     LitellmUserRoles,
     ProxyException,
 )
 from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth
 from litellm.proxy.management_endpoints.customer_endpoints import router
+from litellm.types.proxy.management_endpoints.common_daily_activity import (
+    SpendAnalyticsPaginatedResponse,
+)
+from litellm.types.proxy.management_endpoints.customer_endpoints import (
+    BlockUsersResponse,
+    CustomerResponse,
+    DeleteCustomersResponse,
+    UnblockUsersResponse,
+)
 
 app = FastAPI()
 
@@ -22,9 +32,7 @@ async def openai_exception_handler(request: Request, exc: ProxyException):
     headers = exc.headers
     error_dict = exc.to_dict()
     return JSONResponse(
-        status_code=(
-            int(exc.code) if exc.code else status.HTTP_500_INTERNAL_SERVER_ERROR
-        ),
+        status_code=(int(exc.code) if exc.code else status.HTTP_500_INTERNAL_SERVER_ERROR),
         content={"error": error_dict},
         headers=headers,
     )
@@ -54,30 +62,20 @@ def mock_user_api_key_auth():
 
 def test_update_customer_success(mock_prisma_client, mock_user_api_key_auth):
     # Mock the database responses
-    mock_end_user = LiteLLM_EndUserTable(
-        user_id="test-user-1", alias="Test User", blocked=False
-    )
-    updated_mock_end_user = LiteLLM_EndUserTable(
-        user_id="test-user-1", alias="Updated Test User", blocked=False
-    )
+    mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", alias="Test User", blocked=False)
+    updated_mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", alias="Updated Test User", blocked=False)
 
     # Mock the find_first response
-    mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(
-        return_value=mock_end_user
-    )
+    mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(return_value=mock_end_user)
 
     # Mock the update response
-    mock_prisma_client.db.litellm_endusertable.update = AsyncMock(
-        return_value=updated_mock_end_user
-    )
+    mock_prisma_client.db.litellm_endusertable.update = AsyncMock(return_value=updated_mock_end_user)
 
     # Test data
     test_data = {"user_id": "test-user-1", "alias": "Updated Test User"}
 
     # Make the request
-    response = client.post(
-        "/customer/update", json=test_data, headers={"Authorization": "Bearer test-key"}
-    )
+    response = client.post("/customer/update", json=test_data, headers={"Authorization": "Bearer test-key"})
 
     # Assert response
     assert response.status_code == 200
@@ -106,10 +104,7 @@ def test_update_customer_not_found(mock_prisma_client, mock_user_api_key_auth):
     assert response.status_code == 404
     response_json = response.json()
     assert "error" in response_json
-    assert (
-        response_json["error"]["message"]
-        == "End User Id=non-existent-user does not exist in db"
-    )
+    assert response_json["error"]["message"] == "End User Id=non-existent-user does not exist in db"
     assert response_json["error"]["type"] == "not_found"
     assert response_json["error"]["param"] == "user_id"
     assert response_json["error"]["code"] == "404"
@@ -132,10 +127,7 @@ def test_info_customer_not_found(mock_prisma_client, mock_user_api_key_auth):
     assert response.status_code == 404
     response_json = response.json()
     assert "error" in response_json
-    assert (
-        response_json["error"]["message"]
-        == "End User Id=non-existent-user does not exist in db"
-    )
+    assert response_json["error"]["message"] == "End User Id=non-existent-user does not exist in db"
     assert response_json["error"]["type"] == "not_found"
     assert response_json["error"]["param"] == "end_user_id"
     assert response_json["error"]["code"] == "404"
@@ -220,11 +212,6 @@ def test_error_schema_consistency(mock_prisma_client, mock_user_api_key_auth):
     assert error["code"] == "404"
 
     # Test /customer/new - duplicate user error
-    from unittest.mock import MagicMock
-
-    mock_end_user = LiteLLM_EndUserTable(
-        user_id="existing-user", alias="Existing User", blocked=False
-    )
     mock_prisma_client.db.litellm_endusertable.create = AsyncMock(
         side_effect=Exception("Unique constraint failed on the fields: (`user_id`)")
     )
@@ -238,9 +225,7 @@ def test_error_schema_consistency(mock_prisma_client, mock_user_api_key_auth):
     assert error["code"] == "400"
 
 
-def test_customer_endpoints_error_schema_consistency(
-    mock_prisma_client, mock_user_api_key_auth
-):
+def test_customer_endpoints_error_schema_consistency(mock_prisma_client, mock_user_api_key_auth):
     """
     Test the exact scenarios from the curl examples provided.
 
@@ -307,9 +292,7 @@ def test_customer_endpoints_error_schema_consistency(
     assert "Customer already exists" in error2["message"]
 
     # Verify both errors have the same schema structure
-    assert set(error1.keys()) == set(
-        error2.keys()
-    ), "Both errors should have the same top-level keys"
+    assert set(error1.keys()) == set(error2.keys()), "Both errors should have the same top-level keys"
 
     # Both should have string values for all fields
     for key in ["message", "type", "code"]:
@@ -317,6 +300,153 @@ def test_customer_endpoints_error_schema_consistency(
         assert isinstance(error2[key], str), f"error2[{key}] should be a string"
 
 
+EXPECTED_RESPONSE_MODELS = {
+    "/customer/block": BlockUsersResponse,
+    "/customer/unblock": UnblockUsersResponse,
+    "/customer/new": CustomerResponse,
+    "/customer/update": CustomerResponse,
+    "/customer/delete": DeleteCustomersResponse,
+    "/customer/info": CustomerResponse,
+    "/customer/list": List[CustomerResponse],
+    "/customer/daily/activity": SpendAnalyticsPaginatedResponse,
+}
+
+
+@pytest.mark.parametrize("path, expected_model", EXPECTED_RESPONSE_MODELS.items())
+def test_customer_routes_declare_response_model(path, expected_model):
+    """
+    Every public /customer/* operation must declare a typed response_model so
+    the generated OpenAPI schema documents the response body. Regression for the
+    OpenAPI response-type coverage goal: drop a response_model and this fails.
+    """
+    route = next(r for r in router.routes if isinstance(r, APIRoute) and r.path == path)
+    assert route.response_model == expected_model
+
+
+def test_customer_new_documented_in_openapi_schema():
+    """
+    The response_model must surface in the OpenAPI schema as a concrete ref, not
+    an empty/default response. This is what the coverage metric measures.
+    """
+    schema = app.openapi()["paths"]["/customer/new"]["post"]
+    json_schema = schema["responses"]["200"]["content"]["application/json"]["schema"]
+    assert json_schema["$ref"].endswith("/CustomerResponse")
+
+
+def test_update_customer_response_preserves_budget_id(mock_prisma_client, mock_user_api_key_auth):
+    """
+    Regression for the response_model field-stripping concern: budget_id is a real
+    column on the end-user table that /customer/update echoes. response_model=
+    LiteLLM_EndUserTable must NOT drop it, so budget_id stays in LiteLLM_EndUserTable.
+    """
+    existing = LiteLLM_EndUserTable(user_id="cust-1", blocked=False)
+    updated = LiteLLM_EndUserTable(user_id="cust-1", blocked=False, budget_id="budget-123")
+    mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(return_value=existing)
+    mock_prisma_client.db.litellm_endusertable.update = AsyncMock(return_value=updated)
+
+    response = client.post(
+        "/customer/update",
+        json={"user_id": "cust-1", "budget_id": "budget-123"},
+        headers={"Authorization": "Bearer test-key"},
+    )
+
+    assert response.status_code == 200
+    assert response.json()["budget_id"] == "budget-123"
+
+
+def test_update_customer_response_keeps_nested_budget_server_fields(mock_prisma_client, mock_user_api_key_auth):
+    """
+    Faithfulness regression: /customer/update embeds the full budget row. The
+    response_model must keep the server-managed budget fields the endpoint used
+    to return (budget_reset_at, created_at) instead of the narrow write-allowlist
+    shape. The intentionally-internal audit fields (created_by/updated_by) stay out.
+    """
+    existing = LiteLLM_EndUserTable(user_id="cust-1", blocked=False)
+    raw_row = MagicMock()
+    raw_row.model_dump.return_value = {
+        "user_id": "cust-1",
+        "blocked": False,
+        "alias": "renamed",
+        "spend": 0.0,
+        "allowed_model_region": None,
+        "default_model": None,
+        "budget_id": "b-1",
+        "object_permission_id": None,
+        "object_permission": None,
+        "litellm_budget_table": {
+            "budget_id": "b-1",
+            "max_budget": 10.0,
+            "budget_duration": "30d",
+            "budget_reset_at": "2024-02-01T00:00:00",
+            "created_at": "2024-01-01T00:00:00",
+            "created_by": "admin",
+            "updated_at": "2024-01-02T00:00:00",
+            "updated_by": "admin",
+        },
+    }
+    mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(return_value=existing)
+    mock_prisma_client.db.litellm_endusertable.update = AsyncMock(return_value=raw_row)
+
+    response = client.post(
+        "/customer/update",
+        json={"user_id": "cust-1", "alias": "renamed"},
+        headers={"Authorization": "Bearer test-key"},
+    )
+
+    assert response.status_code == 200
+    budget = response.json()["litellm_budget_table"]
+    assert budget["budget_reset_at"] == "2024-02-01T00:00:00"
+    assert budget["created_at"] == "2024-01-01T00:00:00"
+    assert "created_by" not in budget
+    assert "updated_by" not in budget
+
+
+def test_block_customer_success_serializes_through_response_model(mock_prisma_client, mock_user_api_key_auth):
+    """
+    /customer/block returns {"blocked_users": [<end user rows>]}. With
+    response_model=BlockUsersResponse, a shape mismatch would raise a 500
+    ResponseValidationError, so a clean 200 proves the model matches runtime output.
+    """
+    blocked_row = LiteLLM_EndUserTable(user_id="blocked-1", blocked=True)
+    mock_prisma_client.db.litellm_endusertable.upsert = AsyncMock(return_value=blocked_row)
+
+    response = client.post(
+        "/customer/block",
+        json={"user_ids": ["blocked-1"]},
+        headers={"Authorization": "Bearer test-key"},
+    )
+
+    assert response.status_code == 200
+    body = response.json()
+    assert body["blocked_users"][0]["user_id"] == "blocked-1"
+    assert body["blocked_users"][0]["blocked"] is True
+
+
+def test_delete_customer_success_serializes_through_response_model(mock_prisma_client, mock_user_api_key_auth):
+    """
+    /customer/delete returns {"deleted_customers": <int>, "message": <str>}.
+    response_model=DeleteCustomersResponse enforces that exact shape.
+    """
+    existing = [
+        LiteLLM_EndUserTable(user_id="u1", blocked=False),
+        LiteLLM_EndUserTable(user_id="u2", blocked=False),
+    ]
+    mock_prisma_client.db.litellm_endusertable.find_many = AsyncMock(return_value=existing)
+    mock_prisma_client.db.litellm_endusertable.delete_many = AsyncMock(return_value=2)
+
+    response = client.post(
+        "/customer/delete",
+        json={"user_ids": ["u1", "u2"]},
+        headers={"Authorization": "Bearer test-key"},
+    )
+
+    assert response.status_code == 200
+    assert response.json() == {
+        "deleted_customers": 2,
+        "message": "Successfully deleted customers with ids: ['u1', 'u2']",
+    }
+
+
 @pytest.mark.asyncio
 async def test_get_customer_daily_activity_admin_param_passing(monkeypatch):
     from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
@@ -331,9 +461,7 @@ async def test_get_customer_daily_activity_admin_param_passing(monkeypatch):
 
     mocked_response = MagicMock(name="SpendAnalyticsPaginatedResponse")
     get_daily_activity_mock = AsyncMock(return_value=mocked_response)
-    monkeypatch.setattr(
-        customer_endpoints, "get_daily_activity", get_daily_activity_mock
-    )
+    monkeypatch.setattr(customer_endpoints, "get_daily_activity", get_daily_activity_mock)
 
     auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin1")
     result = await get_customer_daily_activity(
@@ -380,16 +508,12 @@ async def test_get_customer_daily_activity_with_end_user_aliases(monkeypatch):
     mock_end_user2.user_id = "end-user-2"
     mock_end_user2.alias = "Customer Two"
 
-    mock_prisma_client.db.litellm_endusertable.find_many = AsyncMock(
-        return_value=[mock_end_user1, mock_end_user2]
-    )
+    mock_prisma_client.db.litellm_endusertable.find_many = AsyncMock(return_value=[mock_end_user1, mock_end_user2])
     monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     mocked_response = MagicMock(name="SpendAnalyticsPaginatedResponse")
     get_daily_activity_mock = AsyncMock(return_value=mocked_response)
-    monkeypatch.setattr(
-        customer_endpoints, "get_daily_activity", get_daily_activity_mock
-    )
+    monkeypatch.setattr(customer_endpoints, "get_daily_activity", get_daily_activity_mock)
 
     auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin1")
     await get_customer_daily_activity(
@@ -436,9 +560,7 @@ async def test_get_customer_daily_activity_non_admin_is_rejected(monkeypatch):
     monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     get_daily_activity_mock = AsyncMock()
-    monkeypatch.setattr(
-        customer_endpoints, "get_daily_activity", get_daily_activity_mock
-    )
+    monkeypatch.setattr(customer_endpoints, "get_daily_activity", get_daily_activity_mock)
 
     non_admin_key = UserAPIKeyAuth(
         user_id="regular-user-abc",
@@ -482,9 +604,7 @@ async def test_get_customer_daily_activity_service_account_key_is_rejected(monke
     monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
 
     get_daily_activity_mock = AsyncMock()
-    monkeypatch.setattr(
-        customer_endpoints, "get_daily_activity", get_daily_activity_mock
-    )
+    monkeypatch.setattr(customer_endpoints, "get_daily_activity", get_daily_activity_mock)
 
     service_account_key = UserAPIKeyAuth(
         user_id=None,
@@ -507,3 +627,157 @@ async def test_get_customer_daily_activity_service_account_key_is_rejected(monke
     assert exc_info.value.status_code == 401
     assert "Admin-only endpoint" in str(exc_info.value.detail)
     get_daily_activity_mock.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# Characterization (golden-master) tests.
+#
+# These lock the EXACT JSON body every customer-object endpoint returns today,
+# so a type-safety refactor of the handlers is only allowed to land if it
+# reproduces these byte for byte. The input below is what a Prisma row's
+# .model_dump() yields (full nested budget incl. audit fields + object_permission
+# incl. reverse relations); the expected output is what the live endpoint emits.
+# ---------------------------------------------------------------------------
+
+_FULL_DB_ROW = {
+    "user_id": "c1",
+    "blocked": False,
+    "alias": "Acme",
+    "spend": 1.5,
+    "allowed_model_region": None,
+    "default_model": None,
+    "budget_id": "b1",
+    "object_permission_id": "p1",
+    "litellm_budget_table": {
+        "budget_id": "b1",
+        "max_budget": 10.0,
+        "soft_budget": None,
+        "max_parallel_requests": None,
+        "tpm_limit": None,
+        "rpm_limit": None,
+        "model_max_budget": None,
+        "budget_duration": "30d",
+        "allowed_models": [],
+        "budget_reset_at": "2024-02-01T00:00:00",
+        "created_at": "2024-01-01T00:00:00",
+        "created_by": "admin",
+        "updated_at": "2024-01-02T00:00:00",
+        "updated_by": "admin",
+    },
+    "object_permission": {
+        "object_permission_id": "p1",
+        "mcp_servers": ["s1"],
+        "mcp_access_groups": [],
+        "mcp_tool_permissions": None,
+        "vector_stores": [],
+        "agents": [],
+        "agent_access_groups": [],
+        "models": [],
+        "mcp_toolsets": None,
+        "blocked_tools": [],
+        "search_tools": [],
+        "teams": [{"team_id": "t1"}],
+        "users": [{"user_id": "x"}],
+        "end_users": [],
+        "organizations": [],
+        "verification_tokens": [],
+    },
+}
+
+_EXPECTED_CUSTOMER = {
+    "user_id": "c1",
+    "blocked": False,
+    "alias": "Acme",
+    "spend": 1.5,
+    "allowed_model_region": None,
+    "default_model": None,
+    "budget_id": "b1",
+    "litellm_budget_table": {
+        "budget_id": "b1",
+        "soft_budget": None,
+        "max_budget": 10.0,
+        "max_parallel_requests": None,
+        "tpm_limit": None,
+        "rpm_limit": None,
+        "model_max_budget": None,
+        "budget_duration": "30d",
+        "allowed_models": [],
+        "budget_reset_at": "2024-02-01T00:00:00",
+        "created_at": "2024-01-01T00:00:00",
+    },
+    "object_permission_id": "p1",
+    "object_permission": {
+        "object_permission_id": "p1",
+        "mcp_servers": ["s1"],
+        "mcp_access_groups": [],
+        "mcp_tool_permissions": None,
+        "vector_stores": [],
+        "agents": [],
+        "agent_access_groups": [],
+        "models": [],
+        "mcp_toolsets": None,
+        "blocked_tools": [],
+        "search_tools": [],
+    },
+}
+
+
+def _row(dump: dict) -> MagicMock:
+    row = MagicMock()
+    row.model_dump.return_value = dump
+    return row
+
+
+def test_char_info_body(mock_prisma_client, mock_user_api_key_auth):
+    mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(return_value=_row(_FULL_DB_ROW))
+    response = client.get("/customer/info?end_user_id=c1", headers={"Authorization": "Bearer k"})
+    assert response.status_code == 200
+    assert response.json() == _EXPECTED_CUSTOMER
+
+
+def test_char_list_body(mock_prisma_client, mock_user_api_key_auth):
+    mock_prisma_client.db.litellm_endusertable.find_many = AsyncMock(return_value=[_row(_FULL_DB_ROW)])
+    response = client.get("/customer/list", headers={"Authorization": "Bearer k"})
+    assert response.status_code == 200
+    assert response.json() == [_EXPECTED_CUSTOMER]
+
+
+def test_char_new_body(mock_prisma_client, mock_user_api_key_auth):
+    mock_prisma_client.db.litellm_endusertable.create = AsyncMock(return_value=_row(_FULL_DB_ROW))
+    response = client.post("/customer/new", json={"user_id": "c1"}, headers={"Authorization": "Bearer k"})
+    assert response.status_code == 200
+    assert response.json() == _EXPECTED_CUSTOMER
+
+
+def test_char_update_body(mock_prisma_client, mock_user_api_key_auth):
+    mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(
+        return_value=_row({"user_id": "c1", "blocked": False})
+    )
+    mock_prisma_client.db.litellm_endusertable.update = AsyncMock(return_value=_row(_FULL_DB_ROW))
+    response = client.post(
+        "/customer/update",
+        json={"user_id": "c1", "alias": "Acme"},
+        headers={"Authorization": "Bearer k"},
+    )
+    assert response.status_code == 200
+    assert response.json() == _EXPECTED_CUSTOMER
+
+
+def test_char_delete_body(mock_prisma_client, mock_user_api_key_auth):
+    mock_prisma_client.db.litellm_endusertable.find_many = AsyncMock(
+        return_value=[
+            LiteLLM_EndUserTable(user_id="c1", blocked=False),
+            LiteLLM_EndUserTable(user_id="c2", blocked=False),
+        ]
+    )
+    mock_prisma_client.db.litellm_endusertable.delete_many = AsyncMock(return_value=2)
+    response = client.post(
+        "/customer/delete",
+        json={"user_ids": ["c1", "c2"]},
+        headers={"Authorization": "Bearer k"},
+    )
+    assert response.status_code == 200
+    assert response.json() == {
+        "deleted_customers": 2,
+        "message": "Successfully deleted customers with ids: ['c1', 'c2']",
+    }
diff --git a/ui/litellm-dashboard/scripts/gen-api-types.mjs b/ui/litellm-dashboard/scripts/gen-api-types.mjs
index 3c9373ec547..6b9f8581292 100644
--- a/ui/litellm-dashboard/scripts/gen-api-types.mjs
+++ b/ui/litellm-dashboard/scripts/gen-api-types.mjs
@@ -26,15 +26,26 @@ const python = (process.env.LITELLM_PYTHON ?? "python3").split(" ");
 // The dashboard calls internal UI routes that the public /openapi.json hides via
 // include_in_schema=False. Force them in so they get typed here; this mutates a
 // throwaway interpreter, so the spec the proxy actually serves is unchanged.
+// Python 3.13 strips a docstring's common leading indentation at compile time
+// while 3.12 keeps it, so the same model yields differently-indented descriptions
+// depending on the interpreter — enough to make this output non-reproducible
+// across CI and contributors. inspect.cleandoc normalizes every description to one
+// canonical form regardless of interpreter, so the generated file is stable.
 const dumpSpec = [
-  "import json, sys",
+  "import inspect, json, sys",
   "from litellm.proxy.proxy_server import app",
   "from fastapi.routing import APIRoute",
   "for route in app.routes:",
   "    if isinstance(route, APIRoute):",
   "        route.include_in_schema = True",
   "app.openapi_schema = None",
-  "with open(sys.argv[1], 'w') as f: json.dump(app.openapi(), f, sort_keys=True)",
+  "def normalize(node):",
+  "    if isinstance(node, dict):",
+  "        return {k: inspect.cleandoc(v) if k == 'description' and isinstance(v, str) else normalize(v) for k, v in node.items()}",
+  "    if isinstance(node, list):",
+  "        return [normalize(v) for v in node]",
+  "    return node",
+  "with open(sys.argv[1], 'w') as f: json.dump(normalize(app.openapi()), f, sort_keys=True)",
 ].join("\n");
 
 try {
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 7d5d617c824..f15eaf9ea1f 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -3027,8 +3027,8 @@ export interface paths {
         /**
          * Get Active Tasks Stats
          * @description Returns:
-         *       total_active_tasks: int
-         *       by_name: { coroutine_name: count }
+         *     total_active_tasks: int
+         *     by_name: { coroutine_name: count }
          */
         get: operations["get_active_tasks_stats_debug_asyncio_tasks_get"];
         put?: never;
@@ -21003,6 +21003,11 @@ export interface components {
             /** User Ids */
             user_ids: string[];
         };
+        /** BlockUsersResponse */
+        BlockUsersResponse: {
+            /** Blocked Users */
+            blocked_users: components["schemas"]["LiteLLM_EndUserTable"][];
+        };
         /**
          * BlockedWord
          * @description Represents a blocked word with its action and optional description
@@ -22651,10 +22656,10 @@ export interface components {
         /**
          * ContentFilterCategoryConfig
          * @description category: "harmful_self_harm"
-         *                   enabled: true
-         *                   action: "BLOCK"
-         *                   severity_threshold: "medium"
-         *                   category_file: "/path/to/custom_file.yaml"  # optional override
+         *     enabled: true
+         *     action: "BLOCK"
+         *     severity_threshold: "medium"
+         *     category_file: "/path/to/custom_file.yaml"  # optional override
          */
         ContentFilterCategoryConfig: {
             /**
@@ -22879,6 +22884,37 @@ export interface components {
                 [key: string]: unknown;
             };
         };
+        /**
+         * CustomerResponse
+         * @description Customer object returned by the /customer read+write endpoints.
+         *
+         *     Nests the full budget response model so server-managed budget fields
+         *     (budget_reset_at, created_at) survive response_model filtering, rather than
+         *     the narrow write-allowlist shape LiteLLM_EndUserTable carries for internal use.
+         */
+        CustomerResponse: {
+            /** Alias */
+            alias?: string | null;
+            /** Allowed Model Region */
+            allowed_model_region?: ("eu" | "us") | null;
+            /** Blocked */
+            blocked: boolean;
+            /** Budget Id */
+            budget_id?: string | null;
+            /** Default Model */
+            default_model?: string | null;
+            litellm_budget_table?: components["schemas"]["LiteLLM_BudgetTableFull"] | null;
+            object_permission?: components["schemas"]["LiteLLM_ObjectPermissionTable"] | null;
+            /** Object Permission Id */
+            object_permission_id?: string | null;
+            /**
+             * Spend
+             * @default 0
+             */
+            spend: number;
+            /** User Id */
+            user_id: string;
+        };
         /** DailySpendData */
         DailySpendData: {
             breakdown?: components["schemas"]["BreakdownMetrics"];
@@ -23043,6 +23079,13 @@ export interface components {
             /** User Ids */
             user_ids: string[];
         };
+        /** DeleteCustomersResponse */
+        DeleteCustomersResponse: {
+            /** Deleted Customers */
+            deleted_customers: number;
+            /** Message */
+            message: string;
+        };
         /**
          * DeleteEvalResponse
          * @description Response from deleting an evaluation
@@ -24792,6 +24835,8 @@ export interface components {
             allowed_model_region?: ("eu" | "us") | null;
             /** Blocked */
             blocked: boolean;
+            /** Budget Id */
+            budget_id?: string | null;
             /** Default Model */
             default_model?: string | null;
             litellm_budget_table?: components["schemas"]["LiteLLM_BudgetTable"] | null;
@@ -31477,6 +31522,14 @@ export interface components {
              */
             workers: components["schemas"]["WorkerRegistryEntry"][];
         };
+        /** UnblockUsersResponse */
+        UnblockUsersResponse: {
+            /**
+             * Blocked Users
+             * @description User IDs that remain blocked after this unblock call
+             */
+            blocked_users: string[];
+        };
         /**
          * UpdateCustomerRequest
          * @description Update a Customer, use this to update customer budgets etc
@@ -37482,7 +37535,7 @@ export interface operations {
                     [name: string]: unknown;
                 };
                 content: {
-                    "application/json": unknown;
+                    "application/json": components["schemas"]["BlockUsersResponse"];
                 };
             };
             /** @description Validation Error */
@@ -37553,7 +37606,7 @@ export interface operations {
                     [name: string]: unknown;
                 };
                 content: {
-                    "application/json": unknown;
+                    "application/json": components["schemas"]["DeleteCustomersResponse"];
                 };
             };
             /** @description Validation Error */
@@ -37585,7 +37638,7 @@ export interface operations {
                     [name: string]: unknown;
                 };
                 content: {
-                    "application/json": components["schemas"]["LiteLLM_EndUserTable"];
+                    "application/json": components["schemas"]["CustomerResponse"];
                 };
             };
             /** @description Validation Error */
@@ -37614,7 +37667,7 @@ export interface operations {
                     [name: string]: unknown;
                 };
                 content: {
-                    "application/json": components["schemas"]["LiteLLM_EndUserTable"][];
+                    "application/json": components["schemas"]["CustomerResponse"][];
                 };
             };
         };
@@ -37638,7 +37691,7 @@ export interface operations {
                     [name: string]: unknown;
                 };
                 content: {
-                    "application/json": unknown;
+                    "application/json": components["schemas"]["CustomerResponse"];
                 };
             };
             /** @description Validation Error */
@@ -37671,7 +37724,7 @@ export interface operations {
                     [name: string]: unknown;
                 };
                 content: {
-                    "application/json": unknown;
+                    "application/json": components["schemas"]["UnblockUsersResponse"];
                 };
             };
             /** @description Validation Error */
@@ -37704,7 +37757,7 @@ export interface operations {
                     [name: string]: unknown;
                 };
                 content: {
-                    "application/json": unknown;
+                    "application/json": components["schemas"]["CustomerResponse"];
                 };
             };
             /** @description Validation Error */
@@ -38130,7 +38183,7 @@ export interface operations {
                     [name: string]: unknown;
                 };
                 content: {
-                    "application/json": unknown;
+                    "application/json": components["schemas"]["DeleteCustomersResponse"];
                 };
             };
             /** @description Validation Error */
@@ -38162,7 +38215,7 @@ export interface operations {
                     [name: string]: unknown;
                 };
                 content: {
-                    "application/json": unknown;
+                    "application/json": components["schemas"]["CustomerResponse"];
                 };
             };
             /** @description Validation Error */
@@ -38191,7 +38244,7 @@ export interface operations {
                     [name: string]: unknown;
                 };
                 content: {
-                    "application/json": unknown;
+                    "application/json": components["schemas"]["CustomerResponse"][];
                 };
             };
         };
@@ -38215,7 +38268,7 @@ export interface operations {
                     [name: string]: unknown;
                 };
                 content: {
-                    "application/json": unknown;
+                    "application/json": components["schemas"]["CustomerResponse"];
                 };
             };
             /** @description Validation Error */
@@ -38281,7 +38334,7 @@ export interface operations {
                     [name: string]: unknown;
                 };
                 content: {
-                    "application/json": unknown;
+                    "application/json": components["schemas"]["CustomerResponse"];
                 };
             };
             /** @description Validation Error */
@@ -43890,13 +43943,13 @@ export interface operations {
             /**
              * @description Unified rate-limit error.
              *
-             *         Every rate-limit condition surfaced by litellm — whether it originated from
-             *         an upstream LLM provider, a vendor batch endpoint, or one of litellm's own
-             *         proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,
-             *         max-iterations, etc.) — is raised as an instance of this class.
+             *     Every rate-limit condition surfaced by litellm — whether it originated from
+             *     an upstream LLM provider, a vendor batch endpoint, or one of litellm's own
+             *     proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,
+             *     max-iterations, etc.) — is raised as an instance of this class.
              *
-             *         The :attr:`category` attribute lets callers distinguish the source. See
-             *         :class:`RateLimitErrorCategory` for the available values.
+             *     The :attr:`category` attribute lets callers distinguish the source. See
+             *     :class:`RateLimitErrorCategory` for the available values.
              */
             429: {
                 headers: {

From 2e575d39f2557926c0aecadd67b8afba81c640e2 Mon Sep 17 00:00:00 2001
From: Yassin Kortam <yassin@berri.ai>
Date: Tue, 30 Jun 2026 20:26:20 +0300
Subject: [PATCH 09/51] perf(otel): memoize per-request lazy import of otel
 runtime hooks (#31707)

The proxy auth path calls phase_span() and seed_request_identity() in
litellm/integrations/otel/runtime.py on every request, each doing a
try/except lazy import of litellm.integrations.otel.logger. When the
OpenTelemetry SDK is not installed (the default), that import raises, and
CPython never caches a failed import, so every request re-scanned sys.path
and contended on the import lock. At 750 concurrent users this cost about
12% throughput versus v1.85.0.

Resolve the hooks once and cache the outcome, absence included, with
functools.cache, so the import is attempted a single time instead of per
request. Throughput returns to the v1.85.0 baseline.
---
 litellm/integrations/otel/runtime.py          | 34 ++++++----
 .../integrations/otel/test_runtime.py         | 64 +++++++++++++++++++
 2 files changed, 87 insertions(+), 11 deletions(-)
 create mode 100644 tests/test_litellm/integrations/otel/test_runtime.py

diff --git a/litellm/integrations/otel/runtime.py b/litellm/integrations/otel/runtime.py
index ac3b991c971..eb512375023 100644
--- a/litellm/integrations/otel/runtime.py
+++ b/litellm/integrations/otel/runtime.py
@@ -8,7 +8,23 @@ identity unconditionally.
 """
 
 from contextlib import contextmanager
-from typing import Any, Iterator
+from functools import cache
+from typing import Any, Callable, Iterator, Optional
+
+
+@cache
+def _otel_runtime() -> "Optional[tuple[Callable[[str], Any], Callable[..., None]]]":
+    """Resolve the SDK-backed hooks once and cache the outcome, absence included.
+
+    CPython never caches a failed import, so without this memoization every call
+    site re-attempts the import on each request; when the OTel SDK is not installed
+    that re-scans ``sys.path`` and contends on the import lock on the hot path.
+    """
+    try:
+        from litellm.integrations.otel import logger
+    except Exception:
+        return None
+    return (logger.phase_span, logger.seed_request_identity)
 
 
 @contextmanager
@@ -18,21 +34,17 @@ def phase_span(name: str) -> "Iterator[Any]":
     Yields ``None`` (a plain no-op) when the OTel SDK is unavailable or V2 is not
     the active logger.
     """
-    try:
-        from litellm.integrations.otel.logger import phase_span as _phase_span
-    except Exception:
+    runtime = _otel_runtime()
+    if runtime is None:
         yield None
         return
-    with _phase_span(name) as span:
+    with runtime[0](name) as span:
         yield span
 
 
 def seed_request_identity(user_api_key_dict: Any, model: Any = None) -> None:
     """Seed request-identity Baggage at the auth boundary (no-op without V2)."""
-    try:
-        from litellm.integrations.otel.logger import (
-            seed_request_identity as _seed_request_identity,
-        )
-    except Exception:
+    runtime = _otel_runtime()
+    if runtime is None:
         return
-    _seed_request_identity(user_api_key_dict, model=model)
+    runtime[1](user_api_key_dict, model=model)
diff --git a/tests/test_litellm/integrations/otel/test_runtime.py b/tests/test_litellm/integrations/otel/test_runtime.py
new file mode 100644
index 00000000000..d11f31b2523
--- /dev/null
+++ b/tests/test_litellm/integrations/otel/test_runtime.py
@@ -0,0 +1,64 @@
+"""Regression tests for the SDK-free OTel runtime shim.
+
+The proxy auth hot path calls ``phase_span`` and ``seed_request_identity`` on
+every request. These wrappers resolve the SDK-backed implementations with a
+lazy import. CPython never caches a failed import, so before memoization an
+absent OTel SDK made every request re-scan ``sys.path`` and contend on the
+import lock. These tests pin the import to a single resolution.
+"""
+
+import builtins
+
+import litellm.integrations.otel.runtime as runtime
+
+
+def test_logger_not_reimported_after_first_resolution(monkeypatch):
+    runtime._otel_runtime.cache_clear()
+
+    counts = {"n": 0}
+    real_import = builtins.__import__
+
+    def counting_import(name, globals=None, locals=None, fromlist=(), level=0):
+        if name == "litellm.integrations.otel" and fromlist and "logger" in fromlist:
+            counts["n"] += 1
+        return real_import(name, globals, locals, fromlist, level)
+
+    monkeypatch.setattr(builtins, "__import__", counting_import)
+
+    with runtime.phase_span("auth /v1/chat/completions"):
+        pass
+    after_first = counts["n"]
+
+    for _ in range(49):
+        with runtime.phase_span("auth /v1/chat/completions"):
+            pass
+
+    assert counts["n"] == after_first, (
+        f"otel.logger re-imported {counts['n'] - after_first} times after the first "
+        "resolution; it must be memoized so it does not re-scan sys.path per request"
+    )
+
+    runtime._otel_runtime.cache_clear()
+
+
+def test_resolution_is_memoized():
+    runtime._otel_runtime.cache_clear()
+
+    for _ in range(25):
+        with runtime.phase_span("p"):
+            pass
+
+    info = runtime._otel_runtime.cache_info()
+    assert info.misses == 1
+    assert info.hits >= 24
+
+    runtime._otel_runtime.cache_clear()
+
+
+def test_wrappers_no_op_when_runtime_absent(monkeypatch):
+    monkeypatch.setattr(runtime, "_otel_runtime", lambda: None)
+
+    with runtime.phase_span("auth") as span:
+        assert span is None
+
+    assert runtime.seed_request_identity({"token": "sk-x"}, model="gpt-4o") is None

From 468d11f71d2edeb2118d14de73d7733fbce62511 Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Tue, 30 Jun 2026 10:26:57 -0700
Subject: [PATCH 10/51] feat(otel): emit a tools/list CLIENT span for MCP
 discovery under otel_v2 (#31525)

* feat(otel): emit a tools/list CLIENT span for MCP discovery under otel_v2

Under otel_v2 an MCP tools/call already produced a dedicated CLIENT span, but tools/list produced none. The discovery call surfaced only as the bare POST /{mcp_server_name}/mcp server span with no MCP attributes, indistinguishable from initialize and impossible to query by method

The list success event already reaches the v2 logger with call_type list_mcp_tools, but _emit_mcp_tool_call only matched call_mcp_tool, so listing fell through to the LLM-call path and emitted nothing. This adds a dedicated MCP_LIST_TOOLS span role with its own MCPListToolsSpanData, emitted from a sibling _emit_mcp_list_tools branch that mirrors the tools/call path

Per the OTel GenAI MCP semantic conventions the span is named tools/list (the method name alone, since there is no low-cardinality target), is a CLIENT span parented to the request span, and carries mcp.method.name plus the call id. It deliberately omits gen_ai.operation.name and gen_ai.tool.name, which the convention reserves for tool executions, since listing runs no tool

* fix(otel): anchor MCP spans to params._meta trace context, not the transport span

MCP streamable-HTTP multiplexes many JSON-RPC messages over one session, so the request-root anchor captured on initialize persisted and every later message's span (tools/call, tools/list) nested under it. A tools/list run 44s after the initialize rendered 44s to the right of its parent with a clock-skew warning, because the MCP message and the HTTP transport are independent lifecycles

Following the OTel GenAI MCP semantic conventions, an MCP span now parents to the W3C trace context the client propagated in the request's params._meta (a remote parent, per SEP-414), records the transport/session span as a span link rather than the parent, and starts its own root trace when nothing was propagated. The MCP gateway captures traceparent/tracestate/baggage from each message's params._meta into a per-message contextvar that the otel_v2 emitter reads; opentelemetry stays an optional dependency via guarded lazy imports

This applies to tools/call as well as the new tools/list span, since both shared the same transport-anchoring bug

* fix(otel): drop client baggage from MCP params._meta to prevent identity spoofing

The MCP trace propagation added a W3CBaggagePropagator, so resolve_mcp_span_context
extracted the client's W3C Baggage from params._meta into the span's parent context.
The LiteLLMBaggageSpanProcessor then stamps allowlisted baggage keys onto the span,
and the list-tools/tool-call mappers don't set those identity keys, so nothing
overwrites them. A malicious MCP client could send
params._meta.baggage: litellm.team.id=...,litellm.metadata.user_api_key_user_id=...
and have those identity attributes attributed to its spans.

Extract trace context only (traceparent/tracestate) in the propagator, and stop
collecting the baggage key at the source in _mcp_meta_trace_carrier. Parenting to the
client's trace context, the actual goal, needs only trace context; remote baggage had
no legitimate consumer here. Regression tests at both layers assert a spoofed
params._meta.baggage never lands as a span identity attribute.

* style(mcp): clear ruff strict-budget breach in otel trace-carrier helpers

The otel MCP trace-carrier helpers added in this branch pushed the BLE001 and
UP006 strict-rule totals past their ceilings. Use PEP 585 `dict[str, str]` instead
of `Dict`, and narrow the optional-import guards to `except ImportError` (the only
failure these can hit, matching the "when otel_v2 is unavailable" intent) instead of
a blind `except Exception`.

* fix(otel): stamp authenticated identity baggage onto MCP spans

Parenting MCP spans to the client's params._meta trace context over an empty
Context() meant the tool-call and tools/list spans carried no team/key/metadata
identity at all, so they couldn't be attributed or filtered by team in a traces
backend. The LLM-call span already re-seeds identity from the parsed, authenticated
StandardLoggingPayload rather than trusting ambient/remote context; extract that into
a shared _seed_identity_baggage helper and run both MCP emitters through it.

Identity comes only from the authenticated payload, never the client carrier, so this
keeps the earlier spoofing fix intact while restoring attribution. Regression tests
assert the authenticated team lands on both MCP spans and that a spoofed
params._meta.baggage value can't override it.

* refactor(otel): model MCP spans as roots that link the transport in SPAN_REGISTRY
---
 litellm/integrations/otel/__init__.py         |   4 +
 litellm/integrations/otel/emitter.py          |  21 +-
 litellm/integrations/otel/logger.py           |  85 ++++++--
 litellm/integrations/otel/mappers/base.py     |   3 +-
 litellm/integrations/otel/mappers/genai.py    |  12 ++
 litellm/integrations/otel/model/payloads.py   |  38 ++++
 litellm/integrations/otel/model/spans.py      |  36 +++-
 litellm/integrations/otel/plumbing/context.py |  61 +++++-
 .../proxy/_experimental/mcp_server/server.py  |  56 +++++
 .../integrations/otel/test_otel_v2_logger.py  | 192 +++++++++++++++++-
 .../otel/test_otel_v2_sources_of_truth.py     |  19 +-
 .../mcp_server/test_mcp_server.py             |  51 +++++
 12 files changed, 547 insertions(+), 31 deletions(-)

diff --git a/litellm/integrations/otel/__init__.py b/litellm/integrations/otel/__init__.py
index da3ce4af3e7..7f78f7156b4 100644
--- a/litellm/integrations/otel/__init__.py
+++ b/litellm/integrations/otel/__init__.py
@@ -32,11 +32,13 @@ from litellm.integrations.otel.model.payloads import (
     LLMCallSpanData,
     LLMRequestParams,
     LLMUsage,
+    MCPListToolsSpanData,
     MCPToolCallSpanData,
     ProxyRequestSpanData,
     ServerInfo,
     ServiceSpanData,
     SpanError,
+    is_mcp_list_tools,
     is_mcp_tool_call,
 )
 from litellm.integrations.otel.model.semconv import (
@@ -106,6 +108,7 @@ __all__ = [
     "LLMCallSpanData",
     "LLMRequestParams",
     "LLMUsage",
+    "MCPListToolsSpanData",
     "MCPToolCallSpanData",
     "ProxyRequestSpanData",
     "RequestContext",
@@ -113,6 +116,7 @@ __all__ = [
     "ServerInfo",
     "ServiceSpanData",
     "SpanError",
+    "is_mcp_list_tools",
     "is_mcp_tool_call",
     "promoted_baggage",
 ]
diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py
index 69fc53c5b9d..8441cbae834 100644
--- a/litellm/integrations/otel/emitter.py
+++ b/litellm/integrations/otel/emitter.py
@@ -4,7 +4,7 @@ from collections import OrderedDict
 from typing import Callable, Sequence
 
 from opentelemetry.context import Context
-from opentelemetry.trace import Span, Tracer
+from opentelemetry.trace import Link, Span, Tracer
 from opentelemetry.trace.status import Status, StatusCode
 
 from litellm.integrations.otel.model.config import OpenTelemetryV2Config
@@ -13,6 +13,7 @@ from litellm.integrations.otel.mappers.base import AttributeMapper, SpanData
 from litellm.integrations.otel.model.payloads import (
     GuardrailSpanData,
     LLMCallSpanData,
+    MCPListToolsSpanData,
     MCPToolCallSpanData,
     ServiceSpanData,
 )
@@ -23,6 +24,7 @@ from litellm.integrations.otel.model.spans import (
     SpanRole,
     guardrail_span_name,
     llm_call_span_name,
+    mcp_list_tools_span_name,
     mcp_tool_call_span_name,
     service_span_name,
 )
@@ -33,6 +35,7 @@ from litellm.integrations.otel.model.spans import (
 _NAME_BUILDERS: dict[SpanRole, Callable[..., str]] = {
     SpanRole.LLM_CALL: llm_call_span_name,
     SpanRole.MCP_TOOL_CALL: mcp_tool_call_span_name,
+    SpanRole.MCP_LIST_TOOLS: mcp_list_tools_span_name,
     SpanRole.GUARDRAIL: guardrail_span_name,
     # DB_CALL and SERVICE are both built from ServiceSpanData; they differ only in
     # span kind (CLIENT vs INTERNAL) and attribute vocabulary, not in naming.
@@ -74,18 +77,21 @@ class SpanEmitter:
         start_time_ns: int | None = None,
         *,
         tracer: Tracer | None = None,
+        links: Sequence[Link] | None = None,
     ) -> Span:
         """Start a span for ``role`` without dedup or attribute mapping.
 
         For callers that own and manage their own span lifecycle. ``tracer``
         overrides the bound tracer for this span only, used for per-request
-        multi-tenant credential routing.
+        multi-tenant credential routing. ``links`` records related-but-not-parent
+        spans (e.g. the transport span of an MCP message, per MCP semconv).
         """
         return (tracer or self._tracer).start_span(
             name,
             context=parent_context,
             kind=to_otel_span_kind(SPAN_REGISTRY[role].kind),
             start_time=start_time_ns,
+            links=list(links) if links else None,
         )
 
     def _seen(self, dedup_key: str | None, role: SpanRole) -> bool:
@@ -116,16 +122,23 @@ class SpanEmitter:
         start_time_ns: int | None = None,
         end_time_ns: int | None = None,
         tracer: Tracer | None = None,
+        links: Sequence[Link] | None = None,
     ) -> Span | None:
         """Emit one complete span: dedup, start, map attributes, status, end.
 
         Return the span, or ``None`` if it was deduplicated away. ``tracer``
         overrides the bound tracer for this span, used for per-request routing.
+        ``links`` records related-but-not-parent spans (the transport span of an
+        MCP message).
         """
         # LLM-call and MCP tool-call spans carry a dedup key (their request's
         # call id), so a sync+async double-firing coalesces. ``isinstance`` narrows
         # the type for mypy and keeps the engine free of duck-typed attribute reads.
-        dedup_key = data.identity.call_id if isinstance(data, (LLMCallSpanData, MCPToolCallSpanData)) else None
+        dedup_key = (
+            data.identity.call_id
+            if isinstance(data, (LLMCallSpanData, MCPToolCallSpanData, MCPListToolsSpanData))
+            else None
+        )
         if self._seen(dedup_key, role):
             return None
         span = self.start_span(
@@ -134,6 +147,7 @@ class SpanEmitter:
             parent_context=parent_context,
             start_time_ns=start_time_ns,
             tracer=tracer,
+            links=links,
         )
         self.finish_span(role, span, data, end_time_ns=end_time_ns)
         return span
@@ -166,6 +180,7 @@ class SpanEmitter:
                 (
                     LLMCallSpanData,
                     MCPToolCallSpanData,
+                    MCPListToolsSpanData,
                     ServiceSpanData,
                     GuardrailSpanData,
                 ),
diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py
index 44484559948..5e729e12be0 100644
--- a/litellm/integrations/otel/logger.py
+++ b/litellm/integrations/otel/logger.py
@@ -5,7 +5,7 @@ from contextlib import contextmanager
 from datetime import datetime
 from typing import TYPE_CHECKING, Any, Callable, Iterator, Mapping, Sequence, cast
 
-from opentelemetry.context import attach, get_current
+from opentelemetry.context import Context, attach, get_current
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.trace import Span, Tracer, get_current_span, use_span
 
@@ -17,6 +17,7 @@ from litellm.integrations.otel.model.config import OpenTelemetryV2Config
 from litellm.integrations.otel.plumbing.context import (
     is_recordable_span,
     request_root_span,
+    resolve_mcp_span_context,
     resolve_parent_context,
     resolve_request_span_context,
     set_request_baggage,
@@ -32,9 +33,11 @@ from litellm.integrations.otel.model.metadata import (
 from litellm.integrations.otel.model.payloads import (
     GuardrailSpanData,
     LLMCallSpanData,
+    MCPListToolsSpanData,
     MCPToolCallSpanData,
     ServiceSpanData,
     SpanError,
+    is_mcp_list_tools,
     is_mcp_tool_call,
 )
 from litellm.integrations.otel.plumbing.metrics import (
@@ -218,6 +221,8 @@ class OpenTelemetryV2(CustomLogger):
     async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
         if self._emit_mcp_tool_call(kwargs, start_time, end_time):
             return
+        if self._emit_mcp_list_tools(kwargs, start_time, end_time):
+            return
         self._close_llm_call(kwargs, start_time, end_time)
         self._record_metrics(kwargs, response_obj, start_time, end_time)
 
@@ -242,8 +247,24 @@ class OpenTelemetryV2(CustomLogger):
     async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
         if self._emit_mcp_tool_call(kwargs, start_time, end_time):
             return
+        if self._emit_mcp_list_tools(kwargs, start_time, end_time):
+            return
         self._close_llm_call(kwargs, start_time, end_time)
 
+    def _seed_identity_baggage(self, identity: RequestIdentity, model: str | None, context: Context) -> Context:
+        """Seed authenticated request-identity Baggage onto ``context`` so the Baggage
+        processor stamps team/key/metadata onto the span. Identity is read from the
+        parsed payload, never the client's ``params._meta`` carrier, so it can't be
+        spoofed."""
+        bag = promoted_baggage(
+            identity,
+            model,
+            promoted_keys=tuple(self.config.baggage_promoted_keys),
+            metadata_keys=tuple(self.config.baggage_metadata_keys),
+            team_metadata_keys=tuple(self.config.baggage_team_metadata_keys),
+        )
+        return set_request_baggage(bag, context=context) if bag else context
+
     def _emit_mcp_tool_call(
         self,
         kwargs: Mapping[str, Any],
@@ -254,10 +275,12 @@ class OpenTelemetryV2(CustomLogger):
 
         MCP tool calls reach the success/failure callbacks like any other request
         (with ``call_type`` ``call_mcp_tool``), but they are not LLM calls and have
-        no ``pre_call`` carrier — so they get their own CLIENT span here, parented
-        to the request's server span. Returns whether it handled the event, so the
-        caller skips the LLM-call path. The whole span is emitted at once (there is
-        no boundary to open it at), deduped on the call id by the emitter.
+        no ``pre_call`` carrier — so they get their own CLIENT span here. Per the MCP
+        semconv it parents to the trace context the client propagated in
+        ``params._meta`` (or starts a new root) and links the transport span, rather
+        than nesting under the HTTP/session span. Returns whether it handled the
+        event, so the caller skips the LLM-call path. The whole span is emitted at
+        once (there is no boundary to open it at), deduped on the call id.
         """
         raw_payload = kwargs.get("standard_logging_object")
         if not raw_payload or not is_mcp_tool_call(cast(Mapping[str, object], raw_payload)):
@@ -271,12 +294,51 @@ class OpenTelemetryV2(CustomLogger):
         # as a phantom LLM span.
         if data.identity.call_id:
             self._open_llm_calls.pop(data.identity.call_id, None)
+        parent_context, links = resolve_mcp_span_context()
+        parent_context = self._seed_identity_baggage(data.identity, None, parent_context)
         self._emitter.emit(
             SpanRole.MCP_TOOL_CALL,
             data,
-            parent_context=resolve_request_span_context(),
+            parent_context=parent_context,
             start_time_ns=to_ns(start_time),
             end_time_ns=to_ns(end_time),
+            links=links,
+        )
+        return True
+
+    def _emit_mcp_list_tools(
+        self,
+        kwargs: Mapping[str, object],
+        start_time: datetime | float | None,
+        end_time: datetime | float | None,
+    ) -> bool:
+        """Emit an MCP ``tools/list`` span when the closed request was a discovery call.
+
+        Like a tool call, listing reaches the success/failure callbacks (here with
+        ``call_type`` ``list_mcp_tools``) with no ``pre_call`` carrier, so it gets its
+        own CLIENT span. Per the MCP semconv it parents to the ``params._meta`` trace
+        context (or starts a new root) and links the transport span, rather than
+        nesting under the HTTP/session span. Returns whether it handled the event so
+        the caller skips the LLM-call path.
+        """
+        raw_payload = kwargs.get("standard_logging_object")
+        if not raw_payload or not is_mcp_list_tools(cast(Mapping[str, object], raw_payload)):
+            return False
+        payload = cast("StandardLoggingPayload", raw_payload)
+        data = MCPListToolsSpanData.from_standard_logging_payload(
+            payload, capture_content=self.config.capture_span_content
+        )
+        if data.identity.call_id:
+            self._open_llm_calls.pop(data.identity.call_id, None)
+        parent_context, links = resolve_mcp_span_context()
+        parent_context = self._seed_identity_baggage(data.identity, None, parent_context)
+        self._emitter.emit(
+            SpanRole.MCP_LIST_TOOLS,
+            data,
+            parent_context=parent_context,
+            start_time_ns=to_ns(start_time),
+            end_time_ns=to_ns(end_time),
+            links=links,
         )
         return True
 
@@ -319,16 +381,7 @@ class OpenTelemetryV2(CustomLogger):
         # root span — parent to it (ambient fallback on the SDK path). Seed identity
         # Baggage so the span — and the SDK path, which has none — is labeled
         # consistently.
-        parent_ctx = resolve_request_span_context()
-        bag = promoted_baggage(
-            data.identity,
-            data.request_model,
-            promoted_keys=tuple(self.config.baggage_promoted_keys),
-            metadata_keys=tuple(self.config.baggage_metadata_keys),
-            team_metadata_keys=tuple(self.config.baggage_team_metadata_keys),
-        )
-        if bag:
-            parent_ctx = set_request_baggage(bag, context=parent_ctx)
+        parent_ctx = self._seed_identity_baggage(data.identity, data.request_model, resolve_request_span_context())
         return self._emitter.emit(
             SpanRole.LLM_CALL,
             data,
diff --git a/litellm/integrations/otel/mappers/base.py b/litellm/integrations/otel/mappers/base.py
index 6685e34578b..809d956a9c7 100644
--- a/litellm/integrations/otel/mappers/base.py
+++ b/litellm/integrations/otel/mappers/base.py
@@ -7,6 +7,7 @@ from typing_extensions import Protocol, runtime_checkable
 from litellm.integrations.otel.model.payloads import (
     GuardrailSpanData,
     LLMCallSpanData,
+    MCPListToolsSpanData,
     MCPToolCallSpanData,
     ServiceSpanData,
 )
@@ -20,7 +21,7 @@ AttributeMap = dict[str, AttrValue]
 # The closed set of span-data types the engine routes through the mapper chain.
 # Server spans (PROXY_REQUEST + management routes) belong to the mounted FastAPI
 # instrumentor, not the mapper chain.
-SpanData = LLMCallSpanData | MCPToolCallSpanData | GuardrailSpanData | ServiceSpanData
+SpanData = LLMCallSpanData | MCPToolCallSpanData | MCPListToolsSpanData | GuardrailSpanData | ServiceSpanData
 
 
 @runtime_checkable
diff --git a/litellm/integrations/otel/mappers/genai.py b/litellm/integrations/otel/mappers/genai.py
index ad6d3e7ff21..c5d8c35de7d 100644
--- a/litellm/integrations/otel/mappers/genai.py
+++ b/litellm/integrations/otel/mappers/genai.py
@@ -19,6 +19,7 @@ from litellm.integrations.otel.mappers.utils import (
 from litellm.integrations.otel.model.payloads import (
     GuardrailSpanData,
     LLMCallSpanData,
+    MCPListToolsSpanData,
     MCPToolCallSpanData,
     ServiceSpanData,
     ToolDefinition,
@@ -100,6 +101,15 @@ class GenAIMapper:
         f"{LiteLLM.COST_PREFIX}total": lambda d: d.response_cost,
     }
 
+    # A tools/list discovery span: the method and session only. Per semconv it must
+    # NOT carry gen_ai.operation.name (execute_tool) or gen_ai.tool.name — those are
+    # for tool calls, and listing executes no tool.
+    _MCP_LIST_ATTRS: dict[str, Callable[[MCPListToolsSpanData], AttrValue | None]] = {
+        MCP.METHOD_NAME: lambda d: d.method,
+        MCP.SESSION_ID: lambda d: d.session_id,
+        LiteLLM.CALL_ID: lambda d: d.identity.call_id or None,
+    }
+
     _GUARDRAIL_ATTRS: dict[str, Callable[[GuardrailSpanData], AttrValue | None]] = {
         LiteLLM.GUARDRAIL_NAME: lambda d: d.guardrail_name,
         LiteLLM.GUARDRAIL_MODE: lambda d: d.mode,
@@ -130,6 +140,8 @@ class GenAIMapper:
                 return self._llm_call(data)
             case MCPToolCallSpanData():
                 return collect(self._MCP_ATTRS, data)
+            case MCPListToolsSpanData():
+                return collect(self._MCP_LIST_ATTRS, data)
             case GuardrailSpanData():
                 return self._guardrail(data)
             case ServiceSpanData():
diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py
index a368a862024..b0dcf97b787 100644
--- a/litellm/integrations/otel/model/payloads.py
+++ b/litellm/integrations/otel/model/payloads.py
@@ -37,12 +37,14 @@ __all__ = [
     "LLMCost",
     "LLMRequestParams",
     "LLMUsage",
+    "MCPListToolsSpanData",
     "MCPToolCallSpanData",
     "ProxyRequestSpanData",
     "ServerInfo",
     "ServiceSpanData",
     "SpanError",
     "ToolDefinition",
+    "is_mcp_list_tools",
     "is_mcp_tool_call",
 ]
 
@@ -415,6 +417,42 @@ def is_mcp_tool_call(payload: Mapping[str, object]) -> bool:
     return bool(_mcp_tool_call_metadata(payload)) or (payload.get("call_type") == "call_mcp_tool")
 
 
+@dataclass(frozen=True)
+class MCPListToolsSpanData:
+    """One MCP ``tools/list`` discovery call, parsed from a closed request's payload.
+
+    The proxy is an MCP *client* enumerating an upstream server's tools, so this is
+    a CLIENT span. It carries neither ``gen_ai.operation.name`` nor ``gen_ai.tool.name``:
+    the GenAI semconv sets ``execute_tool`` (and the tool name) only for tool *calls*,
+    and listing executes no tool.
+    """
+
+    method: str
+    session_id: str | None
+    error: SpanError | None
+    identity: RequestIdentity
+
+    @classmethod
+    def from_standard_logging_payload(
+        cls, payload: StandardLoggingPayload, capture_content: bool = False
+    ) -> MCPListToolsSpanData:
+        # The list-tools logging path does not thread an MCP session id into the
+        # payload (only the tool-call path stamps ``mcp_tool_call_metadata``), so
+        # there is none to read here; ``mcp.session.id`` is simply omitted.
+        return cls(
+            method=MCPMethod.TOOLS_LIST.value,
+            session_id=None,
+            error=_parse_error(payload),
+            identity=RequestContext.from_standard_logging_payload(payload).identity,
+        )
+
+
+def is_mcp_list_tools(payload: Mapping[str, object]) -> bool:
+    """Whether a closed request's payload is an MCP ``tools/list`` discovery call
+    rather than a tool call or an LLM call — true when the call type says so."""
+    return payload.get("call_type") == "list_mcp_tools"
+
+
 # --- service event_metadata sanitization ------------------------------------ #
 
 # Substrings (case-insensitive) of keys that must never reach a span: secrets,
diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py
index bc624cf6a57..c93f95ec97d 100644
--- a/litellm/integrations/otel/model/spans.py
+++ b/litellm/integrations/otel/model/spans.py
@@ -18,6 +18,13 @@ before the LLM call even starts), so a guardrail is a sibling of the LLM call,
 not a child of it. The emitter parents every span to the ambient OTel context
 (the active server span), which matches this.
 
+MCP spans (``MCP_TOOL_CALL``, ``MCP_LIST_TOOLS``) are intentionally NOT in this
+tree. Per the OTel GenAI MCP semconv, MCP and the HTTP transport are independent
+contexts, so an MCP span parents to the trace context the client propagated in
+``params._meta`` (or starts its own root when none is propagated) and records the
+``PROXY_REQUEST`` transport span as a span *link*, never a parent. The registry
+encodes this as ``parent=None, links=PROXY_REQUEST``.
+
 Not every service call becomes a span — :func:`span_role_for_service` decides:
 
 - ``DB_CALL`` (CLIENT) — outbound datastores (redis, postgres,
@@ -46,6 +53,7 @@ if TYPE_CHECKING:
     from litellm.integrations.otel.model.payloads import (
         GuardrailSpanData,
         LLMCallSpanData,
+        MCPListToolsSpanData,
         MCPToolCallSpanData,
         ProxyRequestSpanData,
         ServiceSpanData,
@@ -56,6 +64,7 @@ class SpanRole(str, Enum):
     PROXY_REQUEST = "proxy_request"
     LLM_CALL = "llm_call"
     MCP_TOOL_CALL = "mcp_tool_call"
+    MCP_LIST_TOOLS = "mcp_list_tools"
     GUARDRAIL = "guardrail"
     DB_CALL = "db_call"
     SERVICE = "service"
@@ -74,14 +83,24 @@ class SpanSpec:
     role: SpanRole
     kind: LiteLLMSpanKind
     parent: SpanRole | None
+    links: SpanRole | None = None
 
 
 SPAN_REGISTRY: dict[SpanRole, SpanSpec] = {
     SpanRole.PROXY_REQUEST: SpanSpec(SpanRole.PROXY_REQUEST, LiteLLMSpanKind.SERVER, parent=None),
     SpanRole.LLM_CALL: SpanSpec(SpanRole.LLM_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
-    # The proxy is an MCP client to the upstream server it dispatches the tool
-    # call to, so this is a CLIENT span, sibling of the LLM call under the request.
-    SpanRole.MCP_TOOL_CALL: SpanSpec(SpanRole.MCP_TOOL_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
+    # MCP and the HTTP transport are independent contexts (OTel GenAI MCP semconv),
+    # so an MCP span does not nest under the transport span. The proxy is an MCP
+    # client to the upstream server, so it's a CLIENT span; it parents to the trace
+    # context the client propagated in ``params._meta`` (or starts its own root when
+    # none is propagated) and records the PROXY_REQUEST transport span as a span
+    # *link*, never a parent — hence ``parent=None, links=PROXY_REQUEST``.
+    SpanRole.MCP_TOOL_CALL: SpanSpec(
+        SpanRole.MCP_TOOL_CALL, LiteLLMSpanKind.CLIENT, parent=None, links=SpanRole.PROXY_REQUEST
+    ),
+    SpanRole.MCP_LIST_TOOLS: SpanSpec(
+        SpanRole.MCP_LIST_TOOLS, LiteLLMSpanKind.CLIENT, parent=None, links=SpanRole.PROXY_REQUEST
+    ),
     SpanRole.GUARDRAIL: SpanSpec(SpanRole.GUARDRAIL, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST),
     SpanRole.DB_CALL: SpanSpec(SpanRole.DB_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
     SpanRole.SERVICE: SpanSpec(SpanRole.SERVICE, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST),
@@ -163,6 +182,12 @@ def mcp_tool_call_span_name(data: "MCPToolCallSpanData") -> str:
     return f"{data.method} {data.tool_name}".strip()
 
 
+def mcp_list_tools_span_name(data: "MCPListToolsSpanData") -> str:
+    """``"{mcp.method.name}"`` i.e. ``"tools/list"`` — no low-cardinality target, so
+    the method name alone names the span (MCP semconv)."""
+    return data.method
+
+
 def proxy_request_span_name(data: "ProxyRequestSpanData") -> str:
     """``"{method} {route}"`` (HTTP semconv)."""
     return f"{data.http_method} {data.route}".strip()
@@ -179,7 +204,8 @@ def service_span_name(data: "ServiceSpanData") -> str:
 
 
 def root_roles() -> list[SpanRole]:
-    """Roles that start a new trace (no in-process parent)."""
+    """Roles with no in-process parent. They start a new trace unless they adopt a
+    remote parent (e.g. an MCP span joining the client's propagated context)."""
     return [role for role, spec in SPAN_REGISTRY.items() if spec.parent is None]
 
 
@@ -196,6 +222,8 @@ def validate_registry(
             raise ValueError(f"SPAN_REGISTRY[{role}] has mismatched role {spec.role}")
         if spec.parent is not None and spec.parent not in reg:
             raise ValueError(f"span role {role} declares unknown parent {spec.parent}")
+        if spec.links is not None and spec.links not in reg:
+            raise ValueError(f"span role {role} declares unknown link target {spec.links}")
     missing = [role for role in SpanRole if role not in reg]
     if missing:
         raise ValueError(f"SPAN_REGISTRY is missing roles: {missing}")
diff --git a/litellm/integrations/otel/plumbing/context.py b/litellm/integrations/otel/plumbing/context.py
index ff513c84d95..8acac112c3d 100644
--- a/litellm/integrations/otel/plumbing/context.py
+++ b/litellm/integrations/otel/plumbing/context.py
@@ -1,11 +1,11 @@
 """Trace-context + Baggage helpers."""
 
-from contextvars import ContextVar
+from contextvars import ContextVar, Token
 from typing import Mapping
 
 from opentelemetry import baggage
 from opentelemetry.context import Context, get_current
-from opentelemetry.trace import Span, get_current_span, set_span_in_context
+from opentelemetry.trace import Link, Span, get_current_span, set_span_in_context
 from opentelemetry.trace.propagation.tracecontext import (
     TraceContextTextMapPropagator,
 )
@@ -47,6 +47,31 @@ def request_root_span() -> "Span | None":
     return span if is_recordable_span(span) else None
 
 
+# The W3C trace-context carrier (``traceparent``/``tracestate``/``baggage``) the
+# MCP client propagated in the current request's ``params._meta``. The MCP gateway
+# sets it per message so the MCP span can parent to the client's span rather than
+# to the transport. A ``ContextVar`` because, like the root-span anchor, it must
+# ride the request task and be readable by the inline success-logging callback.
+_mcp_message_trace_carrier: "ContextVar[Mapping[str, str] | None]" = ContextVar(
+    "litellm_otel_mcp_message_trace_carrier", default=None
+)
+
+
+def set_mcp_message_trace_carrier(
+    carrier: "Mapping[str, str] | None",
+) -> "Token[Mapping[str, str] | None]":
+    """Stash the current MCP message's propagated trace-context carrier.
+
+    Returns the reset token; the caller must reset it once the message is handled
+    so the carrier never leaks to the next message on the same session task.
+    """
+    return _mcp_message_trace_carrier.set(carrier)
+
+
+def reset_mcp_message_trace_carrier(token: "Token[Mapping[str, str] | None]") -> None:
+    _mcp_message_trace_carrier.reset(token)
+
+
 def set_request_baggage(values: Mapping[str, str], context: Context | None = None) -> Context:
     """Return a context with ``values`` written into Baggage."""
     ctx = context
@@ -104,6 +129,38 @@ def resolve_request_span_context() -> Context:
     return get_current()
 
 
+def resolve_mcp_span_context(
+    carrier: "Mapping[str, str] | None" = None,
+) -> "tuple[Context, tuple[Link, ...]]":
+    """Parent context + links for an MCP message span, per the OTel GenAI MCP semconv.
+
+    MCP and the underlying transport (HTTP) are independent lifecycles — one
+    streamable-HTTP session multiplexes many messages, so nesting the message span
+    under the HTTP/session span is wrong (it renders the message at the session's
+    start, skewed by however long the session has been open). Instead:
+
+    * parent to the trace context the client propagated in the request's
+      ``params._meta`` (a *remote* parent), and
+    * record the transport/session span as a *link*, never the parent.
+
+    Only trace context (``traceparent``/``tracestate``) is extracted, never the
+    client's W3C Baggage: ``params._meta`` is caller-controlled, and the otel
+    baggage processor stamps allowlisted baggage keys (``litellm.team.id``,
+    ``litellm.metadata.*``, ...) onto the span as attributes, so honoring remote
+    baggage would let a client spoof a span's identity attribution.
+
+    With no propagated context the returned context carries no span, so the span
+    starts its own root trace (still linked to the transport). The base context is
+    explicitly empty so an absent ``traceparent`` can never fall through to the
+    ambient (stale session) span.
+    """
+    source = carrier if carrier is not None else _mcp_message_trace_carrier.get()
+    parent = _PROPAGATOR.extract(dict(source or {}), context=Context())
+    transport = request_root_span()
+    links = (Link(transport.get_span_context()),) if transport is not None else ()
+    return parent, links
+
+
 def is_recordable_span(obj: object) -> bool:
     """True if ``obj`` is a live span with a valid context (safe to parent under)."""
     if not isinstance(obj, Span):
diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py
index 4b55510a629..158fdda6c39 100644
--- a/litellm/proxy/_experimental/mcp_server/server.py
+++ b/litellm/proxy/_experimental/mcp_server/server.py
@@ -229,6 +229,56 @@ def _jsonrpc_text_has_top_level_method(text: str) -> bool:
     return False
 
 
+def _mcp_meta_trace_carrier(req_ctx: object) -> Optional[dict[str, str]]:
+    """The W3C trace context (``traceparent``/``tracestate``) the MCP client
+    propagated in the request's ``params._meta`` (SEP-414), or ``None``.
+
+    Per the OTel MCP semconv the MCP span parents to this propagated context rather
+    than to the HTTP/session transport (which is recorded as a link instead), so a
+    streamable-HTTP session that multiplexes many messages does not glue every
+    message under the session's first request. The client's W3C Baggage is
+    deliberately excluded: it is caller-controlled, and the otel baggage processor
+    stamps allowlisted baggage keys (``litellm.team.id``, ``litellm.metadata.*``,
+    ...) onto the span, so honoring remote baggage would let a client spoof a
+    span's identity attribution.
+    """
+    meta = getattr(req_ctx, "meta", None)
+    extra = getattr(meta, "model_extra", None)
+    if not isinstance(extra, dict):
+        return None
+    carrier = {key: extra[key] for key in ("traceparent", "tracestate") if isinstance(extra.get(key), str)}
+    return carrier or None
+
+
+def _otel_set_mcp_trace_carrier(carrier: Optional[dict[str, str]]) -> object:
+    """Stash ``carrier`` for the otel_v2 MCP span and return a reset token, or
+    ``None`` when otel_v2 is unavailable. Lazily imported so opentelemetry stays an
+    optional dependency."""
+    try:
+        from litellm.integrations.otel.plumbing.context import (
+            set_mcp_message_trace_carrier,
+        )
+
+        return set_mcp_message_trace_carrier(carrier)
+    except ImportError:
+        return None
+
+
+def _otel_reset_mcp_trace_carrier(token: object) -> None:
+    """Clear the per-message trace carrier so it never leaks to the next message on
+    the same session task. Paired with ``_otel_set_mcp_trace_carrier``."""
+    if token is None:
+        return
+    try:
+        from litellm.integrations.otel.plumbing.context import (
+            reset_mcp_message_trace_carrier,
+        )
+
+        reset_mcp_message_trace_carrier(token)
+    except ImportError:
+        return
+
+
 def _proxy_exception_to_http_exception(exc: ProxyException) -> HTTPException:
     """Map a ``ProxyException`` to an ``HTTPException`` that preserves its real
     status code and headers.
@@ -595,8 +645,10 @@ if MCP_AVAILABLE:
         _session_reset_token = None
         if req_ctx:
             _session_reset_token = active_mcp_session_var.set(req_ctx.session)
+        _trace_token = None
 
         try:
+            _trace_token = _otel_set_mcp_trace_carrier(_mcp_meta_trace_carrier(req_ctx))
             # Get user authentication from context variable
             (
                 user_api_key_auth,
@@ -632,6 +684,7 @@ if MCP_AVAILABLE:
             # This prevents the HTTP stream from failing and allows the client to get a response
             return []
         finally:
+            _otel_reset_mcp_trace_carrier(_trace_token)
             if _session_reset_token is not None:
                 active_mcp_session_var.reset(_session_reset_token)
 
@@ -658,8 +711,10 @@ if MCP_AVAILABLE:
         _session_reset_token = None
         if req_ctx:
             _session_reset_token = active_mcp_session_var.set(req_ctx.session)
+        _trace_token = None
 
         try:
+            _trace_token = _otel_set_mcp_trace_carrier(_mcp_meta_trace_carrier(req_ctx))
             # Validate arguments
             (
                 user_api_key_auth,
@@ -778,6 +833,7 @@ if MCP_AVAILABLE:
 
             return response
         finally:
+            _otel_reset_mcp_trace_carrier(_trace_token)
             if _session_reset_token is not None:
                 active_mcp_session_var.reset(_session_reset_token)
 
diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py
index 0ceb7efbe0b..674b2bec829 100644
--- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py
+++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py
@@ -27,9 +27,11 @@ from litellm.integrations.otel import (  # noqa: E402
     OpenTelemetryV2Config,
 )
 from litellm.integrations.otel.plumbing import providers  # noqa: E402
-from litellm.integrations.otel.plumbing.context import (
+from litellm.integrations.otel.plumbing.context import (  # noqa: E402
+    reset_mcp_message_trace_carrier,
+    set_mcp_message_trace_carrier,
     set_request_root_span,
-)  # noqa: E402
+)
 from litellm.integrations.otel.logger import OpenTelemetryV2  # noqa: E402
 from litellm.integrations.otel.model.spans import (  # noqa: E402
     LITELLM_PROXY_REQUEST_SPAN_NAME,
@@ -53,8 +55,10 @@ def _reset_request_root_span():
     from litellm.integrations.otel.plumbing import context as _otel_context
 
     _otel_context._request_root_span.set(None)
+    _otel_context._mcp_message_trace_carrier.set(None)
     yield
     _otel_context._request_root_span.set(None)
+    _otel_context._mcp_message_trace_carrier.set(None)
 
 
 def _payload(**overrides):
@@ -387,6 +391,190 @@ def test_mcp_tool_call_metadata_read_from_nested_metadata_not_top_level():
     assert LiteLLM.MCP_SERVER_NAME not in span.attributes
 
 
+def _mcp_list_payload(**overrides):
+    payload = {
+        "call_type": "list_mcp_tools",
+        "status": "success",
+        "litellm_call_id": "mcp_list_1",
+        "metadata": {
+            "user_api_key_team_id": "t1",
+            "spend_logs_metadata": {"mcp_operation": "list_tools"},
+        },
+        "hidden_params": {},
+    }
+    payload.update(overrides)
+    return payload
+
+
+def test_mcp_list_tools_emits_client_span():
+    """An MCP ``tools/list`` discovery call becomes a CLIENT span named ``tools/list``,
+    carrying only the MCP method and the call id. Per the GenAI MCP semconv the list
+    span omits ``gen_ai.operation.name`` and ``gen_ai.tool.name`` (tool-call-only) and
+    ``mcp.session.id`` (the list path threads no session id), so a naive reuse of the
+    tool-call mapper would wrongly stamp them, and the pre-fix code emitted no span at
+    all for a ``list_mcp_tools`` payload."""
+    logger, exporter = _logger()
+    kwargs = {"standard_logging_object": _mcp_list_payload()}
+    asyncio.run(logger.async_log_success_event(kwargs, None, None, None))
+    (span,) = exporter.get_finished_spans()
+    assert span.name == "tools/list"
+    assert span.kind is SpanKind.CLIENT
+    assert span.attributes["mcp.method.name"] == "tools/list"
+    assert span.attributes[LiteLLM.CALL_ID] == "mcp_list_1"
+    assert span.status.status_code is StatusCode.UNSET
+    # Bug-killers: no span pre-fix (empty exporter -> the unpack above raises), and a
+    # tool-call-shaped fix would leak execute_tool / tool name / session id here.
+    assert GenAI.OPERATION_NAME not in span.attributes
+    assert "gen_ai.tool.name" not in span.attributes
+    assert "mcp.session.id" not in span.attributes
+
+
+_MCP_SPAN_CASES = [
+    (_mcp_payload, "tools/call get_weather"),
+    (_mcp_list_payload, "tools/list"),
+]
+
+
+@pytest.mark.parametrize("make_payload, span_name", _MCP_SPAN_CASES)
+def test_mcp_span_roots_and_links_transport_without_propagated_context(
+    make_payload, span_name
+):
+    """MCP and the HTTP transport are independent lifecycles (one streamable-HTTP
+    session multiplexes many messages), so per the MCP semconv the message span
+    must NOT nest under the session/transport span — that is what made it render
+    skewed at the session's start. With no propagated ``params._meta`` context it
+    starts its own root trace and records the transport span as a *link*, never
+    the parent."""
+    logger, exporter = _logger()
+    transport = logger._emitter.start_span(
+        SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
+    )
+    set_request_root_span(transport)
+    asyncio.run(
+        logger.async_log_success_event(
+            {"standard_logging_object": make_payload()}, None, None, None
+        )
+    )
+    transport.end()
+    span = next(s for s in exporter.get_finished_spans() if s.name == span_name)
+    assert span.parent is None
+    assert span.context.trace_id != transport.get_span_context().trace_id
+    assert [link.context.span_id for link in span.links] == [
+        transport.get_span_context().span_id
+    ]
+
+
+@pytest.mark.parametrize("make_payload, span_name", _MCP_SPAN_CASES)
+def test_mcp_span_parents_to_propagated_meta_trace_context(make_payload, span_name):
+    """When the client propagates W3C trace context in the request's
+    ``params._meta`` (SEP-414), the MCP span parents to it (one distributed trace)
+    and still links the transport span — never falling through to the
+    ambient/session span."""
+    logger, exporter = _logger()
+    transport = logger._emitter.start_span(
+        SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
+    )
+    set_request_root_span(transport)
+    token = set_mcp_message_trace_carrier(
+        {"traceparent": "00-11111111111111111111111111111111-2222222222222222-01"}
+    )
+    try:
+        asyncio.run(
+            logger.async_log_success_event(
+                {"standard_logging_object": make_payload()}, None, None, None
+            )
+        )
+    finally:
+        reset_mcp_message_trace_carrier(token)
+    transport.end()
+    span = next(s for s in exporter.get_finished_spans() if s.name == span_name)
+    assert span.context.trace_id == 0x11111111111111111111111111111111
+    assert span.parent is not None
+    assert span.parent.span_id == 0x2222222222222222
+    assert [link.context.span_id for link in span.links] == [
+        transport.get_span_context().span_id
+    ]
+
+
+@pytest.mark.parametrize("make_payload, span_name", _MCP_SPAN_CASES)
+def test_mcp_span_ignores_client_supplied_baggage(make_payload, span_name):
+    """The MCP span must NOT honor W3C Baggage from the client's ``params._meta``.
+
+    ``params._meta`` is caller-controlled and the baggage processor stamps
+    allowlisted baggage keys onto every span, so extracting remote baggage would
+    let a client spoof a span's identity (e.g. ``litellm.team.id``). The propagator
+    extracts trace context only, so the spoofed keys never reach the span while the
+    legitimate traceparent parenting still works."""
+    logger, exporter = _logger()
+    transport = logger._emitter.start_span(
+        SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
+    )
+    set_request_root_span(transport)
+    token = set_mcp_message_trace_carrier(
+        {
+            "traceparent": "00-11111111111111111111111111111111-2222222222222222-01",
+            "baggage": "litellm.team.id=spoofed-team,litellm.metadata.user_api_key_user_id=attacker",
+        }
+    )
+    try:
+        asyncio.run(
+            logger.async_log_success_event(
+                {"standard_logging_object": make_payload()}, None, None, None
+            )
+        )
+    finally:
+        reset_mcp_message_trace_carrier(token)
+    transport.end()
+    span = next(s for s in exporter.get_finished_spans() if s.name == span_name)
+    # Trace context still honored: proves the carrier was processed, not dropped wholesale.
+    assert span.parent is not None and span.parent.span_id == 0x2222222222222222
+    # Identity is the authenticated payload's team, never the client's spoofed value.
+    assert span.attributes[LiteLLM.TEAM_ID] == "t1"
+    assert "litellm.metadata.user_api_key_user_id" not in span.attributes
+
+
+@pytest.mark.parametrize("make_payload, span_name", _MCP_SPAN_CASES)
+def test_mcp_span_carries_authenticated_identity(make_payload, span_name):
+    """An MCP span is labeled with the authenticated request's identity (team/key),
+    seeded from the parsed payload like the LLM-call span. Without this seeding the
+    span — parented to an empty remote context — would carry no team/key attribute at
+    all, so it couldn't be attributed or filtered by team in the traces backend."""
+    logger, exporter = _logger()
+    asyncio.run(
+        logger.async_log_success_event(
+            {"standard_logging_object": make_payload()}, None, None, None
+        )
+    )
+    span = next(s for s in exporter.get_finished_spans() if s.name == span_name)
+    assert span.attributes[LiteLLM.TEAM_ID] == "t1"
+
+
+def test_mcp_span_malformed_traceparent_starts_root():
+    """A malformed traceparent in ``params._meta`` must not crash or parent to a
+    bogus span: the propagator ignores it, so the span starts its own root trace and
+    still links the transport span."""
+    logger, exporter = _logger()
+    transport = logger._emitter.start_span(
+        SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
+    )
+    set_request_root_span(transport)
+    token = set_mcp_message_trace_carrier({"traceparent": "not-a-valid-traceparent"})
+    try:
+        asyncio.run(
+            logger.async_log_success_event(
+                {"standard_logging_object": _mcp_list_payload()}, None, None, None
+            )
+        )
+    finally:
+        reset_mcp_message_trace_carrier(token)
+    transport.end()
+    span = next(s for s in exporter.get_finished_spans() if s.name == "tools/list")
+    assert span.parent is None
+    assert [link.context.span_id for link in span.links] == [
+        transport.get_span_context().span_id
+    ]
+
+
 def test_pre_call_idempotent_keeps_first_span():
     """A retried call may re-enter ``pre_call`` with the same call id; the first
     span (with the true start time) is kept, not replaced."""
diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py
index 4bb26a70b02..834a484090f 100644
--- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py
+++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py
@@ -94,19 +94,32 @@ def test_registry_parent_integrity_no_orphans():
 
 
 def test_registry_hierarchy_shape():
-    assert set(root_roles()) == {SpanRole.PROXY_REQUEST}
+    # MCP roles have no in-process parent: per the MCP semconv they root (or adopt
+    # the client's propagated _meta context), so they sit alongside PROXY_REQUEST.
+    assert set(root_roles()) == {
+        SpanRole.PROXY_REQUEST,
+        SpanRole.MCP_TOOL_CALL,
+        SpanRole.MCP_LIST_TOOLS,
+    }
     # Guardrails parent to the request span, not the LLM call: a pre-call
     # guardrail runs before the LLM call exists, so it's a sibling of it.
     assert set(child_roles(SpanRole.PROXY_REQUEST)) == {
         SpanRole.LLM_CALL,
-        SpanRole.MCP_TOOL_CALL,
         SpanRole.GUARDRAIL,
         SpanRole.DB_CALL,
         SpanRole.SERVICE,
     }
     assert SPAN_REGISTRY[SpanRole.LLM_CALL].kind is LiteLLMSpanKind.CLIENT
-    # The proxy is an MCP client to the upstream tool server: CLIENT span.
+    # The proxy is an MCP client to the upstream tool server: CLIENT span. Listing
+    # tools is the same client relationship, so it's a CLIENT span too.
     assert SPAN_REGISTRY[SpanRole.MCP_TOOL_CALL].kind is LiteLLMSpanKind.CLIENT
+    assert SPAN_REGISTRY[SpanRole.MCP_LIST_TOOLS].kind is LiteLLMSpanKind.CLIENT
+    # MCP spans don't nest under the transport: they link the PROXY_REQUEST span
+    # instead of parenting to it (OTel GenAI MCP semconv).
+    assert SPAN_REGISTRY[SpanRole.MCP_TOOL_CALL].parent is None
+    assert SPAN_REGISTRY[SpanRole.MCP_LIST_TOOLS].parent is None
+    assert SPAN_REGISTRY[SpanRole.MCP_TOOL_CALL].links is SpanRole.PROXY_REQUEST
+    assert SPAN_REGISTRY[SpanRole.MCP_LIST_TOOLS].links is SpanRole.PROXY_REQUEST
     assert SPAN_REGISTRY[SpanRole.PROXY_REQUEST].kind is LiteLLMSpanKind.SERVER
     assert SPAN_REGISTRY[SpanRole.GUARDRAIL].parent is SpanRole.PROXY_REQUEST
     # An outbound datastore call is a CLIENT span; an internal service is INTERNAL.
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py
index 34e932b6ae7..abefb2fd984 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py
@@ -6447,3 +6447,54 @@ class TestStreamableHttpAuthErrorMapping:
             m.get("type") == "http.response.start" and m.get("status") == 500
             for m in sent
         )
+
+
+class TestMCPMetaTraceCarrier:
+    """`_mcp_meta_trace_carrier` extracts the W3C trace context the MCP client
+    propagated in the request's params._meta (SEP-414) so the otel_v2 MCP span can
+    parent to the client's span. Exercises the real MCP SDK `RequestParams.Meta`
+    shape (extra='allow' preserves the unprefixed keys), not just an injected
+    carrier."""
+
+    def test_extracts_trace_context_and_excludes_baggage_and_other_meta(self):
+        """Only traceparent/tracestate are carried. The client's W3C ``baggage`` is
+        deliberately dropped even though it rides in params._meta: it is
+        caller-controlled, and the otel baggage processor stamps allowlisted baggage
+        keys onto the span, so honoring it would let a client spoof a span's identity
+        (e.g. ``litellm.team.id``). Dropping it at the source is the regression guard."""
+        from types import SimpleNamespace
+
+        from mcp.types import RequestParams
+
+        from litellm.proxy._experimental.mcp_server.server import (
+            _mcp_meta_trace_carrier,
+        )
+
+        meta = RequestParams.Meta.model_validate(
+            {
+                "traceparent": "00-11111111111111111111111111111111-2222222222222222-01",
+                "tracestate": "rojo=1",
+                "baggage": "litellm.team.id=spoofed-team,litellm.metadata.user_api_key_user_id=attacker",
+                "progressToken": "p1",
+            }
+        )
+        carrier = _mcp_meta_trace_carrier(SimpleNamespace(meta=meta))
+        assert carrier == {
+            "traceparent": "00-11111111111111111111111111111111-2222222222222222-01",
+            "tracestate": "rojo=1",
+        }
+        assert "baggage" not in carrier
+
+    def test_none_when_no_trace_context(self):
+        from types import SimpleNamespace
+
+        from mcp.types import RequestParams
+
+        from litellm.proxy._experimental.mcp_server.server import (
+            _mcp_meta_trace_carrier,
+        )
+
+        assert _mcp_meta_trace_carrier(None) is None
+        assert _mcp_meta_trace_carrier(SimpleNamespace(meta=None)) is None
+        only_progress = RequestParams.Meta.model_validate({"progressToken": "p1"})
+        assert _mcp_meta_trace_carrier(SimpleNamespace(meta=only_progress)) is None

From 1eb712246579bcf27734099155a3145a6aad6e3b Mon Sep 17 00:00:00 2001
From: Yassin Kortam <yassin@berri.ai>
Date: Tue, 30 Jun 2026 20:27:12 +0300
Subject: [PATCH 11/51] test(benchmarks): add CodSpeed benchmarks for
 inference, MCP and A2A hot paths (#31716)

Guard the per-request CPU cost of the chat completion, MCP tool and A2A
message transforms against regressions on every commit. All benchmarks are
pure in-process work with no network I/O so they stay deterministic under
CodSpeed's simulation mode, and they import under the base dependency set the
benchmark job installs.

Inference covers the full SDK overhead via mock_response (simple, multi-turn,
tools, streaming) plus convert_to_model_response_object as a deterministic
anchor. MCP covers the client-side tool translation and the proxy server-side
tool-name prefix round-trip. A2A covers the client request/response transforms
and the proxy server-ingress message conversion.

Adds the mcp and a2a-sdk packages to the benchmark run since those transform
modules need them, and broadens the workflow triggers to litellm_internal_staging
so the internal branch flow is benchmarked too.
---
 .github/workflows/codspeed.yml                |   4 +
 tests/benchmarks/test_a2a_benchmarks.py       |  76 ++++++++++++
 tests/benchmarks/test_inference_benchmarks.py | 113 ++++++++++++++++++
 tests/benchmarks/test_mcp_benchmarks.py       |  84 +++++++++++++
 4 files changed, 277 insertions(+)
 create mode 100644 tests/benchmarks/test_a2a_benchmarks.py
 create mode 100644 tests/benchmarks/test_inference_benchmarks.py
 create mode 100644 tests/benchmarks/test_mcp_benchmarks.py

diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
index 17efbf90339..1fad82827ff 100644
--- a/.github/workflows/codspeed.yml
+++ b/.github/workflows/codspeed.yml
@@ -4,9 +4,11 @@ on:
   push:
     branches:
       - main
+      - litellm_internal_staging
   pull_request:
     branches:
       - main
+      - litellm_internal_staging
   # Allow CodSpeed to trigger backtest performance analysis
   # in order to generate initial data
   workflow_dispatch:
@@ -48,6 +50,8 @@ jobs:
             uv run --frozen --no-default-groups
             --with pytest==8.3.5
             --with pytest-codspeed==4.3.0
+            --with "mcp>=1.26.0,<2.0"
+            --with "a2a-sdk>=1.1.0,<2.0"
             pytest
             -p pytest_codspeed.plugin
             tests/benchmarks/
diff --git a/tests/benchmarks/test_a2a_benchmarks.py b/tests/benchmarks/test_a2a_benchmarks.py
new file mode 100644
index 00000000000..cf7726230b6
--- /dev/null
+++ b/tests/benchmarks/test_a2a_benchmarks.py
@@ -0,0 +1,76 @@
+"""
+Performance benchmarks for the A2A (agent-to-agent) message-translation hot path.
+
+Both directions are covered: the client direction (litellm.completion talking to
+an upstream A2A agent) converts OpenAI messages into a prompt and extracts text
+from the A2A response, and the proxy server-ingress direction converts an inbound
+A2A message into OpenAI messages before bridging to a completion. All are pure-CPU
+per-request transforms.
+"""
+
+import pytest
+
+from litellm.a2a_protocol.litellm_completion_bridge.transformation import (
+    A2ACompletionBridgeTransformation,
+)
+from litellm.llms.a2a.common_utils import (
+    convert_messages_to_prompt,
+    extract_text_from_a2a_response,
+)
+
+MESSAGES = [
+    {"role": "system", "content": "You are a helpful research assistant."},
+    {"role": "user", "content": "What is the capital of France?"},
+    {"role": "assistant", "content": "The capital of France is Paris."},
+    {"role": "user", "content": "And what is its population?"},
+]
+
+MESSAGE_RESPONSE = {
+    "result": {
+        "kind": "message",
+        "parts": [
+            {"kind": "text", "text": "The population of Paris is about 2.1 million."},
+            {"kind": "text", "text": "The metro area has over 12 million people."},
+        ],
+    }
+}
+
+TASK_RESPONSE = {
+    "result": {
+        "kind": "task",
+        "artifacts": [{"parts": [{"kind": "text", "text": "Paris has a population of about 2.1 million."}]}],
+    }
+}
+
+A2A_INBOUND_MESSAGE = {
+    "role": "user",
+    "parts": [
+        {"kind": "text", "text": "Summarize the latest quarterly report."},
+        {"kind": "text", "text": "Focus on revenue and margins."},
+    ],
+    "messageId": "msg-1",
+}
+
+
+@pytest.mark.benchmark
+def test_convert_messages_to_a2a_prompt():
+    """Benchmark converting OpenAI messages into an A2A prompt string."""
+    convert_messages_to_prompt(messages=MESSAGES)
+
+
+@pytest.mark.benchmark
+def test_extract_text_from_a2a_message_response():
+    """Benchmark extracting text from a direct-message A2A response."""
+    extract_text_from_a2a_response(response_dict=MESSAGE_RESPONSE)
+
+
+@pytest.mark.benchmark
+def test_extract_text_from_a2a_task_response():
+    """Benchmark extracting text from a task-with-artifacts A2A response."""
+    extract_text_from_a2a_response(response_dict=TASK_RESPONSE)
+
+
+@pytest.mark.benchmark
+def test_a2a_inbound_message_to_openai_messages():
+    """Benchmark the proxy converting an inbound A2A message into OpenAI messages."""
+    A2ACompletionBridgeTransformation.a2a_message_to_openai_messages(A2A_INBOUND_MESSAGE)
diff --git a/tests/benchmarks/test_inference_benchmarks.py b/tests/benchmarks/test_inference_benchmarks.py
new file mode 100644
index 00000000000..0a95e34a32c
--- /dev/null
+++ b/tests/benchmarks/test_inference_benchmarks.py
@@ -0,0 +1,113 @@
+"""
+Performance benchmarks for the LLM inference (chat completion) hot path.
+
+The end-to-end cases use ``mock_response`` so the full SDK overhead is exercised
+-- provider resolution, request/response transformation, ``ModelResponse``
+construction, token counting and cost calculation -- without any network I/O. The
+``convert_to_model_response_object`` case isolates the provider-response to
+``ModelResponse`` translation, the single deterministic core every non-streaming
+completion runs.
+"""
+
+import pytest
+
+import litellm
+from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
+    convert_to_model_response_object,
+)
+from litellm.types.utils import ModelResponse
+
+SIMPLE_MESSAGES = [{"role": "user", "content": "Hello, how are you?"}]
+
+MULTI_TURN_MESSAGES = [
+    {"role": "system", "content": "You are a helpful assistant."},
+    {"role": "user", "content": "What is the capital of France?"},
+    {
+        "role": "assistant",
+        "content": "The capital of France is Paris. It is known as the City of Light.",
+    },
+    {"role": "user", "content": "Tell me more about Paris."},
+]
+
+TOOL_DEFINITIONS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "get_weather",
+            "description": "Get the current weather in a given location",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "location": {
+                        "type": "string",
+                        "description": "The city and state, e.g. San Francisco, CA",
+                    },
+                    "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
+                },
+                "required": ["location"],
+            },
+        },
+    }
+]
+
+MOCK_RESPONSE = "The capital of France is Paris, the country's largest city and cultural centre."
+
+PROVIDER_RESPONSE = {
+    "id": "chatcmpl-abc123",
+    "object": "chat.completion",
+    "created": 1700000000,
+    "model": "gpt-4o",
+    "choices": [
+        {
+            "index": 0,
+            "finish_reason": "stop",
+            "message": {"role": "assistant", "content": MOCK_RESPONSE},
+        }
+    ],
+    "usage": {"prompt_tokens": 12, "completion_tokens": 16, "total_tokens": 28},
+}
+
+
+@pytest.mark.benchmark
+def test_completion_simple_message():
+    """Benchmark a single-message completion through the full SDK path."""
+    litellm.completion(model="gpt-4o", messages=SIMPLE_MESSAGES, mock_response=MOCK_RESPONSE)
+
+
+@pytest.mark.benchmark
+def test_completion_multi_turn():
+    """Benchmark a multi-turn completion through the full SDK path."""
+    litellm.completion(model="gpt-4o", messages=MULTI_TURN_MESSAGES, mock_response=MOCK_RESPONSE)
+
+
+@pytest.mark.benchmark
+def test_completion_with_tools():
+    """Benchmark a completion that has to process tool schemas."""
+    litellm.completion(
+        model="gpt-4o",
+        messages=SIMPLE_MESSAGES,
+        tools=TOOL_DEFINITIONS,
+        mock_response=MOCK_RESPONSE,
+    )
+
+
+@pytest.mark.benchmark
+def test_completion_streaming():
+    """Benchmark consuming a full streamed completion (CustomStreamWrapper)."""
+    stream = litellm.completion(
+        model="gpt-4o",
+        messages=SIMPLE_MESSAGES,
+        mock_response=MOCK_RESPONSE,
+        stream=True,
+    )
+    for _ in stream:
+        pass
+
+
+@pytest.mark.benchmark
+def test_response_to_model_response_object():
+    """Benchmark the provider-response to ModelResponse translation core."""
+    convert_to_model_response_object(
+        response_object=PROVIDER_RESPONSE,
+        model_response_object=ModelResponse(),
+    )
diff --git a/tests/benchmarks/test_mcp_benchmarks.py b/tests/benchmarks/test_mcp_benchmarks.py
new file mode 100644
index 00000000000..7e23ab1b4f5
--- /dev/null
+++ b/tests/benchmarks/test_mcp_benchmarks.py
@@ -0,0 +1,84 @@
+"""
+Performance benchmarks for the MCP tool hot path.
+
+Two layers are covered: the client-side translation between MCP and OpenAI
+function-calling formats, and the server-side tool-name prefixing that the proxy
+runs on every list-tools (prefix each tool) and call-tool (strip prefix to route)
+request. Both are pure-CPU and deterministic.
+"""
+
+import pytest
+from mcp.types import Tool as MCPTool
+
+from litellm.experimental_mcp_client.tools import (
+    transform_mcp_tool_to_openai_tool,
+    transform_openai_tool_call_request_to_mcp_tool_call_request,
+)
+from litellm.proxy._experimental.mcp_server.utils import (
+    add_server_prefix_to_name,
+    split_server_prefix_from_name,
+)
+
+
+def _make_tool(index: int) -> MCPTool:
+    return MCPTool(
+        name=f"tool_{index}",
+        description=f"Test tool number {index} that performs an operation",
+        inputSchema={
+            "type": "object",
+            "properties": {
+                "query": {"type": "string", "description": "The search query"},
+                "limit": {"type": "integer", "description": "Max results"},
+            },
+            "required": ["query"],
+        },
+    )
+
+
+SINGLE_TOOL = _make_tool(0)
+TOOL_LIST = tuple(_make_tool(i) for i in range(20))
+TOOL_NAMES = tuple(t.name for t in TOOL_LIST)
+
+SERVER_NAME = "github_mcp"
+PREFIXED_TOOL_NAME = add_server_prefix_to_name("tool_0", SERVER_NAME)
+
+OPENAI_TOOL_CALL = {
+    "id": "call_abc123",
+    "type": "function",
+    "function": {
+        "name": "tool_0",
+        "arguments": '{"query": "weather in San Francisco", "limit": 5}',
+    },
+}
+
+
+@pytest.mark.benchmark
+def test_transform_single_mcp_tool_to_openai():
+    """Benchmark translating one MCP tool into OpenAI tool format."""
+    transform_mcp_tool_to_openai_tool(mcp_tool=SINGLE_TOOL)
+
+
+@pytest.mark.benchmark
+def test_transform_mcp_tool_list_to_openai():
+    """Benchmark translating a full list-tools response into OpenAI format."""
+    for tool in TOOL_LIST:
+        transform_mcp_tool_to_openai_tool(mcp_tool=tool)
+
+
+@pytest.mark.benchmark
+def test_transform_openai_tool_call_to_mcp():
+    """Benchmark translating an OpenAI tool call into an MCP call request."""
+    transform_openai_tool_call_request_to_mcp_tool_call_request(openai_tool=OPENAI_TOOL_CALL)
+
+
+@pytest.mark.benchmark
+def test_mcp_server_prefix_tool_list():
+    """Benchmark the proxy prefixing every tool name on a list-tools response."""
+    for name in TOOL_NAMES:
+        add_server_prefix_to_name(name, SERVER_NAME)
+
+
+@pytest.mark.benchmark
+def test_mcp_server_strip_prefix_on_call():
+    """Benchmark the proxy stripping the server prefix to route a tool call."""
+    split_server_prefix_from_name(PREFIXED_TOOL_NAME)

From 7ed25de12028ba24975c0ac6e43dc77eafc75746 Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Tue, 30 Jun 2026 10:29:49 -0700
Subject: [PATCH 12/51] fix(ui): allow any git host on the skills add form
 (LIT-4053) (#31652)

* fix(ui): allow any git host on the skills add form (LIT-4053)

The skills add form only accepted GitHub URLs: its URL parser bailed on
any host that did not start with github.com, so GitLab, Bitbucket, and
self-hosted repos (and any repo subfolder on them) were rejected before a
request was ever sent. The backend already accepts arbitrary git hosts
via its url and git-subdir sources, with no host allowlist, so this was a
client-side restriction only.

Generalize the parser into an exported, host-agnostic parseSkillSource:
GitHub URLs keep their github / git-subdir shorthand, every other host is
treated as a raw repo url, and an optional Subfolder path field turns any
repo into a git-subdir source (url + path). When a pasted GitHub
tree/blob URL already encodes a subfolder, the field is cleared and
disabled so a contradictory source can never be submitted.

The parser is hardened to match the backend contract: query strings and
fragments are stripped, the host match is case-insensitive and drops a
leading www., the extracted and field-entered subfolder paths are both
validated against the same regex the server uses, a real file-extension
allowlist (not "any dot") decides whether a trailing blob segment is a
file, a branch-only tree URL falls back to the repo, non-GitHub URLs
require at least an org/repo, and the suggested skill name is kebab-cased
so it satisfies the name field's own rule.

The git-subdir source is now handled in the display helpers
(getSourceDisplayText, getSourceLink, formatInstallCommand), which
previously showed it as "Unknown source" with no link. The submit path
is fully typed (RegisterPluginRequest plus an AddPluginFormValues
interface), removing the two prior any usages; as a result an
author with an email but no name is dropped rather than sent, since the
backend requires the author name.

No backend changes. Tests cover the full host/subfolder matrix at the
parser level plus form-submit assertions on the exact source payload.

* refactor(ui): sync skill register types to the generated OpenAPI schema, surface backend errors

Replace the hand-maintained, already-drifted API types for the skills add
flow with the generated ones from schema.d.ts: PluginAuthor now aliases
components["schemas"]["PluginAuthor"], the registration payload is a new
SkillRegisterRequest (the generated RegisterPluginRequest envelope with
source narrowed to our PluginSource union, since the backend types source
as a loose string map, and version kept optional since the backend
defaults it), and the dead, mismatched RegisterPluginResponse is deleted.
registerClaudeCodePlugin's inline payload type (which was missing the
git-subdir path field entirely) is replaced with SkillRegisterRequest, so
the networking layer and the form can no longer drift from the backend.

Error handling: the add-skill form swallowed the real failure and always
showed "Failed to register skill". registerClaudeCodePlugin already
derives the backend message and throws it, so the form now surfaces it
("Failed to register skill: <reason>"), and the networking helper falls
back to the raw body / status when the error response is not JSON instead
of throwing a JSON parse error. A regression test asserts the backend
message reaches the user.

* fix(ui): reject credentialed git URLs on the skills form

A repo URL with embedded user-info (user:token@host) passed the raw-host
parser and was stored verbatim as the skill source, which is served on
the unauthenticated /public/skill_hub and marketplace.json feeds, leaking
the credentials. Reject any host segment containing '@'.

* fix(ui): validate skill repo URLs through one WHATWG URL gate

Replace the ad-hoc string parsing (stripScheme / splitHost / manual
scheme, @, ?# checks) with a single parseRepoUrl gate built on the URL
parser, so every malformed/unsafe class is handled in one place and the
URL stored on the public skill feeds is always canonical. It enforces
https (rejecting http/ssh/git/file/javascript/data and protocol-relative
//host), rejects embedded credentials (user:token@host, including
userinfo-confusion like github.com@evil.com), rejects IP-literal hosts
(loopback/private/metadata and obfuscated/IPv6 forms), and rebuilds the
stored url from origin+pathname so query strings, fragments, and trailing
slashes can never be published. The GitHub org/repo shorthand is now
charset-validated like the other paths, so junk can't reach the stored
repo. Closes both Veria findings (credentialed and http sources) plus the
adversarial-review follow-ups, with regression tests for each class.
---
 ui/litellm-dashboard/eslint-metrics.json      |   4 +-
 ui/litellm-dashboard/eslint-suppressions.json |   5 -
 .../add_plugin_form.test.tsx                  | 190 ++++++++++++-
 .../claude_code_plugins/add_plugin_form.tsx   | 219 ++++++++-------
 .../claude_code_plugins/helpers.test.ts       | 260 +++++++++++++++++-
 .../components/claude_code_plugins/helpers.ts | 191 ++++++++++++-
 .../components/claude_code_plugins/types.ts   |  48 +---
 .../src/components/networking.tsx             |  24 +-
 8 files changed, 750 insertions(+), 191 deletions(-)

diff --git a/ui/litellm-dashboard/eslint-metrics.json b/ui/litellm-dashboard/eslint-metrics.json
index deef1136b43..09ad247391b 100644
--- a/ui/litellm-dashboard/eslint-metrics.json
+++ b/ui/litellm-dashboard/eslint-metrics.json
@@ -1,5 +1,5 @@
 {
-  "@typescript-eslint/no-explicit-any": 2016,
-  "complexity": 127,
+  "@typescript-eslint/no-explicit-any": 2014,
+  "complexity": 126,
   "max-depth": 61
 }
diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json
index e3ae304c41c..7a3c8f4a42c 100644
--- a/ui/litellm-dashboard/eslint-suppressions.json
+++ b/ui/litellm-dashboard/eslint-suppressions.json
@@ -873,11 +873,6 @@
       "count": 1
     }
   },
-  "src/components/claude_code_plugins/helpers.test.ts": {
-    "unused-imports/no-unused-imports": {
-      "count": 1
-    }
-  },
   "src/components/claude_code_plugins/plugin_table.tsx": {
     "no-restricted-imports": {
       "count": 1
diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/add_plugin_form.test.tsx b/ui/litellm-dashboard/src/components/claude_code_plugins/add_plugin_form.test.tsx
index 9aa6ec7a969..0453cd8a695 100644
--- a/ui/litellm-dashboard/src/components/claude_code_plugins/add_plugin_form.test.tsx
+++ b/ui/litellm-dashboard/src/components/claude_code_plugins/add_plugin_form.test.tsx
@@ -3,11 +3,20 @@ import { act, fireEvent, screen, waitFor } from "@testing-library/react";
 import { describe, it, expect, vi, beforeEach } from "vitest";
 import { renderWithProviders } from "../../../tests/test-utils";
 import AddPluginForm from "./add_plugin_form";
+import { registerClaudeCodePlugin } from "../networking";
+import MessageManager from "@/components/molecules/message_manager";
 
 vi.mock("../networking", () => ({
   registerClaudeCodePlugin: vi.fn().mockResolvedValue({ status: "success" }),
 }));
 
+vi.mock("@/components/molecules/message_manager", () => ({
+  default: { error: vi.fn(), success: vi.fn() },
+}));
+
+const mockRegister = vi.mocked(registerClaudeCodePlugin);
+const mockMessageError = vi.mocked(MessageManager.error);
+
 const DEFAULT_PROPS = {
   visible: true,
   onClose: vi.fn(),
@@ -15,22 +24,27 @@ const DEFAULT_PROPS = {
   onSuccess: vi.fn(),
 };
 
+const URL_PLACEHOLDER = "https://github.com/org/repo or https://gitlab.com/org/repo";
+const SUBPATH_PLACEHOLDER = "plugins/my-skill";
+
 describe("AddPluginForm", () => {
   beforeEach(() => {
     vi.clearAllMocks();
   });
 
-  it("renders with GitHub URL input", () => {
+  it("renders the host-agnostic repository URL input and subfolder field", () => {
     renderWithProviders(<AddPluginForm {...DEFAULT_PROPS} />);
 
-    expect(screen.getByText("GitHub URL")).toBeInTheDocument();
-    expect(screen.getByPlaceholderText("https://github.com/org/repo/tree/main/my-skill")).toBeInTheDocument();
+    expect(screen.getByText("Repository URL")).toBeInTheDocument();
+    expect(screen.getByPlaceholderText(URL_PLACEHOLDER)).toBeInTheDocument();
+    expect(screen.getByText("Subfolder path (Optional)")).toBeInTheDocument();
+    expect(screen.getByPlaceholderText(SUBPATH_PLACEHOLDER)).toBeInTheDocument();
   });
 
   it("shows GitHub repo preview for a plain repo URL", async () => {
     renderWithProviders(<AddPluginForm {...DEFAULT_PROPS} />);
 
-    const urlInput = screen.getByPlaceholderText("https://github.com/org/repo/tree/main/my-skill");
+    const urlInput = screen.getByPlaceholderText(URL_PLACEHOLDER);
 
     await act(async () => {
       fireEvent.change(urlInput, {
@@ -43,10 +57,10 @@ describe("AddPluginForm", () => {
     });
   });
 
-  it("shows git-subdir preview for a tree URL", async () => {
+  it("shows git-subdir preview for a tree URL and disables the subfolder field", async () => {
     renderWithProviders(<AddPluginForm {...DEFAULT_PROPS} />);
 
-    const urlInput = screen.getByPlaceholderText("https://github.com/org/repo/tree/main/my-skill");
+    const urlInput = screen.getByPlaceholderText(URL_PLACEHOLDER);
 
     await act(async () => {
       fireEvent.change(urlInput, {
@@ -59,12 +73,51 @@ describe("AddPluginForm", () => {
     await waitFor(() => {
       expect(screen.getByText(/GitHub subdir/)).toBeInTheDocument();
     });
+    expect(screen.getByPlaceholderText(SUBPATH_PLACEHOLDER)).toBeDisabled();
+  });
+
+  it("shows a raw url preview for a non-github host", async () => {
+    renderWithProviders(<AddPluginForm {...DEFAULT_PROPS} />);
+
+    const urlInput = screen.getByPlaceholderText(URL_PLACEHOLDER);
+
+    await act(async () => {
+      fireEvent.change(urlInput, {
+        target: { value: "https://gitlab.com/group/repo" },
+      });
+    });
+
+    await waitFor(() => {
+      expect(screen.getByText(/Git repo/)).toBeInTheDocument();
+    });
+    expect(screen.getByPlaceholderText(SUBPATH_PLACEHOLDER)).not.toBeDisabled();
+  });
+
+  it("combines a repo URL with a subfolder into a git-subdir preview", async () => {
+    renderWithProviders(<AddPluginForm {...DEFAULT_PROPS} />);
+
+    const urlInput = screen.getByPlaceholderText(URL_PLACEHOLDER);
+    await act(async () => {
+      fireEvent.change(urlInput, {
+        target: { value: "https://gitlab.com/group/repo" },
+      });
+    });
+
+    const subPathInput = screen.getByPlaceholderText(SUBPATH_PLACEHOLDER);
+    await act(async () => {
+      fireEvent.change(subPathInput, { target: { value: "plugins/x" } });
+    });
+
+    await waitFor(() => {
+      expect(screen.getByText(/Git subdir/)).toBeInTheDocument();
+      expect(screen.getByText(/plugins\/x/)).toBeInTheDocument();
+    });
   });
 
   it("auto-fills skill name from repo URL", async () => {
     renderWithProviders(<AddPluginForm {...DEFAULT_PROPS} />);
 
-    const urlInput = screen.getByPlaceholderText("https://github.com/org/repo/tree/main/my-skill");
+    const urlInput = screen.getByPlaceholderText(URL_PLACEHOLDER);
 
     await act(async () => {
       fireEvent.change(urlInput, {
@@ -84,7 +137,7 @@ describe("AddPluginForm", () => {
     const nameInput = screen.getByPlaceholderText("my-skill") as HTMLInputElement;
     fireEvent.change(nameInput, { target: { value: "existing-name" } });
 
-    const urlInput = screen.getByPlaceholderText("https://github.com/org/repo/tree/main/my-skill");
+    const urlInput = screen.getByPlaceholderText(URL_PLACEHOLDER);
 
     await act(async () => {
       fireEvent.change(urlInput, {
@@ -96,4 +149,125 @@ describe("AddPluginForm", () => {
       expect(nameInput.value).toBe("existing-name");
     });
   });
+
+  const typeUrl = async (value: string) => {
+    await act(async () => {
+      fireEvent.change(screen.getByPlaceholderText(URL_PLACEHOLDER), { target: { value } });
+    });
+  };
+
+  const typeSubPath = async (value: string) => {
+    await act(async () => {
+      fireEvent.change(screen.getByPlaceholderText(SUBPATH_PLACEHOLDER), { target: { value } });
+    });
+  };
+
+  const submit = async () => {
+    await act(async () => {
+      fireEvent.click(screen.getByRole("button", { name: "Add Skill" }));
+    });
+  };
+
+  it("submits a github repo source", async () => {
+    renderWithProviders(<AddPluginForm {...DEFAULT_PROPS} />);
+
+    await typeUrl("https://github.com/anthropics/claude-code");
+    await submit();
+
+    await waitFor(() => {
+      expect(mockRegister).toHaveBeenCalledWith(
+        "sk-test",
+        expect.objectContaining({ source: { source: "github", repo: "anthropics/claude-code" } }),
+      );
+    });
+  });
+
+  it("submits a github subdir source from a tree URL", async () => {
+    renderWithProviders(<AddPluginForm {...DEFAULT_PROPS} />);
+
+    await typeUrl("https://github.com/anthropics/claude-code/tree/main/plugins/my-skill");
+    await submit();
+
+    await waitFor(() => {
+      expect(mockRegister).toHaveBeenCalledWith(
+        "sk-test",
+        expect.objectContaining({
+          source: { source: "git-subdir", url: "https://github.com/anthropics/claude-code", path: "plugins/my-skill" },
+        }),
+      );
+    });
+  });
+
+  it("submits a raw url source for a gitlab repo", async () => {
+    renderWithProviders(<AddPluginForm {...DEFAULT_PROPS} />);
+
+    await typeUrl("https://gitlab.com/group/repo");
+    await submit();
+
+    await waitFor(() => {
+      expect(mockRegister).toHaveBeenCalledWith(
+        "sk-test",
+        expect.objectContaining({ source: { source: "url", url: "https://gitlab.com/group/repo" } }),
+      );
+    });
+  });
+
+  it("submits a git-subdir source from a gitlab repo plus subfolder field", async () => {
+    renderWithProviders(<AddPluginForm {...DEFAULT_PROPS} />);
+
+    await typeUrl("https://gitlab.com/group/repo");
+    await typeSubPath("plugins/x");
+    await submit();
+
+    await waitFor(() => {
+      expect(mockRegister).toHaveBeenCalledWith(
+        "sk-test",
+        expect.objectContaining({
+          source: { source: "git-subdir", url: "https://gitlab.com/group/repo", path: "plugins/x" },
+        }),
+      );
+    });
+  });
+
+  it("clears the subfolder field and uses the URL path once a tree URL is entered", async () => {
+    renderWithProviders(<AddPluginForm {...DEFAULT_PROPS} />);
+
+    await typeSubPath("plugins/x");
+    const subPathInput = screen.getByPlaceholderText(SUBPATH_PLACEHOLDER) as HTMLInputElement;
+    expect(subPathInput.value).toBe("plugins/x");
+
+    await typeUrl("https://github.com/anthropics/claude-code/tree/main/plugins/from-url");
+
+    await waitFor(() => {
+      expect(subPathInput.value).toBe("");
+      expect(subPathInput).toBeDisabled();
+    });
+
+    await submit();
+
+    await waitFor(() => {
+      expect(mockRegister).toHaveBeenCalledWith(
+        "sk-test",
+        expect.objectContaining({
+          source: {
+            source: "git-subdir",
+            url: "https://github.com/anthropics/claude-code",
+            path: "plugins/from-url",
+          },
+        }),
+      );
+    });
+  });
+
+  it("surfaces the backend error message when registration fails", async () => {
+    mockRegister.mockRejectedValueOnce(new Error("Plugin 'claude-code' already exists"));
+    renderWithProviders(<AddPluginForm {...DEFAULT_PROPS} />);
+
+    await typeUrl("https://github.com/anthropics/claude-code");
+    await submit();
+
+    await waitFor(() => {
+      expect(mockMessageError).toHaveBeenCalledWith(expect.stringContaining("Plugin 'claude-code' already exists"));
+    });
+  });
 });
diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/add_plugin_form.tsx b/ui/litellm-dashboard/src/components/claude_code_plugins/add_plugin_form.tsx
index f90261ae3a3..a1587bc5189 100644
--- a/ui/litellm-dashboard/src/components/claude_code_plugins/add_plugin_form.tsx
+++ b/ui/litellm-dashboard/src/components/claude_code_plugins/add_plugin_form.tsx
@@ -3,7 +3,17 @@ import { Modal, Form, Input, Select } from "antd";
 import MessageManager from "@/components/molecules/message_manager";
 import { Button } from "@tremor/react";
 import { registerClaudeCodePlugin } from "../networking";
-import { validatePluginName, isValidSemanticVersion, isValidEmail, isValidUrl, parseKeywords } from "./helpers";
+import {
+  validatePluginName,
+  isValidSemanticVersion,
+  isValidEmail,
+  isValidUrl,
+  parseKeywords,
+  parseSkillSource,
+  isValidSubPath,
+  SkillSourcePreview,
+} from "./helpers";
+import { PluginAuthor, PluginSource, SkillRegisterRequest } from "./types";
 
 const { TextArea } = Input;
 const { Option } = Select;
@@ -15,6 +25,46 @@ interface AddPluginFormProps {
   onSuccess: () => void;
 }
 
+interface AddPluginFormValues {
+  name: string;
+  skillUrl?: string;
+  subPath?: string;
+  version?: string;
+  description?: string;
+  authorName?: string;
+  authorEmail?: string;
+  homepage?: string;
+  category?: string;
+  keywords?: string;
+  domain?: string;
+  namespace?: string;
+}
+
+const buildAuthor = (values: AddPluginFormValues): PluginAuthor | undefined => {
+  const name = values.authorName?.trim();
+  const email = values.authorEmail?.trim();
+  if (!name) {
+    return undefined;
+  }
+  return email ? { name, email } : { name };
+};
+
+const buildRegisterRequest = (values: AddPluginFormValues, source: PluginSource): SkillRegisterRequest => {
+  const author = buildAuthor(values);
+  return {
+    name: values.name.trim(),
+    source,
+    ...(values.version ? { version: values.version.trim() } : {}),
+    ...(values.description ? { description: values.description.trim() } : {}),
+    ...(author ? { author } : {}),
+    ...(values.homepage ? { homepage: values.homepage.trim() } : {}),
+    ...(values.category ? { category: values.category } : {}),
+    ...(values.keywords ? { keywords: parseKeywords(values.keywords) } : {}),
+    ...(values.domain ? { domain: values.domain.trim() } : {}),
+    ...(values.namespace ? { namespace: values.namespace.trim() } : {}),
+  };
+};
+
 const PREDEFINED_CATEGORIES = [
   "Development",
   "Productivity",
@@ -26,106 +76,41 @@ const PREDEFINED_CATEGORIES = [
   "Documentation",
 ];
 
-interface ParsedSource {
-  source: "github" | "url" | "git-subdir";
-  repo?: string;
-  url?: string;
-  path?: string;
-}
-
-interface ParsePreview {
-  parsed: ParsedSource;
-  label: string;
-  suggestedName: string;
-}
-
-function parseGitHubUrl(raw: string): ParsePreview | null {
-  // Strip protocol and trailing slashes/spaces
-  let s = raw
-    .trim()
-    .replace(/^https?:\/\//, "")
-    .replace(/\/+$/, "");
-
-  if (!s.startsWith("github.com/")) return null;
-
-  // Remove "github.com/"
-  const rest = s.slice("github.com/".length);
-  const parts = rest.split("/");
-
-  if (parts.length < 2) return null;
-
-  const org = parts[0];
-  const repo = parts[1];
-  const repoBase = repo.replace(/\.git$/, "");
-
-  // github.com/org/repo  (exactly 2 parts, or ends with .git)
-  if (parts.length === 2 || (parts.length === 2 && repoBase)) {
-    return {
-      parsed: { source: "github", repo: `${org}/${repoBase}` },
-      label: `GitHub repo — ${org}/${repoBase}`,
-      suggestedName: repoBase,
-    };
-  }
-
-  // github.com/org/repo/tree/branch/folder or /blob/branch/folder/FILE.md
-  if (parts.length >= 5 && (parts[2] === "tree" || parts[2] === "blob")) {
-    // parts[3] = branch, parts[4..] = path segments
-    const pathParts = parts.slice(4);
-    // If last segment looks like a file (has extension), drop it
-    const lastPart = pathParts[pathParts.length - 1];
-    if (lastPart && lastPart.includes(".")) {
-      pathParts.pop();
-    }
-    if (pathParts.length === 0) {
-      // Path resolved to repo root — treat as plain github source
-      return {
-        parsed: { source: "github", repo: `${org}/${repoBase}` },
-        label: `GitHub repo — ${org}/${repoBase}`,
-        suggestedName: repoBase,
-      };
-    }
-    const subPath = pathParts.join("/");
-    const suggestedName = pathParts[pathParts.length - 1];
-    return {
-      parsed: {
-        source: "git-subdir",
-        url: `https://github.com/${org}/${repoBase}`,
-        path: subPath,
-      },
-      label: `GitHub subdir — ${org}/${repoBase} @ ${subPath}`,
-      suggestedName,
-    };
-  }
-
-  return null;
-}
-
 const AddPluginForm: React.FC<AddPluginFormProps> = ({ visible, onClose, accessToken, onSuccess }) => {
   const [form] = Form.useForm();
   const [isSubmitting, setIsSubmitting] = useState(false);
-  const [urlPreview, setUrlPreview] = useState<ParsePreview | null>(null);
+  const [urlPreview, setUrlPreview] = useState<SkillSourcePreview | null>(null);
+  const [urlEncodesSubdir, setUrlEncodesSubdir] = useState(false);
 
-  const handleUrlChange = (e: React.ChangeEvent<HTMLInputElement>) => {
-    const val = e.target.value;
-    const preview = parseGitHubUrl(val);
+  const recomputePreview = (skillUrl: string, subPath: string) => {
+    const encodesSubdir = parseSkillSource(skillUrl)?.parsed.source === "git-subdir";
+    setUrlEncodesSubdir(encodesSubdir);
+    if (encodesSubdir && form.getFieldValue("subPath")) {
+      form.setFieldsValue({ subPath: "" });
+    }
+    const preview = parseSkillSource(skillUrl, encodesSubdir ? undefined : subPath);
     setUrlPreview(preview);
-    if (preview) {
-      // Auto-fill name only if it's currently empty
-      const currentName = form.getFieldValue("name");
-      if (!currentName) {
-        form.setFieldsValue({ name: preview.suggestedName });
-      }
+    if (preview && !form.getFieldValue("name")) {
+      form.setFieldsValue({ name: preview.suggestedName });
     }
   };
 
-  const handleSubmit = async (values: any) => {
+  const handleUrlChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    recomputePreview(e.target.value, form.getFieldValue("subPath") ?? "");
+  };
+
+  const handleSubPathChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    recomputePreview(form.getFieldValue("skillUrl") ?? "", e.target.value);
+  };
+
+  const handleSubmit = async (values: AddPluginFormValues) => {
     if (!accessToken) {
       MessageManager.error("No access token available");
       return;
     }
 
     if (!urlPreview) {
-      MessageManager.error("Please enter a valid GitHub URL");
+      MessageManager.error("Please enter a valid repository URL");
       return;
     }
 
@@ -151,33 +136,17 @@ const AddPluginForm: React.FC<AddPluginFormProps> = ({ visible, onClose, accessT
 
     setIsSubmitting(true);
     try {
-      const pluginData: any = {
-        name: values.name.trim(),
-        source: urlPreview.parsed,
-      };
-
-      if (values.version) pluginData.version = values.version.trim();
-      if (values.description) pluginData.description = values.description.trim();
-      if (values.authorName || values.authorEmail) {
-        pluginData.author = {};
-        if (values.authorName) pluginData.author.name = values.authorName.trim();
-        if (values.authorEmail) pluginData.author.email = values.authorEmail.trim();
-      }
-      if (values.homepage) pluginData.homepage = values.homepage.trim();
-      if (values.category) pluginData.category = values.category;
-      if (values.keywords) pluginData.keywords = parseKeywords(values.keywords);
-      if (values.domain) pluginData.domain = values.domain.trim();
-      if (values.namespace) pluginData.namespace = values.namespace.trim();
-
-      await registerClaudeCodePlugin(accessToken, pluginData);
+      await registerClaudeCodePlugin(accessToken, buildRegisterRequest(values, urlPreview.parsed));
       MessageManager.success("Skill registered successfully");
       form.resetFields();
       setUrlPreview(null);
+      setUrlEncodesSubdir(false);
       onSuccess();
       onClose();
     } catch (error) {
       console.error("Error registering skill:", error);
-      MessageManager.error("Failed to register skill");
+      const reason = error instanceof Error && error.message ? error.message : "Failed to register skill";
+      MessageManager.error(`Failed to register skill: ${reason}`);
     } finally {
       setIsSubmitting(false);
     }
@@ -186,6 +155,7 @@ const AddPluginForm: React.FC<AddPluginFormProps> = ({ visible, onClose, accessT
   const handleCancel = () => {
     form.resetFields();
     setUrlPreview(null);
+    setUrlEncodesSubdir(false);
     onClose();
   };
 
@@ -194,18 +164,45 @@ const AddPluginForm: React.FC<AddPluginFormProps> = ({ visible, onClose, accessT
       <Form form={form} layout="vertical" onFinish={handleSubmit} className="mt-4">
         {/* Smart URL Input */}
         <Form.Item
-          label="GitHub URL"
+          label="Repository URL"
           name="skillUrl"
-          rules={[{ required: true, message: "Please enter a GitHub URL" }]}
-          tooltip="Paste a GitHub URL — repo, folder, or file link. E.g. github.com/org/repo or github.com/org/repo/tree/main/my-skill"
+          rules={[{ required: true, message: "Please enter a repository URL" }]}
+          tooltip="Paste an HTTPS git repository URL from GitHub, GitLab, Bitbucket, or a self-hosted host. E.g. github.com/org/repo, gitlab.com/org/repo, or github.com/org/repo/tree/main/my-skill"
         >
           <Input
-            placeholder="https://github.com/org/repo/tree/main/my-skill"
+            placeholder="https://github.com/org/repo or https://gitlab.com/org/repo"
             className="rounded-lg"
             onChange={handleUrlChange}
           />
         </Form.Item>
 
+        {/* Optional subfolder for monorepos */}
+        <Form.Item
+          label="Subfolder path (Optional)"
+          name="subPath"
+          rules={[
+            {
+              validator: (_, value) =>
+                !value || isValidSubPath(value)
+                  ? Promise.resolve()
+                  : Promise.reject(
+                      new Error(
+                        "Subfolder must be a relative path like plugins/my-skill (letters, numbers, dots, hyphens, underscores)",
+                      ),
+                    ),
+            },
+          ]}
+          tooltip="Path within the repository where the skill lives (e.g., plugins/my-skill). Leave empty if the skill is at the repo root."
+          extra={urlEncodesSubdir ? "The URL already points to a subfolder, so this field is disabled" : undefined}
+        >
+          <Input
+            placeholder="plugins/my-skill"
+            className="rounded-lg"
+            onChange={handleSubPathChange}
+            disabled={urlEncodesSubdir}
+          />
+        </Form.Item>
+
         {/* Parsed preview */}
         {urlPreview && (
           <div className="mb-4 px-3 py-2 bg-blue-50 border border-blue-200 rounded-lg text-sm text-blue-700">
diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.test.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.test.ts
index b3930d15718..4c84db2a97d 100644
--- a/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.test.ts
+++ b/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.test.ts
@@ -15,23 +15,34 @@ import {
   isValidUrl,
   parseKeywords,
   formatKeywords,
+  parseSkillSource,
+  isValidSubPath,
 } from "./helpers";
 import { MarketplacePluginEntry, PluginSource } from "./types";
 
 describe("formatInstallCommand", () => {
   it("formats github source with repo", () => {
-    const plugin = { name: "my-plugin", source: { source: "github" as const, repo: "org/repo" } };
-    expect(formatInstallCommand(plugin)).toBe("/plugin marketplace add org/repo");
+    const source: PluginSource = { source: "github", repo: "org/repo" };
+    expect(formatInstallCommand({ name: "my-plugin", source })).toBe("/plugin marketplace add org/repo");
   });
 
   it("formats url source", () => {
-    const plugin = { name: "my-plugin", source: { source: "url" as const, url: "https://example.com/plugin" } };
-    expect(formatInstallCommand(plugin)).toBe("/plugin marketplace add https://example.com/plugin");
+    const source: PluginSource = { source: "url", url: "https://example.com/plugin" };
+    expect(formatInstallCommand({ name: "my-plugin", source })).toBe(
+      "/plugin marketplace add https://example.com/plugin",
+    );
+  });
+
+  it("formats git-subdir source using its url", () => {
+    const source: PluginSource = { source: "git-subdir", url: "https://github.com/org/repo", path: "plugins/x" };
+    expect(formatInstallCommand({ name: "my-plugin", source })).toBe(
+      "/plugin marketplace add https://github.com/org/repo",
+    );
   });
 
   it("falls back to plugin name when no repo or url", () => {
-    const plugin = { name: "my-plugin", source: { source: "github" as const } };
-    expect(formatInstallCommand(plugin)).toBe("/plugin marketplace add my-plugin");
+    const source: PluginSource = { source: "github" };
+    expect(formatInstallCommand({ name: "my-plugin", source })).toBe("/plugin marketplace add my-plugin");
   });
 });
 
@@ -91,6 +102,18 @@ describe("getSourceDisplayText", () => {
     expect(getSourceDisplayText({ source: "url", url: "https://example.com" })).toBe("https://example.com");
   });
 
+  it("shows git-subdir as url @ path for a github subdir", () => {
+    expect(getSourceDisplayText({ source: "git-subdir", url: "https://github.com/org/repo", path: "plugins/x" })).toBe(
+      "https://github.com/org/repo @ plugins/x",
+    );
+  });
+
+  it("shows git-subdir as url @ path for a gitlab subdir", () => {
+    expect(getSourceDisplayText({ source: "git-subdir", url: "https://gitlab.com/org/repo", path: "sub/dir" })).toBe(
+      "https://gitlab.com/org/repo @ sub/dir",
+    );
+  });
+
   it("returns unknown for missing data", () => {
     expect(getSourceDisplayText({ source: "github" })).toBe("Unknown source");
   });
@@ -105,6 +128,18 @@ describe("getSourceLink", () => {
     expect(getSourceLink({ source: "url", url: "https://example.com" })).toBe("https://example.com");
   });
 
+  it("returns the repo url for a github git-subdir source", () => {
+    expect(getSourceLink({ source: "git-subdir", url: "https://github.com/org/repo", path: "plugins/x" })).toBe(
+      "https://github.com/org/repo",
+    );
+  });
+
+  it("returns the repo url for a gitlab git-subdir source", () => {
+    expect(getSourceLink({ source: "git-subdir", url: "https://gitlab.com/org/repo", path: "sub/dir" })).toBe(
+      "https://gitlab.com/org/repo",
+    );
+  });
+
   it("returns null when no repo or url", () => {
     expect(getSourceLink({ source: "github" })).toBeNull();
   });
@@ -323,3 +358,216 @@ describe("formatKeywords", () => {
     expect(formatKeywords(undefined)).toBe("");
   });
 });
+
+describe("parseSkillSource", () => {
+  it("parses a plain github repo", () => {
+    expect(parseSkillSource("github.com/org/repo")?.parsed).toEqual({ source: "github", repo: "org/repo" });
+  });
+
+  it("strips a .git suffix from the github repo shorthand", () => {
+    expect(parseSkillSource("https://github.com/org/repo.git")?.parsed).toEqual({
+      source: "github",
+      repo: "org/repo",
+    });
+  });
+
+  it("parses a github tree URL into a git-subdir", () => {
+    expect(parseSkillSource("github.com/org/repo/tree/main/plugins/x")?.parsed).toEqual({
+      source: "git-subdir",
+      url: "https://github.com/org/repo",
+      path: "plugins/x",
+    });
+  });
+
+  it("drops a trailing file segment from a github blob URL", () => {
+    expect(parseSkillSource("github.com/org/repo/blob/main/x/SKILL.md")?.parsed).toEqual({
+      source: "git-subdir",
+      url: "https://github.com/org/repo",
+      path: "x",
+    });
+  });
+
+  it("combines a github repo with an explicit subfolder", () => {
+    expect(parseSkillSource("github.com/org/repo", "plugins/x")?.parsed).toEqual({
+      source: "git-subdir",
+      url: "https://github.com/org/repo",
+      path: "plugins/x",
+    });
+  });
+
+  it("treats a gitlab repo as a raw url source", () => {
+    expect(parseSkillSource("gitlab.com/org/repo")?.parsed).toEqual({
+      source: "url",
+      url: "https://gitlab.com/org/repo",
+    });
+  });
+
+  it("keeps the .git suffix on raw urls", () => {
+    expect(parseSkillSource("https://gitlab.com/org/repo.git")?.parsed).toEqual({
+      source: "url",
+      url: "https://gitlab.com/org/repo.git",
+    });
+  });
+
+  it("combines a gitlab repo with an explicit subfolder", () => {
+    expect(parseSkillSource("gitlab.com/org/repo", "plugins/x")?.parsed).toEqual({
+      source: "git-subdir",
+      url: "https://gitlab.com/org/repo",
+      path: "plugins/x",
+    });
+  });
+
+  it("combines a self-hosted host with an explicit subfolder", () => {
+    expect(parseSkillSource("https://git.acme.com/team/repo", "sub/dir")?.parsed).toEqual({
+      source: "git-subdir",
+      url: "https://git.acme.com/team/repo",
+      path: "sub/dir",
+    });
+  });
+
+  it("lets a github URL-encoded subdir win over an also-provided subfolder", () => {
+    expect(parseSkillSource("github.com/org/repo/tree/main/plugins/x", "ignored/path")?.parsed).toEqual({
+      source: "git-subdir",
+      url: "https://github.com/org/repo",
+      path: "plugins/x",
+    });
+  });
+
+  it("rejects traversal, absolute, and double-slash subfolders", () => {
+    expect(parseSkillSource("gitlab.com/org/repo", "../etc")).toBeNull();
+    expect(parseSkillSource("gitlab.com/org/repo", "/abs")).toBeNull();
+    expect(parseSkillSource("gitlab.com/org/repo", "a//b")).toBeNull();
+  });
+
+  it("returns null for empty and garbage input", () => {
+    expect(parseSkillSource("")).toBeNull();
+    expect(parseSkillSource("   ")).toBeNull();
+    expect(parseSkillSource("not a url")).toBeNull();
+  });
+
+  it("suggests a kebab-friendly name from the last path segment", () => {
+    expect(parseSkillSource("github.com/org/my-awesome-skill")?.suggestedName).toBe("my-awesome-skill");
+    expect(parseSkillSource("github.com/org/repo/tree/main/plugins/cool-skill")?.suggestedName).toBe("cool-skill");
+    expect(parseSkillSource("gitlab.com/org/repo", "plugins/x")?.suggestedName).toBe("x");
+  });
+
+  it("rejects a bad explicit subfolder for a github repo", () => {
+    expect(parseSkillSource("github.com/org/repo", "../etc")).toBeNull();
+    expect(parseSkillSource("github.com/org/repo", "/abs")).toBeNull();
+    expect(parseSkillSource("github.com/org/repo", "a//b")).toBeNull();
+  });
+
+  it("treats a blob URL pointing at a root file as the plain repo", () => {
+    expect(parseSkillSource("github.com/org/repo/blob/main/SKILL.md")?.parsed).toEqual({
+      source: "github",
+      repo: "org/repo",
+    });
+  });
+
+  it("strips query strings and fragments before parsing", () => {
+    expect(parseSkillSource("github.com/org/repo?tab=readme")?.parsed).toEqual({ source: "github", repo: "org/repo" });
+    expect(parseSkillSource("github.com/org/repo#section")?.parsed).toEqual({ source: "github", repo: "org/repo" });
+  });
+
+  it("rejects a tree URL whose folder has a space or percent-encoded segment", () => {
+    expect(parseSkillSource("github.com/org/repo/tree/main/a b")).toBeNull();
+    expect(parseSkillSource("github.com/org/repo/tree/main/a%20b")).toBeNull();
+  });
+
+  it("routes uppercase and www github hosts through the github shorthand", () => {
+    expect(parseSkillSource("GitHub.com/org/repo/tree/main/x")?.parsed).toEqual({
+      source: "git-subdir",
+      url: "https://github.com/org/repo",
+      path: "x",
+    });
+    expect(parseSkillSource("www.github.com/org/repo")?.parsed).toEqual({ source: "github", repo: "org/repo" });
+  });
+
+  it("keeps a dotted folder name as the subdir path", () => {
+    expect(parseSkillSource("github.com/org/repo/blob/main/my.skill")?.parsed).toEqual({
+      source: "git-subdir",
+      url: "https://github.com/org/repo",
+      path: "my.skill",
+    });
+  });
+
+  it("falls back to the repo for a tree URL with a branch but no folder", () => {
+    expect(parseSkillSource("github.com/org/repo/tree/main")?.parsed).toEqual({ source: "github", repo: "org/repo" });
+  });
+
+  it("kebab-cases the suggested name from a mixed-case repo", () => {
+    expect(parseSkillSource("github.com/Org/My_Repo")?.suggestedName).toBe("my-repo");
+  });
+
+  it("rejects a bare host or single-segment raw git url", () => {
+    expect(parseSkillSource("gitlab.com")).toBeNull();
+    expect(parseSkillSource("gitlab.com/org")).toBeNull();
+  });
+});
+
+// Skill sources are served on the unauthenticated public feeds and cloned by clients, so the
+// parser must never publish an insecure, credentialed, internal, or malformed clone URL.
+describe("parseSkillSource — security boundary", () => {
+  it("rejects non-https schemes", () => {
+    for (const url of [
+      "http://gitlab.com/org/repo",
+      "HTTP://gitlab.com/org/repo",
+      "ssh://gitlab.com/org/repo",
+      "git://gitlab.com/org/repo",
+      "ftp://gitlab.com/org/repo",
+      "file:///etc/passwd",
+      "javascript:alert(1)",
+      "data:text/plain,hi",
+      "//gitlab.com/org/repo",
+    ]) {
+      expect(parseSkillSource(url)).toBeNull();
+    }
+  });
+
+  it("rejects URLs with embedded credentials", () => {
+    expect(parseSkillSource("https://user:token@gitlab.com/org/repo")).toBeNull();
+    expect(parseSkillSource("https://user@gitlab.com/org/repo")).toBeNull();
+    // userinfo confusion: the real host is evil.com, not github.com
+    expect(parseSkillSource("https://github.com@evil.com/org/repo")).toBeNull();
+  });
+
+  it("rejects IP-literal hosts (loopback, private, metadata, obfuscated, IPv6)", () => {
+    for (const url of [
+      "https://127.0.0.1/org/repo",
+      "https://10.0.0.5/org/repo",
+      "https://169.254.169.254/org/repo",
+      "https://2130706433/org/repo",
+      "https://[::ffff:127.0.0.1]/org/repo",
+    ]) {
+      expect(parseSkillSource(url)).toBeNull();
+    }
+  });
+
+  it("does not grant GitHub shorthand to a look-alike host", () => {
+    expect(parseSkillSource("https://github.com.evil.com/org/repo")?.parsed).toEqual({
+      source: "url",
+      url: "https://github.com.evil.com/org/repo",
+    });
+  });
+
+  it("rejects GitHub org/repo segments with illegal characters", () => {
+    expect(parseSkillSource("github.com/o@x/repo")).toBeNull();
+    expect(parseSkillSource("github.com/org/..%2f..%2fx")).toBeNull();
+  });
+});
+
+describe("isValidSubPath", () => {
+  it("accepts relative segment paths", () => {
+    expect(isValidSubPath("plugins/x")).toBe(true);
+    expect(isValidSubPath("sub/dir")).toBe(true);
+    expect(isValidSubPath("a.b-c_d")).toBe(true);
+    expect(isValidSubPath("plugins/x/")).toBe(true);
+  });
+
+  it("rejects empty, traversal, absolute, and double-slash paths", () => {
+    expect(isValidSubPath("")).toBe(false);
+    expect(isValidSubPath("../etc")).toBe(false);
+    expect(isValidSubPath("/abs")).toBe(false);
+    expect(isValidSubPath("a//b")).toBe(false);
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.ts
index d696a78b4cc..cab3c5cba3c 100644
--- a/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.ts
+++ b/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.ts
@@ -4,15 +4,189 @@
 
 import { PluginSource, MarketplacePluginEntry } from "./types";
 
+export interface SkillSourcePreview {
+  parsed: PluginSource;
+  label: string;
+  suggestedName: string;
+}
+
+export const SUBDIR_PATH_REGEX = /^[a-zA-Z0-9][a-zA-Z0-9._-]*(\/[a-zA-Z0-9][a-zA-Z0-9._-]*)*$/;
+
+export const normalizeSubPath = (subPath: string): string => subPath.trim().replace(/\/+$/, "");
+
+export const isValidSubPath = (subPath: string): boolean => {
+  const normalized = normalizeSubPath(subPath);
+  return normalized !== "" && SUBDIR_PATH_REGEX.test(normalized);
+};
+
+const GITHUB_HOST = "github.com";
+
+const SKILL_FILE_EXTENSION_REGEX = /\.(md|markdown|txt|json|ya?ml|toml)$/i;
+
+// WHATWG normalizes obfuscated IPv4 (e.g. 2130706433, 0x7f.0.0.1) to dotted-decimal, so this
+// catches every IPv4 form; bracketed IPv6 is rejected separately.
+const IPV4_HOST_REGEX = /^\d{1,3}(\.\d{1,3}){3}$/;
+
+const GITHUB_ORG_REGEX = /^[A-Za-z0-9-]+$/;
+const GITHUB_REPO_REGEX = /^[A-Za-z0-9._-]+$/;
+
+const buildRepoUrl = (url: URL): string => `${url.protocol}//${url.host}${url.pathname.replace(/\/+$/, "")}`;
+
+const pathSegments = (url: URL): string[] => url.pathname.split("/").filter((seg) => seg !== "");
+
+/**
+ * Validate and normalize a repository URL into a parsed URL, or null. Enforces https (rejects
+ * http/ssh/git/etc.), rejects embedded credentials, and requires a dotted host, so the public
+ * skill feeds never serve an insecure or credentialed clone URL. Everything downstream parses
+ * this normalized object rather than the raw string.
+ */
+const parseRepoUrl = (raw: string): URL | null => {
+  const trimmed = raw.trim();
+  if (trimmed === "" || trimmed.startsWith("//")) {
+    return null;
+  }
+  const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`;
+  let url: URL;
+  try {
+    url = new URL(withScheme);
+  } catch {
+    return null;
+  }
+  if (
+    url.protocol !== "https:" ||
+    url.username !== "" ||
+    url.password !== "" ||
+    !url.hostname.includes(".") ||
+    url.hostname.startsWith("[") ||
+    IPV4_HOST_REGEX.test(url.hostname)
+  ) {
+    return null;
+  }
+  return url;
+};
+
+const lastSegment = (path: string): string => {
+  const segments = path.split("/").filter((seg) => seg !== "");
+  return segments[segments.length - 1] ?? "";
+};
+
+const toKebabCase = (value: string): string =>
+  value
+    .toLowerCase()
+    .replace(/[^a-z0-9-]+/g, "-")
+    .replace(/-+/g, "-")
+    .replace(/^-+|-+$/g, "");
+
+const parseGitHubSource = (url: URL, subPath?: string): SkillSourcePreview | null => {
+  const parts = pathSegments(url);
+  if (parts.length < 2) {
+    return null;
+  }
+
+  const org = parts[0];
+  const repoBase = parts[1].replace(/\.git$/, "");
+  if (!GITHUB_ORG_REGEX.test(org) || !GITHUB_REPO_REGEX.test(repoBase)) {
+    return null;
+  }
+  const repoFull = `${org}/${repoBase}`;
+  const repoUrl = `https://github.com/${repoFull}`;
+  const repoPreview: SkillSourcePreview = {
+    parsed: { source: "github", repo: repoFull },
+    label: `GitHub repo — ${repoFull}`,
+    suggestedName: toKebabCase(repoBase),
+  };
+
+  const isTreeOrBlob = parts.length >= 4 && (parts[2] === "tree" || parts[2] === "blob");
+  if (isTreeOrBlob) {
+    const pathParts = parts.slice(4);
+    const last = lastSegment(pathParts.join("/"));
+    const effective = SKILL_FILE_EXTENSION_REGEX.test(last) ? pathParts.slice(0, -1) : pathParts;
+    if (effective.length === 0) {
+      return repoPreview;
+    }
+    const path = normalizeSubPath(effective.join("/"));
+    if (!SUBDIR_PATH_REGEX.test(path)) {
+      return null;
+    }
+    return {
+      parsed: { source: "git-subdir", url: repoUrl, path },
+      label: `GitHub subdir — ${repoFull} @ ${path}`,
+      suggestedName: toKebabCase(lastSegment(path)),
+    };
+  }
+
+  if (parts.length !== 2) {
+    return null;
+  }
+
+  const normalized = normalizeSubPath(subPath ?? "");
+  if (normalized !== "") {
+    if (!SUBDIR_PATH_REGEX.test(normalized)) {
+      return null;
+    }
+    return {
+      parsed: { source: "git-subdir", url: repoUrl, path: normalized },
+      label: `GitHub subdir — ${repoFull} @ ${normalized}`,
+      suggestedName: toKebabCase(lastSegment(normalized)),
+    };
+  }
+
+  return repoPreview;
+};
+
+const parseRawGitSource = (url: URL, subPath?: string): SkillSourcePreview | null => {
+  if (pathSegments(url).length < 2) {
+    return null;
+  }
+
+  const repoUrl = buildRepoUrl(url);
+
+  const normalized = normalizeSubPath(subPath ?? "");
+  if (normalized !== "") {
+    if (!SUBDIR_PATH_REGEX.test(normalized)) {
+      return null;
+    }
+    return {
+      parsed: { source: "git-subdir", url: repoUrl, path: normalized },
+      label: `Git subdir — ${repoUrl} @ ${normalized}`,
+      suggestedName: toKebabCase(lastSegment(normalized)),
+    };
+  }
+
+  return {
+    parsed: { source: "url", url: repoUrl },
+    label: `Git repo — ${repoUrl}`,
+    suggestedName: toKebabCase(lastSegment(url.pathname).replace(/\.git$/, "")),
+  };
+};
+
+/**
+ * Parse any git-accessible repository URL into a registerable skill source.
+ * GitHub URLs keep their `github`/`git-subdir` shorthand; every other host is
+ * treated as a raw repo URL, with an optional subfolder turning it into git-subdir.
+ */
+export const parseSkillSource = (rawUrl: string, subPath?: string): SkillSourcePreview | null => {
+  const url = parseRepoUrl(rawUrl);
+  if (!url) {
+    return null;
+  }
+  if (url.hostname.replace(/^www\./, "") === GITHUB_HOST) {
+    return parseGitHubSource(url, subPath);
+  }
+  return parseRawGitSource(url, subPath);
+};
+
 /**
  * Generate install command for Claude Code CLI
  * Format: /plugin marketplace add org/repo OR /plugin marketplace add url
  */
 export const formatInstallCommand = (plugin: { name: string; source: PluginSource }): string => {
-  if (plugin.source.source === "github" && plugin.source.repo) {
-    return `/plugin marketplace add ${plugin.source.repo}`;
-  } else if (plugin.source.source === "url" && plugin.source.url) {
-    return `/plugin marketplace add ${plugin.source.url}`;
+  const { source } = plugin;
+  if (source.source === "github" && source.repo) {
+    return `/plugin marketplace add ${source.repo}`;
+  }
+  if ((source.source === "url" || source.source === "git-subdir") && source.url) {
+    return `/plugin marketplace add ${source.url}`;
   }
   // Fallback to plugin name
   return `/plugin marketplace add ${plugin.name}`;
@@ -55,7 +229,11 @@ export const validatePluginName = (name: string): boolean => {
 export const getSourceDisplayText = (source: PluginSource): string => {
   if (source.source === "github" && source.repo) {
     return `GitHub: ${source.repo}`;
-  } else if (source.source === "url" && source.url) {
+  }
+  if (source.source === "git-subdir" && source.url && source.path) {
+    return `${source.url} @ ${source.path}`;
+  }
+  if (source.source === "url" && source.url) {
     return source.url;
   }
   return "Unknown source";
@@ -67,7 +245,8 @@ export const getSourceDisplayText = (source: PluginSource): string => {
 export const getSourceLink = (source: PluginSource): string | null => {
   if (source.source === "github" && source.repo) {
     return `https://github.com/${source.repo}`;
-  } else if (source.source === "url" && source.url) {
+  }
+  if ((source.source === "url" || source.source === "git-subdir") && source.url) {
     return source.url;
   }
   return null;
diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts
index fcb1146685d..d16c880749b 100644
--- a/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts
+++ b/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts
@@ -1,8 +1,12 @@
 /**
  * TypeScript types for Claude Code Marketplace
- * Matches backend API types from /litellm/types/proxy/claude_code_endpoints.py
+ * API request/response shapes are synced from the generated OpenAPI types in @/lib/http/schema.
  */
 
+import type { components } from "@/lib/http/schema";
+
+// Kept hand-written: the backend types `source` as Dict[str, str], so the generated type is a
+// loose string map; this discriminant union is what the parser and display helpers rely on.
 export interface PluginSource {
   source: "github" | "url" | "git-subdir";
   repo?: string; // Format: "org/repo" for GitHub
@@ -10,10 +14,7 @@ export interface PluginSource {
   path?: string; // Subdirectory path for git-subdir
 }
 
-export interface PluginAuthor {
-  name: string;
-  email?: string;
-}
+export type PluginAuthor = components["schemas"]["PluginAuthor"];
 
 export interface Plugin {
   id: string;
@@ -56,24 +57,12 @@ export interface ListPluginsResponse {
   count: number;
 }
 
-export interface RegisterPluginRequest {
-  name: string;
+// Request envelope synced from the OpenAPI spec, with `source` narrowed to our PluginSource
+// union and `version` kept optional (the backend supplies its default).
+export type SkillRegisterRequest = Omit<components["schemas"]["RegisterPluginRequest"], "source" | "version"> & {
   source: PluginSource;
   version?: string;
-  description?: string;
-  author?: PluginAuthor;
-  homepage?: string;
-  keywords?: string[];
-  category?: string;
-  domain?: string;
-  namespace?: string;
-}
-
-export interface RegisterPluginResponse {
-  plugin: Plugin;
-  action: "created" | "updated";
-  message: string;
-}
+};
 
 // Public marketplace types
 export interface MarketplacePluginEntry {
@@ -104,20 +93,3 @@ export interface CategoryTab {
   label: string;
   count: number;
 }
-
-export interface PluginFormData {
-  name: string;
-  sourceType: "github" | "url" | "git-subdir";
-  repo: string;
-  url: string;
-  path: string;
-  version: string;
-  description: string;
-  authorName: string;
-  authorEmail: string;
-  homepage: string;
-  category: string;
-  keywords: string; // Comma-separated string, will be split into array
-  domain: string;
-  namespace: string;
-}
diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx
index f5bae832e64..0bbf4d2a6e8 100644
--- a/ui/litellm-dashboard/src/components/networking.tsx
+++ b/ui/litellm-dashboard/src/components/networking.tsx
@@ -27,6 +27,7 @@ import { TagNewRequest, TagUpdateRequest, TagListResponse, TagInfoResponse } fro
 import { Team } from "./key_team_helpers/key_list";
 import { UserInfo } from "./view_users/types";
 import { EmailEventSettingsResponse, EmailEventSettingsUpdateRequest } from "./email_events/types";
+import type { SkillRegisterRequest } from "./claude_code_plugins/types";
 import { jsonFields } from "./common_components/check_openapi_schema";
 import NotificationsManager from "./molecules/notifications_manager";
 import type { MCPUserEnvVarsStatus } from "./mcp_tools/types";
@@ -7402,19 +7403,7 @@ export const getClaudeCodePluginDetails = async (accessToken: string, pluginName
  * @param accessToken - Admin access token
  * @param pluginData - Plugin registration data
  */
-export const registerClaudeCodePlugin = async (
-  accessToken: string,
-  pluginData: {
-    name: string;
-    source: { source: string; repo?: string; url?: string };
-    version?: string;
-    description?: string;
-    author?: { name: string; email?: string };
-    homepage?: string;
-    keywords?: string[];
-    category?: string;
-  },
-) => {
+export const registerClaudeCodePlugin = async (accessToken: string, pluginData: SkillRegisterRequest) => {
   try {
     const proxyBaseUrl = getProxyBaseUrl();
     const url = proxyBaseUrl ? `${proxyBaseUrl}/claude-code/plugins` : `/claude-code/plugins`;
@@ -7429,8 +7418,13 @@ export const registerClaudeCodePlugin = async (
     });
 
     if (!response.ok) {
-      const errorData = await response.text();
-      const errorMessage = deriveErrorMessage(JSON.parse(errorData));
+      const errorBody = await response.text();
+      let errorMessage: string;
+      try {
+        errorMessage = deriveErrorMessage(JSON.parse(errorBody));
+      } catch {
+        errorMessage = errorBody || `Request failed with status ${response.status}`;
+      }
       handleError(errorMessage);
       throw new Error(errorMessage);
     }

From 6ab3742fa673467433070c926986aa2600f45928 Mon Sep 17 00:00:00 2001
From: Yassin Kortam <yassin@berri.ai>
Date: Tue, 30 Jun 2026 20:31:02 +0300
Subject: [PATCH 13/51] perf(spend): move cost-callback payload deepcopy off
 the request event loop (#31579)

---
 litellm/proxy/db/db_spend_update_writer.py    |  20 ++-
 .../proxy/db/test_db_spend_update_writer.py   | 118 +++++++++++++++++-
 2 files changed, 124 insertions(+), 14 deletions(-)

diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py
index cf4c3e98f00..ca6875ca800 100644
--- a/litellm/proxy/db/db_spend_update_writer.py
+++ b/litellm/proxy/db/db_spend_update_writer.py
@@ -175,16 +175,9 @@ class DBSpendUpdateWriter:
             if team_id is not None and team_id != "":
                 payload["team_id"] = team_id
 
-            # One deepcopy shared by all 6 daily spend helpers (was 5, fixes agent bug)
-            payload_copy = copy.deepcopy(payload)
-
-            # Deepcopy request_tags for _update_tag_db
-            request_tags = copy.deepcopy(payload.get("request_tags"))
-
-            # Keep _insert_spend_log_to_db awaited inline (not a task, preserve current behavior)
             if disable_spend_logs is False:
                 await self._insert_spend_log_to_db(
-                    payload=copy.deepcopy(payload),
+                    payload=payload,
                     prisma_client=prisma_client,
                 )
             else:
@@ -204,8 +197,7 @@ class DBSpendUpdateWriter:
                     prisma_client=prisma_client,
                     user_api_key_cache=user_api_key_cache,
                     litellm_proxy_budget_name=litellm_proxy_budget_name,
-                    payload_copy=payload_copy,
-                    request_tags=request_tags,
+                    payload=payload,
                 )
             )
 
@@ -336,14 +328,18 @@ class DBSpendUpdateWriter:
         prisma_client: Optional[PrismaClient],
         user_api_key_cache: DualCache,
         litellm_proxy_budget_name: Optional[str],
-        payload_copy: SpendLogsPayload,
-        request_tags: Optional[Any],
+        payload: SpendLogsPayload,
     ):
         """
         Runs all 11 spend-update helpers sequentially inside a single asyncio task.
 
         Each helper is wrapped in try/except so one failure doesn't prevent the others.
+
+        The deepcopy runs here, off the awaited request path, so the daily spend
+        helpers get a payload isolated from the spend-log queue entry and the caller.
         """
+        payload_copy = copy.deepcopy(payload)
+        request_tags = payload_copy.get("request_tags")
         try:
             await self._update_user_db(
                 response_cost=response_cost,
diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py
index 04c93f48ca9..c29cdaf4171 100644
--- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py
+++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py
@@ -1,4 +1,5 @@
 import asyncio
+import copy
 import json
 import os
 import sys
@@ -1419,8 +1420,7 @@ async def test_batch_database_updates_isolation_on_failure():
         prisma_client=MagicMock(),
         user_api_key_cache=MagicMock(),
         litellm_proxy_budget_name="budget",
-        payload_copy={"key": "value"},
-        request_tags=None,
+        payload={"key": "value"},
     )
 
     # _update_key_db raised, but all others should still have been called
@@ -1704,3 +1704,117 @@ async def test_commit_spend_updates_iterates_in_sorted_order(
     )
 
     assert captured_where_values == expected_order
+
+
+@pytest.mark.asyncio
+async def test_update_database_does_not_deepcopy_on_request_path():
+    """
+    Regression for LIT-4088: copy.deepcopy must not run while the caller awaits
+    update_database(). The deepcopy used to isolate the daily-spend helpers is
+    relocated into the _batch_database_updates background task, and the spend-log
+    insert receives the payload directly (all consumers are read-only).
+
+    Asserts:
+    - zero copy.deepcopy calls happen on the awaited request path
+    - the batch background task still hands the daily helpers an isolated copy
+      (mutating the original after the task ran does not bleed into it)
+    - the spend-log insert receives the payload on the request path with the
+      correct content
+    """
+    db_writer = DBSpendUpdateWriter()
+
+    captured_batch_payloads = []
+    captured_spend_log = {}
+
+    async def capture_batch_payload(**kwargs):
+        captured_batch_payloads.append(kwargs.get("payload"))
+
+    async def capture_spend_log(**kwargs):
+        payload = kwargs.get("payload")
+        captured_spend_log["ref"] = payload
+        captured_spend_log["model_at_call"] = payload["model"]
+
+    db_writer._insert_spend_log_to_db = AsyncMock(side_effect=capture_spend_log)
+    db_writer._update_user_db = AsyncMock()
+    db_writer._update_key_db = AsyncMock()
+    db_writer._update_team_db = AsyncMock()
+    db_writer._update_org_db = AsyncMock()
+    db_writer._update_tag_db = AsyncMock()
+    db_writer._update_agent_db = AsyncMock()
+    db_writer.add_spend_log_transaction_to_daily_user_transaction = AsyncMock(
+        side_effect=capture_batch_payload
+    )
+    db_writer.add_spend_log_transaction_to_daily_end_user_transaction = AsyncMock()
+    db_writer.add_spend_log_transaction_to_daily_agent_transaction = AsyncMock()
+    db_writer.add_spend_log_transaction_to_daily_team_transaction = AsyncMock()
+    db_writer.add_spend_log_transaction_to_daily_org_transaction = AsyncMock()
+    db_writer.add_spend_log_transaction_to_daily_tag_transaction = AsyncMock()
+
+    fake_payload = {
+        "startTime": "2024-01-01T00:00:00",
+        "endTime": "2024-01-01T00:01:00",
+        "model": "gpt-4",
+        "custom_llm_provider": "openai",
+        "request_tags": '["prod-tag"]',
+        "spend": 0.0,
+        "nested": {"a": 1},
+    }
+
+    deepcopy_calls = []
+    real_deepcopy = copy.deepcopy
+
+    def counting_deepcopy(obj, *args, **kwargs):
+        deepcopy_calls.append(obj)
+        return real_deepcopy(obj, *args, **kwargs)
+
+    with (
+        patch("litellm.proxy.proxy_server.disable_spend_logs", False),
+        patch("litellm.proxy.proxy_server.prisma_client", MagicMock()),
+        patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()),
+        patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"),
+        patch(
+            "litellm.proxy.spend_tracking.spend_tracking_utils.get_logging_payload",
+            return_value=fake_payload,
+        ),
+        patch(
+            "litellm.proxy.db.db_spend_update_writer.copy.deepcopy",
+            counting_deepcopy,
+        ),
+    ):
+        await db_writer.update_database(
+            token="test-token",
+            user_id="test-user",
+            end_user_id="test-end-user",
+            team_id="test-team",
+            org_id="test-org",
+            kwargs={"model": "gpt-4", "custom_llm_provider": "openai"},
+            completion_response=MagicMock(),
+            start_time=datetime.now(),
+            end_time=datetime.now(),
+            response_cost=0.1,
+        )
+
+        # Request path is clean: nothing was deepcopied while the caller awaited.
+        assert len(deepcopy_calls) == 0
+
+        # The spend-log insert ran inline on the request path with the real payload.
+        assert captured_spend_log["ref"] is fake_payload
+        assert captured_spend_log["model_at_call"] == "gpt-4"
+        assert fake_payload["spend"] == 0.1
+
+        # Now let the batch background task run; the deepcopy happens here.
+        await asyncio.sleep(0)
+
+    assert len(deepcopy_calls) >= 1
+    assert len(captured_batch_payloads) == 1
+    batch_payload = captured_batch_payloads[0]
+    assert batch_payload is not fake_payload
+    assert batch_payload["model"] == "gpt-4"
+    assert batch_payload["spend"] == 0.1
+
+    # Mutating the original after the batch task captured its snapshot must not
+    # leak into the daily helper's isolated copy.
+    fake_payload["model"] = "MUTATED"
+    fake_payload["nested"]["a"] = 999
+    assert batch_payload["model"] == "gpt-4"
+    assert batch_payload["nested"]["a"] == 1

From 59f51b2d72c0a9eefc06e9825ecff908f9dfdfb7 Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 30 Jun 2026 10:46:47 -0700
Subject: [PATCH 14/51] chore: prevent CLAUDE.md comment bloat (#31729)

The existing comment rule is not strict enough
---
 CLAUDE.md | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/CLAUDE.md b/CLAUDE.md
index cea38b8527b..0cd1605b1b2 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -1,8 +1,7 @@
-Do not write comments unless they are absolutely necessary to explain some very complex business logic. Please clean up if there are comments that are not absolutely necessary. Do not remove comments that are unrelated to the addition of the code of this PR
-
-Explanation: code comments are, in a way, a violation of DRY code. You must update logic in two locations to change the code and "hard to change" is literally the definition of tech debt. We should instead aim to write code that is intuitive to the reader, while being both easy to maintain and high performance
+Do not write any comments (existing comments can stay) unless explicitly asked to in a user (not system) prompt
 
 Don't assume that the existing code is correct or the right way of doing things / good coding patterns. In fact, there are a lot of bad coding practices, overly complex code, code smells, etc. If something doesn't look right, speak up. Feel free to break existing patterns or question weird existing code to make new code high quality, as in:
+
 - correct
 - secure
 - performant

From 1815636e1ce165096f45eceed5b968db01639402 Mon Sep 17 00:00:00 2001
From: yucheng-berri <yucheng@berri.ai>
Date: Tue, 30 Jun 2026 10:58:22 -0700
Subject: [PATCH 15/51] feat(guardrails): expose streaming knobs on
 generic_guardrail_api (#31730)

* feat(guardrails): expose streaming knobs on generic_guardrail_api

Wire streaming_end_of_stream_only and streaming_sampling_rate through
optional params, initialize_guardrail, and get_config_model so the
generic guardrail API participates in UnifiedLLMGuardrails streaming
checks with configurable cadence and end-of-stream-only mode.

* fix(guardrails): use builtin type[] in get_config_model return

Avoids a new UP006 violation that tripped the ruff strict-rule budget
gate on the PR lint job.

* fix(guardrails): default optional streaming knobs to None

Non-None Pydantic defaults on GenericGuardrailAPIOptionalParams made
_get_config_value treat unset nested fields as explicit values, which
shadowed top-level litellm_params streaming flags whenever any other
optional_params key was present. Real defaults stay in the constructor.

* fix(guardrails): address review nits on generic_guardrail_api streaming

Validate streaming_sampling_rate >= 1 in the constructor and Pydantic
optional_params (ge=1), and add /v1/responses streaming coverage through
the unified post-call hook so Responses API usage is exercised alongside
chat completions.

* fix(guardrails): read nested streaming config from dict optional_params

Guardrail API/UI delivers optional_params as a plain dict, so getattr was
silently ignoring streaming_sampling_rate and streaming_end_of_stream_only.
Handle both dict and model shapes in _get_config_value with regression tests.

* fix(guardrails): clear ruff findings in generic_guardrail_api tests/types

* style(guardrails): ruff format generic_guardrail_api modules

---------

Co-authored-by: Marton Schneider <marton@schneider.co.nl>
---
 .../generic_guardrail_api/__init__.py         |  18 +-
 .../generic_guardrail_api.py                  |  20 +
 .../guardrail_hooks/generic_guardrail_api.py  |  26 +-
 .../test_generic_guardrail_api.py             | 716 +++++++++++++++++-
 4 files changed, 775 insertions(+), 5 deletions(-)

diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py
index 2386f80e819..63ead52baa6 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py
@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Optional
 
 from litellm.types.guardrails import SupportedGuardrailIntegrations
 
@@ -8,9 +8,23 @@ if TYPE_CHECKING:
     from litellm.types.guardrails import Guardrail, LitellmParams
 
 
+def _get_config_value(litellm_params: Any, optional_params: Any, attribute_name: str) -> Optional[Any]:
+    if optional_params is not None:
+        value = (
+            optional_params.get(attribute_name)
+            if isinstance(optional_params, dict)
+            else getattr(optional_params, attribute_name, None)
+        )
+        if value is not None:
+            return value
+    return getattr(litellm_params, attribute_name, None)
+
+
 def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"):
     import litellm
 
+    optional_params = getattr(litellm_params, "optional_params", None)
+
     _generic_guardrail_api_callback = GenericGuardrailAPI(
         api_base=litellm_params.api_base,
         api_key=litellm_params.api_key,
@@ -22,6 +36,8 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
         guardrail_name=guardrail.get("guardrail_name", ""),
         event_hook=litellm_params.mode,
         default_on=litellm_params.default_on,
+        streaming_end_of_stream_only=_get_config_value(litellm_params, optional_params, "streaming_end_of_stream_only"),
+        streaming_sampling_rate=_get_config_value(litellm_params, optional_params, "streaming_sampling_rate"),
     )
 
     litellm.logging_callback_manager.add_litellm_callback(_generic_guardrail_api_callback)
diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py
index df80ea09de0..dc519f56d1a 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py
@@ -33,6 +33,7 @@ from litellm.types.utils import GenericGuardrailAPIInputs
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+    from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
 
 GUARDRAIL_NAME = "generic_guardrail_api"
 
@@ -178,6 +179,8 @@ class GenericGuardrailAPI(CustomGuardrail):
         unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_closed",
         fail_on_error: Optional[bool] = True,
         extra_headers: Optional[list] = None,
+        streaming_end_of_stream_only: Optional[bool] = None,
+        streaming_sampling_rate: Optional[int] = None,
         **kwargs,
     ):
         self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback)
@@ -209,6 +212,15 @@ class GenericGuardrailAPI(CustomGuardrail):
 
         self.fail_on_error: bool = True if fail_on_error is None else fail_on_error
 
+        # Read by UnifiedLLMGuardrails.async_post_call_streaming_iterator_hook
+        # via getattr(guardrail_to_apply, "streaming_*", default).
+        self.streaming_end_of_stream_only: bool = (
+            False if streaming_end_of_stream_only is None else streaming_end_of_stream_only
+        )
+        if streaming_sampling_rate is not None and streaming_sampling_rate < 1:
+            raise ValueError(f"streaming_sampling_rate must be >= 1 (got {streaming_sampling_rate})")
+        self.streaming_sampling_rate: int = 5 if streaming_sampling_rate is None else streaming_sampling_rate
+
         # Set supported event hooks
         if "supported_event_hooks" not in kwargs:
             kwargs["supported_event_hooks"] = [
@@ -470,3 +482,11 @@ class GenericGuardrailAPI(CustomGuardrail):
             return self._handle_guardrail_request_error(e, inputs, input_type, logging_obj)
         except Exception as e:
             return self._handle_guardrail_request_error(e, inputs, input_type, logging_obj, is_unreachable=False)
+
+    @staticmethod
+    def get_config_model() -> Optional[type["GuardrailConfigModel"]]:
+        from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
+            GenericGuardrailAPIConfigModel,
+        )
+
+        return GenericGuardrailAPIConfigModel
diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py
index 28fb482b3af..d0ac8bb8998 100644
--- a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py
+++ b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py
@@ -1,7 +1,7 @@
 from typing import Any, Dict, List, Literal, Optional, Union
 
 from pydantic import BaseModel, ConfigDict, Field
-from typing_extensions import TYPE_CHECKING, TypedDict
+from typing_extensions import TypedDict
 
 from litellm.types.llms.openai import (
     AllMessageValues,
@@ -60,6 +60,30 @@ class GenericGuardrailAPIOptionalParams(BaseModel):
         ),
     )
 
+    streaming_end_of_stream_only: Optional[bool] = Field(
+        default=None,
+        description=(
+            "If False (default when unset), the guardrail runs on sampled chunks during "
+            "the stream at the cadence set by streaming_sampling_rate, and an in-flight "
+            "BLOCKED stops further chunks from streaming. If True, the guardrail runs "
+            "once at end of stream over the assembled response; lower cost and latency, "
+            "but flagged content has already streamed to the client before the terminal "
+            "block. Defaults are applied in GenericGuardrailAPI.__init__ when None so "
+            "unset optional_params does not shadow top-level litellm_params."
+        ),
+    )
+
+    streaming_sampling_rate: Optional[int] = Field(
+        default=None,
+        ge=1,
+        description=(
+            "When streaming_end_of_stream_only is False, the guardrail runs every Nth "
+            "streamed chunk. Ignored when streaming_end_of_stream_only is True. "
+            "Must be >= 1 when set. Defaults to 5 in GenericGuardrailAPI.__init__ "
+            "when None so unset optional_params does not shadow top-level litellm_params."
+        ),
+    )
+
 
 class GenericGuardrailAPIConfigModel(
     GuardrailConfigModel[GenericGuardrailAPIOptionalParams],
diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py
index 399442a5f71..791fdd4077c 100644
--- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py
+++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py
@@ -609,7 +609,6 @@ class TestImageSupport:
                 request_data=mock_request_data_input,
                 input_type="request",
             )
-            result_texts = guardrailed_inputs.get("texts", [])
             result_images = guardrailed_inputs.get("images", None)
 
             # Verify API was called with images
@@ -943,7 +942,7 @@ class TestMultimodalSupport:
             guardrail.async_handler, "post", return_value=mock_response
         ) as mock_post:
             # This should not raise SerializationIterator error
-            result = await guardrail.apply_guardrail(
+            await guardrail.apply_guardrail(
                 inputs={
                     "texts": ["What's in this image?"],
                     "images": ["https://example.com/image.jpg"],
@@ -1006,7 +1005,7 @@ class TestMultimodalSupport:
         with patch.object(
             guardrail.async_handler, "post", return_value=mock_response
         ) as mock_post:
-            result = await guardrail.apply_guardrail(
+            await guardrail.apply_guardrail(
                 inputs={
                     "texts": ["Hello", "World"],
                     "structured_messages": messages_with_iterable,
@@ -1023,6 +1022,717 @@ class TestMultimodalSupport:
             assert isinstance(json_payload["structured_messages"], list)
 
 
+def _make_stream_chunk(content: str, finish_reason=None):
+    """Build a real ModelResponseStream so the handler's isinstance checks pass."""
+    from litellm.types.utils import Delta, ModelResponseStream
+
+    return ModelResponseStream(
+        model="gpt-4",
+        choices=[
+            litellm.StreamingChoices(
+                index=0,
+                delta=Delta(role="assistant", content=content),
+                finish_reason=finish_reason,
+            )
+        ],
+    )
+
+
+def _make_assembled_model_response(content: str) -> ModelResponse:
+    return ModelResponse(
+        id="mock-response",
+        model="gpt-4",
+        choices=[
+            litellm.Choices(
+                index=0,
+                message=litellm.Message(role="assistant", content=content),
+                finish_reason="stop",
+            )
+        ],
+    )
+
+
+def _mock_guardrail_post_response(action: str = "NONE", texts=None, blocked_reason=None):
+    mock_response = MagicMock()
+    payload = {"action": action}
+    if texts is not None:
+        payload["texts"] = texts
+    if blocked_reason is not None:
+        payload["blocked_reason"] = blocked_reason
+    mock_response.json.return_value = payload
+    mock_response.raise_for_status = MagicMock()
+    return mock_response
+
+
+def _make_responses_stream_events(text: str):
+    """Minimal /v1/responses SSE event sequence ending in response.completed."""
+    return (
+        {"type": "response.created", "response": {"id": "resp_test"}},
+        {
+            "type": "response.output_item.added",
+            "item": {"type": "message", "id": "msg_test"},
+        },
+        {
+            "type": "response.content_part.added",
+            "part": {"type": "output_text", "text": ""},
+        },
+        {"type": "response.output_text.delta", "delta": text},
+        {
+            "type": "response.output_text.done",
+            "text": text,
+        },
+        {
+            "type": "response.completed",
+            "response": {
+                "id": "resp_test",
+                "output": [
+                    {
+                        "type": "message",
+                        "id": "msg_test",
+                        "status": "completed",
+                        "role": "assistant",
+                        "content": [{"type": "output_text", "text": text}],
+                    }
+                ],
+                "status": "completed",
+            },
+        },
+    )
+
+
+class TestGenericGuardrailAPIStreamingConfig:
+    """Streaming knobs on GenericGuardrailAPI and initialize_guardrail plumbing."""
+
+    def test_streaming_defaults(self):
+        guardrail = GenericGuardrailAPI(
+            api_base="https://api.test.guardrail.com",
+            guardrail_name="test-generic-guardrail",
+            event_hook="post_call",
+        )
+        assert guardrail.streaming_end_of_stream_only is False
+        assert guardrail.streaming_sampling_rate == 5
+
+    def test_streaming_overrides(self):
+        guardrail = GenericGuardrailAPI(
+            api_base="https://api.test.guardrail.com",
+            guardrail_name="test-generic-guardrail",
+            event_hook="post_call",
+            streaming_end_of_stream_only=True,
+            streaming_sampling_rate=2,
+        )
+        assert guardrail.streaming_end_of_stream_only is True
+        assert guardrail.streaming_sampling_rate == 2
+
+    @pytest.mark.parametrize("invalid_rate", [0, -1, -5])
+    def test_streaming_sampling_rate_rejects_non_positive(self, invalid_rate):
+        with pytest.raises(ValueError, match="streaming_sampling_rate must be >= 1"):
+            GenericGuardrailAPI(
+                api_base="https://api.test.guardrail.com",
+                guardrail_name="test-generic-guardrail",
+                event_hook="post_call",
+                streaming_sampling_rate=invalid_rate,
+            )
+
+    def test_optional_params_streaming_sampling_rate_ge_one(self):
+        from pydantic import ValidationError
+
+        from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
+            GenericGuardrailAPIOptionalParams,
+        )
+
+        with pytest.raises(ValidationError):
+            GenericGuardrailAPIOptionalParams(streaming_sampling_rate=0)
+
+    def test_get_config_model(self):
+        from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
+            GenericGuardrailAPIConfigModel,
+        )
+
+        assert GenericGuardrailAPI.get_config_model() is GenericGuardrailAPIConfigModel
+
+    def test_initialize_guardrail_forwards_streaming_flags(self):
+        from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
+            initialize_guardrail,
+        )
+        from litellm.types.guardrails import LitellmParams
+
+        litellm_params = LitellmParams(
+            guardrail="generic_guardrail_api",
+            mode="post_call",
+            api_base="https://api.test.guardrail.com",
+            default_on=False,
+        )
+        # LitellmParams uses extra="allow" on the base; set streaming knobs dynamically
+        litellm_params.streaming_end_of_stream_only = False  # type: ignore[attr-defined]
+        litellm_params.streaming_sampling_rate = 3  # type: ignore[attr-defined]
+
+        guardrail_config = {"guardrail_name": "test-generic-streaming"}
+
+        with patch(
+            "litellm.logging_callback_manager.add_litellm_callback"
+        ):
+            guardrail = initialize_guardrail(litellm_params, guardrail_config)
+
+        assert guardrail.streaming_end_of_stream_only is False
+        assert guardrail.streaming_sampling_rate == 3
+
+    def test_initialize_guardrail_optional_params_defaults_do_not_shadow_top_level(
+        self,
+    ):
+        """Top-level streaming knobs win when optional_params only carries siblings."""
+        from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
+            initialize_guardrail,
+        )
+        from litellm.types.guardrails import LitellmParams
+        from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
+            GenericGuardrailAPIOptionalParams,
+        )
+
+        litellm_params = LitellmParams(
+            guardrail="generic_guardrail_api",
+            mode="post_call",
+            api_base="https://api.test.guardrail.com",
+            default_on=False,
+        )
+        litellm_params.streaming_end_of_stream_only = True  # type: ignore[attr-defined]
+        litellm_params.streaming_sampling_rate = 2  # type: ignore[attr-defined]
+        # Sibling optional_params only; streaming fields stay at Pydantic default None.
+        litellm_params.optional_params = GenericGuardrailAPIOptionalParams(  # type: ignore[attr-defined]
+            additional_provider_specific_params={"tenant": "acme"},
+        )
+
+        guardrail_config = {"guardrail_name": "test-generic-streaming-mixed"}
+
+        with patch(
+            "litellm.logging_callback_manager.add_litellm_callback"
+        ):
+            guardrail = initialize_guardrail(litellm_params, guardrail_config)
+
+        assert guardrail.streaming_end_of_stream_only is True
+        assert guardrail.streaming_sampling_rate == 2
+
+    def test_initialize_guardrail_explicit_optional_params_streaming_wins(self):
+        from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
+            initialize_guardrail,
+        )
+        from litellm.types.guardrails import LitellmParams
+        from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
+            GenericGuardrailAPIOptionalParams,
+        )
+
+        litellm_params = LitellmParams(
+            guardrail="generic_guardrail_api",
+            mode="post_call",
+            api_base="https://api.test.guardrail.com",
+            default_on=False,
+        )
+        litellm_params.streaming_end_of_stream_only = False  # type: ignore[attr-defined]
+        litellm_params.streaming_sampling_rate = 9  # type: ignore[attr-defined]
+        litellm_params.optional_params = GenericGuardrailAPIOptionalParams(  # type: ignore[attr-defined]
+            streaming_end_of_stream_only=True,
+            streaming_sampling_rate=1,
+        )
+
+        guardrail_config = {"guardrail_name": "test-generic-streaming-nested-wins"}
+
+        with patch(
+            "litellm.logging_callback_manager.add_litellm_callback"
+        ):
+            guardrail = initialize_guardrail(litellm_params, guardrail_config)
+
+        assert guardrail.streaming_end_of_stream_only is True
+        assert guardrail.streaming_sampling_rate == 1
+
+    def test_initialize_guardrail_dict_optional_params_streaming_wins(self):
+        """Guardrail API/UI delivers optional_params as a plain dict, not a model."""
+        from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
+            initialize_guardrail,
+        )
+        from litellm.types.guardrails import LitellmParams
+
+        litellm_params = LitellmParams(
+            guardrail="generic_guardrail_api",
+            mode="post_call",
+            api_base="https://api.test.guardrail.com",
+            default_on=False,
+        )
+        litellm_params.streaming_end_of_stream_only = False  # type: ignore[attr-defined]
+        litellm_params.streaming_sampling_rate = 9  # type: ignore[attr-defined]
+        # Plain dict mirrors how configs arrive from the guardrail API/UI.
+        litellm_params.optional_params = {  # type: ignore[attr-defined]
+            "streaming_end_of_stream_only": True,
+            "streaming_sampling_rate": 1,
+        }
+
+        guardrail_config = {"guardrail_name": "test-generic-streaming-dict-optional"}
+
+        with patch(
+            "litellm.logging_callback_manager.add_litellm_callback"
+        ):
+            guardrail = initialize_guardrail(litellm_params, guardrail_config)
+
+        assert guardrail.streaming_end_of_stream_only is True
+        assert guardrail.streaming_sampling_rate == 1
+
+    def test_initialize_guardrail_dict_optional_params_sibling_only_falls_through(
+        self,
+    ):
+        """Dict optional_params without streaming keys must not shadow top-level knobs."""
+        from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
+            initialize_guardrail,
+        )
+        from litellm.types.guardrails import LitellmParams
+
+        litellm_params = LitellmParams(
+            guardrail="generic_guardrail_api",
+            mode="post_call",
+            api_base="https://api.test.guardrail.com",
+            default_on=False,
+        )
+        litellm_params.streaming_end_of_stream_only = True  # type: ignore[attr-defined]
+        litellm_params.streaming_sampling_rate = 2  # type: ignore[attr-defined]
+        litellm_params.optional_params = {  # type: ignore[attr-defined]
+            "additional_provider_specific_params": {"tenant": "acme"},
+        }
+
+        guardrail_config = {"guardrail_name": "test-generic-streaming-dict-sibling"}
+
+        with patch(
+            "litellm.logging_callback_manager.add_litellm_callback"
+        ):
+            guardrail = initialize_guardrail(litellm_params, guardrail_config)
+
+        assert guardrail.streaming_end_of_stream_only is True
+        assert guardrail.streaming_sampling_rate == 2
+
+
+class TestGenericGuardrailAPIStreamingViaUnified:
+    """Streaming output checks routed through UnifiedLLMGuardrails."""
+
+    @pytest.mark.asyncio
+    async def test_streaming_safe_content_yields_all_chunks(self):
+        from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
+            UnifiedLLMGuardrails,
+        )
+
+        guardrail = GenericGuardrailAPI(
+            api_base="https://api.test.guardrail.com",
+            guardrail_name="test-generic-guardrail",
+            event_hook="post_call",
+        )
+        unified_guardrail = UnifiedLLMGuardrails()
+
+        async def mock_stream():
+            chunks_data = ["Hello", " ", "world", "!", " Goodbye"]
+            for i, content in enumerate(chunks_data):
+                yield _make_stream_chunk(
+                    content,
+                    finish_reason="stop" if i == len(chunks_data) - 1 else None,
+                )
+
+        mock_post = AsyncMock(
+            return_value=_mock_guardrail_post_response(
+                action="NONE", texts=["Hello world! Goodbye"]
+            )
+        )
+
+        with (
+            patch.object(guardrail.async_handler, "post", mock_post),
+            patch(
+                "litellm.llms.openai.chat.guardrail_translation.handler.stream_chunk_builder",
+                return_value=_make_assembled_model_response("Hello world! Goodbye"),
+            ),
+        ):
+            user_api_key_dict = UserAPIKeyAuth(
+                api_key="test", request_route="/chat/completions"
+            )
+            request_data = {
+                "messages": [{"role": "user", "content": "hi"}],
+                "guardrail_to_apply": guardrail,
+                "metadata": {"guardrails": ["test-generic-guardrail"]},
+            }
+
+            chunks_received = 0
+            async for _ in unified_guardrail.async_post_call_streaming_iterator_hook(
+                user_api_key_dict=user_api_key_dict,
+                response=mock_stream(),
+                request_data=request_data,
+            ):
+                chunks_received += 1
+
+        assert chunks_received == 5
+        assert mock_post.await_count >= 1
+
+    @pytest.mark.asyncio
+    async def test_streaming_blocked_content_raises(self):
+        from litellm.exceptions import GuardrailRaisedException
+        from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
+            UnifiedLLMGuardrails,
+        )
+
+        guardrail = GenericGuardrailAPI(
+            api_base="https://api.test.guardrail.com",
+            guardrail_name="test-generic-guardrail",
+            event_hook="post_call",
+            streaming_sampling_rate=1,
+        )
+        unified_guardrail = UnifiedLLMGuardrails()
+
+        async def mock_stream():
+            chunks_data = ["Hello", " ishaan", " here"]
+            for i, content in enumerate(chunks_data):
+                yield _make_stream_chunk(
+                    content,
+                    finish_reason="stop" if i == len(chunks_data) - 1 else None,
+                )
+
+        mock_post = AsyncMock(
+            return_value=_mock_guardrail_post_response(
+                action="BLOCKED", blocked_reason="Ishaan is not allowed"
+            )
+        )
+
+        with (
+            patch.object(guardrail.async_handler, "post", mock_post),
+            patch(
+                "litellm.llms.openai.chat.guardrail_translation.handler.stream_chunk_builder",
+                return_value=_make_assembled_model_response("Hello ishaan here"),
+            ),
+        ):
+            user_api_key_dict = UserAPIKeyAuth(
+                api_key="test", request_route="/chat/completions"
+            )
+            request_data = {
+                "messages": [{"role": "user", "content": "hi"}],
+                "guardrail_to_apply": guardrail,
+                "metadata": {"guardrails": ["test-generic-guardrail"]},
+            }
+
+            with pytest.raises(GuardrailRaisedException) as exc_info:
+                async for _ in unified_guardrail.async_post_call_streaming_iterator_hook(
+                    user_api_key_dict=user_api_key_dict,
+                    response=mock_stream(),
+                    request_data=request_data,
+                ):
+                    pass
+
+        assert "Ishaan is not allowed" in str(exc_info.value)
+
+    @pytest.mark.asyncio
+    async def test_streaming_default_uses_sampled_cadence(self):
+        """Default samples every 5th chunk + final pass: 10 chunks → calls at 5, 10, and final = 3."""
+        from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
+            UnifiedLLMGuardrails,
+        )
+
+        guardrail = GenericGuardrailAPI(
+            api_base="https://api.test.guardrail.com",
+            guardrail_name="test-generic-guardrail",
+            event_hook="post_call",
+        )
+        unified_guardrail = UnifiedLLMGuardrails()
+
+        async def mock_stream():
+            chunks_data = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"]
+            for i, content in enumerate(chunks_data):
+                yield _make_stream_chunk(
+                    content,
+                    finish_reason="stop" if i == len(chunks_data) - 1 else None,
+                )
+
+        mock_post = AsyncMock(
+            return_value=_mock_guardrail_post_response(
+                action="NONE", texts=["ABCDEFGHIJ"]
+            )
+        )
+
+        with (
+            patch.object(guardrail.async_handler, "post", mock_post),
+            patch(
+                "litellm.llms.openai.chat.guardrail_translation.handler.stream_chunk_builder",
+                return_value=_make_assembled_model_response("ABCDEFGHIJ"),
+            ),
+        ):
+            user_api_key_dict = UserAPIKeyAuth(
+                api_key="test", request_route="/chat/completions"
+            )
+            request_data = {
+                "messages": [{"role": "user", "content": "hi"}],
+                "guardrail_to_apply": guardrail,
+                "metadata": {"guardrails": ["test-generic-guardrail"]},
+            }
+
+            async for _ in unified_guardrail.async_post_call_streaming_iterator_hook(
+                user_api_key_dict=user_api_key_dict,
+                response=mock_stream(),
+                request_data=request_data,
+            ):
+                pass
+
+        assert mock_post.await_count == 3, (
+            f"Expected 3 guardrail calls (2 sampled at chunks 5 / 10 + 1 final), "
+            f"got {mock_post.await_count}"
+        )
+        for call in mock_post.await_args_list:
+            assert call.kwargs["json"]["input_type"] == "response"
+
+    @pytest.mark.asyncio
+    async def test_streaming_end_of_stream_only_calls_guardrail_once(self):
+        from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
+            UnifiedLLMGuardrails,
+        )
+
+        guardrail = GenericGuardrailAPI(
+            api_base="https://api.test.guardrail.com",
+            guardrail_name="test-generic-guardrail",
+            event_hook="post_call",
+            streaming_end_of_stream_only=True,
+        )
+        unified_guardrail = UnifiedLLMGuardrails()
+
+        async def mock_stream():
+            chunks_data = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"]
+            for i, content in enumerate(chunks_data):
+                yield _make_stream_chunk(
+                    content,
+                    finish_reason="stop" if i == len(chunks_data) - 1 else None,
+                )
+
+        mock_post = AsyncMock(
+            return_value=_mock_guardrail_post_response(
+                action="NONE", texts=["ABCDEFGHIJ"]
+            )
+        )
+
+        with (
+            patch.object(guardrail.async_handler, "post", mock_post),
+            patch(
+                "litellm.llms.openai.chat.guardrail_translation.handler.stream_chunk_builder",
+                return_value=_make_assembled_model_response("ABCDEFGHIJ"),
+            ),
+        ):
+            user_api_key_dict = UserAPIKeyAuth(
+                api_key="test", request_route="/chat/completions"
+            )
+            request_data = {
+                "messages": [{"role": "user", "content": "hi"}],
+                "guardrail_to_apply": guardrail,
+                "metadata": {"guardrails": ["test-generic-guardrail"]},
+            }
+
+            async for _ in unified_guardrail.async_post_call_streaming_iterator_hook(
+                user_api_key_dict=user_api_key_dict,
+                response=mock_stream(),
+                request_data=request_data,
+            ):
+                pass
+
+        assert mock_post.await_count == 1, (
+            f"Expected exactly one guardrail call at end of stream, "
+            f"got {mock_post.await_count}"
+        )
+
+    @pytest.mark.asyncio
+    async def test_streaming_sampling_rate_override(self):
+        """sampling_rate=2 on 6 chunks → in-stream at 2,4,6 plus final = 4 calls."""
+        from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
+            UnifiedLLMGuardrails,
+        )
+
+        guardrail = GenericGuardrailAPI(
+            api_base="https://api.test.guardrail.com",
+            guardrail_name="test-generic-guardrail",
+            event_hook="post_call",
+            streaming_end_of_stream_only=False,
+            streaming_sampling_rate=2,
+        )
+        unified_guardrail = UnifiedLLMGuardrails()
+
+        async def mock_stream():
+            chunks_data = ["A", "B", "C", "D", "E", "F"]
+            for i, content in enumerate(chunks_data):
+                yield _make_stream_chunk(
+                    content,
+                    finish_reason="stop" if i == len(chunks_data) - 1 else None,
+                )
+
+        mock_post = AsyncMock(
+            return_value=_mock_guardrail_post_response(action="NONE", texts=["ABCDEF"])
+        )
+
+        with (
+            patch.object(guardrail.async_handler, "post", mock_post),
+            patch(
+                "litellm.llms.openai.chat.guardrail_translation.handler.stream_chunk_builder",
+                return_value=_make_assembled_model_response("ABCDEF"),
+            ),
+        ):
+            user_api_key_dict = UserAPIKeyAuth(
+                api_key="test", request_route="/chat/completions"
+            )
+            request_data = {
+                "messages": [{"role": "user", "content": "hi"}],
+                "guardrail_to_apply": guardrail,
+                "metadata": {"guardrails": ["test-generic-guardrail"]},
+            }
+
+            async for _ in unified_guardrail.async_post_call_streaming_iterator_hook(
+                user_api_key_dict=user_api_key_dict,
+                response=mock_stream(),
+                request_data=request_data,
+            ):
+                pass
+
+        assert mock_post.await_count == 4, (
+            f"Expected 4 guardrail calls (3 sampled + 1 final aggregate), "
+            f"got {mock_post.await_count}"
+        )
+
+    @pytest.mark.asyncio
+    async def test_streaming_fail_open_on_unreachable_continues_stream(self):
+        from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
+            UnifiedLLMGuardrails,
+        )
+
+        guardrail = GenericGuardrailAPI(
+            api_base="https://api.test.guardrail.com",
+            guardrail_name="test-generic-guardrail",
+            event_hook="post_call",
+            unreachable_fallback="fail_open",
+            streaming_end_of_stream_only=True,
+        )
+        unified_guardrail = UnifiedLLMGuardrails()
+
+        async def mock_stream():
+            for i, content in enumerate(["A", "B", "C"]):
+                yield _make_stream_chunk(
+                    content, finish_reason="stop" if i == 2 else None
+                )
+
+        mock_post = AsyncMock(side_effect=httpx.ConnectError("connection refused"))
+
+        with (
+            patch.object(guardrail.async_handler, "post", mock_post),
+            patch(
+                "litellm.llms.openai.chat.guardrail_translation.handler.stream_chunk_builder",
+                return_value=_make_assembled_model_response("ABC"),
+            ),
+        ):
+            user_api_key_dict = UserAPIKeyAuth(
+                api_key="test", request_route="/chat/completions"
+            )
+            request_data = {
+                "messages": [{"role": "user", "content": "hi"}],
+                "guardrail_to_apply": guardrail,
+                "metadata": {"guardrails": ["test-generic-guardrail"]},
+            }
+
+            chunks_received = 0
+            async for _ in unified_guardrail.async_post_call_streaming_iterator_hook(
+                user_api_key_dict=user_api_key_dict,
+                response=mock_stream(),
+                request_data=request_data,
+            ):
+                chunks_received += 1
+
+        assert chunks_received == 3
+
+    @pytest.mark.asyncio
+    async def test_responses_api_streaming_end_of_stream_only_calls_guardrail_once(self):
+        """/v1/responses path through unified hook; end-of-stream-only = one call."""
+        from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
+            UnifiedLLMGuardrails,
+        )
+
+        guardrail = GenericGuardrailAPI(
+            api_base="https://api.test.guardrail.com",
+            guardrail_name="test-generic-guardrail",
+            event_hook="post_call",
+            streaming_end_of_stream_only=True,
+        )
+        unified_guardrail = UnifiedLLMGuardrails()
+
+        async def mock_responses_stream():
+            for event in _make_responses_stream_events("Hello world"):
+                yield event
+
+        mock_post = AsyncMock(
+            return_value=_mock_guardrail_post_response(
+                action="NONE", texts=["Hello world"]
+            )
+        )
+
+        with patch.object(guardrail.async_handler, "post", mock_post):
+            user_api_key_dict = UserAPIKeyAuth(
+                api_key="test", request_route="/v1/responses"
+            )
+            request_data = {
+                "input": "hi",
+                "guardrail_to_apply": guardrail,
+                "metadata": {"guardrails": ["test-generic-guardrail"]},
+            }
+
+            events_received = 0
+            async for _ in unified_guardrail.async_post_call_streaming_iterator_hook(
+                user_api_key_dict=user_api_key_dict,
+                response=mock_responses_stream(),
+                request_data=request_data,
+            ):
+                events_received += 1
+
+        assert events_received == 6
+        assert mock_post.await_count == 1, (
+            f"Expected exactly one guardrail call at end of /v1/responses stream, "
+            f"got {mock_post.await_count}"
+        )
+        assert mock_post.await_args.kwargs["json"]["input_type"] == "response"
+
+    @pytest.mark.asyncio
+    async def test_responses_api_streaming_blocked_raises(self):
+        """Mid-stream BLOCKED on /v1/responses surfaces GuardrailRaisedException."""
+        from litellm.exceptions import GuardrailRaisedException
+        from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
+            UnifiedLLMGuardrails,
+        )
+
+        guardrail = GenericGuardrailAPI(
+            api_base="https://api.test.guardrail.com",
+            guardrail_name="test-generic-guardrail",
+            event_hook="post_call",
+            streaming_sampling_rate=1,
+        )
+        unified_guardrail = UnifiedLLMGuardrails()
+
+        async def mock_responses_stream():
+            for event in _make_responses_stream_events("blocked content"):
+                yield event
+
+        mock_post = AsyncMock(
+            return_value=_mock_guardrail_post_response(
+                action="BLOCKED", blocked_reason="Responses content not allowed"
+            )
+        )
+
+        with patch.object(guardrail.async_handler, "post", mock_post):
+            user_api_key_dict = UserAPIKeyAuth(
+                api_key="test", request_route="/v1/responses"
+            )
+            request_data = {
+                "input": "hi",
+                "guardrail_to_apply": guardrail,
+                "metadata": {"guardrails": ["test-generic-guardrail"]},
+            }
+
+            with pytest.raises(GuardrailRaisedException) as exc_info:
+                async for _ in unified_guardrail.async_post_call_streaming_iterator_hook(
+                    user_api_key_dict=user_api_key_dict,
+                    response=mock_responses_stream(),
+                    request_data=request_data,
+                ):
+                    pass
+
+        assert "Responses content not allowed" in str(exc_info.value)
+
 class TestToolSupport:
     """Test tool handling in guardrail requests"""
 

From fecaf5c9e525a50d1d49b4ce9df5ec1e5f699cfc Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 30 Jun 2026 11:00:30 -0700
Subject: [PATCH 16/51] feat(router): tag routing denylist support via ! prefix
 (#31728)

Adds `!` prefix negation to tag-based routing so callers can exclude deployments by exact tag value without enumerating every allowed alternative. `!provider:anthropic` removes all deployments tagged exactly `provider:anthropic` before routing, and positive and negation tags compose. Matching is exact literal membership (frozenset intersection), so there is no regex or ReDoS surface for client-supplied tags. Ban-only requests that carry only negation tags stay within the default pool, mirroring untagged-request semantics so callers can't use negation to escape it. Fallback chains keep working because get_deployments_for_tag runs on each routing hop

Copy of #31680; implementation credit to @deepanshululla

Co-authored-by: deepanshululla <15312873+deepanshululla@users.noreply.github.com>
---
 litellm/router_strategy/tag_based_routing.py  | 107 ++-
 .../test_router_tag_routing.py                | 662 ++++++++++++++++--
 2 files changed, 674 insertions(+), 95 deletions(-)

diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py
index 65e76ba909b..6ca4e1de322 100644
--- a/litellm/router_strategy/tag_based_routing.py
+++ b/litellm/router_strategy/tag_based_routing.py
@@ -7,7 +7,7 @@ Use this to route requests between Teams
 """
 
 import re
-from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union
+from typing import TYPE_CHECKING, Any, Literal, Optional, Union
 
 from litellm._logging import verbose_logger
 from litellm.types.router import RouterErrors
@@ -21,8 +21,8 @@ else:
 
 
 def _is_valid_deployment_tag_regex(
-    tag_regexes: List[str],
-    header_strings: List[str],
+    tag_regexes: list[str],
+    header_strings: list[str],
 ) -> Optional[str]:
     """
     Test compiled regex patterns against "Header-Name: value" strings.
@@ -43,7 +43,7 @@ def _is_valid_deployment_tag_regex(
     return None
 
 
-def is_valid_deployment_tag(deployment_tags: List[str], request_tags: List[str], match_any: bool = True) -> bool:
+def is_valid_deployment_tag(deployment_tags: list[str], request_tags: list[str], match_any: bool = True) -> bool:
     """
     Check if a tag is valid, the matching can be either any or all based on `match_any` flag
     """
@@ -71,10 +71,10 @@ def is_valid_deployment_tag(deployment_tags: List[str], request_tags: List[str],
 
 def _match_deployment(
     deployment: Any,
-    request_tags: Optional[List[str]],
-    header_strings: List[str],
+    request_tags: Optional[list[str]],
+    header_strings: list[str],
     match_any: bool,
-) -> Optional[Dict[str, str]]:
+) -> Optional[dict[str, str]]:
     """
     Determine whether *deployment* matches the current request.
 
@@ -87,8 +87,8 @@ def _match_deployment(
          ran and failed, so the regex cannot override strict-tag policy.
     """
     litellm_params = deployment.get("litellm_params", {})
-    deployment_tags: Optional[List[str]] = litellm_params.get("tags")
-    deployment_tag_regex: Optional[List[str]] = litellm_params.get("tag_regex")
+    deployment_tags: Optional[list[str]] = litellm_params.get("tags")
+    deployment_tag_regex: Optional[list[str]] = litellm_params.get("tag_regex")
 
     # 1. Exact tag match (existing behaviour).
     if deployment_tags and request_tags:
@@ -114,11 +114,46 @@ def _match_deployment(
     return None
 
 
+def _split_tags(tags: list[str]) -> tuple[list[str], list[str]]:
+    positive = [t for t in tags if not t.startswith("!")]
+    excluded = [tag[1:] for tag in tags if tag.startswith("!") and len(tag) > 1]
+    return positive, excluded
+
+
+def _exclude_deployments(
+    deployments: Union[list[Any], dict[Any, Any]],
+    excluded_set: frozenset[str],
+) -> list[Any]:
+    if not excluded_set:
+        return list(deployments)
+    return [d for d in deployments if not excluded_set.intersection(d.get("litellm_params", {}).get("tags") or [])]
+
+
+def _require_candidates(
+    candidates: list[Any],
+    model: str,
+    request_tags: Any,
+) -> list[Any]:
+    if not candidates:
+        raise ValueError(
+            f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model={model} and tags={request_tags}"
+        )
+    return candidates
+
+
+def _ban_only_base_pool(
+    deployments: Union[list[Any], dict[Any, Any]],
+) -> list[Any]:
+    # Mirrors untagged-request semantics so callers can't use !tags to escape the default pool.
+    defaults = [d for d in deployments if "default" in (d.get("litellm_params", {}).get("tags") or [])]
+    return defaults if defaults else list(deployments)
+
+
 async def get_deployments_for_tag(
     llm_router_instance: LitellmRouter,
     model: str,  # used to raise the correct error
-    healthy_deployments: Union[List[Any], Dict[Any, Any]],
-    request_kwargs: Optional[Dict[Any, Any]] = None,
+    healthy_deployments: Union[list[Any], dict[Any, Any]],
+    request_kwargs: Optional[dict[Any, Any]] = None,
     metadata_variable_name: Literal["metadata", "litellm_metadata"] = "metadata",
 ):
     """
@@ -136,13 +171,8 @@ async def get_deployments_for_tag(
         )
         return healthy_deployments
 
-    if healthy_deployments is None:
-        verbose_logger.debug("get_deployments_for_tag: healthy_deployments is None returning healthy_deployments")
-        return healthy_deployments
-
-    # Tag filtering applies only when there is at least one deployment to evaluate.
-    if isinstance(healthy_deployments, list) and len(healthy_deployments) == 0:
-        verbose_logger.debug("get_deployments_for_tag: empty candidate set; skipping tag filter")
+    if not healthy_deployments:
+        verbose_logger.debug("get_deployments_for_tag: empty or None healthy_deployments; skipping tag filter")
         return healthy_deployments
 
     verbose_logger.debug("request metadata: %s", request_kwargs.get(metadata_variable_name))
@@ -154,30 +184,36 @@ async def get_deployments_for_tag(
         # Build header strings for regex matching from what the proxy already stores.
         # Currently we match against User-Agent; format matches "^User-Agent: claude-code/..."
         user_agent = metadata.get("user_agent", "")
-        header_strings: List[str] = [f"User-Agent: {user_agent}"] if user_agent else []
+        header_strings: list[str] = [f"User-Agent: {user_agent}"] if user_agent else []
 
-        new_healthy_deployments: List[Any] = []
-        default_deployments: List[Any] = []
+        positive_tags, excluded_patterns = _split_tags(request_tags or [])
+
+        excluded_set = frozenset(excluded_patterns)
+        candidates = _exclude_deployments(healthy_deployments, excluded_set)
+
+        has_regex_deployments = any(d.get("litellm_params", {}).get("tag_regex") for d in candidates)
+        has_tag_filter = bool(positive_tags) or (bool(header_strings) and has_regex_deployments)
+        ban_only = bool(excluded_set) and not has_tag_filter
+
+        if ban_only:
+            pool = _exclude_deployments(_ban_only_base_pool(healthy_deployments), excluded_set)
+            return _require_candidates(pool, model, request_tags)
+
+        new_healthy_deployments: list[Any] = []
+        default_deployments: list[Any] = []
 
-        # Only activate header-based regex filtering when at least one deployment in
-        # the candidate set has tag_regex configured.  This preserves existing
-        # behaviour for operators who use plain tags: a request that carries a
-        # User-Agent (all proxy requests do) but targets deployments with no
-        # tag_regex will continue to use the original tag-only code path.
-        has_regex_deployments = any(d.get("litellm_params", {}).get("tag_regex") for d in healthy_deployments)
-        has_tag_filter = bool(request_tags) or (bool(header_strings) and has_regex_deployments)
         if has_tag_filter:
             verbose_logger.debug(
                 "get_deployments_for_tag routing: request_tags=%s user_agent=%s",
                 request_tags,
                 user_agent,
             )
-            for deployment in healthy_deployments:
+            for deployment in candidates:
                 deployment_tags = deployment.get("litellm_params", {}).get("tags")
 
                 match_result = _match_deployment(
                     deployment=deployment,
-                    request_tags=request_tags,
+                    request_tags=positive_tags,
                     header_strings=header_strings,
                     match_any=match_any,
                 )
@@ -189,10 +225,6 @@ async def get_deployments_for_tag(
                         match_result["matched_via"],
                         match_result["matched_value"],
                     )
-                    # Record provenance in metadata so it flows to SpendLogs.
-                    # Written only for the first match — load balancer selects one
-                    # deployment from new_healthy_deployments, so overwriting on
-                    # subsequent matches would produce misleading observability data.
                     if "tag_routing" not in metadata:
                         metadata["tag_routing"] = {
                             "matched_deployment": deployment.get("model_name"),
@@ -208,7 +240,8 @@ async def get_deployments_for_tag(
 
             if len(new_healthy_deployments) == 0 and len(default_deployments) == 0:
                 raise ValueError(
-                    f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model={model} and tags={request_tags}"
+                    f"{RouterErrors.no_deployments_with_tag_routing.value}."
+                    f" Passed model={model} and tags={request_tags}"
                 )
 
             return new_healthy_deployments if len(new_healthy_deployments) > 0 else default_deployments
@@ -231,9 +264,9 @@ async def get_deployments_for_tag(
 
 
 def _get_tags_from_request_kwargs(
-    request_kwargs: Optional[Dict[Any, Any]] = None,
+    request_kwargs: Optional[dict[Any, Any]] = None,
     metadata_variable_name: Literal["metadata", "litellm_metadata"] = "metadata",
-) -> List[str]:
+) -> list[str]:
     """
     Helper to get tags from request kwargs
 
diff --git a/tests/test_litellm/router_strategy/test_router_tag_routing.py b/tests/test_litellm/router_strategy/test_router_tag_routing.py
index a6e39ec3c0a..eb289095c51 100644
--- a/tests/test_litellm/router_strategy/test_router_tag_routing.py
+++ b/tests/test_litellm/router_strategy/test_router_tag_routing.py
@@ -1,29 +1,17 @@
 #### What this tests ####
 # This tests litellm router
 
-import asyncio
 import os
 import sys
-import time
-import traceback
 
-import openai
 import pytest
 
-sys.path.insert(
-    0, os.path.abspath("../..")
-)  # Adds the parent directory to the system path
+sys.path.insert(0, os.path.abspath("../.."))  # Adds the parent directory to the system path
 import logging
 import os
-from collections import defaultdict
-from concurrent.futures import ThreadPoolExecutor
-from unittest.mock import AsyncMock, MagicMock, patch
 
-import httpx
-from dotenv import load_dotenv
 
 import litellm
-from litellm import Router
 from litellm._logging import verbose_logger
 
 
@@ -66,10 +54,7 @@ async def test_router_free_paid_tier():
             mock_response="Tell me a joke.",
         )
 
-        print("Response: ", response)
-
         response_extra_info = response._hidden_params
-        print("response_extra_info: ", response_extra_info)
 
         assert response_extra_info["model_id"] == "very-cheap-model"
 
@@ -82,10 +67,7 @@ async def test_router_free_paid_tier():
             mock_response="Tell me a joke.",
         )
 
-        print("Response: ", response)
-
         response_extra_info = response._hidden_params
-        print("response_extra_info: ", response_extra_info)
 
         assert response_extra_info["model_id"] == "very-expensive-model"
 
@@ -141,10 +123,7 @@ async def test_router_free_paid_tier_embeddings():
             mock_response=[1, 2, 3],
         )
 
-        print("Response: ", response)
-
         response_extra_info = response._hidden_params
-        print("response_extra_info: ", response_extra_info)
 
         assert response_extra_info["model_id"] == "very-cheap-model"
 
@@ -157,10 +136,7 @@ async def test_router_free_paid_tier_embeddings():
             mock_response=[1, 2, 3],
         )
 
-        print("Response: ", response)
-
         response_extra_info = response._hidden_params
-        print("response_extra_info: ", response_extra_info)
 
         assert response_extra_info["model_id"] == "very-expensive-model"
 
@@ -212,10 +188,7 @@ async def test_default_tagged_deployments():
             mock_response="Tell me a joke.",
         )
 
-        print("Response: ", response)
-
         response_extra_info = response._hidden_params
-        print("response_extra_info: ", response_extra_info)
 
         assert response_extra_info["model_id"] == "default-model"
 
@@ -228,10 +201,7 @@ async def test_default_tagged_deployments():
             mock_response="Tell me a joke.",
         )
 
-        print("Response: ", response)
-
         response_extra_info = response._hidden_params
-        print("response_extra_info: ", response_extra_info)
 
         assert response_extra_info["model_id"] == "default-model"
 
@@ -244,10 +214,7 @@ async def test_default_tagged_deployments():
             mock_response="Tell me a joke.",
         )
 
-        print("Response: ", response)
-
         response_extra_info = response._hidden_params
-        print("response_extra_info: ", response_extra_info)
 
         assert response_extra_info["model_id"] == "default-model"
 
@@ -257,10 +224,6 @@ async def test_error_from_tag_routing():
     """
     Tests the correct error raised when no deployments found for tag
     """
-    import logging
-
-    from litellm._logging import verbose_logger
-
     verbose_logger.setLevel(logging.DEBUG)
     router = litellm.Router(
         model_list=[
@@ -294,7 +257,7 @@ async def test_error_from_tag_routing():
     )
 
     try:
-        response = await router.acompletion(
+        await router.acompletion(
             model="gpt-4",
             messages=[{"role": "user", "content": "Tell me a joke."}],
             metadata={"tags": ["paid"]},
@@ -306,7 +269,6 @@ async def test_error_from_tag_routing():
         from litellm.types.router import RouterErrors
 
         assert RouterErrors.no_deployments_with_tag_routing.value in str(e)
-        print("got expected exception = ", e)
         pass
 
 
@@ -332,16 +294,10 @@ def test_tag_routing_with_list_of_tags_match_all():
     from litellm.router_strategy.tag_based_routing import is_valid_deployment_tag
 
     assert is_valid_deployment_tag(["teamA", "teamB"], ["teamA"], match_any=False)
-    assert is_valid_deployment_tag(
-        ["teamA", "teamB"], ["teamA", "teamB"], match_any=False
-    )
-    assert not is_valid_deployment_tag(
-        ["teamA", "teamB", "teamC"], ["teamA", "teamD"], match_any=False
-    )
+    assert is_valid_deployment_tag(["teamA", "teamB"], ["teamA", "teamB"], match_any=False)
+    assert not is_valid_deployment_tag(["teamA", "teamB", "teamC"], ["teamA", "teamD"], match_any=False)
     assert not is_valid_deployment_tag(["teamA"], ["teamA", "teamB"], match_any=False)
-    assert not is_valid_deployment_tag(
-        ["teamA", "teamB"], ["teamA", "teamC"], match_any=False
-    )
+    assert not is_valid_deployment_tag(["teamA", "teamB"], ["teamA", "teamC"], match_any=False)
     assert not is_valid_deployment_tag(["teamA", "teamB"], [], match_any=False)
     assert not is_valid_deployment_tag(["default"], ["teamA"], match_any=False)
 
@@ -413,10 +369,7 @@ async def test_router_free_paid_tier_with_responses_api():
             mock_response="Tell me a joke.",
         )
 
-        print("Response: ", response)
-
         response_extra_info = response._hidden_params
-        print("response_extra_info: ", response_extra_info)
 
         assert response_extra_info["model_id"] == "very-cheap-model"
 
@@ -429,10 +382,7 @@ async def test_router_free_paid_tier_with_responses_api():
             mock_response="Tell me a joke.",
         )
 
-        print("Response: ", response)
-
         response_extra_info = response._hidden_params
-        print("response_extra_info: ", response_extra_info)
 
         assert response_extra_info["model_id"] == "very-expensive-model"
 
@@ -455,9 +405,7 @@ def test_get_tags_from_request_kwargs_various_inputs():
     assert _get_tags_from_request_kwargs({"metadata": None}) == []
 
     # Indirect via "litellm_params" - metadata inside
-    assert _get_tags_from_request_kwargs(
-        {"litellm_params": {"metadata": {"tags": ["paid"]}}}
-    ) == ["paid"]
+    assert _get_tags_from_request_kwargs({"litellm_params": {"metadata": {"tags": ["paid"]}}}) == ["paid"]
     assert _get_tags_from_request_kwargs({"litellm_params": {"metadata": None}}) == []
     assert _get_tags_from_request_kwargs({"litellm_params": {}}) == []
 
@@ -473,3 +421,601 @@ def test_get_tags_from_request_kwargs_various_inputs():
 
     # No relevant keys present
     assert _get_tags_from_request_kwargs({"foo": "bar"}) == []
+
+
+# --- _split_tags unit tests ---
+
+
+def test_split_tags_positive_only():
+    from litellm.router_strategy.tag_based_routing import _split_tags
+
+    positive, excluded = _split_tags(["paid", "teamA"])
+    assert positive == ["paid", "teamA"]
+    assert excluded == []
+
+
+def test_split_tags_negation_only():
+    from litellm.router_strategy.tag_based_routing import _split_tags
+
+    positive, excluded = _split_tags(["!provider:anthropic"])
+    assert positive == []
+    assert excluded == ["provider:anthropic"]
+
+
+def test_split_tags_mixed():
+    from litellm.router_strategy.tag_based_routing import _split_tags
+
+    positive, excluded = _split_tags(["paid", "!provider:anthropic", "!inference:cerebras"])
+    assert positive == ["paid"]
+    assert len(excluded) == 2
+
+
+def test_split_tags_bare_bang_skipped():
+    from litellm.router_strategy.tag_based_routing import _split_tags
+
+    # A bare "!" with nothing after it is not a valid negation tag; skip it
+    positive, excluded = _split_tags(["paid", "!"])
+    assert positive == ["paid"]
+    assert excluded == []
+
+
+def test_split_tags_empty():
+    from litellm.router_strategy.tag_based_routing import _split_tags
+
+    positive, excluded = _split_tags([])
+    assert positive == []
+    assert excluded == []
+
+
+# --- get_deployments_for_tag negation integration tests ---
+
+
+@pytest.mark.asyncio()
+async def test_negation_excludes_matching_deployments():
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["provider:anthropic", "model:claude-sonnet-4-6"],
+                },
+                "model_info": {"id": "anthropic-model"},
+            },
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o-mini",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["provider:openai", "model:gpt-4o"],
+                },
+                "model_info": {"id": "openai-model"},
+            },
+        ],
+        enable_tag_filtering=True,
+    )
+
+    for _ in range(5):
+        response = await router.acompletion(
+            model="gpt-4",
+            messages=[{"role": "user", "content": "hi"}],
+            metadata={"tags": ["!provider:anthropic"]},
+            mock_response="hi",
+        )
+        assert response._hidden_params["model_id"] == "openai-model"
+
+
+@pytest.mark.asyncio()
+async def test_negation_multiple_tags_exclude_multiple_providers():
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["provider:anthropic"],
+                },
+                "model_info": {"id": "anthropic-model"},
+            },
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o-mini",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["provider:openai"],
+                },
+                "model_info": {"id": "openai-model"},
+            },
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["provider:vertex"],
+                },
+                "model_info": {"id": "vertex-model"},
+            },
+        ],
+        enable_tag_filtering=True,
+    )
+
+    for _ in range(5):
+        response = await router.acompletion(
+            model="gpt-4",
+            messages=[{"role": "user", "content": "hi"}],
+            metadata={"tags": ["!provider:anthropic", "!provider:openai"]},
+            mock_response="hi",
+        )
+        assert response._hidden_params["model_id"] == "vertex-model"
+
+
+@pytest.mark.asyncio()
+async def test_negation_with_positive_tag():
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["paid", "provider:anthropic"],
+                },
+                "model_info": {"id": "anthropic-paid"},
+            },
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o-mini",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["paid", "provider:openai"],
+                },
+                "model_info": {"id": "openai-paid"},
+            },
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["free", "provider:openai"],
+                },
+                "model_info": {"id": "openai-free"},
+            },
+        ],
+        enable_tag_filtering=True,
+    )
+
+    for _ in range(5):
+        response = await router.acompletion(
+            model="gpt-4",
+            messages=[{"role": "user", "content": "hi"}],
+            metadata={"tags": ["paid", "!provider:anthropic"]},
+            mock_response="hi",
+        )
+        assert response._hidden_params["model_id"] == "openai-paid"
+
+
+@pytest.mark.asyncio()
+async def test_negation_all_excluded_raises():
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["provider:anthropic"],
+                },
+                "model_info": {"id": "anthropic-model"},
+            },
+        ],
+        enable_tag_filtering=True,
+    )
+
+    with pytest.raises(Exception) as exc_info:
+        await router.acompletion(
+            model="gpt-4",
+            messages=[{"role": "user", "content": "hi"}],
+            metadata={"tags": ["!provider:anthropic"]},
+            mock_response="hi",
+        )
+
+    from litellm.types.router import RouterErrors
+
+    assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value)
+
+
+@pytest.mark.asyncio()
+async def test_negation_ban_only_cannot_escape_default_pool():
+    # A ban-only request must not route to tagged deployments outside the default pool.
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["default"],
+                },
+                "model_info": {"id": "default-model"},
+            },
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o-mini",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["paid"],
+                },
+                "model_info": {"id": "paid-model"},
+            },
+        ],
+        enable_tag_filtering=True,
+    )
+
+    # Sending only "!default" must NOT route to the paid deployment.
+    # The base pool for ban-only is the default pool; banning the only
+    # default deployment should raise rather than falling through to paid.
+    with pytest.raises(Exception) as exc_info:
+        await router.acompletion(
+            model="gpt-4",
+            messages=[{"role": "user", "content": "hi"}],
+            metadata={"tags": ["!default"]},
+            mock_response="hi",
+        )
+
+    from litellm.types.router import RouterErrors
+
+    assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value)
+
+
+@pytest.mark.asyncio()
+async def test_negation_ban_only_respects_default_pool():
+    # A ban-only request stays within the default pool; non-default deployments
+    # remain unreachable even when the negation tag is unrelated to the default.
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["default"],
+                },
+                "model_info": {"id": "default-model"},
+            },
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o-mini",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["paid"],
+                },
+                "model_info": {"id": "paid-model"},
+            },
+        ],
+        enable_tag_filtering=True,
+    )
+
+    # "!paid" bans the paid deployment, but the base pool for ban-only is
+    # already restricted to defaults; default-model must still be returned.
+    for _ in range(5):
+        response = await router.acompletion(
+            model="gpt-4",
+            messages=[{"role": "user", "content": "hi"}],
+            metadata={"tags": ["!paid"]},
+            mock_response="hi",
+        )
+        assert response._hidden_params["model_id"] == "default-model"
+
+
+@pytest.mark.asyncio()
+async def test_negation_untagged_deployment_kept():
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["provider:anthropic"],
+                },
+                "model_info": {"id": "anthropic-model"},
+            },
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o-mini",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                },
+                "model_info": {"id": "untagged-model"},
+            },
+        ],
+        enable_tag_filtering=True,
+    )
+
+    for _ in range(5):
+        response = await router.acompletion(
+            model="gpt-4",
+            messages=[{"role": "user", "content": "hi"}],
+            metadata={"tags": ["!provider:anthropic"]},
+            mock_response="hi",
+        )
+        assert response._hidden_params["model_id"] == "untagged-model"
+
+
+@pytest.mark.asyncio()
+async def test_negation_literal_only_no_partial_match():
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["provider:anthropic-haiku"],
+                },
+                "model_info": {"id": "anthropic-haiku-model"},
+            },
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o-mini",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["provider:openai"],
+                },
+                "model_info": {"id": "openai-model"},
+            },
+        ],
+        enable_tag_filtering=True,
+    )
+
+    # "!provider:anthropic" should NOT match "provider:anthropic-haiku" — exact tag match only
+    for _ in range(5):
+        response = await router.acompletion(
+            model="gpt-4",
+            messages=[{"role": "user", "content": "hi"}],
+            metadata={"tags": ["!provider:anthropic"]},
+            mock_response="hi",
+        )
+        assert response._hidden_params["model_id"] in (
+            "anthropic-haiku-model",
+            "openai-model",
+        )
+
+
+@pytest.mark.asyncio()
+async def test_negation_regex_pattern_treated_as_literal():
+    # "!provider:(anthropic|openai)" looks like a regex but is treated as a literal string.
+    # It does NOT exclude deployments tagged "provider:anthropic" or "provider:openai".
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["provider:anthropic"],
+                },
+                "model_info": {"id": "anthropic-model"},
+            },
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o-mini",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["provider:openai"],
+                },
+                "model_info": {"id": "openai-model"},
+            },
+        ],
+        enable_tag_filtering=True,
+    )
+
+    # The regex-like string matches no deployment tag literally, so all
+    # candidates survive and both model IDs are reachable.
+    seen_ids = set()
+    for _ in range(10):
+        response = await router.acompletion(
+            model="gpt-4",
+            messages=[{"role": "user", "content": "hi"}],
+            metadata={"tags": ["!provider:(anthropic|openai)"]},
+            mock_response="hi",
+        )
+        seen_ids.add(response._hidden_params["model_id"])
+
+    assert seen_ids == {"anthropic-model", "openai-model"}
+
+
+@pytest.mark.asyncio()
+async def test_positive_tags_unchanged_by_negation():
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["free"],
+                },
+                "model_info": {"id": "free-model"},
+            },
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o-mini",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["paid"],
+                },
+                "model_info": {"id": "paid-model"},
+            },
+        ],
+        enable_tag_filtering=True,
+    )
+
+    for _ in range(5):
+        response = await router.acompletion(
+            model="gpt-4",
+            messages=[{"role": "user", "content": "hi"}],
+            metadata={"tags": ["free"]},
+            mock_response="hi",
+        )
+        assert response._hidden_params["model_id"] == "free-model"
+
+
+@pytest.mark.asyncio()
+async def test_negation_skips_banned_group_and_uses_fallback():
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "primary",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["provider:anthropic"],
+                },
+                "model_info": {"id": "anthropic-primary"},
+            },
+            {
+                "model_name": "fallback",
+                "litellm_params": {
+                    "model": "gpt-4o-mini",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["provider:openai"],
+                },
+                "model_info": {"id": "openai-fallback"},
+            },
+        ],
+        fallbacks=[{"primary": ["fallback"]}],
+        enable_tag_filtering=True,
+    )
+
+    response = await router.acompletion(
+        model="primary",
+        messages=[{"role": "user", "content": "hi"}],
+        metadata={"tags": ["!provider:anthropic"]},
+        mock_response="hi",
+    )
+    assert response._hidden_params["model_id"] == "openai-fallback"
+
+
+@pytest.mark.asyncio()
+async def test_negation_exhausts_entire_fallback_chain():
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "primary",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["provider:anthropic"],
+                },
+                "model_info": {"id": "anthropic-primary"},
+            },
+            {
+                "model_name": "fallback",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["provider:anthropic"],
+                },
+                "model_info": {"id": "anthropic-fallback"},
+            },
+        ],
+        fallbacks=[{"primary": ["fallback"]}],
+        enable_tag_filtering=True,
+    )
+
+    with pytest.raises(Exception) as exc_info:
+        await router.acompletion(
+            model="primary",
+            messages=[{"role": "user", "content": "hi"}],
+            metadata={"tags": ["!provider:anthropic"]},
+            mock_response="hi",
+        )
+
+    from litellm.types.router import RouterErrors
+
+    assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value)
+
+
+@pytest.mark.asyncio()
+async def test_tag_regex_survives_when_negation_removes_other_deployment():
+    # Negation removes a plain-tagged deployment; the surviving tag_regex deployment
+    # is still matched by User-Agent and selected.
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tag_regex": ["^User-Agent: claude-code\\/"],
+                },
+                "model_info": {"id": "claude-code-deployment"},
+            },
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o-mini",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["provider:anthropic"],
+                },
+                "model_info": {"id": "anthropic-deployment"},
+            },
+        ],
+        enable_tag_filtering=True,
+        tag_filtering_match_any=True,
+    )
+
+    for _ in range(5):
+        response = await router.acompletion(
+            model="gpt-4",
+            messages=[{"role": "user", "content": "hi"}],
+            metadata={"tags": ["!provider:anthropic"], "user_agent": "claude-code/1.2.3"},
+            mock_response="hi",
+        )
+        assert response._hidden_params["model_id"] == "claude-code-deployment"
+
+
+@pytest.mark.asyncio()
+async def test_negation_removes_tag_regex_deployment_falls_to_ban_only():
+    # When a negation tag removes the only tag_regex deployment, no regex deployments
+    # remain in the candidate pool. has_tag_filter becomes False, ban_only fires,
+    # and the remaining plain-tagged deployment is returned via the ban-only path.
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tag_regex": ["^User-Agent: claude-code\\/"],
+                    "tags": ["group:claude"],
+                },
+                "model_info": {"id": "claude-code-deployment"},
+            },
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {
+                    "model": "gpt-4o-mini",
+                    "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
+                    "tags": ["provider:openai"],
+                },
+                "model_info": {"id": "openai-deployment"},
+            },
+        ],
+        enable_tag_filtering=True,
+        tag_filtering_match_any=True,
+    )
+
+    # !group:claude removes the tag_regex deployment from candidates, so no regex
+    # deployments remain. The ban-only path fires and returns the openai deployment.
+    for _ in range(5):
+        response = await router.acompletion(
+            model="gpt-4",
+            messages=[{"role": "user", "content": "hi"}],
+            metadata={"tags": ["!group:claude"], "user_agent": "claude-code/1.2.3"},
+            mock_response="hi",
+        )
+        assert response._hidden_params["model_id"] == "openai-deployment"

From 9968499aabf6d6b4e36579c05979fc1deda44098 Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Tue, 30 Jun 2026 11:28:51 -0700
Subject: [PATCH 17/51] fix(ui): fix Router Settings Loadbalancing tab save
 (LIT-4057)

The Loadbalancing tab rendered routing_groups as a generic text input and
sent its array value back as the JSON string "[]", which fails Pydantic
list validation on POST /config/update and returns 422. routing_groups has
its own dedicated Routing Groups tab, so this tab must neither render nor
write it; exclude it the same way retry_policy and model_group_retry_policy
are excluded for the Model Retry Settings tab.

The save was also fire-and-forget: setCallbacksCall was not awaited, so the
rejected promise escaped the try/catch and the success toast fired
unconditionally, showing success even when the backend rejected the change.
Await the call, gate the success toast on resolution, and surface the error.
---
 .../ReliabilityRetriesSection.tsx             |  5 ++-
 .../components/router_settings/index.test.tsx | 40 +++++++++++++++++++
 .../src/components/router_settings/index.tsx  | 13 +++---
 3 files changed, 49 insertions(+), 9 deletions(-)

diff --git a/ui/litellm-dashboard/src/components/router_settings/ReliabilityRetriesSection.tsx b/ui/litellm-dashboard/src/components/router_settings/ReliabilityRetriesSection.tsx
index fa48c1c97b9..da089552b11 100644
--- a/ui/litellm-dashboard/src/components/router_settings/ReliabilityRetriesSection.tsx
+++ b/ui/litellm-dashboard/src/components/router_settings/ReliabilityRetriesSection.tsx
@@ -20,14 +20,15 @@ const ReliabilityRetriesSection: React.FC<ReliabilityRetriesSectionProps> = ({
       <div className="grid grid-cols-1 gap-6 lg:grid-cols-2 xl:grid-cols-3">
         {Object.entries(routerSettings)
           .filter(
-            ([param, value]) =>
+            ([param]) =>
               param != "fallbacks" &&
               param != "context_window_fallbacks" &&
               param != "routing_strategy_args" &&
               param != "routing_strategy" &&
               param != "enable_tag_filtering" &&
               param != "retry_policy" &&
-              param != "model_group_retry_policy",
+              param != "model_group_retry_policy" &&
+              param != "routing_groups",
           )
           .map(([param, value]) => (
             <div key={param} className="space-y-2">
diff --git a/ui/litellm-dashboard/src/components/router_settings/index.test.tsx b/ui/litellm-dashboard/src/components/router_settings/index.test.tsx
index 78a0b4b0dff..0df1cd81d6b 100644
--- a/ui/litellm-dashboard/src/components/router_settings/index.test.tsx
+++ b/ui/litellm-dashboard/src/components/router_settings/index.test.tsx
@@ -146,4 +146,44 @@ describe("RouterSettings", () => {
 
     expect(NotificationsManager.success).toHaveBeenCalledWith("router settings updated successfully");
   });
+
+  it("should not render or save routing_groups (owned by the Routing Groups tab) (LIT-4057)", async () => {
+    const user = userEvent.setup();
+    vi.mocked(getCallbacksCall).mockResolvedValue({
+      router_settings: {
+        routing_strategy: "simple-shuffle",
+        num_retries: 3,
+        routing_groups: [{ group_name: "g1", models: ["gpt-4"], routing_strategy: "simple-shuffle" }],
+      },
+    });
+    renderWithProviders(<RouterSettings {...defaultProps} />);
+
+    await waitFor(() => {
+      expect(screen.getByTestId("strategy-select")).toBeInTheDocument();
+    });
+    expect(document.querySelector('input[name="routing_groups"]')).toBeNull();
+
+    await user.click(screen.getByRole("button", { name: /save changes/i }));
+
+    const payload = vi.mocked(setCallbacksCall).mock.calls[0][1] as {
+      router_settings: Record<string, unknown>;
+    };
+    expect(payload.router_settings).not.toHaveProperty("routing_groups");
+  });
+
+  it("should surface an error and not claim success when saving fails (LIT-4057)", async () => {
+    const user = userEvent.setup();
+    vi.mocked(setCallbacksCall).mockRejectedValue(new Error("422 Unprocessable Entity"));
+    renderWithProviders(<RouterSettings {...defaultProps} />);
+
+    await waitFor(() => {
+      expect(screen.getByTestId("strategy-select")).toBeInTheDocument();
+    });
+    await user.click(screen.getByRole("button", { name: /save changes/i }));
+
+    await waitFor(() => {
+      expect(NotificationsManager.fromBackend).toHaveBeenCalled();
+    });
+    expect(NotificationsManager.success).not.toHaveBeenCalled();
+  });
 });
diff --git a/ui/litellm-dashboard/src/components/router_settings/index.tsx b/ui/litellm-dashboard/src/components/router_settings/index.tsx
index d3753529058..360c7f41138 100644
--- a/ui/litellm-dashboard/src/components/router_settings/index.tsx
+++ b/ui/litellm-dashboard/src/components/router_settings/index.tsx
@@ -81,7 +81,7 @@ const RouterSettings: React.FC<RouterSettingsProps> = ({ accessToken, userRole,
     });
   }, [accessToken, userRole, userID]);
 
-  const handleSaveChanges = () => {
+  const handleSaveChanges = async () => {
     if (!accessToken) {
       return;
     }
@@ -91,9 +91,9 @@ const RouterSettings: React.FC<RouterSettingsProps> = ({ accessToken, userRole,
 
     const numberKeys = new Set(["allowed_fails", "cooldown_time", "num_retries", "timeout", "retry_after"]);
     const jsonKeys = new Set(["model_group_alias"]);
-    // retry_policy and model_group_retry_policy are owned exclusively by the
-    // Model Retry Settings tab; this page must not read or write them.
-    const tabOwnedKeys = new Set(["retry_policy", "model_group_retry_policy"]);
+    // retry_policy and model_group_retry_policy are owned by the Model Retry Settings tab;
+    // routing_groups is owned by the Routing Groups tab. This page must not read or write them.
+    const tabOwnedKeys = new Set(["retry_policy", "model_group_retry_policy", "routing_groups"]);
 
     const parseInputValue = (key: string, raw: string | undefined, fallback: unknown) => {
       if (raw === undefined) return fallback;
@@ -172,12 +172,11 @@ const RouterSettings: React.FC<RouterSettingsProps> = ({ accessToken, userRole,
     };
 
     try {
-      setCallbacksCall(accessToken, payload);
+      await setCallbacksCall(accessToken, payload);
+      NotificationsManager.success("router settings updated successfully");
     } catch (error) {
       NotificationsManager.fromBackend("Failed to update router settings: " + error);
     }
-
-    NotificationsManager.success("router settings updated successfully");
   };
 
   if (!accessToken) {

From 30141f86f824cdc53e425a4b27fe07e34cc61c64 Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Tue, 30 Jun 2026 11:40:09 -0700
Subject: [PATCH 18/51] test(ui): make router settings save tests resilient to
 async timing

Address Greptile P2: the routing_groups test read setCallbacksCall.mock.calls[0][1]
immediately after the now-async save handler, so any latency in the mock would throw
an opaque TypeError instead of a clean assertion failure. Assert through
toHaveBeenCalledWith inside waitFor with expect.not.objectContaining, dropping the
index access and the cast. Also drop the ticket id from the test names.
---
 .../src/components/router_settings/index.test.tsx   | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/ui/litellm-dashboard/src/components/router_settings/index.test.tsx b/ui/litellm-dashboard/src/components/router_settings/index.test.tsx
index 0df1cd81d6b..94cbb94d164 100644
--- a/ui/litellm-dashboard/src/components/router_settings/index.test.tsx
+++ b/ui/litellm-dashboard/src/components/router_settings/index.test.tsx
@@ -147,7 +147,7 @@ describe("RouterSettings", () => {
     expect(NotificationsManager.success).toHaveBeenCalledWith("router settings updated successfully");
   });
 
-  it("should not render or save routing_groups (owned by the Routing Groups tab) (LIT-4057)", async () => {
+  it("should not render or save routing_groups (owned by the Routing Groups tab)", async () => {
     const user = userEvent.setup();
     vi.mocked(getCallbacksCall).mockResolvedValue({
       router_settings: {
@@ -165,13 +165,14 @@ describe("RouterSettings", () => {
 
     await user.click(screen.getByRole("button", { name: /save changes/i }));
 
-    const payload = vi.mocked(setCallbacksCall).mock.calls[0][1] as {
-      router_settings: Record<string, unknown>;
-    };
-    expect(payload.router_settings).not.toHaveProperty("routing_groups");
+    await waitFor(() =>
+      expect(setCallbacksCall).toHaveBeenCalledWith("test-token", {
+        router_settings: expect.not.objectContaining({ routing_groups: expect.anything() }),
+      }),
+    );
   });
 
-  it("should surface an error and not claim success when saving fails (LIT-4057)", async () => {
+  it("should surface an error and not claim success when saving fails", async () => {
     const user = userEvent.setup();
     vi.mocked(setCallbacksCall).mockRejectedValue(new Error("422 Unprocessable Entity"));
     renderWithProviders(<RouterSettings {...defaultProps} />);

From a126cdf5b79362aea1c76fec2c1c654ad5175fcb Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Tue, 30 Jun 2026 18:47:08 +0000
Subject: [PATCH 19/51] feat(anthropic): add Claude Sonnet 5

Register claude-sonnet-5 across the Anthropic, Bedrock (base + global/us/eu/au/jp
cross-region inference profiles), Vertex AI, and Azure AI cost-map entries in both
the root and bundled-backup model maps, plus BEDROCK_CONVERSE_MODELS and the
setup-wizard provider list.

Sonnet 5 ships with the gen-5 adaptive-thinking profile (adaptive thinking always
on, no extended thinking, effort defaults to high), so the entries mirror the
Fable 5 / Opus 4.8 sampling-param and prefill restrictions rather than the older
Sonnet 4.6 behavior: supports_sampling_params and supports_assistant_prefill are
false while supports_adaptive_thinking, supports_xhigh_reasoning_effort, and
supports_max_reasoning_effort are true. Pricing follows standard Sonnet rates
($3 / $15 per MTok) with the 10% regional premium on the us/eu/au/jp profiles.

Add a reasoning-effort grid entry for the Anthropic direct route and a regression
test pinning pricing, capabilities, regional premiums, backup parity, and bare-name
provider resolution.

Co-authored-by: Mateo Wang <mateo-berri@users.noreply.github.com>
---
 litellm/constants.py                          |   1 +
 ...odel_prices_and_context_window_backup.json | 325 ++++++++++++++++++
 litellm/setup_wizard.py                       |   1 +
 model_prices_and_context_window.json          | 325 ++++++++++++++++++
 .../reasoning_effort_grid/grid_spec.py        |   7 +
 .../test_claude_sonnet_5_config.py            | 174 ++++++++++
 6 files changed, 833 insertions(+)
 create mode 100644 tests/test_litellm/test_claude_sonnet_5_config.py

diff --git a/litellm/constants.py b/litellm/constants.py
index aeb74a65839..5cc41bd7954 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -1123,6 +1123,7 @@ BEDROCK_CONVERSE_MODELS = [
     "anthropic.claude-haiku-4-5-20251001-v1:0",
     "anthropic.claude-sonnet-4-5-20250929-v1:0",
     "anthropic.claude-fable-5",
+    "anthropic.claude-sonnet-5",
     "anthropic.claude-opus-4-8",
     "anthropic.claude-opus-4-7",
     "anthropic.claude-opus-4-6-v1:0",
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 21132db93cb..5bd70690d48 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -1671,6 +1671,204 @@
         "supports_max_reasoning_effort": true,
         "supports_minimal_reasoning_effort": true
     },
+    "anthropic.claude-sonnet-5": {
+        "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "global.anthropic.claude-sonnet-5": {
+        "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "us.anthropic.claude-sonnet-5": {
+        "cache_creation_input_token_cost": 4.125e-06,
+        "cache_creation_input_token_cost_above_1hr": 6.6e-06,
+        "cache_read_input_token_cost": 3.3e-07,
+        "input_cost_per_token": 3.3e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.65e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "eu.anthropic.claude-sonnet-5": {
+        "cache_creation_input_token_cost": 4.125e-06,
+        "cache_creation_input_token_cost_above_1hr": 6.6e-06,
+        "cache_read_input_token_cost": 3.3e-07,
+        "input_cost_per_token": 3.3e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.65e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "au.anthropic.claude-sonnet-5": {
+        "cache_creation_input_token_cost": 4.125e-06,
+        "cache_creation_input_token_cost_above_1hr": 6.6e-06,
+        "cache_read_input_token_cost": 3.3e-07,
+        "input_cost_per_token": 3.3e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.65e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "jp.anthropic.claude-sonnet-5": {
+        "cache_creation_input_token_cost": 4.125e-06,
+        "cache_creation_input_token_cost_above_1hr": 6.6e-06,
+        "cache_read_input_token_cost": 3.3e-07,
+        "input_cost_per_token": 3.3e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.65e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
     "anthropic.claude-sonnet-4-6": {
         "supports_adaptive_thinking": true,
         "cache_creation_input_token_cost": 3.75e-06,
@@ -2511,6 +2709,37 @@
         "supports_tool_choice": true,
         "supports_vision": true
     },
+    "azure_ai/claude-sonnet-5": {
+        "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "azure_ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true
+    },
     "azure_ai/claude-sonnet-4-6": {
         "supports_adaptive_thinking": true,
         "cache_creation_input_token_cost": 3.75e-06,
@@ -10245,6 +10474,40 @@
         "supports_vision": true,
         "supports_web_search": true
     },
+    "claude-sonnet-5": {
+        "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "anthropic",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "provider_specific_entry": {
+            "us": 1.1
+        },
+        "supports_output_config": true
+    },
     "claude-sonnet-4-6": {
         "cache_creation_input_token_cost": 3.75e-06,
         "cache_creation_input_token_cost_above_1hr": 6e-06,
@@ -34944,6 +35207,37 @@
         "supports_tool_choice": true,
         "supports_vision": true
     },
+    "vertex_ai/claude-sonnet-5": {
+        "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "vertex_ai-anthropic_models",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true
+    },
     "vertex_ai/claude-sonnet-4-6": {
         "supports_adaptive_thinking": true,
         "cache_creation_input_token_cost": 3.75e-06,
@@ -42381,6 +42675,37 @@
             "search_context_size_high": 0.035
         }
     },
+    "vertex_ai/claude-sonnet-5@default": {
+        "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "vertex_ai-anthropic_models",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true
+    },
     "vertex_ai/claude-sonnet-4-6@default": {
         "supports_adaptive_thinking": true,
         "cache_creation_input_token_cost": 3.75e-06,
diff --git a/litellm/setup_wizard.py b/litellm/setup_wizard.py
index b43590079fa..10b4fb30f22 100644
--- a/litellm/setup_wizard.py
+++ b/litellm/setup_wizard.py
@@ -58,6 +58,7 @@ PROVIDERS: List[Dict] = [
         "test_model": "claude-haiku-4-5-20251001",
         "models": [
             "claude-fable-5",
+            "claude-sonnet-5",
             "claude-opus-4-8",
             "claude-opus-4-7",
             "claude-opus-4-6",
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 73cefeb7c77..5fc431d721b 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -1671,6 +1671,204 @@
         "supports_max_reasoning_effort": true,
         "supports_minimal_reasoning_effort": true
     },
+    "anthropic.claude-sonnet-5": {
+        "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "global.anthropic.claude-sonnet-5": {
+        "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "us.anthropic.claude-sonnet-5": {
+        "cache_creation_input_token_cost": 4.125e-06,
+        "cache_creation_input_token_cost_above_1hr": 6.6e-06,
+        "cache_read_input_token_cost": 3.3e-07,
+        "input_cost_per_token": 3.3e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.65e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "eu.anthropic.claude-sonnet-5": {
+        "cache_creation_input_token_cost": 4.125e-06,
+        "cache_creation_input_token_cost_above_1hr": 6.6e-06,
+        "cache_read_input_token_cost": 3.3e-07,
+        "input_cost_per_token": 3.3e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.65e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "au.anthropic.claude-sonnet-5": {
+        "cache_creation_input_token_cost": 4.125e-06,
+        "cache_creation_input_token_cost_above_1hr": 6.6e-06,
+        "cache_read_input_token_cost": 3.3e-07,
+        "input_cost_per_token": 3.3e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.65e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "jp.anthropic.claude-sonnet-5": {
+        "cache_creation_input_token_cost": 4.125e-06,
+        "cache_creation_input_token_cost_above_1hr": 6.6e-06,
+        "cache_read_input_token_cost": 3.3e-07,
+        "input_cost_per_token": 3.3e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.65e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
     "anthropic.claude-sonnet-4-6": {
         "supports_adaptive_thinking": true,
         "cache_creation_input_token_cost": 3.75e-06,
@@ -2511,6 +2709,37 @@
         "supports_tool_choice": true,
         "supports_vision": true
     },
+    "azure_ai/claude-sonnet-5": {
+        "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "azure_ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true
+    },
     "azure_ai/claude-sonnet-4-6": {
         "supports_adaptive_thinking": true,
         "cache_creation_input_token_cost": 3.75e-06,
@@ -10245,6 +10474,40 @@
         "supports_vision": true,
         "supports_web_search": true
     },
+    "claude-sonnet-5": {
+        "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "anthropic",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "provider_specific_entry": {
+            "us": 1.1
+        },
+        "supports_output_config": true
+    },
     "claude-sonnet-4-6": {
         "cache_creation_input_token_cost": 3.75e-06,
         "cache_creation_input_token_cost_above_1hr": 6e-06,
@@ -35121,6 +35384,37 @@
         "supports_tool_choice": true,
         "supports_vision": true
     },
+    "vertex_ai/claude-sonnet-5": {
+        "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "vertex_ai-anthropic_models",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true
+    },
     "vertex_ai/claude-sonnet-4-6": {
         "supports_adaptive_thinking": true,
         "cache_creation_input_token_cost": 3.75e-06,
@@ -42616,6 +42910,37 @@
             "search_context_size_high": 0.035
         }
     },
+    "vertex_ai/claude-sonnet-5@default": {
+        "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "vertex_ai-anthropic_models",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true
+    },
     "vertex_ai/claude-sonnet-4-6@default": {
         "supports_adaptive_thinking": true,
         "cache_creation_input_token_cost": 3.75e-06,
diff --git a/tests/llm_translation/reasoning_effort_grid/grid_spec.py b/tests/llm_translation/reasoning_effort_grid/grid_spec.py
index 1bb468d15ad..762606bb3a0 100644
--- a/tests/llm_translation/reasoning_effort_grid/grid_spec.py
+++ b/tests/llm_translation/reasoning_effort_grid/grid_spec.py
@@ -181,6 +181,13 @@ ANTHROPIC_DIRECT_MODELS: Tuple[ModelEntry, ...] = (
         required_env=_ANTHROPIC_REQ,
         caps=_CAPS_XHIGH_MAX,
     ),
+    ModelEntry(
+        alias="claude-sonnet-5",
+        model="anthropic/claude-sonnet-5",
+        mode="adaptive",
+        required_env=_ANTHROPIC_REQ,
+        caps=_CAPS_XHIGH_MAX,
+    ),
     ModelEntry(
         alias="claude-sonnet-4-6",
         model="anthropic/claude-sonnet-4-6",
diff --git a/tests/test_litellm/test_claude_sonnet_5_config.py b/tests/test_litellm/test_claude_sonnet_5_config.py
new file mode 100644
index 00000000000..88a06aea95f
--- /dev/null
+++ b/tests/test_litellm/test_claude_sonnet_5_config.py
@@ -0,0 +1,174 @@
+"""
+Validate Claude Sonnet 5 model configuration entries.
+
+Sonnet 5 ships with the gen-5 adaptive-thinking profile (adaptive thinking
+always on, no extended thinking, ``effort`` defaults to ``high``), so it must
+mirror the sampling-param and prefill restrictions that Fable 5 / Opus 4.8 carry
+rather than the older Sonnet 4.6 behavior. The cost-map entries are also what
+populate ``litellm.anthropic_models`` at import, which is what lets a bare
+``claude-sonnet-5`` name resolve to the ``anthropic`` provider (and match an
+``anthropic/*`` wildcard deployment).
+"""
+
+import json
+import os
+
+import pytest
+
+import litellm
+from litellm.constants import BEDROCK_CONVERSE_MODELS
+from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
+
+REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
+
+ALL_SONNET_5_VARIANTS = (
+    "claude-sonnet-5",
+    "anthropic.claude-sonnet-5",
+    "global.anthropic.claude-sonnet-5",
+    "us.anthropic.claude-sonnet-5",
+    "eu.anthropic.claude-sonnet-5",
+    "au.anthropic.claude-sonnet-5",
+    "jp.anthropic.claude-sonnet-5",
+    "vertex_ai/claude-sonnet-5",
+    "vertex_ai/claude-sonnet-5@default",
+    "azure_ai/claude-sonnet-5",
+)
+
+
+def _load_root_cost_map() -> dict:
+    json_path = os.path.join(REPO_ROOT, "model_prices_and_context_window.json")
+    with open(json_path) as f:
+        return json.load(f)
+
+
+@pytest.fixture
+def local_model_cost_map(monkeypatch):
+    """Force the bundled backup cost map so assertions don't depend on the
+    network-fetched ``main`` copy (which lags this branch until merge)."""
+    original_model_cost = litellm.model_cost
+    monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
+    litellm.model_cost = litellm.get_model_cost_map(url="")
+    litellm.get_model_info.cache_clear()
+    try:
+        yield
+    finally:
+        litellm.model_cost = original_model_cost
+        litellm.get_model_info.cache_clear()
+
+
+def test_sonnet_5_pricing_and_capabilities():
+    model_data = _load_root_cost_map()
+
+    expected_providers = {
+        "claude-sonnet-5": "anthropic",
+        "anthropic.claude-sonnet-5": "bedrock_converse",
+        "vertex_ai/claude-sonnet-5": "vertex_ai-anthropic_models",
+        "azure_ai/claude-sonnet-5": "azure_ai",
+    }
+
+    for model_name, provider in expected_providers.items():
+        assert model_name in model_data, f"Missing model entry: {model_name}"
+        info = model_data[model_name]
+
+        assert info["litellm_provider"] == provider
+        assert info["mode"] == "chat"
+        assert info["max_input_tokens"] == 1000000
+        assert info["max_output_tokens"] == 128000
+        assert info["max_tokens"] == 128000
+
+        # Standard Sonnet pricing: $3 / $15 per MTok, with the 1.25x cache-write
+        # and 0.1x cache-read multipliers.
+        assert info["input_cost_per_token"] == 3e-06
+        assert info["output_cost_per_token"] == 1.5e-05
+        assert info["cache_creation_input_token_cost"] == 3.75e-06
+        assert info["cache_read_input_token_cost"] == 3e-07
+
+        # gen-5 adaptive-thinking profile: effort-driven, no sampling params, no
+        # assistant prefill.
+        assert info["supports_adaptive_thinking"] is True
+        assert info["supports_reasoning"] is True
+        assert info["supports_sampling_params"] is False
+        assert info["supports_assistant_prefill"] is False
+
+        assert info["supports_function_calling"] is True
+        assert info["supports_prompt_caching"] is True
+        assert info["supports_tool_choice"] is True
+        assert info["supports_vision"] is True
+
+
+def test_sonnet_5_bedrock_regional_pricing():
+    """Global/base endpoints use base pricing; the us./eu./au./jp. regional
+    cross-region inference profiles carry a 10% premium."""
+    model_data = _load_root_cost_map()
+
+    base_pricing = {
+        "input_cost_per_token": 3e-06,
+        "output_cost_per_token": 1.5e-05,
+        "cache_creation_input_token_cost": 3.75e-06,
+        "cache_read_input_token_cost": 3e-07,
+    }
+    regional_pricing = {
+        "input_cost_per_token": 3.3e-06,
+        "output_cost_per_token": 1.65e-05,
+        "cache_creation_input_token_cost": 4.125e-06,
+        "cache_read_input_token_cost": 3.3e-07,
+    }
+
+    expected = {
+        "anthropic.claude-sonnet-5": base_pricing,
+        "global.anthropic.claude-sonnet-5": base_pricing,
+        "us.anthropic.claude-sonnet-5": regional_pricing,
+        "eu.anthropic.claude-sonnet-5": regional_pricing,
+        "au.anthropic.claude-sonnet-5": regional_pricing,
+        "jp.anthropic.claude-sonnet-5": regional_pricing,
+    }
+
+    for model_name, pricing in expected.items():
+        assert model_name in model_data, f"Missing model entry: {model_name}"
+        info = model_data[model_name]
+        assert info["litellm_provider"] == "bedrock_converse"
+        assert info["bedrock_output_config_effort_ceiling"] == "xhigh"
+        for key, value in pricing.items():
+            assert info[key] == value, f"{model_name}.{key} = {info[key]}, want {value}"
+
+
+def test_sonnet_5_present_in_bundled_backup():
+    """The bundled backup is the runtime fallback (and what tests load with
+    ``LITELLM_LOCAL_MODEL_COST_MAP=True``); it must carry the same entries as the
+    root cost map, otherwise the model resolves on one path but not the other."""
+    backup = GetModelCostMap.load_local_model_cost_map()
+    for model_name in ALL_SONNET_5_VARIANTS:
+        assert model_name in backup, f"Missing from backup cost map: {model_name}"
+
+
+def test_sonnet_5_registered_for_bedrock_converse():
+    assert "anthropic.claude-sonnet-5" in BEDROCK_CONVERSE_MODELS
+
+
+def test_sonnet_5_provider_resolves_via_model_info(local_model_cost_map):
+    """Regression: ``claude-sonnet-5`` must resolve to provider ``anthropic``.
+
+    Before the cost-map entry existed, the model was unknown to LiteLLM, so it
+    could not be tied to the ``anthropic`` provider and an ``anthropic/*``
+    wildcard deployment would not match it."""
+    info = litellm.get_model_info(model="claude-sonnet-5")
+    assert info["litellm_provider"] == "anthropic"
+    assert info["max_input_tokens"] == 1000000
+    assert info["max_output_tokens"] == 128000
+
+
+@pytest.mark.parametrize(
+    "cost_map",
+    [_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
+    ids=["root", "bundled_backup"],
+)
+def test_sonnet_5_all_variants_carry_adaptive_thinking_flag(cost_map):
+    """Every Sonnet 5 entry must advertise ``supports_adaptive_thinking``.
+
+    Adaptive-thinking detection is cost-map driven, so a single variant missing
+    the flag silently sends the legacy ``thinking.type='enabled'`` shape and the
+    provider 400s. This guards against a future variant being added without it."""
+    variants = [k for k in cost_map if "claude-sonnet-5" in k]
+    assert variants, "no claude-sonnet-5 entries found in cost map"
+    missing = [k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True]
+    assert not missing, f"missing supports_adaptive_thinking: {missing}"

From 540c860a9737838e5fa2146aa8d4cd2fab445548 Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Tue, 30 Jun 2026 11:47:35 -0700
Subject: [PATCH 20/51] test(ui): add typed e2e for Router Settings
 Loadbalancing save (LIT-4057)

Drives the real save flow against a live proxy: seeds a present routing_groups
array (the LIT-4057 trigger) via the typed /config/update contract, changes
num_retries on the Loadbalancing tab, and asserts the POST returns 200 instead
of 422, the success toast appears, and the value still shows after a reload (the
ticket's "refresh shows old values" symptom). The round-trip is typed against the
OpenAPI-generated backend schema (ConfigYAML write, RouterSettingsResponse read)
through a type-only import, so a backend contract drift fails the type check.
---
 .../tests/settings/routerSettings.spec.ts     | 84 +++++++++++++++++++
 1 file changed, 84 insertions(+)

diff --git a/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts
index 98b86ec9b11..e560500fca6 100644
--- a/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts
+++ b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts
@@ -3,6 +3,10 @@ import { ADMIN_STORAGE_PATH } from "../../constants";
 import { navigateToPage } from "../../helpers/navigation";
 import { Page } from "../../fixtures/pages";
 import { Role, users } from "../../fixtures/users";
+// Type-only import of the OpenAPI-generated backend schema; esbuild erases it at
+// runtime, so the round-trip below is checked against the real /config/update and
+// /router/settings contracts without bundling the 2 MB definition file.
+import type { components } from "../../../src/lib/http/schema";
 
 const PRIMARY = "fake-openai-gpt-4";
 const FALLBACK = "fake-anthropic-claude";
@@ -99,3 +103,83 @@ test.describe("Router Settings - Fallbacks", () => {
     await expect(newRow).toHaveCount(1, { timeout: 10_000 });
   });
 });
+
+type ConfigYAML = components["schemas"]["ConfigYAML"];
+type RouterSettingsResponse = components["schemas"]["RouterSettingsResponse"];
+
+const BASE_URL = "http://localhost:4000";
+const ADMIN_AUTH = { Authorization: `Bearer ${users[Role.ProxyAdmin].password}` };
+
+/**
+ * Merge a router_settings patch into the live config through the typed
+ * /config/update contract, preserving any other settings already present.
+ */
+async function patchRouterSettings(
+  request: import("@playwright/test").APIRequestContext,
+  patch: Partial<NonNullable<ConfigYAML["router_settings"]>>,
+) {
+  const current = await request.get(`${BASE_URL}/get/config/callbacks`, { headers: ADMIN_AUTH });
+  const existing = current.ok() ? (await current.json())?.router_settings ?? {} : {};
+  const payload = { router_settings: { ...(existing as Record<string, unknown>), ...patch } };
+  await request.post(`${BASE_URL}/config/update`, { headers: ADMIN_AUTH, data: payload });
+}
+
+test.describe("Router Settings - Loadbalancing", () => {
+  test.use({ storageState: ADMIN_STORAGE_PATH });
+
+  // Seed a present routing_groups array (the LIT-4057 trigger) plus a known
+  // num_retries so the UI assertions are deterministic across reruns.
+  const ROUTING_GROUP = { group_name: "e2e-lit-4057", models: [PRIMARY], routing_strategy: "simple-shuffle" };
+
+  test.beforeEach(async ({ request }) => {
+    await patchRouterSettings(request, { num_retries: 3, routing_groups: [ROUTING_GROUP] });
+  });
+
+  test.afterEach(async ({ request }) => {
+    await patchRouterSettings(request, { num_retries: 3, routing_groups: [] });
+  });
+
+  test("saves the Loadbalancing tab without a 422 when routing_groups is present, and persists", async ({
+    page,
+    request,
+  }) => {
+    await navigateToPage(page, Page.RouterSettings);
+    await page.getByRole("tab", { name: "Loadbalancing" }).click();
+
+    const numRetries = page.locator('input[name="num_retries"]');
+    await expect(numRetries).toHaveValue("3", { timeout: 15_000 });
+    // routing_groups belongs to its own tab and must not leak into this form.
+    await expect(page.locator('input[name="routing_groups"]')).toHaveCount(0);
+
+    await numRetries.fill("5");
+
+    // LIT-4057: the tab used to serialize routing_groups as the string "[]",
+    // which the backend rejects with 422 while the UI still claimed success.
+    // Assert the save actually succeeds at the network level.
+    const saveResponse = page.waitForResponse(
+      (res) => res.url().includes("/config/update") && res.request().method() === "POST",
+      { timeout: 15_000 },
+    );
+    await page.getByRole("button", { name: /save changes/i }).click();
+    expect((await saveResponse).status()).toBe(200);
+
+    await expect(page.getByText(/router settings updated successfully/i).first()).toBeVisible({ timeout: 10_000 });
+
+    // The ticket's core symptom was that a refresh showed the old value.
+    await navigateToPage(page, Page.RouterSettings);
+    await page.getByRole("tab", { name: "Loadbalancing" }).click();
+    await expect(page.locator('input[name="num_retries"]')).toHaveValue("5", { timeout: 15_000 });
+
+    // The typed backend read agrees the change persisted.
+    await expect
+      .poll(
+        async () => {
+          const res = await request.get(`${BASE_URL}/router/settings`, { headers: ADMIN_AUTH });
+          const data = (await res.json()) as RouterSettingsResponse;
+          return data.current_values?.num_retries;
+        },
+        { timeout: 10_000 },
+      )
+      .toBe(5);
+  });
+});

From 87de0e80a83628885aa68775e1cf07e8a140c1d7 Mon Sep 17 00:00:00 2001
From: tin-berri <tin@berri.ai>
Date: Tue, 30 Jun 2026 11:58:47 -0700
Subject: [PATCH 21/51] fix(mcp): stop one unauthenticated server from emptying
 the aggregate tools/list (#31684)

* fix(mcp): stop one unauthenticated server from emptying the aggregate tools/list

On the aggregate MCP route (/mcp), the gateway fans out to every server the caller can access and
flattens their tools. _fetch_and_filter_server_tools re-raises MCPUpstreamAuthError unconditionally
(added with the OAuth passthrough feature in #28356) so it surfaces a 401 on single-server routes,
but on the aggregate route that exception propagates through the asyncio.gather fan-out and the
outer handler turns it into an empty list. The result: a single delegate/passthrough OAuth server
the user has not authenticated (e.g. a delegate-auth server) zeroes the tools of every other server,
including the ones that resolve fine, so the client connects and sees no tools.

Surface the upstream auth error only when a single server was explicitly targeted (so that route
still drives the upstream OAuth flow); across the aggregate, absorb it to [] for that one server so
the rest still list their tools. This restores the graceful per-server degradation that predated
#28356.

Adds regression tests: the aggregate keeps a healthy server's tools when a sibling raises
MCPUpstreamAuthError, and a single-server listing still surfaces it.

* fix(mcp): decide aggregate vs single-server listing by route scope, not server count

Addresses review: keying the surface-vs-absorb decision off the server count (len(allowed_mcp_servers),
and even len(mcp_servers)) misclassifies an aggregate /mcp request from a key that can access exactly
one server as a targeted single-server listing, so that one server's MCPUpstreamAuthError re-raises and
empties the aggregate again for one-server permission sets.

Use the path-derived single-server scope instead: _mcp_gateway_server_name, set by
_gateway_initialize_instructions_request_scope only when the request path names exactly one upstream
server (/<server>/mcp) and never from client headers, is None on the aggregate route (/mcp) regardless
of how many servers the key can access. Single-server routes still surface the upstream-auth challenge;
the aggregate absorbs it per server.

Adds a regression test that an aggregate request with a single accessible server still absorbs, plus
renames the single-server test to drive the route scope explicitly. The new test fails on the
count-based logic.

* fixing aggregation error

* style(mcp): collapse single-line debug log to satisfy ruff format
---
 .../proxy/_experimental/mcp_server/server.py  |  13 +-
 .../test_mcp_oauth_passthrough_tools.py       | 128 ++++++++++++++++++
 2 files changed, 135 insertions(+), 6 deletions(-)

diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py
index 158fdda6c39..607e676524e 100644
--- a/litellm/proxy/_experimental/mcp_server/server.py
+++ b/litellm/proxy/_experimental/mcp_server/server.py
@@ -1699,12 +1699,13 @@ if MCP_AVAILABLE:
                     )
                     return filtered_tools
                 except MCPUpstreamAuthError:
-                    # Surface upstream 401/403 to the outer handler so the
-                    # client receives a proper WWW-Authenticate challenge
-                    # instead of a silently empty tool list. Without this
-                    # re-raise the broad ``except Exception`` below would
-                    # swallow the auth error.
-                    raise
+                    # Absorb so one unauthenticated server does not empty every other server's
+                    # tools. Surfacing the upstream 401 to the client as a re-auth challenge is
+                    # intentionally not done here: raising from this list handler cannot produce a
+                    # 401 + WWW-Authenticate (the MCP session manager serializes it as a JSON-RPC
+                    # error), so that belongs in a request-scope preemptive check, tracked separately.
+                    verbose_logger.debug(f"MCP list_tools: omitting {server.name}; it needs upstream auth")
+                    return []
                 except Exception as e:
                     verbose_logger.exception(f"Error getting tools from server {server.name}: {str(e)}")
                     return []
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py
index d51cf8c5b72..b836c3aef33 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py
@@ -265,3 +265,131 @@ async def test_fetch_tools_from_gateway_managed_swallows_errors():
     )
     assert tools == []
     mock_client.list_tools.assert_awaited_with(raise_on_error=False)
+
+
+def _http_server(server_id: str, name: str, **kwargs) -> MCPServer:
+    return MCPServer(
+        server_id=server_id,
+        name=name,
+        url=f"https://{name}/mcp",
+        transport=MCPTransport.http,
+        **kwargs,
+    )
+
+
+@pytest.mark.asyncio
+async def test_aggregate_list_tools_absorbs_one_unauthenticated_server():
+    """Regression: across the aggregate (/mcp), a delegate/passthrough server that raises
+    MCPUpstreamAuthError must not empty every other server's tools. Re-raising it on the
+    aggregate path (introduced with the passthrough feature) zeroed the whole list because the
+    fan-out gather propagated it."""
+    from unittest.mock import patch
+
+    from mcp.types import Tool as MCPTool
+
+    from litellm.proxy._experimental.mcp_server import server as mcp_server
+    from litellm.proxy._types import UserAPIKeyAuth
+
+    delegate = _http_server(
+        "s1", "delegate_docs", auth_type=MCPAuth.oauth2, delegate_auth_to_upstream=True
+    )
+    working = _http_server("s2", "working_docs", auth_type=MCPAuth.none)
+    good_tool = MCPTool(name="working_docs-read", description="d", inputSchema={"type": "object"})
+
+    async def fake_get_tools(server, **kwargs):
+        if server.server_id == delegate.server_id:
+            raise MCPUpstreamAuthError(status_code=401, www_authenticate=None, server_name=server.name)
+        return [good_tool]
+
+    with patch.object(mcp_server, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate, working])), patch.object(
+        mcp_server, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={})
+    ), patch.object(mcp_server, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object(
+        mcp_server, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None)
+    ), patch.object(
+        mcp_server, "filter_tools_by_key_team_permissions", AsyncMock(side_effect=lambda tools, **k: tools)
+    ), patch.object(
+        mcp_server.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools)
+    ):
+        tools = await mcp_server._get_tools_from_mcp_servers(
+            user_api_key_auth=UserAPIKeyAuth(token="h", user_id="u1"),
+            mcp_auth_header=None,
+            mcp_servers=None,
+        )
+
+    assert [t.name for t in tools] == ["working_docs-read"]
+
+
+@pytest.mark.asyncio
+async def test_single_server_route_also_absorbs_upstream_auth_error():
+    """A single-server route (/<server>/mcp) absorbs an upstream-auth error just like the aggregate:
+    the failing server is omitted (empty list) rather than re-raised. Surfacing it to the client as a
+    401 + WWW-Authenticate challenge cannot be done from this list handler — the MCP session manager
+    serializes a raise into a JSON-RPC error, not an HTTP 401 — so re-auth surfacing is handled by a
+    request-scope preemptive check, tracked separately."""
+    from unittest.mock import patch
+
+    from litellm.proxy._experimental.mcp_server import server as mcp_server
+    from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_gateway_server_name
+    from litellm.proxy._types import UserAPIKeyAuth
+
+    delegate = _http_server(
+        "s1", "delegate_docs", auth_type=MCPAuth.oauth2, delegate_auth_to_upstream=True
+    )
+
+    async def fake_get_tools(server, **kwargs):
+        raise MCPUpstreamAuthError(status_code=401, www_authenticate=None, server_name=server.name)
+
+    # /<server>/mcp sets the path-derived single-server scope; absorption must hold even then.
+    token = _mcp_gateway_server_name.set("delegate_docs")
+    try:
+        with patch.object(mcp_server, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate])), patch.object(
+            mcp_server, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={})
+        ), patch.object(mcp_server, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object(
+            mcp_server, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None)
+        ), patch.object(
+            mcp_server.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools)
+        ):
+            tools = await mcp_server._get_tools_from_mcp_servers(
+                user_api_key_auth=UserAPIKeyAuth(token="h", user_id="u1"),
+                mcp_auth_header=None,
+                mcp_servers=["delegate_docs"],
+            )
+        assert tools == []
+    finally:
+        _mcp_gateway_server_name.reset(token)
+
+
+@pytest.mark.asyncio
+async def test_aggregate_with_single_accessible_server_still_absorbs():
+    """Regression for the route-misclassification: an aggregate request (/mcp, mcp_servers=None)
+    from a key that can access exactly one server must still absorb that server's
+    MCPUpstreamAuthError, not surface it. Keying the surface decision off the allowed count rather
+    than the request filter would re-raise here and leave the aggregate broken for one-server
+    permission sets."""
+    from unittest.mock import patch
+
+    from litellm.proxy._experimental.mcp_server import server as mcp_server
+    from litellm.proxy._types import UserAPIKeyAuth
+
+    delegate = _http_server(
+        "s1", "delegate_docs", auth_type=MCPAuth.oauth2, delegate_auth_to_upstream=True
+    )
+
+    async def fake_get_tools(server, **kwargs):
+        raise MCPUpstreamAuthError(status_code=401, www_authenticate=None, server_name=server.name)
+
+    with patch.object(mcp_server, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate])), patch.object(
+        mcp_server, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={})
+    ), patch.object(mcp_server, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object(
+        mcp_server, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None)
+    ), patch.object(
+        mcp_server.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools)
+    ):
+        # Aggregate route: no explicit server filter, even though only one server is accessible.
+        tools = await mcp_server._get_tools_from_mcp_servers(
+            user_api_key_auth=UserAPIKeyAuth(token="h", user_id="u1"),
+            mcp_auth_header=None,
+            mcp_servers=None,
+        )
+
+    assert tools == []

From d6f09c4f245bcf4cfa1f5be2abab6e70b910e525 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 30 Jun 2026 19:04:19 +0000
Subject: [PATCH 22/51] test(reasoning-effort-grid): bump cell-count assertion
 for claude-sonnet-5

The Sonnet 5 grid entry raised the Anthropic direct route to 30 model
combos, so test_grid_cell_count now expects 330 cells instead of 319.
---
 .../reasoning_effort_grid/test_reasoning_effort_grid.py       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py
index 304743b1f3c..2409067ebbe 100644
--- a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py
+++ b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py
@@ -201,8 +201,8 @@ async def test_reasoning_effort_grid(
 
 
 def test_grid_cell_count() -> None:
-    assert len(_PARAMS) == 29 * 11, (
-        f"expected 319 cells (29 provider x model combos x 11 efforts), "
+    assert len(_PARAMS) == 30 * 11, (
+        f"expected 330 cells (30 provider x model combos x 11 efforts), "
         f"got {len(_PARAMS)}"
     )
 

From 87f035b58f31e1d43b7e5ed49d29974bbc919191 Mon Sep 17 00:00:00 2001
From: Yassin Kortam <yassin@berri.ai>
Date: Tue, 30 Jun 2026 22:07:47 +0300
Subject: [PATCH 23/51] perf(spend): gather independent per-scope spend-counter
 increments (#31578)

---
 litellm/proxy/proxy_server.py                 | 187 ++++++++++--------
 .../proxy/proxy_server/test_spend_counters.py | 186 +++++++++++++++++
 2 files changed, 291 insertions(+), 82 deletions(-)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 6298a5a98b1..57814de6e3f 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -470,6 +470,7 @@ from litellm.proxy.response_api_endpoints.endpoints import router as response_ro
 from litellm.proxy.route_llm_request import route_request
 from litellm.proxy.search_endpoints.endpoints import router as search_router
 from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
+from litellm.proxy.spend_tracking.budget_reservation import get_budget_window_start
 from litellm.proxy.spend_tracking.spend_management_endpoints import (
     router as spend_management_router,
 )
@@ -2263,111 +2264,133 @@ async def increment_spend_counters(
             budget_reservation["finalized"] = True
         return
 
-    if token is not None:
-        # token arrives pre-hashed from metadata["user_api_key"] (auth flow
+    cost: float = response_cost
+
+    async def _key_scope(key_token: str) -> None:
+        # key_token arrives pre-hashed from metadata["user_api_key"] (auth flow
         # hashes raw "sk-..." keys before they reach the callback). The
         # startswith("sk-") check is a safety net matching update_cache —
         # if a raw key somehow arrives, hash it; otherwise use as-is to
         # avoid double-hashing (budget checks read valid_token.token which
         # is single-hashed).
-        hashed_token = hash_token(token=token) if isinstance(token, str) and token.startswith("sk-") else token
+        hashed_token = (
+            hash_token(token=key_token) if isinstance(key_token, str) and key_token.startswith("sk-") else key_token
+        )
         key_counter_key = f"spend:key:{hashed_token}"
         if key_counter_key not in reserved_counter_keys:
             await _init_and_increment_spend_counter(
                 counter_key=key_counter_key,
                 source_cache_key=hashed_token,
-                increment=response_cost,
+                increment=cost,
             )
 
-        # Increment per-window budget counters for multi-budget keys
         key_obj = await user_api_key_cache.async_get_cache(key=hashed_token)
-        if key_obj is not None:
-            key_budget_limits = getattr(key_obj, "budget_limits", None) or (
-                key_obj.get("budget_limits") if isinstance(key_obj, dict) else None
-            )
-            if isinstance(key_budget_limits, str):
-                key_budget_limits = json.loads(key_budget_limits)
-            if isinstance(key_budget_limits, list):
-                for window in key_budget_limits:
-                    duration = window["budget_duration"] if isinstance(window, dict) else window.budget_duration
-                    key_window_counter = f"spend:key:{hashed_token}:window:{duration}"
-                    if key_window_counter not in reserved_counter_keys:
-                        from litellm.proxy.spend_tracking.budget_reservation import (
-                            get_budget_window_start,
-                        )
+        if key_obj is None:
+            return
+        key_budget_limits = getattr(key_obj, "budget_limits", None) or (
+            key_obj.get("budget_limits") if isinstance(key_obj, dict) else None
+        )
+        if isinstance(key_budget_limits, str):
+            key_budget_limits = json.loads(key_budget_limits)
+        if not isinstance(key_budget_limits, list):
+            return
+        for window in key_budget_limits:
+            duration = window["budget_duration"] if isinstance(window, dict) else window.budget_duration
+            key_window_counter = f"spend:key:{hashed_token}:window:{duration}"
+            if key_window_counter not in reserved_counter_keys:
+                await _init_and_increment_window_spend_counter(
+                    counter_key=key_window_counter,
+                    entity_type="Key",
+                    entity_id=hashed_token,
+                    window_start=get_budget_window_start(window),
+                    increment=cost,
+                )
 
-                        await _init_and_increment_window_spend_counter(
-                            counter_key=key_window_counter,
-                            entity_type="Key",
-                            entity_id=hashed_token,
-                            window_start=get_budget_window_start(window),
-                            increment=response_cost,
-                        )
-
-    if team_id is not None:
-        team_counter_key = f"spend:team:{team_id}"
+    async def _team_scope(scope_team_id: str) -> None:
+        team_counter_key = f"spend:team:{scope_team_id}"
         if team_counter_key not in reserved_counter_keys:
             await _init_and_increment_spend_counter(
                 counter_key=team_counter_key,
-                source_cache_key=f"team_id:{team_id}",
-                increment=response_cost,
+                source_cache_key=f"team_id:{scope_team_id}",
+                increment=cost,
             )
 
-        # Increment per-window budget counters for multi-budget teams
-        team_obj = await user_api_key_cache.async_get_cache(key=f"team_id:{team_id}")
-        if team_obj is not None:
-            team_budget_limits = getattr(team_obj, "budget_limits", None) or (
-                team_obj.get("budget_limits") if isinstance(team_obj, dict) else None
+        team_obj = await user_api_key_cache.async_get_cache(key=f"team_id:{scope_team_id}")
+        if team_obj is None:
+            return
+        team_budget_limits = getattr(team_obj, "budget_limits", None) or (
+            team_obj.get("budget_limits") if isinstance(team_obj, dict) else None
+        )
+        if isinstance(team_budget_limits, str):
+            team_budget_limits = json.loads(team_budget_limits)
+        if not isinstance(team_budget_limits, list):
+            return
+        for window in team_budget_limits:
+            duration = window["budget_duration"] if isinstance(window, dict) else window.budget_duration
+            team_window_counter = f"spend:team:{scope_team_id}:window:{duration}"
+            if team_window_counter not in reserved_counter_keys:
+                await _init_and_increment_window_spend_counter(
+                    counter_key=team_window_counter,
+                    entity_type="Team",
+                    entity_id=scope_team_id,
+                    window_start=get_budget_window_start(window),
+                    increment=cost,
+                )
+
+    async def _team_member_scope(scope_user_id: str, scope_team_id: str) -> None:
+        team_member_counter_key = f"spend:team_member:{scope_user_id}:{scope_team_id}"
+        if team_member_counter_key in reserved_counter_keys:
+            return
+        await _init_and_increment_spend_counter(
+            counter_key=team_member_counter_key,
+            source_cache_key=f"team_membership:{scope_user_id}:{scope_team_id}",
+            increment=cost,
+        )
+
+    async def _user_scope(scope_user_id: str) -> None:
+        user_counter_key = f"spend:user:{scope_user_id}"
+        if user_counter_key in reserved_counter_keys:
+            return
+        await _init_and_increment_spend_counter(
+            counter_key=user_counter_key,
+            source_cache_key=scope_user_id,
+            increment=cost,
+        )
+
+    scope_coros = tuple(
+        coro
+        for coro in (
+            _key_scope(token) if token is not None else None,
+            _team_scope(team_id) if team_id is not None else None,
+            _team_member_scope(user_id, team_id) if user_id is not None and team_id is not None else None,
+            _user_scope(user_id) if user_id is not None else None,
+            _increment_end_user_and_tag_spend_counters(
+                end_user_id=end_user_id,
+                tags=tags,
+                response_cost=cost,
+                reserved_counter_keys=reserved_counter_keys,
             )
-            if isinstance(team_budget_limits, str):
-                team_budget_limits = json.loads(team_budget_limits)
-            if isinstance(team_budget_limits, list):
-                for window in team_budget_limits:
-                    duration = window["budget_duration"] if isinstance(window, dict) else window.budget_duration
-                    team_window_counter = f"spend:team:{team_id}:window:{duration}"
-                    if team_window_counter not in reserved_counter_keys:
-                        from litellm.proxy.spend_tracking.budget_reservation import (
-                            get_budget_window_start,
-                        )
-
-                        await _init_and_increment_window_spend_counter(
-                            counter_key=team_window_counter,
-                            entity_type="Team",
-                            entity_id=team_id,
-                            window_start=get_budget_window_start(window),
-                            increment=response_cost,
-                        )
-
-    if user_id is not None and team_id is not None:
-        team_member_counter_key = f"spend:team_member:{user_id}:{team_id}"
-        if team_member_counter_key not in reserved_counter_keys:
-            await _init_and_increment_spend_counter(
-                counter_key=team_member_counter_key,
-                source_cache_key=f"team_membership:{user_id}:{team_id}",
-                increment=response_cost,
+            if end_user_id is not None or tags is not None
+            else None,
+            _increment_org_spend_counter(
+                org_id=org_id,
+                response_cost=cost,
+                reserved_counter_keys=reserved_counter_keys,
             )
-
-    if user_id is not None:
-        user_counter_key = f"spend:user:{user_id}"
-        if user_counter_key not in reserved_counter_keys:
-            await _init_and_increment_spend_counter(
-                counter_key=user_counter_key,
-                source_cache_key=user_id,
-                increment=response_cost,
-            )
-
-    await _increment_end_user_and_tag_spend_counters(
-        end_user_id=end_user_id,
-        tags=tags,
-        response_cost=response_cost,
-        reserved_counter_keys=reserved_counter_keys,
+            if org_id is not None
+            else None,
+        )
+        if coro is not None
     )
 
-    await _increment_org_spend_counter(
-        org_id=org_id,
-        response_cost=response_cost,
-        reserved_counter_keys=reserved_counter_keys,
-    )
+    # return_exceptions so a failing scope does not leave its siblings running
+    # as orphaned tasks that race the caller's reservation-counter invalidation;
+    # all scopes settle, then the first error propagates as before.
+    scope_results = await asyncio.gather(*scope_coros, return_exceptions=True)
+    scope_errors = [r for r in scope_results if isinstance(r, BaseException)]
+    if scope_errors:
+        raise scope_errors[0]
+
     if budget_reservation is not None:
         budget_reservation["finalized"] = True
 
diff --git a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py
index a839d82984c..51980342a1d 100644
--- a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py
+++ b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py
@@ -20,6 +20,7 @@ Pins covered:
 
 from __future__ import annotations
 
+import asyncio
 from datetime import datetime
 from unittest.mock import AsyncMock, MagicMock
 
@@ -420,6 +421,191 @@ async def test_increment_spend_counters_increments_all_buckets(monkeypatch):
     }
 
 
+class _ConcurrencyProbe:
+    """Stand-in for redis_cache.async_increment that pins concurrency.
+
+    Each call registers itself as in-flight and blocks on ``release`` until the
+    test lets it proceed. ``all_arrived`` fires once ``expected`` distinct scope
+    increments are simultaneously suspended here, which can only happen if the
+    per-scope increments are gathered rather than awaited one after another.
+    """
+
+    def __init__(self, expected_concurrency: int):
+        self.expected = expected_concurrency
+        self.in_flight = 0
+        self.max_in_flight = 0
+        self.all_arrived = asyncio.Event()
+        self.release = asyncio.Event()
+        self.values: dict[str, float] = {}
+
+    async def async_increment(self, *, key, value, refresh_ttl=True):
+        self.in_flight += 1
+        self.max_in_flight = max(self.max_in_flight, self.in_flight)
+        if self.in_flight >= self.expected:
+            self.all_arrived.set()
+        if not self.release.is_set():
+            await self.release.wait()
+        self.in_flight -= 1
+        self.values[key] = self.values.get(key, 0.0) + value
+        return self.values[key]
+
+
+@pytest.mark.asyncio
+async def test_increment_spend_counters_runs_scopes_concurrently(monkeypatch):
+    """The six independent scopes (key, team, team_member, user, end_user+tags,
+    org) must be incremented concurrently. The probe only fires once all six are
+    suspended in async_increment at the same time, which is impossible if the
+    awaits are chained sequentially."""
+    probe = _ConcurrencyProbe(expected_concurrency=6)
+    fake_cache = _make_spend_counter_cache(redis_get_value=None)
+    fake_cache.redis_cache.async_increment = probe.async_increment
+    fake_user_cache = _make_user_api_key_cache(get_value=None)
+    monkeypatch.setattr(ps, "spend_counter_cache", fake_cache)
+    monkeypatch.setattr(ps, "user_api_key_cache", fake_user_cache)
+    monkeypatch.setattr(ps, "prisma_client", None)
+    monkeypatch.setattr(
+        ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None)
+    )
+
+    task = asyncio.create_task(
+        ps.increment_spend_counters(
+            token="hashed-tok",
+            team_id="t1",
+            user_id="u1",
+            org_id="org1",
+            end_user_id="eu1",
+            tags=["a", "b"],
+            response_cost=5.0,
+        )
+    )
+
+    try:
+        await asyncio.wait_for(probe.all_arrived.wait(), timeout=2.0)
+    except asyncio.TimeoutError:
+        probe.release.set()
+        await task
+        pytest.fail(
+            "scope increments did not run concurrently; sequential awaits "
+            f"detected (peak in-flight was {probe.max_in_flight}, expected 6)"
+        )
+
+    assert probe.in_flight == 6
+    assert probe.max_in_flight == 6
+    probe.release.set()
+    await task
+
+    assert probe.values == {
+        "spend:key:hashed-tok": 5.0,
+        "spend:team:t1": 5.0,
+        "spend:team_member:u1:t1": 5.0,
+        "spend:user:u1": 5.0,
+        "spend:end_user:eu1": 5.0,
+        "spend:tag:a": 5.0,
+        "spend:tag:b": 5.0,
+        "spend:org:org1": 5.0,
+    }
+
+
+@pytest.mark.asyncio
+async def test_increment_spend_counters_skips_reserved_counter_keys(monkeypatch):
+    """Counters already reserved by a budget reservation are skipped, every
+    other scope is still incremented exactly once, and the reservation is
+    finalized after the gathered work completes."""
+    import litellm.proxy.spend_tracking.budget_reservation as br
+
+    reserved = {"spend:key:hashed-tok", "spend:org:org1"}
+    monkeypatch.setattr(
+        br, "get_reserved_counter_keys", MagicMock(return_value=set(reserved))
+    )
+    monkeypatch.setattr(br, "reconcile_budget_reservation", AsyncMock())
+
+    recorded: dict[str, float] = {}
+
+    async def _record_increment(*, key, value, refresh_ttl=True):
+        recorded[key] = recorded.get(key, 0.0) + value
+        return recorded[key]
+
+    fake_cache = _make_spend_counter_cache(redis_get_value=None)
+    fake_cache.redis_cache.async_increment = _record_increment
+    fake_user_cache = _make_user_api_key_cache(get_value=None)
+    monkeypatch.setattr(ps, "spend_counter_cache", fake_cache)
+    monkeypatch.setattr(ps, "user_api_key_cache", fake_user_cache)
+    monkeypatch.setattr(ps, "prisma_client", None)
+    monkeypatch.setattr(
+        ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None)
+    )
+
+    reservation = {"finalized": False}
+    await ps.increment_spend_counters(
+        token="hashed-tok",
+        team_id="t1",
+        user_id="u1",
+        org_id="org1",
+        end_user_id="eu1",
+        tags=["a"],
+        response_cost=5.0,
+        budget_reservation=reservation,
+    )
+
+    assert reservation["finalized"] is True
+    assert recorded == {
+        "spend:team:t1": 5.0,
+        "spend:team_member:u1:t1": 5.0,
+        "spend:user:u1": 5.0,
+        "spend:end_user:eu1": 5.0,
+        "spend:tag:a": 5.0,
+    }
+
+
+@pytest.mark.asyncio
+async def test_increment_spend_counters_failing_scope_propagates_after_siblings_settle(
+    monkeypatch,
+):
+    """A failure in one scope must propagate to the caller (so it can invalidate
+    reserved counters) while every other scope still settles rather than being
+    left as an orphaned background task, and the reservation is not finalized."""
+    recorded: dict[str, float] = {}
+
+    async def _increment(*, key, value, refresh_ttl=True):
+        if key == "spend:team:t1":
+            raise RuntimeError("redis increment failed")
+        recorded[key] = recorded.get(key, 0.0) + value
+        return recorded[key]
+
+    fake_cache = _make_spend_counter_cache(redis_get_value=None)
+    fake_cache.redis_cache.async_increment = _increment
+    fake_user_cache = _make_user_api_key_cache(get_value=None)
+    monkeypatch.setattr(ps, "spend_counter_cache", fake_cache)
+    monkeypatch.setattr(ps, "user_api_key_cache", fake_user_cache)
+    monkeypatch.setattr(ps, "prisma_client", None)
+    monkeypatch.setattr(
+        ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None)
+    )
+
+    reservation = {"finalized": False}
+    with pytest.raises(RuntimeError, match="redis increment failed"):
+        await ps.increment_spend_counters(
+            token="hashed-tok",
+            team_id="t1",
+            user_id="u1",
+            org_id="org1",
+            end_user_id="eu1",
+            tags=["a"],
+            response_cost=5.0,
+            budget_reservation=reservation,
+        )
+
+    assert reservation["finalized"] is False
+    assert recorded == {
+        "spend:key:hashed-tok": 5.0,
+        "spend:team_member:u1:t1": 5.0,
+        "spend:user:u1": 5.0,
+        "spend:end_user:eu1": 5.0,
+        "spend:tag:a": 5.0,
+        "spend:org:org1": 5.0,
+    }
+
+
 @pytest.mark.asyncio
 async def test_increment_spend_counters_zero_cost_is_noop_finalizes_reservation(
     monkeypatch,

From 3971469b71e4454bc02dc5ee1d5b9bd1618566ad Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Tue, 30 Jun 2026 12:08:15 -0700
Subject: [PATCH 24/51] test(ui): harden Router Settings e2e and make its
 typing a real CI gate

Address an adversarial review of the Loadbalancing e2e:

- The "typed against the backend schema" claim was hollow: nothing type-checked
  e2e_tests (the root tsconfig excludes it and no CI step runs tsc), so a
  contract drift would compile and run unchanged. Add e2e_tests/tsconfig.json, a
  typecheck:e2e script, and a CircleCI step so the schema typing actually gates.
- The two describe blocks both mutate the proxy's shared router_settings, and the
  Loadbalancing save echoes the whole settings object, so they could clobber each
  other under local fullyParallel. Run the file serially.
- patchRouterSettings swallowed a failed seed, which surfaced later as a
  misleading UI timeout. Assert the write succeeded, and rely on the server-side
  merge instead of echoing the whole settings object back (drops a cast and a GET).
- Empty routing_groups already reproduces the bug, so drop the non-empty seed and
  its model coupling.
---
 .circleci/config.yml                          |  8 +++++
 .../tests/settings/routerSettings.spec.ts     | 36 +++++++++++--------
 ui/litellm-dashboard/e2e_tests/tsconfig.json  |  9 +++++
 ui/litellm-dashboard/package.json             |  1 +
 4 files changed, 39 insertions(+), 15 deletions(-)
 create mode 100644 ui/litellm-dashboard/e2e_tests/tsconfig.json

diff --git a/.circleci/config.yml b/.circleci/config.yml
index f13e9bf66f1..009884cbfe4 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -2687,6 +2687,14 @@ jobs:
           paths:
             - ui/litellm-dashboard/node_modules
             - ~/.cache/ms-playwright
+      - run:
+          name: Type-check E2E specs
+          # The specs type their request/response round-trips against the generated
+          # OpenAPI schema; this step turns that typing into a real gate, so a backend
+          # contract drift fails here instead of silently passing at runtime.
+          command: |
+            cd ui/litellm-dashboard
+            npm run typecheck:e2e
       - run:
           name: Build UI from source
           # Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`.
diff --git a/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts
index e560500fca6..64dab6d7cf6 100644
--- a/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts
+++ b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts
@@ -3,11 +3,16 @@ import { ADMIN_STORAGE_PATH } from "../../constants";
 import { navigateToPage } from "../../helpers/navigation";
 import { Page } from "../../fixtures/pages";
 import { Role, users } from "../../fixtures/users";
-// Type-only import of the OpenAPI-generated backend schema; esbuild erases it at
-// runtime, so the round-trip below is checked against the real /config/update and
-// /router/settings contracts without bundling the 2 MB definition file.
+// Type-only import of the OpenAPI-generated backend schema. esbuild erases it at
+// runtime; the round-trip below is enforced by the `typecheck:e2e` CI step (tsc over
+// e2e_tests), so a drift in the /config/update or /router/settings contract fails the
+// build rather than silently passing here.
 import type { components } from "../../../src/lib/http/schema";
 
+// These tests mutate the proxy's shared router_settings, and the Loadbalancing save
+// echoes the whole settings object, so they must not run concurrently.
+test.describe.configure({ mode: "serial" });
+
 const PRIMARY = "fake-openai-gpt-4";
 const FALLBACK = "fake-anthropic-claude";
 
@@ -111,32 +116,33 @@ const BASE_URL = "http://localhost:4000";
 const ADMIN_AUTH = { Authorization: `Bearer ${users[Role.ProxyAdmin].password}` };
 
 /**
- * Merge a router_settings patch into the live config through the typed
- * /config/update contract, preserving any other settings already present.
+ * Apply a router_settings patch through the typed /config/update contract. The
+ * server merges it over existing settings (request wins), so only the passed keys
+ * change. Fails loudly if the write is rejected instead of leaving a silent bad seed.
  */
 async function patchRouterSettings(
   request: import("@playwright/test").APIRequestContext,
   patch: Partial<NonNullable<ConfigYAML["router_settings"]>>,
 ) {
-  const current = await request.get(`${BASE_URL}/get/config/callbacks`, { headers: ADMIN_AUTH });
-  const existing = current.ok() ? (await current.json())?.router_settings ?? {} : {};
-  const payload = { router_settings: { ...(existing as Record<string, unknown>), ...patch } };
-  await request.post(`${BASE_URL}/config/update`, { headers: ADMIN_AUTH, data: payload });
+  const res = await request.post(`${BASE_URL}/config/update`, {
+    headers: ADMIN_AUTH,
+    data: { router_settings: patch },
+  });
+  expect(res.ok(), `seed /config/update failed: ${res.status()} ${await res.text()}`).toBeTruthy();
 }
 
 test.describe("Router Settings - Loadbalancing", () => {
   test.use({ storageState: ADMIN_STORAGE_PATH });
 
-  // Seed a present routing_groups array (the LIT-4057 trigger) plus a known
-  // num_retries so the UI assertions are deterministic across reruns.
-  const ROUTING_GROUP = { group_name: "e2e-lit-4057", models: [PRIMARY], routing_strategy: "simple-shuffle" };
-
+  // Pin num_retries and an empty routing_groups so the assertions are deterministic.
+  // Empty already reproduces LIT-4057: the old tab serialized [] to the string "[]"
+  // and the save 422'd.
   test.beforeEach(async ({ request }) => {
-    await patchRouterSettings(request, { num_retries: 3, routing_groups: [ROUTING_GROUP] });
+    await patchRouterSettings(request, { num_retries: 3, routing_groups: [] });
   });
 
   test.afterEach(async ({ request }) => {
-    await patchRouterSettings(request, { num_retries: 3, routing_groups: [] });
+    await patchRouterSettings(request, { num_retries: 3 });
   });
 
   test("saves the Loadbalancing tab without a 422 when routing_groups is present, and persists", async ({
diff --git a/ui/litellm-dashboard/e2e_tests/tsconfig.json b/ui/litellm-dashboard/e2e_tests/tsconfig.json
new file mode 100644
index 00000000000..abda3fdb8b6
--- /dev/null
+++ b/ui/litellm-dashboard/e2e_tests/tsconfig.json
@@ -0,0 +1,9 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "noEmit": true,
+    "types": ["node"]
+  },
+  "include": ["**/*.ts"],
+  "exclude": ["node_modules"]
+}
diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json
index a8948f4be34..1d2b505a172 100644
--- a/ui/litellm-dashboard/package.json
+++ b/ui/litellm-dashboard/package.json
@@ -19,6 +19,7 @@
     "e2e:ui": "playwright test --ui --config e2e_tests/playwright.config.ts",
     "e2e:migration": "playwright test e2e_tests/tests/migration/migratedPages.spec.ts --config e2e_tests/playwright.config.ts",
     "e2e:migration:root": "playwright test --config e2e_tests/migration.serverRootPath.config.ts",
+    "typecheck:e2e": "tsc -p e2e_tests/tsconfig.json --noEmit",
     "knip": "knip",
     "knip:fix": "knip --fix",
     "gen:api": "node scripts/gen-api-types.mjs"

From 88c34a5bad735c98804d427ce6d30a0bd5bea124 Mon Sep 17 00:00:00 2001
From: michelligabriele <gabriele.michelli@icloud.com>
Date: Tue, 30 Jun 2026 21:11:50 +0200
Subject: [PATCH 25/51] fix(email): apply EMAIL_SIGNATURE to budget alert
 emails (#31712)

---
 .../send_emails/base_email.py                 |   4 +
 .../integrations/email_templates/templates.py |   9 +-
 .../send_emails/test_base_email.py            | 130 ++++++++++++++++++
 3 files changed, 137 insertions(+), 6 deletions(-)

diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py b/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py
index 89c3b854686..9d15f45079f 100644
--- a/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py
+++ b/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py
@@ -239,6 +239,7 @@ class BaseEmailLogger(CustomLogger):
             max_budget_info=max_budget_info,
             base_url=email_params.base_url,
             email_support_contact=email_params.support_contact,
+            email_footer=email_params.signature,
         )
         await self.send_email(
             from_email=self.DEFAULT_LITELLM_EMAIL,
@@ -311,6 +312,7 @@ class BaseEmailLogger(CustomLogger):
             max_budget_info=max_budget_info,
             base_url=email_params.base_url,
             email_support_contact=email_params.support_contact,
+            email_footer=email_params.signature,
         )
 
         # Send email to all recipients
@@ -379,6 +381,7 @@ class BaseEmailLogger(CustomLogger):
                 alert_threshold=alert_threshold_str,
                 base_url=email_params.base_url,
                 email_support_contact=email_params.support_contact,
+                email_footer=email_params.signature,
             )
             await self.send_email(
                 from_email=self.DEFAULT_LITELLM_EMAIL,
@@ -403,6 +406,7 @@ class BaseEmailLogger(CustomLogger):
                 alert_threshold=alert_threshold_str,
                 base_url=email_params.base_url,
                 email_support_contact=email_params.support_contact,
+                email_footer=email_params.signature,
             )
             await self.send_email(
                 from_email=self.DEFAULT_LITELLM_EMAIL,
diff --git a/litellm/integrations/email_templates/templates.py b/litellm/integrations/email_templates/templates.py
index 8df816dfecd..b4f39074a94 100644
--- a/litellm/integrations/email_templates/templates.py
+++ b/litellm/integrations/email_templates/templates.py
@@ -81,8 +81,7 @@ SOFT_BUDGET_ALERT_EMAIL_TEMPLATE = """
 
                     If you have any questions, please send an email to {email_support_contact} <br /> <br />
 
-                    Best, <br />
-                    The LiteLLM team <br />
+                    {email_footer}
 """
 
 TEAM_SOFT_BUDGET_ALERT_EMAIL_TEMPLATE = """
@@ -105,8 +104,7 @@ TEAM_SOFT_BUDGET_ALERT_EMAIL_TEMPLATE = """
 
                     If you have any questions, please send an email to {email_support_contact} <br /> <br />
 
-                    Best, <br />
-                    The LiteLLM team <br />
+                    {email_footer}
 """
 
 MAX_BUDGET_ALERT_EMAIL_TEMPLATE = """
@@ -129,6 +127,5 @@ MAX_BUDGET_ALERT_EMAIL_TEMPLATE = """
 
                     If you have any questions, please send an email to {email_support_contact} <br /> <br />
 
-                    Best, <br />
-                    The LiteLLM team <br />
+                    {email_footer}
 """
diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py
index 5cabfe5fb7f..db23b712125 100644
--- a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py
+++ b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py
@@ -1112,3 +1112,133 @@ async def test_no_map_preserves_old_single_threshold(
         # Old path cache key has no threshold percentage
         cache_key = mock_cache.async_set_cache.call_args[1]["key"]
         assert cache_key == "email_budget_alerts:max_budget_alert:test_user"
+
+
+CUSTOM_SIGNATURE = "<div>Best,<br/>The Acme Platform Team</div>"
+
+
+@pytest.mark.asyncio
+async def test_send_soft_budget_alert_email_uses_custom_signature(
+    base_email_logger, mock_send_email, mock_lookup_user_email
+):
+    """Soft budget alert honors EMAIL_SIGNATURE for premium users."""
+    event = WebhookEvent(
+        user_id="test_user",
+        user_email="test@example.com",
+        event_group=Litellm_EntityType.USER,
+        event="soft_budget_crossed",
+        event_message="Soft Budget Crossed",
+        spend=105.0,
+        max_budget=200.0,
+        soft_budget=100.0,
+    )
+    with mock.patch.dict(
+        os.environ,
+        {"PROXY_BASE_URL": "http://test.com", "EMAIL_SIGNATURE": CUSTOM_SIGNATURE},
+    ), patch("litellm.proxy.proxy_server.premium_user", True):
+        await base_email_logger.send_soft_budget_alert_email(event)
+
+        html_body = mock_send_email.call_args[1]["html_body"]
+        assert CUSTOM_SIGNATURE in html_body
+        assert "The LiteLLM team" not in html_body
+
+
+@pytest.mark.asyncio
+async def test_send_team_soft_budget_alert_email_uses_custom_signature(
+    base_email_logger, mock_send_email, mock_lookup_user_email
+):
+    """Team soft budget alert honors EMAIL_SIGNATURE for premium users."""
+    event = WebhookEvent(
+        user_id="test_user",
+        event_group=Litellm_EntityType.TEAM,
+        event="soft_budget_crossed",
+        event_message="Team Soft Budget Crossed",
+        spend=105.0,
+        max_budget=200.0,
+        soft_budget=100.0,
+        team_alias="Acme",
+        alert_emails=["teamlead@example.com"],
+    )
+    with mock.patch.dict(
+        os.environ,
+        {"PROXY_BASE_URL": "http://test.com", "EMAIL_SIGNATURE": CUSTOM_SIGNATURE},
+    ), patch("litellm.proxy.proxy_server.premium_user", True):
+        await base_email_logger.send_team_soft_budget_alert_email(event)
+
+        html_body = mock_send_email.call_args[1]["html_body"]
+        assert CUSTOM_SIGNATURE in html_body
+        assert "The LiteLLM team" not in html_body
+
+
+@pytest.mark.asyncio
+async def test_send_max_budget_alert_email_single_recipient_uses_custom_signature(
+    base_email_logger, mock_send_email, mock_lookup_user_email
+):
+    """Max budget alert (single-recipient path) honors EMAIL_SIGNATURE."""
+    event = WebhookEvent(
+        user_id="test_user",
+        user_email="test@example.com",
+        event_group=Litellm_EntityType.USER,
+        event="max_budget_alert",
+        event_message="Max Budget Alert",
+        spend=165.0,
+        max_budget=200.0,
+    )
+    with mock.patch.dict(
+        os.environ,
+        {"PROXY_BASE_URL": "http://test.com", "EMAIL_SIGNATURE": CUSTOM_SIGNATURE},
+    ), patch("litellm.proxy.proxy_server.premium_user", True):
+        await base_email_logger.send_max_budget_alert_email(event)
+
+        html_body = mock_send_email.call_args[1]["html_body"]
+        assert CUSTOM_SIGNATURE in html_body
+        assert "The LiteLLM team" not in html_body
+
+
+@pytest.mark.asyncio
+async def test_send_max_budget_alert_email_multi_recipient_uses_custom_signature(
+    base_email_logger, mock_send_email, mock_lookup_user_email
+):
+    """Max budget alert (multi-threshold/recipient path) honors EMAIL_SIGNATURE."""
+    event = WebhookEvent(
+        user_id="test_user",
+        user_email="owner@example.com",
+        event_group=Litellm_EntityType.USER,
+        event="max_budget_alert",
+        event_message="Max Budget Alert",
+        spend=165.0,
+        max_budget=200.0,
+    )
+    with mock.patch.dict(
+        os.environ,
+        {"PROXY_BASE_URL": "http://test.com", "EMAIL_SIGNATURE": CUSTOM_SIGNATURE},
+    ), patch("litellm.proxy.proxy_server.premium_user", True):
+        await base_email_logger.send_max_budget_alert_email(
+            event, threshold_pct=75, recipient_emails=["a@example.com", "b@example.com"]
+        )
+
+        html_body = mock_send_email.call_args[1]["html_body"]
+        assert CUSTOM_SIGNATURE in html_body
+        assert "The LiteLLM team" not in html_body
+
+
+@pytest.mark.asyncio
+async def test_send_soft_budget_alert_email_default_footer_when_no_signature(
+    base_email_logger, mock_send_email, mock_lookup_user_email
+):
+    """Without EMAIL_SIGNATURE, budget alert falls back to the default EMAIL_FOOTER."""
+    event = WebhookEvent(
+        user_id="test_user",
+        user_email="test@example.com",
+        event_group=Litellm_EntityType.USER,
+        event="soft_budget_crossed",
+        event_message="Soft Budget Crossed",
+        spend=105.0,
+        max_budget=200.0,
+        soft_budget=100.0,
+    )
+    with mock.patch.dict(os.environ, {"PROXY_BASE_URL": "http://test.com"}):
+        await base_email_logger.send_soft_budget_alert_email(event)
+
+        html_body = mock_send_email.call_args[1]["html_body"]
+        assert EMAIL_FOOTER in html_body

From 6d828e5759dc76ecc5d582861fd56e27a5763c2d Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 30 Jun 2026 12:17:33 -0700
Subject: [PATCH 26/51] feat(messages): passthrough /v1/messages to native
 endpoints via supported_endpoints (#31685)

* feat(messages): passthrough /v1/messages to native endpoints via supported_endpoints

The unified /v1/messages proxy endpoint always translated inbound Anthropic
requests down to /v1/chat/completions (or the Responses API for openai) when the
deployment's provider lacked a native Anthropic-messages config, dropping
Anthropic-only features like cache_control and thinking. Some customers run
OpenAI-compatible servers (self-hosted vLLM, DeepSeek's Anthropic endpoint, etc.)
that also natively expose /v1/messages and want the raw Anthropic payload
forwarded untranslated, while keeping provider openai so /v1/chat/completions to
the same deployment stays native.

Opt in per deployment via model_info.supported_endpoints containing
/v1/messages. When present, the gate routes to a generic, provider-agnostic
OpenAILikeAnthropicMessagesConfig that POSTs the Anthropic payload to
{api_base}/v1/messages with Bearer auth, instead of translating. Default
behavior is unchanged. Generalizes and supersedes the hosted_vllm-only,
env-var-toggled PR #28745.

* fix(messages): preserve standard-cased caller headers in native passthrough

The OpenAI-like Anthropic passthrough config only checked for lowercase header
names before injecting Bearer auth, anthropic-version, and content-type
defaults. A caller sending standard-cased Authorization, Anthropic-Version, or
Content-Type was treated as missing those headers, so LiteLLM added duplicate
lowercase variants and overwrote the caller's credential/version at the HTTP
layer. Header presence is now checked case-insensitively and the merge no longer
mutates the caller dict.

Also moves the feature docs out of the main repo (docs live in litellm-docs).

* fix(openai_like/messages): delegate to parent transform and inject anthropic-beta headers

The passthrough config bypassed the parent transform and skipped header beta injection. Both gaps cause native /v1/messages features (context management, advisor tool, fast mode, structured outputs, reasoning_effort, advisor stripping) to silently degrade on opted-in deployments. Reuse the parent's pipeline and call _update_headers_with_anthropic_beta after merging defaults

* fix: normalize anthropic-beta header key case before beta injection

* style: collapse anthropic-beta header normalization to single line

ruff format --check requires the comprehension on one line (it fits within
the 120 char limit); fixes the lint job failure on the bugbot autofix commit

* fix(messages): forward anthropic-beta to native passthrough upstream

The shared anthropic_messages HTTP handler ran update_headers_with_filtered_beta
with the deployment's custom_llm_provider after validate. For the native
/v1/messages passthrough that provider is openai, which has no beta-header
mapping, so every anthropic-beta value (caller-supplied or feature-derived for
speed/context_management/etc.) was stripped to empty before the upstream
request, breaking beta passthrough to the Anthropic-compatible endpoint.

Beta filtering only makes sense on cross-provider translation paths where the
upstream cannot understand Anthropic betas. Gate it on a new
should_filter_anthropic_beta_headers() that defaults to True (bedrock, vertex_ai,
native anthropic unchanged) and is overridden to False by
OpenAILikeAnthropicMessagesConfig, whose upstream is a native Anthropic endpoint,
so betas pass through verbatim.

* chore: remove accidentally committed local QA logs and config

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
---
 .../messages/handler.py                       |  21 ++
 .../anthropic_messages/transformation.py      |  11 +
 litellm/llms/custom_httpx/llm_http_handler.py |   3 +-
 litellm/llms/openai_like/messages/__init__.py |   0
 .../openai_like/messages/transformation.py    |  69 ++++
 ...erimental_pass_through_messages_handler.py | 106 ++++++
 .../llms/openai_like/messages/__init__.py     |   0
 ..._like_anthropic_messages_transformation.py | 301 ++++++++++++++++++
 8 files changed, 510 insertions(+), 1 deletion(-)
 create mode 100644 litellm/llms/openai_like/messages/__init__.py
 create mode 100644 litellm/llms/openai_like/messages/transformation.py
 create mode 100644 tests/test_litellm/llms/openai_like/messages/__init__.py
 create mode 100644 tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py

diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py
index 9c9427c7302..547ddd9b8d3 100644
--- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py
+++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py
@@ -61,6 +61,19 @@ def _should_route_to_responses_api(custom_llm_provider: Optional[str]) -> bool:
     return custom_llm_provider in _RESPONSES_API_PROVIDERS
 
 
+def _deployment_passes_through_anthropic_messages(model_info: object) -> bool:
+    """Whether the deployment opted into forwarding /v1/messages untranslated.
+
+    The opt-in is ``model_info.supported_endpoints`` containing ``"/v1/messages"``,
+    declared per deployment in config.yaml and plumbed here as ``kwargs["model_info"]``
+    by the router.
+    """
+    if not isinstance(model_info, dict):
+        return False
+    supported_endpoints = model_info.get("supported_endpoints")
+    return isinstance(supported_endpoints, (list, tuple)) and "/v1/messages" in supported_endpoints
+
+
 ####### ENVIRONMENT VARIABLES ###################
 # Initialize any necessary instances or variables here
 base_llm_http_handler = BaseLLMHTTPHandler()
@@ -456,6 +469,14 @@ def anthropic_messages_handler(
             model=model,
             provider=litellm.LlmProviders(custom_llm_provider),
         )
+    if anthropic_messages_provider_config is None and _deployment_passes_through_anthropic_messages(
+        kwargs.get("model_info")
+    ):
+        from litellm.llms.openai_like.messages.transformation import (
+            OpenAILikeAnthropicMessagesConfig,
+        )
+
+        anthropic_messages_provider_config = OpenAILikeAnthropicMessagesConfig()
     if anthropic_messages_provider_config is None:
         # Route to Responses API for OpenAI / Azure, chat/completions for everything else.
         _shared_kwargs = dict(
diff --git a/litellm/llms/base_llm/anthropic_messages/transformation.py b/litellm/llms/base_llm/anthropic_messages/transformation.py
index 7f8403c0223..966995bc571 100644
--- a/litellm/llms/base_llm/anthropic_messages/transformation.py
+++ b/litellm/llms/base_llm/anthropic_messages/transformation.py
@@ -103,6 +103,17 @@ class BaseAnthropicMessagesConfig(ABC):
         """
         return headers, None
 
+    def should_filter_anthropic_beta_headers(self) -> bool:
+        """
+        Whether ``anthropic-beta`` header values should be filtered down to the
+        ones the routed provider supports before the upstream request.
+
+        Cross-provider translation paths (bedrock, vertex_ai, ...) need this so
+        unsupported betas are dropped. Configs that forward natively to an
+        Anthropic-compatible endpoint return False to pass betas through verbatim.
+        """
+        return True
+
     def get_async_streaming_response_iterator(
         self,
         model: str,
diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index 9f18b669124..9bb956d0808 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -1986,7 +1986,8 @@ class BaseLLMHTTPHandler:
             api_base=api_base,
         )
 
-        headers = update_headers_with_filtered_beta(headers=headers, provider=custom_llm_provider)
+        if anthropic_messages_provider_config.should_filter_anthropic_beta_headers():
+            headers = update_headers_with_filtered_beta(headers=headers, provider=custom_llm_provider)
 
         logging_obj.update_from_kwargs(
             kwargs=kwargs,
diff --git a/litellm/llms/openai_like/messages/__init__.py b/litellm/llms/openai_like/messages/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py
new file mode 100644
index 00000000000..0df8c6e830b
--- /dev/null
+++ b/litellm/llms/openai_like/messages/transformation.py
@@ -0,0 +1,69 @@
+from typing import Any, Optional
+
+from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
+    AnthropicMessagesConfig,
+)
+
+DEFAULT_ANTHROPIC_API_VERSION = "2023-06-01"
+
+
+class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig):
+    """
+    Forwards Anthropic /v1/messages requests to an OpenAI-compatible server that
+    also natively exposes the Anthropic Messages API, with no translation.
+
+    Opted into per deployment via ``model_info.supported_endpoints`` containing
+    ``"/v1/messages"``. The inbound Anthropic payload (system, cache_control,
+    thinking, tools, ...) is forwarded essentially unchanged to
+    ``{api_base}/v1/messages``, so Anthropic-only features that the
+    Anthropic->OpenAI translation would otherwise drop are preserved. Response
+    parsing and streaming are inherited from the native Anthropic config.
+    """
+
+    def validate_anthropic_messages_environment(
+        self,
+        headers: dict[str, str],
+        model: str,
+        messages: list[Any],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> tuple[dict[str, str], Optional[str]]:
+        present = {key.lower() for key in headers}
+        needs_auth = bool(api_key) and "authorization" not in present and "x-api-key" not in present
+        defaults: dict[str, str] = {
+            **({"authorization": f"Bearer {api_key}"} if needs_auth else {}),
+            **({"anthropic-version": DEFAULT_ANTHROPIC_API_VERSION} if "anthropic-version" not in present else {}),
+            **({"content-type": "application/json"} if "content-type" not in present else {}),
+        }
+        combined = {**headers, **defaults}
+        normalized = {
+            ("anthropic-beta" if key.lower() == "anthropic-beta" else key): value for key, value in combined.items()
+        }
+        merged = self._update_headers_with_anthropic_beta(
+            headers=normalized,
+            optional_params=optional_params,
+        )
+        return merged, api_base
+
+    def should_filter_anthropic_beta_headers(self) -> bool:
+        return False
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        if not api_base:
+            raise ValueError("api_base is required to forward Anthropic /v1/messages to a native endpoint")
+        base = api_base.rstrip("/")
+        if base.endswith("/v1/messages"):
+            return base
+        if base.endswith("/v1"):
+            base = base[: -len("/v1")]
+        return f"{base}/v1/messages"
diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
index 7bcaf07c5bb..3327fc39f73 100644
--- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
+++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
@@ -715,3 +715,109 @@ async def test_async_wrapper_sets_presanitized_and_sanitizes_once():
     assert spy.call_count == 1
     assert captured["presanitized"] is True
     assert [b["type"] for b in captured["messages"][0]["content"]] == ["tool_use"]
+
+
+def _gate_stubs(monkeypatch):
+    """Patch the gate's downstream dispatch targets so config selection can be
+    observed without making a network call.
+
+    Returns ``(captured, translation_calls)`` where ``captured["config"]`` is the
+    provider config handed to the native passthrough path and ``translation_calls``
+    counts hits on the Anthropic->OpenAI translation handlers.
+    """
+    from litellm.llms.anthropic.experimental_pass_through.messages import handler
+
+    captured = {}
+    translation_calls = {"count": 0}
+
+    def fake_native(**kwargs):
+        captured["config"] = kwargs.get("anthropic_messages_provider_config")
+        return "native-passthrough"
+
+    def fake_translation(**kwargs):
+        translation_calls["count"] += 1
+        return "translated"
+
+    monkeypatch.setattr(handler.base_llm_http_handler, "anthropic_messages_handler", fake_native)
+    monkeypatch.setattr(
+        handler.LiteLLMMessagesToResponsesAPIHandler,
+        "anthropic_messages_handler",
+        staticmethod(fake_translation),
+    )
+    monkeypatch.setattr(
+        handler.LiteLLMMessagesToCompletionTransformationHandler,
+        "anthropic_messages_handler",
+        staticmethod(fake_translation),
+    )
+    return captured, translation_calls
+
+
+def test_gate_passthrough_when_supported_endpoints_opts_in(monkeypatch):
+    """provider=openai + model_info.supported_endpoints containing /v1/messages
+    must route to the native passthrough config, NOT the translation handlers."""
+    from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
+        anthropic_messages_handler,
+    )
+    from litellm.llms.openai_like.messages.transformation import (
+        OpenAILikeAnthropicMessagesConfig,
+    )
+
+    captured, translation_calls = _gate_stubs(monkeypatch)
+
+    result = anthropic_messages_handler(
+        max_tokens=100,
+        messages=[{"role": "user", "content": "Hello"}],
+        model="openai/some-model",
+        api_key="sk-test",
+        api_base="https://host/v1",
+        model_info={"supported_endpoints": ["/v1/chat/completions", "/v1/messages"]},
+    )
+
+    assert result == "native-passthrough"
+    assert isinstance(captured["config"], OpenAILikeAnthropicMessagesConfig)
+    assert translation_calls["count"] == 0
+
+
+def test_gate_translates_when_supported_endpoints_absent(monkeypatch):
+    """Default behavior is unchanged: without the /v1/messages opt-in, an openai
+    deployment is translated (Responses API), never passed through natively."""
+    from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
+        anthropic_messages_handler,
+    )
+
+    captured, translation_calls = _gate_stubs(monkeypatch)
+
+    result = anthropic_messages_handler(
+        max_tokens=100,
+        messages=[{"role": "user", "content": "Hello"}],
+        model="openai/some-model",
+        api_key="sk-test",
+        api_base="https://host/v1",
+    )
+
+    assert result == "translated"
+    assert translation_calls["count"] == 1
+    assert "config" not in captured
+
+
+def test_gate_passthrough_skipped_when_only_chat_completions_supported(monkeypatch):
+    """A deployment that lists only /v1/chat/completions is still translated;
+    the opt-in is specifically the /v1/messages entry."""
+    from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
+        anthropic_messages_handler,
+    )
+
+    captured, translation_calls = _gate_stubs(monkeypatch)
+
+    result = anthropic_messages_handler(
+        max_tokens=100,
+        messages=[{"role": "user", "content": "Hello"}],
+        model="openai/some-model",
+        api_key="sk-test",
+        api_base="https://host/v1",
+        model_info={"supported_endpoints": ["/v1/chat/completions"]},
+    )
+
+    assert result == "translated"
+    assert translation_calls["count"] == 1
+    assert "config" not in captured
diff --git a/tests/test_litellm/llms/openai_like/messages/__init__.py b/tests/test_litellm/llms/openai_like/messages/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py b/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py
new file mode 100644
index 00000000000..534d7aefda4
--- /dev/null
+++ b/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py
@@ -0,0 +1,301 @@
+import pytest
+
+from litellm.llms.anthropic.common_utils import AnthropicError
+from litellm.llms.openai_like.messages.transformation import (
+    OpenAILikeAnthropicMessagesConfig,
+)
+from litellm.types.router import GenericLiteLLMParams
+
+
+@pytest.fixture
+def config() -> OpenAILikeAnthropicMessagesConfig:
+    return OpenAILikeAnthropicMessagesConfig()
+
+
+@pytest.mark.parametrize(
+    "api_base, expected",
+    [
+        ("https://host/v1", "https://host/v1/messages"),
+        ("https://host/v1/", "https://host/v1/messages"),
+        ("https://host", "https://host/v1/messages"),
+        ("https://host/v1/messages", "https://host/v1/messages"),
+        ("https://api.deepseek.com/anthropic", "https://api.deepseek.com/anthropic/v1/messages"),
+        ("https://api.deepseek.com/anthropic/v1", "https://api.deepseek.com/anthropic/v1/messages"),
+    ],
+)
+def test_get_complete_url_handles_api_base_variants(config, api_base, expected):
+    url = config.get_complete_url(
+        api_base=api_base,
+        api_key="sk-test",
+        model="some-model",
+        optional_params={},
+        litellm_params={},
+    )
+    assert url == expected
+
+
+def test_get_complete_url_requires_api_base(config):
+    with pytest.raises(ValueError, match="api_base is required"):
+        config.get_complete_url(
+            api_base=None,
+            api_key="sk-test",
+            model="some-model",
+            optional_params={},
+            litellm_params={},
+        )
+
+
+def test_request_stays_in_anthropic_shape(config):
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "Summarize this",
+                    "cache_control": {"type": "ephemeral"},
+                }
+            ],
+        }
+    ]
+    optional_params = {
+        "max_tokens": 256,
+        "system": "You are a careful assistant",
+        "thinking": {"type": "enabled", "budget_tokens": 1024},
+        "temperature": 0.3,
+        "tools": [{"name": "lookup", "input_schema": {"type": "object"}}],
+        "stream": False,
+    }
+
+    payload = config.transform_anthropic_messages_request(
+        model="some-model",
+        messages=messages,
+        anthropic_messages_optional_request_params=optional_params,
+        litellm_params=GenericLiteLLMParams(),
+        headers={},
+    )
+
+    assert payload["model"] == "some-model"
+    assert payload["messages"] == messages
+    assert payload["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"}
+    assert payload["system"] == "You are a careful assistant"
+    assert payload["thinking"] == {"type": "enabled", "budget_tokens": 1024}
+    assert payload["max_tokens"] == 256
+    assert payload["tools"] == optional_params["tools"]
+
+    openai_only_keys = {
+        "max_completion_tokens",
+        "stop",
+        "n",
+        "logprobs",
+        "response_format",
+        "frequency_penalty",
+    }
+    assert openai_only_keys.isdisjoint(payload.keys())
+
+
+def test_request_requires_max_tokens(config):
+    with pytest.raises(AnthropicError, match="max_tokens is required"):
+        config.transform_anthropic_messages_request(
+            model="some-model",
+            messages=[{"role": "user", "content": "hi"}],
+            anthropic_messages_optional_request_params={"system": "s"},
+            litellm_params=GenericLiteLLMParams(),
+            headers={},
+        )
+
+
+def test_validate_environment_sets_bearer_and_anthropic_defaults(config):
+    headers, api_base = config.validate_anthropic_messages_environment(
+        headers={},
+        model="some-model",
+        messages=[],
+        optional_params={},
+        litellm_params={},
+        api_key="sk-test",
+        api_base="https://host/v1",
+    )
+    assert headers["authorization"] == "Bearer sk-test"
+    assert headers["anthropic-version"] == "2023-06-01"
+    assert headers["content-type"] == "application/json"
+    assert api_base == "https://host/v1"
+
+
+def test_validate_environment_does_not_overwrite_caller_headers(config):
+    headers, _ = config.validate_anthropic_messages_environment(
+        headers={
+            "authorization": "Bearer caller-token",
+            "anthropic-version": "2024-10-22",
+            "content-type": "application/json",
+        },
+        model="some-model",
+        messages=[],
+        optional_params={},
+        litellm_params={},
+        api_key="sk-test",
+        api_base="https://host/v1",
+    )
+    assert headers["authorization"] == "Bearer caller-token"
+    assert headers["anthropic-version"] == "2024-10-22"
+
+
+def test_validate_environment_preserves_standard_cased_caller_headers(config):
+    headers, _ = config.validate_anthropic_messages_environment(
+        headers={
+            "Authorization": "Bearer caller-token",
+            "Anthropic-Version": "2024-10-22",
+            "Content-Type": "application/json",
+        },
+        model="some-model",
+        messages=[],
+        optional_params={},
+        litellm_params={},
+        api_key="sk-test",
+        api_base="https://host/v1",
+    )
+    lowercased = {key.lower() for key in headers}
+    assert len(lowercased) == len(headers)
+    assert headers["Authorization"] == "Bearer caller-token"
+    assert headers["Anthropic-Version"] == "2024-10-22"
+    assert headers["Content-Type"] == "application/json"
+
+
+def test_validate_environment_honors_x_api_key_when_present(config):
+    headers, _ = config.validate_anthropic_messages_environment(
+        headers={"X-Api-Key": "caller-key"},
+        model="some-model",
+        messages=[],
+        optional_params={},
+        litellm_params={},
+        api_key="sk-test",
+        api_base="https://host/v1",
+    )
+    assert "authorization" not in {key.lower() for key in headers}
+    assert headers["X-Api-Key"] == "caller-key"
+
+
+def test_validate_environment_injects_anthropic_beta_for_context_management(config):
+    headers, _ = config.validate_anthropic_messages_environment(
+        headers={},
+        model="some-model",
+        messages=[],
+        optional_params={
+            "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]},
+        },
+        litellm_params={},
+        api_key="sk-test",
+        api_base="https://host/v1",
+    )
+    assert "context-management-2025-06-27" in headers["anthropic-beta"].split(",")
+
+
+def test_validate_environment_injects_anthropic_beta_for_fast_mode(config):
+    headers, _ = config.validate_anthropic_messages_environment(
+        headers={},
+        model="some-model",
+        messages=[],
+        optional_params={"speed": "fast"},
+        litellm_params={},
+        api_key="sk-test",
+        api_base="https://host/v1",
+    )
+    assert "fast-mode-2026-02-01" in headers["anthropic-beta"].split(",")
+
+
+def test_validate_environment_merges_existing_anthropic_beta(config):
+    headers, _ = config.validate_anthropic_messages_environment(
+        headers={"anthropic-beta": "caller-flag"},
+        model="some-model",
+        messages=[],
+        optional_params={"speed": "fast"},
+        litellm_params={},
+        api_key="sk-test",
+        api_base="https://host/v1",
+    )
+    beta_values = set(headers["anthropic-beta"].split(","))
+    assert "caller-flag" in beta_values
+    assert "fast-mode-2026-02-01" in beta_values
+
+
+def test_request_strips_advisor_blocks_when_advisor_tool_absent(config):
+    messages = [
+        {"role": "user", "content": "hello"},
+        {
+            "role": "assistant",
+            "content": [
+                {"type": "text", "text": "thinking out loud"},
+                {"type": "server_tool_use", "id": "advisor_1", "name": "advisor", "input": {}},
+                {"type": "advisor_tool_result", "tool_use_id": "advisor_1", "content": "stale"},
+            ],
+        },
+    ]
+
+    payload = config.transform_anthropic_messages_request(
+        model="some-model",
+        messages=messages,
+        anthropic_messages_optional_request_params={"max_tokens": 64},
+        litellm_params=GenericLiteLLMParams(),
+        headers={},
+    )
+
+    flattened_types = [
+        block.get("type")
+        for message in payload["messages"]
+        if isinstance(message.get("content"), list)
+        for block in message["content"]
+        if isinstance(block, dict)
+    ]
+    assert "advisor_tool_result" not in flattened_types
+    assert "server_tool_use" not in flattened_types
+
+
+def test_request_maps_reasoning_effort_to_thinking(config):
+    payload = config.transform_anthropic_messages_request(
+        model="claude-sonnet-4-20250514",
+        messages=[{"role": "user", "content": "hi"}],
+        anthropic_messages_optional_request_params={
+            "max_tokens": 1024,
+            "reasoning_effort": "medium",
+        },
+        litellm_params=GenericLiteLLMParams(),
+        headers={},
+    )
+
+    assert "reasoning_effort" not in payload
+    assert isinstance(payload.get("thinking"), dict)
+    assert payload["thinking"].get("type") == "enabled"
+
+
+def test_passthrough_disables_anthropic_beta_filtering(config):
+    from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
+        AnthropicMessagesConfig,
+    )
+
+    assert config.should_filter_anthropic_beta_headers() is False
+    assert AnthropicMessagesConfig().should_filter_anthropic_beta_headers() is True
+
+
+def test_anthropic_beta_survives_provider_filter_on_passthrough_path(config):
+    from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
+
+    headers, _ = config.validate_anthropic_messages_environment(
+        headers={"Anthropic-Beta": "caller-flag"},
+        model="some-model",
+        messages=[],
+        optional_params={"speed": "fast"},
+        litellm_params={},
+        api_key="sk-test",
+        api_base="https://host/v1",
+    )
+
+    # The deployment routes as provider "openai", which has no beta mapping, so an
+    # unconditional filter would drop every anthropic-beta value. The handler must
+    # skip filtering for this config so the native upstream still receives them.
+    if config.should_filter_anthropic_beta_headers():
+        headers = update_headers_with_filtered_beta(headers=dict(headers), provider="openai")
+
+    survived = set(headers.get("anthropic-beta", "").split(","))
+    assert {"caller-flag", "fast-mode-2026-02-01"} <= survived
+
+    stripped = update_headers_with_filtered_beta(headers=dict(headers), provider="openai")
+    assert "anthropic-beta" not in stripped

From 6d43c21ec641baee73e486ec3f9b72c1c36a850d Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 30 Jun 2026 19:19:48 +0000
Subject: [PATCH 27/51] fix(anthropic): drop redundant supports_output_config
 from Vertex/Azure Sonnet 5

The Vertex AI and Azure AI Sonnet 5 entries carried supports_output_config:
true, which the gen-5 siblings (vertex_ai/claude-opus-4-8, azure_ai/claude-fable-5,
etc.) do not. The flag only feeds AnthropicConfig._model_supports_effort_param,
which already returns true for these entries via supports_xhigh/max_reasoning_effort,
so output_config.effort still forwards on both routes. Removing it is behavior
neutral and matches the existing per-platform convention for gen-5 Claude.
---
 litellm/model_prices_and_context_window_backup.json | 9 +++------
 model_prices_and_context_window.json                | 9 +++------
 2 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 5bd70690d48..b16e1015255 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -2737,8 +2737,7 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true,
-        "supports_output_config": true
+        "supports_max_reasoning_effort": true
     },
     "azure_ai/claude-sonnet-4-6": {
         "supports_adaptive_thinking": true,
@@ -35235,8 +35234,7 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true,
-        "supports_output_config": true
+        "supports_max_reasoning_effort": true
     },
     "vertex_ai/claude-sonnet-4-6": {
         "supports_adaptive_thinking": true,
@@ -42703,8 +42701,7 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true,
-        "supports_output_config": true
+        "supports_max_reasoning_effort": true
     },
     "vertex_ai/claude-sonnet-4-6@default": {
         "supports_adaptive_thinking": true,
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 5fc431d721b..3e4c5b947a1 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -2737,8 +2737,7 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true,
-        "supports_output_config": true
+        "supports_max_reasoning_effort": true
     },
     "azure_ai/claude-sonnet-4-6": {
         "supports_adaptive_thinking": true,
@@ -35412,8 +35411,7 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true,
-        "supports_output_config": true
+        "supports_max_reasoning_effort": true
     },
     "vertex_ai/claude-sonnet-4-6": {
         "supports_adaptive_thinking": true,
@@ -42938,8 +42936,7 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true,
-        "supports_output_config": true
+        "supports_max_reasoning_effort": true
     },
     "vertex_ai/claude-sonnet-4-6@default": {
         "supports_adaptive_thinking": true,

From 52dc15adfefc9385e68c6a4a635f659cfb466f9e Mon Sep 17 00:00:00 2001
From: Yassin Kortam <yassin@berri.ai>
Date: Tue, 30 Jun 2026 22:21:14 +0300
Subject: [PATCH 28/51] fix(proxy): isolate poison spend-log rows so one bad
 record can't drop the whole batch (#31705)

update_spend_logs flushes the queue with a single create_many per batch, so one
row carrying bytes Postgres refuses (a residual NUL byte is the canonical case)
fails the entire insert and drops every good spend log alongside it. PR #29515
strips NUL bytes from the JSON columns, but the scalar string columns (end_user,
model, session_id, ...) still flow through unsanitized, so a poisoned row can
still reach the write and take a batch of up to 1000 good rows down with it.

On a genuine data-layer rejection the batch is now bisected so the good rows
still persist and only the offending row is dropped and logged with its
request_id. The classification lives in PrismaDBExceptionHandler.is_prisma_data_error
(matched by exact type so systemic subclasses like a missing table are not
mistaken for a single poison row), which keeps prisma an in-function import and
litellm.proxy.utils importable without the proxy extra. Transport failures,
including the "can't reach database server" outage that prisma mislabels as a
DataError, are re-raised unchanged so the existing connection-retry path still
runs and a transient outage never turns into silent per-row data loss.

The bisection carries a per-batch isolation budget so an authenticated caller
flooding poisoned rows cannot amplify one failed bulk insert into ~2N failed
inserts and N log lines; once the budget is spent the still-failing remainder
is dropped wholesale under a single log line.

Resolves LIT-4103
---
 litellm/proxy/db/exception_handler.py         |  23 ++++
 litellm/proxy/utils.py                        |  67 ++++++++++-
 .../proxy/db/test_exception_handler.py        |  31 +++++
 .../test_proxy_update_spend.py                | 106 ++++++++++++++++++
 4 files changed, 225 insertions(+), 2 deletions(-)

diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py
index 48066945131..3a93896a206 100644
--- a/litellm/proxy/db/exception_handler.py
+++ b/litellm/proxy/db/exception_handler.py
@@ -66,6 +66,29 @@ class PrismaDBExceptionHandler:
             return True
         return False
 
+    @staticmethod
+    def is_prisma_data_error(e: Exception) -> bool:
+        """True iff ``e`` is a base prisma ``DataError``: the database processed
+        the statement and refused the data itself (e.g. ``invalid byte sequence
+        for encoding "UTF8": 0x00``), as opposed to a connectivity failure.
+
+        Matched by exact type, not ``isinstance``: the specific data-layer
+        subclasses (``UniqueViolationError``, ``TableNotFoundError``,
+        ``MissingRequiredValueError`` ...) all derive from ``DataError`` but
+        carry their own semantics, and a systemic one like a missing table must
+        not be mistaken for a single poison row and bisected away. A raw
+        Postgres execution error with no prisma P-code surfaces as the base
+        ``DataError``.
+
+        prisma also wraps the P1001 "can't reach database server" outage as a
+        base ``DataError``, so a caller that must not treat an outage as a
+        per-row data rejection has to additionally consult
+        ``is_database_service_unavailable_error`` before acting on a True here.
+        """
+        import prisma
+
+        return type(e) is prisma.errors.DataError
+
     @staticmethod
     def is_database_transport_error(e: Exception) -> bool:
         """
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 154a17bc4db..4433a35f5d0 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -23,7 +23,9 @@ from typing import (
     Dict,
     List,
     Literal,
+    Mapping,
     Optional,
+    Sequence,
     Tuple,
     Union,
     cast,
@@ -5194,8 +5196,10 @@ class ProxyUpdateSpend:
                         for j in range(0, len(logs_to_process), BATCH_SIZE):
                             batch = logs_to_process[j : j + BATCH_SIZE]
                             batch_with_dates = [prisma_client.jsonify_object({**entry}) for entry in batch]
-                            await SpendLogsRepository(prisma_client).table.create_many(
-                                data=batch_with_dates, skip_duplicates=True
+                            await _create_spend_logs_with_poison_isolation(
+                                SpendLogsRepository(prisma_client),
+                                batch_with_dates,
+                                MAX_SPEND_LOG_ISOLATION_ATTEMPTS_PER_BATCH,
                             )
                             verbose_proxy_logger.debug(f"Flushed {len(batch)} logs to the DB.")
                             # Explicitly clear batch memory
@@ -5462,6 +5466,65 @@ async def _monitor_spend_logs_queue(
             await asyncio.sleep(current_interval)
 
 
+MAX_SPEND_LOG_ISOLATION_ATTEMPTS_PER_BATCH = 256
+
+
+async def _create_spend_logs_with_poison_isolation(
+    repo: SpendLogsRepository,
+    rows: Sequence[Mapping[str, object]],
+    attempts_left: int,
+) -> int:
+    """Write spend-log rows, isolating any row Postgres rejects on its data.
+
+    ``create_many`` writes the whole batch in a single statement, so one row
+    carrying bytes Postgres refuses (a residual NUL byte is the canonical case)
+    fails the entire insert and drops every good row alongside it. On a genuine
+    data-layer rejection the batch is bisected so the good rows still persist
+    and only the offending row is dropped and logged. Transport failures,
+    including the "can't reach database server" outage that prisma mislabels as
+    a ``DataError``, are re-raised unchanged so the caller's connection-retry
+    path still runs.
+
+    ``attempts_left`` is a hard ceiling on the number of ``create_many`` calls
+    the isolation may issue for this batch, so an authenticated caller flooding
+    poisoned rows cannot amplify one failed bulk insert into unbounded failed
+    inserts and log lines. It is checked before any insert (so an exhausted
+    budget never even attempts a write), decremented once per ``create_many``
+    call, and threaded through the recursion so the whole bisection shares one
+    allowance; total inserts are therefore bounded by the initial value
+    regardless of how many rows are poisoned. When it runs out the still-failing
+    remainder is dropped wholesale (the pre-existing drop-the-batch behavior)
+    under one log line. Returns the budget left after this subtree.
+    """
+    if attempts_left <= 0:
+        spend_log_error(
+            "Spend tracking - dropping %d spend log rows without per-row isolation; "
+            "isolation attempt budget exhausted for this flush",
+            len(rows),
+        )
+        return 0
+    try:
+        await repo.table.create_many(data=rows, skip_duplicates=True)
+        return attempts_left - 1
+    except Exception as e:
+        if not PrismaDBExceptionHandler.is_prisma_data_error(e):
+            raise
+        if PrismaDBExceptionHandler.is_database_service_unavailable_error(e):
+            raise
+        if len(rows) == 1:
+            request_id = rows[0].get("request_id")
+            spend_log_error(
+                "Spend tracking - dropping spend log row Postgres rejected. request_id=%s error=%s",
+                request_id,
+                str(e),
+                exc=e,
+            )
+            return attempts_left - 1
+        mid = len(rows) // 2
+        remaining = await _create_spend_logs_with_poison_isolation(repo, rows[:mid], attempts_left - 1)
+        return await _create_spend_logs_with_poison_isolation(repo, rows[mid:], remaining)
+
+
 def _raise_failed_update_spend_exception(e: Exception, start_time: float, proxy_logging_obj: ProxyLogging):
     """
     Raise an exception for failed update spend logs
diff --git a/tests/test_litellm/proxy/db/test_exception_handler.py b/tests/test_litellm/proxy/db/test_exception_handler.py
index 6021c221426..0634a01326c 100644
--- a/tests/test_litellm/proxy/db/test_exception_handler.py
+++ b/tests/test_litellm/proxy/db/test_exception_handler.py
@@ -148,6 +148,37 @@ def test_is_database_service_unavailable_error_prisma_p1001_masquerades_as_datae
     )
 
 
+def test_is_prisma_data_error_only_true_for_dataerror():
+    """The spend-log poison-row isolation gates on this: only a prisma
+    ``DataError`` (the DB refused the data, e.g. a NUL byte) may be bisected
+    into a per-row drop. A connectivity failure or any non-prisma exception
+    must not be treated as a data rejection, so the whole batch surfaces."""
+    import httpx
+
+    data_error = DataError(data={"user_facing_error": {"message": "invalid byte sequence for encoding UTF8: 0x00"}})
+    assert PrismaDBExceptionHandler.is_prisma_data_error(data_error) is True
+
+    for non_data in (
+        httpx.ConnectError("conn refused"),
+        PrismaError("can't reach database server"),
+        UniqueViolationError(data={"user_facing_error": {"meta": {"table": "t"}}}),
+        RuntimeError("boom"),
+    ):
+        assert PrismaDBExceptionHandler.is_prisma_data_error(non_data) is False
+
+
+def test_is_prisma_data_error_true_for_connection_masquerade_dataerror():
+    """The P1001 outage prisma mislabels as a ``DataError`` is still a
+    ``DataError`` by type, so this returns True; the spend-log helper relies on
+    ``is_database_service_unavailable_error`` (not this check) to keep that
+    outage on the retry path instead of dropping rows."""
+    p1001_as_dataerror = DataError(
+        data={"user_facing_error": {"message": "Can't reach database server at `127.0.0.1`:`5499`"}}
+    )
+    assert PrismaDBExceptionHandler.is_prisma_data_error(p1001_as_dataerror) is True
+    assert PrismaDBExceptionHandler.is_database_service_unavailable_error(p1001_as_dataerror) is True
+
+
 def test_is_database_service_unavailable_error_cached_plan_escapes_as_503():
     """Composes with the cached-plan retry: when that recovery fails and the
     Postgres "cached plan must not change result type" error escapes (raised by
diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py
index 6a4fd516c9b..d5d4de7f2cf 100644
--- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py
+++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py
@@ -231,6 +231,112 @@ async def test_update_spend_logs_failure_raises_after_retries(
         )
 
 
+def _data_error(message: str) -> Any:
+    from prisma.errors import DataError
+
+    return DataError({"user_facing_error": {"message": message}})
+
+
+@pytest.mark.asyncio
+async def test_update_spend_logs_isolates_poison_row_and_persists_good_rows(
+    mock_prisma_client: Any, make_spend_log_row: Any
+) -> None:
+    """One row Postgres rejects (22P05) must not drop the whole batch.
+
+    The good rows still persist and only the offending row is dropped, with no
+    exception bubbling up. On the unfixed single-shot ``create_many`` the first
+    write raises and the entire batch is lost.
+    """
+    poison_id = "r1"
+    written: List[str] = []
+
+    async def _create_many(*, data: Any, skip_duplicates: bool) -> None:
+        ids = [row["request_id"] for row in data]
+        if poison_id in ids:
+            raise _data_error(
+                "Inconsistent column data: 22P05 invalid byte sequence for encoding UTF8: 0x00"
+            )
+        written.extend(ids)
+
+    mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_create_many)
+    proxy_logging = MagicMock()
+    proxy_logging.failure_handler = AsyncMock()
+
+    logs = [make_spend_log_row(request_id=f"r{i}") for i in range(4)]
+    await ProxyUpdateSpend.update_spend_logs(
+        n_retry_times=0,
+        prisma_client=mock_prisma_client,
+        db_writer_client=None,
+        proxy_logging_obj=proxy_logging,
+        logs_to_process=logs,
+    )
+    assert sorted(written) == ["r0", "r2", "r3"]
+
+
+@pytest.mark.asyncio
+async def test_update_spend_logs_reraises_connection_masquerade_dataerror(
+    mock_prisma_client: Any, make_spend_log_row: Any
+) -> None:
+    """A P1001 "can't reach database server" outage that prisma mislabels as a
+    ``DataError`` is transient, not a poison row: it must propagate so the batch
+    is surfaced/retried rather than bisected into silent per-row drops.
+    """
+    err = _data_error("Can't reach database server at db-host:5432")
+    mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=err)
+    proxy_logging = MagicMock()
+    proxy_logging.failure_handler = AsyncMock()
+
+    with pytest.raises(type(err)):
+        await ProxyUpdateSpend.update_spend_logs(
+            n_retry_times=0,
+            prisma_client=mock_prisma_client,
+            db_writer_client=None,
+            proxy_logging_obj=proxy_logging,
+            logs_to_process=[
+                make_spend_log_row(request_id="a"),
+                make_spend_log_row(request_id="b"),
+            ],
+        )
+
+
+@pytest.mark.asyncio
+async def test_update_spend_logs_caps_isolation_attempts_under_poison_flood(
+    mock_prisma_client: Any, make_spend_log_row: Any
+) -> None:
+    """A flood of poisoned rows must not amplify one failed bulk insert into
+    unbounded failed inserts. The per-batch attempt budget hard-caps the number
+    of ``create_many`` calls regardless of how many rows are poisoned, so the DB
+    work stays bounded and well below the input row count, and the helper still
+    completes without raising.
+    """
+    import litellm.proxy.utils as utils_mod
+
+    attempt_cap = utils_mod.MAX_SPEND_LOG_ISOLATION_ATTEMPTS_PER_BATCH
+    # single create_many batch (< BATCH_SIZE) whose row count exceeds the attempt
+    # cap, so the bound bites and attempts stay below the input row count
+    n_rows = attempt_cap * 3
+
+    async def _always_poison(*, data: Any, skip_duplicates: bool) -> None:
+        raise _data_error("invalid byte sequence for encoding UTF8: 0x00")
+
+    mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_always_poison)
+    proxy_logging = MagicMock()
+    proxy_logging.failure_handler = AsyncMock()
+    logs = [make_spend_log_row(request_id=f"r{i}") for i in range(n_rows)]
+
+    await ProxyUpdateSpend.update_spend_logs(
+        n_retry_times=0,
+        prisma_client=mock_prisma_client,
+        db_writer_client=None,
+        proxy_logging_obj=proxy_logging,
+        logs_to_process=logs,
+    )
+
+    attempts = mock_prisma_client.db.litellm_spendlogs.create_many.await_count
+    assert attempts <= attempt_cap
+    assert attempts < n_rows
+
+
 def test_disable_spend_updates_reflects_general_settings(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:

From be4d0d8439ad6bea5b7a310824c74f2df0c73884 Mon Sep 17 00:00:00 2001
From: Yassin Kortam <yassin@berri.ai>
Date: Tue, 30 Jun 2026 22:25:15 +0300
Subject: [PATCH 29/51] fix(redis): re-establish async cluster connections
 after a node restart (#31577)

When redis_startup_nodes is set the async cluster client was built with no health check and no TCP keepalive, so a connection silently dropped by a cluster restart (e.g. ElastiCache Serverless maintenance) stayed in the pool and got reused while dead; the first command after the restart stalled in re-initialization until the LoggingWorker timeout cancelled it, surfacing as CancelledError then TimeoutError on the spend-counter path

Build the async cluster client with a 25s health_check_interval and socket_keepalive so an idle connection is PING-validated and reconnected before reuse, and expose both through the cluster kwarg allow-list so an explicit value from config still wins

Resolves LIT-4083
---
 litellm/_redis.py                | 15 +++++++++++-
 litellm/constants.py             |  4 ++++
 tests/test_litellm/test_redis.py | 41 ++++++++++++++++++++++++++++++++
 3 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/litellm/_redis.py b/litellm/_redis.py
index 2bcce0e1083..bb3a0974241 100644
--- a/litellm/_redis.py
+++ b/litellm/_redis.py
@@ -23,7 +23,11 @@ from litellm._redis_credential_provider import (
     GCPIAMCredentialProvider,
     _generate_gcp_iam_access_token,
 )
-from litellm.constants import REDIS_CONNECTION_POOL_TIMEOUT, REDIS_SOCKET_TIMEOUT
+from litellm.constants import (
+    REDIS_CLUSTER_HEALTH_CHECK_INTERVAL,
+    REDIS_CONNECTION_POOL_TIMEOUT,
+    REDIS_SOCKET_TIMEOUT,
+)
 from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
 
 from ._logging import verbose_logger
@@ -102,6 +106,8 @@ def _get_redis_cluster_kwargs(client=None):
         "max_connections",
         "socket_timeout",
         "socket_connect_timeout",
+        "health_check_interval",
+        "socket_keepalive",
     }
 
     return available_args
@@ -579,6 +585,13 @@ def get_redis_async_client(
             new_startup_nodes.append(ClusterNode(**item))
         cluster_kwargs.pop("startup_nodes", None)
 
+        # Default to a periodic health check + TCP keepalive so a connection silently dropped
+        # by a cluster restart (e.g. ElastiCache Serverless maintenance) is revalidated and
+        # reconnected before reuse instead of stalling in re-initialization; an explicit value
+        # from config still wins.
+        cluster_kwargs.setdefault("health_check_interval", REDIS_CLUSTER_HEALTH_CHECK_INTERVAL)
+        cluster_kwargs.setdefault("socket_keepalive", True)
+
         # Create async RedisCluster with IAM token as password if available
         cluster_client = async_redis.RedisCluster(
             startup_nodes=new_startup_nodes,
diff --git a/litellm/constants.py b/litellm/constants.py
index aeb74a65839..f6add8da3ea 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -332,6 +332,10 @@ REDIS_CONNECTION_POOL_TIMEOUT = int(os.getenv("REDIS_CONNECTION_POOL_TIMEOUT", 5
 REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD = int(os.getenv("REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", 5))
 REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT = int(os.getenv("REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", 60))
 REDIS_CIRCUIT_BREAKER_ENABLED = os.getenv("REDIS_CIRCUIT_BREAKER_ENABLED", "true").lower() == "true"
+# Seconds of idle before a Redis cluster connection is validated with a PING and
+# reconnected if dead, so a connection silently dropped by a cluster restart
+# (e.g. ElastiCache Serverless maintenance) is not reused while broken
+REDIS_CLUSTER_HEALTH_CHECK_INTERVAL = 25
 # Default Redis major version to assume when version cannot be determined
 # Using 7 as it's the modern version that supports LPOP with count parameter
 DEFAULT_REDIS_MAJOR_VERSION = int(os.getenv("DEFAULT_REDIS_MAJOR_VERSION", 7))
diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py
index a89e30a0e06..0081e1c819f 100644
--- a/tests/test_litellm/test_redis.py
+++ b/tests/test_litellm/test_redis.py
@@ -12,6 +12,7 @@ from litellm._redis import (
     get_redis_connection_pool,
     get_redis_url_from_environment,
 )
+from litellm.constants import REDIS_CLUSTER_HEALTH_CHECK_INTERVAL
 from litellm._redis_credential_provider import (
     GCPIAMCredentialProvider,
     _token_cache,
@@ -171,6 +172,46 @@ def test_socket_timeouts_in_cluster_kwargs():
     assert "socket_connect_timeout" in kwargs
 
 
+def test_reconnect_kwargs_in_cluster_kwargs():
+    """Health check and keepalive must survive the cluster kwarg allow-list so
+    operators can tune Redis cluster reconnection behavior via config."""
+    kwargs = _get_redis_cluster_kwargs()
+    assert "health_check_interval" in kwargs
+    assert "socket_keepalive" in kwargs
+
+
+@patch("litellm._redis.async_redis.RedisCluster")
+def test_async_cluster_sets_reconnect_defaults(mock_cluster_cls):
+    """
+    The async RedisCluster client must be built with a periodic health check and
+    TCP keepalive so a connection silently dropped by a cluster restart (e.g.
+    ElastiCache Serverless maintenance) is revalidated and reconnected before
+    reuse instead of stalling in re-initialization. Regression for LIT-4083.
+    """
+    get_redis_async_client(startup_nodes=[{"host": "cluster-node", "port": 6379}])
+
+    mock_cluster_cls.assert_called_once()
+    call_kwargs = mock_cluster_cls.call_args[1]
+    assert call_kwargs["health_check_interval"] == REDIS_CLUSTER_HEALTH_CHECK_INTERVAL
+    assert call_kwargs["health_check_interval"] > 0
+    assert call_kwargs["socket_keepalive"] is True
+
+
+@patch("litellm._redis.async_redis.RedisCluster")
+def test_async_cluster_reconnect_defaults_are_overridable(mock_cluster_cls):
+    """An explicit health_check_interval / socket_keepalive from config must win
+    over the built-in reconnect defaults."""
+    get_redis_async_client(
+        startup_nodes=[{"host": "cluster-node", "port": 6379}],
+        health_check_interval=7,
+        socket_keepalive=False,
+    )
+
+    call_kwargs = mock_cluster_cls.call_args[1]
+    assert call_kwargs["health_check_interval"] == 7
+    assert call_kwargs["socket_keepalive"] is False
+
+
 def test_get_redis_async_client_with_connection_pool():
     """Test that connection_pool parameter is properly passed to Redis client"""
     # Create a mock connection pool

From 4f41a9e140ebcf12ec56bfc8d24b4dd5ba78ed4f Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Tue, 30 Jun 2026 12:49:46 -0700
Subject: [PATCH 30/51] test(ui): drop the e2e typecheck CI gate, keep the
 typed import for the editor

The e2e runs against the real proxy, so a contract drift already fails the test at
runtime; tsc only checks the spec against schema.d.ts, a generated snapshot, so a
backend change with a stale snapshot would pass tsc while the live test still
catches it. The dedicated tsconfig + script + CI step were circular ceremony for
that. Keep the zero-runtime-cost type-only import, which still catches mistakes in
the editor, and make its comment honest about what enforces the contract.
---
 .circleci/config.yml                                     | 8 --------
 .../e2e_tests/tests/settings/routerSettings.spec.ts      | 7 +++----
 ui/litellm-dashboard/e2e_tests/tsconfig.json             | 9 ---------
 ui/litellm-dashboard/package.json                        | 1 -
 4 files changed, 3 insertions(+), 22 deletions(-)
 delete mode 100644 ui/litellm-dashboard/e2e_tests/tsconfig.json

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 009884cbfe4..f13e9bf66f1 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -2687,14 +2687,6 @@ jobs:
           paths:
             - ui/litellm-dashboard/node_modules
             - ~/.cache/ms-playwright
-      - run:
-          name: Type-check E2E specs
-          # The specs type their request/response round-trips against the generated
-          # OpenAPI schema; this step turns that typing into a real gate, so a backend
-          # contract drift fails here instead of silently passing at runtime.
-          command: |
-            cd ui/litellm-dashboard
-            npm run typecheck:e2e
       - run:
           name: Build UI from source
           # Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`.
diff --git a/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts
index 64dab6d7cf6..3e140b9ab56 100644
--- a/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts
+++ b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts
@@ -3,10 +3,9 @@ import { ADMIN_STORAGE_PATH } from "../../constants";
 import { navigateToPage } from "../../helpers/navigation";
 import { Page } from "../../fixtures/pages";
 import { Role, users } from "../../fixtures/users";
-// Type-only import of the OpenAPI-generated backend schema. esbuild erases it at
-// runtime; the round-trip below is enforced by the `typecheck:e2e` CI step (tsc over
-// e2e_tests), so a drift in the /config/update or /router/settings contract fails the
-// build rather than silently passing here.
+// Type-only import of the OpenAPI-generated backend schema, erased at runtime by
+// esbuild. It types the round-trips below so mistakes surface in the editor; the live
+// test against the real proxy is what actually enforces the contract.
 import type { components } from "../../../src/lib/http/schema";
 
 // These tests mutate the proxy's shared router_settings, and the Loadbalancing save
diff --git a/ui/litellm-dashboard/e2e_tests/tsconfig.json b/ui/litellm-dashboard/e2e_tests/tsconfig.json
deleted file mode 100644
index abda3fdb8b6..00000000000
--- a/ui/litellm-dashboard/e2e_tests/tsconfig.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
-  "extends": "../tsconfig.json",
-  "compilerOptions": {
-    "noEmit": true,
-    "types": ["node"]
-  },
-  "include": ["**/*.ts"],
-  "exclude": ["node_modules"]
-}
diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json
index 1d2b505a172..a8948f4be34 100644
--- a/ui/litellm-dashboard/package.json
+++ b/ui/litellm-dashboard/package.json
@@ -19,7 +19,6 @@
     "e2e:ui": "playwright test --ui --config e2e_tests/playwright.config.ts",
     "e2e:migration": "playwright test e2e_tests/tests/migration/migratedPages.spec.ts --config e2e_tests/playwright.config.ts",
     "e2e:migration:root": "playwright test --config e2e_tests/migration.serverRootPath.config.ts",
-    "typecheck:e2e": "tsc -p e2e_tests/tsconfig.json --noEmit",
     "knip": "knip",
     "knip:fix": "knip --fix",
     "gen:api": "node scripts/gen-api-types.mjs"

From d4c33b2b5922cdc780c7dac31c73a2a21f342540 Mon Sep 17 00:00:00 2001
From: mubashir1osmani <mubashir.osmani777@gmail.com>
Date: Tue, 30 Jun 2026 12:54:47 -0700
Subject: [PATCH 31/51] fix(logging): route realtime success logging through
 the bounded worker (#31733)

RealTimeStreaming.log_messages dispatched the success handler with a bare
asyncio.create_task, bypassing GLOBAL_LOGGING_WORKER (which gives a per-coroutine
timeout and a concurrency cap). On a long-lived realtime websocket a slow logging
callback left one suspended task per logged turn, each pinning that turn's
assembled response, accumulating without bound (~12-15k in-flight under load in a
repro) until OOM. Route realtime success logging through the bounded worker so
in-flight logging is capped and a hung callback is cancelled at the worker
timeout.

The chat and responses streaming success-logging paths are intentionally left
unchanged: their success callbacks must complete within the call's event-loop run
(the non-streaming path pairs the worker with a synchronous callback; the
streaming path has no such companion), so deferring them through the worker would
drop logs for one-shot SDK calls and breaks test_async_custom_handler_stream.
Bounding those paths needs a load-shedding approach and is left to a follow-up.
---
 .../litellm_core_utils/realtime_streaming.py  |  7 ++++--
 .../test_realtime_streaming.py                | 23 +++++++++++++++++++
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py
index bd6406c6241..a1a070eb5b7 100644
--- a/litellm/litellm_core_utils/realtime_streaming.py
+++ b/litellm/litellm_core_utils/realtime_streaming.py
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Protocol, Union, ca
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
 from litellm.types.llms.openai import (
     OpenAIRealtimeEvents,
@@ -315,8 +316,10 @@ class RealTimeStreaming:
                 self.logging_obj.model_call_details["realtime_tools"] = self.session_tools
                 self.logging_obj.model_call_details["realtime_tool_calls"] = self.tool_calls
             ## ASYNC LOGGING
-            # Create an event loop for the new thread
-            asyncio.create_task(self.logging_obj.async_success_handler(self.messages))
+            # Route through the bounded logging worker (per-coroutine timeout +
+            # concurrency cap) instead of a bare create_task, so a slow callback
+            # can't leave suspended tasks pinning each call's response in memory.
+            GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(self.logging_obj.async_success_handler(self.messages))
             ## SYNC LOGGING
             executor.submit(self.logging_obj.success_handler(self.messages))
 
diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py
index 2ad9b919a1f..766befd1a99 100644
--- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py
+++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py
@@ -2945,3 +2945,26 @@ def test_non_bidi_setup_left_untouched_for_followup_capable_providers():
         assert streaming._maybe_inject_guardrail_auto_response_disable(msg) == msg
     finally:
         litellm.callbacks = []
+
+
+@pytest.mark.asyncio
+async def test_log_messages_routes_async_logging_through_bounded_worker():
+    """Realtime success logging must go through GLOBAL_LOGGING_WORKER (bounded
+    queue + per-coroutine timeout), not a bare asyncio.create_task. A bare task
+    has no timeout/concurrency cap, so when a logging callback is slow every
+    realtime turn leaves a suspended task pinning its response in memory -> an
+    unbounded leak. Regression for that fix."""
+    logging_obj = MagicMock()
+    streaming = RealTimeStreaming(MagicMock(), MagicMock(), logging_obj)
+    streaming.messages = [{"type": "session.created"}]
+
+    with (
+        patch("litellm.litellm_core_utils.realtime_streaming.GLOBAL_LOGGING_WORKER") as mock_worker,
+        patch("litellm.litellm_core_utils.realtime_streaming.asyncio.create_task") as mock_create_task,
+        patch("litellm.litellm_core_utils.realtime_streaming.executor.submit"),
+    ):
+        await streaming.log_messages()
+
+        mock_worker.ensure_initialized_and_enqueue.assert_called_once()
+        # the bare create_task path must no longer be used for success logging
+        mock_create_task.assert_not_called()

From 94936a3922aeb8aa9a4d5928e63ea6f15be6f098 Mon Sep 17 00:00:00 2001
From: Yassin Kortam <yassin@berri.ai>
Date: Tue, 30 Jun 2026 22:59:18 +0300
Subject: [PATCH 32/51] fix(presidio): stream SSE output incrementally instead
 of buffering the whole response (#31503)

The Presidio streaming post-call hooks (_stream_apply_output_masking for
apply_to_output and _stream_pii_unmasking for output_parse_pii) collected every
upstream chunk, reassembled the full completion with stream_chunk_builder at
end-of-stream, ran Presidio over it, then emitted one reconstructed SSE chunk.
Time-to-first-token collapsed to the total generation time and token-by-token
streaming was lost whenever Presidio output handling was enabled. With the
default presidio_filter_scope both, an apply_to_output masking instance is always
created, so even the unmask configuration buffered the stream.

Both paths now transform and forward chunks as they arrive. The unmask path
replaces placeholder tokens per chunk, holding back only the trailing run that
could still grow into a token so a placeholder split across SSE chunks
(<PER + SON_1>) is still rewritten atomically. The mask path emits a prefix only
when masking it in isolation matches the corresponding prefix of masking the
whole buffer, with a lookahead margin still buffered past the cut, so an entity
straddling the cut is detected and held until complete; past
_PRESIDIO_STREAM_MAX_BUFFER the run is bounded without splitting an entity.
Tool-call and legacy function-call argument fragments are accumulated per choice
and transformed once the choice closes, content is buffered independently per
choice index for correct n>1 streaming, raw Anthropic SSE bytes and /v1/responses
events pass through with any held content flushed first so events never reorder,
and a masking error redacts only the affected chunk (fail closed, keeping
finish_reason) while the stream continues.

Resolves LIT-3222
---
 .../guardrails/guardrail_hooks/presidio.py    | 529 +++++++++----
 .../guardrail_hooks/test_presidio.py          | 712 +++++++++++++++++-
 2 files changed, 1086 insertions(+), 155 deletions(-)

diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py
index 95876a55eab..e60b6233038 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py
@@ -17,6 +17,8 @@ from typing import (
     TYPE_CHECKING,
     Any,
     AsyncGenerator,
+    Awaitable,
+    Callable,
     Dict,
     List,
     Literal,
@@ -54,7 +56,14 @@ from litellm.types.proxy.guardrails.guardrail_hooks.presidio import (
     PresidioAnalyzeRequest,
     PresidioAnalyzeResponseItem,
 )
-from litellm.types.utils import GuardrailStatus, StreamingChoices
+from litellm.types.utils import (
+    ChatCompletionDeltaToolCall,
+    Delta,
+    Function,
+    FunctionCall,
+    GuardrailStatus,
+    StreamingChoices,
+)
 from litellm.utils import (
     EmbeddingResponse,
     ImageResponse,
@@ -62,6 +71,17 @@ from litellm.utils import (
     ModelResponseStream,
 )
 
+# Trailing context (chars) the streaming output-masking path keeps buffered past
+# a sentence boundary before emitting, so a PII entity that straddles the
+# boundary is seen in full by Presidio and is never split across two analyze
+# calls. It bounds the largest single entity the incremental path can mask
+# without leaking; an entity longer than this could still be split.
+_PRESIDIO_STREAM_MARGIN = 96
+# Hard cap on buffered un-emitted output. Past this with no sentence boundary,
+# stable prefixes are flushed; if stability cannot be proven, the ambiguous
+# prefix is dropped while retaining the trailing margin.
+_PRESIDIO_STREAM_MAX_BUFFER = 2000
+
 
 class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
     user_api_key_cache = None
@@ -93,6 +113,10 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
         self.mock_redacted_text = mock_redacted_text
         self.output_parse_pii = output_parse_pii or False
         self.apply_to_output = apply_to_output
+        # Streaming output-masking safety window; instance attributes so tests can
+        # exercise incremental flushing with short content (see _mask_emit_decision).
+        self._stream_mask_margin = _PRESIDIO_STREAM_MARGIN
+        self._stream_mask_max_buffer = _PRESIDIO_STREAM_MAX_BUFFER
 
         # When output_parse_pii or apply_to_output is enabled, the guardrail must
         # also run on post_call to unmask/mask the response.  Expand the event_hook
@@ -1048,81 +1072,353 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
         )
         return response
 
+    @staticmethod
+    def _unmask_hold_len(text: str, token_keys: Any) -> int:
+        """Length of the trailing run of ``text`` that could still grow into a
+        PII placeholder token, so the unmask path holds it until the next chunk
+        completes (or aborts) the token instead of emitting a half-written
+        ``<PERSON_1>``."""
+        keys = tuple(token_keys)
+        if not text or not keys:
+            return 0
+        longest = max(len(key) for key in keys)
+        for start in range(max(0, len(text) - (longest - 1)), len(text)):
+            suffix = text[start:]
+            if any(key.startswith(suffix) for key in keys if len(suffix) < len(key)):
+                return len(text) - start
+        return 0
+
+    @staticmethod
+    def _mask_boundaries(text: str) -> tuple[int, ...]:
+        """Candidate flush points: a newline, or a sentence terminator already
+        followed by whitespace. A terminator at the very end of the buffer is
+        excluded because the next chunk may continue the token (``jane.`` +
+        ``doe@example.com``); it becomes a boundary once the whitespace arrives.
+        A boundary is only a *candidate* here; ``_mask_emit_decision`` still
+        confirms via a stability check that no entity straddles it."""
+        return tuple(
+            i + 1
+            for i in range(len(text))
+            if text[i] == "\n" or (text[i] in ".!?" and i + 1 < len(text) and text[i + 1].isspace())
+        )
+
+    async def _mask_emit_decision(
+        self,
+        buffer: str,
+        terminal: bool,
+        transform: "Callable[[str], Awaitable[str]]",
+    ) -> "tuple[str, str]":
+        """Decide how much of ``buffer`` is safe to mask and emit now, returning
+        ``(masked_emit, hold_raw)``.
+
+        A sentence boundary is not trusted blindly (it can fall inside a name
+        with an initial or an address spanning a newline). Instead a prefix is
+        emitted only when masking it in isolation matches the corresponding
+        prefix of masking the whole buffer, with at least ``_PRESIDIO_STREAM_MARGIN``
+        characters of lookahead still buffered past the cut. That guarantees any
+        entity overlapping the cut is present in full when the buffer is analyzed,
+        so a straddling entity makes the prefixes differ and the cut is held.
+        Past ``_PRESIDIO_STREAM_MAX_BUFFER`` with no sentence boundary the buffer
+        first tries stable forced cuts and then drops the ambiguous prefix while
+        retaining the trailing margin, so a failed stability check cannot grow
+        the held buffer without bound."""
+        if terminal:
+            return (await transform(buffer) if buffer else ""), ""
+        margin = self._stream_mask_margin
+        forced_cut = (
+            len(buffer) - margin if len(buffer) > self._stream_mask_max_buffer and len(buffer) > margin else None
+        )
+        cuts = [index for index in self._mask_boundaries(buffer) if len(buffer) - index >= margin]
+        if forced_cut is not None:
+            verbose_proxy_logger.warning(
+                "Presidio apply_to_output: buffered %d streamed chars with no "
+                "sentence boundary; bounding held stream state.",
+                len(buffer),
+            )
+            cuts.append(forced_cut)
+            cuts.extend(index for index in range(forced_cut, len(buffer)) if buffer[index].isspace())
+        if cuts:
+            masked_full = await transform(buffer)
+            for index in sorted(set(cuts), reverse=True):
+                masked_prefix = await transform(buffer[:index])
+                if masked_full.startswith(masked_prefix):
+                    return masked_prefix, buffer[index:]
+        if forced_cut is not None:
+            return "", buffer[forced_cut:]
+        return "", buffer
+
+    @staticmethod
+    def _accumulate_tool_calls(
+        tool_acc: dict[int, dict[int, dict[str, Optional[str]]]],
+        choice_index: int,
+        tool_calls: list[Any],
+    ) -> None:
+        choice_acc = tool_acc.setdefault(choice_index, {})  # mutable-ok: streaming tool-call accumulator
+        for tool_call in tool_calls:
+            entry = choice_acc.setdefault(  # mutable-ok: streaming tool-call accumulator
+                getattr(tool_call, "index", 0) or 0,
+                {"id": None, "type": None, "name": None, "args": ""},
+            )
+            if getattr(tool_call, "id", None):
+                entry["id"] = tool_call.id
+            if getattr(tool_call, "type", None):
+                entry["type"] = tool_call.type
+            function = getattr(tool_call, "function", None)
+            if function is not None:
+                if getattr(function, "name", None):
+                    entry["name"] = function.name
+                arguments = getattr(function, "arguments", None)
+                if isinstance(arguments, str):
+                    entry["args"] = (entry["args"] or "") + arguments
+
+    @staticmethod
+    def _accumulate_function_call(
+        func_acc: dict[int, dict[str, Optional[str]]],
+        choice_index: int,
+        function_call: Any,
+    ) -> None:
+        entry = func_acc.setdefault(  # mutable-ok: streaming function-call accumulator
+            choice_index, {"name": None, "args": ""}
+        )
+        if getattr(function_call, "name", None):
+            entry["name"] = function_call.name
+        arguments = getattr(function_call, "arguments", None)
+        if isinstance(arguments, str):
+            entry["args"] = (entry["args"] or "") + arguments
+
+    @staticmethod
+    async def _build_tool_calls(
+        choice_acc: dict[int, dict[str, Optional[str]]],
+        transform: "Callable[[str], Awaitable[str]]",
+    ) -> list[ChatCompletionDeltaToolCall]:
+        return [
+            ChatCompletionDeltaToolCall(
+                index=tool_index,
+                id=entry["id"],
+                type=entry["type"],
+                function=Function(
+                    name=entry["name"],
+                    arguments=(await transform(entry["args"]) if entry["args"] else ""),
+                ),
+            )
+            for tool_index, entry in sorted(choice_acc.items())
+        ]
+
+    @staticmethod
+    async def _build_function_call(
+        entry: Optional[dict[str, Optional[str]]],
+        transform: "Callable[[str], Awaitable[str]]",
+    ) -> Optional[FunctionCall]:
+        if entry is None:
+            return None
+        return FunctionCall(
+            name=entry["name"],
+            arguments=await transform(entry["args"]) if entry["args"] else "",
+        )
+
+    async def _rewrite_chat_chunk(
+        self,
+        chunk: ModelResponseStream,
+        content_buffers: dict[int, str],
+        tool_acc: dict[int, dict[int, dict[str, Optional[str]]]],
+        func_acc: dict[int, dict[str, Optional[str]]],
+        transform: "Callable[[str], Awaitable[str]]",
+        emit_content: "Callable[[str, bool], Awaitable[tuple[str, str]]]",
+    ) -> None:
+        """Transform one streaming chat chunk in place: text content is masked /
+        unmasked and emitted as soon as ``emit_content`` deems a prefix safe (it
+        returns the already-transformed text to emit plus the raw remainder to
+        hold), while tool-call and function-call argument fragments are
+        accumulated and emitted, fully transformed, on the chunk that closes the
+        choice."""
+        for choice in chunk.choices:
+            index = getattr(choice, "index", 0)
+            delta = getattr(choice, "delta", None)
+            if delta is None:
+                continue
+            terminal = bool(getattr(choice, "finish_reason", None))
+
+            tool_calls = getattr(delta, "tool_calls", None)
+            if tool_calls:
+                self._accumulate_tool_calls(tool_acc, index, tool_calls)
+                delta.tool_calls = None
+            function_call = getattr(delta, "function_call", None)
+            if function_call is not None:
+                self._accumulate_function_call(func_acc, index, function_call)
+                delta.function_call = None
+
+            raw_content = getattr(delta, "content", None)
+            content = raw_content if isinstance(raw_content, str) else None
+            if content is not None or terminal:
+                emitted, hold = await emit_content(content_buffers.pop(index, "") + (content or ""), terminal)
+                if hold:
+                    content_buffers[index] = hold
+                if emitted:
+                    delta.content = emitted
+                else:
+                    delta.content = None if content is None else ""
+
+            if terminal:
+                built_tool_calls = await self._build_tool_calls(tool_acc.get(index, {}), transform)
+                built_function_call = await self._build_function_call(func_acc.get(index), transform)
+                if built_tool_calls:
+                    delta.tool_calls = built_tool_calls
+                if built_function_call is not None:
+                    delta.function_call = built_function_call
+                tool_acc.pop(index, None)
+                func_acc.pop(index, None)
+
+    @staticmethod
+    async def _build_tail_chunk(
+        template: Optional[ModelResponseStream],
+        content_buffers: dict[int, str],
+        tool_acc: dict[int, dict[int, dict[str, Optional[str]]]],
+        func_acc: dict[int, dict[str, Optional[str]]],
+        transform: "Callable[[str], Awaitable[str]]",
+    ) -> Optional[ModelResponseStream]:
+        """Flush any content / tool-call state still held when a stream ends
+        without a finish-reason chunk to attach it to."""
+        if template is None:
+            return None
+        cls = _OPTIONAL_PresidioPIIMasking
+        choices: list[StreamingChoices] = []
+        for index in sorted(set(content_buffers) | set(tool_acc) | set(func_acc)):
+            held = content_buffers.get(index, "")
+            masked_content = await transform(held) if held else None
+            built_tool_calls = await cls._build_tool_calls(tool_acc.get(index, {}), transform)
+            built_function_call = await cls._build_function_call(func_acc.get(index), transform)
+            if masked_content is None and not built_tool_calls and built_function_call is None:
+                continue
+            choices.append(
+                StreamingChoices(
+                    index=index,
+                    delta=Delta(
+                        content=masked_content,
+                        tool_calls=built_tool_calls or None,
+                        function_call=built_function_call,
+                    ),
+                )
+            )
+        if not choices:
+            return None
+        return ModelResponseStream(
+            id=getattr(template, "id", None),
+            created=getattr(template, "created", None),
+            model=getattr(template, "model", None),
+            object="chat.completion.chunk",
+            choices=choices,
+        )
+
+    @staticmethod
+    def _redacted_chunk(chunk: ModelResponseStream) -> ModelResponseStream:
+        """Fail closed when masking a chunk raises: rebuild it with empty content
+        but its original ``finish_reason`` and choice indices preserved, so
+        possibly-unmasked PII never reaches the client yet a terminal chunk still
+        carries the completion signal instead of being dropped."""
+        return ModelResponseStream(
+            id=chunk.id,
+            created=chunk.created,
+            model=chunk.model,
+            object="chat.completion.chunk",
+            choices=[
+                StreamingChoices(
+                    index=choice.index,
+                    delta=Delta(content=None),
+                    finish_reason=choice.finish_reason,
+                )
+                for choice in chunk.choices
+            ],
+        )
+
     async def _stream_apply_output_masking(
         self,
         response: Any,
         request_data: dict,
     ) -> AsyncGenerator[Union[ModelResponseStream, bytes], None]:
         """Apply Presidio masking to streaming output (apply_to_output=True path)."""
-        from litellm.llms.base_llm.base_model_iterator import (
-            convert_model_response_to_streaming,
-        )
-        from litellm.main import stream_chunk_builder
-        from litellm.types.utils import ModelResponse
+        presidio_config = self.get_presidio_settings_from_request_data(request_data or {})
 
-        all_chunks: List[ModelResponseStream] = []
-        passthrough_due_to_unknown_stream_shape = False
-        try:
-            async for chunk in response:
-                if isinstance(chunk, ModelResponseStream):
-                    if passthrough_due_to_unknown_stream_shape:
-                        yield chunk
-                    else:
-                        all_chunks.append(chunk)
-                elif isinstance(chunk, bytes):
-                    yield chunk  # type: ignore[misc]
-                    continue
-                else:
-                    if all_chunks:
-                        # Flush buffered chunks and switch to transparent passthrough for this stream shape.
-                        # NOTE: these buffered chunks are emitted unmasked because this
-                        # stream mixed chunk types and cannot be safely reconstructed.
-                        verbose_proxy_logger.warning(
-                            "Presidio apply_to_output: mixed stream detected (ModelResponseStream + unknown event). "
-                            "Flushing %d buffered chunks without PII masking and switching to transparent passthrough.",
-                            len(all_chunks),
-                        )
-                        for buffered_chunk in all_chunks:
-                            yield buffered_chunk
-                        all_chunks = []
-                    passthrough_due_to_unknown_stream_shape = True
-                    yield chunk
-            if passthrough_due_to_unknown_stream_shape:
-                verbose_proxy_logger.warning(
-                    "Presidio apply_to_output: streaming response contained unknown event objects "
-                    "(e.g. /v1/responses events). Output PII masking was skipped for this response."
-                )
-                return
-            if not all_chunks:
-                verbose_proxy_logger.warning(
-                    "Presidio apply_to_output: streaming response contained no "
-                    "ModelResponseStream chunks (e.g. raw SSE bytes or an empty "
-                    "upstream stream). Output PII masking was skipped for this "
-                    "response."
-                )
-                return
-
-            assembled_model_response = stream_chunk_builder(chunks=all_chunks, messages=request_data.get("messages"))
-
-            if not isinstance(assembled_model_response, ModelResponse):
-                for chunk in all_chunks:
-                    yield chunk
-                return
-
-            await self._process_response_for_pii(
-                response=assembled_model_response,
+        async def transform(text: str) -> str:
+            return await self.check_pii(
+                text=text,
+                output_parse_pii=False,
+                presidio_config=presidio_config,
                 request_data=request_data,
-                mode="mask",
             )
 
-            mock_response_stream = convert_model_response_to_streaming(assembled_model_response)
-            yield mock_response_stream
+        async def emit_content(text: str, terminal: bool) -> tuple[str, str]:
+            return await self._mask_emit_decision(text, terminal, transform)
 
-        except Exception as e:
-            verbose_proxy_logger.error(f"Error masking streaming PII output: {str(e)}")
-            for chunk in all_chunks:
+        async def flush_held() -> Optional[ModelResponseStream]:
+            """Build the held-content tail, failing closed (drop held content)
+            on a masking error instead of letting it abort the whole stream."""
+            try:
+                return await self._build_tail_chunk(last_chunk, content_buffers, tool_acc, func_acc, transform)
+            except Exception as e:
+                if self._is_guardrail_intervention(e):
+                    raise
+                verbose_proxy_logger.error(f"Error masking streaming PII tail: {str(e)}")
+                return None
+
+        content_buffers: dict[int, str] = {}
+        tool_acc: dict[int, dict[int, dict[str, Optional[str]]]] = {}
+        func_acc: dict[int, dict[str, Optional[str]]] = {}
+        last_chunk: Optional[ModelResponseStream] = None
+        masked_any_content = False
+        saw_unmaskable_shape = False
+        try:
+            async for chunk in response:
+                if not isinstance(chunk, ModelResponseStream):
+                    # Flush buffered masked content before forwarding a non-chat
+                    # shape (raw bytes / a /v1/responses event) so the client
+                    # never sees a later event ahead of earlier masked text.
+                    tail = await flush_held()
+                    if tail is not None:
+                        yield tail
+                    content_buffers.clear()
+                    tool_acc.clear()
+                    func_acc.clear()
+                    saw_unmaskable_shape = True
+                    yield chunk
+                    continue
+                masked_any_content = True
+                last_chunk = chunk
+                try:
+                    await self._rewrite_chat_chunk(
+                        chunk,
+                        content_buffers,
+                        tool_acc,
+                        func_acc,
+                        transform,
+                        emit_content,
+                    )
+                except Exception as e:
+                    if self._is_guardrail_intervention(e):
+                        raise
+                    # Fail closed: a transient masking error redacts this chunk's
+                    # content (so possibly-unmasked PII never reaches the client)
+                    # but keeps its finish_reason and keeps the stream flowing,
+                    # rather than truncating the response or dropping a terminal
+                    # chunk's completion signal.
+                    verbose_proxy_logger.error(f"Error masking streaming PII chunk: {str(e)}")
+                    yield self._redacted_chunk(chunk)
+                    continue
                 yield chunk
 
+            tail = await flush_held()
+            if tail is not None:
+                yield tail
+            if not masked_any_content and saw_unmaskable_shape:
+                verbose_proxy_logger.warning(
+                    "Presidio apply_to_output: streaming response contained no "
+                    "maskable chat content (e.g. raw SSE bytes or /v1/responses "
+                    "events). Output PII masking was skipped for this response."
+                )
+        except Exception as e:
+            if self._is_guardrail_intervention(e):
+                raise
+            verbose_proxy_logger.error(f"Error masking streaming PII output: {str(e)}")
+
     @staticmethod
     def _unmask_sse_bytes_chunk(chunk: bytes, pii_tokens: Dict[str, str]) -> bytes:
         try:
@@ -1183,74 +1479,56 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
         request_data: dict,
     ) -> AsyncGenerator[Union[ModelResponseStream, bytes], None]:
         """Apply PII unmasking to streaming output (output_parse_pii=True path)."""
-        from litellm.llms.base_llm.base_model_iterator import (
-            convert_model_response_to_streaming,
-        )
-        from litellm.main import stream_chunk_builder
-        from litellm.types.utils import ModelResponse
-
         metadata = (request_data.get("metadata") or {}) if request_data else {}
         pii_tokens: Dict[str, str] = metadata.get("pii_tokens", {})
 
-        remaining_chunks: List[ModelResponseStream] = []
-        saw_non_chat_chunk = False
+        async def transform(text: str) -> str:
+            return self._unmask_pii_text(text, pii_tokens)
+
+        async def emit_content(text: str, terminal: bool) -> tuple[str, str]:
+            if terminal:
+                return (await transform(text) if text else ""), ""
+            hold = self._unmask_hold_len(text, pii_tokens.keys())
+            emit_raw, held = text[: len(text) - hold], text[len(text) - hold :]
+            return (await transform(emit_raw) if emit_raw else ""), held
+
+        content_buffers: dict[int, str] = {}
+        tool_acc: dict[int, dict[int, dict[str, Optional[str]]]] = {}
+        func_acc: dict[int, dict[str, Optional[str]]] = {}
+        last_chunk: Optional[ModelResponseStream] = None
         try:
             async for chunk in response:
-                if isinstance(chunk, ModelResponseStream):
-                    if saw_non_chat_chunk:
-                        yield chunk
-                    else:
-                        remaining_chunks.append(chunk)
-                elif isinstance(chunk, bytes):
-                    if pii_tokens:
-                        yield self._unmask_sse_bytes_chunk(chunk, pii_tokens)  # type: ignore[misc]
-                    else:
-                        yield chunk  # type: ignore[misc]
+                if isinstance(chunk, bytes):
+                    tail = await self._build_tail_chunk(last_chunk, content_buffers, tool_acc, func_acc, transform)
+                    if tail is not None:
+                        yield tail
+                    content_buffers.clear()
+                    tool_acc.clear()
+                    func_acc.clear()
+                    yield (  # type: ignore[misc]
+                        self._unmask_sse_bytes_chunk(chunk, pii_tokens) if pii_tokens else chunk
+                    )
                     continue
-                else:
-                    # /v1/responses events: unmask response.completed text in-place.
-                    # A mixed stream can't be reassembled, so flush buffered chat
-                    # chunks in order before passthrough instead of dropping them.
-                    if remaining_chunks and not saw_non_chat_chunk:
-                        for buffered_chunk in remaining_chunks:
-                            yield buffered_chunk
-                        remaining_chunks = []
-                    chunk_type = getattr(chunk, "type", None)
-                    if chunk_type == "response.completed" and pii_tokens:
+                if not isinstance(chunk, ModelResponseStream):
+                    tail = await self._build_tail_chunk(last_chunk, content_buffers, tool_acc, func_acc, transform)
+                    if tail is not None:
+                        yield tail
+                    content_buffers.clear()
+                    tool_acc.clear()
+                    func_acc.clear()
+                    if getattr(chunk, "type", None) == "response.completed" and pii_tokens:
                         self._unmask_responses_api_completed_chunk(chunk, pii_tokens)
-                    saw_non_chat_chunk = True
                     yield chunk
+                    continue
+                last_chunk = chunk
+                await self._rewrite_chat_chunk(chunk, content_buffers, tool_acc, func_acc, transform, emit_content)
+                yield chunk
 
-            if saw_non_chat_chunk:
-                return
-
-            if not remaining_chunks:
-                return
-
-            assembled_model_response = stream_chunk_builder(
-                chunks=remaining_chunks, messages=request_data.get("messages")
-            )
-
-            if not isinstance(assembled_model_response, ModelResponse):
-                for chunk in remaining_chunks:
-                    yield chunk
-                return
-
-            self._preserve_usage_from_last_chunk(assembled_model_response, remaining_chunks)
-
-            await self._process_response_for_pii(
-                response=assembled_model_response,
-                request_data=request_data,
-                mode="unmask",
-            )
-
-            mock_response_stream = convert_model_response_to_streaming(assembled_model_response)
-            yield mock_response_stream
-
+            tail = await self._build_tail_chunk(last_chunk, content_buffers, tool_acc, func_acc, transform)
+            if tail is not None:
+                yield tail
         except Exception as e:
             verbose_proxy_logger.error(f"Error in PII streaming processing: {str(e)}")
-            for chunk in remaining_chunks:
-                yield chunk
 
     async def async_post_call_streaming_iterator_hook(  # type: ignore[override]
         self,
@@ -1282,17 +1560,6 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
         async for chunk in self._stream_pii_unmasking(response, request_data):
             yield chunk
 
-    @staticmethod
-    def _preserve_usage_from_last_chunk(
-        assembled_model_response: Any,
-        chunks: List[Any],
-    ) -> None:
-        """Copy usage metadata from the last chunk when stream_chunk_builder misses it."""
-        if not getattr(assembled_model_response, "usage", None) and chunks:
-            last_chunk_usage = getattr(chunks[-1], "usage", None)
-            if last_chunk_usage:
-                setattr(assembled_model_response, "usage", last_chunk_usage)
-
     def get_presidio_settings_from_request_data(self, data: dict) -> Optional[PresidioPerRequestConfig]:
         if "metadata" in data:
             _metadata = data.get("metadata", None)
diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py
index 253d989f203..ce4d6d4a8b0 100644
--- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py
+++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py
@@ -19,7 +19,7 @@ from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.guardrails.guardrail_hooks.presidio import (
     _OPTIONAL_PresidioPIIMasking,
 )
-from litellm.exceptions import GuardrailRaisedException
+from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
 from litellm.types.guardrails import LitellmParams, PiiAction, PiiEntityType
 from litellm.types.utils import Choices, Message, ModelResponse
 
@@ -2207,11 +2207,12 @@ async def test_apply_to_output_streaming_unknown_events_passthrough():
 
 
 @pytest.mark.asyncio
-async def test_apply_to_output_streaming_mixed_chunks_flushes_and_warns():
+async def test_apply_to_output_streaming_mixed_chunks_preserve_order():
     """
-    Regression test for mixed stream shape:
-    a buffered ModelResponseStream chunk followed by unknown responses-style
-    events should be preserved, and masking skip should be visible via warnings.
+    Regression test for mixed stream shape: a ModelResponseStream chat chunk
+    followed by an unknown responses-style event must be forwarded in order.
+    Incremental masking forwards chat chunks as they arrive, so a responses
+    event after them does not buffer or drop anything.
     """
     guardrail = _OPTIONAL_PresidioPIIMasking(
         mock_testing=True,
@@ -2238,26 +2239,14 @@ async def test_apply_to_output_streaming_mixed_chunks_flushes_and_warns():
 
     mock_user_api_key = UserAPIKeyAuth(api_key="test-key")
     received = []
-    with patch(
-        "litellm.proxy.guardrails.guardrail_hooks.presidio.verbose_proxy_logger"
-    ) as mock_logger:
-        async for chunk in guardrail.async_post_call_streaming_iterator_hook(
-            user_api_key_dict=mock_user_api_key,
-            response=mock_stream(),
-            request_data={},
-        ):
-            received.append(chunk)
+    async for chunk in guardrail.async_post_call_streaming_iterator_hook(
+        user_api_key_dict=mock_user_api_key,
+        response=mock_stream(),
+        request_data={},
+    ):
+        received.append(chunk)
 
-        # Preserve original ordering across mixed stream types.
-        assert received == [model_chunk, response_completed]
-
-        # Two warnings are expected:
-        # 1) mixed stream detected + unmasked flush
-        # 2) passthrough mode skipped output masking
-        assert mock_logger.warning.call_count == 2
-        warning_messages = [call.args[0] for call in mock_logger.warning.call_args_list]
-        assert any("mixed stream detected" in msg for msg in warning_messages)
-        assert any("unknown event objects" in msg for msg in warning_messages)
+    assert received == [model_chunk, response_completed]
 
 
 # ---------------------------------------------------------------------------
@@ -2849,3 +2838,678 @@ async def test_stream_pii_unmasking_passthrough_when_no_tokens(mock_user_api_key
         chunks.append(chunk)
 
     assert chunks == [raw_chunk]
+
+
+# ---------------------------------------------------------------------------
+# LIT-3222: incremental SSE streaming for Presidio output masking / unmasking
+# ---------------------------------------------------------------------------
+
+from litellm.types.utils import (
+    ChatCompletionDeltaToolCall,
+    Delta,
+    Function,
+    StreamingChoices,
+)
+
+
+def _content_chunk(text, index=0, finish_reason=None):
+    return ModelResponseStream(
+        id="chatcmpl-lit3222",
+        created=1,
+        model="gpt-4o-mini",
+        object="chat.completion.chunk",
+        choices=[
+            StreamingChoices(
+                index=index, delta=Delta(content=text), finish_reason=finish_reason
+            )
+        ],
+    )
+
+
+def _non_empty_content(chunks):
+    out = []
+    for chunk in chunks:
+        for choice in chunk.choices:
+            piece = getattr(choice.delta, "content", None)
+            if piece:
+                out.append(piece)
+    return out
+
+
+async def _drive(guardrail, stream, request_data):
+    collected = []
+    async for chunk in guardrail.async_post_call_streaming_iterator_hook(
+        user_api_key_dict=UserAPIKeyAuth(api_key="test-key"),
+        response=stream,
+        request_data=request_data,
+    ):
+        collected.append(chunk)
+    return collected
+
+
+@pytest.mark.asyncio
+async def test_unmask_streaming_is_incremental_not_buffered():
+    """
+    output_parse_pii streaming must forward each content chunk as it arrives
+    (unmasked), not collapse the whole completion into a single end-of-stream
+    chunk. The buffering implementation yielded exactly one content chunk.
+    """
+    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, output_parse_pii=True)
+    request_data = {"metadata": {"pii_tokens": {"<PERSON_1>": "John Smith"}}}
+
+    pieces = ["Hello ", "<PERSON_1>", " is here."]
+
+    async def stream():
+        for i, piece in enumerate(pieces):
+            yield _content_chunk(piece)
+        yield _content_chunk("", finish_reason="stop")
+
+    collected = await _drive(guardrail, stream(), request_data)
+    content = _non_empty_content(collected)
+
+    assert len(content) >= 3, f"expected progressive chunks, got {content}"
+    assert "".join(content) == "Hello John Smith is here."
+    assert all("<PERSON_1>" not in piece for piece in content)
+
+
+@pytest.mark.asyncio
+async def test_unmask_streaming_token_split_across_chunks():
+    """
+    A placeholder token split across SSE chunks (``<PER`` + ``SON_1>``) must
+    still be unmasked atomically via the cross-chunk carry buffer.
+    """
+    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, output_parse_pii=True)
+    request_data = {"metadata": {"pii_tokens": {"<PERSON_1>": "John Smith"}}}
+
+    pieces = ["Hi ", "<PER", "SON_1>", "!"]
+
+    async def stream():
+        for piece in pieces:
+            yield _content_chunk(piece)
+        yield _content_chunk("", finish_reason="stop")
+
+    collected = await _drive(guardrail, stream(), request_data)
+    reassembled = "".join(_non_empty_content(collected))
+
+    assert reassembled == "Hi John Smith!"
+    assert "<PER" not in reassembled and "SON_1>" not in reassembled
+
+
+@pytest.mark.asyncio
+async def test_unmask_streaming_independent_per_choice_buffers():
+    """
+    With n>1 each choice keeps its own carry buffer, so a token split across
+    chunks on choice 1 does not corrupt choice 0 and vice versa.
+    """
+    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, output_parse_pii=True)
+    request_data = {
+        "metadata": {
+            "pii_tokens": {"<PERSON_1>": "John", "<PERSON_2>": "Jane"}
+        }
+    }
+
+    def two_choice_chunk(c0, c1):
+        return ModelResponseStream(
+            id="chatcmpl-lit3222",
+            created=1,
+            model="gpt-4o-mini",
+            object="chat.completion.chunk",
+            choices=[
+                StreamingChoices(index=0, delta=Delta(content=c0)),
+                StreamingChoices(index=1, delta=Delta(content=c1)),
+            ],
+        )
+
+    async def stream():
+        yield two_choice_chunk("<PERSON", "<PER")
+        yield two_choice_chunk("_1> ok", "SON_2>!")
+        yield two_choice_chunk("", "")
+
+    collected = []
+    async for chunk in guardrail.async_post_call_streaming_iterator_hook(
+        user_api_key_dict=UserAPIKeyAuth(api_key="test-key"),
+        response=stream(),
+        request_data=request_data,
+    ):
+        collected.append(chunk)
+
+    per_choice = {0: "", 1: ""}
+    for chunk in collected:
+        for choice in chunk.choices:
+            if choice.delta.content:
+                per_choice[choice.index] += choice.delta.content
+
+    assert per_choice[0] == "John ok"
+    assert per_choice[1] == "Jane!"
+
+
+@pytest.mark.asyncio
+async def test_unmask_streaming_tool_call_arguments_unmasked_at_finish():
+    """
+    Tool-call argument fragments carrying a placeholder token must be
+    reassembled and unmasked (the tool would otherwise receive ``<EMAIL...>``).
+    """
+    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, output_parse_pii=True)
+    request_data = {
+        "metadata": {"pii_tokens": {"<EMAIL_ADDRESS_1>": "real@example.com"}}
+    }
+
+    def tool_chunk(*, id=None, name=None, args, finish_reason=None):
+        return ModelResponseStream(
+            id="chatcmpl-lit3222",
+            created=1,
+            model="gpt-4o-mini",
+            object="chat.completion.chunk",
+            choices=[
+                StreamingChoices(
+                    index=0,
+                    delta=Delta(
+                        tool_calls=[
+                            ChatCompletionDeltaToolCall(
+                                index=0,
+                                id=id,
+                                type="function" if id else None,
+                                function=Function(name=name, arguments=args),
+                            )
+                        ]
+                    ),
+                    finish_reason=finish_reason,
+                )
+            ],
+        )
+
+    async def stream():
+        yield tool_chunk(id="call_1", name="send_email", args="")
+        yield tool_chunk(args='{"to": "<EMAIL_ADD')
+        yield tool_chunk(args='RESS_1>"}')
+        yield ModelResponseStream(
+            id="chatcmpl-lit3222",
+            created=1,
+            model="gpt-4o-mini",
+            object="chat.completion.chunk",
+            choices=[
+                StreamingChoices(index=0, delta=Delta(), finish_reason="tool_calls")
+            ],
+        )
+
+    collected = await _drive(guardrail, stream(), request_data)
+
+    tool_calls = [
+        tc
+        for chunk in collected
+        for choice in chunk.choices
+        for tc in (getattr(choice.delta, "tool_calls", None) or [])
+    ]
+    assert len(tool_calls) == 1
+    assert tool_calls[0].id == "call_1"
+    assert tool_calls[0].function.name == "send_email"
+    assert tool_calls[0].function.arguments == '{"to": "real@example.com"}'
+
+
+@pytest.mark.asyncio
+async def test_mask_streaming_is_incremental_per_sentence():
+    """
+    apply_to_output streaming must mask and emit a completed sentence once enough
+    following context confirms it, instead of buffering the whole completion into
+    one chunk. A small safety margin keeps the test content short.
+    """
+    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
+    guardrail._stream_mask_margin = 4
+
+    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
+        return text.replace("secret@example.com", "<EMAIL>")
+
+    guardrail.check_pii = mock_check_pii
+
+    pieces = ["My email is ", "secret@example.com. ", "Call me later."]
+
+    async def stream():
+        for piece in pieces:
+            yield _content_chunk(piece)
+        yield _content_chunk("", finish_reason="stop")
+
+    collected = await _drive(guardrail, stream(), {"metadata": {}})
+    content = _non_empty_content(collected)
+
+    assert len(content) >= 2, f"expected per-sentence chunks, got {content}"
+    reassembled = "".join(content)
+    assert reassembled == "My email is <EMAIL>. Call me later."
+    assert "secret@example.com" not in reassembled
+
+
+@pytest.mark.asyncio
+async def test_mask_streaming_tool_call_arguments_masked_at_finish():
+    """
+    Model-generated PII inside streamed tool-call arguments must be masked
+    before reaching the client, not passed through unmasked.
+    """
+    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
+
+    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
+        return text.replace("secret@example.com", "<EMAIL>")
+
+    guardrail.check_pii = mock_check_pii
+
+    def tool_chunk(*, id=None, name=None, args, finish_reason=None):
+        return ModelResponseStream(
+            id="chatcmpl-lit3222",
+            created=1,
+            model="gpt-4o-mini",
+            object="chat.completion.chunk",
+            choices=[
+                StreamingChoices(
+                    index=0,
+                    delta=Delta(
+                        tool_calls=[
+                            ChatCompletionDeltaToolCall(
+                                index=0,
+                                id=id,
+                                type="function" if id else None,
+                                function=Function(name=name, arguments=args),
+                            )
+                        ]
+                    ),
+                    finish_reason=finish_reason,
+                )
+            ],
+        )
+
+    async def stream():
+        yield tool_chunk(id="call_1", name="save", args='{"email": "sec')
+        yield tool_chunk(args='ret@example.com"}')
+        yield ModelResponseStream(
+            id="chatcmpl-lit3222",
+            created=1,
+            model="gpt-4o-mini",
+            object="chat.completion.chunk",
+            choices=[
+                StreamingChoices(index=0, delta=Delta(), finish_reason="tool_calls")
+            ],
+        )
+
+    collected = await _drive(guardrail, stream(), {"metadata": {}})
+
+    all_args = [
+        tc.function.arguments
+        for chunk in collected
+        for choice in chunk.choices
+        for tc in (getattr(choice.delta, "tool_calls", None) or [])
+    ]
+    assert all_args == ['{"email": "<EMAIL>"}']
+    assert all("secret@example.com" not in args for args in all_args)
+
+
+@pytest.mark.asyncio
+async def test_mask_streaming_flushes_buffered_content_before_passthrough_event():
+    """
+    Regression test (Greptile 4/5 finding): in the apply_to_output path, masked
+    content held in the buffer (no sentence boundary yet) must be flushed BEFORE
+    a non-chat passthrough event (e.g. a /v1/responses completion) is forwarded,
+    so the client never observes stream completion ahead of the final text.
+    """
+    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
+
+    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
+        return text.replace("secret@example.com", "<EMAIL>")
+
+    guardrail.check_pii = mock_check_pii
+
+    class FakeResponsesEvent:
+        def __init__(self, event_type: str):
+            self.type = event_type
+
+    completed = FakeResponsesEvent("response.completed")
+
+    async def stream():
+        # No sentence terminator -> held in the mask buffer, not yet emitted.
+        yield _content_chunk("My email is secret@example.com")
+        yield completed
+
+    collected = await _drive(guardrail, stream(), {"metadata": {}})
+
+    event_index = collected.index(completed)
+    masked_indexes = [
+        i
+        for i, chunk in enumerate(collected)
+        if not isinstance(chunk, FakeResponsesEvent)
+        and any(getattr(c.delta, "content", None) for c in chunk.choices)
+    ]
+    assert masked_indexes, "buffered masked content was never emitted"
+    assert max(masked_indexes) < event_index, "masked content must precede the event"
+
+    masked_text = "".join(
+        c.delta.content
+        for chunk in collected
+        if not isinstance(chunk, FakeResponsesEvent)
+        for c in chunk.choices
+        if getattr(c.delta, "content", None)
+    )
+    assert masked_text == "My email is <EMAIL>"
+    assert "secret@example.com" not in masked_text
+
+
+@pytest.mark.asyncio
+async def test_mask_streaming_does_not_split_entity_on_long_unpunctuated_run():
+    """
+    A long punctuation-free run must never force-flush mid-entity. The old
+    fixed-window fallback emitted at the last whitespace once the buffer grew
+    past a cap, so a space-bearing entity (SSN, phone) straddling that cut was
+    analyzed in two halves and leaked unmasked. Content past the last sentence
+    boundary is now held until a boundary or end-of-stream so each analyze call
+    sees the whole entity.
+    """
+    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
+
+    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
+        return text.replace("123 45 6789", "<US_SSN>")
+
+    guardrail.check_pii = mock_check_pii
+
+    filler = "data " * 90  # >400 chars, spaces only, no .!?\n boundary
+    async def stream():
+        yield _content_chunk(filler + "my ssn is 123 45 6789")
+        yield _content_chunk("", finish_reason="stop")
+
+    collected = await _drive(guardrail, stream(), {"metadata": {}})
+    reassembled = "".join(_non_empty_content(collected))
+
+    assert "<US_SSN>" in reassembled
+    assert "123 45 6789" not in reassembled
+    assert "456789" not in reassembled
+
+
+@pytest.mark.asyncio
+async def test_mask_streaming_preserves_stream_on_check_pii_error():
+    """
+    A transient Presidio failure mid-stream must not truncate the response. The
+    failing run is dropped (fail closed, never leaking the PII it could not mask)
+    while content that already flushed safely, later chunks, and the finish chunk
+    still reach the client.
+    """
+    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
+    guardrail._stream_mask_margin = 4
+
+    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
+        if "boom@example.com" in text:
+            raise RuntimeError("presidio down")
+        return text.replace("ok@example.com", "<EMAIL>")
+
+    guardrail.check_pii = mock_check_pii
+
+    pieces = [
+        "First ok@example.com. ",
+        "filler text here. ",
+        "Second boom@example.com. ",
+        "Third part here.",
+    ]
+
+    async def stream():
+        for piece in pieces:
+            yield _content_chunk(piece)
+        yield _content_chunk("", finish_reason="stop")
+
+    collected = await _drive(guardrail, stream(), {"metadata": {}})
+    reassembled = "".join(_non_empty_content(collected))
+
+    assert "<EMAIL>" in reassembled
+    assert "boom@example.com" not in reassembled
+    assert "Third part" in reassembled, "stream truncated after a masking error"
+    assert any(
+        getattr(choice, "finish_reason", None)
+        for chunk in collected
+        for choice in getattr(chunk, "choices", [])
+    ), "finish chunk dropped after a masking error"
+
+
+@pytest.mark.asyncio
+async def test_mask_streaming_error_preserves_tool_call_accumulators():
+    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
+
+    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
+        if "bad@example.com" in text:
+            raise RuntimeError("presidio down")
+        return text.replace("secret@example.com", "<EMAIL>")
+
+    guardrail.check_pii = mock_check_pii
+
+    def tool_chunk(*, id=None, name=None, args=None, finish_reason=None):
+        return ModelResponseStream(
+            id="chatcmpl-lit3222",
+            created=1,
+            model="gpt-4o-mini",
+            object="chat.completion.chunk",
+            choices=[
+                StreamingChoices(
+                    index=1,
+                    delta=(
+                        Delta(
+                            tool_calls=[
+                                ChatCompletionDeltaToolCall(
+                                    index=0,
+                                    id=id,
+                                    type="function" if id else None,
+                                    function=Function(name=name, arguments=args),
+                                )
+                            ]
+                        )
+                        if args is not None
+                        else Delta()
+                    ),
+                    finish_reason=finish_reason,
+                )
+            ],
+        )
+
+    async def stream():
+        yield tool_chunk(
+            id="call_1",
+            name="save",
+            args='{"email": "secret@example.com"}',
+        )
+        yield _content_chunk("bad@example.com", index=0, finish_reason="stop")
+        yield tool_chunk(finish_reason="tool_calls")
+
+    collected = await _drive(guardrail, stream(), {"metadata": {}})
+    tool_args = [
+        tc.function.arguments
+        for chunk in collected
+        for choice in chunk.choices
+        for tc in (getattr(choice.delta, "tool_calls", None) or [])
+    ]
+
+    assert tool_args == ['{"email": "<EMAIL>"}']
+
+
+@pytest.mark.asyncio
+async def test_mask_emit_decision_caps_buffer_when_stability_fails():
+    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
+    guardrail._stream_mask_margin = 3
+    guardrail._stream_mask_max_buffer = 6
+
+    async def unstable_transform(text):
+        return text[::-1]
+
+    emitted, held = await guardrail._mask_emit_decision(
+        "abcdefghij", False, unstable_transform
+    )
+
+    assert emitted == ""
+    assert held == "hij"
+
+
+@pytest.mark.parametrize(
+    "exception",
+    [
+        BlockedPiiEntityError(entity_type="EMAIL_ADDRESS", guardrail_name="presidio"),
+        GuardrailRaisedException(guardrail_name="presidio", message="invalid response"),
+    ],
+)
+@pytest.mark.asyncio
+async def test_mask_streaming_propagates_guardrail_interventions(exception):
+    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
+
+    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
+        raise exception
+
+    guardrail.check_pii = mock_check_pii
+
+    async def stream():
+        yield _content_chunk("blocked", finish_reason="stop")
+
+    with pytest.raises(type(exception)):
+        await _drive(guardrail, stream(), {"metadata": {}})
+
+
+@pytest.mark.asyncio
+async def test_mask_streaming_holds_terminator_at_chunk_end_until_whitespace():
+    """
+    A sentence terminator at the very end of a chunk is not a safe boundary: the
+    next chunk may continue the token. "Contact jane." followed by
+    "doe@example.com" must mask the whole email rather than flushing "jane." and
+    analyzing the two halves separately, which would leak the address.
+    """
+    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
+
+    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
+        return text.replace("jane.doe@example.com", "<EMAIL>")
+
+    guardrail.check_pii = mock_check_pii
+
+    async def stream():
+        yield _content_chunk("Contact jane.")
+        yield _content_chunk("doe@example.com")
+        yield _content_chunk("", finish_reason="stop")
+
+    collected = await _drive(guardrail, stream(), {"metadata": {}})
+    reassembled = "".join(_non_empty_content(collected))
+
+    assert "<EMAIL>" in reassembled
+    assert "jane.doe@example.com" not in reassembled
+    assert "jane." not in reassembled
+
+
+@pytest.mark.asyncio
+async def test_mask_streaming_does_not_split_entity_across_sentence_boundary():
+    """
+    An entity that straddles a sentence boundary (a name with a middle initial,
+    an address across a newline) must not be flushed in halves. The stability
+    check holds the prefix until masking it alone matches masking the whole
+    buffer, so the straddling entity is analyzed and masked as one unit instead
+    of leaking the part before the boundary.
+    """
+    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
+    guardrail._stream_mask_margin = 8
+
+    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
+        return text.replace("John Q. Public", "<PERSON>")
+
+    guardrail.check_pii = mock_check_pii
+
+    async def stream():
+        yield _content_chunk("Please greet John Q. Public warmly when they arrive.")
+        yield _content_chunk("", finish_reason="stop")
+
+    collected = await _drive(guardrail, stream(), {"metadata": {}})
+    reassembled = "".join(_non_empty_content(collected))
+
+    assert "<PERSON>" in reassembled
+    assert "John Q. Public" not in reassembled
+    assert "John Q." not in reassembled
+
+
+@pytest.mark.asyncio
+async def test_mask_streaming_caps_runaway_buffer_without_splitting_entity():
+    """
+    A punctuation-free run past the buffer cap must be flushed to bound memory,
+    but the forced flush still cuts a margin back from the end so a PII value
+    split exactly at the cap (``secret@`` in one chunk, ``example.com`` in the
+    next) stays buffered and is masked whole instead of leaking its raw halves.
+    The early flush plus the terminal flush yields more than one content chunk.
+    """
+    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
+    guardrail._stream_mask_margin = 8
+    guardrail._stream_mask_max_buffer = 20
+
+    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
+        return text.replace("secret@example.com", "<EMAIL>")
+
+    guardrail.check_pii = mock_check_pii
+
+    async def stream():
+        yield _content_chunk("please email me at secret@")  # 26 > cap, email cut
+        yield _content_chunk("example.com now")
+        yield _content_chunk("", finish_reason="stop")
+
+    collected = await _drive(guardrail, stream(), {"metadata": {}})
+    content = _non_empty_content(collected)
+    reassembled = "".join(content)
+
+    assert "<EMAIL>" in reassembled
+    assert "secret@example.com" not in reassembled
+    assert "secret@" not in reassembled
+    assert len(content) >= 2, f"cap did not flush before end of stream, got {content}"
+
+
+@pytest.mark.asyncio
+async def test_mask_streaming_preserves_finish_reason_when_terminal_chunk_fails():
+    """
+    When the masking call fails on the terminal chunk itself, that chunk must be
+    redacted in place (content dropped, fail closed) but keep its finish_reason,
+    so the client still receives the completion signal instead of a stream that
+    ends without one.
+    """
+    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
+    guardrail._stream_mask_margin = 4
+
+    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
+        if "boom@example.com" in text:
+            raise RuntimeError("presidio down")
+        return text
+
+    guardrail.check_pii = mock_check_pii
+
+    async def stream():
+        yield _content_chunk("Hello world. ")
+        yield _content_chunk("more text here. ")
+        yield _content_chunk("boom@example.com", finish_reason="stop")
+
+    collected = await _drive(guardrail, stream(), {"metadata": {}})
+    reassembled = "".join(_non_empty_content(collected))
+
+    assert "boom@example.com" not in reassembled
+    assert "Hello world" in reassembled
+    finish_reasons = [
+        choice.finish_reason
+        for chunk in collected
+        for choice in getattr(chunk, "choices", [])
+        if getattr(choice, "finish_reason", None)
+    ]
+    assert "stop" in finish_reasons, "finish_reason dropped when terminal chunk failed"
+
+
+@pytest.mark.asyncio
+async def test_unmask_streaming_flushes_held_content_before_bytes():
+    """
+    When a held placeholder prefix is buffered and the next upstream item is a
+    raw SSE byte chunk, the held chat text must be flushed before the bytes so
+    the client never sees the byte chunk ahead of earlier content.
+    """
+    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, output_parse_pii=True)
+    request_data = {"metadata": {"pii_tokens": {"<PERSON_1>": "Jane"}}}
+
+    async def stream():
+        yield _content_chunk("Hi <PERSON")  # "<PERSON" held as a partial token
+        yield b"data: {\"type\": \"ping\"}\n\n"
+
+    collected = await _drive(guardrail, stream(), request_data)
+
+    bytes_index = next(
+        i for i, chunk in enumerate(collected) if isinstance(chunk, bytes)
+    )
+    held_index = next(
+        i
+        for i, chunk in enumerate(collected)
+        if not isinstance(chunk, bytes)
+        and any("Jane" in (getattr(c.delta, "content", None) or "") for c in chunk.choices)
+    )
+    assert held_index < bytes_index, "raw bytes were sent before held chat content"

From 86fbe90f459ec527a1ca408895d48a523a125185 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Tue, 30 Jun 2026 13:20:57 -0700
Subject: [PATCH 33/51] =?UTF-8?q?bump:=20version=201.91.0=20=E2=86=92=201.?=
 =?UTF-8?q?92.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pyproject.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 2ad96c4936b..63afd87f455 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "litellm"
-version = "1.91.0"
+version = "1.92.0"
 description = "Library to easily interface with LLM API providers"
 readme = "README.md"
 requires-python = ">=3.10, <3.14"
@@ -274,7 +274,7 @@ source-exclude = [
 profile = "black"
 
 [tool.commitizen]
-version = "1.91.0"
+version = "1.92.0"
 version_files = [
     "pyproject.toml:^version",
 ]

From c736ec52859f650c1b20a6f7772a78a02ce842d5 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Tue, 30 Jun 2026 13:21:13 -0700
Subject: [PATCH 34/51] =?UTF-8?q?bump:=20version=200.1.44=20=E2=86=92=200.?=
 =?UTF-8?q?1.45?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 enterprise/pyproject.toml | 4 ++--
 pyproject.toml            | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml
index 66f6aeb7abc..f2ad04510a8 100644
--- a/enterprise/pyproject.toml
+++ b/enterprise/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.44"
+version = "0.1.45"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.44"
+version = "0.1.45"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",
diff --git a/pyproject.toml b/pyproject.toml
index 63afd87f455..5165859b1b6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -63,7 +63,7 @@ proxy = [
     "azure-storage-blob>=12.28.0,<13.0",
     "mcp>=1.26.0,<2.0",
     "litellm-proxy-extras==0.4.74",
-    "litellm-enterprise==0.1.44",
+    "litellm-enterprise==0.1.45",
     "RestrictedPython>=8.1,<9.0",
     "rich>=13.9.4,<14.0",
     "polars>=1.38.1,<2.0",

From aaa58a72f763b5ce1462418013d14b9b24528d52 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Tue, 30 Jun 2026 13:21:49 -0700
Subject: [PATCH 35/51] chore: rebuild uv lock for version bumps

---
 uv.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/uv.lock b/uv.lock
index f76be69505f..768a2176d53 100644
--- a/uv.lock
+++ b/uv.lock
@@ -9,7 +9,7 @@ resolution-markers = [
 ]
 
 [options]
-exclude-newer = "2026-06-23T00:31:52.495979Z"
+exclude-newer = "2026-06-27T20:21:25.609736Z"
 exclude-newer-span = "P3D"
 
 [manifest]
@@ -3274,7 +3274,7 @@ wheels = [
 
 [[package]]
 name = "litellm"
-version = "1.91.0"
+version = "1.92.0"
 source = { editable = "." }
 dependencies = [
     { name = "aiohttp" },
@@ -3639,7 +3639,7 @@ proxy-dev = [
 
 [[package]]
 name = "litellm-enterprise"
-version = "0.1.44"
+version = "0.1.45"
 source = { editable = "enterprise" }
 
 [[package]]

From 5d4bb7548fa25a3d240b446e393d64dc788841fb Mon Sep 17 00:00:00 2001
From: yucheng-berri <yucheng@berri.ai>
Date: Tue, 30 Jun 2026 14:23:09 -0700
Subject: [PATCH 36/51] fix(token_counter): count legacy
 function_call.arguments (VERIA-492) (#31741)

* fix(token_counter): count legacy function_call.arguments (VERIA-492)

token_counter handled the modern assistant tool_calls field but had no
branch for the legacy OpenAI function_call payload. The value is a dict,
so it skipped every special-cased branch in _count_messages and fell
through to the unsupported-key continue, letting arbitrary text in
function_call.arguments slip past the count.

Resolves VERIA-492

* refactor(token_counter): raise on unexpected key in _count_function_call_tokens

Address Greptile P2: the helper's fallback branch previously applied
function_call logic to any key that wasn't tool_calls. Make the contract
explicit so a future caller can't silently miscount.
---
 litellm/litellm_core_utils/token_counter.py   | 43 +++++++++++++-----
 .../litellm_core_utils/test_token_counter.py  | 44 +++++++++++++++++++
 2 files changed, 77 insertions(+), 10 deletions(-)

diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py
index 56b9d42092c..071b16c8378 100644
--- a/litellm/litellm_core_utils/token_counter.py
+++ b/litellm/litellm_core_utils/token_counter.py
@@ -407,6 +407,37 @@ def token_counter(
     return num_tokens
 
 
+def _count_function_call_tokens(
+    key: str,
+    value: Any,
+    message: Mapping[str, Any],
+    count_function: TokenCounterFunction,
+) -> int:
+    """
+    Count tokens contributed by an assistant message's tool/function call payload.
+
+    Handles both the modern `tool_calls` list and the legacy OpenAI
+    `function_call` dict. Only the `arguments` string is counted (matching the
+    existing tool_calls behavior); names are accounted for elsewhere via the
+    tool/function definitions and `tool_choice`.
+    """
+    if key == "tool_calls":
+        if not isinstance(value, List):
+            raise ValueError(f"Unsupported type {type(value)} for key tool_calls in message {message}")
+        total = 0
+        for tool_call in value:
+            if "function" not in tool_call:
+                raise ValueError(f"Unsupported tool call {tool_call} must contain a function key")
+            function_arguments = tool_call["function"].get("arguments", "")
+            total += count_function(str(function_arguments))
+        return total
+    if key == "function_call":
+        if not isinstance(value, Mapping):
+            raise ValueError(f"Unsupported type {type(value)} for key function_call in message {message}")
+        return count_function(str(value.get("arguments", "")))
+    raise ValueError(f"Unexpected key {key!r}; expected 'tool_calls' or 'function_call'")
+
+
 def _count_messages(
     params: _MessageCountParams,
     messages: List[AllMessageValues],
@@ -430,16 +461,8 @@ def _count_messages(
         for key, value in message.items():
             if value is None:
                 pass
-            elif key == "tool_calls":
-                if isinstance(value, List):
-                    for tool_call in value:
-                        if "function" in tool_call:
-                            function_arguments = tool_call["function"].get("arguments", [])
-                            num_tokens += params.count_function(str(function_arguments))
-                        else:
-                            raise ValueError(f"Unsupported tool call {tool_call} must contain a function key")
-                else:
-                    raise ValueError(f"Unsupported type {type(value)} for key tool_calls in message {message}")
+            elif key in ("tool_calls", "function_call"):
+                num_tokens += _count_function_call_tokens(key, value, message, params.count_function)
             elif isinstance(value, str):
                 num_tokens += params.count_function(value)
                 if key == "name":
diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py
index 60e5a797627..71e686563a5 100644
--- a/tests/test_litellm/litellm_core_utils/test_token_counter.py
+++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py
@@ -97,6 +97,50 @@ def test_token_counter_normal_plus_function_calling():
 # test_token_counter_normal_plus_function_calling()
 
 
+def test_token_counter_legacy_function_call_counts_arguments():
+    """
+    Regression for VERIA-492 (Token-counter function_call bypass).
+
+    The legacy OpenAI assistant `function_call` field carries arbitrary text in
+    `arguments`. Before the fix, `_count_messages` had no branch for
+    `function_call` and fell through to the unsupported-key `continue`, so an
+    assistant turn could smuggle unlimited text past `token_counter` and the
+    proxy `/utils/token_counter` endpoint (and downstream pre-call budget /
+    `get_modified_max_tokens` math). After the fix it must be counted the
+    same as the equivalent `tool_calls` payload.
+    """
+    long_arg = "A" * 4000
+    fc_messages = [
+        {"role": "user", "content": "hi"},
+        {
+            "role": "assistant",
+            "content": None,
+            "function_call": {"name": "search", "arguments": long_arg},
+        },
+    ]
+    tc_messages = [
+        {"role": "user", "content": "hi"},
+        {
+            "role": "assistant",
+            "content": None,
+            "tool_calls": [
+                {
+                    "id": "call_1",
+                    "type": "function",
+                    "function": {"name": "search", "arguments": long_arg},
+                }
+            ],
+        },
+    ]
+    fc_tokens = token_counter(model="gpt-3.5-turbo", messages=fc_messages)
+    tc_tokens = token_counter(model="gpt-3.5-turbo", messages=tc_messages)
+    assert fc_tokens == tc_tokens, (
+        f"function_call arguments must count like tool_calls arguments; "
+        f"got function_call={fc_tokens}, tool_calls={tc_tokens}"
+    )
+    assert fc_tokens > 500, f"4000-char arguments payload must contribute real tokens, got {fc_tokens}"
+
+
 @pytest.mark.parametrize(
     "message_count_pair",
     MESSAGES_TEXT,

From a0b26d2c3c498f6c11ec4548aa986183eec18f20 Mon Sep 17 00:00:00 2001
From: tin-berri <tin@berri.ai>
Date: Tue, 30 Jun 2026 14:37:11 -0700
Subject: [PATCH 37/51] =?UTF-8?q?Revert=20"fix(presidio):=20stream=20SSE?=
 =?UTF-8?q?=20output=20incrementally=20instead=20of=20buffering=20t?=
 =?UTF-8?q?=E2=80=A6"=20(#31764)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This reverts commit 94936a3922aeb8aa9a4d5928e63ea6f15be6f098.
---
 .../guardrails/guardrail_hooks/presidio.py    | 523 ++++---------
 .../guardrail_hooks/test_presidio.py          | 712 +-----------------
 2 files changed, 152 insertions(+), 1083 deletions(-)

diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py
index e60b6233038..95876a55eab 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py
@@ -17,8 +17,6 @@ from typing import (
     TYPE_CHECKING,
     Any,
     AsyncGenerator,
-    Awaitable,
-    Callable,
     Dict,
     List,
     Literal,
@@ -56,14 +54,7 @@ from litellm.types.proxy.guardrails.guardrail_hooks.presidio import (
     PresidioAnalyzeRequest,
     PresidioAnalyzeResponseItem,
 )
-from litellm.types.utils import (
-    ChatCompletionDeltaToolCall,
-    Delta,
-    Function,
-    FunctionCall,
-    GuardrailStatus,
-    StreamingChoices,
-)
+from litellm.types.utils import GuardrailStatus, StreamingChoices
 from litellm.utils import (
     EmbeddingResponse,
     ImageResponse,
@@ -71,17 +62,6 @@ from litellm.utils import (
     ModelResponseStream,
 )
 
-# Trailing context (chars) the streaming output-masking path keeps buffered past
-# a sentence boundary before emitting, so a PII entity that straddles the
-# boundary is seen in full by Presidio and is never split across two analyze
-# calls. It bounds the largest single entity the incremental path can mask
-# without leaking; an entity longer than this could still be split.
-_PRESIDIO_STREAM_MARGIN = 96
-# Hard cap on buffered un-emitted output. Past this with no sentence boundary,
-# stable prefixes are flushed; if stability cannot be proven, the ambiguous
-# prefix is dropped while retaining the trailing margin.
-_PRESIDIO_STREAM_MAX_BUFFER = 2000
-
 
 class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
     user_api_key_cache = None
@@ -113,10 +93,6 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
         self.mock_redacted_text = mock_redacted_text
         self.output_parse_pii = output_parse_pii or False
         self.apply_to_output = apply_to_output
-        # Streaming output-masking safety window; instance attributes so tests can
-        # exercise incremental flushing with short content (see _mask_emit_decision).
-        self._stream_mask_margin = _PRESIDIO_STREAM_MARGIN
-        self._stream_mask_max_buffer = _PRESIDIO_STREAM_MAX_BUFFER
 
         # When output_parse_pii or apply_to_output is enabled, the guardrail must
         # also run on post_call to unmask/mask the response.  Expand the event_hook
@@ -1072,352 +1048,80 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
         )
         return response
 
-    @staticmethod
-    def _unmask_hold_len(text: str, token_keys: Any) -> int:
-        """Length of the trailing run of ``text`` that could still grow into a
-        PII placeholder token, so the unmask path holds it until the next chunk
-        completes (or aborts) the token instead of emitting a half-written
-        ``<PERSON_1>``."""
-        keys = tuple(token_keys)
-        if not text or not keys:
-            return 0
-        longest = max(len(key) for key in keys)
-        for start in range(max(0, len(text) - (longest - 1)), len(text)):
-            suffix = text[start:]
-            if any(key.startswith(suffix) for key in keys if len(suffix) < len(key)):
-                return len(text) - start
-        return 0
-
-    @staticmethod
-    def _mask_boundaries(text: str) -> tuple[int, ...]:
-        """Candidate flush points: a newline, or a sentence terminator already
-        followed by whitespace. A terminator at the very end of the buffer is
-        excluded because the next chunk may continue the token (``jane.`` +
-        ``doe@example.com``); it becomes a boundary once the whitespace arrives.
-        A boundary is only a *candidate* here; ``_mask_emit_decision`` still
-        confirms via a stability check that no entity straddles it."""
-        return tuple(
-            i + 1
-            for i in range(len(text))
-            if text[i] == "\n" or (text[i] in ".!?" and i + 1 < len(text) and text[i + 1].isspace())
-        )
-
-    async def _mask_emit_decision(
-        self,
-        buffer: str,
-        terminal: bool,
-        transform: "Callable[[str], Awaitable[str]]",
-    ) -> "tuple[str, str]":
-        """Decide how much of ``buffer`` is safe to mask and emit now, returning
-        ``(masked_emit, hold_raw)``.
-
-        A sentence boundary is not trusted blindly (it can fall inside a name
-        with an initial or an address spanning a newline). Instead a prefix is
-        emitted only when masking it in isolation matches the corresponding
-        prefix of masking the whole buffer, with at least ``_PRESIDIO_STREAM_MARGIN``
-        characters of lookahead still buffered past the cut. That guarantees any
-        entity overlapping the cut is present in full when the buffer is analyzed,
-        so a straddling entity makes the prefixes differ and the cut is held.
-        Past ``_PRESIDIO_STREAM_MAX_BUFFER`` with no sentence boundary the buffer
-        first tries stable forced cuts and then drops the ambiguous prefix while
-        retaining the trailing margin, so a failed stability check cannot grow
-        the held buffer without bound."""
-        if terminal:
-            return (await transform(buffer) if buffer else ""), ""
-        margin = self._stream_mask_margin
-        forced_cut = (
-            len(buffer) - margin if len(buffer) > self._stream_mask_max_buffer and len(buffer) > margin else None
-        )
-        cuts = [index for index in self._mask_boundaries(buffer) if len(buffer) - index >= margin]
-        if forced_cut is not None:
-            verbose_proxy_logger.warning(
-                "Presidio apply_to_output: buffered %d streamed chars with no "
-                "sentence boundary; bounding held stream state.",
-                len(buffer),
-            )
-            cuts.append(forced_cut)
-            cuts.extend(index for index in range(forced_cut, len(buffer)) if buffer[index].isspace())
-        if cuts:
-            masked_full = await transform(buffer)
-            for index in sorted(set(cuts), reverse=True):
-                masked_prefix = await transform(buffer[:index])
-                if masked_full.startswith(masked_prefix):
-                    return masked_prefix, buffer[index:]
-        if forced_cut is not None:
-            return "", buffer[forced_cut:]
-        return "", buffer
-
-    @staticmethod
-    def _accumulate_tool_calls(
-        tool_acc: dict[int, dict[int, dict[str, Optional[str]]]],
-        choice_index: int,
-        tool_calls: list[Any],
-    ) -> None:
-        choice_acc = tool_acc.setdefault(choice_index, {})  # mutable-ok: streaming tool-call accumulator
-        for tool_call in tool_calls:
-            entry = choice_acc.setdefault(  # mutable-ok: streaming tool-call accumulator
-                getattr(tool_call, "index", 0) or 0,
-                {"id": None, "type": None, "name": None, "args": ""},
-            )
-            if getattr(tool_call, "id", None):
-                entry["id"] = tool_call.id
-            if getattr(tool_call, "type", None):
-                entry["type"] = tool_call.type
-            function = getattr(tool_call, "function", None)
-            if function is not None:
-                if getattr(function, "name", None):
-                    entry["name"] = function.name
-                arguments = getattr(function, "arguments", None)
-                if isinstance(arguments, str):
-                    entry["args"] = (entry["args"] or "") + arguments
-
-    @staticmethod
-    def _accumulate_function_call(
-        func_acc: dict[int, dict[str, Optional[str]]],
-        choice_index: int,
-        function_call: Any,
-    ) -> None:
-        entry = func_acc.setdefault(  # mutable-ok: streaming function-call accumulator
-            choice_index, {"name": None, "args": ""}
-        )
-        if getattr(function_call, "name", None):
-            entry["name"] = function_call.name
-        arguments = getattr(function_call, "arguments", None)
-        if isinstance(arguments, str):
-            entry["args"] = (entry["args"] or "") + arguments
-
-    @staticmethod
-    async def _build_tool_calls(
-        choice_acc: dict[int, dict[str, Optional[str]]],
-        transform: "Callable[[str], Awaitable[str]]",
-    ) -> list[ChatCompletionDeltaToolCall]:
-        return [
-            ChatCompletionDeltaToolCall(
-                index=tool_index,
-                id=entry["id"],
-                type=entry["type"],
-                function=Function(
-                    name=entry["name"],
-                    arguments=(await transform(entry["args"]) if entry["args"] else ""),
-                ),
-            )
-            for tool_index, entry in sorted(choice_acc.items())
-        ]
-
-    @staticmethod
-    async def _build_function_call(
-        entry: Optional[dict[str, Optional[str]]],
-        transform: "Callable[[str], Awaitable[str]]",
-    ) -> Optional[FunctionCall]:
-        if entry is None:
-            return None
-        return FunctionCall(
-            name=entry["name"],
-            arguments=await transform(entry["args"]) if entry["args"] else "",
-        )
-
-    async def _rewrite_chat_chunk(
-        self,
-        chunk: ModelResponseStream,
-        content_buffers: dict[int, str],
-        tool_acc: dict[int, dict[int, dict[str, Optional[str]]]],
-        func_acc: dict[int, dict[str, Optional[str]]],
-        transform: "Callable[[str], Awaitable[str]]",
-        emit_content: "Callable[[str, bool], Awaitable[tuple[str, str]]]",
-    ) -> None:
-        """Transform one streaming chat chunk in place: text content is masked /
-        unmasked and emitted as soon as ``emit_content`` deems a prefix safe (it
-        returns the already-transformed text to emit plus the raw remainder to
-        hold), while tool-call and function-call argument fragments are
-        accumulated and emitted, fully transformed, on the chunk that closes the
-        choice."""
-        for choice in chunk.choices:
-            index = getattr(choice, "index", 0)
-            delta = getattr(choice, "delta", None)
-            if delta is None:
-                continue
-            terminal = bool(getattr(choice, "finish_reason", None))
-
-            tool_calls = getattr(delta, "tool_calls", None)
-            if tool_calls:
-                self._accumulate_tool_calls(tool_acc, index, tool_calls)
-                delta.tool_calls = None
-            function_call = getattr(delta, "function_call", None)
-            if function_call is not None:
-                self._accumulate_function_call(func_acc, index, function_call)
-                delta.function_call = None
-
-            raw_content = getattr(delta, "content", None)
-            content = raw_content if isinstance(raw_content, str) else None
-            if content is not None or terminal:
-                emitted, hold = await emit_content(content_buffers.pop(index, "") + (content or ""), terminal)
-                if hold:
-                    content_buffers[index] = hold
-                if emitted:
-                    delta.content = emitted
-                else:
-                    delta.content = None if content is None else ""
-
-            if terminal:
-                built_tool_calls = await self._build_tool_calls(tool_acc.get(index, {}), transform)
-                built_function_call = await self._build_function_call(func_acc.get(index), transform)
-                if built_tool_calls:
-                    delta.tool_calls = built_tool_calls
-                if built_function_call is not None:
-                    delta.function_call = built_function_call
-                tool_acc.pop(index, None)
-                func_acc.pop(index, None)
-
-    @staticmethod
-    async def _build_tail_chunk(
-        template: Optional[ModelResponseStream],
-        content_buffers: dict[int, str],
-        tool_acc: dict[int, dict[int, dict[str, Optional[str]]]],
-        func_acc: dict[int, dict[str, Optional[str]]],
-        transform: "Callable[[str], Awaitable[str]]",
-    ) -> Optional[ModelResponseStream]:
-        """Flush any content / tool-call state still held when a stream ends
-        without a finish-reason chunk to attach it to."""
-        if template is None:
-            return None
-        cls = _OPTIONAL_PresidioPIIMasking
-        choices: list[StreamingChoices] = []
-        for index in sorted(set(content_buffers) | set(tool_acc) | set(func_acc)):
-            held = content_buffers.get(index, "")
-            masked_content = await transform(held) if held else None
-            built_tool_calls = await cls._build_tool_calls(tool_acc.get(index, {}), transform)
-            built_function_call = await cls._build_function_call(func_acc.get(index), transform)
-            if masked_content is None and not built_tool_calls and built_function_call is None:
-                continue
-            choices.append(
-                StreamingChoices(
-                    index=index,
-                    delta=Delta(
-                        content=masked_content,
-                        tool_calls=built_tool_calls or None,
-                        function_call=built_function_call,
-                    ),
-                )
-            )
-        if not choices:
-            return None
-        return ModelResponseStream(
-            id=getattr(template, "id", None),
-            created=getattr(template, "created", None),
-            model=getattr(template, "model", None),
-            object="chat.completion.chunk",
-            choices=choices,
-        )
-
-    @staticmethod
-    def _redacted_chunk(chunk: ModelResponseStream) -> ModelResponseStream:
-        """Fail closed when masking a chunk raises: rebuild it with empty content
-        but its original ``finish_reason`` and choice indices preserved, so
-        possibly-unmasked PII never reaches the client yet a terminal chunk still
-        carries the completion signal instead of being dropped."""
-        return ModelResponseStream(
-            id=chunk.id,
-            created=chunk.created,
-            model=chunk.model,
-            object="chat.completion.chunk",
-            choices=[
-                StreamingChoices(
-                    index=choice.index,
-                    delta=Delta(content=None),
-                    finish_reason=choice.finish_reason,
-                )
-                for choice in chunk.choices
-            ],
-        )
-
     async def _stream_apply_output_masking(
         self,
         response: Any,
         request_data: dict,
     ) -> AsyncGenerator[Union[ModelResponseStream, bytes], None]:
         """Apply Presidio masking to streaming output (apply_to_output=True path)."""
-        presidio_config = self.get_presidio_settings_from_request_data(request_data or {})
+        from litellm.llms.base_llm.base_model_iterator import (
+            convert_model_response_to_streaming,
+        )
+        from litellm.main import stream_chunk_builder
+        from litellm.types.utils import ModelResponse
 
-        async def transform(text: str) -> str:
-            return await self.check_pii(
-                text=text,
-                output_parse_pii=False,
-                presidio_config=presidio_config,
-                request_data=request_data,
-            )
-
-        async def emit_content(text: str, terminal: bool) -> tuple[str, str]:
-            return await self._mask_emit_decision(text, terminal, transform)
-
-        async def flush_held() -> Optional[ModelResponseStream]:
-            """Build the held-content tail, failing closed (drop held content)
-            on a masking error instead of letting it abort the whole stream."""
-            try:
-                return await self._build_tail_chunk(last_chunk, content_buffers, tool_acc, func_acc, transform)
-            except Exception as e:
-                if self._is_guardrail_intervention(e):
-                    raise
-                verbose_proxy_logger.error(f"Error masking streaming PII tail: {str(e)}")
-                return None
-
-        content_buffers: dict[int, str] = {}
-        tool_acc: dict[int, dict[int, dict[str, Optional[str]]]] = {}
-        func_acc: dict[int, dict[str, Optional[str]]] = {}
-        last_chunk: Optional[ModelResponseStream] = None
-        masked_any_content = False
-        saw_unmaskable_shape = False
+        all_chunks: List[ModelResponseStream] = []
+        passthrough_due_to_unknown_stream_shape = False
         try:
             async for chunk in response:
-                if not isinstance(chunk, ModelResponseStream):
-                    # Flush buffered masked content before forwarding a non-chat
-                    # shape (raw bytes / a /v1/responses event) so the client
-                    # never sees a later event ahead of earlier masked text.
-                    tail = await flush_held()
-                    if tail is not None:
-                        yield tail
-                    content_buffers.clear()
-                    tool_acc.clear()
-                    func_acc.clear()
-                    saw_unmaskable_shape = True
+                if isinstance(chunk, ModelResponseStream):
+                    if passthrough_due_to_unknown_stream_shape:
+                        yield chunk
+                    else:
+                        all_chunks.append(chunk)
+                elif isinstance(chunk, bytes):
+                    yield chunk  # type: ignore[misc]
+                    continue
+                else:
+                    if all_chunks:
+                        # Flush buffered chunks and switch to transparent passthrough for this stream shape.
+                        # NOTE: these buffered chunks are emitted unmasked because this
+                        # stream mixed chunk types and cannot be safely reconstructed.
+                        verbose_proxy_logger.warning(
+                            "Presidio apply_to_output: mixed stream detected (ModelResponseStream + unknown event). "
+                            "Flushing %d buffered chunks without PII masking and switching to transparent passthrough.",
+                            len(all_chunks),
+                        )
+                        for buffered_chunk in all_chunks:
+                            yield buffered_chunk
+                        all_chunks = []
+                    passthrough_due_to_unknown_stream_shape = True
                     yield chunk
-                    continue
-                masked_any_content = True
-                last_chunk = chunk
-                try:
-                    await self._rewrite_chat_chunk(
-                        chunk,
-                        content_buffers,
-                        tool_acc,
-                        func_acc,
-                        transform,
-                        emit_content,
-                    )
-                except Exception as e:
-                    if self._is_guardrail_intervention(e):
-                        raise
-                    # Fail closed: a transient masking error redacts this chunk's
-                    # content (so possibly-unmasked PII never reaches the client)
-                    # but keeps its finish_reason and keeps the stream flowing,
-                    # rather than truncating the response or dropping a terminal
-                    # chunk's completion signal.
-                    verbose_proxy_logger.error(f"Error masking streaming PII chunk: {str(e)}")
-                    yield self._redacted_chunk(chunk)
-                    continue
-                yield chunk
-
-            tail = await flush_held()
-            if tail is not None:
-                yield tail
-            if not masked_any_content and saw_unmaskable_shape:
+            if passthrough_due_to_unknown_stream_shape:
+                verbose_proxy_logger.warning(
+                    "Presidio apply_to_output: streaming response contained unknown event objects "
+                    "(e.g. /v1/responses events). Output PII masking was skipped for this response."
+                )
+                return
+            if not all_chunks:
                 verbose_proxy_logger.warning(
                     "Presidio apply_to_output: streaming response contained no "
-                    "maskable chat content (e.g. raw SSE bytes or /v1/responses "
-                    "events). Output PII masking was skipped for this response."
+                    "ModelResponseStream chunks (e.g. raw SSE bytes or an empty "
+                    "upstream stream). Output PII masking was skipped for this "
+                    "response."
                 )
+                return
+
+            assembled_model_response = stream_chunk_builder(chunks=all_chunks, messages=request_data.get("messages"))
+
+            if not isinstance(assembled_model_response, ModelResponse):
+                for chunk in all_chunks:
+                    yield chunk
+                return
+
+            await self._process_response_for_pii(
+                response=assembled_model_response,
+                request_data=request_data,
+                mode="mask",
+            )
+
+            mock_response_stream = convert_model_response_to_streaming(assembled_model_response)
+            yield mock_response_stream
+
         except Exception as e:
-            if self._is_guardrail_intervention(e):
-                raise
             verbose_proxy_logger.error(f"Error masking streaming PII output: {str(e)}")
+            for chunk in all_chunks:
+                yield chunk
 
     @staticmethod
     def _unmask_sse_bytes_chunk(chunk: bytes, pii_tokens: Dict[str, str]) -> bytes:
@@ -1479,56 +1183,74 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
         request_data: dict,
     ) -> AsyncGenerator[Union[ModelResponseStream, bytes], None]:
         """Apply PII unmasking to streaming output (output_parse_pii=True path)."""
+        from litellm.llms.base_llm.base_model_iterator import (
+            convert_model_response_to_streaming,
+        )
+        from litellm.main import stream_chunk_builder
+        from litellm.types.utils import ModelResponse
+
         metadata = (request_data.get("metadata") or {}) if request_data else {}
         pii_tokens: Dict[str, str] = metadata.get("pii_tokens", {})
 
-        async def transform(text: str) -> str:
-            return self._unmask_pii_text(text, pii_tokens)
-
-        async def emit_content(text: str, terminal: bool) -> tuple[str, str]:
-            if terminal:
-                return (await transform(text) if text else ""), ""
-            hold = self._unmask_hold_len(text, pii_tokens.keys())
-            emit_raw, held = text[: len(text) - hold], text[len(text) - hold :]
-            return (await transform(emit_raw) if emit_raw else ""), held
-
-        content_buffers: dict[int, str] = {}
-        tool_acc: dict[int, dict[int, dict[str, Optional[str]]]] = {}
-        func_acc: dict[int, dict[str, Optional[str]]] = {}
-        last_chunk: Optional[ModelResponseStream] = None
+        remaining_chunks: List[ModelResponseStream] = []
+        saw_non_chat_chunk = False
         try:
             async for chunk in response:
-                if isinstance(chunk, bytes):
-                    tail = await self._build_tail_chunk(last_chunk, content_buffers, tool_acc, func_acc, transform)
-                    if tail is not None:
-                        yield tail
-                    content_buffers.clear()
-                    tool_acc.clear()
-                    func_acc.clear()
-                    yield (  # type: ignore[misc]
-                        self._unmask_sse_bytes_chunk(chunk, pii_tokens) if pii_tokens else chunk
-                    )
+                if isinstance(chunk, ModelResponseStream):
+                    if saw_non_chat_chunk:
+                        yield chunk
+                    else:
+                        remaining_chunks.append(chunk)
+                elif isinstance(chunk, bytes):
+                    if pii_tokens:
+                        yield self._unmask_sse_bytes_chunk(chunk, pii_tokens)  # type: ignore[misc]
+                    else:
+                        yield chunk  # type: ignore[misc]
                     continue
-                if not isinstance(chunk, ModelResponseStream):
-                    tail = await self._build_tail_chunk(last_chunk, content_buffers, tool_acc, func_acc, transform)
-                    if tail is not None:
-                        yield tail
-                    content_buffers.clear()
-                    tool_acc.clear()
-                    func_acc.clear()
-                    if getattr(chunk, "type", None) == "response.completed" and pii_tokens:
+                else:
+                    # /v1/responses events: unmask response.completed text in-place.
+                    # A mixed stream can't be reassembled, so flush buffered chat
+                    # chunks in order before passthrough instead of dropping them.
+                    if remaining_chunks and not saw_non_chat_chunk:
+                        for buffered_chunk in remaining_chunks:
+                            yield buffered_chunk
+                        remaining_chunks = []
+                    chunk_type = getattr(chunk, "type", None)
+                    if chunk_type == "response.completed" and pii_tokens:
                         self._unmask_responses_api_completed_chunk(chunk, pii_tokens)
+                    saw_non_chat_chunk = True
                     yield chunk
-                    continue
-                last_chunk = chunk
-                await self._rewrite_chat_chunk(chunk, content_buffers, tool_acc, func_acc, transform, emit_content)
-                yield chunk
 
-            tail = await self._build_tail_chunk(last_chunk, content_buffers, tool_acc, func_acc, transform)
-            if tail is not None:
-                yield tail
+            if saw_non_chat_chunk:
+                return
+
+            if not remaining_chunks:
+                return
+
+            assembled_model_response = stream_chunk_builder(
+                chunks=remaining_chunks, messages=request_data.get("messages")
+            )
+
+            if not isinstance(assembled_model_response, ModelResponse):
+                for chunk in remaining_chunks:
+                    yield chunk
+                return
+
+            self._preserve_usage_from_last_chunk(assembled_model_response, remaining_chunks)
+
+            await self._process_response_for_pii(
+                response=assembled_model_response,
+                request_data=request_data,
+                mode="unmask",
+            )
+
+            mock_response_stream = convert_model_response_to_streaming(assembled_model_response)
+            yield mock_response_stream
+
         except Exception as e:
             verbose_proxy_logger.error(f"Error in PII streaming processing: {str(e)}")
+            for chunk in remaining_chunks:
+                yield chunk
 
     async def async_post_call_streaming_iterator_hook(  # type: ignore[override]
         self,
@@ -1560,6 +1282,17 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
         async for chunk in self._stream_pii_unmasking(response, request_data):
             yield chunk
 
+    @staticmethod
+    def _preserve_usage_from_last_chunk(
+        assembled_model_response: Any,
+        chunks: List[Any],
+    ) -> None:
+        """Copy usage metadata from the last chunk when stream_chunk_builder misses it."""
+        if not getattr(assembled_model_response, "usage", None) and chunks:
+            last_chunk_usage = getattr(chunks[-1], "usage", None)
+            if last_chunk_usage:
+                setattr(assembled_model_response, "usage", last_chunk_usage)
+
     def get_presidio_settings_from_request_data(self, data: dict) -> Optional[PresidioPerRequestConfig]:
         if "metadata" in data:
             _metadata = data.get("metadata", None)
diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py
index ce4d6d4a8b0..253d989f203 100644
--- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py
+++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py
@@ -19,7 +19,7 @@ from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.guardrails.guardrail_hooks.presidio import (
     _OPTIONAL_PresidioPIIMasking,
 )
-from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
+from litellm.exceptions import GuardrailRaisedException
 from litellm.types.guardrails import LitellmParams, PiiAction, PiiEntityType
 from litellm.types.utils import Choices, Message, ModelResponse
 
@@ -2207,12 +2207,11 @@ async def test_apply_to_output_streaming_unknown_events_passthrough():
 
 
 @pytest.mark.asyncio
-async def test_apply_to_output_streaming_mixed_chunks_preserve_order():
+async def test_apply_to_output_streaming_mixed_chunks_flushes_and_warns():
     """
-    Regression test for mixed stream shape: a ModelResponseStream chat chunk
-    followed by an unknown responses-style event must be forwarded in order.
-    Incremental masking forwards chat chunks as they arrive, so a responses
-    event after them does not buffer or drop anything.
+    Regression test for mixed stream shape:
+    a buffered ModelResponseStream chunk followed by unknown responses-style
+    events should be preserved, and masking skip should be visible via warnings.
     """
     guardrail = _OPTIONAL_PresidioPIIMasking(
         mock_testing=True,
@@ -2239,14 +2238,26 @@ async def test_apply_to_output_streaming_mixed_chunks_preserve_order():
 
     mock_user_api_key = UserAPIKeyAuth(api_key="test-key")
     received = []
-    async for chunk in guardrail.async_post_call_streaming_iterator_hook(
-        user_api_key_dict=mock_user_api_key,
-        response=mock_stream(),
-        request_data={},
-    ):
-        received.append(chunk)
+    with patch(
+        "litellm.proxy.guardrails.guardrail_hooks.presidio.verbose_proxy_logger"
+    ) as mock_logger:
+        async for chunk in guardrail.async_post_call_streaming_iterator_hook(
+            user_api_key_dict=mock_user_api_key,
+            response=mock_stream(),
+            request_data={},
+        ):
+            received.append(chunk)
 
-    assert received == [model_chunk, response_completed]
+        # Preserve original ordering across mixed stream types.
+        assert received == [model_chunk, response_completed]
+
+        # Two warnings are expected:
+        # 1) mixed stream detected + unmasked flush
+        # 2) passthrough mode skipped output masking
+        assert mock_logger.warning.call_count == 2
+        warning_messages = [call.args[0] for call in mock_logger.warning.call_args_list]
+        assert any("mixed stream detected" in msg for msg in warning_messages)
+        assert any("unknown event objects" in msg for msg in warning_messages)
 
 
 # ---------------------------------------------------------------------------
@@ -2838,678 +2849,3 @@ async def test_stream_pii_unmasking_passthrough_when_no_tokens(mock_user_api_key
         chunks.append(chunk)
 
     assert chunks == [raw_chunk]
-
-
-# ---------------------------------------------------------------------------
-# LIT-3222: incremental SSE streaming for Presidio output masking / unmasking
-# ---------------------------------------------------------------------------
-
-from litellm.types.utils import (
-    ChatCompletionDeltaToolCall,
-    Delta,
-    Function,
-    StreamingChoices,
-)
-
-
-def _content_chunk(text, index=0, finish_reason=None):
-    return ModelResponseStream(
-        id="chatcmpl-lit3222",
-        created=1,
-        model="gpt-4o-mini",
-        object="chat.completion.chunk",
-        choices=[
-            StreamingChoices(
-                index=index, delta=Delta(content=text), finish_reason=finish_reason
-            )
-        ],
-    )
-
-
-def _non_empty_content(chunks):
-    out = []
-    for chunk in chunks:
-        for choice in chunk.choices:
-            piece = getattr(choice.delta, "content", None)
-            if piece:
-                out.append(piece)
-    return out
-
-
-async def _drive(guardrail, stream, request_data):
-    collected = []
-    async for chunk in guardrail.async_post_call_streaming_iterator_hook(
-        user_api_key_dict=UserAPIKeyAuth(api_key="test-key"),
-        response=stream,
-        request_data=request_data,
-    ):
-        collected.append(chunk)
-    return collected
-
-
-@pytest.mark.asyncio
-async def test_unmask_streaming_is_incremental_not_buffered():
-    """
-    output_parse_pii streaming must forward each content chunk as it arrives
-    (unmasked), not collapse the whole completion into a single end-of-stream
-    chunk. The buffering implementation yielded exactly one content chunk.
-    """
-    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, output_parse_pii=True)
-    request_data = {"metadata": {"pii_tokens": {"<PERSON_1>": "John Smith"}}}
-
-    pieces = ["Hello ", "<PERSON_1>", " is here."]
-
-    async def stream():
-        for i, piece in enumerate(pieces):
-            yield _content_chunk(piece)
-        yield _content_chunk("", finish_reason="stop")
-
-    collected = await _drive(guardrail, stream(), request_data)
-    content = _non_empty_content(collected)
-
-    assert len(content) >= 3, f"expected progressive chunks, got {content}"
-    assert "".join(content) == "Hello John Smith is here."
-    assert all("<PERSON_1>" not in piece for piece in content)
-
-
-@pytest.mark.asyncio
-async def test_unmask_streaming_token_split_across_chunks():
-    """
-    A placeholder token split across SSE chunks (``<PER`` + ``SON_1>``) must
-    still be unmasked atomically via the cross-chunk carry buffer.
-    """
-    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, output_parse_pii=True)
-    request_data = {"metadata": {"pii_tokens": {"<PERSON_1>": "John Smith"}}}
-
-    pieces = ["Hi ", "<PER", "SON_1>", "!"]
-
-    async def stream():
-        for piece in pieces:
-            yield _content_chunk(piece)
-        yield _content_chunk("", finish_reason="stop")
-
-    collected = await _drive(guardrail, stream(), request_data)
-    reassembled = "".join(_non_empty_content(collected))
-
-    assert reassembled == "Hi John Smith!"
-    assert "<PER" not in reassembled and "SON_1>" not in reassembled
-
-
-@pytest.mark.asyncio
-async def test_unmask_streaming_independent_per_choice_buffers():
-    """
-    With n>1 each choice keeps its own carry buffer, so a token split across
-    chunks on choice 1 does not corrupt choice 0 and vice versa.
-    """
-    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, output_parse_pii=True)
-    request_data = {
-        "metadata": {
-            "pii_tokens": {"<PERSON_1>": "John", "<PERSON_2>": "Jane"}
-        }
-    }
-
-    def two_choice_chunk(c0, c1):
-        return ModelResponseStream(
-            id="chatcmpl-lit3222",
-            created=1,
-            model="gpt-4o-mini",
-            object="chat.completion.chunk",
-            choices=[
-                StreamingChoices(index=0, delta=Delta(content=c0)),
-                StreamingChoices(index=1, delta=Delta(content=c1)),
-            ],
-        )
-
-    async def stream():
-        yield two_choice_chunk("<PERSON", "<PER")
-        yield two_choice_chunk("_1> ok", "SON_2>!")
-        yield two_choice_chunk("", "")
-
-    collected = []
-    async for chunk in guardrail.async_post_call_streaming_iterator_hook(
-        user_api_key_dict=UserAPIKeyAuth(api_key="test-key"),
-        response=stream(),
-        request_data=request_data,
-    ):
-        collected.append(chunk)
-
-    per_choice = {0: "", 1: ""}
-    for chunk in collected:
-        for choice in chunk.choices:
-            if choice.delta.content:
-                per_choice[choice.index] += choice.delta.content
-
-    assert per_choice[0] == "John ok"
-    assert per_choice[1] == "Jane!"
-
-
-@pytest.mark.asyncio
-async def test_unmask_streaming_tool_call_arguments_unmasked_at_finish():
-    """
-    Tool-call argument fragments carrying a placeholder token must be
-    reassembled and unmasked (the tool would otherwise receive ``<EMAIL...>``).
-    """
-    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, output_parse_pii=True)
-    request_data = {
-        "metadata": {"pii_tokens": {"<EMAIL_ADDRESS_1>": "real@example.com"}}
-    }
-
-    def tool_chunk(*, id=None, name=None, args, finish_reason=None):
-        return ModelResponseStream(
-            id="chatcmpl-lit3222",
-            created=1,
-            model="gpt-4o-mini",
-            object="chat.completion.chunk",
-            choices=[
-                StreamingChoices(
-                    index=0,
-                    delta=Delta(
-                        tool_calls=[
-                            ChatCompletionDeltaToolCall(
-                                index=0,
-                                id=id,
-                                type="function" if id else None,
-                                function=Function(name=name, arguments=args),
-                            )
-                        ]
-                    ),
-                    finish_reason=finish_reason,
-                )
-            ],
-        )
-
-    async def stream():
-        yield tool_chunk(id="call_1", name="send_email", args="")
-        yield tool_chunk(args='{"to": "<EMAIL_ADD')
-        yield tool_chunk(args='RESS_1>"}')
-        yield ModelResponseStream(
-            id="chatcmpl-lit3222",
-            created=1,
-            model="gpt-4o-mini",
-            object="chat.completion.chunk",
-            choices=[
-                StreamingChoices(index=0, delta=Delta(), finish_reason="tool_calls")
-            ],
-        )
-
-    collected = await _drive(guardrail, stream(), request_data)
-
-    tool_calls = [
-        tc
-        for chunk in collected
-        for choice in chunk.choices
-        for tc in (getattr(choice.delta, "tool_calls", None) or [])
-    ]
-    assert len(tool_calls) == 1
-    assert tool_calls[0].id == "call_1"
-    assert tool_calls[0].function.name == "send_email"
-    assert tool_calls[0].function.arguments == '{"to": "real@example.com"}'
-
-
-@pytest.mark.asyncio
-async def test_mask_streaming_is_incremental_per_sentence():
-    """
-    apply_to_output streaming must mask and emit a completed sentence once enough
-    following context confirms it, instead of buffering the whole completion into
-    one chunk. A small safety margin keeps the test content short.
-    """
-    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
-    guardrail._stream_mask_margin = 4
-
-    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
-        return text.replace("secret@example.com", "<EMAIL>")
-
-    guardrail.check_pii = mock_check_pii
-
-    pieces = ["My email is ", "secret@example.com. ", "Call me later."]
-
-    async def stream():
-        for piece in pieces:
-            yield _content_chunk(piece)
-        yield _content_chunk("", finish_reason="stop")
-
-    collected = await _drive(guardrail, stream(), {"metadata": {}})
-    content = _non_empty_content(collected)
-
-    assert len(content) >= 2, f"expected per-sentence chunks, got {content}"
-    reassembled = "".join(content)
-    assert reassembled == "My email is <EMAIL>. Call me later."
-    assert "secret@example.com" not in reassembled
-
-
-@pytest.mark.asyncio
-async def test_mask_streaming_tool_call_arguments_masked_at_finish():
-    """
-    Model-generated PII inside streamed tool-call arguments must be masked
-    before reaching the client, not passed through unmasked.
-    """
-    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
-
-    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
-        return text.replace("secret@example.com", "<EMAIL>")
-
-    guardrail.check_pii = mock_check_pii
-
-    def tool_chunk(*, id=None, name=None, args, finish_reason=None):
-        return ModelResponseStream(
-            id="chatcmpl-lit3222",
-            created=1,
-            model="gpt-4o-mini",
-            object="chat.completion.chunk",
-            choices=[
-                StreamingChoices(
-                    index=0,
-                    delta=Delta(
-                        tool_calls=[
-                            ChatCompletionDeltaToolCall(
-                                index=0,
-                                id=id,
-                                type="function" if id else None,
-                                function=Function(name=name, arguments=args),
-                            )
-                        ]
-                    ),
-                    finish_reason=finish_reason,
-                )
-            ],
-        )
-
-    async def stream():
-        yield tool_chunk(id="call_1", name="save", args='{"email": "sec')
-        yield tool_chunk(args='ret@example.com"}')
-        yield ModelResponseStream(
-            id="chatcmpl-lit3222",
-            created=1,
-            model="gpt-4o-mini",
-            object="chat.completion.chunk",
-            choices=[
-                StreamingChoices(index=0, delta=Delta(), finish_reason="tool_calls")
-            ],
-        )
-
-    collected = await _drive(guardrail, stream(), {"metadata": {}})
-
-    all_args = [
-        tc.function.arguments
-        for chunk in collected
-        for choice in chunk.choices
-        for tc in (getattr(choice.delta, "tool_calls", None) or [])
-    ]
-    assert all_args == ['{"email": "<EMAIL>"}']
-    assert all("secret@example.com" not in args for args in all_args)
-
-
-@pytest.mark.asyncio
-async def test_mask_streaming_flushes_buffered_content_before_passthrough_event():
-    """
-    Regression test (Greptile 4/5 finding): in the apply_to_output path, masked
-    content held in the buffer (no sentence boundary yet) must be flushed BEFORE
-    a non-chat passthrough event (e.g. a /v1/responses completion) is forwarded,
-    so the client never observes stream completion ahead of the final text.
-    """
-    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
-
-    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
-        return text.replace("secret@example.com", "<EMAIL>")
-
-    guardrail.check_pii = mock_check_pii
-
-    class FakeResponsesEvent:
-        def __init__(self, event_type: str):
-            self.type = event_type
-
-    completed = FakeResponsesEvent("response.completed")
-
-    async def stream():
-        # No sentence terminator -> held in the mask buffer, not yet emitted.
-        yield _content_chunk("My email is secret@example.com")
-        yield completed
-
-    collected = await _drive(guardrail, stream(), {"metadata": {}})
-
-    event_index = collected.index(completed)
-    masked_indexes = [
-        i
-        for i, chunk in enumerate(collected)
-        if not isinstance(chunk, FakeResponsesEvent)
-        and any(getattr(c.delta, "content", None) for c in chunk.choices)
-    ]
-    assert masked_indexes, "buffered masked content was never emitted"
-    assert max(masked_indexes) < event_index, "masked content must precede the event"
-
-    masked_text = "".join(
-        c.delta.content
-        for chunk in collected
-        if not isinstance(chunk, FakeResponsesEvent)
-        for c in chunk.choices
-        if getattr(c.delta, "content", None)
-    )
-    assert masked_text == "My email is <EMAIL>"
-    assert "secret@example.com" not in masked_text
-
-
-@pytest.mark.asyncio
-async def test_mask_streaming_does_not_split_entity_on_long_unpunctuated_run():
-    """
-    A long punctuation-free run must never force-flush mid-entity. The old
-    fixed-window fallback emitted at the last whitespace once the buffer grew
-    past a cap, so a space-bearing entity (SSN, phone) straddling that cut was
-    analyzed in two halves and leaked unmasked. Content past the last sentence
-    boundary is now held until a boundary or end-of-stream so each analyze call
-    sees the whole entity.
-    """
-    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
-
-    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
-        return text.replace("123 45 6789", "<US_SSN>")
-
-    guardrail.check_pii = mock_check_pii
-
-    filler = "data " * 90  # >400 chars, spaces only, no .!?\n boundary
-    async def stream():
-        yield _content_chunk(filler + "my ssn is 123 45 6789")
-        yield _content_chunk("", finish_reason="stop")
-
-    collected = await _drive(guardrail, stream(), {"metadata": {}})
-    reassembled = "".join(_non_empty_content(collected))
-
-    assert "<US_SSN>" in reassembled
-    assert "123 45 6789" not in reassembled
-    assert "456789" not in reassembled
-
-
-@pytest.mark.asyncio
-async def test_mask_streaming_preserves_stream_on_check_pii_error():
-    """
-    A transient Presidio failure mid-stream must not truncate the response. The
-    failing run is dropped (fail closed, never leaking the PII it could not mask)
-    while content that already flushed safely, later chunks, and the finish chunk
-    still reach the client.
-    """
-    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
-    guardrail._stream_mask_margin = 4
-
-    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
-        if "boom@example.com" in text:
-            raise RuntimeError("presidio down")
-        return text.replace("ok@example.com", "<EMAIL>")
-
-    guardrail.check_pii = mock_check_pii
-
-    pieces = [
-        "First ok@example.com. ",
-        "filler text here. ",
-        "Second boom@example.com. ",
-        "Third part here.",
-    ]
-
-    async def stream():
-        for piece in pieces:
-            yield _content_chunk(piece)
-        yield _content_chunk("", finish_reason="stop")
-
-    collected = await _drive(guardrail, stream(), {"metadata": {}})
-    reassembled = "".join(_non_empty_content(collected))
-
-    assert "<EMAIL>" in reassembled
-    assert "boom@example.com" not in reassembled
-    assert "Third part" in reassembled, "stream truncated after a masking error"
-    assert any(
-        getattr(choice, "finish_reason", None)
-        for chunk in collected
-        for choice in getattr(chunk, "choices", [])
-    ), "finish chunk dropped after a masking error"
-
-
-@pytest.mark.asyncio
-async def test_mask_streaming_error_preserves_tool_call_accumulators():
-    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
-
-    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
-        if "bad@example.com" in text:
-            raise RuntimeError("presidio down")
-        return text.replace("secret@example.com", "<EMAIL>")
-
-    guardrail.check_pii = mock_check_pii
-
-    def tool_chunk(*, id=None, name=None, args=None, finish_reason=None):
-        return ModelResponseStream(
-            id="chatcmpl-lit3222",
-            created=1,
-            model="gpt-4o-mini",
-            object="chat.completion.chunk",
-            choices=[
-                StreamingChoices(
-                    index=1,
-                    delta=(
-                        Delta(
-                            tool_calls=[
-                                ChatCompletionDeltaToolCall(
-                                    index=0,
-                                    id=id,
-                                    type="function" if id else None,
-                                    function=Function(name=name, arguments=args),
-                                )
-                            ]
-                        )
-                        if args is not None
-                        else Delta()
-                    ),
-                    finish_reason=finish_reason,
-                )
-            ],
-        )
-
-    async def stream():
-        yield tool_chunk(
-            id="call_1",
-            name="save",
-            args='{"email": "secret@example.com"}',
-        )
-        yield _content_chunk("bad@example.com", index=0, finish_reason="stop")
-        yield tool_chunk(finish_reason="tool_calls")
-
-    collected = await _drive(guardrail, stream(), {"metadata": {}})
-    tool_args = [
-        tc.function.arguments
-        for chunk in collected
-        for choice in chunk.choices
-        for tc in (getattr(choice.delta, "tool_calls", None) or [])
-    ]
-
-    assert tool_args == ['{"email": "<EMAIL>"}']
-
-
-@pytest.mark.asyncio
-async def test_mask_emit_decision_caps_buffer_when_stability_fails():
-    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
-    guardrail._stream_mask_margin = 3
-    guardrail._stream_mask_max_buffer = 6
-
-    async def unstable_transform(text):
-        return text[::-1]
-
-    emitted, held = await guardrail._mask_emit_decision(
-        "abcdefghij", False, unstable_transform
-    )
-
-    assert emitted == ""
-    assert held == "hij"
-
-
-@pytest.mark.parametrize(
-    "exception",
-    [
-        BlockedPiiEntityError(entity_type="EMAIL_ADDRESS", guardrail_name="presidio"),
-        GuardrailRaisedException(guardrail_name="presidio", message="invalid response"),
-    ],
-)
-@pytest.mark.asyncio
-async def test_mask_streaming_propagates_guardrail_interventions(exception):
-    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
-
-    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
-        raise exception
-
-    guardrail.check_pii = mock_check_pii
-
-    async def stream():
-        yield _content_chunk("blocked", finish_reason="stop")
-
-    with pytest.raises(type(exception)):
-        await _drive(guardrail, stream(), {"metadata": {}})
-
-
-@pytest.mark.asyncio
-async def test_mask_streaming_holds_terminator_at_chunk_end_until_whitespace():
-    """
-    A sentence terminator at the very end of a chunk is not a safe boundary: the
-    next chunk may continue the token. "Contact jane." followed by
-    "doe@example.com" must mask the whole email rather than flushing "jane." and
-    analyzing the two halves separately, which would leak the address.
-    """
-    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
-
-    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
-        return text.replace("jane.doe@example.com", "<EMAIL>")
-
-    guardrail.check_pii = mock_check_pii
-
-    async def stream():
-        yield _content_chunk("Contact jane.")
-        yield _content_chunk("doe@example.com")
-        yield _content_chunk("", finish_reason="stop")
-
-    collected = await _drive(guardrail, stream(), {"metadata": {}})
-    reassembled = "".join(_non_empty_content(collected))
-
-    assert "<EMAIL>" in reassembled
-    assert "jane.doe@example.com" not in reassembled
-    assert "jane." not in reassembled
-
-
-@pytest.mark.asyncio
-async def test_mask_streaming_does_not_split_entity_across_sentence_boundary():
-    """
-    An entity that straddles a sentence boundary (a name with a middle initial,
-    an address across a newline) must not be flushed in halves. The stability
-    check holds the prefix until masking it alone matches masking the whole
-    buffer, so the straddling entity is analyzed and masked as one unit instead
-    of leaking the part before the boundary.
-    """
-    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
-    guardrail._stream_mask_margin = 8
-
-    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
-        return text.replace("John Q. Public", "<PERSON>")
-
-    guardrail.check_pii = mock_check_pii
-
-    async def stream():
-        yield _content_chunk("Please greet John Q. Public warmly when they arrive.")
-        yield _content_chunk("", finish_reason="stop")
-
-    collected = await _drive(guardrail, stream(), {"metadata": {}})
-    reassembled = "".join(_non_empty_content(collected))
-
-    assert "<PERSON>" in reassembled
-    assert "John Q. Public" not in reassembled
-    assert "John Q." not in reassembled
-
-
-@pytest.mark.asyncio
-async def test_mask_streaming_caps_runaway_buffer_without_splitting_entity():
-    """
-    A punctuation-free run past the buffer cap must be flushed to bound memory,
-    but the forced flush still cuts a margin back from the end so a PII value
-    split exactly at the cap (``secret@`` in one chunk, ``example.com`` in the
-    next) stays buffered and is masked whole instead of leaking its raw halves.
-    The early flush plus the terminal flush yields more than one content chunk.
-    """
-    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
-    guardrail._stream_mask_margin = 8
-    guardrail._stream_mask_max_buffer = 20
-
-    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
-        return text.replace("secret@example.com", "<EMAIL>")
-
-    guardrail.check_pii = mock_check_pii
-
-    async def stream():
-        yield _content_chunk("please email me at secret@")  # 26 > cap, email cut
-        yield _content_chunk("example.com now")
-        yield _content_chunk("", finish_reason="stop")
-
-    collected = await _drive(guardrail, stream(), {"metadata": {}})
-    content = _non_empty_content(collected)
-    reassembled = "".join(content)
-
-    assert "<EMAIL>" in reassembled
-    assert "secret@example.com" not in reassembled
-    assert "secret@" not in reassembled
-    assert len(content) >= 2, f"cap did not flush before end of stream, got {content}"
-
-
-@pytest.mark.asyncio
-async def test_mask_streaming_preserves_finish_reason_when_terminal_chunk_fails():
-    """
-    When the masking call fails on the terminal chunk itself, that chunk must be
-    redacted in place (content dropped, fail closed) but keep its finish_reason,
-    so the client still receives the completion signal instead of a stream that
-    ends without one.
-    """
-    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True)
-    guardrail._stream_mask_margin = 4
-
-    async def mock_check_pii(text, output_parse_pii, presidio_config, request_data):
-        if "boom@example.com" in text:
-            raise RuntimeError("presidio down")
-        return text
-
-    guardrail.check_pii = mock_check_pii
-
-    async def stream():
-        yield _content_chunk("Hello world. ")
-        yield _content_chunk("more text here. ")
-        yield _content_chunk("boom@example.com", finish_reason="stop")
-
-    collected = await _drive(guardrail, stream(), {"metadata": {}})
-    reassembled = "".join(_non_empty_content(collected))
-
-    assert "boom@example.com" not in reassembled
-    assert "Hello world" in reassembled
-    finish_reasons = [
-        choice.finish_reason
-        for chunk in collected
-        for choice in getattr(chunk, "choices", [])
-        if getattr(choice, "finish_reason", None)
-    ]
-    assert "stop" in finish_reasons, "finish_reason dropped when terminal chunk failed"
-
-
-@pytest.mark.asyncio
-async def test_unmask_streaming_flushes_held_content_before_bytes():
-    """
-    When a held placeholder prefix is buffered and the next upstream item is a
-    raw SSE byte chunk, the held chat text must be flushed before the bytes so
-    the client never sees the byte chunk ahead of earlier content.
-    """
-    guardrail = _OPTIONAL_PresidioPIIMasking(mock_testing=True, output_parse_pii=True)
-    request_data = {"metadata": {"pii_tokens": {"<PERSON_1>": "Jane"}}}
-
-    async def stream():
-        yield _content_chunk("Hi <PERSON")  # "<PERSON" held as a partial token
-        yield b"data: {\"type\": \"ping\"}\n\n"
-
-    collected = await _drive(guardrail, stream(), request_data)
-
-    bytes_index = next(
-        i for i, chunk in enumerate(collected) if isinstance(chunk, bytes)
-    )
-    held_index = next(
-        i
-        for i, chunk in enumerate(collected)
-        if not isinstance(chunk, bytes)
-        and any("Jane" in (getattr(c.delta, "content", None) or "") for c in chunk.choices)
-    )
-    assert held_index < bytes_index, "raw bytes were sent before held chat content"

From ae8084de747c2e152d2594ddcc97b851da2d98f0 Mon Sep 17 00:00:00 2001
From: yuneng-jiang <yuneng@berri.ai>
Date: Tue, 30 Jun 2026 15:25:55 -0700
Subject: [PATCH 38/51] ci(codspeed): pin benchmark runner to ubuntu-24.04
 (#31746)

* ci(codspeed): pin benchmark runner to ubuntu-24.04

ubuntu-latest resolves to different runner images between the BASE
(main/staging) and HEAD (PR) runs, so CodSpeed reports 'Different
runtime environments detected' and emits false-positive regressions
(e.g. a -25.2% swing on test_completion_multi_turn in #31684, an MCP
auth fix with no LLM code changes). Pinning the runner to a fixed
image keeps BASE and HEAD on the same hardware so 1 ms swings on a
~3 ms benchmark stop blocking unrelated PRs.

Fixes #31738

* ci(codspeed): stop running benchmarks on litellm_internal_staging

The CodSpeed check flip-flops on internal staging and on PRs targeting
it (e.g. "+11.75% improvement" on one run, "-25.36% regression" on the
next) because the comparison flags "different runtime environments" and
the benchmarks are only 3-4 ms, so sub-millisecond runner noise swings
the result by 25-30%. Pinning the runner to ubuntu-24.04 in this PR
helps the head side, but the internal_staging base is still recorded on
the old unpinned runner, so comparisons keep flapping until the pin
merges and the base is re-baselined.

Until that settles, the red X's on internal staging make the OSS
project look unhealthy and confuse contributors, so drop the
litellm_internal_staging push and pull_request triggers and keep
CodSpeed running on main only.

---------

Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
---
 .github/workflows/codspeed.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
index 1fad82827ff..49f1d906069 100644
--- a/.github/workflows/codspeed.yml
+++ b/.github/workflows/codspeed.yml
@@ -4,11 +4,9 @@ on:
   push:
     branches:
       - main
-      - litellm_internal_staging
   pull_request:
     branches:
       - main
-      - litellm_internal_staging
   # Allow CodSpeed to trigger backtest performance analysis
   # in order to generate initial data
   workflow_dispatch:
@@ -23,7 +21,7 @@ concurrency:
 
 jobs:
   benchmarks:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 15
 
     steps:

From a7d8c6f46760eae1051d92129507b17e50642234 Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 30 Jun 2026 15:27:48 -0700
Subject: [PATCH 39/51] test(pass-through): de-flake vertex spend-log test by
 routing through the proxy (#31689)

* test(pass-through): de-flake vertex spend-log assertion by re-billing

The vertex pass-through spend-log test asserted that a single billed
generateContent call moved the global spend aggregate within a fixed
wait. CI failures show the call returning a valid response with real
usage, yet spend never increasing over a 240s poll.

Pass-through spend logging is best-effort: the success handler is
enqueued on a background worker that can drop or time out an individual
event under load and never retries it, so one billed call occasionally
never reaches LiteLLM_SpendLogs. Waiting longer cannot recover a dropped
event; only re-issuing the call can.

Re-bill the call up to a few times and require at least one to be
tracked, mirroring the sibling jest test that already retries. The test
still fails hard if cost tracking is actually broken, since then every
call records nothing. Also sum spend across all returned days instead of
matching the runner's local 'today', removing a separate UTC-rollover
flake.

* test(pass-through): route vertex spend-log test through proxy via direct HTTP

The vertexai SDK, configured with location="global" and an http api_endpoint
override, intermittently sends generateContent to the public Vertex endpoint
instead of the proxy. Proxy logs from a failing run show all 46 of the test's
own spend-log polls reaching the proxy while zero generateContent calls did, so
LiteLLM never saw the billed call and no spend was ever recorded; re-billing
through the SDK could not help because every retry bypassed the proxy too.

Issue the pass-through request directly over HTTP so it always hits the proxy,
minting a Google token from the same service-account credentials, then assert
that the specific call's own spend log lands with spend > 0, a gemini model, and
custom_llm_provider vertex_ai. A small best-effort retry covers the rare case
where the background logging worker drops a single event; failing every attempt
still fails hard so the test keeps its teeth if cost tracking breaks.

* test(pass-through): reuse LITE_LLM_ENDPOINT and drop needless async in get_tracked_spend
---
 tests/pass_through_tests/test_vertex_ai.py | 175 +++++++++++----------
 1 file changed, 96 insertions(+), 79 deletions(-)

diff --git a/tests/pass_through_tests/test_vertex_ai.py b/tests/pass_through_tests/test_vertex_ai.py
index e8223f2219c..35cb5f49c56 100644
--- a/tests/pass_through_tests/test_vertex_ai.py
+++ b/tests/pass_through_tests/test_vertex_ai.py
@@ -11,6 +11,7 @@ import json
 import os
 import pytest
 import asyncio
+import requests
 
 # Path to your service account JSON file
 SERVICE_ACCOUNT_FILE = "path/to/your/service-account.json"
@@ -57,98 +58,114 @@ def load_vertex_ai_credentials():
     os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(temp_file.name)
 
 
-async def call_spend_logs_endpoint():
-    """
-    Call this
-    curl -X GET "http://0.0.0.0:4000/spend/logs" -H "Authorization: Bearer sk-1234"
-    """
-    import datetime
-    import requests
-
-    todays_date = datetime.datetime.now().strftime("%Y-%m-%d")
-    url = f"http://0.0.0.0:4000/global/spend/logs?api_key=best-api-key-ever"
-    headers = {"Authorization": f"Bearer sk-1234"}
-    response = requests.get(url, headers=headers)
-    print("response from call_spend_logs_endpoint", response)
-
-    if response.status_code != 200:
-        print(f"spend logs endpoint returned {response.status_code}: {response.text}")
-        return None
-
-    json_response = response.json()
-
-    # get spend for today
-    """
-    json response looks like this
-
-    [{'date': '2024-08-30', 'spend': 0.00016600000000000002, 'api_key': 'best-api-key-ever'}]
-    """
-    print("json_response", json_response)
-
-    todays_date = datetime.datetime.now().strftime("%Y-%m-%d")
-    for spend_log in json_response:
-        if spend_log["date"] == todays_date:
-            return spend_log["spend"]
-
-
 LITE_LLM_ENDPOINT = "http://localhost:4000"
 
+SPEND_LOG_API_KEY = "best-api-key-ever"
 
-def _is_vertex_quota_error(exc: Exception) -> bool:
-    message = str(exc)
-    return (
-        "429" in message
-        or "Too Many Requests" in message
-        or "RESOURCE_EXHAUSTED" in message
+
+def get_tracked_spend() -> float:
+    """
+    Total spend recorded under the pass-through key in the global spend view.
+
+    Sums every day the endpoint returns instead of matching the runner's local
+    "today" so a UTC date rollover mid-test can't hide a freshly billed call, and
+    treats an unreachable endpoint as "nothing recorded yet" (0.0).
+    """
+    url = f"{LITE_LLM_ENDPOINT}/global/spend/logs?api_key={SPEND_LOG_API_KEY}"
+    response = requests.get(url, headers={"Authorization": "Bearer sk-1234"})
+    if response.status_code != 200:
+        print(f"global spend logs endpoint returned {response.status_code}: {response.text}")
+        return 0.0
+
+    rows = response.json()
+    print("global spend logs rows", rows)
+    return sum(float(row.get("spend") or 0.0) for row in rows)
+
+
+VERTEX_PROJECT = "litellm-ci-cd"
+VERTEX_MODEL = "gemini-3.1-flash-lite"
+VERTEX_GENERATE_CONTENT_URL = (
+    f"{LITE_LLM_ENDPOINT}/vertex_ai/v1/projects/{VERTEX_PROJECT}"
+    f"/locations/global/publishers/google/models/{VERTEX_MODEL}:generateContent"
+)
+
+
+def _vertex_access_token() -> str:
+    import google.auth
+    import google.auth.transport.requests
+
+    credentials, _ = google.auth.default(
+        scopes=["https://www.googleapis.com/auth/cloud-platform"]
     )
+    credentials.refresh(google.auth.transport.requests.Request())
+    return credentials.token
+
+
+def _spend_log_for_request(call_id: str) -> dict | None:
+    response = requests.get(
+        f"{LITE_LLM_ENDPOINT}/spend/logs?request_id={call_id}",
+        headers={"Authorization": "Bearer sk-1234"},
+        timeout=30,
+    )
+    if response.status_code != 200:
+        return None
+    rows = response.json()
+    return rows[0] if rows else None
+
+
+def _is_vertex_quota_error(response: requests.Response) -> bool:
+    return response.status_code == 429 or "RESOURCE_EXHAUSTED" in response.text
 
 
 @pytest.mark.asyncio()
 async def test_basic_vertex_ai_pass_through_with_spendlog():
-
-    spend_before = await call_spend_logs_endpoint() or 0.0
     load_vertex_ai_credentials()
+    access_token = _vertex_access_token()
 
-    vertexai.init(
-        project="litellm-ci-cd",
-        location="global",
-        api_endpoint=f"{LITE_LLM_ENDPOINT}/vertex_ai",
-        api_transport="rest",
-    )
+    # Drive the pass-through over HTTP instead of the vertexai SDK: the SDK intermittently
+    # routes generateContent to the public Vertex endpoint rather than the proxy override,
+    # so the call never reaches LiteLLM and no spend is logged. A direct request always
+    # hits the proxy. Spend logging then runs on a best-effort background worker that can
+    # drop a single event, so retry a few billed calls and assert that one specific call's
+    # spend log lands. Failing every attempt still fails hard, which is the signal we want
+    # if cost tracking is broken.
+    max_attempts = 3
+    poll_seconds = 60
+    poll_interval = 5
 
-    model = GenerativeModel(model_name="gemini-3.1-flash-lite")
-    try:
-        response = model.generate_content("hi")
-    except Exception as exc:
-        if _is_vertex_quota_error(exc):
+    for attempt in range(1, max_attempts + 1):
+        response = requests.post(
+            VERTEX_GENERATE_CONTENT_URL,
+            headers={
+                "Authorization": f"Bearer {access_token}",
+                "Content-Type": "application/json",
+            },
+            json={"contents": [{"role": "user", "parts": [{"text": "hi"}]}]},
+            timeout=60,
+        )
+        if _is_vertex_quota_error(response):
             pytest.skip("Vertex AI quota exhausted")
-        raise
+        assert (
+            response.status_code == 200
+        ), f"vertex pass-through call failed: {response.status_code} {response.text}"
 
-    print("response", response)
+        call_id = response.headers.get("x-litellm-call-id")
+        assert call_id, "proxy response missing x-litellm-call-id header"
 
-    # Spend logging is async/batched and can lag under CI load, so poll instead of
-    # sleeping a fixed amount. A transient empty read is skipped, not counted as 0.0
-    # spend, which would spuriously fail the assertion on an otherwise-billed call.
-    max_wait = 240  # total seconds to wait
-    poll_interval = 10  # seconds between checks
-    elapsed = 0
-    spend_after = spend_before
-    while elapsed < max_wait:
-        await asyncio.sleep(poll_interval)
-        elapsed += poll_interval
-        latest_spend = await call_spend_logs_endpoint()
-        if latest_spend is None:
-            print(f"spend logs unavailable (elapsed={elapsed}s), retrying")
-            continue
-        spend_after = latest_spend
-        print(f"spend_after (elapsed={elapsed}s)", spend_after)
-        if spend_after > spend_before:
-            break
+        for _ in range(poll_seconds // poll_interval):
+            await asyncio.sleep(poll_interval)
+            row = _spend_log_for_request(call_id)
+            if row is not None and float(row.get("spend") or 0) > 0:
+                assert "gemini" in row["model"], f"unexpected model in spend log: {row}"
+                assert (
+                    row["custom_llm_provider"] == "vertex_ai"
+                ), f"unexpected provider in spend log: {row}"
+                return
 
-    assert (
-        spend_after > spend_before
-    ), "Spend should be greater than before after {}s. spend_before: {}, spend_after: {}".format(
-        elapsed, spend_before, spend_after
+        print(f"attempt {attempt}: spend log for call {call_id} not found yet, re-billing")
+
+    pytest.fail(
+        f"Vertex pass-through spend never recorded after {max_attempts} billed calls"
     )
 
 
@@ -156,7 +173,7 @@ async def test_basic_vertex_ai_pass_through_with_spendlog():
 @pytest.mark.skip(reason="skip flaky test - vertex pass through streaming is flaky")
 async def test_basic_vertex_ai_pass_through_streaming_with_spendlog():
 
-    spend_before = await call_spend_logs_endpoint() or 0.0
+    spend_before = get_tracked_spend()
     print("spend_before", spend_before)
     load_vertex_ai_credentials()
 
@@ -176,7 +193,7 @@ async def test_basic_vertex_ai_pass_through_streaming_with_spendlog():
     print("response", response)
 
     await asyncio.sleep(20)
-    spend_after = await call_spend_logs_endpoint()
+    spend_after = get_tracked_spend()
     print("spend_after", spend_after)
     assert (
         spend_after > spend_before

From 41f9d8de7b16516808bbad3b5be5da9dd736e698 Mon Sep 17 00:00:00 2001
From: yucheng-berri <yucheng@berri.ai>
Date: Tue, 30 Jun 2026 15:30:08 -0700
Subject: [PATCH 40/51] fix(proxy): extend banned-params + admin-clear lists
 for NVIDIA Riva (VERIA-493) (#31742)

Two NVIDIA-Riva-specific fields consumed by the audio-transcription
handler via the provider's `optional_params` passthrough were not
covered by the proxy's existing banned-request-body list or the
admin-config clearing list applied on `api_base` BYOK override:

* `nvcf_function_id`
* `use_ssl`

Add both to `_BANNED_REQUEST_BODY_PARAMS` in
`litellm/proxy/auth/auth_utils.py` and to the kwargs-only list in
`_admin_config_fields_to_clear_on_base_override()` in
`litellm/router_utils/clientside_credential_handler.py`, next to the
analogous provider-specific entries already there (`aws_bedrock_*`,
OCI provider fields, etc.). Same admin opt-ins as every other entry
on those lists (`general_settings.allow_client_side_credentials`
proxy-wide, or `configurable_clientside_auth_params` per deployment).

Regression tests in `tests/test_litellm/proxy/auth/test_auth_utils.py`
cover root-level rejection, the historical `api_key` bypass, both
admin opt-in paths (proxy-wide and per-deployment), nested-container
smuggling via the existing recursive walk, and clearing on
`api_base` override. Mutation check verified.

Resolves VERIA-493
---
 litellm/proxy/auth/auth_utils.py              |   6 +
 .../clientside_credential_handler.py          |   7 +
 .../proxy/auth/test_auth_utils.py             | 173 ++++++++++++++++++
 3 files changed, 186 insertions(+)

diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py
index b1bce352784..2bf0acc7232 100644
--- a/litellm/proxy/auth/auth_utils.py
+++ b/litellm/proxy/auth/auth_utils.py
@@ -278,6 +278,12 @@ _BANNED_REQUEST_BODY_PARAMS: Tuple[str, ...] = (
     "s3_endpoint_url",
     "sagemaker_base_url",
     "deployment_url",
+    # NVIDIA Riva fields consumed by the audio-transcription handler
+    # via ``optional_params``. Banned for the same reason as the
+    # provider-specific entries above: a caller-supplied value retargets
+    # the request away from the admin's pinned configuration.
+    "nvcf_function_id",
+    "use_ssl",
     # SDK-only field; also rejected outright in is_request_body_safe.
     "model_list",
     # Observability credentials, hosts, and project identifiers: derived
diff --git a/litellm/router_utils/clientside_credential_handler.py b/litellm/router_utils/clientside_credential_handler.py
index e992ef63658..8234d89e248 100644
--- a/litellm/router_utils/clientside_credential_handler.py
+++ b/litellm/router_utils/clientside_credential_handler.py
@@ -52,6 +52,13 @@ def _admin_config_fields_to_clear_on_base_override() -> List[str]:
         "oci_tenancy",
         "oci_key",
         "oci_key_file",
+        # NVIDIA Riva fields — consumed by
+        # ``litellm/llms/nvidia_riva/audio_transcription/handler.py`` via
+        # optional_params and not declared on CredentialLiteLLMParams.
+        # Admin-pinned values must not flow through on a caller-redirected
+        # ``api_base`` for the same reason as the OCI entries above.
+        "nvcf_function_id",
+        "use_ssl",
     ]
     return typed_fields + kwargs_only_fields
 
diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py
index cd8cf10d037..d5d2d27cb7e 100644
--- a/tests/test_litellm/proxy/auth/test_auth_utils.py
+++ b/tests/test_litellm/proxy/auth/test_auth_utils.py
@@ -1520,6 +1520,42 @@ class TestGetDynamicLitellmParamsClearsAdminConfigOnBaseOverride:
         assert "vertex_credentials" not in out
         assert "vertex_project" not in out
 
+    def test_clears_nvcf_function_id_on_base_override(self):
+        from litellm.router_utils.clientside_credential_handler import (
+            get_dynamic_litellm_params,
+        )
+
+        admin_params = {
+            "model": "nvidia_riva/parakeet",
+            "api_base": "grpc.nvcf.nvidia.com:443",
+            "api_key": "nvapi-admin",
+            "nvcf_function_id": "admin-pinned-function",
+        }
+        out = get_dynamic_litellm_params(
+            litellm_params=dict(admin_params),
+            request_kwargs={"api_base": "self-hosted.example.com:50051"},
+        )
+        assert out["api_base"] == "self-hosted.example.com:50051"
+        assert "nvcf_function_id" not in out
+
+    def test_clears_use_ssl_on_base_override(self):
+        from litellm.router_utils.clientside_credential_handler import (
+            get_dynamic_litellm_params,
+        )
+
+        admin_params = {
+            "model": "nvidia_riva/parakeet",
+            "api_base": "grpc.nvcf.nvidia.com:443",
+            "api_key": "nvapi-admin",
+            "use_ssl": True,
+        }
+        out = get_dynamic_litellm_params(
+            litellm_params=dict(admin_params),
+            request_kwargs={"api_base": "self-hosted.example.com:50051"},
+        )
+        assert out["api_base"] == "self-hosted.example.com:50051"
+        assert "use_ssl" not in out
+
     def test_caller_resupplied_value_overrides_admin_value_on_base_override(self):
         # When the caller redirects ``api_base`` and *also* supplies their
         # own value for one of the admin fields (e.g. ``organization``),
@@ -1712,6 +1748,127 @@ class TestIsRequestBodySafeBlocksBedrockProjectOverride:
         )
 
 
+class TestIsRequestBodySafeBlocksNVCFFunctionOverride:
+    """``nvcf_function_id`` is rejected as a request-body param unless the
+    admin opted in proxy-wide or per-deployment."""
+
+    def test_nvcf_function_id_in_request_body_is_rejected(self):
+        with pytest.raises(ValueError, match="nvcf_function_id"):
+            is_request_body_safe(
+                request_body={
+                    "model": "nvidia_riva/parakeet",
+                    "nvcf_function_id": "caller-supplied",
+                },
+                general_settings={},
+                llm_router=None,
+                model="nvidia_riva/parakeet",
+            )
+
+    def test_nvcf_function_id_with_api_key_still_rejected(self):
+        with pytest.raises(ValueError, match="nvcf_function_id"):
+            is_request_body_safe(
+                request_body={
+                    "model": "nvidia_riva/parakeet",
+                    "api_key": "sk-anything",
+                    "nvcf_function_id": "caller-supplied",
+                },
+                general_settings={},
+                llm_router=None,
+                model="nvidia_riva/parakeet",
+            )
+
+    def test_admin_opt_in_proxy_wide_allows_nvcf_function_id(self):
+        assert (
+            is_request_body_safe(
+                request_body={
+                    "model": "nvidia_riva/parakeet",
+                    "nvcf_function_id": "byok-function-id",
+                },
+                general_settings={"allow_client_side_credentials": True},
+                llm_router=None,
+                model="nvidia_riva/parakeet",
+            )
+            is True
+        )
+
+    def test_admin_opt_in_per_deployment_allows_nvcf_function_id(self, monkeypatch):
+        """The error message lists per-deployment ``configurable_clientside_auth_params``
+        as a second opt-in. Cover that path too so it can't silently regress."""
+        from litellm.proxy.auth import auth_utils
+
+        monkeypatch.setattr(
+            auth_utils,
+            "_allow_model_level_clientside_configurable_parameters",
+            lambda model, param, request_body_value, llm_router: param == "nvcf_function_id",
+        )
+
+        assert (
+            is_request_body_safe(
+                request_body={
+                    "model": "nvidia_riva/parakeet",
+                    "nvcf_function_id": "byok-function-id",
+                },
+                general_settings={},
+                llm_router=None,
+                model="nvidia_riva/parakeet",
+            )
+            is True
+        )
+
+
+class TestIsRequestBodySafeBlocksRivaUseSsl:
+    """``use_ssl`` is rejected as a request-body param unless the admin
+    opted in proxy-wide or per-deployment."""
+
+    def test_use_ssl_in_request_body_is_rejected(self):
+        with pytest.raises(ValueError, match="use_ssl"):
+            is_request_body_safe(
+                request_body={
+                    "model": "nvidia_riva/parakeet",
+                    "use_ssl": False,
+                },
+                general_settings={},
+                llm_router=None,
+                model="nvidia_riva/parakeet",
+            )
+
+    def test_admin_opt_in_proxy_wide_allows_use_ssl(self):
+        assert (
+            is_request_body_safe(
+                request_body={
+                    "model": "nvidia_riva/parakeet",
+                    "use_ssl": True,
+                },
+                general_settings={"allow_client_side_credentials": True},
+                llm_router=None,
+                model="nvidia_riva/parakeet",
+            )
+            is True
+        )
+
+    def test_admin_opt_in_per_deployment_allows_use_ssl(self, monkeypatch):
+        from litellm.proxy.auth import auth_utils
+
+        monkeypatch.setattr(
+            auth_utils,
+            "_allow_model_level_clientside_configurable_parameters",
+            lambda model, param, request_body_value, llm_router: param == "use_ssl",
+        )
+
+        assert (
+            is_request_body_safe(
+                request_body={
+                    "model": "nvidia_riva/parakeet",
+                    "use_ssl": True,
+                },
+                general_settings={},
+                llm_router=None,
+                model="nvidia_riva/parakeet",
+            )
+            is True
+        )
+
+
 # ── is_request_body_safe nested-config recursion (VERIA-6) ────────────────────
 
 
@@ -1748,6 +1905,22 @@ class TestIsRequestBodySafeNestedConfig:
                 model="milvus-store",
             )
 
+    def test_nested_nvcf_function_id_in_metadata_blocked(self):
+        """Smuggling ``nvcf_function_id`` via ``metadata`` / ``extra_body``
+        is the same shape as the VERIA-6 ``api_base`` bypass — must be
+        rejected by the recursive walk so the NVCF override gate cannot
+        be sidestepped with nesting."""
+        with pytest.raises(ValueError, match="nvcf_function_id"):
+            is_request_body_safe(
+                request_body={
+                    "model": "nvidia_riva/parakeet",
+                    "litellm_metadata": {"nvcf_function_id": "attacker-via-metadata"},
+                },
+                general_settings={},
+                llm_router=None,
+                model="nvidia_riva/parakeet",
+            )
+
     def test_nested_langfuse_host_in_embedding_config_blocked(self):
         """The recursion uses the *full* banned-param list, not a special
         subset — so any flag that's banned at the root is also banned

From 833406a711111e8e1347eead3aac5a831de39ea8 Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Tue, 30 Jun 2026 17:20:24 -0700
Subject: [PATCH 41/51] fix(ui): rotate model credentials in a dedicated modal
 so a normal save can't overwrite secrets (#28089)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* feat(ui): add provider auth editing to the model edit view

Provider API keys / auth could previously only be changed by hand-editing
the raw litellm_params JSON, so there was no first-class way to rotate a
model's key. Adds an Authentication section that renders the correct
provider-specific fields (reusing ProviderSpecificFields) keyed off the
model's custom_llm_provider; fields are blank ("leave blank to keep
current") so untouched secrets are preserved and only entered values are
PATCHed and encrypted at rest.

Resolves LIT-3169

* refactor(ui): simplify model auth editing; fix stale credential branch

Drop the onFieldsResolved/authFieldKeys round trip: the parent now resolves
provider auth field keys itself via the new useProviderAuthFieldKeys hook
(same metadata ProviderSpecificFields renders), removing the report-up effect
and its stable-reference footgun. ProviderSpecificFields keeps only
excludeKeys (real need: suppress duplicate visible inputs).

Fix the stale Authentication branch: derive it from the live
litellm_credential_name form value (Form.useWatch) instead of the server
snapshot, so clearing/adding a credential mid-edit shows the right UI. Also
skip inline auth updates entirely when a named credential is selected, so we
never submit a credential name and raw inline auth together.

* fix(ui): don't leak freshly-entered model auth secrets to display/console

The auth values a user types are still sent in the PATCH request, but:
- strip them from the locally-stored litellm_params after save so the
  read-only LiteLLM Params JSON doesn't render the plaintext key
- remove the debug console.log in modelPatchUpdateCall that dumped the
  full update payload (incl. api_key / vertex_credentials) to the browser
  console on every model update

Backend stores these encrypted and returns them masked on refetch.

* fix(ui): don't require blank auth fields in model edit context

Auth fields render blank ('leave blank to keep'), but required metadata
(e.g. OpenAI api_key) added a required validation rule that blocked
onFinish entirely — making it impossible to save any unrelated edit
without re-entering the secret. Add a disableRequired prop to
ProviderSpecificFields and set it in the model edit Authentication section.

* fix(ui): rotate model credentials in a dedicated modal so a normal save can't overwrite secrets

The model edit form seeded the read-only LiteLLM Params textarea with the whole
litellm_params blob and re-sent all of it on every save. Because /model/info
redacts secrets by masking them ("azur****BBCC") rather than removing them, any
save re-encrypted the asterisk mask over the real value and silently destroyed
credentials such as azure_ad_token, aws_session_token, watsonx token/zen_api_key
and the OCI key fields. api_key, client_secret, vertex_credentials and the AWS
access/secret keys were safe only because the backend strips those entirely

Credential rotation now lives in a dedicated UpdateModelCredentialsModal that
PATCHes only the fields the user types, decoupled from the params blob; the
backend already merges partial litellm_params, so the rest of the deployment is
left untouched. The general edit form drops masked values from both the textarea
seed and the outbound payload, so a normal save can never carry a redacted secret

Also removes the now-unused inline auth section and its excludeKeys and
useProviderAuthFieldKeys plumbing, strips secret-leaking console.logs from the
provider upload handler and the model-update response, and fixes a
react-hooks/use-memo error that was failing the frontend-lint CI job

* chore(ui): ratchet no-explicit-any lint metric to 2013

Removing the credential-echoing console.log (and its info: any param) from the
provider upload handler dropped the tracked count by one; update the committed
baseline so the Check lint budgets CI step is not stale

* refactor(ui): scope the model credential modal to api-key rotation only

Narrows UpdateModelCredentialsModal to a single API Key field. On submit it
PATCHes only { api_key }, so the backend merge leaves every other deployment
param untouched; a model authed via azure_ad_token, AWS keys, or a Vertex JSON
won't have anything to rotate here yet, which is the intended scope for now.

Drops the multi-field provider rendering this added earlier, which also removes
the now-unused disableRequired prop from ProviderSpecificFields and reverts that
shared component to its prior shape. The "Update API Key" trigger button is now
an antd Button rather than a TremorButton, so the feature introduces no tremor.

* refactor(ui): convert the model detail toolbar buttons from tremor to antd

Switches Test Connection, Re-use Credentials and Delete Model to antd Button so
the toolbar matches the Update API Key button and no longer mixes libraries;
Delete Model uses antd's danger styling instead of hand-rolled red classes

* style(ui): make the api-key modal submit button primary and drop the Need Help link
---
 ui/litellm-dashboard/eslint-metrics.json      |  2 +-
 .../add_model/provider_specific_fields.tsx    | 17 ----
 .../src/components/model_info_view.test.tsx   | 45 ++++++++++
 .../src/components/model_info_view.tsx        | 74 +++++++++++++----
 .../src/components/networking.tsx             |  5 +-
 .../update_model_credentials_modal.test.tsx   | 83 +++++++++++++++++++
 .../update_model_credentials_modal.tsx        | 76 +++++++++++++++++
 7 files changed, 265 insertions(+), 37 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/components/update_model_credentials_modal.test.tsx
 create mode 100644 ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx

diff --git a/ui/litellm-dashboard/eslint-metrics.json b/ui/litellm-dashboard/eslint-metrics.json
index 09ad247391b..92c5a991eb6 100644
--- a/ui/litellm-dashboard/eslint-metrics.json
+++ b/ui/litellm-dashboard/eslint-metrics.json
@@ -1,5 +1,5 @@
 {
-  "@typescript-eslint/no-explicit-any": 2014,
+  "@typescript-eslint/no-explicit-any": 2013,
   "complexity": 126,
   "max-depth": 61
 }
diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx
index 045a9b0c1b6..205292edfb4 100644
--- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx
@@ -212,9 +212,7 @@ const ProviderSpecificFields: React.FC<ProviderSpecificFieldsProps> = ({ selecte
         reader.onload = (e) => {
           if (e.target) {
             const jsonStr = e.target.result as string;
-            console.log(`Setting field value from JSON, length: ${jsonStr.length}`);
             form.setFieldsValue({ vertex_credentials: jsonStr });
-            console.log("Form values after setting:", form.getFieldsValue());
           }
         };
         reader.readAsText(file);
@@ -222,14 +220,6 @@ const ProviderSpecificFields: React.FC<ProviderSpecificFieldsProps> = ({ selecte
       // Prevent upload
       return false;
     },
-    onChange(info: any) {
-      console.log("Upload onChange triggered in ProviderSpecificFields");
-      console.log("Current form values:", form.getFieldsValue());
-
-      if (info.file.status !== "uploading") {
-        console.log(info.file, info.fileList);
-      }
-    },
   };
 
   return (
@@ -271,16 +261,9 @@ const ProviderSpecificFields: React.FC<ProviderSpecificFieldsProps> = ({ selecte
               <Upload
                 {...handleUpload}
                 onChange={(info) => {
-                  // First call the original onChange
                   if (uploadProps?.onChange) {
                     uploadProps.onChange(info);
                   }
-
-                  // Check the field value after a short delay
-                  setTimeout(() => {
-                    const value = form.getFieldValue(field.key);
-                    console.log(`${field.key} value after upload:`, JSON.stringify(value));
-                  }, 500);
                 }}
               >
                 <Button2 icon={<UploadOutlined />}>Click to Upload</Button2>
diff --git a/ui/litellm-dashboard/src/components/model_info_view.test.tsx b/ui/litellm-dashboard/src/components/model_info_view.test.tsx
index d91d5ec307e..2546601b4db 100644
--- a/ui/litellm-dashboard/src/components/model_info_view.test.tsx
+++ b/ui/litellm-dashboard/src/components/model_info_view.test.tsx
@@ -633,6 +633,51 @@ describe("ModelInfoView", () => {
     expect(updatePayload.litellm_params).not.toHaveProperty("output_cost_per_token");
   });
 
+  it("never re-sends a masked secret on save (regression: masked auth value must not overwrite the real secret)", async () => {
+    // /model/info redacts secrets by masking (e.g. "azur****BBCC"), not removing them.
+    // A plain save re-PATCHes the whole litellm_params blob; if the masked value were
+    // sent, the backend would encrypt the asterisks over the real azure_ad_token and
+    // silently destroy the credential. The edit form must strip masked values entirely.
+    const maskedSecret = "azur********************************************BBCC";
+    const maskedModelData = {
+      ...defaultModelData,
+      litellm_params: {
+        model: "azure/gpt-4o",
+        api_base: "https://example-az.openai.azure.com",
+        custom_llm_provider: "azure",
+        azure_ad_token: maskedSecret,
+      },
+    };
+    mockUseModelsInfo.mockReturnValue({
+      data: { data: [maskedModelData] },
+      isLoading: false,
+      error: null,
+    });
+    mockModelInfoV1Call.mockResolvedValue({ data: [maskedModelData] });
+
+    const user = userEvent.setup();
+    render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} />, { wrapper });
+
+    await waitFor(() => {
+      expect(screen.getByRole("button", { name: /edit settings/i })).toBeInTheDocument();
+    });
+    await user.click(screen.getByRole("button", { name: /edit settings/i }));
+
+    await waitFor(() => {
+      expect(screen.getByRole("button", { name: /save changes/i })).toBeInTheDocument();
+    });
+    await user.click(screen.getByRole("button", { name: /save changes/i }));
+
+    await waitFor(() => {
+      expect(mockModelPatchUpdateCall).toHaveBeenCalled();
+    });
+
+    const updatePayload = mockModelPatchUpdateCall.mock.calls[0][1];
+    expect(updatePayload.litellm_params.azure_ad_token).not.toBe(maskedSecret);
+    // No masked value may appear anywhere in the outbound params.
+    expect(JSON.stringify(updatePayload.litellm_params)).not.toContain("**");
+  });
+
   it("should display health check model field for wildcard models", async () => {
     const wildcardModelData = {
       ...defaultModelData,
diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx
index 66a00b9bbe3..45c5b0fd9b6 100644
--- a/ui/litellm-dashboard/src/components/model_info_view.tsx
+++ b/ui/litellm-dashboard/src/components/model_info_view.tsx
@@ -1,5 +1,6 @@
 import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap";
 import { useModelHub, useModelsInfo } from "@/app/(dashboard)/hooks/models/useModels";
+import { useQueryClient } from "@tanstack/react-query";
 import { transformModelData } from "@/app/(dashboard)/models-and-endpoints/utils/modelDataTransformer";
 import { InfoCircleOutlined } from "@ant-design/icons";
 import { ArrowLeftIcon, KeyIcon, RefreshIcon, TrashIcon } from "@heroicons/react/outline";
@@ -40,6 +41,7 @@ import {
   testConnectionRequest,
 } from "./networking";
 import { getProviderLogoAndName } from "./provider_info_helpers";
+import UpdateModelCredentialsModal from "./update_model_credentials_modal";
 import NumericalInput from "./shared/numerical_input";
 import { Tag } from "./tag_management/types";
 import { getDisplayModelName } from "./view_model/model_name_display";
@@ -54,6 +56,18 @@ interface ModelInfoViewProps {
   modelAccessGroups: string[] | null;
 }
 
+// The /model/info response redacts secrets by masking them (e.g. "sk-1****2345"),
+// not by removing them. The edit form must never echo a masked value back on save:
+// the backend would encrypt the asterisks and overwrite the real secret. A run of
+// 2+ mask chars only appears in masker output (real config — incl. wildcard model
+// names like "openai/*" — carries at most a single "*"), so this reliably detects a
+// redacted value without a provider-metadata lookup. API-key rotation goes through
+// UpdateModelCredentialsModal instead, which sends only the new key.
+const isMaskedSecret = (value: unknown): boolean => typeof value === "string" && /\*{2,}/.test(value);
+
+const stripMaskedSecrets = (params: Record<string, unknown>): Record<string, unknown> =>
+  Object.fromEntries(Object.entries(params).filter(([, value]) => !isMaskedSecret(value)));
+
 export default function ModelInfoView({
   modelId,
   onClose,
@@ -64,10 +78,12 @@ export default function ModelInfoView({
   modelAccessGroups,
 }: ModelInfoViewProps) {
   const [form] = Form.useForm();
+  const queryClient = useQueryClient();
   const [localModelData, setLocalModelData] = useState<any>(null);
   const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false);
   const [deleteLoading, setDeleteLoading] = useState(false);
   const [isCredentialModalOpen, setIsCredentialModalOpen] = useState(false);
+  const [isUpdateCredentialsModalOpen, setIsUpdateCredentialsModalOpen] = useState(false);
   const [isDirty, setIsDirty] = useState(false);
   const [isSaving, setIsSaving] = useState(false);
   const [isEditing, setIsEditing] = useState(false);
@@ -351,9 +367,15 @@ export default function ModelInfoView({
         return;
       }
 
+      // Final guard: never PATCH a redacted secret. The /model/info snapshot that
+      // seeds this form masks secrets, and any save re-sends the whole params blob;
+      // without this strip a masked value would be re-encrypted over the real secret.
+      // Credential rotation has its own dedicated path (UpdateModelCredentialsModal).
+      const safeLitellmParams = stripMaskedSecrets(updatedLitellmParams);
+
       const updateData = {
         model_name: values.model_name,
-        litellm_params: updatedLitellmParams,
+        litellm_params: safeLitellmParams,
         model_info: updatedModelInfo,
       };
 
@@ -363,7 +385,7 @@ export default function ModelInfoView({
         ...localModelData,
         model_name: values.model_name,
         litellm_model_name: values.litellm_model_name,
-        litellm_params: updatedLitellmParams,
+        litellm_params: safeLitellmParams,
         model_info: updatedModelInfo,
       };
 
@@ -511,36 +533,44 @@ export default function ModelInfoView({
           </div>
         </div>
         <div className="flex gap-2">
-          <TremorButton
-            variant="secondary"
-            icon={RefreshIcon}
+          <Button
+            icon={<RefreshIcon className="h-4 w-4" />}
             onClick={handleTestConnection}
             className="flex items-center gap-2"
             data-testid="test-connection-button"
           >
             Test Connection
-          </TremorButton>
+          </Button>
 
-          <TremorButton
-            icon={KeyIcon}
-            variant="secondary"
+          <Button
+            icon={<KeyIcon className="h-4 w-4" />}
+            onClick={() => setIsUpdateCredentialsModalOpen(true)}
+            className="flex items-center"
+            disabled={!canEditModel}
+            data-testid="update-api-key-button"
+          >
+            Update API Key
+          </Button>
+
+          <Button
+            icon={<KeyIcon className="h-4 w-4" />}
             onClick={() => setIsCredentialModalOpen(true)}
             className="flex items-center"
             disabled={!isAdmin}
             data-testid="reuse-credentials-button"
           >
             Re-use Credentials
-          </TremorButton>
-          <TremorButton
-            icon={TrashIcon}
-            variant="secondary"
+          </Button>
+          <Button
+            danger
+            icon={<TrashIcon className="h-4 w-4" />}
             onClick={() => setIsDeleteModalOpen(true)}
-            className="flex items-center text-red-500 border-red-500 hover:text-red-700"
+            className="flex items-center"
             disabled={!canEditModel}
             data-testid="delete-model-button"
           >
             Delete Model
-          </TremorButton>
+          </Button>
         </div>
       </div>
 
@@ -715,7 +745,7 @@ export default function ModelInfoView({
                     litellm_extra_params: JSON.stringify(
                       Object.fromEntries(
                         Object.entries(localModelData.litellm_params || {}).filter(
-                          ([key]) => key !== "litellm_credential_name",
+                          ([key, value]) => key !== "litellm_credential_name" && !isMaskedSecret(value),
                         ),
                       ),
                       null,
@@ -1375,6 +1405,18 @@ export default function ModelInfoView({
         </Modal>
       )}
 
+      {isUpdateCredentialsModalOpen && accessToken && (
+        <UpdateModelCredentialsModal
+          open={isUpdateCredentialsModalOpen}
+          onCancel={() => setIsUpdateCredentialsModalOpen(false)}
+          accessToken={accessToken}
+          modelId={modelId}
+          onUpdated={() => {
+            queryClient.invalidateQueries({ queryKey: ["models", "list"] });
+          }}
+        />
+      )}
+
       {/* Edit Auto Router Modal */}
       <EditAutoRouterModal
         isVisible={isAutoRouterModalOpen}
diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx
index 0bbf4d2a6e8..d982956d7a3 100644
--- a/ui/litellm-dashboard/src/components/networking.tsx
+++ b/ui/litellm-dashboard/src/components/networking.tsx
@@ -2781,8 +2781,8 @@ export const modelPatchUpdateCall = async (
   modelId: string,
 ) => {
   try {
-    console.log("Form Values in modelUpateCall:", formValues); // Log the form values before making the API call
-
+    // Intentionally not logging the payload: it can contain freshly-entered
+    // provider secrets (api_key, vertex_credentials, AWS creds).
     const url = proxyBaseUrl ? `${proxyBaseUrl}/model/${modelId}/update` : `/model/${modelId}/update`;
     const response = await fetch(url, {
       method: "PATCH",
@@ -2802,7 +2802,6 @@ export const modelPatchUpdateCall = async (
       throw new Error("Network response was not ok");
     }
     const data = await response.json();
-    console.log("Update model Response:", data);
     return data;
     // Handle success - you might want to update some state or UI based on the created key
   } catch (error) {
diff --git a/ui/litellm-dashboard/src/components/update_model_credentials_modal.test.tsx b/ui/litellm-dashboard/src/components/update_model_credentials_modal.test.tsx
new file mode 100644
index 00000000000..ab18ae71203
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/update_model_credentials_modal.test.tsx
@@ -0,0 +1,83 @@
+import { render, screen, waitFor } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import UpdateModelCredentialsModal from "./update_model_credentials_modal";
+import * as networking from "./networking";
+
+vi.mock("./networking", async () => {
+  const actual = await vi.importActual("./networking");
+  return {
+    ...actual,
+    modelPatchUpdateCall: vi.fn().mockResolvedValue({}),
+  };
+});
+
+vi.mock("./molecules/notifications_manager", () => ({
+  default: { success: vi.fn(), error: vi.fn(), info: vi.fn(), fromBackend: vi.fn() },
+}));
+
+const mockModelPatchUpdateCall = vi.mocked(networking.modelPatchUpdateCall);
+
+beforeAll(() => {
+  Object.defineProperty(window, "matchMedia", {
+    writable: true,
+    value: (query: string) => ({
+      matches: false,
+      media: query,
+      onchange: null,
+      addListener: () => {},
+      removeListener: () => {},
+      addEventListener: () => {},
+      removeEventListener: () => {},
+      dispatchEvent: () => false,
+    }),
+  });
+});
+
+const renderModal = (overrides: Partial<Parameters<typeof UpdateModelCredentialsModal>[0]> = {}) =>
+  render(
+    <UpdateModelCredentialsModal
+      open
+      onCancel={vi.fn()}
+      accessToken="test-token"
+      modelId="model-123"
+      onUpdated={vi.fn()}
+      {...overrides}
+    />,
+  );
+
+describe("UpdateModelCredentialsModal", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("sends a minimal PATCH with only the new api_key", async () => {
+    const user = userEvent.setup();
+    const onUpdated = vi.fn();
+    const onCancel = vi.fn();
+    renderModal({ onUpdated, onCancel });
+
+    await user.type(screen.getByLabelText(/new api key/i), "sk-rotated-9988");
+    await user.click(screen.getByRole("button", { name: /update api key/i }));
+
+    await waitFor(() => expect(mockModelPatchUpdateCall).toHaveBeenCalledTimes(1));
+    const [token, payload, modelId] = mockModelPatchUpdateCall.mock.calls[0];
+    expect(token).toBe("test-token");
+    expect(modelId).toBe("model-123");
+    // Exactly the new key plus the id — nothing else from the deployment.
+    expect(payload).toEqual({ litellm_params: { api_key: "sk-rotated-9988" }, model_info: { id: "model-123" } });
+    expect(onUpdated).toHaveBeenCalledTimes(1);
+    expect(onCancel).toHaveBeenCalledTimes(1);
+  });
+
+  it("does not call the update API when the field is left blank", async () => {
+    const user = userEvent.setup();
+    renderModal();
+
+    await user.click(screen.getByRole("button", { name: /update api key/i }));
+
+    // Required-field validation blocks submit; give it a tick then assert no call.
+    await new Promise((resolve) => setTimeout(resolve, 50));
+    expect(mockModelPatchUpdateCall).not.toHaveBeenCalled();
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx b/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx
new file mode 100644
index 00000000000..238207a4aa8
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx
@@ -0,0 +1,76 @@
+import { Button, Form, Input, Modal, Typography } from "antd";
+import { useState } from "react";
+import { modelPatchUpdateCall } from "./networking";
+import NotificationsManager from "./molecules/notifications_manager";
+
+const { Text } = Typography;
+
+interface UpdateModelCredentialsModalProps {
+  open: boolean;
+  onCancel: () => void;
+  accessToken: string;
+  modelId: string;
+  onUpdated: () => void;
+}
+
+export default function UpdateModelCredentialsModal({
+  open,
+  onCancel,
+  accessToken,
+  modelId,
+  onUpdated,
+}: UpdateModelCredentialsModalProps) {
+  const [form] = Form.useForm();
+  const [isSaving, setIsSaving] = useState(false);
+
+  const close = () => {
+    form.resetFields();
+    onCancel();
+  };
+
+  const handleSubmit = async (values: { api_key?: string }) => {
+    const apiKey = values.api_key?.trim();
+    if (!apiKey) {
+      NotificationsManager.fromBackend("Enter a new API key");
+      return;
+    }
+    setIsSaving(true);
+    try {
+      await modelPatchUpdateCall(
+        accessToken,
+        { litellm_params: { api_key: apiKey }, model_info: { id: modelId } },
+        modelId,
+      );
+      NotificationsManager.success("API key updated");
+      form.resetFields();
+      onUpdated();
+      onCancel();
+    } catch (error) {
+      console.error("Error updating API key:", error);
+      NotificationsManager.fromBackend("Failed to update API key");
+    } finally {
+      setIsSaving(false);
+    }
+  };
+
+  return (
+    <Modal title="Update API Key" open={open} onCancel={close} footer={null} width={520} destroyOnHidden={true}>
+      <Text className="block mb-4 text-gray-500">
+        Rotate this model&apos;s API key. Only the new key is sent; the rest of the deployment is left untouched.
+      </Text>
+      <Form form={form} onFinish={handleSubmit} layout="vertical">
+        <Form.Item label="New API Key" name="api_key" rules={[{ required: true, message: "Enter a new API key" }]}>
+          <Input.Password placeholder="Enter the new API key" autoComplete="new-password" />
+        </Form.Item>
+        <div className="flex justify-end items-center mt-4">
+          <Button onClick={close} style={{ marginRight: 10 }}>
+            Cancel
+          </Button>
+          <Button type="primary" htmlType="submit" loading={isSaving}>
+            Update API Key
+          </Button>
+        </div>
+      </Form>
+    </Modal>
+  );
+}

From 2860dad5145772070c6607883989454dbb2943d4 Mon Sep 17 00:00:00 2001
From: yucheng-berri <yucheng@berri.ai>
Date: Tue, 30 Jun 2026 18:17:56 -0700
Subject: [PATCH 42/51] feat(proxy): audit default user settings updates
 (#31753)

* feat(proxy): audit default user settings updates

Adds audit logging for the customer-impacting path: PATCH
/update/internal_user_settings, which is what the admin dashboard hits
when an admin changes Default User Settings and which today leaves no
record of who changed what.

Introduces the small framework that future system-wide settings audits
will share: a CONFIG_TABLE_NAME enum value, a create_config_audit_log
helper that reuses the existing create_object_audit_log path (so the
enterprise gate and store_audit_logs flag still apply), and a
_dump_redacted_config helper that strips secret leaves before the row is
written using the same matcher /config/field/info applies for non-admins.
The helper handles environment_variables as a special case where every
value is redacted, since that section carries credentials under
non-secret-looking uppercase keys (e.g. DATABASE_URL).

Only update_internal_user_settings is wired up in this change. Coverage
for the other LiteLLM_Config writers (/config/update sections,
/config/field/update, /config/field/delete, /config/callback/delete,
default_team_settings, mcp_semantic_filter, allowed_ip, sso_settings,
ui_theme, ui_settings) is intentionally a follow-up so each can be
verified live against the credential-bearing fields it actually carries.

The audit-actor parameter on _update_litellm_setting is optional today so
non-audited callers keep working unchanged; the follow-up will make it
required once every caller is wired up.

* fix(proxy): make audit-log call non-blocking and serializer defensive

Greptile review of #31753 surfaced three robustness issues with the
audit-log call path. The settings change always commits; these fixes
prevent post-commit audit failures from surfacing as 500 responses.

Switch the audit-log call in _update_litellm_setting from a blocking
await to asyncio.create_task, matching the create_object_audit_log
pattern every other call site uses (model_management_endpoints etc.).
A transient prisma blip or a JSON serialization error in the audit row
no longer turns a successful save_config into a 500 the caller sees.

Add default=str to both json.dumps calls in _dump_redacted_config so a
YAML-loaded value with a non-JSON-native leaf (datetime, custom object)
serializes cleanly. The sibling audit-log serializers in
team_endpoints.py already pass default=str for the same reason.

Tighten the redact_all_values branch to redact wholesale for non-dict
inputs rather than silently falling through to the key-name matcher;
defensive against a future change that stores a section as a list or
scalar.

Each fix has a regression test mutation-checked against reverting the
fix.

* refactor(proxy): drop unreachable non-dict redact_all_values branch

The defensive non-dict fallback in _dump_redacted_config emitted
json.dumps("REDACTED") which, if ever hit, would crash LiteLLM_AuditLogs
construction (mask_api_keys validator calls json.loads on the already-
parsed bare string). Reachability is zero: redact_all_values is True
only for param_name=="environment_variables", which is always a dict.
Delete the dead branch and its test rather than ship provably-wrong
defensive code with a test that green-lights it.
---
 litellm/proxy/_types.py                       |   1 +
 litellm/proxy/proxy_server.py                 |  46 +++++++-
 .../proxy_setting_endpoints.py                |  28 ++++-
 tests/test_litellm/proxy/test_proxy_server.py | 106 ++++++++++++++++++
 .../test_proxy_setting_endpoints.py           |  91 +++++++++++++++
 5 files changed, 270 insertions(+), 2 deletions(-)

diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 5fe17d79ab5..a6ef7de07ae 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -189,6 +189,7 @@ class LitellmTableNames(str, enum.Enum):
     TOOL_TABLE_NAME = "LiteLLM_ToolTable"
     CACHE_CONFIG_TABLE_NAME = "LiteLLM_CacheConfig"
     CONFIG_OVERRIDES_TABLE_NAME = "LiteLLM_ConfigOverrides"
+    CONFIG_TABLE_NAME = "LiteLLM_Config"
 
 
 class Litellm_EntityType(enum.Enum):
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 57814de6e3f..0158f601d32 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -425,7 +425,10 @@ from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import (
 from litellm.proxy.management_endpoints.workflow_management_endpoints import (
     router as workflow_management_router,
 )
-from litellm.proxy.management_helpers.audit_logs import create_audit_log_for_update
+from litellm.proxy.management_helpers.audit_logs import (
+    create_audit_log_for_update,
+    create_object_audit_log,
+)
 from litellm.proxy.memory.memory_endpoints import router as memory_router
 from litellm.proxy.plugin_routes import (
     router as plugin_router,
@@ -14228,6 +14231,47 @@ def _redact_general_setting_value(field_name: str, value: JsonValue, is_full_adm
     return value
 
 
+def _dump_redacted_config(value: Optional[JsonValue], *, redact_all_values: bool = False) -> Optional[str]:
+    # `default=str` matches the sibling audit-log serializers in
+    # team_endpoints.py and the LiteLLM_AuditLogs validator, so a YAML-loaded
+    # value with a non-JSON-native leaf (datetime, custom object) cannot turn
+    # an audit write into a 500.
+    if value is None:
+        return None
+    if redact_all_values and isinstance(value, dict):
+        return json.dumps({key: "REDACTED" for key in value}, default=str)
+    return json.dumps(_redact_secret_values_in_obj(value), default=str)
+
+
+async def create_config_audit_log(
+    param_name: str,
+    action: AUDIT_ACTIONS,
+    before_value: Optional[JsonValue],
+    after_value: Optional[JsonValue],
+    user_api_key_dict: UserAPIKeyAuth,
+    table_name: LitellmTableNames = LitellmTableNames.CONFIG_TABLE_NAME,
+) -> None:
+    """Record a system-wide settings change in LiteLLM_AuditLog.
+
+    Secret leaves are redacted before the row is written. environment_variables
+    hold arbitrary credentials under non-secret-looking uppercase keys (e.g.
+    DATABASE_URL), so every value in that section is redacted rather than
+    relying on key-name matching; other sections reuse the same matcher
+    /config/field/info applies for non-admins.
+    """
+    redact_all_values = param_name == "environment_variables"
+    await create_object_audit_log(
+        object_id=param_name,
+        action=action,
+        table_name=table_name,
+        before_value=_dump_redacted_config(before_value, redact_all_values=redact_all_values),
+        after_value=_dump_redacted_config(after_value, redact_all_values=redact_all_values),
+        user_api_key_dict=user_api_key_dict,
+        litellm_changed_by=None,
+        litellm_proxy_admin_name=LITELLM_PROXY_ADMIN_NAME,
+    )
+
+
 @router.get(
     "/config/field/info",
     tags=["config.yaml"],
diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
index e4f68a1e9db..1be17c86123 100644
--- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
+++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
@@ -1,4 +1,5 @@
 #### CRUD ENDPOINTS for UI Settings #####
+import asyncio
 import json
 from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
 from urllib.parse import urlparse
@@ -553,6 +554,7 @@ async def _update_litellm_setting(
     settings: Union[DefaultInternalUserParams, DefaultTeamSSOParams, MCPSemanticFilterSettings],
     settings_key: str,
     success_message: str,
+    user_api_key_dict: Optional[UserAPIKeyAuth] = None,
 ):
     """
     Common utility function to update `litellm_settings` in both memory and config.
@@ -561,8 +563,15 @@ async def _update_litellm_setting(
         settings: The settings object to update
         settings_key: The key in litellm_settings to update
         success_message: Message to return on success
+        user_api_key_dict: The acting admin, recorded as the audit-log actor.
+            Optional today so callers that have not been wired for auditing
+            keep working; the audit row is only written when an actor is passed.
     """
-    from litellm.proxy.proxy_server import proxy_config, store_model_in_db
+    from litellm.proxy.proxy_server import (
+        create_config_audit_log,
+        proxy_config,
+        store_model_in_db,
+    )
 
     if store_model_in_db is not True:
         raise HTTPException(
@@ -576,6 +585,7 @@ async def _update_litellm_setting(
     # because get_config() may overwrite litellm.<key> with stale DB values
     # via LITELLM_SETTINGS_SAFE_DB_OVERRIDES.
     config = await proxy_config.get_config()
+    before_value = config.get("litellm_settings", {}).get(settings_key)
 
     # Update the in-memory settings (after get_config to avoid stale override)
     setattr(litellm, settings_key, in_memory_var)
@@ -589,6 +599,21 @@ async def _update_litellm_setting(
     # Save the updated config
     await proxy_config.save_config(new_config=config)
 
+    if user_api_key_dict is not None:
+        # Fire-and-forget so an audit-log failure (transient DB blip, etc.)
+        # never surfaces as a 500 after save_config has already committed,
+        # matching the create_object_audit_log pattern used elsewhere
+        # (e.g. model_management_endpoints).
+        asyncio.create_task(
+            create_config_audit_log(
+                param_name=settings_key,
+                action="updated",
+                before_value=before_value,
+                after_value=in_memory_var,
+                user_api_key_dict=user_api_key_dict,
+            )
+        )
+
     return {
         "message": success_message,
         "status": "success",
@@ -619,6 +644,7 @@ async def update_internal_user_settings(
         settings=settings,
         settings_key="default_internal_user_params",
         success_message="Internal user settings updated successfully",
+        user_api_key_dict=user_api_key_dict,
     )
 
 
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index 019a7dc90d2..dc35d71ccbd 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -8658,3 +8658,109 @@ def test_config_field_info_returns_raw_secrets_for_full_admin(monkeypatch):
         )
     finally:
         app.dependency_overrides.clear()
+
+
+def _fake_prisma_with_config(existing_param_value):
+    """MagicMock prisma whose litellm_config row returns existing_param_value and
+    whose litellm_auditlog.create records the written audit row."""
+    fake = MagicMock()
+    config_row = MagicMock()
+    config_row.param_value = existing_param_value
+    fake.db.litellm_config.find_first = AsyncMock(return_value=config_row)
+    fake.db.litellm_config.upsert = AsyncMock(return_value=config_row)
+    fake.db.litellm_auditlog.create = AsyncMock()
+    return fake
+
+
+def test_dump_redacted_config_redacts_secret_leaves():
+    from litellm.proxy.proxy_server import _dump_redacted_config
+
+    assert _dump_redacted_config(None) is None
+
+    restored = json.loads(
+        _dump_redacted_config(
+            {
+                "api_key": "sk-leak",
+                "model": "gpt-4",
+                "nested": {"aws_secret_access_key": "abc", "region": "us-east-1"},
+            }
+        )
+    )
+    assert restored["api_key"] == "REDACTED"
+    assert restored["model"] == "gpt-4"
+    assert restored["nested"]["aws_secret_access_key"] == "REDACTED"
+    assert restored["nested"]["region"] == "us-east-1"
+
+
+@pytest.mark.asyncio
+async def test_create_config_audit_log_writes_redacted_entry(monkeypatch):
+    import litellm.proxy.proxy_server as proxy_server_module
+    from litellm.proxy._types import LitellmTableNames
+    from litellm.proxy.proxy_server import create_config_audit_log
+
+    fake = _fake_prisma_with_config({})
+    monkeypatch.setattr(proxy_server_module, "prisma_client", fake)
+    monkeypatch.setattr(proxy_server_module, "premium_user", True)
+    monkeypatch.setattr(litellm, "store_audit_logs", True)
+
+    caller = UserAPIKeyAuth(api_key="hashed-key-abc", user_id="admin-7")
+    await create_config_audit_log(
+        "router_settings",
+        "updated",
+        {"routing_strategy": "simple-shuffle", "api_key": "sk-old"},
+        {"routing_strategy": "latency-based", "api_key": "sk-new"},
+        caller,
+    )
+
+    fake.db.litellm_auditlog.create.assert_awaited_once()
+    written = fake.db.litellm_auditlog.create.call_args.kwargs["data"]
+    assert written["table_name"] == LitellmTableNames.CONFIG_TABLE_NAME.value
+    assert written["object_id"] == "router_settings"
+    assert written["action"] == "updated"
+    assert written["changed_by"] == "admin-7"
+    assert written["changed_by_api_key"] == "hashed-key-abc"
+
+    before = json.loads(written["before_value"])
+    after = json.loads(written["updated_values"])
+    assert before["routing_strategy"] == "simple-shuffle"
+    assert after["routing_strategy"] == "latency-based"
+    assert "sk-old" not in written["before_value"]
+    assert "sk-new" not in written["updated_values"]
+    assert before["api_key"] != "sk-old"
+    assert after["api_key"] != "sk-new"
+
+
+@pytest.mark.asyncio
+async def test_create_config_audit_log_noop_when_store_audit_logs_disabled(monkeypatch):
+    import litellm.proxy.proxy_server as proxy_server_module
+    from litellm.proxy.proxy_server import create_config_audit_log
+
+    fake = _fake_prisma_with_config({})
+    monkeypatch.setattr(proxy_server_module, "prisma_client", fake)
+    monkeypatch.setattr(proxy_server_module, "premium_user", True)
+    monkeypatch.setattr(litellm, "store_audit_logs", False)
+
+    await create_config_audit_log(
+        "router_settings",
+        "updated",
+        {},
+        {"a": 1},
+        UserAPIKeyAuth(api_key="k", user_id="u"),
+    )
+    fake.db.litellm_auditlog.create.assert_not_called()
+
+
+def test_dump_redacted_config_serializes_non_json_native_values():
+    """YAML-loaded config can contain datetime/date/custom values that plain
+    json.dumps refuses. Without default=str the audit write turns into a 500
+    after the config change has already committed; the sibling audit-log
+    serializers in team_endpoints.py use default=str for the same reason."""
+    from datetime import datetime, timezone
+
+    from litellm.proxy.proxy_server import _dump_redacted_config
+
+    out = _dump_redacted_config({"updated_at": datetime(2026, 6, 30, tzinfo=timezone.utc)})
+    assert out is not None
+    restored = json.loads(out)
+    assert "2026-06-30" in restored["updated_at"]
+
diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
index ae217aca16e..7a586f758f4 100644
--- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
+++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
@@ -1869,3 +1869,94 @@ class TestProxySettingEndpoints:
         assert "field_schema" in data
         assert "properties" in data["field_schema"]
         assert "role_mappings" in data["field_schema"]["properties"]
+
+
+def test_update_internal_user_settings_writes_audit_log(mock_proxy_config, monkeypatch):
+    """Regression for the reported scenario: an admin changes Default User
+    Settings from the dashboard, which issues PATCH /update/internal_user_settings
+    (NOT /config/update). An audit row must record who changed it and what
+    changed."""
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm
+    import litellm.proxy.proxy_server as proxy_server_module
+    from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+
+    audit_create = AsyncMock()
+    fake_prisma = MagicMock()
+    fake_prisma.db.litellm_auditlog.create = audit_create
+
+    monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma)
+    monkeypatch.setattr(proxy_server_module, "premium_user", True)
+    monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True)
+    monkeypatch.setattr(litellm, "store_audit_logs", True)
+    monkeypatch.setattr(litellm, "default_internal_user_params", {})
+
+    async def _admin_auth():
+        return UserAPIKeyAuth(
+            user_id="audit-admin",
+            api_key="hashed-admin-key",
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+        )
+
+    app.dependency_overrides[user_api_key_auth] = _admin_auth
+    try:
+        resp = client.patch(
+            "/update/internal_user_settings",
+            json={"max_budget": 999.0, "models": ["gpt-4"]},
+        )
+        assert resp.status_code == 200, resp.text
+
+        audit_create.assert_awaited_once()
+        written = audit_create.await_args.kwargs["data"]
+        assert written["object_id"] == "default_internal_user_params"
+        assert written["action"] == "updated"
+        assert written["table_name"] == "LiteLLM_Config"
+        assert written["changed_by"] == "audit-admin"
+        assert written["changed_by_api_key"] == "hashed-admin-key"
+
+        before = json.loads(written["before_value"])
+        after = json.loads(written["updated_values"])
+        assert before["max_budget"] == 100.0
+        assert after["max_budget"] == 999.0
+    finally:
+        app.dependency_overrides.pop(user_api_key_auth, None)
+
+
+def test_update_internal_user_settings_returns_200_when_audit_write_raises(
+    mock_proxy_config, monkeypatch
+):
+    """The settings change is already committed by save_config, so an
+    audit-log failure must never surface as a 500. Scheduling via
+    asyncio.create_task keeps the audit call off the request path; this
+    test asserts that contract by making the audit helper raise."""
+    import litellm
+    import litellm.proxy.proxy_server as proxy_server_module
+    from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+
+    monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True)
+    monkeypatch.setattr(litellm, "default_internal_user_params", {})
+
+    async def _raise(**_kwargs):
+        raise RuntimeError("audit prisma blip")
+
+    monkeypatch.setattr(proxy_server_module, "create_config_audit_log", _raise)
+
+    async def _admin_auth():
+        return UserAPIKeyAuth(
+            user_id="audit-admin",
+            api_key="hashed-admin-key",
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+        )
+
+    app.dependency_overrides[user_api_key_auth] = _admin_auth
+    try:
+        resp = client.patch(
+            "/update/internal_user_settings", json={"max_budget": 42.0}
+        )
+        assert resp.status_code == 200, resp.text
+        assert resp.json()["status"] == "success"
+    finally:
+        app.dependency_overrides.pop(user_api_key_auth, None)

From ada9ef88ac1a18d7c3073bad951cea9be4a3f981 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrish+github@berri.ai>
Date: Tue, 30 Jun 2026 18:36:40 -0700
Subject: [PATCH 43/51] fix(websearch): websearch_interception agentic loop
 fixes for chat completions and anthropic messages (#31669)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix(websearch): wire chat completion agentic loop to correct hooks

maybe_run_chat_completion_agentic_loop was calling async_should_run_agentic_loop (Anthropic format) and async_run_agentic_loop (Anthropic path) instead of the chat-completion variants. This meant WebSearchInterceptionLogger never intercepted chat completion requests — the LLM returned a litellm_web_search tool_call but the agentic loop never executed, so the raw tool_calls response was returned to the caller.

Fix: gate on async_should_run_chat_completion_agentic_loop override, call that hook and async_build_chat_completion_agentic_loop_plan / async_run_chat_completion_agentic_loop in the execution path.

Regression test added.

* fix(websearch): strip tool_choice from follow-up request

When the original request forces tool_choice to litellm_web_search,
the follow-up request after search execution inherited that tool_choice,
causing the model to call the search tool again instead of synthesizing
an answer from the results.

* fix(websearch): inject api_key into agentic hook kwargs for anthropic messages

Follow-up calls inside async_run_agentic_loop (e.g. websearch interception's
synthesis call after executing Exa/Perplexity searches) were missing api_key
because the named api_key param in async_anthropic_messages_handler was never
merged into the kwargs dict forwarded downstream. Result: every /v1/messages
websearch follow-up failed with "x-api-key header is required" and the caller
received the raw tool_use response instead of the synthesized answer.

* ci: trigger CI run

* fix(websearch): support unified agentic hooks alongside chat-completion-specific hooks

CodeInterpreterInterceptionLogger uses async_should_run_agentic_loop with
_agentic_loop_api_surface to handle both surfaces from one hook. The chat
completion loop must also check _gate_overridden so callbacks using the
unified hook pattern still fire for chat completions.

* fix(websearch): strip tool_choice from legacy chat completion follow-up call

The _execute_chat_completion_agentic_loop path merged original optional_params
(which includes forced tool_choice) into follow-up params without explicit
removal. _build_chat_completion_request_patch already excluded tool_choice from
its optional_params output, but dict.update() with a missing key leaves the
original value intact. Explicit pop after the merge removes it.

* fix(websearch): always strip tool_choice from plan-path follow-up params

The tool_choice removal was gated on patch.tools is not None. WebSearch sets
tools via patch.optional_params not patch.tools, so the gate was False and
forced tool_choice from the original request survived into the synthesis call.
Move the pop outside the patch.tools branch so it applies unconditionally.
---
 .../websearch_interception/handler.py         |  31 +-
 .../chat_completion_agentic_loop.py           |  18 +-
 litellm/llms/custom_httpx/llm_http_handler.py |   8 +-
 .../test_websearch_chat_completion.py         | 281 ++++++++++++++----
 .../test_chat_completion_agentic_loop.py      |  18 +-
 .../custom_httpx/test_llm_http_handler.py     |  67 +++++
 6 files changed, 325 insertions(+), 98 deletions(-)

diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py
index 2e11405af3f..bfae6d5b7b0 100644
--- a/litellm/integrations/websearch_interception/handler.py
+++ b/litellm/integrations/websearch_interception/handler.py
@@ -31,6 +31,7 @@ from litellm.types.integrations.websearch_interception import (
     WebSearchInterceptionConfig,
 )
 from litellm.types.integrations.custom_logger import (
+    CHAT_COMPLETION_AGENTIC_SURFACE,
     AgenticLoopPlan,
     AgenticLoopRequestPatch,
 )
@@ -440,12 +441,16 @@ class WebSearchInterceptionLogger(CustomLogger):
         custom_llm_provider: str,
         kwargs: Dict,
     ) -> Tuple[bool, Dict]:
-        """
-        Check if WebSearch tool interception is needed for Anthropic Messages API.
-
-        This is the legacy method for Anthropic-style responses.
-        For chat completions, use async_should_run_chat_completion_agentic_loop instead.
-        """
+        if kwargs.get("_agentic_loop_api_surface") == CHAT_COMPLETION_AGENTIC_SURFACE:
+            return await self.async_should_run_chat_completion_agentic_loop(
+                response=response,
+                model=model,
+                messages=messages,
+                tools=tools,
+                stream=stream,
+                custom_llm_provider=custom_llm_provider,
+                kwargs=kwargs,
+            )
 
         verbose_logger.debug(f"WebSearchInterception: Hook called! provider={custom_llm_provider}, stream={stream}")
         verbose_logger.debug(f"WebSearchInterception: Response type: {type(response)}")
@@ -629,6 +634,18 @@ class WebSearchInterceptionLogger(CustomLogger):
         stream: bool,
         kwargs: Dict,
     ) -> AgenticLoopPlan:
+        if kwargs.get("_agentic_loop_api_surface") == CHAT_COMPLETION_AGENTIC_SURFACE:
+            return await self.async_build_chat_completion_agentic_loop_plan(
+                tools=tools,
+                model=model,
+                messages=messages,
+                response=response,
+                optional_params=anthropic_messages_optional_request_params,
+                logging_obj=logging_obj,
+                stream=stream,
+                kwargs=kwargs,
+            )
+
         tool_calls = tools["tool_calls"]
         thinking_blocks = tools.get("thinking_blocks", [])
         request_patch, structured_results = await self._build_anthropic_request_patch(
@@ -1088,6 +1105,7 @@ class WebSearchInterceptionLogger(CustomLogger):
             raise ValueError("WebSearchInterception: missing follow-up messages")
         params = dict(optional_params)
         params.update(request_patch.optional_params)
+        params.pop("tool_choice", None)
         return await litellm.acompletion(
             model=request_patch.model or model,
             messages=request_patch.messages,
@@ -1203,6 +1221,7 @@ class WebSearchInterceptionLogger(CustomLogger):
             if k
             not in {
                 "tools",
+                "tool_choice",
                 "extra_body",
                 "model_alias_map",
                 "stream_response",
diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py
index 828605d5ef8..b7262a42324 100644
--- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py
+++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py
@@ -137,8 +137,8 @@ async def _execute_chat_completion_agentic_plan(
     optional_params_for_followup = {**optional_params, **patch.optional_params}
     if patch.tools is not None:
         optional_params_for_followup["tools"] = patch.tools
-        if "tool_choice" not in patch.optional_params:
-            optional_params_for_followup.pop("tool_choice", None)
+    if "tool_choice" not in patch.optional_params:
+        optional_params_for_followup.pop("tool_choice", None)
 
     kwargs_for_followup = _filter_followup_kwargs(kwargs)
     kwargs_for_followup.update(
@@ -206,10 +206,11 @@ async def maybe_run_chat_completion_agentic_loop(
     for callback in callbacks:
         if not isinstance(callback, CustomLogger):
             continue
+
         if not _gate_overridden(callback):
             continue
 
-        gate_kwargs = {
+        hook_kwargs = {
             **kwargs,
             "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE,
             "custom_llm_provider": custom_llm_provider,
@@ -222,7 +223,7 @@ async def maybe_run_chat_completion_agentic_loop(
                 tools=tools,
                 stream=stream,
                 custom_llm_provider=custom_llm_provider,
-                kwargs=gate_kwargs,
+                kwargs=hook_kwargs,
             )
         except Exception as e:
             verbose_logger.exception(
@@ -243,11 +244,6 @@ async def maybe_run_chat_completion_agentic_loop(
         )
 
         try:
-            plan_kwargs = {
-                **kwargs,
-                "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE,
-                "custom_llm_provider": custom_llm_provider,
-            }
             if not _build_plan_overridden(callback):
                 return await callback.async_run_agentic_loop(
                     tools=tool_calls,
@@ -258,7 +254,7 @@ async def maybe_run_chat_completion_agentic_loop(
                     anthropic_messages_optional_request_params=optional_params,
                     logging_obj=logging_obj,
                     stream=stream,
-                    kwargs=plan_kwargs,
+                    kwargs=hook_kwargs,
                 )
 
             plan = await callback.async_build_agentic_loop_plan(
@@ -270,7 +266,7 @@ async def maybe_run_chat_completion_agentic_loop(
                 anthropic_messages_optional_request_params=optional_params,
                 logging_obj=logging_obj,
                 stream=stream,
-                kwargs=plan_kwargs,
+                kwargs=hook_kwargs,
             )
 
             if plan.response_override is not None:
diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index 9bb956d0808..3c10239f868 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -2112,7 +2112,7 @@ class BaseLLMHTTPHandler:
                 anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
                 logging_obj=logging_obj,
                 custom_llm_provider=custom_llm_provider,
-                kwargs=kwargs,
+                kwargs={**kwargs, "api_key": api_key} if api_key else kwargs,
             )
             return initial_response
         else:
@@ -2122,6 +2122,10 @@ class BaseLLMHTTPHandler:
                 logging_obj=logging_obj,
             )
 
+        # Inject api_key into kwargs so follow-up calls in agentic hooks can
+        # authenticate. api_key is a named param here (not in kwargs), so
+        # _prepare_followup_kwargs would miss it otherwise.
+        kwargs_for_agentic = {**kwargs, "api_key": api_key} if api_key else kwargs
         # Call agentic completion hooks (non-streaming path only)
         final_response = await self._call_agentic_completion_hooks(
             response=initial_response,
@@ -2132,7 +2136,7 @@ class BaseLLMHTTPHandler:
             logging_obj=logging_obj,
             stream=False,
             custom_llm_provider=custom_llm_provider,
-            kwargs=kwargs,
+            kwargs=kwargs_for_agentic,
         )
 
         return self._maybe_wrap_in_fake_stream(
diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_chat_completion.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_chat_completion.py
index 34555d76554..7ef43e2eadf 100644
--- a/tests/test_litellm/integrations/websearch_interception/test_websearch_chat_completion.py
+++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_chat_completion.py
@@ -6,7 +6,7 @@ litellm.acompletion() for transparent server-side web search execution.
 """
 
 import os
-from unittest.mock import AsyncMock, MagicMock, patch
+from unittest.mock import MagicMock
 
 import pytest
 
@@ -34,9 +34,7 @@ def mock_search_response():
 @pytest.fixture
 def websearch_logger():
     """Create a WebSearchInterceptionLogger instance"""
-    return WebSearchInterceptionLogger(
-        enabled_providers=[LlmProviders.OPENAI, LlmProviders.MINIMAX]
-    )
+    return WebSearchInterceptionLogger(enabled_providers=[LlmProviders.OPENAI, LlmProviders.MINIMAX])
 
 
 @pytest.mark.asyncio
@@ -55,9 +53,7 @@ async def test_websearch_chat_completion_with_openai():
     """
     # Configure WebSearch interception
     original_callbacks = litellm.callbacks.copy() if litellm.callbacks else []
-    websearch_logger = WebSearchInterceptionLogger(
-        enabled_providers=[LlmProviders.OPENAI]
-    )
+    websearch_logger = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.OPENAI])
     litellm.callbacks = [websearch_logger]
 
     try:
@@ -100,9 +96,7 @@ async def test_websearch_chat_completion_with_openai():
         if hasattr(response.choices[0].message, "tool_calls"):
             # If tool_calls exist, it means agentic loop didn't run
             # This could happen if search tool is not configured
-            pytest.skip(
-                "Agentic loop did not execute - search tool may not be configured"
-            )
+            pytest.skip("Agentic loop did not execute - search tool may not be configured")
 
         # Verify we got a meaningful response
         assert response.choices[0].finish_reason in ["stop", "end_turn"]
@@ -122,9 +116,7 @@ async def test_websearch_chat_completion_hook_detection():
         Message,
     )
 
-    websearch_logger = WebSearchInterceptionLogger(
-        enabled_providers=[LlmProviders.OPENAI]
-    )
+    websearch_logger = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.OPENAI])
 
     # Mock response with litellm_web_search tool call
     mock_response = ModelResponse(
@@ -155,21 +147,19 @@ async def test_websearch_chat_completion_hook_detection():
     )
 
     # Test should_run_chat_completion_agentic_loop
-    should_run, tools_dict = (
-        await websearch_logger.async_should_run_chat_completion_agentic_loop(
-            response=mock_response,
-            model="gpt-4o",
-            messages=[{"role": "user", "content": "What's the weather?"}],
-            tools=[
-                {
-                    "type": "function",
-                    "function": {"name": "litellm_web_search"},
-                }
-            ],
-            stream=False,
-            custom_llm_provider="openai",
-            kwargs={},
-        )
+    should_run, tools_dict = await websearch_logger.async_should_run_chat_completion_agentic_loop(
+        response=mock_response,
+        model="gpt-4o",
+        messages=[{"role": "user", "content": "What's the weather?"}],
+        tools=[
+            {
+                "type": "function",
+                "function": {"name": "litellm_web_search"},
+            }
+        ],
+        stream=False,
+        custom_llm_provider="openai",
+        kwargs={},
     )
 
     # Verify hook detected the tool call
@@ -185,9 +175,7 @@ async def test_websearch_not_triggered_without_tool():
     """Test that websearch hook is NOT triggered when no web search tool in request."""
     from litellm.types.utils import Choices, Message
 
-    websearch_logger = WebSearchInterceptionLogger(
-        enabled_providers=[LlmProviders.OPENAI]
-    )
+    websearch_logger = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.OPENAI])
 
     mock_response = ModelResponse(
         id="test-123",
@@ -208,21 +196,19 @@ async def test_websearch_not_triggered_without_tool():
     )
 
     # Test without web search tool
-    should_run, tools_dict = (
-        await websearch_logger.async_should_run_chat_completion_agentic_loop(
-            response=mock_response,
-            model="gpt-4o",
-            messages=[{"role": "user", "content": "Hello"}],
-            tools=[
-                {
-                    "type": "function",
-                    "function": {"name": "some_other_tool"},
-                }
-            ],
-            stream=False,
-            custom_llm_provider="openai",
-            kwargs={},
-        )
+    should_run, tools_dict = await websearch_logger.async_should_run_chat_completion_agentic_loop(
+        response=mock_response,
+        model="gpt-4o",
+        messages=[{"role": "user", "content": "Hello"}],
+        tools=[
+            {
+                "type": "function",
+                "function": {"name": "some_other_tool"},
+            }
+        ],
+        stream=False,
+        custom_llm_provider="openai",
+        kwargs={},
     )
 
     # Verify hook did NOT trigger
@@ -241,9 +227,7 @@ async def test_websearch_not_triggered_for_disabled_provider():
     )
 
     # Only enable bedrock
-    websearch_logger = WebSearchInterceptionLogger(
-        enabled_providers=[LlmProviders.BEDROCK]
-    )
+    websearch_logger = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.BEDROCK])
 
     mock_response = ModelResponse(
         id="test-123",
@@ -273,21 +257,19 @@ async def test_websearch_not_triggered_for_disabled_provider():
     )
 
     # Test with OpenAI provider (not enabled)
-    should_run, tools_dict = (
-        await websearch_logger.async_should_run_chat_completion_agentic_loop(
-            response=mock_response,
-            model="gpt-4o",
-            messages=[{"role": "user", "content": "test"}],
-            tools=[
-                {
-                    "type": "function",
-                    "function": {"name": "litellm_web_search"},
-                }
-            ],
-            stream=False,
-            custom_llm_provider="openai",  # Not in enabled_providers
-            kwargs={},
-        )
+    should_run, tools_dict = await websearch_logger.async_should_run_chat_completion_agentic_loop(
+        response=mock_response,
+        model="gpt-4o",
+        messages=[{"role": "user", "content": "test"}],
+        tools=[
+            {
+                "type": "function",
+                "function": {"name": "litellm_web_search"},
+            }
+        ],
+        stream=False,
+        custom_llm_provider="openai",  # Not in enabled_providers
+        kwargs={},
     )
 
     # Verify hook did NOT trigger
@@ -341,8 +323,7 @@ async def test_websearch_json_serialization_fix():
 
 @pytest.mark.asyncio
 @pytest.mark.skipif(
-    os.environ.get("OPENAI_API_KEY") is None
-    or os.environ.get("PERPLEXITY_API_KEY") is None,
+    os.environ.get("OPENAI_API_KEY") is None or os.environ.get("PERPLEXITY_API_KEY") is None,
     reason="OPENAI_API_KEY or PERPLEXITY_API_KEY not set",
 )
 async def test_websearch_streaming_conversion():
@@ -395,6 +376,174 @@ async def test_websearch_streaming_conversion():
         litellm.callbacks = []
 
 
+@pytest.mark.asyncio
+async def test_maybe_run_chat_completion_agentic_loop_calls_chat_completion_hook():
+    """Regression test: maybe_run_chat_completion_agentic_loop must call
+    async_should_run_chat_completion_agentic_loop, not async_should_run_agentic_loop.
+
+    Before the fix, the function used the wrong gate check and wrong hook,
+    causing WebSearchInterceptionLogger to never intercept chat completion requests
+    even when the LLM returned a litellm_web_search tool call.
+    """
+    from litellm.litellm_core_utils.chat_completion_agentic_loop import (
+        maybe_run_chat_completion_agentic_loop,
+    )
+    from litellm.types.utils import (
+        ChatCompletionMessageToolCall,
+        Choices,
+        Function,
+        Message,
+    )
+
+    mock_response = ModelResponse(
+        id="test-regression-123",
+        choices=[
+            Choices(
+                finish_reason="tool_calls",
+                index=0,
+                message=Message(
+                    role="assistant",
+                    content=None,
+                    tool_calls=[
+                        ChatCompletionMessageToolCall(
+                            id="call_abc",
+                            type="function",
+                            function=Function(
+                                name="litellm_web_search",
+                                arguments='{"query": "latest news"}',
+                            ),
+                        )
+                    ],
+                ),
+            )
+        ],
+        model="gpt-4o",
+        object="chat.completion",
+        created=1234567890,
+    )
+
+    sentinel = ModelResponse(
+        id="sentinel-final",
+        choices=[
+            Choices(
+                finish_reason="stop",
+                index=0,
+                message=Message(role="assistant", content="Here is the news."),
+            )
+        ],
+        model="gpt-4o",
+        object="chat.completion",
+        created=1234567890,
+    )
+
+    websearch_logger = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.OPENAI])
+
+    chat_completion_hook_called = False
+
+    async def fake_should_run_chat_completion(response, model, messages, tools, stream, custom_llm_provider, kwargs):
+        nonlocal chat_completion_hook_called
+        chat_completion_hook_called = True
+        return True, {
+            "tool_calls": [{"id": "call_abc", "name": "litellm_web_search", "input": {"query": "latest news"}}],
+            "tool_type": "websearch",
+            "provider": "openai",
+            "response_format": "openai",
+        }
+
+    async def fake_build_plan(tools, model, messages, response, optional_params, logging_obj, stream, kwargs):
+        from litellm.types.integrations.custom_logger import AgenticLoopPlan
+
+        return AgenticLoopPlan(run_agentic_loop=False, response_override=sentinel)
+
+    websearch_logger.async_should_run_chat_completion_agentic_loop = fake_should_run_chat_completion
+    websearch_logger.async_build_chat_completion_agentic_loop_plan = fake_build_plan
+
+    import litellm as _litellm
+
+    original_callbacks = _litellm.callbacks[:]
+    _litellm.callbacks = [websearch_logger]
+
+    mock_logging_obj = MagicMock()
+    mock_logging_obj.dynamic_success_callbacks = None
+
+    try:
+        result = await maybe_run_chat_completion_agentic_loop(
+            response=mock_response,
+            model="gpt-4o",
+            messages=[{"role": "user", "content": "Latest news?"}],
+            optional_params={
+                "tools": [
+                    {
+                        "type": "function",
+                        "function": {"name": "litellm_web_search"},
+                    }
+                ]
+            },
+            kwargs={},
+            logging_obj=mock_logging_obj,
+            custom_llm_provider="openai",
+            stream=False,
+        )
+    finally:
+        _litellm.callbacks = original_callbacks
+
+    assert chat_completion_hook_called, (
+        "async_should_run_chat_completion_agentic_loop was never called; "
+        "maybe_run_chat_completion_agentic_loop used the wrong hook"
+    )
+    assert result is sentinel, "Expected agentic loop to return sentinel final response"
+
+
+@pytest.mark.asyncio
+async def test_execute_chat_completion_agentic_loop_strips_tool_choice():
+    """Regression: _execute_chat_completion_agentic_loop must not forward tool_choice
+    from the original request into the follow-up synthesis call.
+
+    When the original request forces tool_choice to litellm_web_search, merging
+    optional_params into the follow-up params without explicit removal causes the
+    model to call the search tool again instead of synthesizing an answer.
+    """
+    from unittest.mock import patch
+
+    websearch_logger = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.OPENAI])
+
+    captured_kwargs: dict = {}
+
+    async def fake_acompletion(**kwargs):
+        captured_kwargs.update(kwargs)
+        return ModelResponse(id="followup", model="gpt-4o", object="chat.completion")
+
+    async def fake_search(query):
+        return ("Bitcoin price is $60,000", None)
+
+    with patch.object(websearch_logger, "_execute_search", side_effect=fake_search):
+        with patch("litellm.acompletion", side_effect=fake_acompletion):
+            await websearch_logger._execute_chat_completion_agentic_loop(
+                model="gpt-4o",
+                messages=[{"role": "user", "content": "What is Bitcoin price?"}],
+                tool_calls=[
+                    {
+                        "id": "call_1",
+                        "name": "litellm_web_search",
+                        "input": {"query": "bitcoin price"},
+                    }
+                ],
+                optional_params={
+                    "tools": [{"type": "function", "function": {"name": "litellm_web_search"}}],
+                    "tool_choice": {"type": "function", "function": {"name": "litellm_web_search"}},
+                    "max_tokens": 512,
+                },
+                logging_obj=MagicMock(),
+                stream=False,
+                kwargs={},
+            )
+
+    assert "tool_choice" not in captured_kwargs, (
+        "tool_choice must not appear in follow-up acompletion kwargs; "
+        "it would force the model to call the search tool again instead of synthesizing"
+    )
+
+
 if __name__ == "__main__":
     # Run with: pytest test_websearch_chat_completion.py -v -s
     pytest.main([__file__, "-v", "-s"])
diff --git a/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py
index f1196ab4692..cc16ad558e4 100644
--- a/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py
+++ b/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py
@@ -181,8 +181,7 @@ async def test_internal_control_fields_never_leak_into_provider_body(restore_cal
 
     # The loop must have actually fired (sanity: two provider calls).
     assert create.await_count == 2, (
-        "expected the agentic loop to issue a follow-up provider call; "
-        f"got {create.await_count} call(s)"
+        f"expected the agentic loop to issue a follow-up provider call; got {create.await_count} call(s)"
     )
 
     for idx, call in enumerate(create.await_args_list):
@@ -194,8 +193,7 @@ async def test_internal_control_fields_never_leak_into_provider_body(restore_cal
                 f"top-level request body: {sorted(body.keys())}"
             )
             assert field not in extra_body, (
-                f"provider call #{idx}: internal field {field!r} leaked into "
-                f"extra_body: {sorted(extra_body.keys())}"
+                f"provider call #{idx}: internal field {field!r} leaked into extra_body: {sorted(extra_body.keys())}"
             )
         # The native code_interpreter tool must have been swapped for the
         # function tool, never sent raw to OpenAI as a chat-completions request.
@@ -254,9 +252,7 @@ class _GateOnlyLogger(CustomLogger):
     ) -> AgenticLoopPlan:
         return self._plan
 
-    async def async_agentic_loop_cleanup_hook(
-        self, plan: AgenticLoopPlan, kwargs: Dict[str, Any]
-    ) -> None:
+    async def async_agentic_loop_cleanup_hook(self, plan: AgenticLoopPlan, kwargs: Dict[str, Any]) -> None:
         self.cleanup_calls += 1
 
 
@@ -343,9 +339,7 @@ async def test_dispatcher_runs_followup_with_incremented_depth_and_patched_messa
     assert call_kwargs["max_agentic_loops"] >= 1
     assert "_agentic_loop_fingerprints" in call_kwargs
     # Interception markers are mirrored into litellm_metadata for the follow-up.
-    assert (
-        call_kwargs["litellm_metadata"]["_code_interpreter_interception_active"] is True
-    )
+    assert call_kwargs["litellm_metadata"]["_code_interpreter_interception_active"] is True
     # The transient surface marker is NOT forwarded to the follow-up call.
     assert "_agentic_loop_api_surface" not in call_kwargs
     # Cleanup hook always runs.
@@ -390,9 +384,7 @@ async def test_dispatcher_raises_on_repeated_tool_call_fingerprint(restore_callb
 
     # The dispatcher fingerprints the whole value the gate returns as its second
     # tuple element, so the seeded fingerprint must mirror that dict exactly.
-    gate_tool_calls = {
-        "tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}]
-    }
+    gate_tool_calls = {"tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}]}
     fingerprint = json.dumps(gate_tool_calls, sort_keys=True, default=str)
 
     logger = _GateOnlyLogger(
diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
index 8c934f9c21e..b18af060a20 100644
--- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
+++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
@@ -1212,6 +1212,73 @@ def test_async_compact_handler_sends_json_when_not_signed():
     assert "data" not in kwargs
 
 
+@pytest.mark.asyncio
+async def test_async_anthropic_messages_handler_passes_api_key_to_agentic_hooks():
+    """
+    Regression: async_anthropic_messages_handler must inject api_key into the
+    kwargs dict forwarded to _call_agentic_completion_hooks.
+
+    Without this, follow-up calls made by agentic hooks (e.g. websearch
+    interception's second LLM call after executing searches) have no api_key
+    and fail with "x-api-key header is required".
+    """
+    handler = BaseLLMHTTPHandler()
+
+    mock_config = Mock()
+    mock_config.validate_anthropic_messages_environment = Mock(
+        return_value=({"x-api-key": "sk-test"}, "https://api.anthropic.com")
+    )
+    mock_config.transform_anthropic_messages_request = Mock(
+        return_value={"model": "claude-haiku", "messages": [], "max_tokens": 16}
+    )
+    mock_config.sign_request = Mock(return_value=({}, None))
+
+    fake_raw_response = {"id": "msg_1", "type": "message", "role": "assistant", "content": [], "stop_reason": "end_turn"}
+    mock_config.transform_anthropic_messages_response = Mock(return_value=fake_raw_response)
+
+    mock_logging_obj = Mock()
+    mock_logging_obj.update_environment_variables = Mock()
+    mock_logging_obj.model_call_details = {}
+    mock_logging_obj.stream = False
+    mock_logging_obj.dynamic_success_callbacks = None
+
+    captured_kwargs: dict = {}
+    sentinel_response = object()
+
+    async def fake_agentic_hooks(**call_kwargs):
+        captured_kwargs.update(call_kwargs)
+        return sentinel_response
+
+    mock_httpx_response = Mock()
+    mock_httpx_response.status_code = 200
+
+    with (
+        patch.object(handler, "_async_post_anthropic_messages_with_http_error_retry", new=AsyncMock(return_value=mock_httpx_response)),
+        patch.object(handler, "_call_agentic_completion_hooks", side_effect=fake_agentic_hooks),
+        patch("litellm.llms.custom_httpx.llm_http_handler.get_async_httpx_client"),
+        patch("litellm.litellm_core_utils.get_provider_specific_headers.ProviderSpecificHeaderUtils.get_provider_specific_headers", return_value=None),
+    ):
+        result = await handler.async_anthropic_messages_handler(
+            model="claude-haiku",
+            messages=[{"role": "user", "content": "hi"}],
+            anthropic_messages_provider_config=mock_config,
+            anthropic_messages_optional_request_params={"stream": False},
+            custom_llm_provider="anthropic",
+            litellm_params=GenericLiteLLMParams(api_key="sk-real-anthropic-key"),
+            logging_obj=mock_logging_obj,
+            api_key="sk-real-anthropic-key",
+            stream=False,
+        )
+
+    assert result is sentinel_response
+    assert "kwargs" in captured_kwargs, "_call_agentic_completion_hooks not called"
+    forwarded = captured_kwargs["kwargs"]
+    assert forwarded.get("api_key") == "sk-real-anthropic-key", (
+        "api_key must be injected into kwargs passed to _call_agentic_completion_hooks "
+        "so follow-up calls in agentic hooks (e.g. websearch) can authenticate"
+    )
+
+
 class _FakeWSExceptions:
     class WebSocketException(Exception):
         pass

From 6c21029cb7e8fe827ea9f8d108f1ff18ae3b9e4b Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrish+github@berri.ai>
Date: Tue, 30 Jun 2026 18:58:09 -0700
Subject: [PATCH 44/51] feat(sandbox): reuse e2b container across requests when
 metadata.session_id is set (#31688)

* feat(sandbox): reuse e2b container across requests when metadata.session_id is set

When a client passes `metadata.session_id` in a /chat/completions request
alongside a code_interpreter tool, the proxy now routes all requests sharing
that session_id to the same sandbox container. State (variables, imports,
installed packages) persists across requests within the session.

Without a session_id the existing ephemeral behavior is unchanged: one
container per agentic loop, deleted immediately after.

The sandbox key is derived from session_id rather than a per-request UUID.
The cleanup and post-loop hooks skip deletion for session-scoped containers.
TTL-based pruning (15 min idle) still applies and refreshes on every use,
so an active session never expires mid-use. The session_id-scoped key is
registered in all_litellm_params and the proxy strip-list so it never
leaks to the upstream LLM provider.

* fix(sandbox): scope session sandbox key to API key identity; add per-identity LRU cap

Two security issues addressed:

1. Cross-user sandbox isolation: the session_id supplied by the client is now
   combined with the server-minted user_api_key_hash to form the cache key
   (format: "{hash}:{session_id}" when authenticated, bare session_id for
   non-proxy use). Two tenants sharing the same session_id no longer share a
   sandbox.

2. Bounded session allocation: each API key identity is capped at
   _SESSION_SCOPED_PER_IDENTITY_CAP (10) live session-scoped containers. When
   a new session is opened beyond the cap, the least-recently-used entry for
   that identity is evicted and its sandbox deleted, preventing unbounded
   accumulation via rotating session IDs.

The container cache tuple gains a fourth element (identity: str | None) so
eviction can filter by identity without parsing key formats. Tests added for
both properties.
---
 .../code_interpreter_interception/handler.py  |  71 +++-
 litellm/proxy/dev_config.yaml                 |  15 +
 litellm/proxy/litellm_pre_call_utils.py       |   1 +
 litellm/types/utils.py                        |   1 +
 qa_sticky_session.sh                          |  59 +++
 .../test_handler.py                           | 369 ++++++++++++++----
 6 files changed, 421 insertions(+), 95 deletions(-)
 create mode 100755 qa_sticky_session.sh

diff --git a/litellm/integrations/code_interpreter_interception/handler.py b/litellm/integrations/code_interpreter_interception/handler.py
index cd7b211f1a5..759b2be3a84 100644
--- a/litellm/integrations/code_interpreter_interception/handler.py
+++ b/litellm/integrations/code_interpreter_interception/handler.py
@@ -40,9 +40,11 @@ from litellm.types.utils import (
 LITELLM_CODE_EXECUTION_TOOL_NAME = "litellm_code_execution"
 _INTERCEPTION_ACTIVE_KEY = "_code_interpreter_interception_active"
 _SANDBOX_KEY = "_code_interpreter_interception_sandbox_key"
+_SESSION_SCOPED_KEY = "_code_interpreter_interception_session_scoped"
 _CONVERTED_STREAM_KEY = "_code_interpreter_interception_converted_stream"
 _LITELLM_METADATA_KEY = "litellm_metadata"
 _CACHE_TTL_SECONDS = 15 * 60
+_SESSION_SCOPED_PER_IDENTITY_CAP = 10
 
 
 class CodeExecutionToolCall(TypedDict, total=False):
@@ -107,6 +109,20 @@ class ChatCompletionFunctionToolChoice(TypedDict):
 CodeExecutionFunctionToolChoice = ResponsesFunctionToolChoice | ChatCompletionFunctionToolChoice
 
 
+def _extract_session_id(kwargs: dict[str, Any]) -> str | None:
+    for meta_key in ("metadata", "litellm_metadata"):
+        meta = kwargs.get(meta_key)
+        if isinstance(meta, dict):
+            sid = meta.get("session_id")
+            if sid and isinstance(sid, str):
+                return sid
+    return None
+
+
+def _extract_identity(kwargs: dict[str, Any]) -> str:
+    return kwargs.get("user_api_key_hash") or ""
+
+
 def _resolve_sandbox_tool(sandbox_tool_name: str | None) -> dict[str, Any] | None:
     try:
         from litellm.sandbox.sandbox_tools import resolve_sandbox_tool
@@ -140,7 +156,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
         self.enabled_providers = enabled_providers
         self.sandbox_tool_name = sandbox_tool_name
         self.sandbox_config = sandbox_config
-        self._container_cache: dict[str, tuple[Any, dict[str, Any] | None, float]] = {}
+        self._container_cache: dict[str, tuple[Any, dict[str, Any] | None, float, str | None]] = {}
 
     @classmethod
     def from_config_yaml(cls, config: CodeInterpreterInterceptionConfig) -> "CodeInterpreterInterceptionLogger":
@@ -191,7 +207,13 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
             return None
 
         kwargs[_INTERCEPTION_ACTIVE_KEY] = True
-        kwargs[_SANDBOX_KEY] = uuid.uuid4().hex
+        session_id = _extract_session_id(kwargs)
+        if session_id:
+            identity = _extract_identity(kwargs)
+            kwargs[_SANDBOX_KEY] = f"{identity}:{session_id}" if identity else session_id
+            kwargs[_SESSION_SCOPED_KEY] = True
+        else:
+            kwargs[_SANDBOX_KEY] = uuid.uuid4().hex
         if kwargs.get("stream"):
             kwargs["stream"] = False
             kwargs[_CONVERTED_STREAM_KEY] = True
@@ -217,6 +239,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
             if not is_interception_internal_key(key)
             and not key.startswith("_agentic_loop")
             and key != "max_agentic_loops"
+            and key != _SESSION_SCOPED_KEY
         }
         if filtered_metadata:
             kwargs[_LITELLM_METADATA_KEY] = filtered_metadata
@@ -227,7 +250,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
     def _write_interception_metadata(kwargs: dict[str, Any]) -> None:
         metadata = kwargs.get(_LITELLM_METADATA_KEY)
         metadata = dict(metadata) if isinstance(metadata, dict) else {}
-        for key in (_INTERCEPTION_ACTIVE_KEY, _SANDBOX_KEY, _CONVERTED_STREAM_KEY):
+        for key in (_INTERCEPTION_ACTIVE_KEY, _SANDBOX_KEY, _SESSION_SCOPED_KEY, _CONVERTED_STREAM_KEY):
             if key in kwargs:
                 metadata[key] = kwargs[key]
         kwargs[_LITELLM_METADATA_KEY] = metadata
@@ -347,7 +370,9 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
         await self._prune_expired_cache()
         tool_calls = cast(list[CodeExecutionToolCall], tools.get("tool_calls", []))
         sandbox_key = kwargs.get(_SANDBOX_KEY)
-        container, params = await self._get_or_create_container(cache_key=sandbox_key)
+        is_session = bool(kwargs.get(_SESSION_SCOPED_KEY))
+        identity = _extract_identity(kwargs) if is_session else None
+        container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity)
 
         try:
             container_id = cast(str | None, getattr(container, "id", None))
@@ -404,6 +429,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
             metadata={
                 "tool_type": "code_interpreter",
                 "sandbox_key": sandbox_key or "",
+                "is_session_scoped": bool(kwargs.get(_SESSION_SCOPED_KEY)),
                 "code_interpreter_calls": code_interpreter_calls,
             },
         )
@@ -419,7 +445,9 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
         await self._prune_expired_cache()
         tool_calls = cast(list[CodeExecutionToolCall], tools.get("tool_calls", []))
         sandbox_key = cast(str | None, kwargs.get(_SANDBOX_KEY))
-        container, params = await self._get_or_create_container(cache_key=sandbox_key)
+        is_session = bool(kwargs.get(_SESSION_SCOPED_KEY))
+        identity = _extract_identity(cast(dict[str, Any], kwargs)) if is_session else None
+        container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity)
 
         try:
             container_id = cast(str | None, getattr(container, "id", None))
@@ -455,6 +483,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
             metadata={
                 "tool_type": "code_interpreter",
                 "sandbox_key": sandbox_key or "",
+                "is_session_scoped": bool(kwargs.get(_SESSION_SCOPED_KEY)),
                 "code_interpreter_calls": code_interpreter_calls,
                 "response_format": "openai",
             },
@@ -489,6 +518,8 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
 
     async def async_agentic_loop_cleanup_hook(self, plan: AgenticLoopPlan, kwargs: dict) -> None:
         metadata = plan.metadata or {} if plan else {}
+        if metadata.get("is_session_scoped"):
+            return
         await self._delete_container_for_cache_key(metadata.get("sandbox_key"))
 
     @staticmethod
@@ -520,7 +551,8 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
 
     async def async_post_agentic_loop_response_hook(self, response: Any, plan: AgenticLoopPlan, kwargs: dict) -> Any:
         metadata = plan.metadata or {} if plan else {}
-        await self._delete_container_for_cache_key(metadata.get("sandbox_key"))
+        if not metadata.get("is_session_scoped"):
+            await self._delete_container_for_cache_key(metadata.get("sandbox_key"))
 
         calls = metadata.get("code_interpreter_calls")
         if not calls:
@@ -565,17 +597,32 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
             return f"[execution error] {message}"
         return getattr(result, "stdout", "") or ""
 
-    async def _get_or_create_container(self, cache_key: str | None) -> tuple[Any, dict[str, Any] | None]:
+    async def _get_or_create_container(
+        self,
+        cache_key: str | None,
+        identity: str | None = None,
+    ) -> tuple[Any, dict[str, Any] | None]:
         if cache_key:
             cached = self._container_cache.get(cache_key)
             if cached is not None:
+                self._container_cache[cache_key] = (cached[0], cached[1], time.time(), cached[3])
                 return cached[0], cached[1]
 
         container, params = await self._create_container()
         if cache_key:
-            self._container_cache[cache_key] = (container, params, time.time())
+            if identity is not None:
+                await self._evict_lru_session_if_over_cap(identity)
+            self._container_cache[cache_key] = (container, params, time.time(), identity)
         return container, params
 
+    async def _evict_lru_session_if_over_cap(self, identity: str) -> None:
+        identity_entries = [(k, v) for k, v in self._container_cache.items() if v[3] == identity]
+        if len(identity_entries) < _SESSION_SCOPED_PER_IDENTITY_CAP:
+            return
+        lru_key, lru_entry = min(identity_entries, key=lambda item: item[1][2])
+        self._container_cache.pop(lru_key, None)
+        await self._delete_container(container=lru_entry[0], params=lru_entry[1])
+
     async def _create_container(self) -> tuple[Any, dict[str, Any] | None]:
         if self.sandbox_config is not None:
             return await self.sandbox_config.acreate_sandbox(), None
@@ -739,12 +786,8 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
         now = time.time()
         expired = [
             (cache_key, container, params)
-            for cache_key, (
-                container,
-                params,
-                created_at,
-            ) in self._container_cache.items()
-            if now - created_at > _CACHE_TTL_SECONDS
+            for cache_key, (container, params, last_accessed, *_) in self._container_cache.items()
+            if now - last_accessed > _CACHE_TTL_SECONDS
         ]
         for cache_key, container, params in expired:
             self._container_cache.pop(cache_key, None)
diff --git a/litellm/proxy/dev_config.yaml b/litellm/proxy/dev_config.yaml
index e437ed7a118..6078161c780 100644
--- a/litellm/proxy/dev_config.yaml
+++ b/litellm/proxy/dev_config.yaml
@@ -182,10 +182,25 @@ model_list:
     litellm_params:
       model: openai/gpt-5.5
       api_key: os.environ/OPENAI_API_KEY
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: openai/gpt-4o-mini
+      api_key: os.environ/OPENAI_API_KEY
 
 general_settings:
   master_key: sk-1234
 
+sandbox_tools:
+  - sandbox_tool_name: e2b_sandbox
+    litellm_params:
+      sandbox_provider: e2b
+      api_key: os.environ/E2B_API_KEY
+
 litellm_settings:
   drop_params: True
   telemetry: False
+  code_interpreter_interception_params:
+    enabled: true
+    sandbox_tool_name: e2b_sandbox
+  callbacks:
+    - code_interpreter_interception
diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py
index 0ffb0337545..6f5c82530e3 100644
--- a/litellm/proxy/litellm_pre_call_utils.py
+++ b/litellm/proxy/litellm_pre_call_utils.py
@@ -153,6 +153,7 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS = (
     "_code_interpreter_interception_active",
     "_code_interpreter_interception_converted_stream",
     "_code_interpreter_interception_sandbox_key",
+    "_code_interpreter_interception_session_scoped",
     "max_agentic_loops",
 )
 
diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index dff4e4af89e..4f0c0c21bce 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -3062,6 +3062,7 @@ agentic_loop_internal_litellm_params = [
     "max_agentic_loops",
     "_code_interpreter_interception_active",
     "_code_interpreter_interception_sandbox_key",
+    "_code_interpreter_interception_session_scoped",
     "_code_interpreter_interception_converted_stream",
 ]
 
diff --git a/qa_sticky_session.sh b/qa_sticky_session.sh
new file mode 100755
index 00000000000..326bb8117c7
--- /dev/null
+++ b/qa_sticky_session.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# QA: code interpreter sandbox stickiness via metadata.session_id
+#   bash qa_sticky_session.sh
+#   LITELLM_BASE_URL=http://localhost:4000 LITELLM_KEY=sk-1234 bash qa_sticky_session.sh
+
+set -euo pipefail
+
+BASE="${LITELLM_BASE_URL:-http://localhost:4000}"
+KEY="${LITELLM_KEY:-sk-1234}"
+MODEL="${LITELLM_MODEL:-gpt-4o-mini}"
+# proxy running at http://localhost:4000 (master key: sk-1234)
+SESSION_A="qa-session-$(date +%s)-A"
+SESSION_B="qa-session-$(date +%s)-B"
+
+content() {
+  echo "$1" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('choices',[{}])[0].get('message',{}).get('content','<error>'))"
+}
+
+call() {
+  local session="${1:-}" code="$2" meta=""
+  [[ -n "$session" ]] && meta=", \"metadata\": {\"session_id\": \"$session\"}"
+  curl -s -X POST "$BASE/chat/completions" \
+    -H "Content-Type: application/json" \
+    -H "Authorization: Bearer $KEY" \
+    -d "{\"model\":\"$MODEL\"$meta,\"tools\":[{\"type\":\"code_interpreter\"}],\"messages\":[{\"role\":\"user\",\"content\":\"Run this Python code and tell me the result: $code\"}]}"
+}
+
+assert_match() {
+  local label="$1" body="$2" pattern="$3"
+  if echo "$body" | grep -qiE "$pattern"; then
+    echo "PASS  $label"
+  else
+    echo "FAIL  $label (expected /$pattern/)"
+    echo "      $(content "$body")"
+    exit 1
+  fi
+}
+
+echo "=== Sticky Session Sandbox QA ==="
+echo "base: $BASE  session A: $SESSION_A  session B: $SESSION_B"
+echo
+
+R=$(call "$SESSION_A" "x = 42; print(x)")
+assert_match "same session_id reuses sandbox (set x=42)" "$R" "42"
+
+R=$(call "$SESSION_A" "print(x)")
+assert_match "same session_id keeps state (x still 42)" "$R" "42"
+
+R=$(call "$SESSION_B" "print(x)")
+assert_match "different session_id is isolated" "$R" "not defined|NameError|undefined|error"
+
+R=$(call "" "y = 99; print(y)")
+assert_match "no session_id runs code" "$R" "99"
+
+R=$(call "" "print(y)")
+assert_match "no session_id gets fresh sandbox each request" "$R" "not defined|NameError|undefined|error"
+
+echo
+echo "All checks passed."
diff --git a/tests/test_litellm/integrations/code_interpreter_interception/test_handler.py b/tests/test_litellm/integrations/code_interpreter_interception/test_handler.py
index 7ff58ba6324..9d308ac1989 100644
--- a/tests/test_litellm/integrations/code_interpreter_interception/test_handler.py
+++ b/tests/test_litellm/integrations/code_interpreter_interception/test_handler.py
@@ -14,6 +14,7 @@ from litellm.integrations.code_interpreter_interception.handler import (
     LITELLM_CODE_EXECUTION_TOOL_NAME,
     _INTERCEPTION_ACTIVE_KEY as _ACTIVE_KEY,
     _SANDBOX_KEY,
+    _SESSION_SCOPED_KEY,
 )
 from litellm.types.integrations.custom_logger import (
     CHAT_COMPLETION_AGENTIC_SURFACE,
@@ -138,11 +139,7 @@ async def test_build_plan_runs_code_and_feeds_output_back():
     assert sandbox.run_calls[0]["code"] == "print(40 + 2)"
 
     messages = _iter_messages(plan)
-    outputs = [
-        m
-        for m in messages
-        if isinstance(m, dict) and m.get("type") == "function_call_output"
-    ]
+    outputs = [m for m in messages if isinstance(m, dict) and m.get("type") == "function_call_output"]
     assert outputs, "expected a function_call_output item appended"
     output_item = next(m for m in outputs if m.get("call_id") == "c1")
     assert "42" in str(output_item["output"])
@@ -160,9 +157,7 @@ async def test_pre_call_converts_code_interpreter_tool():
 
     assert result is not None
     tools = result["tools"]
-    assert not any(
-        t.get("type") == "code_interpreter" for t in tools
-    ), "code_interpreter tool must be removed"
+    assert not any(t.get("type") == "code_interpreter" for t in tools), "code_interpreter tool must be removed"
     names = [t.get("name") or (t.get("function") or {}).get("name") for t in tools]
     assert LITELLM_CODE_EXECUTION_TOOL_NAME in names
 
@@ -267,9 +262,7 @@ async def test_should_run_detects_only_matching_function_call():
     logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox())
 
     active_kwargs = {"_code_interpreter_interception_active": True}
-    match = FakeResponse(
-        output=[_function_call_item(name=LITELLM_CODE_EXECUTION_TOOL_NAME)]
-    )
+    match = FakeResponse(output=[_function_call_item(name=LITELLM_CODE_EXECUTION_TOOL_NAME)])
     should_run, payload = await logger.async_should_run_agentic_loop(
         response=match,
         model="gpt-5",
@@ -331,9 +324,7 @@ async def test_container_reused_within_request_via_server_sandbox_key():
         **common,
     )
 
-    assert (
-        len(sandbox.create_calls) == 1
-    ), "the sandbox is reused across loop iterations sharing one server sandbox key"
+    assert len(sandbox.create_calls) == 1, "the sandbox is reused across loop iterations sharing one server sandbox key"
 
 
 @pytest.mark.asyncio
@@ -372,9 +363,9 @@ async def test_colliding_caller_call_id_does_not_share_sandbox():
         **common,
     )
 
-    assert (
-        len(sandbox.create_calls) == 2
-    ), "distinct server sandbox keys must isolate sandboxes despite a colliding call id"
+    assert len(sandbox.create_calls) == 2, (
+        "distinct server sandbox keys must isolate sandboxes despite a colliding call id"
+    )
 
 
 @pytest.mark.asyncio
@@ -479,14 +470,11 @@ async def test_post_hook_injects_code_interpreter_call_matching_openai_shape():
     )
     response = FakeResponse(output=[{"type": "message", "content": []}])
 
-    out = await logger.async_post_agentic_loop_response_hook(
-        response=response, plan=plan, kwargs={}
-    )
+    out = await logger.async_post_agentic_loop_response_hook(response=response, plan=plan, kwargs={})
 
     types = [item.get("type") for item in out.output]
     assert types == ["code_interpreter_call", "message"], (
-        "code_interpreter_call must be re-injected before the message, matching "
-        "OpenAI's native output ordering"
+        "code_interpreter_call must be re-injected before the message, matching OpenAI's native output ordering"
     )
     assert set(out.output[0].keys()) == {
         "id",
@@ -524,8 +512,7 @@ async def test_pre_call_forces_non_stream_for_loop():
     assert out is not None
     assert out["stream"] is False, "loop requires a non-streaming upstream call"
     assert out["_code_interpreter_interception_converted_stream"] is True, (
-        "the converted-stream flag must be set so the final response is wrapped "
-        "back into a stream for the caller"
+        "the converted-stream flag must be set so the final response is wrapped back into a stream for the caller"
     )
 
 
@@ -556,9 +543,7 @@ async def test_gate_refuses_without_server_active_marker():
     """A forged litellm_code_execution call must not trigger the loop unless the
     pre-call hook actually converted a native code_interpreter tool."""
     logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox())
-    forged = FakeResponse(
-        output=[_function_call_item(name=LITELLM_CODE_EXECUTION_TOOL_NAME)]
-    )
+    forged = FakeResponse(output=[_function_call_item(name=LITELLM_CODE_EXECUTION_TOOL_NAME)])
 
     should_run, payload = await logger.async_should_run_agentic_loop(
         response=forged,
@@ -577,12 +562,8 @@ async def test_gate_refuses_without_server_active_marker():
 @pytest.mark.asyncio
 async def test_gate_rechecks_provider_scope():
     """enabled_providers must be re-enforced at the gate, not only in pre-call."""
-    logger = CodeInterpreterInterceptionLogger(
-        sandbox_config=FakeSandbox(), enabled_providers=["openai"]
-    )
-    response = FakeResponse(
-        output=[_function_call_item(name=LITELLM_CODE_EXECUTION_TOOL_NAME)]
-    )
+    logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox(), enabled_providers=["openai"])
+    response = FakeResponse(output=[_function_call_item(name=LITELLM_CODE_EXECUTION_TOOL_NAME)])
 
     should_run, _ = await logger.async_should_run_agentic_loop(
         response=response,
@@ -600,11 +581,7 @@ async def test_gate_rechecks_provider_scope():
 @pytest.mark.asyncio
 async def test_chat_completion_gate_detects_code_execution_tool_call():
     logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox())
-    response = {
-        "choices": [
-            {"message": {"tool_calls": [_chat_function_call_item(call_id="call_123")]}}
-        ]
-    }
+    response = {"choices": [{"message": {"tool_calls": [_chat_function_call_item(call_id="call_123")]}}]}
 
     should_run, payload = await logger.async_should_run_agentic_loop(
         response=response,
@@ -661,9 +638,7 @@ async def test_chat_completion_build_plan_runs_code_and_appends_tool_message():
         },
         model="gpt-5",
         messages=[{"role": "user", "content": "x"}],
-        response={
-            "choices": [{"message": {"tool_calls": [_chat_function_call_item()]}}]
-        },
+        response={"choices": [{"message": {"tool_calls": [_chat_function_call_item()]}}]},
         anthropic_messages_provider_config=None,
         anthropic_messages_optional_request_params={
             "tools": [native_chat_tool],
@@ -738,8 +713,7 @@ async def test_pre_call_strips_client_forged_marker_on_initial_request():
     await logger.async_pre_call_deployment_hook(kwargs, CallTypes.aresponses)
 
     assert _ACTIVE_KEY not in kwargs, (
-        "no native code_interpreter tool was present, so a client-supplied "
-        "active marker must be cleared"
+        "no native code_interpreter tool was present, so a client-supplied active marker must be cleared"
     )
     assert kwargs["litellm_metadata"] == {"safe_user_value": "kept"}
 
@@ -774,8 +748,7 @@ async def test_pre_call_strips_forged_loop_controls_then_mints_own_markers():
     assert metadata[_ACTIVE_KEY] is True
     assert metadata[_SANDBOX_KEY] == result[_SANDBOX_KEY]
     assert metadata[_SANDBOX_KEY] != "client-forged", (
-        "the surviving sandbox key must be the server-minted one, not the forged "
-        "value the client supplied"
+        "the surviving sandbox key must be the server-minted one, not the forged value the client supplied"
     )
 
 
@@ -793,8 +766,7 @@ async def test_pre_call_preserves_marker_on_server_followup():
     await logger.async_pre_call_deployment_hook(kwargs, CallTypes.aresponses)
 
     assert kwargs.get(_ACTIVE_KEY) is True, (
-        "the server-set marker must survive followup requests so multi-round "
-        "code execution keeps working"
+        "the server-set marker must survive followup requests so multi-round code execution keeps working"
     )
 
 
@@ -805,9 +777,7 @@ async def test_sandbox_deleted_after_loop_completes():
 
     plan = await _build_plan(logger, sandbox, call_id="k1")
     assert sandbox.create_calls, "sandbox must be created during the loop"
-    assert (
-        not sandbox.delete_calls
-    ), "sandbox must outlive the loop until the final hook"
+    assert not sandbox.delete_calls, "sandbox must outlive the loop until the final hook"
 
     await logger.async_post_agentic_loop_response_hook(
         response=FakeResponse(output=[{"type": "message", "content": []}]),
@@ -816,8 +786,7 @@ async def test_sandbox_deleted_after_loop_completes():
     )
 
     assert len(sandbox.delete_calls) == 1, (
-        "the sandbox must be deleted once the final response is assembled, "
-        "otherwise it keeps running and billing"
+        "the sandbox must be deleted once the final response is assembled, otherwise it keeps running and billing"
     )
     assert "sbxkey1" not in logger._container_cache
 
@@ -829,16 +798,11 @@ async def test_post_hook_delete_is_idempotent_across_loop_levels():
     plan = await _build_plan(logger, sandbox, call_id="k1")
     response = FakeResponse(output=[{"type": "message", "content": []}])
 
-    await logger.async_post_agentic_loop_response_hook(
-        response=response, plan=plan, kwargs={}
-    )
-    await logger.async_post_agentic_loop_response_hook(
-        response=response, plan=plan, kwargs={}
-    )
+    await logger.async_post_agentic_loop_response_hook(response=response, plan=plan, kwargs={})
+    await logger.async_post_agentic_loop_response_hook(response=response, plan=plan, kwargs={})
 
     assert len(sandbox.delete_calls) == 1, (
-        "deleting an already-removed container must be a no-op so unwinding "
-        "loop levels do not double-delete"
+        "deleting an already-removed container must be a no-op so unwinding loop levels do not double-delete"
     )
 
 
@@ -860,8 +824,7 @@ async def test_build_plan_deletes_sandbox_when_execution_raises():
 
     assert len(sandbox.create_calls) == 1, "the sandbox must have been created"
     assert len(sandbox.delete_calls) == 1, (
-        "a build failure must delete the cached sandbox so it does not keep "
-        "running and billing"
+        "a build failure must delete the cached sandbox so it does not keep running and billing"
     )
     assert "sbxkey1" not in logger._container_cache
 
@@ -875,8 +838,7 @@ async def test_cleanup_hook_deletes_sandbox():
     await logger.async_agentic_loop_cleanup_hook(plan=plan, kwargs={})
 
     assert len(sandbox.delete_calls) == 1, (
-        "the cleanup hook must delete the sandbox so a rerun failure cannot "
-        "leak a running container"
+        "the cleanup hook must delete the sandbox so a rerun failure cannot leak a running container"
     )
     assert "sbxkey1" not in logger._container_cache
 
@@ -895,8 +857,7 @@ async def test_cleanup_hook_is_idempotent_with_post_hook():
     await logger.async_agentic_loop_cleanup_hook(plan=plan, kwargs={})
 
     assert len(sandbox.delete_calls) == 1, (
-        "cleanup running in finally after the success-path post hook already "
-        "deleted the sandbox must not double-delete"
+        "cleanup running in finally after the success-path post hook already deleted the sandbox must not double-delete"
     )
 
 
@@ -923,9 +884,7 @@ async def test_responses_plan_cleans_up_sandbox_when_followup_raises():
 
     plan = AgenticLoopPlan(
         run_agentic_loop=True,
-        request_patch=AgenticLoopRequestPatch(
-            model="gpt-5", messages=[{"role": "user", "content": "x"}]
-        ),
+        request_patch=AgenticLoopRequestPatch(model="gpt-5", messages=[{"role": "user", "content": "x"}]),
         metadata={"sandbox_key": "sbxkey1"},
     )
 
@@ -995,9 +954,7 @@ async def test_run_code_does_not_re_resolve_registry(monkeypatch):
 
         sandbox_tools.clear_sandbox_tools()
 
-        stdout = await logger._run_tool_call(
-            container=container, params=params, arguments='{"code":"print(1)"}'
-        )
+        stdout = await logger._run_tool_call(container=container, params=params, arguments='{"code":"print(1)"}')
     finally:
         sandbox_tools.clear_sandbox_tools()
 
@@ -1013,9 +970,7 @@ async def test_run_tool_call_surfaces_execution_error():
     class ErroringSandbox(FakeSandbox):
         async def arun_code(self, *, container, code, **kwargs):
             self.run_calls.append({"container": container, "code": code})
-            return CodeExecutionResult(
-                stdout="", error={"name": "ValueError", "value": "boom"}
-            )
+            return CodeExecutionResult(stdout="", error={"name": "ValueError", "value": "boom"})
 
     sandbox = ErroringSandbox()
     logger = CodeInterpreterInterceptionLogger(sandbox_config=sandbox)
@@ -1036,9 +991,7 @@ async def test_run_tool_call_reports_unparseable_arguments():
     logger = CodeInterpreterInterceptionLogger(sandbox_config=sandbox)
     container = await logger._create_container()
 
-    stdout = await logger._run_tool_call(
-        container=container[0], params=None, arguments="not-json"
-    )
+    stdout = await logger._run_tool_call(container=container[0], params=None, arguments="not-json")
 
     assert stdout == "[invalid tool arguments: could not parse code]"
     assert not sandbox.run_calls, "code must not run when arguments cannot be parsed"
@@ -1048,9 +1001,7 @@ async def test_run_tool_call_reports_unparseable_arguments():
 async def test_pre_call_skips_provider_outside_scope():
     """enabled_providers must filter the pre-call conversion so a request to an
     out-of-scope provider is left untouched."""
-    logger = CodeInterpreterInterceptionLogger(
-        sandbox_config=FakeSandbox(), enabled_providers=["openai"]
-    )
+    logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox(), enabled_providers=["openai"])
     kwargs = {
         "tools": [{"type": "code_interpreter", "container": {"type": "auto"}}],
         "custom_llm_provider": "anthropic",
@@ -1119,6 +1070,7 @@ async def test_prune_expired_cache_deletes_underlying_container():
         container,
         params,
         time.time() - handler_mod._CACHE_TTL_SECONDS - 1,
+        None,
     )
 
     await logger._prune_expired_cache()
@@ -1217,3 +1169,258 @@ async def test_extract_tool_calls_reads_object_attributes():
     assert len(calls) == 1
     assert calls[0]["call_id"] == "c9"
     assert calls[0]["arguments"] == '{"code":"print(1)"}'
+
+
+# ---------------------------------------------------------------------------
+# Sticky session tests
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_pre_call_uses_session_id_from_metadata_as_sandbox_key():
+    """When session_id is in request metadata, it becomes the sandbox key so the
+    container is shared across requests in the same session."""
+    logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox())
+    session_id = "conv-abc-123"
+    kwargs = {
+        "tools": [{"type": "code_interpreter", "container": {"type": "auto"}}],
+        "custom_llm_provider": "openai",
+        "metadata": {"session_id": session_id},
+    }
+
+    result = await logger.async_pre_call_deployment_hook(kwargs, CallTypes.acompletion)
+
+    assert result is not None
+    assert result[_SANDBOX_KEY] == session_id
+    assert result[_SESSION_SCOPED_KEY] is True
+    assert result["litellm_metadata"][_SANDBOX_KEY] == session_id
+    assert result["litellm_metadata"][_SESSION_SCOPED_KEY] is True
+
+
+@pytest.mark.asyncio
+async def test_pre_call_uses_session_id_from_litellm_metadata():
+    """session_id in litellm_metadata also works as the sticky key."""
+    logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox())
+    session_id = "sess-xyz-789"
+    kwargs = {
+        "tools": [{"type": "code_interpreter", "container": {"type": "auto"}}],
+        "custom_llm_provider": "openai",
+        "litellm_metadata": {"session_id": session_id},
+    }
+
+    result = await logger.async_pre_call_deployment_hook(kwargs, CallTypes.acompletion)
+
+    assert result is not None
+    assert result[_SANDBOX_KEY] == session_id
+    assert result[_SESSION_SCOPED_KEY] is True
+
+
+@pytest.mark.asyncio
+async def test_pre_call_without_session_id_still_mints_random_key():
+    """Requests without a session_id still get a server-minted random sandbox key."""
+    logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox())
+    kwargs = {
+        "tools": [{"type": "code_interpreter", "container": {"type": "auto"}}],
+        "custom_llm_provider": "openai",
+    }
+
+    result = await logger.async_pre_call_deployment_hook(kwargs, CallTypes.acompletion)
+
+    assert result is not None
+    assert _SESSION_SCOPED_KEY not in result or result[_SESSION_SCOPED_KEY] is False
+    assert len(result[_SANDBOX_KEY]) >= 16
+
+
+@pytest.mark.asyncio
+async def test_session_scoped_sandbox_survives_agentic_loop_cleanup():
+    """A session-scoped sandbox must NOT be deleted by the cleanup or post hooks;
+    it needs to persist across requests within the same session."""
+    sandbox = FakeSandbox(stdout="42")
+    logger = CodeInterpreterInterceptionLogger(sandbox_config=sandbox)
+    session_id = "conv-persist-me"
+
+    plan = await logger.async_build_agentic_loop_plan(
+        tools={
+            "tool_calls": [
+                {
+                    "call_id": "c1",
+                    "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+                    "arguments": '{"code":"x = 10"}',
+                }
+            ]
+        },
+        model="gpt-4o-mini",
+        messages=[{"role": "user", "content": "set x"}],
+        response=FakeResponse(output=[_function_call_item()]),
+        anthropic_messages_provider_config=None,
+        anthropic_messages_optional_request_params={"tools": []},
+        logging_obj=FakeLogging(litellm_call_id="k1"),
+        stream=False,
+        kwargs={
+            "litellm_call_id": "k1",
+            _SANDBOX_KEY: session_id,
+            _SESSION_SCOPED_KEY: True,
+        },
+    )
+
+    assert plan.metadata["is_session_scoped"] is True
+
+    await logger.async_post_agentic_loop_response_hook(
+        response=FakeResponse(output=[{"type": "message", "content": []}]),
+        plan=plan,
+        kwargs={},
+    )
+    await logger.async_agentic_loop_cleanup_hook(plan=plan, kwargs={})
+
+    assert not sandbox.delete_calls, (
+        "session-scoped sandbox must not be deleted after a single agentic loop; "
+        "it must persist for the next request in the session"
+    )
+    assert session_id in logger._container_cache, "session-scoped container must remain in cache after loop ends"
+
+
+@pytest.mark.asyncio
+async def test_session_scoped_sandbox_reused_across_sequential_requests():
+    """Two sequential requests with the same session_id must share one container,
+    confirming state (e.g. assigned variables) can persist across HTTP requests."""
+    sandbox = FakeSandbox(stdout="42")
+    logger = CodeInterpreterInterceptionLogger(sandbox_config=sandbox)
+    session_id = "conv-reuse-me"
+
+    common_plan_args = dict(
+        tools={
+            "tool_calls": [
+                {
+                    "call_id": "c1",
+                    "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+                    "arguments": '{"code":"print(1)"}',
+                }
+            ]
+        },
+        model="gpt-4o-mini",
+        messages=[{"role": "user", "content": "x"}],
+        response=FakeResponse(output=[_function_call_item()]),
+        anthropic_messages_provider_config=None,
+        anthropic_messages_optional_request_params={"tools": []},
+        stream=False,
+    )
+    session_kwargs = {_SANDBOX_KEY: session_id, _SESSION_SCOPED_KEY: True}
+
+    plan1 = await logger.async_build_agentic_loop_plan(
+        logging_obj=FakeLogging(litellm_call_id="req1"),
+        kwargs={"litellm_call_id": "req1", **session_kwargs},
+        **common_plan_args,
+    )
+    await logger.async_post_agentic_loop_response_hook(
+        response=FakeResponse(output=[{"type": "message", "content": []}]),
+        plan=plan1,
+        kwargs={},
+    )
+
+    plan2 = await logger.async_build_agentic_loop_plan(
+        logging_obj=FakeLogging(litellm_call_id="req2"),
+        kwargs={"litellm_call_id": "req2", **session_kwargs},
+        **common_plan_args,
+    )
+    await logger.async_post_agentic_loop_response_hook(
+        response=FakeResponse(output=[{"type": "message", "content": []}]),
+        plan=plan2,
+        kwargs={},
+    )
+
+    assert len(sandbox.create_calls) == 1, (
+        "a single container must serve both requests in the same session; "
+        "two creates means state cannot persist between requests"
+    )
+    assert len(sandbox.delete_calls) == 0, "the session container must still be alive after both requests complete"
+
+
+@pytest.mark.asyncio
+async def test_non_session_sandbox_still_deleted_after_loop():
+    """Without a session_id, the existing per-request ephemeral behavior is unchanged."""
+    sandbox = FakeSandbox(stdout="42")
+    logger = CodeInterpreterInterceptionLogger(sandbox_config=sandbox)
+
+    plan = await _build_plan(logger, sandbox, call_id="k1")
+    await logger.async_post_agentic_loop_response_hook(
+        response=FakeResponse(output=[{"type": "message", "content": []}]),
+        plan=plan,
+        kwargs={},
+    )
+
+    assert len(sandbox.delete_calls) == 1, "non-session sandbox must still be cleaned up after each request"
+
+
+@pytest.mark.asyncio
+async def test_sandbox_key_scoped_to_api_key_hash_isolates_users():
+    """Two callers supplying the same session_id but different API key hashes must
+    each get their own sandbox; sharing across tenants would let one read or mutate
+    the other's interpreter state."""
+    logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox())
+    session_id = "same-session-id"
+
+    result_a = await logger.async_pre_call_deployment_hook(
+        {
+            "tools": [{"type": "code_interpreter"}],
+            "custom_llm_provider": "openai",
+            "metadata": {"session_id": session_id},
+            "user_api_key_hash": "hash-for-tenant-a",
+        },
+        CallTypes.acompletion,
+    )
+    result_b = await logger.async_pre_call_deployment_hook(
+        {
+            "tools": [{"type": "code_interpreter"}],
+            "custom_llm_provider": "openai",
+            "metadata": {"session_id": session_id},
+            "user_api_key_hash": "hash-for-tenant-b",
+        },
+        CallTypes.acompletion,
+    )
+
+    assert result_a is not None and result_b is not None
+    assert result_a[_SANDBOX_KEY] != result_b[_SANDBOX_KEY], (
+        "same session_id from different API keys must yield different sandbox keys; "
+        "otherwise tenant A can read tenant B's sandbox state"
+    )
+    assert "hash-for-tenant-a" in result_a[_SANDBOX_KEY]
+    assert "hash-for-tenant-b" in result_b[_SANDBOX_KEY]
+
+
+@pytest.mark.asyncio
+async def test_per_identity_cap_evicts_lru_session():
+    """When a single identity holds the cap limit of session sandboxes and opens a
+    new one, the least-recently-used session is evicted so the allocation stays
+    bounded.  Without this, rotating session IDs is an unbounded sandbox leak."""
+    from litellm.integrations.code_interpreter_interception.handler import _SESSION_SCOPED_PER_IDENTITY_CAP
+
+    sandbox = FakeSandbox(stdout="ok")
+    logger = CodeInterpreterInterceptionLogger(sandbox_config=sandbox)
+    identity = "hash-for-identity-x"
+
+    for i in range(_SESSION_SCOPED_PER_IDENTITY_CAP):
+        await logger._get_or_create_container(
+            cache_key=f"{identity}:session-{i}",
+            identity=identity,
+        )
+        logger._container_cache[f"{identity}:session-{i}"] = (
+            logger._container_cache[f"{identity}:session-{i}"][0],
+            logger._container_cache[f"{identity}:session-{i}"][1],
+            float(i),
+            identity,
+        )
+
+    assert len(logger._container_cache) == _SESSION_SCOPED_PER_IDENTITY_CAP
+
+    await logger._get_or_create_container(
+        cache_key=f"{identity}:session-new",
+        identity=identity,
+    )
+
+    assert len(logger._container_cache) == _SESSION_SCOPED_PER_IDENTITY_CAP, (
+        "adding a new session beyond the cap must evict one entry so total stays bounded"
+    )
+    assert f"{identity}:session-0" not in logger._container_cache, (
+        "the entry with the oldest last_accessed timestamp must be evicted first (LRU)"
+    )
+    assert len(sandbox.delete_calls) == 1, "evicted sandbox must be deleted, not just removed from cache"

From 846dbecbf2bcf3e11c60f56971c16c21f13f40fd Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrish+github@berri.ai>
Date: Tue, 30 Jun 2026 19:02:00 -0700
Subject: [PATCH 45/51] feat(proxy): support object_permission in
 default_key_generate_params (#31776)

* feat(proxy): support object_permission in default_key_generate_params

default_key_generate_params filled in a fixed whitelist of scalar fields
plus a full-replace for models/metadata, but never touched object_permission,
so admins had no way to set a default (e.g. mcp_tool_search_enabled,
vector_stores) applied to every new key. Merge object_permission field-by-field
instead of replacing it wholesale, so a caller-supplied field (e.g. mcp_servers)
is preserved alongside defaulted fields the caller left unset.

* ci: retrigger proxy_pass_through_endpoint_tests (suspected flake, unrelated to this PR's diff)

* fix(proxy): apply default object_permission after team-scope validation

Injecting the default before validate_key_vector_stores_against_team /
validate_key_search_tools_against_team ran meant a default containing a
team-scoped field (e.g. vector_stores) looked like a caller-requested
permission, turning ordinary non-admin personal key creation into a 403.
Merge the default into data_json after those checks instead, and guard
against a non-dict default value.
---
 .../key_management_endpoints.py               |  17 ++
 .../test_key_management_endpoints.py          | 258 ++++++++++++++++++
 2 files changed, 275 insertions(+)

diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py
index 4106eae606c..523cb9b74e5 100644
--- a/litellm/proxy/management_endpoints/key_management_endpoints.py
+++ b/litellm/proxy/management_endpoints/key_management_endpoints.py
@@ -965,6 +965,23 @@ async def _common_key_generation_helper(
         is_proxy_admin=_is_proxy_admin_caller,
     )
 
+    # Merge default_key_generate_params.object_permission in *after* the team-scope
+    # checks above, so an admin-configured default (e.g. vector_stores, search_tools)
+    # is never mistaken for a caller-requested permission and rejected by those
+    # non-admin/no-team checks. Only fields the caller left unset are filled in.
+    _default_object_permission = (
+        litellm.default_key_generate_params.get("object_permission")
+        if litellm.default_key_generate_params is not None
+        else None
+    )
+    if isinstance(_default_object_permission, dict):
+        _caller_object_permission = data_json.get("object_permission")
+        if _caller_object_permission is None:
+            data_json["object_permission"] = dict(_default_object_permission)
+        elif isinstance(_caller_object_permission, dict):
+            for _op_field, _op_default_value in _default_object_permission.items():
+                _caller_object_permission.setdefault(_op_field, _op_default_value)
+
     data_json = await _set_object_permission(
         data_json=data_json,
         prisma_client=prisma_client,
diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
index 04048020e18..ae62be8799f 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
@@ -7781,6 +7781,264 @@ async def test_default_key_generate_params_duration(monkeypatch):
         litellm.default_key_generate_params = original_value
 
 
+async def test_default_key_generate_params_object_permission_applied_when_absent(
+    monkeypatch,
+):
+    """
+    default_key_generate_params.object_permission is applied to a key that
+    doesn't specify object_permission at all.
+    """
+    import litellm
+
+    mock_prisma_client = AsyncMock()
+    mock_insert_data = AsyncMock(
+        return_value=MagicMock(
+            token="hashed_token_123", litellm_budget_table=None, object_permission=None
+        )
+    )
+    mock_prisma_client.insert_data = mock_insert_data
+    mock_prisma_client.db = MagicMock()
+    mock_prisma_client.db.litellm_verificationtoken = MagicMock()
+    mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(
+        return_value=None
+    )
+    mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(
+        return_value=[]
+    )
+    mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0)
+    mock_prisma_client.db.litellm_verificationtoken.update = AsyncMock(
+        return_value=MagicMock(
+            token="hashed_token_123", litellm_budget_table=None, object_permission=None
+        )
+    )
+    mock_prisma_client.db.litellm_objectpermissiontable = MagicMock()
+    mock_prisma_client.db.litellm_objectpermissiontable.create = AsyncMock(
+        return_value=MagicMock(object_permission_id="objperm-1")
+    )
+
+    monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
+
+    original_value = litellm.default_key_generate_params
+    litellm.default_key_generate_params = {
+        "object_permission": {"vector_stores": ["default-vs"]}
+    }
+
+    try:
+        request = GenerateKeyRequest()  # No object_permission specified
+        await _common_key_generation_helper(
+            data=request,
+            user_api_key_dict=UserAPIKeyAuth(
+                user_role=LitellmUserRoles.PROXY_ADMIN,
+                api_key="sk-1234",
+                user_id="1234",
+            ),
+            litellm_changed_by=None,
+            team_table=None,
+        )
+
+        created_data = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"]
+        assert created_data["vector_stores"] == ["default-vs"]
+    finally:
+        litellm.default_key_generate_params = original_value
+
+
+async def test_default_key_generate_params_object_permission_merges_partial(
+    monkeypatch,
+):
+    """
+    default_key_generate_params.object_permission fills only the fields the
+    caller left unset - an explicitly supplied field (agents here) is
+    preserved alongside the defaulted field (vector_stores).
+    """
+    import litellm
+    from litellm.proxy._types import LiteLLM_ObjectPermissionBase
+
+    mock_prisma_client = AsyncMock()
+    mock_insert_data = AsyncMock(
+        return_value=MagicMock(
+            token="hashed_token_123", litellm_budget_table=None, object_permission=None
+        )
+    )
+    mock_prisma_client.insert_data = mock_insert_data
+    mock_prisma_client.db = MagicMock()
+    mock_prisma_client.db.litellm_verificationtoken = MagicMock()
+    mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(
+        return_value=None
+    )
+    mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(
+        return_value=[]
+    )
+    mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0)
+    mock_prisma_client.db.litellm_verificationtoken.update = AsyncMock(
+        return_value=MagicMock(
+            token="hashed_token_123", litellm_budget_table=None, object_permission=None
+        )
+    )
+    mock_prisma_client.db.litellm_objectpermissiontable = MagicMock()
+    mock_prisma_client.db.litellm_objectpermissiontable.create = AsyncMock(
+        return_value=MagicMock(object_permission_id="objperm-2")
+    )
+
+    monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
+
+    original_value = litellm.default_key_generate_params
+    litellm.default_key_generate_params = {
+        "object_permission": {"vector_stores": ["default-vs"]}
+    }
+
+    try:
+        request = GenerateKeyRequest(
+            object_permission=LiteLLM_ObjectPermissionBase(agents=["agent-1"])
+        )
+        await _common_key_generation_helper(
+            data=request,
+            user_api_key_dict=UserAPIKeyAuth(
+                user_role=LitellmUserRoles.PROXY_ADMIN,
+                api_key="sk-1234",
+                user_id="1234",
+            ),
+            litellm_changed_by=None,
+            team_table=None,
+        )
+
+        created_data = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"]
+        assert created_data["agents"] == ["agent-1"]
+        assert created_data["vector_stores"] == ["default-vs"]
+    finally:
+        litellm.default_key_generate_params = original_value
+
+
+async def test_default_key_generate_params_object_permission_does_not_override_explicit(
+    monkeypatch,
+):
+    """
+    A field the caller explicitly set on object_permission must win over the
+    same field in default_key_generate_params.
+    """
+    import litellm
+    from litellm.proxy._types import LiteLLM_ObjectPermissionBase
+
+    mock_prisma_client = AsyncMock()
+    mock_insert_data = AsyncMock(
+        return_value=MagicMock(
+            token="hashed_token_123", litellm_budget_table=None, object_permission=None
+        )
+    )
+    mock_prisma_client.insert_data = mock_insert_data
+    mock_prisma_client.db = MagicMock()
+    mock_prisma_client.db.litellm_verificationtoken = MagicMock()
+    mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(
+        return_value=None
+    )
+    mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(
+        return_value=[]
+    )
+    mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0)
+    mock_prisma_client.db.litellm_verificationtoken.update = AsyncMock(
+        return_value=MagicMock(
+            token="hashed_token_123", litellm_budget_table=None, object_permission=None
+        )
+    )
+    mock_prisma_client.db.litellm_objectpermissiontable = MagicMock()
+    mock_prisma_client.db.litellm_objectpermissiontable.create = AsyncMock(
+        return_value=MagicMock(object_permission_id="objperm-3")
+    )
+
+    monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
+
+    original_value = litellm.default_key_generate_params
+    litellm.default_key_generate_params = {
+        "object_permission": {"vector_stores": ["default-vs"]}
+    }
+
+    try:
+        request = GenerateKeyRequest(
+            object_permission=LiteLLM_ObjectPermissionBase(
+                vector_stores=["explicit-vs"]
+            )
+        )
+        await _common_key_generation_helper(
+            data=request,
+            user_api_key_dict=UserAPIKeyAuth(
+                user_role=LitellmUserRoles.PROXY_ADMIN,
+                api_key="sk-1234",
+                user_id="1234",
+            ),
+            litellm_changed_by=None,
+            team_table=None,
+        )
+
+        created_data = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"]
+        assert created_data["vector_stores"] == ["explicit-vs"]
+    finally:
+        litellm.default_key_generate_params = original_value
+
+
+async def test_default_key_generate_params_object_permission_not_rejected_for_non_admin_personal_key(
+    monkeypatch,
+):
+    """
+    Regression test: a default_key_generate_params.object_permission containing
+    a team-scoped field (vector_stores) must not turn ordinary non-admin
+    personal key creation into a 403. The default is merged in *after* the
+    caller-scope validation, so it is never mistaken for a caller-requested
+    permission.
+    """
+    import litellm
+
+    mock_prisma_client = AsyncMock()
+    mock_insert_data = AsyncMock(
+        return_value=MagicMock(
+            token="hashed_token_123", litellm_budget_table=None, object_permission=None
+        )
+    )
+    mock_prisma_client.insert_data = mock_insert_data
+    mock_prisma_client.db = MagicMock()
+    mock_prisma_client.db.litellm_verificationtoken = MagicMock()
+    mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(
+        return_value=None
+    )
+    mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(
+        return_value=[]
+    )
+    mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0)
+    mock_prisma_client.db.litellm_verificationtoken.update = AsyncMock(
+        return_value=MagicMock(
+            token="hashed_token_123", litellm_budget_table=None, object_permission=None
+        )
+    )
+    mock_prisma_client.db.litellm_objectpermissiontable = MagicMock()
+    mock_prisma_client.db.litellm_objectpermissiontable.create = AsyncMock(
+        return_value=MagicMock(object_permission_id="objperm-4")
+    )
+
+    monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
+
+    original_value = litellm.default_key_generate_params
+    litellm.default_key_generate_params = {
+        "object_permission": {"vector_stores": ["default-vs"]}
+    }
+
+    try:
+        request = GenerateKeyRequest(user_id="alice")  # No object_permission specified
+        response = await _common_key_generation_helper(
+            data=request,
+            user_api_key_dict=UserAPIKeyAuth(
+                user_role=LitellmUserRoles.INTERNAL_USER,
+                api_key="sk-alice",
+                user_id="alice",
+            ),
+            litellm_changed_by=None,
+            team_table=None,
+        )
+
+        assert response is not None
+        created_data = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"]
+        assert created_data["vector_stores"] == ["default-vs"]
+    finally:
+        litellm.default_key_generate_params = original_value
+
+
 @pytest.mark.asyncio
 async def test_build_key_filter_member_team_service_accounts():
     """

From 50b936c75e8d7066968b8b6e31b73969d95790ae Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrish+github@berri.ai>
Date: Tue, 30 Jun 2026 19:19:34 -0700
Subject: [PATCH 46/51] feat(guardrails/headroom): add CCR
 (compress-cache-retrieve) via agentic loop (#31681)

* feat(guardrails/headroom): add CCR (compress-cache-retrieve) support via agentic loop

When Headroom's /v1/compress returns messages containing hash markers
(hash=[a-f0-9]{24}), inject a headroom_retrieve tool into the request.
When the LLM calls that tool, intercept via async_should_run_agentic_loop
and async_build_agentic_loop_plan, call GET /v1/retrieve/{hash} on the
Headroom sidecar, and replay the LLM with the original content as a tool
result -- all transparent to the caller.

* style: run ruff format on headroom guardrail and tests

* fix(guardrails/headroom): detect headroom_retrieve calls in both OpenAI and Anthropic response formats

* test(guardrails/headroom): add test for Anthropic content block format detection in CCR loop

* ci: trigger CI checks

* fix(guardrails/headroom): replace List/Dict with list/dict to fix UP006 ruff violations

* fix(guardrails/headroom): replace except Exception with except ValueError to fix BLE001

* fix(guardrails/headroom): add Responses API output format detection for CCR tool calls

* refactor(guardrails/headroom): extract format-specific helpers to fix C901 complexity

* fix(guardrails/headroom): scope CCR retrieval to hashes produced by current request

Previously any LLM-supplied hash in a headroom_retrieve tool call was
forwarded to the Headroom retrieve API, letting a crafted tool call
fetch arbitrary cached content. Validate the hash against the set
produced by compressing the current request's messages before calling
retrieve.

* fix(guardrails/headroom): track issued hashes server-side, fix Responses API replay shape

Hash validation now also checks an in-memory cache of hashes actually
returned by /v1/compress, not just whether the hash text appears
somewhere in the request's messages. The message-text check alone is
forgeable: an attacker can plant a hash-shaped string in their own
prompt and have it treated as valid.

Responses API follow-up now emits function_call/function_call_output
items keyed by call_id instead of chat-style assistant/tool messages,
since the Responses API does not accept the latter as input. Also
fixes call_id/id field priority when extracting tool calls from
Responses API output, since call_id (not id) is what must match
between the function_call and its output.

* fix(guardrails/headroom): drop redundant quoted type annotations

UP037 flags quotes on annotations that are already lazily evaluated
via `from __future__ import annotations`.

* test(guardrails/headroom): add missing pytest.mark.asyncio decorators

Functional under asyncio_mode=auto, but every other async test in the
file has the decorator for consistency.

* fix(guardrails/headroom): scope CCR hashes per call_id, fix Anthropic replay shape

Two real gaps found in review:

1. The instance-wide issued-hash cache combined with a message-text
   check did not actually scope retrieval to the request that produced
   the hash. A hash issued for request A stays in the shared cache
   until TTL expiry, and the message-text check is satisfied by any
   request whose own messages happen to echo that hash string. Request
   B could plant A's hash in its own prompt and retrieve A's content.

   Fixed by keying the issued-hash cache by litellm_call_id, matching
   the pattern already used in compression_interception: a hash is
   only honored when it was issued under the exact call_id resolving
   for the current request.

2. The Anthropic Messages replay path fell through to the chat-style
   assistant/tool-message builder, which Anthropic does not accept.
   Anthropic requires the tool_use block echoed in an assistant message
   paired with a tool_result block in a user message, keyed by
   tool_use_id. Added a dedicated branch for this shape.

* docs: note proactive API-fragmentation helper convention

Add a bullet to the coding-conventions list: look for or add a shared
helper when logic branches on API surface (chat completions vs
Anthropic Messages vs Responses API), instead of duplicating
format-detection per module.

* fix(guardrails/headroom): fix Anthropic tool-shape detection, extract shared cross-API tool util

Live e2e testing against the real Anthropic API surfaced two bugs the
mocked unit tests couldn't catch because they used MagicMock responses
instead of realistic response shapes:

1. has_headroom_retrieve_tool only recognized OpenAI-shaped function
   tools. By the time an Anthropic Messages response reaches the
   agentic-loop gate, the tool this guardrail injected has already been
   transformed into Anthropic's native shape (type: "custom", top-level
   "name"), so the gate never fired for real Anthropic requests.

2. AnthropicMessagesResponse is a TypedDict, so real responses are
   plain dicts at runtime, not objects with attribute access. The
   extractors and format detectors used bare getattr(), which silently
   returns nothing for dict responses instead of reading the actual
   key.

Extracted the cross-API-surface tool-call extraction and tool-presence
check into litellm/litellm_core_utils/prompt_templates/factory.py
(get_tool_calls_from_response, has_tool_with_name) so this format
fragmentation is handled in one place instead of being duplicated
per-guardrail, and reused the existing repair-aware
parse_tool_call_arguments from common_utils instead of a naive
json.loads. headroom.py now delegates to these shared helpers.

Confirmed live against the real Anthropic API: the retrieve loop now
fires and successfully retrieves the correct hash's content through
the full compress -> tool-call -> retrieve -> replay round-trip.

* fix(guardrails/headroom): fix ruff-strict UP006/I001 budget violations

Use lowercase list/dict generics in the new factory.py tool-call
helpers instead of typing.List/Dict, drop the now-unused Tuple import
in headroom.py, and reorder the new factory import ahead of the
llms.custom_httpx import to satisfy import sorting.

* fix(guardrails/headroom): match Anthropic tools without a type field

Anthropic's documented client tool format is just name + input_schema;
type: "custom" is only one possible value, not a requirement. Match
any non-OpenAI-shaped tool on its top-level name instead of requiring
type == "custom".
---
 CLAUDE.md                                     |   1 +
 .../prompt_templates/factory.py               | 145 +++-
 .../guardrail_hooks/headroom/headroom.py      | 355 ++++++++-
 tests/llm_translation/test_prompt_factory.py  |  99 +++
 .../guardrail_hooks/test_headroom.py          | 741 +++++++++++++++++-
 5 files changed, 1319 insertions(+), 22 deletions(-)

diff --git a/CLAUDE.md b/CLAUDE.md
index 0cd1605b1b2..83bf3e22d27 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -69,6 +69,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
 - No monster files or god objects
 - No file sprawl: deliberate file and folder structure
 - Standard over hand-rolled: use the official SDK or a library where one exists; where none does, follow industry standards instead of inventing local conventions
+- API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module; if none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration
 
 Follow conventional commits for commit names and PR titles
 
diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py
index 1f0df51d7de..c2448430387 100644
--- a/litellm/litellm_core_utils/prompt_templates/factory.py
+++ b/litellm/litellm_core_utils/prompt_templates/factory.py
@@ -6,7 +6,7 @@ import mimetypes
 import re
 import xml.etree.ElementTree as ET
 from enum import Enum
-from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast, overload
+from typing import Any, Dict, List, Optional, Set, Tuple, TypedDict, Union, cast, overload
 
 from jinja2.sandbox import ImmutableSandboxedEnvironment
 
@@ -5322,3 +5322,146 @@ def get_attribute_or_key(tool_or_function, attribute, default=None):
     if hasattr(tool_or_function, attribute):
         return getattr(tool_or_function, attribute)
     return tool_or_function.get(attribute, default)
+
+
+class NormalizedToolCall(TypedDict):
+    id: Optional[str]
+    name: Optional[str]
+    arguments: dict[str, Any]
+
+
+def _parse_tool_call_arguments(raw: Any, tool_name: Optional[str], context: str) -> dict[str, Any]:
+    # Anthropic's tool_use blocks already carry a parsed dict in "input";
+    # chat completions and the Responses API carry a JSON string that may be
+    # truncated by the model, so route those through the repair-aware parser.
+    if isinstance(raw, dict):
+        return raw
+    if not isinstance(raw, str):
+        return {}
+    from litellm.litellm_core_utils.prompt_templates.common_utils import (
+        parse_tool_call_arguments,
+    )
+
+    try:
+        parsed = parse_tool_call_arguments(raw, tool_name=tool_name, context=context)
+    except ValueError as e:
+        verbose_logger.warning("Failed to parse tool call arguments: %s", e)
+        return {}
+    return parsed if isinstance(parsed, dict) else {}
+
+
+def _tool_calls_from_chat_completion_response(response: Any) -> list[NormalizedToolCall]:
+    choices = get_attribute_or_key(response, "choices", None)
+    if not (isinstance(choices, list) and choices):
+        return []
+    message = get_attribute_or_key(choices[0], "message", None)
+    tool_calls = get_attribute_or_key(message, "tool_calls", None) if message else None
+    if not isinstance(tool_calls, list):
+        return []
+    result: list[NormalizedToolCall] = []
+    for tc in tool_calls:
+        fn = get_attribute_or_key(tc, "function", None)
+        if fn is None:
+            continue
+        name = get_attribute_or_key(fn, "name")
+        result.append(
+            NormalizedToolCall(
+                id=get_attribute_or_key(tc, "id"),
+                name=name,
+                arguments=_parse_tool_call_arguments(
+                    get_attribute_or_key(fn, "arguments", "{}"),
+                    tool_name=name,
+                    context="chat completions",
+                ),
+            )
+        )
+    return result
+
+
+def _tool_calls_from_responses_api_response(response: Any) -> list[NormalizedToolCall]:
+    output = get_attribute_or_key(response, "output", None)
+    if not isinstance(output, list):
+        return []
+    result: list[NormalizedToolCall] = []
+    for item in output:
+        if get_attribute_or_key(item, "type") != "function_call":
+            continue
+        name = get_attribute_or_key(item, "name")
+        result.append(
+            NormalizedToolCall(
+                id=get_attribute_or_key(item, "call_id") or get_attribute_or_key(item, "id"),
+                name=name,
+                arguments=_parse_tool_call_arguments(
+                    get_attribute_or_key(item, "arguments", "{}"),
+                    tool_name=name,
+                    context="responses API",
+                ),
+            )
+        )
+    return result
+
+
+def _tool_calls_from_anthropic_messages_response(response: Any) -> list[NormalizedToolCall]:
+    content = get_attribute_or_key(response, "content", None)
+    if not isinstance(content, list):
+        return []
+    result: list[NormalizedToolCall] = []
+    for block in content:
+        if get_attribute_or_key(block, "type") != "tool_use":
+            continue
+        raw_input = get_attribute_or_key(block, "input", {})
+        result.append(
+            NormalizedToolCall(
+                id=get_attribute_or_key(block, "id"),
+                name=get_attribute_or_key(block, "name"),
+                arguments=raw_input if isinstance(raw_input, dict) else {},
+            )
+        )
+    return result
+
+
+def get_tool_calls_from_response(response: Any) -> list[NormalizedToolCall]:
+    """
+    Extract tool/function calls from a response object into a normalized
+    ``{"id", "name", "arguments"}`` shape, regardless of which API surface
+    produced it: chat completions (``choices[].message.tool_calls``),
+    the Responses API (``output`` items of type ``function_call``), or the
+    Anthropic Messages API (``content`` blocks of type ``tool_use``).
+
+    Callers that only care about a specific tool should filter the result by
+    ``name`` themselves -- this returns every tool call found.
+    """
+    for extractor in (
+        _tool_calls_from_chat_completion_response,
+        _tool_calls_from_responses_api_response,
+        _tool_calls_from_anthropic_messages_response,
+    ):
+        tool_calls = extractor(response)
+        if tool_calls:
+            return tool_calls
+    return []
+
+
+def has_tool_with_name(tools: Any, tool_name: str) -> bool:
+    """
+    Check whether a tools list (as sent to an LLM) includes a tool with the
+    given name, regardless of shape: OpenAI-style function tools
+    (``{"type": "function", "function": {"name": ...}}``) or Anthropic's
+    native tool shape (a top-level ``"name"``, e.g.
+    ``{"name": ..., "input_schema": ...}``). Anthropic's documented client
+    tool format doesn't require a ``"type"`` key at all -- ``"custom"`` is
+    only one of several possible values -- so any non-OpenAI-shaped tool is
+    matched on its top-level ``"name"``.
+    """
+    if not isinstance(tools, list):
+        return False
+    for tool in tools:
+        if not isinstance(tool, dict):
+            continue
+        function = tool.get("function")
+        if tool.get("type") == "function" and isinstance(function, dict):
+            if function.get("name") == tool_name:
+                return True
+        elif tool.get("name") == tool_name:
+            return True
+    return False
diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
index 2228ccf3997..4badb48e2eb 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
@@ -1,6 +1,10 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Literal
+import json
+import re
+import time
+import uuid
+from typing import TYPE_CHECKING, Any, Literal, Optional
 
 import httpx
 from fastapi import HTTPException
@@ -12,12 +16,18 @@ from litellm.integrations.custom_guardrail import (
     CustomGuardrail,
     log_guardrail_information,
 )
+from litellm.litellm_core_utils.prompt_templates.factory import (
+    get_attribute_or_key,
+    get_tool_calls_from_response,
+    has_tool_with_name,
+)
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,  # pyright: ignore[reportUnknownVariableType]
     httpxSpecialProvider,
 )
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.guardrails import GuardrailEventHooks, Mode
+from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch
 from litellm.types.utils import GenericGuardrailAPIInputs
 
 if TYPE_CHECKING:
@@ -25,6 +35,9 @@ if TYPE_CHECKING:
     from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
 
 BYPASS_HEADER = "x-headroom-bypass"
+HEADROOM_RETRIEVE_TOOL_NAME = "headroom_retrieve"
+_HASH_PATTERN = re.compile(r"hash=([a-f0-9]{24})")
+_HASH_CACHE_TTL_SECONDS = 15 * 60
 
 
 def _is_str_object_dict(value: object) -> TypeGuard[dict[str, object]]:  # guard-ok: isinstance narrows correctly; predicate is trivially correct  # fmt: skip
@@ -35,6 +48,163 @@ def _is_object_list(value: object) -> TypeGuard[list[object]]:  # guard-ok: isin
     return isinstance(value, list)
 
 
+def extract_hashes_from_messages(messages: list[dict[str, object]]) -> list[str]:
+    hashes: list[str] = []
+    for msg in messages:
+        content = msg.get("content")
+        if isinstance(content, str):
+            hashes.extend(_HASH_PATTERN.findall(content))
+        elif isinstance(content, list):
+            for block in content:
+                if isinstance(block, dict):
+                    text = block.get("text")
+                    if isinstance(text, str):
+                        hashes.extend(_HASH_PATTERN.findall(text))
+    return hashes
+
+
+def _build_headroom_retrieve_tool() -> dict[str, object]:
+    return {
+        "type": "function",
+        "function": {
+            "name": HEADROOM_RETRIEVE_TOOL_NAME,
+            "description": (
+                "Retrieve original content that was compressed by Headroom. "
+                "Call this when you encounter a compression marker containing a hash."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "hash": {
+                        "type": "string",
+                        "description": "The 24-character hex hash from the compression marker.",
+                    },
+                    "query": {
+                        "type": "string",
+                        "description": "Optional search query for BM25-ranked retrieval.",
+                    },
+                },
+                "required": ["hash"],
+            },
+        },
+    }
+
+
+def _resolve_call_id(logging_obj: object, request_state: dict[str, object]) -> Optional[str]:
+    """Resolve the litellm_call_id shared by a request's pre-call hook and its
+    agentic-loop hooks, so CCR hash validation can be scoped per call instead
+    of trusting any hash-shaped string that shows up in message text."""
+    logging_call_id = getattr(logging_obj, "litellm_call_id", None)
+    if isinstance(logging_call_id, str) and logging_call_id:
+        return logging_call_id
+    kwargs_call_id = request_state.get("litellm_call_id")
+    return kwargs_call_id if isinstance(kwargs_call_id, str) else None
+
+
+def has_headroom_retrieve_tool(tools: object) -> bool:
+    return has_tool_with_name(tools, HEADROOM_RETRIEVE_TOOL_NAME)
+
+
+def _extract_headroom_tool_calls(response: object) -> list[dict[str, object]]:
+    return [
+        {"id": tc["id"], "type": "function", "name": tc["name"], "arguments": tc["arguments"]}
+        for tc in get_tool_calls_from_response(response)
+        if tc["name"] == HEADROOM_RETRIEVE_TOOL_NAME
+    ]
+
+
+def _build_assistant_message_from_response(response: object) -> dict[str, object]:
+    choices = getattr(response, "choices", None)
+    if not isinstance(choices, list) or not choices:
+        return {"role": "assistant", "content": None, "tool_calls": []}
+    message = getattr(choices[0], "message", None)
+    if message is None:
+        return {"role": "assistant", "content": None, "tool_calls": []}
+    content = getattr(message, "content", None)
+    tool_calls = getattr(message, "tool_calls", None)
+    raw_tool_calls: list[dict[str, object]] = []
+    if isinstance(tool_calls, list):
+        for tc in tool_calls:
+            fn = getattr(tc, "function", None)
+            raw_tool_calls.append(
+                {
+                    "id": getattr(tc, "id", None),
+                    "type": "function",
+                    "function": {
+                        "name": getattr(fn, "name", None) if fn else None,
+                        "arguments": getattr(fn, "arguments", "{}") if fn else "{}",
+                    },
+                }
+            )
+    return {"role": "assistant", "content": content, "tool_calls": raw_tool_calls}
+
+
+def _is_responses_api_response(response: object) -> bool:
+    # Real response objects can be plain dicts at runtime (e.g. TypedDict-based
+    # response types), so getattr alone would silently miss the key -- use the
+    # same dict-or-object accessor as the tool-call extractors.
+    return isinstance(get_attribute_or_key(response, "output", None), list)
+
+
+def _is_anthropic_messages_response(response: object) -> bool:
+    return isinstance(get_attribute_or_key(response, "content", None), list)
+
+
+def _build_anthropic_followup_messages(
+    retrieved: list[tuple[dict[str, object], str]],
+) -> list[dict[str, object]]:
+    """Build Anthropic Messages API follow-up messages for a tool round-trip.
+
+    Anthropic requires the tool_use block to be echoed back in an assistant
+    message, paired with a tool_result block in a user message keyed by the
+    same tool_use_id -- it does not accept chat-style tool-role messages.
+    """
+    assistant_message: dict[str, object] = {
+        "role": "assistant",
+        "content": [
+            {
+                "type": "tool_use",
+                "id": tool_call.get("id"),
+                "name": tool_call.get("name"),
+                "input": tool_call.get("arguments", {}),
+            }
+            for tool_call, _ in retrieved
+        ],
+    }
+    user_message: dict[str, object] = {
+        "role": "user",
+        "content": [
+            {"type": "tool_result", "tool_use_id": tool_call.get("id"), "content": content}
+            for tool_call, content in retrieved
+        ],
+    }
+    return [assistant_message, user_message]
+
+
+def _build_responses_followup_items(
+    retrieved: list[tuple[dict[str, object], str]],
+) -> list[dict[str, object]]:
+    """Build Responses API input items for a tool round-trip.
+
+    The Responses API does not accept chat-style assistant/tool messages as
+    follow-up input; it requires the model's function_call to be echoed back
+    paired with a function_call_output keyed by the same call_id.
+    """
+    items: list[dict[str, object]] = []
+    for tool_call, content in retrieved:
+        call_id = tool_call.get("id")
+        items.append(
+            {
+                "type": "function_call",
+                "call_id": call_id,
+                "name": tool_call.get("name"),
+                "arguments": json.dumps(tool_call.get("arguments", {})),
+            }
+        )
+        items.append({"type": "function_call_output", "call_id": call_id, "output": content})
+    return items
+
+
 class HeadroomGuardrail(CustomGuardrail):
     def __init__(
         self,
@@ -56,6 +226,7 @@ class HeadroomGuardrail(CustomGuardrail):
         self.async_handler = get_async_httpx_client(
             llm_provider=httpxSpecialProvider.GuardrailCallback,
         )
+        self._issued_hashes_by_call_id: dict[str, tuple[frozenset[str], float]] = {}
         super().__init__(  # pyright: ignore[reportUnknownMemberType]
             guardrail_name=guardrail_name,
             event_hook=event_hook,
@@ -72,6 +243,20 @@ class HeadroomGuardrail(CustomGuardrail):
         value = headers.get(BYPASS_HEADER)
         return str(value).lower() == "true"
 
+    def _request_headers(self) -> dict[str, str]:
+        headers: dict[str, str] = {"Content-Type": "application/json"}
+        if self.headroom_api_key:
+            headers["Authorization"] = f"Bearer {self.headroom_api_key}"
+        return headers
+
+    def _prune_expired_hashes(self) -> None:
+        now = time.monotonic()
+        self._issued_hashes_by_call_id = {
+            call_id: (hashes, expiry)
+            for call_id, (hashes, expiry) in self._issued_hashes_by_call_id.items()
+            if expiry > now
+        }
+
     async def _call_compress(
         self,
         messages: list[dict[str, object]],
@@ -81,15 +266,11 @@ class HeadroomGuardrail(CustomGuardrail):
         if model:
             payload["model"] = model
 
-        request_headers: dict[str, str] = {"Content-Type": "application/json"}
-        if self.headroom_api_key:
-            request_headers["Authorization"] = f"Bearer {self.headroom_api_key}"
-
         try:
             raw_response: HttpxResponse | None = await self.async_handler.post(  # pyright: ignore[reportUnknownMemberType]
                 url=f"{self.headroom_api_base}/v1/compress",
                 json=payload,
-                headers=request_headers,
+                headers=self._request_headers(),
             )
         except (httpx.ConnectError, httpx.TimeoutException, httpx.TransportError) as e:
             raise HTTPException(
@@ -118,7 +299,7 @@ class HeadroomGuardrail(CustomGuardrail):
 
         try:
             body: object = response.json()
-        except Exception:
+        except ValueError:
             raise HTTPException(
                 status_code=502,
                 detail={
@@ -163,6 +344,44 @@ class HeadroomGuardrail(CustomGuardrail):
         )
         return filtered
 
+    async def _call_retrieve(self, hash_value: str, query: str | None = None) -> str:
+        params: dict[str, str] = {}
+        if query:
+            params["query"] = query
+
+        try:
+            raw_response: HttpxResponse | None = await self.async_handler.get(  # pyright: ignore[reportUnknownMemberType]
+                url=f"{self.headroom_api_base}/v1/retrieve/{hash_value}",
+                params=params,
+                headers=self._request_headers(),
+            )
+        except (httpx.ConnectError, httpx.TimeoutException, httpx.TransportError) as e:
+            verbose_proxy_logger.warning("Headroom: retrieve failed for hash=%s: %s", hash_value, e)
+            return f"[Headroom: retrieval failed for hash={hash_value}]"
+
+        if raw_response is None or raw_response.status_code == 404:
+            return f"[Headroom: hash={hash_value} not found or expired]"
+
+        if raw_response.status_code != 200:
+            verbose_proxy_logger.warning(
+                "Headroom: retrieve returned %s for hash=%s",
+                raw_response.status_code,
+                hash_value,
+            )
+            return f"[Headroom: retrieval error {raw_response.status_code} for hash={hash_value}]"
+
+        try:
+            body: object = raw_response.json()
+        except ValueError:
+            return raw_response.text
+
+        if _is_str_object_dict(body):
+            original_content = body.get("original_content")
+            if isinstance(original_content, str):
+                return original_content
+
+        return str(body)
+
     @log_guardrail_information
     async def apply_guardrail(
         self,
@@ -192,7 +411,127 @@ class HeadroomGuardrail(CustomGuardrail):
             model=model if isinstance(model, str) else None,
         )
 
-        return {**inputs, "structured_messages": compressed}  # pyright: ignore[reportReturnType]
+        hashes = extract_hashes_from_messages(compressed)
+        if not hashes:
+            return {**inputs, "structured_messages": compressed}  # pyright: ignore[reportReturnType]
+
+        self._prune_expired_hashes()
+        call_id = _resolve_call_id(logging_obj, request_data)
+        if not call_id:
+            call_id = str(uuid.uuid4())
+            request_data["litellm_call_id"] = call_id
+        self._issued_hashes_by_call_id[call_id] = (frozenset(hashes), time.monotonic() + _HASH_CACHE_TTL_SECONDS)
+
+        existing_tools = inputs.get("tools")
+        retrieve_tool = _build_headroom_retrieve_tool()
+        if isinstance(existing_tools, list) and not has_headroom_retrieve_tool(existing_tools):
+            merged_tools: list[object] = list(existing_tools) + [retrieve_tool]
+        elif existing_tools is None:
+            merged_tools = [retrieve_tool]
+        else:
+            merged_tools = list(existing_tools) if isinstance(existing_tools, list) else [retrieve_tool]
+
+        return {**inputs, "structured_messages": compressed, "tools": merged_tools}  # pyright: ignore[reportReturnType]
+
+    async def async_should_run_agentic_loop(
+        self,
+        response: Any,
+        model: str,
+        messages: list[dict],
+        tools: Optional[list[dict]],
+        stream: bool,
+        custom_llm_provider: str,
+        kwargs: dict,
+    ) -> tuple[bool, dict]:
+        if not has_headroom_retrieve_tool(tools):
+            return False, {}
+
+        tool_calls = _extract_headroom_tool_calls(response)
+        if not tool_calls:
+            return False, {}
+
+        return True, {"tool_calls": tool_calls}
+
+    async def async_build_agentic_loop_plan(
+        self,
+        tools: dict,
+        model: str,
+        messages: list[dict],
+        response: Any,
+        anthropic_messages_provider_config: Any,
+        anthropic_messages_optional_request_params: dict,
+        logging_obj: Any,
+        stream: bool,
+        kwargs: dict,
+    ) -> AgenticLoopPlan:
+        tool_calls: list[dict[str, object]] = tools.get("tool_calls", [])  # type: ignore[assignment]
+
+        self._prune_expired_hashes()
+        call_id = _resolve_call_id(logging_obj, kwargs)
+        valid_hashes = self._issued_hashes_by_call_id.get(call_id, (frozenset(), 0.0))[0] if call_id else frozenset()
+
+        retrieved: list[tuple[dict[str, object], str]] = []
+        for tc in tool_calls:
+            arguments = tc.get("arguments", {})
+            hash_value = arguments.get("hash", "") if isinstance(arguments, dict) else ""
+            query = arguments.get("query") if isinstance(arguments, dict) else None
+            # A hash is only honored if it was issued by *this request's own*
+            # Headroom /v1/compress call, scoped by litellm_call_id. Scoping by
+            # message text alone is forgeable -- an attacker can plant a
+            # hash-shaped string in their own prompt, and a hash issued for one
+            # request would validate for any other request that echoes it back.
+            if str(hash_value) not in valid_hashes:
+                verbose_proxy_logger.warning(
+                    "Headroom CCR: rejecting hash=%s not produced by current request compression",
+                    hash_value,
+                )
+                content = f"[Headroom: hash={hash_value} was not produced by the current request]"
+            else:
+                content = await self._call_retrieve(
+                    hash_value=str(hash_value),
+                    query=str(query) if query else None,
+                )
+            verbose_proxy_logger.debug("Headroom CCR: retrieved hash=%s (%d chars)", hash_value, len(content))
+            retrieved.append((tc, content))
+
+        if _is_responses_api_response(response):
+            follow_up_messages = list(messages) + _build_responses_followup_items(retrieved)
+        elif _is_anthropic_messages_response(response):
+            follow_up_messages = list(messages) + _build_anthropic_followup_messages(retrieved)
+        else:
+            assistant_message = _build_assistant_message_from_response(response)
+            tool_results = [
+                {"role": "tool", "tool_call_id": tc.get("id"), "content": content} for tc, content in retrieved
+            ]
+            follow_up_messages = list(messages) + [assistant_message] + tool_results
+
+        max_tokens: Optional[int] = anthropic_messages_optional_request_params.get("max_tokens") or kwargs.get(
+            "max_tokens"
+        )
+        optional_params_without_max_tokens = {
+            k: v for k, v in anthropic_messages_optional_request_params.items() if k != "max_tokens"
+        }
+
+        full_model_name = model
+        if logging_obj is not None:
+            agentic_params = getattr(logging_obj, "model_call_details", {}).get("agentic_loop_params", {})
+            candidate = agentic_params.get("model", model)
+            if isinstance(candidate, str) and candidate:
+                full_model_name = candidate
+
+        return AgenticLoopPlan(
+            run_agentic_loop=True,
+            request_patch=AgenticLoopRequestPatch(
+                model=full_model_name,
+                messages=follow_up_messages,
+                max_tokens=max_tokens,
+                optional_params=optional_params_without_max_tokens,
+                kwargs={
+                    k: v for k, v in kwargs.items() if not k.startswith("_headroom") and k != "litellm_logging_obj"
+                },
+            ),
+            metadata={"tool_type": "headroom_ccr"},
+        )
 
     @staticmethod
     def get_config_model() -> type[GuardrailConfigModel[object]] | None:
diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py
index 05a58a135d2..ae215602e31 100644
--- a/tests/llm_translation/test_prompt_factory.py
+++ b/tests/llm_translation/test_prompt_factory.py
@@ -20,6 +20,8 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
     convert_to_anthropic_tool_invoke,
     convert_url_to_base64,
     create_anthropic_image_param,
+    get_tool_calls_from_response,
+    has_tool_with_name,
     llama_2_chat_pt,
     prompt_factory,
 )
@@ -2385,3 +2387,100 @@ def test_anthropic_messages_pt_list_content_with_thinking_preserves_order():
     # Verify signatures preserved in correct positions
     assert content[0]["signature"] == "sig_1"
     assert content[3]["signature"] == "sig_2"
+
+
+def test_get_tool_calls_from_response_chat_completions():
+    response = MagicMock()
+    response.output = None
+    response.content = None
+    tool_call = MagicMock()
+    tool_call.id = "call_abc"
+    tool_call.function.name = "my_tool"
+    tool_call.function.arguments = '{"x": 1}'
+    response.choices = [MagicMock(message=MagicMock(tool_calls=[tool_call]))]
+
+    result = get_tool_calls_from_response(response)
+
+    assert result == [{"id": "call_abc", "name": "my_tool", "arguments": {"x": 1}}]
+
+
+def test_get_tool_calls_from_response_responses_api():
+    response = MagicMock()
+    response.choices = None
+    response.content = None
+    response.output = [
+        {
+            "type": "function_call",
+            "id": "fc_1",
+            "call_id": "call_1",
+            "name": "my_tool",
+            "arguments": '{"x": 2}',
+        }
+    ]
+
+    result = get_tool_calls_from_response(response)
+
+    assert result == [{"id": "call_1", "name": "my_tool", "arguments": {"x": 2}}]
+
+
+def test_get_tool_calls_from_response_anthropic_messages():
+    response = MagicMock()
+    response.choices = None
+    response.output = None
+    response.content = [
+        {"type": "tool_use", "id": "toolu_1", "name": "my_tool", "input": {"x": 3}},
+    ]
+
+    result = get_tool_calls_from_response(response)
+
+    assert result == [{"id": "toolu_1", "name": "my_tool", "arguments": {"x": 3}}]
+
+
+def test_get_tool_calls_from_response_anthropic_messages_plain_dict():
+    # AnthropicMessagesResponse is a TypedDict -- real responses are plain
+    # dicts at runtime, not objects with attribute access. A MagicMock-only
+    # test would pass even if the extractor used bare getattr() and silently
+    # returned nothing for a real response.
+    response = {
+        "content": [
+            {"type": "tool_use", "id": "toolu_1", "name": "my_tool", "input": {"x": 3}},
+        ]
+    }
+
+    result = get_tool_calls_from_response(response)
+
+    assert result == [{"id": "toolu_1", "name": "my_tool", "arguments": {"x": 3}}]
+
+
+def test_get_tool_calls_from_response_no_tool_calls():
+    response = MagicMock()
+    response.choices = None
+    response.output = None
+    response.content = None
+
+    assert get_tool_calls_from_response(response) == []
+
+
+def test_has_tool_with_name_openai_function_shape():
+    tools = [{"type": "function", "function": {"name": "my_tool"}}]
+    assert has_tool_with_name(tools, "my_tool")
+    assert not has_tool_with_name(tools, "other_tool")
+
+
+def test_has_tool_with_name_anthropic_custom_shape():
+    tools = [{"type": "custom", "name": "my_tool", "input_schema": {}}]
+    assert has_tool_with_name(tools, "my_tool")
+    assert not has_tool_with_name(tools, "other_tool")
+
+
+def test_has_tool_with_name_anthropic_shape_without_type_field():
+    # Anthropic's documented client tool format is just name + input_schema;
+    # "type" isn't required at all (type: "custom" is only one possible value).
+    tools = [{"name": "my_tool", "input_schema": {}}]
+    assert has_tool_with_name(tools, "my_tool")
+    assert not has_tool_with_name(tools, "other_tool")
+
+
+def test_has_tool_with_name_not_a_list():
+    assert not has_tool_with_name(None, "my_tool")
+    assert not has_tool_with_name("not a list", "my_tool")
diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
index 66395035384..f5ce6cedf64 100644
--- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
+++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
@@ -8,15 +8,25 @@ Tests cover:
 - response-type input is passed through unchanged
 - /v1/compress HTTP error raises HTTPException
 - /v1/compress returning malformed JSON raises HTTPException
+- CCR: headroom_retrieve tool injected when compressed messages contain hashes
+- CCR: async_should_run_agentic_loop returns True when response has headroom_retrieve tool calls
+- CCR: async_build_agentic_loop_plan calls retrieve endpoint and builds follow-up messages
 """
 
+import json
+import time
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import httpx
 import pytest
 from fastapi import HTTPException
 
-from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import HeadroomGuardrail
+from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import (
+    HeadroomGuardrail,
+    extract_hashes_from_messages,
+    has_headroom_retrieve_tool,
+    HEADROOM_RETRIEVE_TOOL_NAME,
+)
 from litellm.types.utils import GenericGuardrailAPIInputs
 
 FAKE_API_BASE = "https://headroom.example.com"
@@ -30,6 +40,13 @@ COMPRESSED_MESSAGES = [
     {"role": "system", "content": "You are a helpful assistant."},
     {"role": "user", "content": "A" * 500},
 ]
+COMPRESSED_MESSAGES_WITH_HASH = [
+    {"role": "system", "content": "You are a helpful assistant."},
+    {
+        "role": "user",
+        "content": "Summary. Retrieve more: hash=b573993006976af767214fac",
+    },
+]
 
 
 def _make_guardrail(**kwargs) -> HeadroomGuardrail:
@@ -57,6 +74,36 @@ def _make_compress_response(messages: list, status: int = 200) -> MagicMock:
     return mock
 
 
+def _make_retrieve_response(original_content: str, status: int = 200) -> MagicMock:
+    mock = MagicMock()
+    mock.status_code = status
+    mock.json.return_value = {"original_content": original_content}
+    mock.text = original_content
+    return mock
+
+
+def _make_openai_response_with_tool_call(tool_name: str, arguments: dict, tool_id: str = "call_abc123") -> MagicMock:
+    fn = MagicMock()
+    fn.name = tool_name
+    fn.arguments = json.dumps(arguments)
+
+    tc = MagicMock()
+    tc.id = tool_id
+    tc.type = "function"
+    tc.function = fn
+
+    message = MagicMock()
+    message.content = None
+    message.tool_calls = [tc]
+
+    choice = MagicMock()
+    choice.message = message
+
+    response = MagicMock()
+    response.choices = [choice]
+    return response
+
+
 @pytest.fixture
 def guardrail() -> HeadroomGuardrail:
     return _make_guardrail()
@@ -87,6 +134,567 @@ async def test_apply_guardrail_compresses_and_returns_structured_messages(
     assert result.get("structured_messages") == COMPRESSED_MESSAGES
 
 
+@pytest.mark.asyncio
+async def test_apply_guardrail_injects_retrieve_tool_when_hashes_present(
+    guardrail: HeadroomGuardrail,
+):
+    inputs = GenericGuardrailAPIInputs(
+        texts=["A" * 5000],
+        structured_messages=ORIGINAL_MESSAGES,
+    )
+    mock_response = _make_compress_response(COMPRESSED_MESSAGES_WITH_HASH)
+
+    with patch.object(
+        guardrail.async_handler,
+        "post",
+        new_callable=AsyncMock,
+        return_value=mock_response,
+    ):
+        result = await guardrail.apply_guardrail(
+            inputs=inputs,
+            request_data={"model": "gpt-4o"},
+            input_type="request",
+        )
+
+    tools = result.get("tools")
+    assert tools is not None
+    assert has_headroom_retrieve_tool(tools)
+
+
+@pytest.mark.asyncio
+async def test_apply_guardrail_no_tool_injected_when_no_hashes(
+    guardrail: HeadroomGuardrail,
+):
+    inputs = GenericGuardrailAPIInputs(
+        texts=["A" * 5000],
+        structured_messages=ORIGINAL_MESSAGES,
+    )
+    mock_response = _make_compress_response(COMPRESSED_MESSAGES)
+
+    with patch.object(
+        guardrail.async_handler,
+        "post",
+        new_callable=AsyncMock,
+        return_value=mock_response,
+    ):
+        result = await guardrail.apply_guardrail(
+            inputs=inputs,
+            request_data={"model": "gpt-4o"},
+            input_type="request",
+        )
+
+    tools = result.get("tools")
+    assert not has_headroom_retrieve_tool(tools or [])
+
+
+@pytest.mark.asyncio
+async def test_apply_guardrail_preserves_existing_tools_when_injecting(
+    guardrail: HeadroomGuardrail,
+):
+    existing_tool = {"type": "function", "function": {"name": "my_tool", "parameters": {}}}
+    inputs = GenericGuardrailAPIInputs(
+        texts=["A" * 5000],
+        structured_messages=ORIGINAL_MESSAGES,
+        tools=[existing_tool],
+    )
+    mock_response = _make_compress_response(COMPRESSED_MESSAGES_WITH_HASH)
+
+    with patch.object(
+        guardrail.async_handler,
+        "post",
+        new_callable=AsyncMock,
+        return_value=mock_response,
+    ):
+        result = await guardrail.apply_guardrail(
+            inputs=inputs,
+            request_data={"model": "gpt-4o"},
+            input_type="request",
+        )
+
+    tools = result.get("tools")
+    assert tools is not None
+    assert isinstance(tools, list)
+    assert any(isinstance(t, dict) and t.get("function", {}).get("name") == "my_tool" for t in tools)
+    assert has_headroom_retrieve_tool(tools)
+
+
+@pytest.mark.asyncio
+async def test_async_should_run_agentic_loop_returns_true_for_retrieve_call(
+    guardrail: HeadroomGuardrail,
+):
+    retrieve_tool_def = [{"type": "function", "function": {"name": HEADROOM_RETRIEVE_TOOL_NAME}}]
+    response = _make_openai_response_with_tool_call(
+        tool_name=HEADROOM_RETRIEVE_TOOL_NAME,
+        arguments={"hash": "b573993006976af767214fac"},
+    )
+
+    should_run, ctx = await guardrail.async_should_run_agentic_loop(
+        response=response,
+        model="gpt-4o",
+        messages=[],
+        tools=retrieve_tool_def,
+        stream=False,
+        custom_llm_provider="openai",
+        kwargs={},
+    )
+
+    assert should_run is True
+    assert len(ctx["tool_calls"]) == 1
+    assert ctx["tool_calls"][0]["arguments"]["hash"] == "b573993006976af767214fac"
+
+
+@pytest.mark.asyncio
+async def test_async_should_run_agentic_loop_returns_false_without_retrieve_tool(
+    guardrail: HeadroomGuardrail,
+):
+    other_tools = [{"type": "function", "function": {"name": "other_tool"}}]
+    response = _make_openai_response_with_tool_call(
+        tool_name="other_tool",
+        arguments={},
+    )
+
+    should_run, _ = await guardrail.async_should_run_agentic_loop(
+        response=response,
+        model="gpt-4o",
+        messages=[],
+        tools=other_tools,
+        stream=False,
+        custom_llm_provider="openai",
+        kwargs={},
+    )
+
+    assert should_run is False
+
+
+@pytest.mark.asyncio
+async def test_async_should_run_agentic_loop_returns_false_when_no_retrieve_calls(
+    guardrail: HeadroomGuardrail,
+):
+    retrieve_tool_def = [{"type": "function", "function": {"name": HEADROOM_RETRIEVE_TOOL_NAME}}]
+    response = _make_openai_response_with_tool_call(
+        tool_name="some_other_function",
+        arguments={},
+    )
+
+    should_run, _ = await guardrail.async_should_run_agentic_loop(
+        response=response,
+        model="gpt-4o",
+        messages=[],
+        tools=retrieve_tool_def,
+        stream=False,
+        custom_llm_provider="openai",
+        kwargs={},
+    )
+
+    assert should_run is False
+
+
+@pytest.mark.asyncio
+async def test_async_build_agentic_loop_plan_calls_retrieve_and_builds_messages(
+    guardrail: HeadroomGuardrail,
+):
+    original_content = "This is the full compressed content."
+    mock_retrieve = _make_retrieve_response(original_content)
+
+    tool_calls = [
+        {
+            "id": "call_abc123",
+            "type": "function",
+            "name": HEADROOM_RETRIEVE_TOOL_NAME,
+            "arguments": {"hash": "b573993006976af767214fac"},
+        }
+    ]
+    response = _make_openai_response_with_tool_call(
+        tool_name=HEADROOM_RETRIEVE_TOOL_NAME,
+        arguments={"hash": "b573993006976af767214fac"},
+        tool_id="call_abc123",
+    )
+    messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}]
+    guardrail._issued_hashes_by_call_id["call-1"] = (
+        frozenset({"b573993006976af767214fac"}),
+        time.monotonic() + 999,
+    )
+
+    with patch.object(
+        guardrail.async_handler,
+        "get",
+        new_callable=AsyncMock,
+        return_value=mock_retrieve,
+    ) as mock_get:
+        plan = await guardrail.async_build_agentic_loop_plan(
+            tools={"tool_calls": tool_calls},
+            model="gpt-4o",
+            messages=messages,
+            response=response,
+            anthropic_messages_provider_config=None,
+            anthropic_messages_optional_request_params={},
+            logging_obj=None,
+            stream=False,
+            kwargs={"litellm_call_id": "call-1"},
+        )
+
+    assert plan.run_agentic_loop is True
+    assert plan.request_patch is not None
+
+    follow_up = plan.request_patch.messages
+    assert follow_up is not None
+
+    tool_result_message = next((m for m in follow_up if m.get("role") == "tool"), None)
+    assert tool_result_message is not None
+    assert tool_result_message["content"] == original_content
+    assert tool_result_message["tool_call_id"] == "call_abc123"
+
+    mock_get.assert_called_once()
+    call_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args.args[0]
+    assert "b573993006976af767214fac" in call_url
+
+
+@pytest.mark.asyncio
+async def test_async_build_agentic_loop_plan_handles_retrieve_404(
+    guardrail: HeadroomGuardrail,
+):
+    mock_retrieve = MagicMock()
+    mock_retrieve.status_code = 404
+
+    tool_calls = [
+        {
+            "id": "call_xyz",
+            "type": "function",
+            "name": HEADROOM_RETRIEVE_TOOL_NAME,
+            "arguments": {"hash": "deadbeef000000000000dead"},
+        }
+    ]
+    response = _make_openai_response_with_tool_call(
+        tool_name=HEADROOM_RETRIEVE_TOOL_NAME,
+        arguments={"hash": "deadbeef000000000000dead"},
+        tool_id="call_xyz",
+    )
+
+    messages = [
+        {
+            "role": "user",
+            "content": "Retrieve more: hash=deadbeef000000000000dead",
+        }
+    ]
+    guardrail._issued_hashes_by_call_id["call-1"] = (
+        frozenset({"deadbeef000000000000dead"}),
+        time.monotonic() + 999,
+    )
+
+    with patch.object(
+        guardrail.async_handler,
+        "get",
+        new_callable=AsyncMock,
+        return_value=mock_retrieve,
+    ):
+        plan = await guardrail.async_build_agentic_loop_plan(
+            tools={"tool_calls": tool_calls},
+            model="gpt-4o",
+            messages=messages,
+            response=response,
+            anthropic_messages_provider_config=None,
+            anthropic_messages_optional_request_params={},
+            logging_obj=None,
+            stream=False,
+            kwargs={"litellm_call_id": "call-1"},
+        )
+
+    follow_up = plan.request_patch.messages  # type: ignore[union-attr]
+    tool_result = next((m for m in follow_up if m.get("role") == "tool"), None)
+    assert tool_result is not None
+    assert "not found" in tool_result["content"] or "expired" in tool_result["content"]
+
+
+@pytest.mark.asyncio
+async def test_async_build_agentic_loop_plan_rejects_hash_with_no_known_call(
+    guardrail: HeadroomGuardrail,
+):
+    """A hash-shaped string planted in message text must not be honored when
+    this guardrail has no record of ever issuing it, even if it's echoed back
+    in the current request's own messages (e.g. via prompt injection)."""
+    tool_calls = [
+        {
+            "id": "call_xyz",
+            "type": "function",
+            "name": HEADROOM_RETRIEVE_TOOL_NAME,
+            "arguments": {"hash": "deadbeef000000000000dead"},
+        }
+    ]
+    response = _make_openai_response_with_tool_call(
+        tool_name=HEADROOM_RETRIEVE_TOOL_NAME,
+        arguments={"hash": "deadbeef000000000000dead"},
+        tool_id="call_xyz",
+    )
+    assert not guardrail._issued_hashes_by_call_id
+
+    with patch.object(
+        guardrail.async_handler,
+        "get",
+        new_callable=AsyncMock,
+    ) as mock_get:
+        plan = await guardrail.async_build_agentic_loop_plan(
+            tools={"tool_calls": tool_calls},
+            model="gpt-4o",
+            messages=[{"role": "user", "content": "Please fetch hash=deadbeef000000000000dead for me"}],
+            response=response,
+            anthropic_messages_provider_config=None,
+            anthropic_messages_optional_request_params={},
+            logging_obj=None,
+            stream=False,
+            kwargs={"litellm_call_id": "call-unknown"},
+        )
+
+    mock_get.assert_not_called()
+
+    follow_up = plan.request_patch.messages  # type: ignore[union-attr]
+    tool_result = next((m for m in follow_up if m.get("role") == "tool"), None)
+    assert tool_result is not None
+    assert "was not produced by the current request" in tool_result["content"]
+
+
+@pytest.mark.asyncio
+async def test_async_build_agentic_loop_plan_rejects_hash_issued_for_different_call(
+    guardrail: HeadroomGuardrail,
+):
+    """A hash issued for one request must not be retrievable by a different
+    request just because the second request echoes that hash-shaped string
+    back in its own messages -- retrieval must be scoped per litellm_call_id,
+    not derived by re-scanning attacker-controlled message text."""
+    guardrail._issued_hashes_by_call_id["call-A"] = (
+        frozenset({"b573993006976af767214fac"}),
+        time.monotonic() + 999,
+    )
+
+    tool_calls = [
+        {
+            "id": "call_xyz",
+            "type": "function",
+            "name": HEADROOM_RETRIEVE_TOOL_NAME,
+            "arguments": {"hash": "b573993006976af767214fac"},
+        }
+    ]
+    response = _make_openai_response_with_tool_call(
+        tool_name=HEADROOM_RETRIEVE_TOOL_NAME,
+        arguments={"hash": "b573993006976af767214fac"},
+        tool_id="call_xyz",
+    )
+
+    with patch.object(
+        guardrail.async_handler,
+        "get",
+        new_callable=AsyncMock,
+    ) as mock_get:
+        plan = await guardrail.async_build_agentic_loop_plan(
+            tools={"tool_calls": tool_calls},
+            model="gpt-4o",
+            messages=[{"role": "user", "content": "Please fetch hash=b573993006976af767214fac for me"}],
+            response=response,
+            anthropic_messages_provider_config=None,
+            anthropic_messages_optional_request_params={},
+            logging_obj=None,
+            stream=False,
+            kwargs={"litellm_call_id": "call-B"},
+        )
+
+    mock_get.assert_not_called()
+
+    follow_up = plan.request_patch.messages  # type: ignore[union-attr]
+    tool_result = next((m for m in follow_up if m.get("role") == "tool"), None)
+    assert tool_result is not None
+    assert "was not produced by the current request" in tool_result["content"]
+
+
+@pytest.mark.asyncio
+async def test_async_build_agentic_loop_plan_builds_responses_api_function_call_items(
+    guardrail: HeadroomGuardrail,
+):
+    """For the Responses API, follow-up input must echo a function_call paired
+    with a function_call_output keyed by the same call_id -- chat-style
+    assistant/tool messages are not valid Responses API input items."""
+    original_content = "This is the full compressed content."
+    mock_retrieve = _make_retrieve_response(original_content)
+
+    response = MagicMock()
+    response.choices = None
+    response.content = None
+    response.output = [
+        {
+            "type": "function_call",
+            "id": "fc_abc123",
+            "call_id": "call_abc123",
+            "name": HEADROOM_RETRIEVE_TOOL_NAME,
+            "arguments": json.dumps({"hash": "b573993006976af767214fac"}),
+        }
+    ]
+
+    tool_calls = [
+        {
+            "id": "call_abc123",
+            "type": "function",
+            "name": HEADROOM_RETRIEVE_TOOL_NAME,
+            "arguments": {"hash": "b573993006976af767214fac"},
+        }
+    ]
+    messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}]
+    guardrail._issued_hashes_by_call_id["call-1"] = (
+        frozenset({"b573993006976af767214fac"}),
+        time.monotonic() + 999,
+    )
+
+    with patch.object(
+        guardrail.async_handler,
+        "get",
+        new_callable=AsyncMock,
+        return_value=mock_retrieve,
+    ):
+        plan = await guardrail.async_build_agentic_loop_plan(
+            tools={"tool_calls": tool_calls},
+            model="gpt-4o",
+            messages=messages,
+            response=response,
+            anthropic_messages_provider_config=None,
+            anthropic_messages_optional_request_params={},
+            logging_obj=None,
+            stream=False,
+            kwargs={"litellm_call_id": "call-1"},
+        )
+
+    follow_up = plan.request_patch.messages  # type: ignore[union-attr]
+    assert all("role" not in item for item in follow_up if item not in messages)
+
+    function_call_item = next((i for i in follow_up if i.get("type") == "function_call"), None)
+    assert function_call_item is not None
+    assert function_call_item["call_id"] == "call_abc123"
+    assert function_call_item["name"] == HEADROOM_RETRIEVE_TOOL_NAME
+
+    output_item = next((i for i in follow_up if i.get("type") == "function_call_output"), None)
+    assert output_item is not None
+    assert output_item["call_id"] == "call_abc123"
+    assert output_item["output"] == original_content
+
+
+@pytest.mark.asyncio
+async def test_async_build_agentic_loop_plan_builds_anthropic_tool_result_messages(
+    guardrail: HeadroomGuardrail,
+):
+    """For the Anthropic Messages API, follow-up must echo a tool_use content
+    block in an assistant message paired with a tool_result content block in a
+    user message keyed by the same tool_use_id -- chat-style tool-role
+    messages are not valid Anthropic input.
+
+    AnthropicMessagesResponse is a TypedDict, so real responses are plain
+    dicts at runtime; a MagicMock response here would pass even if branch
+    selection used bare getattr() and silently fell through to the
+    chat-completions replay shape for every real Anthropic response.
+    """
+    original_content = "This is the full compressed content."
+    mock_retrieve = _make_retrieve_response(original_content)
+
+    response = {
+        "content": [
+            {
+                "type": "tool_use",
+                "id": "toolu_abc123",
+                "name": HEADROOM_RETRIEVE_TOOL_NAME,
+                "input": {"hash": "b573993006976af767214fac"},
+            }
+        ]
+    }
+
+    tool_calls = [
+        {
+            "id": "toolu_abc123",
+            "type": "function",
+            "name": HEADROOM_RETRIEVE_TOOL_NAME,
+            "arguments": {"hash": "b573993006976af767214fac"},
+        }
+    ]
+    messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}]
+    guardrail._issued_hashes_by_call_id["call-1"] = (
+        frozenset({"b573993006976af767214fac"}),
+        time.monotonic() + 999,
+    )
+
+    with patch.object(
+        guardrail.async_handler,
+        "get",
+        new_callable=AsyncMock,
+        return_value=mock_retrieve,
+    ):
+        plan = await guardrail.async_build_agentic_loop_plan(
+            tools={"tool_calls": tool_calls},
+            model="claude-sonnet-4-5",
+            messages=messages,
+            response=response,
+            anthropic_messages_provider_config=None,
+            anthropic_messages_optional_request_params={},
+            logging_obj=None,
+            stream=False,
+            kwargs={"litellm_call_id": "call-1"},
+        )
+
+    follow_up = plan.request_patch.messages  # type: ignore[union-attr]
+    assert all(m.get("role") != "tool" for m in follow_up)
+
+    assistant_message = next((m for m in follow_up if m.get("role") == "assistant"), None)
+    assert assistant_message is not None
+    tool_use_block = next((b for b in assistant_message["content"] if b.get("type") == "tool_use"), None)
+    assert tool_use_block is not None
+    assert tool_use_block["id"] == "toolu_abc123"
+
+    user_message = follow_up[-1]
+    assert user_message["role"] == "user"
+    tool_result_block = next((b for b in user_message["content"] if b.get("type") == "tool_result"), None)
+    assert tool_result_block is not None
+    assert tool_result_block["tool_use_id"] == "toolu_abc123"
+    assert tool_result_block["content"] == original_content
+
+
+def test_extract_hashes_from_messages_finds_hashes():
+    messages = [
+        {"role": "user", "content": "Retrieve more: hash=b573993006976af767214fac"},
+        {"role": "assistant", "content": "Also: hash=aabbccdd001122334455aabb"},
+    ]
+    hashes = extract_hashes_from_messages(messages)
+    assert "b573993006976af767214fac" in hashes
+    assert "aabbccdd001122334455aabb" in hashes
+
+
+def test_extract_hashes_from_messages_ignores_short_hashes():
+    messages = [{"role": "user", "content": "hash=tooshort"}]
+    hashes = extract_hashes_from_messages(messages)
+    assert not hashes
+
+
+def test_extract_hashes_from_list_content_blocks():
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "hash=b573993006976af767214fac found here"},
+            ],
+        }
+    ]
+    hashes = extract_hashes_from_messages(messages)
+    assert "b573993006976af767214fac" in hashes
+
+
+def test_has_headroom_retrieve_tool_recognizes_anthropic_native_shape():
+    """By the time an Anthropic Messages API response reaches the agentic-loop
+    gate, the OpenAI-shaped tool this guardrail injects (type: "function")
+    has already been transformed into Anthropic's native tool shape
+    (type: "custom", top-level "name", no nested "function" object)."""
+    anthropic_native_tools = [
+        {
+            "type": "custom",
+            "name": HEADROOM_RETRIEVE_TOOL_NAME,
+            "input_schema": {"type": "object", "properties": {"hash": {"type": "string"}}},
+        }
+    ]
+    assert has_headroom_retrieve_tool(anthropic_native_tools)
+    assert not has_headroom_retrieve_tool([{"type": "custom", "name": "some_other_tool"}])
+
+
 @pytest.mark.asyncio
 async def test_apply_guardrail_bypass_header_skips_compression(
     guardrail: HeadroomGuardrail,
@@ -97,9 +705,7 @@ async def test_apply_guardrail_bypass_header_skips_compression(
     )
     request_data = {"proxy_server_request": {"headers": {"x-headroom-bypass": "true"}}}
 
-    with patch.object(
-        guardrail.async_handler, "post", new_callable=AsyncMock
-    ) as mock_post:
+    with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post:
         result = await guardrail.apply_guardrail(
             inputs=inputs,
             request_data=request_data,
@@ -119,9 +725,7 @@ async def test_apply_guardrail_response_type_passthrough(
         structured_messages=ORIGINAL_MESSAGES,
     )
 
-    with patch.object(
-        guardrail.async_handler, "post", new_callable=AsyncMock
-    ) as mock_post:
+    with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post:
         result = await guardrail.apply_guardrail(
             inputs=inputs,
             request_data={},
@@ -138,9 +742,7 @@ async def test_apply_guardrail_empty_structured_messages_passthrough(
 ):
     inputs = GenericGuardrailAPIInputs(texts=["hello"])
 
-    with patch.object(
-        guardrail.async_handler, "post", new_callable=AsyncMock
-    ) as mock_post:
+    with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post:
         result = await guardrail.apply_guardrail(
             inputs=inputs,
             request_data={},
@@ -277,9 +879,7 @@ def test_bypass_header_case_insensitive():
     guardrail = _make_guardrail()
 
     for header_value in ("true", "True", "TRUE"):
-        data = {
-            "proxy_server_request": {"headers": {"x-headroom-bypass": header_value}}
-        }
+        data = {"proxy_server_request": {"headers": {"x-headroom-bypass": header_value}}}
         assert guardrail._should_bypass(data) is True
 
     data = {"proxy_server_request": {"headers": {"x-headroom-bypass": "false"}}}
@@ -344,3 +944,118 @@ async def test_apply_guardrail_sends_model_from_request_data_when_no_config_mode
     call_kwargs = mock_post.call_args
     sent_payload = call_kwargs.kwargs.get("json") or call_kwargs.args[1]
     assert sent_payload.get("model") == "gpt-4o"
+
+
+@pytest.mark.asyncio
+async def test_async_should_run_agentic_loop_detects_anthropic_content_block_format(
+    guardrail: HeadroomGuardrail,
+):
+    # Anthropic's native tool format (type: "custom", top-level "name") --
+    # by the time a Messages API response reaches this gate, the OpenAI-shaped
+    # tool this guardrail injects has already been transformed into this shape.
+    retrieve_tool_def = [
+        {
+            "type": "custom",
+            "name": HEADROOM_RETRIEVE_TOOL_NAME,
+            "input_schema": {"type": "object", "properties": {"hash": {"type": "string"}}},
+        }
+    ]
+
+    response = MagicMock()
+    response.choices = None
+    response.content = [
+        {
+            "type": "tool_use",
+            "id": "toolu_abc",
+            "name": HEADROOM_RETRIEVE_TOOL_NAME,
+            "input": {"hash": "b573993006976af767214fac"},
+        }
+    ]
+
+    should_run, ctx = await guardrail.async_should_run_agentic_loop(
+        response=response,
+        model="claude-sonnet-4-6",
+        messages=[],
+        tools=retrieve_tool_def,
+        stream=False,
+        custom_llm_provider="anthropic",
+        kwargs={},
+    )
+
+    assert should_run is True
+    assert len(ctx["tool_calls"]) == 1
+    assert ctx["tool_calls"][0]["arguments"]["hash"] == "b573993006976af767214fac"
+
+
+@pytest.mark.asyncio
+async def test_async_should_run_agentic_loop_detects_anthropic_response_as_plain_dict(
+    guardrail: HeadroomGuardrail,
+):
+    """AnthropicMessagesResponse is a TypedDict -- real Messages API responses
+    are plain dicts at runtime, not objects with attribute access. A
+    MagicMock-only test would pass even if detection used bare getattr() and
+    silently treated every real response as having no tool calls."""
+    retrieve_tool_def = [
+        {
+            "type": "custom",
+            "name": HEADROOM_RETRIEVE_TOOL_NAME,
+            "input_schema": {"type": "object", "properties": {"hash": {"type": "string"}}},
+        }
+    ]
+    response = {
+        "content": [
+            {
+                "type": "tool_use",
+                "id": "toolu_abc",
+                "name": HEADROOM_RETRIEVE_TOOL_NAME,
+                "input": {"hash": "b573993006976af767214fac"},
+            }
+        ]
+    }
+
+    should_run, ctx = await guardrail.async_should_run_agentic_loop(
+        response=response,
+        model="claude-sonnet-4-6",
+        messages=[],
+        tools=retrieve_tool_def,
+        stream=False,
+        custom_llm_provider="anthropic",
+        kwargs={},
+    )
+
+    assert should_run is True
+    assert len(ctx["tool_calls"]) == 1
+    assert ctx["tool_calls"][0]["arguments"]["hash"] == "b573993006976af767214fac"
+
+
+@pytest.mark.asyncio
+async def test_async_should_run_agentic_loop_detects_responses_api_output_format(
+    guardrail: HeadroomGuardrail,
+):
+    retrieve_tool_def = [{"type": "function", "function": {"name": HEADROOM_RETRIEVE_TOOL_NAME}}]
+
+    response = MagicMock()
+    response.choices = None
+    response.content = None
+    response.output = [
+        {
+            "type": "function_call",
+            "id": "fc_abc123",
+            "name": HEADROOM_RETRIEVE_TOOL_NAME,
+            "arguments": json.dumps({"hash": "b573993006976af767214fac"}),
+        }
+    ]
+
+    should_run, ctx = await guardrail.async_should_run_agentic_loop(
+        response=response,
+        model="gpt-4o",
+        messages=[],
+        tools=retrieve_tool_def,
+        stream=False,
+        custom_llm_provider="openai",
+        kwargs={},
+    )
+
+    assert should_run is True
+    assert len(ctx["tool_calls"]) == 1
+    assert ctx["tool_calls"][0]["arguments"]["hash"] == "b573993006976af767214fac"

From 23af78465c877f5f7f02c53d9f04cf1af612f7a9 Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 30 Jun 2026 19:31:51 -0700
Subject: [PATCH 47/51] feat: add cache control injection support for
 v1/messages endpoint (#31778)

* feat: add cache control injection support for v1/messages endpoint

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix: normalize string content to list for Anthropic-native cache_control injection

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor: simplify cache control injection, fix system=[] bug, fix handler system type

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor: extract cache control logic into static helper on AnthropicCacheControlHook

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Krrish Dholakia <krrishdholakia@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../anthropic_cache_control_hook.py           |  97 ++++-
 .../messages/handler.py                       |  16 +-
 .../test_anthropic_cache_control_hook.py      | 400 +++++++++++-------
 3 files changed, 356 insertions(+), 157 deletions(-)

diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py
index 1314fd82255..608fdebc1d9 100644
--- a/litellm/integrations/anthropic_cache_control_hook.py
+++ b/litellm/integrations/anthropic_cache_control_hook.py
@@ -1,9 +1,12 @@
 """
-This hook is used to inject cache control directives into the messages of a chat completion.
+This hook is used to inject cache control directives into messages.
 
 Users can define
 - `cache_control_injection_points` in the completion params and litellm will inject the cache control directives into the messages at the specified injection points.
 
+Supported for both `v1/chat/completions` (via the prompt-management hook) and
+`v1/messages` (via `apply_to_anthropic_messages_request`).
+
 """
 
 import copy
@@ -225,6 +228,98 @@ class AnthropicCacheControlHook(CustomPromptManagement):
                 message_content[-1]["cache_control"] = control  # type: ignore
         return message
 
+    @staticmethod
+    def apply_to_anthropic_messages_request(
+        messages: List[Dict],
+        system: str | list | None,
+        injection_points: List[CacheControlInjectionPoint],
+    ) -> Tuple[List[Dict], str | list | None, List[CacheControlInjectionPoint]]:
+        """Apply cache control injection for the Anthropic-native v1/messages endpoint.
+
+        Returns (messages, system, remaining_non_message_points).
+        """
+        if not injection_points:
+            return messages, system, []
+
+        processed_messages: List[Dict] = copy.deepcopy(messages)
+        processed_system = copy.deepcopy(system) if system is not None else None
+
+        message_points: List[CacheControlMessageInjectionPoint] = []
+        system_points: List[CacheControlMessageInjectionPoint] = []
+        remaining_points: List[CacheControlInjectionPoint] = []
+
+        for point in injection_points:
+            if point.get("location") == "message":
+                msg_point = cast(CacheControlMessageInjectionPoint, point)
+                if msg_point.get("role") == "system":
+                    system_points.append(msg_point)
+                else:
+                    message_points.append(msg_point)
+            else:
+                remaining_points.append(point)
+
+        reserved_blocks = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0
+        max_blocks = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks
+
+        used_blocks = sum(
+            AnthropicCacheControlHook._count_cache_control_blocks(cast(AllMessageValues, msg))
+            for msg in processed_messages
+        )
+        if isinstance(processed_system, list):
+            used_blocks += sum(
+                1 for b in processed_system if isinstance(b, dict) and b.get("cache_control") is not None
+            )
+
+        if system_points and processed_system is not None and used_blocks < max_blocks:
+            system_already_has_cc = isinstance(processed_system, list) and any(
+                isinstance(b, dict) and b.get("cache_control") is not None for b in processed_system
+            )
+            if not system_already_has_cc:
+                control = system_points[0].get("control") or ChatCompletionCachedContent(type="ephemeral")
+                if isinstance(processed_system, str):
+                    processed_system = [{"type": "text", "text": processed_system, "cache_control": control}]
+                    used_blocks += 1
+                elif len(processed_system) > 0 and isinstance(processed_system[-1], dict):
+                    processed_system[-1] = {**processed_system[-1], "cache_control": control}
+                    used_blocks += 1
+
+        for i, msg in enumerate(processed_messages):
+            content = msg.get("content")
+            if isinstance(content, str):
+                processed_messages[i] = {**msg, "content": [{"type": "text", "text": content}]}
+
+        processed_messages = AnthropicCacheControlHook._apply_message_injections(
+            points=message_points,
+            messages=cast(List[AllMessageValues], processed_messages),
+            max_blocks=max_blocks - used_blocks,
+        )
+
+        return processed_messages, processed_system, remaining_points
+
+    @staticmethod
+    def maybe_inject_cache_control(
+        messages: List[Dict],
+        system: str | list | None,
+        kwargs: Dict[str, Any],
+    ) -> Tuple[List[Dict], str | list | None]:
+        """Extract cache_control_injection_points from kwargs and apply if present.
+
+        Pops the key from kwargs; if remaining (non-message) points exist they
+        are written back so downstream transforms can handle them.
+        """
+        injection_points = kwargs.pop("cache_control_injection_points", None)
+        if not injection_points:
+            return messages, system
+
+        messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
+            messages=messages,
+            system=system,
+            injection_points=injection_points,
+        )
+        if remaining:
+            kwargs["cache_control_injection_points"] = remaining
+        return messages, system
+
     @property
     def integration_name(self) -> str:
         """Return the integration name for this hook."""
diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py
index 547ddd9b8d3..effd7dda6a0 100644
--- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py
+++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py
@@ -199,7 +199,7 @@ async def anthropic_messages(
     metadata: Optional[Dict] = None,
     stop_sequences: Optional[List[str]] = None,
     stream: Optional[bool] = False,
-    system: Optional[str] = None,
+    system: Optional[Union[str, list]] = None,
     temperature: Optional[float] = None,
     thinking: Optional[Dict] = None,
     tool_choice: Optional[Dict] = None,
@@ -230,6 +230,12 @@ async def anthropic_messages(
     # ids like ``functions.Bash:0`` that violate Anthropic's id pattern.
     messages = sanitize_tool_use_ids_in_anthropic_messages(messages)
 
+    from litellm.integrations.anthropic_cache_control_hook import (
+        AnthropicCacheControlHook,
+    )
+
+    messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(messages, system, kwargs)
+
     original_stream = stream or kwargs.get("_websearch_interception_converted_stream", False)
 
     # Execute pre-request hooks to allow CustomLoggers to modify request.
@@ -375,7 +381,7 @@ def anthropic_messages_handler(
     metadata: Optional[Dict] = None,
     stop_sequences: Optional[List[str]] = None,
     stream: Optional[bool] = False,
-    system: Optional[str] = None,
+    system: Optional[Union[str, list]] = None,
     temperature: Optional[float] = None,
     thinking: Optional[Dict] = None,
     tool_choice: Optional[Dict] = None,
@@ -412,6 +418,12 @@ def anthropic_messages_handler(
         messages = strip_empty_text_blocks_from_anthropic_messages(messages)
         messages = sanitize_tool_use_ids_in_anthropic_messages(messages)
 
+    from litellm.integrations.anthropic_cache_control_hook import (
+        AnthropicCacheControlHook,
+    )
+
+    messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(messages, system, kwargs)
+
     metadata = validate_anthropic_api_metadata(metadata)
 
     local_vars = locals()
diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py
index 6afe5efc54d..4664cc86303 100644
--- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py
+++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py
@@ -1,3 +1,4 @@
+import copy
 import datetime
 import json
 import os
@@ -9,9 +10,7 @@ from unittest.mock import ANY, MagicMock, Mock, patch
 import httpx
 import pytest
 
-sys.path.insert(
-    0, os.path.abspath("../..")
-)  # Adds the parent directory to the system-path
+sys.path.insert(0, os.path.abspath("../.."))  # Adds the parent directory to the system-path
 import litellm
 from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
@@ -93,13 +92,9 @@ async def test_anthropic_cache_control_hook_system_message():
 
             # Verify that cache control was applied (Bedrock transforms it to a separate item)
             cache_control_count = sum(
-                1
-                for item in request_body["system"]
-                if isinstance(item, dict) and "cachePoint" in item
+                1 for item in request_body["system"] if isinstance(item, dict) and "cachePoint" in item
             )
-            assert (
-                cache_control_count == 1
-            ), f"Expected exactly 1 cache control point, found {cache_control_count}"
+            assert cache_control_count == 1, f"Expected exactly 1 cache control point, found {cache_control_count}"
 
 
 @pytest.mark.asyncio
@@ -171,9 +166,7 @@ async def test_anthropic_cache_control_hook_user_message():
             print("request_body: ", json.dumps(request_body, indent=4))
 
             # Verify the request body
-            assert request_body["messages"][1]["content"][1]["cachePoint"] == {
-                "type": "default"
-            }
+            assert request_body["messages"][1]["content"][1]["cachePoint"] == {"type": "default"}
 
 
 @pytest.mark.asyncio
@@ -262,14 +255,10 @@ async def test_anthropic_cache_control_hook_negative_indices():
 
             # Verify the last message (input index -1 -> request index 2) has cache control
             last_message_content = request_body["messages"][2]["content"]
-            assert isinstance(
-                last_message_content, list
-            ), "Last message content should be a list"
-            assert any(
-                "cachePoint" in item
-                for item in last_message_content
-                if isinstance(item, dict)
-            ), "CachePoint missing in last message"
+            assert isinstance(last_message_content, list), "Last message content should be a list"
+            assert any("cachePoint" in item for item in last_message_content if isinstance(item, dict)), (
+                "CachePoint missing in last message"
+            )
 
             # Note: Based on debug output, the hook correctly applies cache control to both messages,
             # but the Bedrock API transformation appears to only preserve cache control for user messages,
@@ -278,30 +267,20 @@ async def test_anthropic_cache_control_hook_negative_indices():
             # The second-to-last message (assistant) gets cache_control from the hook but loses it
             # during API transformation. This test documents this behavior.
             second_last_message_content = request_body["messages"][1]["content"]
-            assert isinstance(
-                second_last_message_content, list
-            ), "Second-to-last message content should be a list"
+            assert isinstance(second_last_message_content, list), "Second-to-last message content should be a list"
 
             # Check if assistant message cache control is preserved (currently it's not)
             assistant_has_cache_control = any(
-                "cachePoint" in item
-                for item in second_last_message_content
-                if isinstance(item, dict)
-            )
-            print(
-                f"Assistant message has cache control in final request: {assistant_has_cache_control}"
+                "cachePoint" in item for item in second_last_message_content if isinstance(item, dict)
             )
+            print(f"Assistant message has cache control in final request: {assistant_has_cache_control}")
 
             # Verify the first user message (request index 0) was NOT modified
             first_user_message_content = request_body["messages"][0]["content"]
-            assert isinstance(
-                first_user_message_content, list
-            ), "First user message content should be a list"
-            assert not any(
-                "cachePoint" in item
-                for item in first_user_message_content
-                if isinstance(item, dict)
-            ), "CachePoint unexpectedly found in first user message"
+            assert isinstance(first_user_message_content, list), "First user message content should be a list"
+            assert not any("cachePoint" in item for item in first_user_message_content if isinstance(item, dict)), (
+                "CachePoint unexpectedly found in first user message"
+            )
 
 
 @pytest.mark.asyncio
@@ -342,9 +321,7 @@ async def test_anthropic_cache_control_hook_out_of_bounds_logging():
         client = AsyncHTTPHandler()
 
         # Mock the verbose_logger to capture warning calls
-        with patch(
-            "litellm.integrations.anthropic_cache_control_hook.verbose_logger"
-        ) as mock_logger:
+        with patch("litellm.integrations.anthropic_cache_control_hook.verbose_logger") as mock_logger:
             with patch.object(client, "post", return_value=mock_response) as mock_post:
                 messages = [
                     {"role": "user", "content": "Message 1"},
@@ -354,9 +331,7 @@ async def test_anthropic_cache_control_hook_out_of_bounds_logging():
                 await litellm.acompletion(
                     model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                     messages=messages,
-                    cache_control_injection_points=[
-                        {"location": "message", "index": 10}
-                    ],  # Out of bounds index
+                    cache_control_injection_points=[{"location": "message", "index": 10}],  # Out of bounds index
                     client=client,
                 )
 
@@ -365,10 +340,7 @@ async def test_anthropic_cache_control_hook_out_of_bounds_logging():
                 warning_call = mock_logger.warning.call_args[0][0]
 
                 # Check that the warning message contains the expected information
-                assert (
-                    "AnthropicCacheControlHook: Provided index 10 is out of bounds"
-                    in warning_call
-                )
+                assert "AnthropicCacheControlHook: Provided index 10 is out of bounds" in warning_call
                 assert "message list of length 2" in warning_call
                 assert "Targeted index was 10" in warning_call
                 assert "Skipping cache control injection for this point" in warning_call
@@ -411,9 +383,7 @@ async def test_anthropic_cache_control_hook_negative_out_of_bounds_logging():
         client = AsyncHTTPHandler()
 
         # Mock the verbose_logger to capture warning calls
-        with patch(
-            "litellm.integrations.anthropic_cache_control_hook.verbose_logger"
-        ) as mock_logger:
+        with patch("litellm.integrations.anthropic_cache_control_hook.verbose_logger") as mock_logger:
             with patch.object(client, "post", return_value=mock_response) as mock_post:
                 messages = [
                     {"role": "user", "content": "Single message"},
@@ -436,14 +406,9 @@ async def test_anthropic_cache_control_hook_negative_out_of_bounds_logging():
                 warning_call = mock_logger.warning.call_args[0][0]
 
                 # Check that the warning message contains the original negative index
-                assert (
-                    "AnthropicCacheControlHook: Provided index -5 is out of bounds"
-                    in warning_call
-                )
+                assert "AnthropicCacheControlHook: Provided index -5 is out of bounds" in warning_call
                 assert "message list of length 1" in warning_call
-                assert (
-                    "Targeted index was -4" in warning_call
-                )  # -5 + 1 = -4 (converted index)
+                assert "Targeted index was -4" in warning_call  # -5 + 1 = -4 (converted index)
                 assert "Skipping cache control injection for this point" in warning_call
 
 
@@ -531,15 +496,11 @@ async def test_anthropic_cache_control_hook_multiple_user_messages():
 
             # Count cache control points - should have 2 since both injection points were applied
             cache_control_count = sum(
-                1
-                for item in combined_message_content
-                if isinstance(item, dict) and "cachePoint" in item
+                1 for item in combined_message_content if isinstance(item, dict) and "cachePoint" in item
             )
             assert cache_control_count == 2
 
-            print(
-                f"Found {cache_control_count} cache control points in the combined message"
-            )
+            print(f"Found {cache_control_count} cache control points in the combined message")
 
 
 @pytest.mark.asyncio
@@ -588,9 +549,7 @@ async def test_anthropic_cache_control_hook_out_of_bounds(bad_index):
             await litellm.acompletion(
                 model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                 messages=messages,
-                cache_control_injection_points=[
-                    {"location": "message", "index": bad_index}
-                ],
+                cache_control_injection_points=[{"location": "message", "index": bad_index}],
                 client=client,
             )
 
@@ -601,19 +560,13 @@ async def test_anthropic_cache_control_hook_out_of_bounds(bad_index):
             for msg in request_body["messages"]:
                 content = msg.get("content", [])
                 if isinstance(content, list):
-                    assert not any(
-                        "cachePoint" in item
-                        for item in content
-                        if isinstance(item, dict)
-                    )
+                    assert not any("cachePoint" in item for item in content if isinstance(item, dict))
 
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "message_list",
-    [
-        [{"role": "user", "content": "Single message"}]
-    ],  # Single message only - empty list will fail at API level
+    [[{"role": "user", "content": "Single message"}]],  # Single message only - empty list will fail at API level
 )
 async def test_anthropic_cache_control_hook_single_message(message_list):
     """
@@ -662,9 +615,7 @@ async def test_anthropic_cache_control_hook_single_message(message_list):
             # For the single message, verify cache control was applied
             content = request_body["messages"][0]["content"]
             assert isinstance(content, list)
-            assert any(
-                "cachePoint" in item for item in content if isinstance(item, dict)
-            )
+            assert any("cachePoint" in item for item in content if isinstance(item, dict))
 
 
 @pytest.mark.asyncio
@@ -693,9 +644,7 @@ async def test_anthropic_cache_control_hook_empty_message_list():
                 await litellm.acompletion(
                     model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                     messages=[],
-                    cache_control_injection_points=[
-                        {"location": "message", "index": -1}
-                    ],
+                    cache_control_injection_points=[{"location": "message", "index": -1}],
                     client=client,
                 )
 
@@ -755,11 +704,7 @@ async def test_anthropic_cache_control_hook_no_op():
             for msg in request_body["messages"]:
                 content = msg.get("content", [])
                 if isinstance(content, list):
-                    assert not any(
-                        "cachePoint" in item
-                        for item in content
-                        if isinstance(item, dict)
-                    )
+                    assert not any("cachePoint" in item for item in content if isinstance(item, dict))
 
 
 @pytest.mark.asyncio
@@ -827,14 +772,10 @@ async def test_anthropic_cache_control_hook_multiple_content_items_last_only():
             message_content = request_body["messages"][0]["content"]
             assert isinstance(message_content, list)
 
-            cache_control_count = sum(
-                1
-                for item in message_content
-                if isinstance(item, dict) and "cachePoint" in item
+            cache_control_count = sum(1 for item in message_content if isinstance(item, dict) and "cachePoint" in item)
+            assert cache_control_count == 1, (
+                f"Expected exactly 1 cache control point, found {cache_control_count}. This test verifies the fix for issue 15696 where cache_control was incorrectly applied to ALL content items."
             )
-            assert (
-                cache_control_count == 1
-            ), f"Expected exactly 1 cache control point, found {cache_control_count}. This test verifies the fix for issue 15696 where cache_control was incorrectly applied to ALL content items."
 
 
 @pytest.mark.asyncio
@@ -891,30 +832,22 @@ async def test_anthropic_cache_control_hook_document_analysis_multiple_pages():
                         ],
                     }
                 ],
-                cache_control_injection_points=[
-                    {"location": "message", "role": "user"}
-                ],
+                cache_control_injection_points=[{"location": "message", "role": "user"}],
                 client=client,
             )
 
             mock_post.assert_called_once()
             request_body = json.loads(mock_post.call_args.kwargs["data"])
 
-            print(
-                "Document analysis request_body: ", json.dumps(request_body, indent=4)
-            )
+            print("Document analysis request_body: ", json.dumps(request_body, indent=4))
 
             message_content = request_body["messages"][0]["content"]
             assert isinstance(message_content, list)
 
-            cache_control_count = sum(
-                1
-                for item in message_content
-                if isinstance(item, dict) and "cachePoint" in item
+            cache_control_count = sum(1 for item in message_content if isinstance(item, dict) and "cachePoint" in item)
+            assert cache_control_count == 1, (
+                f"Expected exactly 1 cache control point (last item only), found {cache_control_count}. Before fix, this would be 6 (one for each content item)."
             )
-            assert (
-                cache_control_count == 1
-            ), f"Expected exactly 1 cache control point (last item only), found {cache_control_count}. Before fix, this would be 6 (one for each content item)."
 
 
 def test_gemini_cache_control_injection_points_detected():
@@ -1076,13 +1009,8 @@ async def test_anthropic_cache_control_hook_string_negative_index():
             # The last user message should have cache control applied
             last_message = request_body["messages"][-1]
             last_message_content = last_message["content"]
-            assert isinstance(
-                last_message_content, list
-            ), f"Expected list content, got {type(last_message_content)}"
-            has_cache_point = any(
-                isinstance(item, dict) and "cachePoint" in item
-                for item in last_message_content
-            )
+            assert isinstance(last_message_content, list), f"Expected list content, got {type(last_message_content)}"
+            has_cache_point = any(isinstance(item, dict) and "cachePoint" in item for item in last_message_content)
             assert has_cache_point, (
                 f"Expected cachePoint in last message content, got: {last_message_content}. "
                 "String index '-1' was not parsed correctly (str.isdigit() returns False for negative strings)."
@@ -1146,17 +1074,13 @@ def test_cache_control_hook_caps_at_four_blocks_with_client_cache_control():
     _, processed, _ = hook.get_chat_completion_prompt(
         model="bedrock/us.anthropic.claude-opus-4-6-v1:0",
         messages=messages,
-        non_default_params={
-            "cache_control_injection_points": _build_injection_points()
-        },
+        non_default_params={"cache_control_injection_points": _build_injection_points()},
         prompt_id=None,
         prompt_variables=None,
         dynamic_callback_params={},
     )
 
-    assert (
-        _count_cache_control(processed) == 4
-    ), "Hook must cap cache_control at Anthropic's limit of 4 blocks"
+    assert _count_cache_control(processed) == 4, "Hook must cap cache_control at Anthropic's limit of 4 blocks"
 
     # Client TTL on system blocks must be preserved (not overwritten by config).
     for i in range(4):
@@ -1170,11 +1094,7 @@ def test_cache_control_hook_caps_at_four_blocks_with_client_cache_control():
     assert user_message.get("cache_control") is None
     user_content = user_message.get("content")
     if isinstance(user_content, list):
-        assert all(
-            block.get("cache_control") is None
-            for block in user_content
-            if isinstance(block, dict)
-        )
+        assert all(block.get("cache_control") is None for block in user_content if isinstance(block, dict))
 
 
 def test_cache_control_hook_caps_at_four_blocks_without_client_cache_control():
@@ -1184,17 +1104,13 @@ def test_cache_control_hook_caps_at_four_blocks_without_client_cache_control():
     """
     hook = AnthropicCacheControlHook()
 
-    messages: List[AllMessageValues] = [
-        {"role": "system", "content": f"System {i}"} for i in range(4)
-    ]
+    messages: List[AllMessageValues] = [{"role": "system", "content": f"System {i}"} for i in range(4)]
     messages.append({"role": "user", "content": "hello"})
 
     _, processed, _ = hook.get_chat_completion_prompt(
         model="bedrock/us.anthropic.claude-opus-4-6-v1:0",
         messages=messages,
-        non_default_params={
-            "cache_control_injection_points": _build_injection_points()
-        },
+        non_default_params={"cache_control_injection_points": _build_injection_points()},
         prompt_id=None,
         prompt_variables=None,
         dynamic_callback_params={},
@@ -1303,18 +1219,12 @@ async def test_cache_control_hook_bedrock_payload_caps_cachepoints_at_four():
             request_body = json.loads(mock_post.call_args.kwargs["data"])
 
             cache_points = sum(
-                1
-                for block in request_body.get("system", [])
-                if isinstance(block, dict) and "cachePoint" in block
+                1 for block in request_body.get("system", []) if isinstance(block, dict) and "cachePoint" in block
             )
             for msg in request_body.get("messages", []):
                 content = msg.get("content", [])
                 if isinstance(content, list):
-                    cache_points += sum(
-                        1
-                        for block in content
-                        if isinstance(block, dict) and "cachePoint" in block
-                    )
+                    cache_points += sum(1 for block in content if isinstance(block, dict) and "cachePoint" in block)
 
             assert cache_points <= 4, (
                 f"Bedrock payload exceeded Anthropic's 4 cache_control block limit: "
@@ -1331,9 +1241,7 @@ def test_cache_control_hook_reserves_slot_for_tool_config_point():
     """
     hook = AnthropicCacheControlHook()
 
-    messages: List[AllMessageValues] = [
-        {"role": "system", "content": f"System {i}"} for i in range(4)
-    ]
+    messages: List[AllMessageValues] = [{"role": "system", "content": f"System {i}"} for i in range(4)]
     messages.append({"role": "user", "content": "hello"})
 
     _, processed, non_default_params = hook.get_chat_completion_prompt(
@@ -1356,9 +1264,7 @@ def test_cache_control_hook_reserves_slot_for_tool_config_point():
 
     assert _count_cache_control(processed) == 3
     # The tool_config point is passed through for the provider transform.
-    assert non_default_params["cache_control_injection_points"] == [
-        {"location": "tool_config"}
-    ]
+    assert non_default_params["cache_control_injection_points"] == [{"location": "tool_config"}]
 
 
 @pytest.mark.asyncio
@@ -1384,9 +1290,7 @@ async def test_cache_control_hook_bedrock_payload_caps_with_tool_config_point():
 
         client = AsyncHTTPHandler()
         with patch.object(client, "post", return_value=mock_response) as mock_post:
-            messages = [
-                {"role": "system", "content": f"System block {i}"} for i in range(4)
-            ]
+            messages = [{"role": "system", "content": f"System block {i}"} for i in range(4)]
             messages.append({"role": "user", "content": "What is the weather?"})
 
             await litellm.acompletion(
@@ -1421,18 +1325,12 @@ async def test_cache_control_hook_bedrock_payload_caps_with_tool_config_point():
             request_body = json.loads(mock_post.call_args.kwargs["data"])
 
             cache_points = sum(
-                1
-                for block in request_body.get("system", [])
-                if isinstance(block, dict) and "cachePoint" in block
+                1 for block in request_body.get("system", []) if isinstance(block, dict) and "cachePoint" in block
             )
             for msg in request_body.get("messages", []):
                 content = msg.get("content", [])
                 if isinstance(content, list):
-                    cache_points += sum(
-                        1
-                        for block in content
-                        if isinstance(block, dict) and "cachePoint" in block
-                    )
+                    cache_points += sum(1 for block in content if isinstance(block, dict) and "cachePoint" in block)
             for tool in request_body.get("toolConfig", {}).get("tools", []):
                 if isinstance(tool, dict) and "cachePoint" in tool:
                     cache_points += 1
@@ -1441,3 +1339,197 @@ async def test_cache_control_hook_bedrock_payload_caps_with_tool_config_point():
                 f"Bedrock payload exceeded Anthropic's 4 cache_control block limit "
                 f"when mixing message and tool_config injection: found {cache_points}"
             )
+
+
+class TestApplyToAnthropicMessagesRequest:
+    """Tests for apply_to_anthropic_messages_request (v1/messages cache control)."""
+
+    def test_system_string_injection(self):
+        messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}]
+        system = "You are helpful"
+        injection_points = [{"location": "message", "role": "system"}]
+
+        result_msgs, result_sys, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
+            messages=messages,
+            system=system,
+            injection_points=injection_points,
+        )
+
+        assert result_sys == [{"type": "text", "text": "You are helpful", "cache_control": {"type": "ephemeral"}}]
+        assert result_msgs == messages
+        assert remaining == []
+
+    def test_system_list_injection(self):
+        messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}]
+        system = [
+            {"type": "text", "text": "Part 1"},
+            {"type": "text", "text": "Part 2"},
+        ]
+        injection_points = [{"location": "message", "role": "system"}]
+
+        _, result_sys, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
+            messages=messages,
+            system=system,
+            injection_points=injection_points,
+        )
+
+        assert result_sys[0] == {"type": "text", "text": "Part 1"}
+        assert result_sys[1] == {"type": "text", "text": "Part 2", "cache_control": {"type": "ephemeral"}}
+
+    def test_user_message_injection_by_role(self):
+        messages = [
+            {"role": "user", "content": [{"type": "text", "text": "First"}]},
+            {"role": "assistant", "content": [{"type": "text", "text": "Response"}]},
+            {"role": "user", "content": [{"type": "text", "text": "Second"}]},
+        ]
+        injection_points = [{"location": "message", "role": "user"}]
+
+        result_msgs, _, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
+            messages=messages,
+            system=None,
+            injection_points=injection_points,
+        )
+
+        assert result_msgs[0]["content"][-1].get("cache_control") == {"type": "ephemeral"}
+        assert result_msgs[2]["content"][-1].get("cache_control") == {"type": "ephemeral"}
+        assert result_msgs[1]["content"][-1].get("cache_control") is None
+
+    def test_message_injection_by_index(self):
+        messages = [
+            {"role": "user", "content": [{"type": "text", "text": "First"}]},
+            {"role": "assistant", "content": [{"type": "text", "text": "Response"}]},
+            {"role": "user", "content": [{"type": "text", "text": "Second"}]},
+        ]
+        injection_points = [{"location": "message", "index": -1}]
+
+        result_msgs, _, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
+            messages=messages,
+            system=None,
+            injection_points=injection_points,
+        )
+
+        assert result_msgs[2]["content"][-1].get("cache_control") == {"type": "ephemeral"}
+        assert result_msgs[0]["content"][-1].get("cache_control") is None
+        assert result_msgs[1]["content"][-1].get("cache_control") is None
+
+    def test_mixed_system_and_message_injection(self):
+        messages = [
+            {"role": "user", "content": [{"type": "text", "text": "Hello"}]},
+            {"role": "assistant", "content": [{"type": "text", "text": "Hi"}]},
+            {"role": "user", "content": [{"type": "text", "text": "Question"}]},
+        ]
+        system = "System prompt"
+        injection_points = [
+            {"location": "message", "role": "system"},
+            {"location": "message", "index": -1},
+        ]
+
+        result_msgs, result_sys, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
+            messages=messages,
+            system=system,
+            injection_points=injection_points,
+        )
+
+        assert result_sys[0]["cache_control"] == {"type": "ephemeral"}
+        assert result_msgs[2]["content"][-1].get("cache_control") == {"type": "ephemeral"}
+
+    def test_respects_max_4_blocks(self):
+        messages = [{"role": "user", "content": [{"type": "text", "text": f"Msg {i}"}]} for i in range(6)]
+        system = "System"
+        injection_points = [
+            {"location": "message", "role": "system"},
+            {"location": "message", "role": "user"},
+        ]
+
+        result_msgs, result_sys, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
+            messages=messages,
+            system=system,
+            injection_points=injection_points,
+        )
+
+        sys_blocks = sum(1 for b in (result_sys or []) if isinstance(b, dict) and b.get("cache_control") is not None)
+        total_blocks = sys_blocks + sum(AnthropicCacheControlHook._count_cache_control_blocks(m) for m in result_msgs)
+        assert total_blocks <= 4
+
+    def test_tool_config_points_forwarded_as_remaining(self):
+        messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}]
+        injection_points = [
+            {"location": "message", "role": "user"},
+            {"location": "tool_config"},
+        ]
+
+        _, _, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
+            messages=messages,
+            system=None,
+            injection_points=injection_points,
+        )
+
+        assert remaining == [{"location": "tool_config"}]
+
+    def test_no_injection_points_returns_unchanged(self):
+        messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}]
+        system = "System"
+
+        result_msgs, result_sys, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
+            messages=messages,
+            system=system,
+            injection_points=[],
+        )
+
+        assert result_msgs == messages
+        assert result_sys == system
+        assert remaining == []
+
+    def test_does_not_mutate_input(self):
+        messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}]
+        system = [{"type": "text", "text": "System"}]
+        injection_points = [{"location": "message", "role": "system"}]
+
+        original_system = copy.deepcopy(system)
+        original_messages = copy.deepcopy(messages)
+
+        AnthropicCacheControlHook.apply_to_anthropic_messages_request(
+            messages=messages,
+            system=system,
+            injection_points=injection_points,
+        )
+
+        assert messages == original_messages
+        assert system == original_system
+
+    def test_system_none_with_system_point_skipped(self):
+        messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}]
+        injection_points = [{"location": "message", "role": "system"}]
+
+        result_msgs, result_sys, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
+            messages=messages,
+            system=None,
+            injection_points=injection_points,
+        )
+
+        assert result_sys is None
+
+    def test_existing_cache_control_counted_toward_limit(self):
+        messages = [
+            {"role": "user", "content": [{"type": "text", "text": "A", "cache_control": {"type": "ephemeral"}}]},
+            {"role": "assistant", "content": [{"type": "text", "text": "B", "cache_control": {"type": "ephemeral"}}]},
+            {"role": "user", "content": [{"type": "text", "text": "C", "cache_control": {"type": "ephemeral"}}]},
+            {"role": "user", "content": [{"type": "text", "text": "D"}]},
+            {"role": "user", "content": [{"type": "text", "text": "E"}]},
+        ]
+        system = "System"
+        injection_points = [
+            {"location": "message", "role": "system"},
+            {"location": "message", "index": 3},
+            {"location": "message", "index": 4},
+        ]
+
+        result_msgs, result_sys, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
+            messages=messages,
+            system=system,
+            injection_points=injection_points,
+        )
+
+        sys_blocks = sum(1 for b in (result_sys or []) if isinstance(b, dict) and b.get("cache_control") is not None)
+        total_blocks = sys_blocks + sum(AnthropicCacheControlHook._count_cache_control_blocks(m) for m in result_msgs)
+        assert total_blocks <= 4

From bfb8ffccb8e42b69533d95605c5821d88324c870 Mon Sep 17 00:00:00 2001
From: yucheng-berri <yucheng@berri.ai>
Date: Tue, 30 Jun 2026 19:32:27 -0700
Subject: [PATCH 48/51] feat(proxy): audit remaining system-wide settings
 updates (#31754)

* feat(proxy): audit remaining system-wide settings updates

Extends the audit logging framework introduced in the parent PR to the
rest of the LiteLLM_Config writers and the two adjacent settings tables:

  /config/update (general, environment_variables, litellm_settings,
  router_settings sections), /config/field/update, /config/field/delete,
  /config/callback/delete, /update/default_team_settings,
  /update/mcp_semantic_filter_settings, /add/allowed_ip,
  /delete/allowed_ip, /update/sso_settings, /update/ui_theme_settings,
  /update/ui_settings.

Each writer records the actor, action, the affected config section, and
a redacted before/after snapshot. SSO and UI settings rows use their own
table_name (LiteLLM_SSOConfig, LiteLLM_UISettings). The /config/callback
and /update/sso_settings audits fire BEFORE the proxy reload and the env
cleanup step respectively, so a failure in either leaves the audit row
intact.

The audit-actor parameter on _update_litellm_setting is now required
rather than optional; the chokepoint covers default_team and
mcp_semantic_filter for free, and a future caller that forgets the
actor fails loudly instead of silently skipping the audit. The two
direct-calling tests pass a dummy actor.

The environment_variables section redacts every value rather than
relying on key-name matching, because it carries credentials under
non-secret-looking uppercase keys (e.g. DATABASE_URL).

* fix(proxy): capture redacted SSO before-snapshot in audit log

Greptile review of #31754 flagged update_sso_settings as the one endpoint
where before_value is permanently None, so the LiteLLM_SSOConfig audit
trail has no pre-change state. An auditor reviewing a secret-rotation
event could see what the SSO settings were changed to but not what they
were before.

Read the existing SSO row before the upsert, decrypt it via
proxy_config._decrypt_db_variables, and pass it as before_value.
create_config_audit_log's secret-name redaction then masks the
*_client_secret fields, so neither the old nor the new plaintext secret
lands in the audit row.

Add a regression test asserting the before-snapshot reflects the
pre-change values for non-secret fields (google_client_id) and is
redacted for secret fields (google_client_secret). Mutation-checked
against reverting to before_value=None.

The pre-existing SSO tests now also mock litellm_ssoconfig.find_unique
since the endpoint reads it; the read returns None for tests that do not
care about the before-state.

* fix: remove committed zero init migration

* refactor(proxy): audit config writes via asyncio.create_task everywhere

PR A's chokepoint audit call was refactored from a blocking await to
asyncio.create_task so that a post-save audit-log failure could not
surface as a 500 to the caller. The 12 other audit call sites added in
this PR were still using await, reintroducing the exact 500-after-commit
exposure at every sibling endpoint. Wrap them all in asyncio.create_task
to match the model_management_endpoints / key_management_endpoints /
hooks / config_override_endpoints / team_callback_endpoints /
cache_settings_endpoints house pattern, so the codebase tells one story.

The two direct-invocation tests (test_update_config_general_settings and
test_delete_config_general_settings, which call the handler in-process
rather than via TestClient) yield with `await asyncio.sleep(0)` after the
handler returns so the scheduled audit task runs before the assertion.

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
---
 litellm/proxy/_types.py                       |   2 +
 litellm/proxy/proxy_server.py                 |  53 ++-
 .../proxy_setting_endpoints.py                | 145 +++++--
 .../scim/test_scim_v2_endpoints.py            |   2 +
 .../test_team_default_params.py               |   2 +
 tests/test_litellm/proxy/test_proxy_server.py | 275 ++++++++++++
 .../test_proxy_setting_endpoints.py           | 407 ++++++++++++++++++
 7 files changed, 861 insertions(+), 25 deletions(-)

diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index a6ef7de07ae..643f8d69300 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -190,6 +190,8 @@ class LitellmTableNames(str, enum.Enum):
     CACHE_CONFIG_TABLE_NAME = "LiteLLM_CacheConfig"
     CONFIG_OVERRIDES_TABLE_NAME = "LiteLLM_ConfigOverrides"
     CONFIG_TABLE_NAME = "LiteLLM_Config"
+    SSO_CONFIG_TABLE_NAME = "LiteLLM_SSOConfig"
+    UI_SETTINGS_TABLE_NAME = "LiteLLM_UISettings"
 
 
 class Litellm_EntityType(enum.Enum):
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 0158f601d32..2f6c48a751b 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -13962,6 +13962,7 @@ async def update_config(
         # effect of auto-enabling slack alerting.
         if config_info.general_settings is not None:
             existing = await _read_section("general_settings")
+            before_general_settings = copy.deepcopy(existing)
             updates = config_info.general_settings.dict(exclude_none=True)
             for k, v in updates.items():
                 if k == "alert_to_webhook_url":
@@ -13971,6 +13972,11 @@ async def update_config(
                         existing["alerting"].append("slack")
                 existing[k] = v
             await _upsert_section("general_settings", existing)
+            asyncio.create_task(
+                create_config_audit_log(
+                    "general_settings", "updated", before_general_settings, existing, user_api_key_dict
+                )
+            )
 
         # environment_variables: idempotently encrypt the request values
         # (plaintext on first write, OR ciphertext the UI read back via
@@ -13979,10 +13985,16 @@ async def update_config(
         # their stored ciphertext byte-for-byte.
         if config_info.environment_variables is not None:
             existing = await _read_section("environment_variables")
+            before_environment_variables = copy.deepcopy(existing)
             existing.update(
                 proxy_config._encrypt_env_variables_for_db(environment_variables=config_info.environment_variables)
             )
             await _upsert_section("environment_variables", existing)
+            asyncio.create_task(
+                create_config_audit_log(
+                    "environment_variables", "updated", before_environment_variables, existing, user_api_key_dict
+                )
+            )
 
         # litellm_settings: merge existing + request, request wins (matching
         # router_settings semantics — the caller's value for any given key is
@@ -13994,6 +14006,7 @@ async def update_config(
         # entries that delete_callback (lowercase lookup) cannot find.
         if config_info.litellm_settings is not None:
             existing = await _read_section("litellm_settings")
+            before_litellm_settings = copy.deepcopy(existing)
             updated_litellm_settings = dict(config_info.litellm_settings)
 
             incoming_cb = updated_litellm_settings.get("success_callback")
@@ -14015,12 +14028,24 @@ async def update_config(
                     merged["success_callback"] = list(set(incoming_cb))
 
             await _upsert_section("litellm_settings", merged)
+            asyncio.create_task(
+                create_config_audit_log(
+                    "litellm_settings", "updated", before_litellm_settings, merged, user_api_key_dict
+                )
+            )
 
         # router_settings: merge existing + request, request wins.
         if config_info.router_settings is not None:
             existing = await _read_section("router_settings")
+            before_router_settings = copy.deepcopy(existing)
             updates = config_info.router_settings.dict(exclude_none=True)
-            await _upsert_section("router_settings", {**existing, **updates})
+            new_router_settings = {**existing, **updates}
+            await _upsert_section("router_settings", new_router_settings)
+            asyncio.create_task(
+                create_config_audit_log(
+                    "router_settings", "updated", before_router_settings, new_router_settings, user_api_key_dict
+                )
+            )
 
         await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj)
 
@@ -14152,6 +14177,8 @@ async def update_config_general_settings(
     else:
         general_settings = dict(db_general_settings.param_value)
 
+    before_general_settings = copy.deepcopy(general_settings)
+
     ## update db
 
     field_value = data.field_value
@@ -14171,6 +14198,11 @@ async def update_config_general_settings(
         },
     )
     await invalidate_config_param("general_settings")
+    asyncio.create_task(
+        create_config_audit_log(
+            "general_settings", "updated", before_general_settings, general_settings, user_api_key_dict
+        )
+    )
 
     if data.field_name == "plugins":
         register_plugins_from_config(general_settings)
@@ -14555,6 +14587,8 @@ async def delete_config_general_settings(
     else:
         general_settings = dict(db_general_settings.param_value)
 
+    before_general_settings = copy.deepcopy(general_settings)
+
     ## update db
 
     general_settings.pop(data.field_name, None)
@@ -14570,6 +14604,11 @@ async def delete_config_general_settings(
         },
     )
     await invalidate_config_param("general_settings")
+    asyncio.create_task(
+        create_config_audit_log(
+            "general_settings", "deleted", before_general_settings, general_settings, user_api_key_dict
+        )
+    )
 
     return response
 
@@ -14627,6 +14666,8 @@ async def delete_callback(
                 detail={"error": f"Callback '{callback_name}' not found in active configuration"},
             )
 
+        before_success_callbacks = list(success_callbacks)
+
         # Remove callback from success_callback list
         success_callbacks.remove(callback_name)
         config.setdefault("litellm_settings", {})["success_callback"] = success_callbacks
@@ -14634,6 +14675,16 @@ async def delete_callback(
         # Save the updated configuration
         await proxy_config.save_config(new_config=config)
 
+        asyncio.create_task(
+            create_config_audit_log(
+                "litellm_settings",
+                "deleted",
+                {"success_callback": before_success_callbacks},
+                {"success_callback": success_callbacks},
+                user_api_key_dict,
+            )
+        )
+
         # Restart the proxy to apply changes
         await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj)
 
diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
index 1be17c86123..0fc303737b4 100644
--- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
+++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
@@ -323,8 +323,12 @@ async def get_allowed_ips():
     tags=["Budget & Spend Tracking"],
     dependencies=[Depends(user_api_key_auth)],
 )
-async def add_allowed_ip(ip_address: IPAddress):
+async def add_allowed_ip(
+    ip_address: IPAddress,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
     from litellm.proxy.proxy_server import (
+        create_config_audit_log,
         general_settings,
         prisma_client,
         proxy_config,
@@ -356,11 +360,22 @@ async def add_allowed_ip(ip_address: IPAddress):
     if "allowed_ips" not in config["general_settings"]:
         config["general_settings"]["allowed_ips"] = []
 
+    before_allowed_ips = list(config["general_settings"]["allowed_ips"])
     if ip_address.ip not in config["general_settings"]["allowed_ips"]:
         config["general_settings"]["allowed_ips"].append(ip_address.ip)
 
     await proxy_config.save_config(new_config=config)
 
+    asyncio.create_task(
+        create_config_audit_log(
+            param_name="general_settings",
+            action="updated",
+            before_value={"allowed_ips": before_allowed_ips},
+            after_value={"allowed_ips": config["general_settings"]["allowed_ips"]},
+            user_api_key_dict=user_api_key_dict,
+        )
+    )
+
     return {
         "message": f"IP {ip_address.ip} address added successfully",
         "status": "success",
@@ -372,8 +387,15 @@ async def add_allowed_ip(ip_address: IPAddress):
     tags=["Budget & Spend Tracking"],
     dependencies=[Depends(user_api_key_auth)],
 )
-async def delete_allowed_ip(ip_address: IPAddress):
-    from litellm.proxy.proxy_server import general_settings, proxy_config
+async def delete_allowed_ip(
+    ip_address: IPAddress,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    from litellm.proxy.proxy_server import (
+        create_config_audit_log,
+        general_settings,
+        proxy_config,
+    )
 
     _allowed_ips: List = general_settings.get("allowed_ips", [])
     if ip_address.ip in _allowed_ips:
@@ -391,11 +413,22 @@ async def delete_allowed_ip(ip_address: IPAddress):
     if "allowed_ips" not in config["general_settings"]:
         config["general_settings"]["allowed_ips"] = []
 
+    before_allowed_ips = list(config["general_settings"]["allowed_ips"])
     if ip_address.ip in config["general_settings"]["allowed_ips"]:
         config["general_settings"]["allowed_ips"].remove(ip_address.ip)
 
     await proxy_config.save_config(new_config=config)
 
+    asyncio.create_task(
+        create_config_audit_log(
+            param_name="general_settings",
+            action="deleted",
+            before_value={"allowed_ips": before_allowed_ips},
+            after_value={"allowed_ips": config["general_settings"]["allowed_ips"]},
+            user_api_key_dict=user_api_key_dict,
+        )
+    )
+
     return {"message": f"IP {ip_address.ip} deleted successfully", "status": "success"}
 
 
@@ -554,7 +587,7 @@ async def _update_litellm_setting(
     settings: Union[DefaultInternalUserParams, DefaultTeamSSOParams, MCPSemanticFilterSettings],
     settings_key: str,
     success_message: str,
-    user_api_key_dict: Optional[UserAPIKeyAuth] = None,
+    user_api_key_dict: UserAPIKeyAuth,
 ):
     """
     Common utility function to update `litellm_settings` in both memory and config.
@@ -564,8 +597,6 @@ async def _update_litellm_setting(
         settings_key: The key in litellm_settings to update
         success_message: Message to return on success
         user_api_key_dict: The acting admin, recorded as the audit-log actor.
-            Optional today so callers that have not been wired for auditing
-            keep working; the audit row is only written when an actor is passed.
     """
     from litellm.proxy.proxy_server import (
         create_config_audit_log,
@@ -599,20 +630,19 @@ async def _update_litellm_setting(
     # Save the updated config
     await proxy_config.save_config(new_config=config)
 
-    if user_api_key_dict is not None:
-        # Fire-and-forget so an audit-log failure (transient DB blip, etc.)
-        # never surfaces as a 500 after save_config has already committed,
-        # matching the create_object_audit_log pattern used elsewhere
-        # (e.g. model_management_endpoints).
-        asyncio.create_task(
-            create_config_audit_log(
-                param_name=settings_key,
-                action="updated",
-                before_value=before_value,
-                after_value=in_memory_var,
-                user_api_key_dict=user_api_key_dict,
-            )
+    # Fire-and-forget so an audit-log failure (transient DB blip, etc.)
+    # never surfaces as a 500 after save_config has already committed,
+    # matching the create_object_audit_log pattern used elsewhere
+    # (e.g. model_management_endpoints).
+    asyncio.create_task(
+        create_config_audit_log(
+            param_name=settings_key,
+            action="updated",
+            before_value=before_value,
+            after_value=in_memory_var,
+            user_api_key_dict=user_api_key_dict,
         )
+    )
 
     return {
         "message": success_message,
@@ -653,7 +683,10 @@ async def update_internal_user_settings(
     tags=["SSO Settings"],
     dependencies=[Depends(user_api_key_auth)],
 )
-async def update_default_team_settings(settings: DefaultTeamSSOParams):
+async def update_default_team_settings(
+    settings: DefaultTeamSSOParams,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
     """
     Update the default team parameters for SSO users.
     These settings will be applied to new teams created from SSO.
@@ -662,6 +695,7 @@ async def update_default_team_settings(settings: DefaultTeamSSOParams):
         settings=settings,
         settings_key="default_team_params",
         success_message="Default team settings updated successfully",
+        user_api_key_dict=user_api_key_dict,
     )
 
 
@@ -772,7 +806,10 @@ async def get_sso_settings():
     tags=["SSO Settings"],
     dependencies=[Depends(user_api_key_auth)],
 )
-async def update_sso_settings(sso_config: SSOConfig):
+async def update_sso_settings(
+    sso_config: SSOConfig,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
     """
     Update SSO configuration by saving to the dedicated SSO table.
     """
@@ -780,6 +817,7 @@ async def update_sso_settings(sso_config: SSOConfig):
     import os
 
     from litellm.proxy.proxy_server import (
+        create_config_audit_log,
         prisma_client,
         proxy_config,
         store_model_in_db,
@@ -812,6 +850,20 @@ async def update_sso_settings(sso_config: SSOConfig):
         "proxy_base_url": "PROXY_BASE_URL",
     }
 
+    # Read the existing SSO row first so the audit log captures a real
+    # before/after diff. Stored values are encrypted; decrypt them so the
+    # before-snapshot has the same shape as after_value, and rely on
+    # create_config_audit_log's secret-name redaction to mask the
+    # *_client_secret fields before the audit row is written.
+    existing_sso_record = await SSOConfigRepository(prisma_client).table.find_unique(where={"id": "sso_config"})
+    before_sso_data: Optional[Dict[str, Any]] = None
+    if existing_sso_record and existing_sso_record.sso_settings:
+        stored = existing_sso_record.sso_settings
+        if isinstance(stored, str):
+            stored = json.loads(stored)
+        if isinstance(stored, dict):
+            before_sso_data = proxy_config._decrypt_db_variables(stored)
+
     # Load existing config
     config = await proxy_config.get_config()
 
@@ -850,6 +902,17 @@ async def update_sso_settings(sso_config: SSOConfig):
         },
     )
 
+    asyncio.create_task(
+        create_config_audit_log(
+            param_name="sso_config",
+            action="updated",
+            before_value=before_sso_data,
+            after_value=sso_data,
+            user_api_key_dict=user_api_key_dict,
+            table_name=LitellmTableNames.SSO_CONFIG_TABLE_NAME,
+        )
+    )
+
     # Remove SSO-related env vars from config.environment_variables
     try:
         env_var_entry = await ConfigRepository(prisma_client).table.find_unique(
@@ -943,14 +1006,21 @@ def _validate_public_image_url(value: Optional[str], field_name: str) -> None:
     tags=["UI Theme Settings"],
     dependencies=[Depends(user_api_key_auth)],
 )
-async def update_ui_theme_settings(theme_config: UIThemeConfig):
+async def update_ui_theme_settings(
+    theme_config: UIThemeConfig,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
     """
     Update UI theme configuration.
     Updates logo settings for the admin UI.
     """
     import os
 
-    from litellm.proxy.proxy_server import proxy_config, store_model_in_db
+    from litellm.proxy.proxy_server import (
+        create_config_audit_log,
+        proxy_config,
+        store_model_in_db,
+    )
 
     _validate_public_image_url(theme_config.logo_url, "logo_url")
     _validate_public_image_url(theme_config.favicon_url, "favicon_url")
@@ -963,6 +1033,7 @@ async def update_ui_theme_settings(theme_config: UIThemeConfig):
 
     # Load existing config
     config = await proxy_config.get_config()
+    before_theme = config.get("litellm_settings", {}).get("ui_theme_config")
 
     # Update config with UI theme settings
     if "general_settings" not in config:
@@ -1029,6 +1100,16 @@ async def update_ui_theme_settings(theme_config: UIThemeConfig):
     # Save the updated config
     await proxy_config.save_config(new_config=stored_config)
 
+    asyncio.create_task(
+        create_config_audit_log(
+            param_name="ui_theme_config",
+            action="updated",
+            before_value=before_theme,
+            after_value=theme_data,
+            user_api_key_dict=user_api_key_dict,
+        )
+    )
+
     return {
         "message": "UI theme settings updated successfully.",
         "status": "success",
@@ -1083,6 +1164,7 @@ async def update_mcp_semantic_filter_settings(
         settings=settings,
         settings_key="mcp_semantic_tool_filter",
         success_message="MCP Semantic Filter settings updated successfully. Changes will be applied across all pods within 10 seconds.",
+        user_api_key_dict=user_api_key_dict,
     )
     try:
         from litellm.proxy.proxy_server import prisma_client, proxy_config
@@ -1200,7 +1282,11 @@ async def update_ui_settings(
     Update UI-specific configuration flags.
     Only proxy admins are allowed to modify these settings.
     """
-    from litellm.proxy.proxy_server import prisma_client, store_model_in_db
+    from litellm.proxy.proxy_server import (
+        create_config_audit_log,
+        prisma_client,
+        store_model_in_db,
+    )
 
     if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
         raise HTTPException(status_code=403, detail="Only proxy admins can update UI settings.")
@@ -1282,6 +1368,17 @@ async def update_ui_settings(
     sanitized = {k: v for k, v in ui_settings.items() if k in ALLOWED_UI_SETTINGS_FIELDS}
     await user_api_key_cache.async_set_cache(key=UI_SETTINGS_CACHE_KEY, value=sanitized, ttl=UI_SETTINGS_CACHE_TTL)
 
+    asyncio.create_task(
+        create_config_audit_log(
+            param_name="ui_settings",
+            action="updated",
+            before_value=existing,
+            after_value=ui_settings,
+            user_api_key_dict=user_api_key_dict,
+            table_name=LitellmTableNames.UI_SETTINGS_TABLE_NAME,
+        )
+    )
+
     return {
         "message": "UI settings updated successfully",
         "status": "success",
diff --git a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py
index 7f5aee51f51..f39ff93cee7 100644
--- a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py
@@ -258,6 +258,7 @@ async def test_scim_create_user_respects_default_role_set_via_ui(mocker, monkeyp
     )
 
     import litellm
+    from litellm.proxy._types import UserAPIKeyAuth
 
     settings = DefaultInternalUserParams(
         user_role=LitellmUserRoles.INTERNAL_USER,
@@ -266,6 +267,7 @@ async def test_scim_create_user_respects_default_role_set_via_ui(mocker, monkeyp
         settings=settings,
         settings_key="default_internal_user_params",
         success_message="ok",
+        user_api_key_dict=UserAPIKeyAuth(user_id="test-admin"),
     )
 
     # Verify the in-memory variable was actually updated
diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py b/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py
index 443089b5f01..e0b90332ca0 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py
@@ -426,6 +426,7 @@ class TestUpdateLitellmSettingOrdering:
             settings=new_settings,
             settings_key="default_team_params",
             success_message="Updated",
+            user_api_key_dict=UserAPIKeyAuth(user_id="test-admin"),
         )
 
         # In-memory value should be the NEW value, not the stale one
@@ -459,6 +460,7 @@ class TestUpdateLitellmSettingOrdering:
                 settings=DefaultTeamSSOParams(max_budget=100.0),
                 settings_key="default_team_params",
                 success_message="Updated",
+                user_api_key_dict=UserAPIKeyAuth(user_id="test-admin"),
             )
 
         assert exc_info.value.status_code == 500
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index dc35d71ccbd..88d9ad0d968 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -8764,3 +8764,278 @@ def test_dump_redacted_config_serializes_non_json_native_values():
     restored = json.loads(out)
     assert "2026-06-30" in restored["updated_at"]
 
+
+@pytest.mark.asyncio
+async def test_update_config_general_settings_emits_audit_log(monkeypatch):
+    import litellm.proxy.proxy_server as proxy_server_module
+    from litellm.proxy._types import ConfigFieldUpdate
+    from litellm.proxy.proxy_server import update_config_general_settings
+
+    existing = {"max_parallel_requests": 5, "some_api_key": "sk-stored-secret"}
+    fake = _fake_prisma_with_config(existing)
+    monkeypatch.setattr(proxy_server_module, "prisma_client", fake)
+    monkeypatch.setattr(proxy_server_module, "premium_user", True)
+    monkeypatch.setattr(litellm, "store_audit_logs", True)
+
+    admin = UserAPIKeyAuth(
+        api_key="hashed-admin",
+        user_id="admin-1",
+        user_role=LitellmUserRoles.PROXY_ADMIN,
+    )
+    await update_config_general_settings(
+        data=ConfigFieldUpdate(
+            field_name="max_parallel_requests",
+            field_value=42,
+            config_type="general_settings",
+        ),
+        user_api_key_dict=admin,
+    )
+    # Audit is scheduled via asyncio.create_task; yield so it runs.
+    await asyncio.sleep(0)
+
+    fake.db.litellm_auditlog.create.assert_awaited_once()
+    written = fake.db.litellm_auditlog.create.call_args.kwargs["data"]
+    assert written["table_name"] == "LiteLLM_Config"
+    assert written["object_id"] == "general_settings"
+    assert written["action"] == "updated"
+    assert written["changed_by"] == "admin-1"
+
+    before = json.loads(written["before_value"])
+    after = json.loads(written["updated_values"])
+    assert before["max_parallel_requests"] == 5
+    assert after["max_parallel_requests"] == 42
+    assert "sk-stored-secret" not in written["before_value"]
+    assert "sk-stored-secret" not in written["updated_values"]
+    assert before["some_api_key"] != "sk-stored-secret"
+
+
+@pytest.mark.asyncio
+async def test_delete_config_general_settings_emits_deleted_audit_log(monkeypatch):
+    import litellm.proxy.proxy_server as proxy_server_module
+    from litellm.proxy._types import ConfigFieldDelete
+    from litellm.proxy.proxy_server import delete_config_general_settings
+
+    existing = {"max_parallel_requests": 5}
+    fake = _fake_prisma_with_config(existing)
+    monkeypatch.setattr(proxy_server_module, "prisma_client", fake)
+    monkeypatch.setattr(proxy_server_module, "premium_user", True)
+    monkeypatch.setattr(litellm, "store_audit_logs", True)
+
+    admin = UserAPIKeyAuth(
+        api_key="hashed-admin",
+        user_id="admin-1",
+        user_role=LitellmUserRoles.PROXY_ADMIN,
+    )
+    await delete_config_general_settings(
+        data=ConfigFieldDelete(
+            field_name="max_parallel_requests", config_type="general_settings"
+        ),
+        user_api_key_dict=admin,
+    )
+    # Audit is scheduled via asyncio.create_task; yield so it runs.
+    await asyncio.sleep(0)
+
+    fake.db.litellm_auditlog.create.assert_awaited_once()
+    written = fake.db.litellm_auditlog.create.call_args.kwargs["data"]
+    assert written["object_id"] == "general_settings"
+    assert written["action"] == "deleted"
+    before = json.loads(written["before_value"])
+    after = json.loads(written["updated_values"])
+    assert before["max_parallel_requests"] == 5
+    assert "max_parallel_requests" not in after
+
+
+def test_update_config_audits_every_written_section(_update_config_setup, monkeypatch):
+    """/config/update must emit one audit row per section it writes, so each
+    of the four call sites (general_settings, environment_variables,
+    litellm_settings, router_settings) is mutation-protected. litellm_settings
+    is the row that holds default_internal_user_params ("default user settings")."""
+    import litellm.proxy.proxy_server as proxy_server_module
+
+    client, prisma, restore = _update_config_setup(
+        initial_rows={"litellm_settings": {"drop_params": True}}
+    )
+    audit_create = AsyncMock()
+    prisma.db.litellm_auditlog.create = audit_create
+    monkeypatch.setattr(proxy_server_module, "premium_user", True)
+    monkeypatch.setattr(litellm, "store_audit_logs", True)
+    try:
+        resp = client.post(
+            "/config/update",
+            json={
+                "general_settings": {"store_prompts_in_spend_logs": True},
+                "environment_variables": {"FOO": "bar"},
+                "litellm_settings": {
+                    "default_internal_user_params": {"max_budget": 10}
+                },
+                "router_settings": {"routing_strategy": "latency-based-routing"},
+            },
+        )
+        assert resp.status_code == 200, resp.text
+
+        audited = {
+            call.kwargs["data"]["object_id"]: call.kwargs["data"]["action"]
+            for call in audit_create.await_args_list
+        }
+        assert audited == {
+            "general_settings": "updated",
+            "environment_variables": "updated",
+            "litellm_settings": "updated",
+            "router_settings": "updated",
+        }
+        for call in audit_create.await_args_list:
+            assert call.kwargs["data"]["table_name"] == "LiteLLM_Config"
+            assert call.kwargs["data"]["changed_by"] == "test_admin"
+
+        ls_call = next(
+            c
+            for c in audit_create.await_args_list
+            if c.kwargs["data"]["object_id"] == "litellm_settings"
+        )
+        after = json.loads(ls_call.kwargs["data"]["updated_values"])
+        assert after["default_internal_user_params"] == {"max_budget": 10}
+    finally:
+        restore()
+
+
+def test_delete_callback_audits_litellm_settings_deletion(
+    _update_config_setup, monkeypatch
+):
+    """/config/callback/delete must emit a deleted audit row for litellm_settings
+    capturing the success_callback list before and after removal."""
+    import litellm.proxy.proxy_server as proxy_server_module
+
+    client, prisma, restore = _update_config_setup()
+    audit_create = AsyncMock()
+    prisma.db.litellm_auditlog.create = audit_create
+    monkeypatch.setattr(proxy_server_module, "premium_user", True)
+    monkeypatch.setattr(litellm, "store_audit_logs", True)
+
+    from litellm.proxy.proxy_server import proxy_config as real_proxy_config
+
+    monkeypatch.setattr(
+        real_proxy_config,
+        "get_config",
+        AsyncMock(
+            return_value={
+                "litellm_settings": {"success_callback": ["langfuse", "datadog"]}
+            }
+        ),
+    )
+    monkeypatch.setattr(
+        real_proxy_config, "save_config", AsyncMock(return_value=None)
+    )
+    try:
+        resp = client.post(
+            "/config/callback/delete", json={"callback_name": "datadog"}
+        )
+        assert resp.status_code == 200, resp.text
+
+        audit_create.assert_awaited_once()
+        written = audit_create.await_args.kwargs["data"]
+        assert written["object_id"] == "litellm_settings"
+        assert written["action"] == "deleted"
+        before = json.loads(written["before_value"])
+        after = json.loads(written["updated_values"])
+        assert before["success_callback"] == ["langfuse", "datadog"]
+        assert after["success_callback"] == ["langfuse"]
+    finally:
+        restore()
+
+
+def test_delete_callback_audits_before_reload_failure(_update_config_setup, monkeypatch):
+    import litellm.proxy.proxy_server as proxy_server_module
+
+    client, prisma, restore = _update_config_setup()
+    audit_create = AsyncMock()
+    prisma.db.litellm_auditlog.create = audit_create
+    monkeypatch.setattr(proxy_server_module, "premium_user", True)
+    monkeypatch.setattr(litellm, "store_audit_logs", True)
+
+    from litellm.proxy.proxy_server import proxy_config as real_proxy_config
+
+    monkeypatch.setattr(
+        real_proxy_config,
+        "get_config",
+        AsyncMock(
+            return_value={
+                "litellm_settings": {"success_callback": ["langfuse", "datadog"]}
+            }
+        ),
+    )
+    monkeypatch.setattr(
+        real_proxy_config, "save_config", AsyncMock(return_value=None)
+    )
+    monkeypatch.setattr(
+        real_proxy_config,
+        "add_deployment",
+        AsyncMock(side_effect=RuntimeError("reload failed")),
+    )
+    try:
+        resp = client.post(
+            "/config/callback/delete", json={"callback_name": "datadog"}
+        )
+        assert resp.status_code == 500, resp.text
+
+        audit_create.assert_awaited_once()
+        written = audit_create.await_args.kwargs["data"]
+        assert written["object_id"] == "litellm_settings"
+        assert written["action"] == "deleted"
+    finally:
+        restore()
+
+
+def test_update_config_redacts_all_environment_variable_values(
+    _update_config_setup, monkeypatch
+):
+    """environment_variables hold credentials under arbitrary uppercase keys
+    (DATABASE_URL) that key-name secret matching misses, so every value in the
+    section must be redacted before the audit row is written; a plaintext
+    secret must never reach LiteLLM_AuditLog."""
+    import litellm.proxy.proxy_server as proxy_server_module
+
+    # DATABASE_URL is the bug class: an uppercase env key that key-name secret
+    # matching does NOT flag, so only whole-section value redaction protects it.
+    client, prisma, restore = _update_config_setup(
+        initial_rows={
+            "environment_variables": {
+                "DATABASE_URL": "enc:postgresql://OLDsecret@old.host:5432/db"
+            }
+        }
+    )
+    audit_create = AsyncMock()
+    prisma.db.litellm_auditlog.create = audit_create
+    monkeypatch.setattr(proxy_server_module, "premium_user", True)
+    monkeypatch.setattr(litellm, "store_audit_logs", True)
+    try:
+        resp = client.post(
+            "/config/update",
+            json={
+                "environment_variables": {
+                    "DATABASE_URL": "postgresql://u:p@db.internal:5432/litellm",
+                    "LOG_LEVEL": "debug",
+                }
+            },
+        )
+        assert resp.status_code == 200, resp.text
+
+        env_call = next(
+            c
+            for c in audit_create.await_args_list
+            if c.kwargs["data"]["object_id"] == "environment_variables"
+        )
+        data = env_call.kwargs["data"]
+
+        # the pre-existing secret must be redacted in the before snapshot
+        before = json.loads(data["before_value"])
+        assert before == {"DATABASE_URL": "REDACTED"}
+        assert "OLDsecret" not in data["before_value"]
+        assert "old.host" not in data["before_value"]
+
+        # the newly-written values must be redacted in the after snapshot
+        after = json.loads(data["updated_values"])
+        assert after == {"DATABASE_URL": "REDACTED", "LOG_LEVEL": "REDACTED"}
+        assert "postgresql://" not in data["updated_values"]
+        assert "db.internal" not in data["updated_values"]
+    finally:
+        restore()
diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
index 7a586f758f4..cb77c42fe9b 100644
--- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
+++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
@@ -396,6 +396,7 @@ class TestProxySettingEndpoints:
 
         # Mock the prisma client
         mock_prisma = MagicMock()
+        mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None)
         mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock()
         mock_prisma.db.litellm_config = MagicMock()
         mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None)
@@ -466,6 +467,62 @@ class TestProxySettingEndpoints:
         create_sso_settings = json.loads(create_data["sso_settings"])
         assert create_sso_settings["google_client_id"] == "new_google_client_id"
 
+    def test_update_sso_settings_audits_when_env_cleanup_fails(
+        self, mock_proxy_config, mock_auth, monkeypatch
+    ):
+        import json
+        from unittest.mock import AsyncMock, MagicMock
+
+        monkeypatch.setenv("LITELLM_SALT_KEY", "test_salt_key")
+        monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True)
+
+        mock_prisma = MagicMock()
+        mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None)
+        mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock()
+        mock_prisma.db.litellm_config = MagicMock()
+        mock_prisma.db.litellm_config.find_unique = AsyncMock(
+            side_effect=ValueError("cleanup failed")
+        )
+        mock_prisma.db.litellm_config.update = AsyncMock()
+        monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma)
+
+        from litellm.proxy.proxy_server import proxy_config
+
+        monkeypatch.setattr(
+            proxy_config,
+            "_encrypt_env_variables",
+            lambda environment_variables: environment_variables,
+        )
+
+        create_config_audit_log = AsyncMock()
+        monkeypatch.setattr(
+            "litellm.proxy.proxy_server.create_config_audit_log",
+            create_config_audit_log,
+        )
+
+        response = client.patch(
+            "/update/sso_settings",
+            json={"google_client_id": "new_google_client_id"},
+        )
+
+        assert response.status_code == 500
+        assert mock_prisma.db.litellm_ssoconfig.upsert.called
+        create_config_audit_log.assert_awaited_once()
+        audit_log_kwargs = create_config_audit_log.await_args.kwargs
+        assert audit_log_kwargs["param_name"] == "sso_config"
+        assert (
+            audit_log_kwargs["after_value"]["google_client_id"]
+            == "new_google_client_id"
+        )
+        assert (
+            json.loads(
+                mock_prisma.db.litellm_ssoconfig.upsert.call_args.kwargs["data"][
+                    "create"
+                ]["sso_settings"]
+            )["google_client_id"]
+            == "new_google_client_id"
+        )
+
     def test_update_sso_settings_with_null_values_clears_env_vars(
         self, mock_proxy_config, mock_auth, monkeypatch
     ):
@@ -478,6 +535,7 @@ class TestProxySettingEndpoints:
 
         # Mock the prisma client
         mock_prisma = MagicMock()
+        mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None)
         mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock()
         mock_prisma.db.litellm_config = MagicMock()
 
@@ -557,6 +615,7 @@ class TestProxySettingEndpoints:
 
         # Mock the prisma client
         mock_prisma = MagicMock()
+        mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None)
         mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock()
         mock_prisma.db.litellm_config = MagicMock()
         env_var_entry = MagicMock()
@@ -627,6 +686,7 @@ class TestProxySettingEndpoints:
 
         # Mock the prisma client
         mock_prisma = MagicMock()
+        mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None)
         mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock()
         mock_prisma.db.litellm_config = MagicMock()
 
@@ -704,6 +764,7 @@ class TestProxySettingEndpoints:
 
         # Mock the prisma client
         mock_prisma = MagicMock()
+        mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None)
         mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock()
         mock_prisma.db.litellm_config = MagicMock()
         mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None)
@@ -1350,6 +1411,7 @@ class TestProxySettingEndpoints:
         # Mock the prisma client
         mock_prisma = MagicMock()
         upsert_mock = AsyncMock()
+        mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None)
         mock_prisma.db.litellm_ssoconfig.upsert = upsert_mock
         mock_prisma.db.litellm_config = MagicMock()
         mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None)
@@ -1429,6 +1491,7 @@ class TestProxySettingEndpoints:
         mock_prisma = MagicMock()
         mock_prisma.db = MagicMock()
         mock_prisma.db.litellm_ssoconfig = MagicMock()
+        mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None)
         mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock()
 
         env_var_entry = MagicMock()
@@ -1480,6 +1543,7 @@ class TestProxySettingEndpoints:
         mock_prisma = MagicMock()
         mock_prisma.db = MagicMock()
         mock_prisma.db.litellm_ssoconfig = MagicMock()
+        mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None)
         mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock()
 
         env_var_entry = MagicMock()
@@ -1651,6 +1715,7 @@ class TestProxySettingEndpoints:
 
         # Mock the prisma client
         mock_prisma = MagicMock()
+        mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None)
         mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock()
         mock_prisma.db.litellm_config = MagicMock()
         mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None)
@@ -1960,3 +2025,345 @@ def test_update_internal_user_settings_returns_200_when_audit_write_raises(
         assert resp.json()["status"] == "success"
     finally:
         app.dependency_overrides.pop(user_api_key_auth, None)
+
+
+def test_update_sso_settings_writes_redacted_audit_log(mock_proxy_config, monkeypatch):
+    """Updating SSO settings must write an audit row to the SSO config table
+    with the client secret redacted."""
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm
+    import litellm.proxy.proxy_server as proxy_server_module
+    from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+
+    audit_create = AsyncMock()
+    fake_prisma = MagicMock()
+    fake_prisma.db.litellm_auditlog.create = audit_create
+    fake_prisma.db.litellm_ssoconfig.upsert = AsyncMock()
+    # No prior SSO row, so before_value resolves to None.
+    fake_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None)
+    fake_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None)
+
+    monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma)
+    monkeypatch.setattr(proxy_server_module, "premium_user", True)
+    monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True)
+    monkeypatch.setattr(litellm, "store_audit_logs", True)
+    monkeypatch.setattr(
+        proxy_server_module.proxy_config,
+        "_encrypt_env_variables",
+        lambda environment_variables: environment_variables,
+    )
+
+    async def _admin_auth():
+        return UserAPIKeyAuth(
+            user_id="audit-admin",
+            api_key="hashed-admin-key",
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+        )
+
+    app.dependency_overrides[user_api_key_auth] = _admin_auth
+    try:
+        resp = client.patch(
+            "/update/sso_settings",
+            json={
+                "google_client_id": "client-id-123",
+                "google_client_secret": "super-secret-xyz",
+            },
+        )
+        assert resp.status_code == 200, resp.text
+
+        audit_create.assert_awaited_once()
+        written = audit_create.await_args.kwargs["data"]
+        assert written["object_id"] == "sso_config"
+        assert written["table_name"] == "LiteLLM_SSOConfig"
+        assert written["changed_by"] == "audit-admin"
+
+        after = json.loads(written["updated_values"])
+        assert after["google_client_id"] == "client-id-123"
+        assert after["google_client_secret"] == "REDACTED"
+        assert "super-secret-xyz" not in written["updated_values"]
+    finally:
+        app.dependency_overrides.pop(user_api_key_auth, None)
+
+
+def test_update_sso_settings_audit_captures_redacted_before_snapshot(
+    mock_proxy_config, monkeypatch
+):
+    """An auditor reviewing an SSO secret rotation needs to see a real
+    before/after diff in the audit row, not before_value=None. The endpoint
+    reads the existing (encrypted) SSO row, decrypts it, and lets the audit
+    helper redact the *_client_secret fields before persistence so neither
+    the old nor the new plaintext secret is recorded."""
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm
+    import litellm.proxy.proxy_server as proxy_server_module
+    from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+
+    audit_create = AsyncMock()
+    fake_prisma = MagicMock()
+    fake_prisma.db.litellm_auditlog.create = audit_create
+    fake_prisma.db.litellm_ssoconfig.upsert = AsyncMock()
+
+    # Pre-existing SSO row contains the *prior* secret (would be ciphertext in
+    # production; the test patches _decrypt_db_variables to pass through).
+    existing_record = MagicMock()
+    existing_record.sso_settings = {
+        "google_client_id": "old-client-id",
+        "google_client_secret": "OLD-SUPER-SECRET",
+    }
+    fake_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=existing_record)
+    fake_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None)
+
+    monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma)
+    monkeypatch.setattr(proxy_server_module, "premium_user", True)
+    monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True)
+    monkeypatch.setattr(litellm, "store_audit_logs", True)
+    monkeypatch.setattr(
+        proxy_server_module.proxy_config,
+        "_encrypt_env_variables",
+        lambda environment_variables: environment_variables,
+    )
+    # Pretend the stored value is already plaintext for the test (production
+    # decrypts via Fernet); the audit helper still has to redact it.
+    monkeypatch.setattr(
+        proxy_server_module.proxy_config,
+        "_decrypt_db_variables",
+        lambda variables_dict: dict(variables_dict),
+    )
+
+    async def _admin_auth():
+        return UserAPIKeyAuth(
+            user_id="audit-admin",
+            api_key="hashed-admin-key",
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+        )
+
+    app.dependency_overrides[user_api_key_auth] = _admin_auth
+    try:
+        resp = client.patch(
+            "/update/sso_settings",
+            json={
+                "google_client_id": "new-client-id",
+                "google_client_secret": "NEW-SUPER-SECRET",
+            },
+        )
+        assert resp.status_code == 200, resp.text
+
+        audit_create.assert_awaited_once()
+        written = audit_create.await_args.kwargs["data"]
+        before = json.loads(written["before_value"])
+        after = json.loads(written["updated_values"])
+
+        # Non-secret field shows the diff
+        assert before["google_client_id"] == "old-client-id"
+        assert after["google_client_id"] == "new-client-id"
+
+        # Secret field is redacted in BOTH snapshots — auditor sees the
+        # rotation event without ever seeing either plaintext secret.
+        assert before["google_client_secret"] == "REDACTED"
+        assert after["google_client_secret"] == "REDACTED"
+        assert "OLD-SUPER-SECRET" not in written["before_value"]
+        assert "NEW-SUPER-SECRET" not in written["updated_values"]
+    finally:
+        app.dependency_overrides.pop(user_api_key_auth, None)
+
+
+def test_add_allowed_ip_writes_audit_log(mock_proxy_config, monkeypatch):
+    """Adding an allowed IP is a system-wide security setting change and must
+    be audited with the before and after IP list."""
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm
+    import litellm.proxy.proxy_server as proxy_server_module
+    from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+
+    audit_create = AsyncMock()
+    fake_prisma = MagicMock()
+    fake_prisma.db.litellm_auditlog.create = audit_create
+
+    monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma)
+    monkeypatch.setattr(proxy_server_module, "premium_user", True)
+    monkeypatch.setattr(proxy_server_module, "general_settings", {})
+    monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True)
+    monkeypatch.setattr(litellm, "store_audit_logs", True)
+
+    async def _admin_auth():
+        return UserAPIKeyAuth(
+            user_id="audit-admin",
+            api_key="hashed-admin-key",
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+        )
+
+    app.dependency_overrides[user_api_key_auth] = _admin_auth
+    try:
+        resp = client.post("/add/allowed_ip", json={"ip": "203.0.113.77"})
+        assert resp.status_code == 200, resp.text
+
+        audit_create.assert_awaited_once()
+        written = audit_create.await_args.kwargs["data"]
+        assert written["object_id"] == "general_settings"
+        assert written["action"] == "updated"
+        assert written["changed_by"] == "audit-admin"
+
+        before = json.loads(written["before_value"])
+        after = json.loads(written["updated_values"])
+        assert "203.0.113.77" not in before["allowed_ips"]
+        assert "203.0.113.77" in after["allowed_ips"]
+    finally:
+        app.dependency_overrides.pop(user_api_key_auth, None)
+
+
+def test_delete_allowed_ip_writes_deleted_audit_log(monkeypatch):
+    """Removing an allowed IP must be audited as a deletion, symmetric with the
+    add path."""
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm
+    import litellm.proxy.proxy_server as proxy_server_module
+    from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+
+    audit_create = AsyncMock()
+    fake_prisma = MagicMock()
+    fake_prisma.db.litellm_auditlog.create = audit_create
+
+    config = {"general_settings": {"allowed_ips": ["203.0.113.77", "198.51.100.1"]}}
+
+    async def _get_config():
+        return config
+
+    async def _save_config(new_config=None):
+        nonlocal config
+        if new_config is not None:
+            config = new_config
+        return config
+
+    monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma)
+    monkeypatch.setattr(proxy_server_module, "premium_user", True)
+    monkeypatch.setattr(
+        proxy_server_module, "general_settings", {"allowed_ips": ["203.0.113.77"]}
+    )
+    monkeypatch.setattr(litellm, "store_audit_logs", True)
+    monkeypatch.setattr(proxy_server_module.proxy_config, "get_config", _get_config)
+    monkeypatch.setattr(proxy_server_module.proxy_config, "save_config", _save_config)
+
+    async def _admin_auth():
+        return UserAPIKeyAuth(
+            user_id="audit-admin",
+            api_key="hashed-admin-key",
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+        )
+
+    app.dependency_overrides[user_api_key_auth] = _admin_auth
+    try:
+        resp = client.post("/delete/allowed_ip", json={"ip": "203.0.113.77"})
+        assert resp.status_code == 200, resp.text
+
+        audit_create.assert_awaited_once()
+        written = audit_create.await_args.kwargs["data"]
+        assert written["object_id"] == "general_settings"
+        assert written["action"] == "deleted"
+        before = json.loads(written["before_value"])
+        after = json.loads(written["updated_values"])
+        assert "203.0.113.77" in before["allowed_ips"]
+        assert "203.0.113.77" not in after["allowed_ips"]
+    finally:
+        app.dependency_overrides.pop(user_api_key_auth, None)
+
+
+def test_update_ui_theme_settings_writes_audit_log(mock_proxy_config, monkeypatch):
+    """Updating the UI theme must be audited under ui_theme_config."""
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm
+    import litellm.proxy.proxy_server as proxy_server_module
+    from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+
+    audit_create = AsyncMock()
+    fake_prisma = MagicMock()
+    fake_prisma.db.litellm_auditlog.create = audit_create
+
+    monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma)
+    monkeypatch.setattr(proxy_server_module, "premium_user", True)
+    monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True)
+    monkeypatch.setattr(litellm, "store_audit_logs", True)
+    monkeypatch.setattr(
+        proxy_server_module.proxy_config,
+        "_encrypt_env_variables",
+        lambda environment_variables: environment_variables,
+    )
+
+    async def _admin_auth():
+        return UserAPIKeyAuth(
+            user_id="audit-admin",
+            api_key="hashed-admin-key",
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+        )
+
+    app.dependency_overrides[user_api_key_auth] = _admin_auth
+    try:
+        resp = client.patch(
+            "/update/ui_theme_settings",
+            json={"logo_url": "https://example.com/logo.png"},
+        )
+        assert resp.status_code == 200, resp.text
+
+        audit_create.assert_awaited_once()
+        written = audit_create.await_args.kwargs["data"]
+        assert written["object_id"] == "ui_theme_config"
+        assert written["action"] == "updated"
+        assert written["changed_by"] == "audit-admin"
+        after = json.loads(written["updated_values"])
+        assert after["logo_url"] == "https://example.com/logo.png"
+    finally:
+        app.dependency_overrides.pop(user_api_key_auth, None)
+
+
+def test_update_ui_settings_writes_audit_log(monkeypatch):
+    """Updating UI settings must be audited under the UI settings table."""
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm
+    import litellm.proxy.proxy_server as proxy_server_module
+    from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+
+    audit_create = AsyncMock()
+    fake_prisma = MagicMock()
+    fake_prisma.db.litellm_auditlog.create = audit_create
+    fake_prisma.db.litellm_uisettings.find_unique = AsyncMock(return_value=None)
+    fake_prisma.db.litellm_uisettings.upsert = AsyncMock()
+
+    monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma)
+    monkeypatch.setattr(proxy_server_module, "premium_user", True)
+    monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True)
+    monkeypatch.setattr(litellm, "store_audit_logs", True)
+
+    async def _admin_auth():
+        return UserAPIKeyAuth(
+            user_id="audit-admin",
+            api_key="hashed-admin-key",
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+        )
+
+    app.dependency_overrides[user_api_key_auth] = _admin_auth
+    try:
+        resp = client.patch(
+            "/update/ui_settings", json={"disable_custom_api_keys": True}
+        )
+        assert resp.status_code == 200, resp.text
+
+        audit_create.assert_awaited_once()
+        written = audit_create.await_args.kwargs["data"]
+        assert written["object_id"] == "ui_settings"
+        assert written["table_name"] == "LiteLLM_UISettings"
+        assert written["changed_by"] == "audit-admin"
+        after = json.loads(written["updated_values"])
+        assert after["disable_custom_api_keys"] is True
+    finally:
+        app.dependency_overrides.pop(user_api_key_auth, None)

From c4a77bded7b3e21e0ca8bf52caaa75b9442f6145 Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Wed, 1 Jul 2026 10:44:02 +0800
Subject: [PATCH 49/51] fix(prometheus): expose project_alias in custom
 metadata labels (LIT-3741) (#31784)

Include top-level scalar fields from standard_logging_metadata in the
combined metadata dict used by custom_prometheus_metadata_labels. Previously
only nested sub-dicts (requester_metadata, user_api_key_auth_metadata,
spend_logs_metadata) were spread into combined_metadata, so fields like
user_api_key_project_alias were inaccessible and always resolved to None.

Co-authored-by: unknown <>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/integrations/prometheus.py            |  5 +
 .../test_prometheus_spend_logs_metadata.py    | 98 ++++++++++++++++++-
 2 files changed, 102 insertions(+), 1 deletion(-)

diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py
index 1f516e9dc93..8eb6eaa8e2b 100644
--- a/litellm/integrations/prometheus.py
+++ b/litellm/integrations/prometheus.py
@@ -3804,6 +3804,10 @@ def _get_combined_custom_metadata_from_standard_logging_payload(
 ) -> Dict[str, Any]:
     """
     Combine the metadata sources that can supply custom Prometheus labels.
+
+    Includes top-level scalar fields from the standard logging metadata (e.g.
+    user_api_key_project_alias, user_api_key_team_alias) so they are accessible
+    via custom_prometheus_metadata_labels configuration.
     """
     if not isinstance(standard_logging_payload, dict):
         return {}
@@ -3817,6 +3821,7 @@ def _get_combined_custom_metadata_from_standard_logging_payload(
     spend_logs_metadata = standard_logging_metadata.get("spend_logs_metadata")
 
     return {
+        **{k: v for k, v in standard_logging_metadata.items() if not isinstance(v, dict)},
         **(requester_metadata if isinstance(requester_metadata, dict) else {}),
         **(user_api_key_auth_metadata if isinstance(user_api_key_auth_metadata, dict) else {}),
         **(spend_logs_metadata if isinstance(spend_logs_metadata, dict) else {}),
diff --git a/tests/test_litellm/integrations/test_prometheus_spend_logs_metadata.py b/tests/test_litellm/integrations/test_prometheus_spend_logs_metadata.py
index 31934e5fd8e..e2af6fd2daf 100644
--- a/tests/test_litellm/integrations/test_prometheus_spend_logs_metadata.py
+++ b/tests/test_litellm/integrations/test_prometheus_spend_logs_metadata.py
@@ -5,7 +5,10 @@ Verifies that metadata from x-litellm-spend-logs-metadata header is available
 in Prometheus custom labels via combined_metadata.
 """
 
-from litellm.integrations.prometheus import get_custom_labels_from_metadata
+from litellm.integrations.prometheus import (
+    _get_combined_custom_metadata_from_standard_logging_payload,
+    get_custom_labels_from_metadata,
+)
 
 
 def test_get_custom_labels_includes_spend_logs_metadata(monkeypatch):
@@ -109,3 +112,96 @@ def test_combined_metadata_with_none_spend_logs(monkeypatch):
 
     result = get_custom_labels_from_metadata(combined_metadata)
     assert result == {"metadata_foo": "bar"}
+
+
+def test_combined_metadata_includes_top_level_fields():
+    """
+    Regression test for LIT-3741: user_api_key_project_alias (and other
+    top-level metadata fields) must be included in the combined metadata
+    so they can be referenced via custom_prometheus_metadata_labels.
+    """
+    standard_logging_payload = {
+        "metadata": {
+            "user_api_key_hash": "sk-abc123",
+            "user_api_key_alias": "hotel-key",
+            "user_api_key_team_id": "team-1",
+            "user_api_key_team_alias": "hotel-team",
+            "user_api_key_project_id": "proj-1",
+            "user_api_key_project_alias": "hotel-recommendations",
+            "user_api_key_user_id": "user-1",
+            "user_api_key_user_email": "user@example.com",
+            "user_api_key_end_user_id": None,
+            "user_api_key_org_id": None,
+            "user_api_key_org_alias": None,
+            "user_api_key_request_route": "/v1/chat/completions",
+            "requester_metadata": {"custom_field": "custom_value"},
+            "user_api_key_auth_metadata": {"auth_field": "auth_value"},
+            "spend_logs_metadata": None,
+        }
+    }
+
+    combined = _get_combined_custom_metadata_from_standard_logging_payload(
+        standard_logging_payload
+    )
+
+    assert combined["user_api_key_project_alias"] == "hotel-recommendations"
+    assert combined["user_api_key_project_id"] == "proj-1"
+    assert combined["user_api_key_team_alias"] == "hotel-team"
+    assert combined["user_api_key_request_route"] == "/v1/chat/completions"
+    assert combined["custom_field"] == "custom_value"
+    assert combined["auth_field"] == "auth_value"
+
+
+def test_project_alias_accessible_via_custom_prometheus_labels(monkeypatch):
+    """
+    Regression test for LIT-3741: configuring
+    custom_prometheus_metadata_labels with "metadata.user_api_key_project_alias"
+    should produce a label with the project's alias value.
+    """
+    monkeypatch.setattr(
+        "litellm.custom_prometheus_metadata_labels",
+        ["metadata.user_api_key_project_alias"],
+    )
+
+    standard_logging_payload = {
+        "metadata": {
+            "user_api_key_project_alias": "hotel-recommendations",
+            "requester_metadata": None,
+            "user_api_key_auth_metadata": None,
+            "spend_logs_metadata": None,
+        }
+    }
+
+    combined = _get_combined_custom_metadata_from_standard_logging_payload(
+        standard_logging_payload
+    )
+    result = get_custom_labels_from_metadata(combined)
+
+    assert result == {"metadata_user_api_key_project_alias": "hotel-recommendations"}
+
+
+def test_project_alias_accessible_without_prefix(monkeypatch):
+    """
+    user_api_key_project_alias should also be accessible without
+    the "metadata." prefix in custom_prometheus_metadata_labels config.
+    """
+    monkeypatch.setattr(
+        "litellm.custom_prometheus_metadata_labels",
+        ["user_api_key_project_alias"],
+    )
+
+    standard_logging_payload = {
+        "metadata": {
+            "user_api_key_project_alias": "hotel-recommendations",
+            "requester_metadata": None,
+            "user_api_key_auth_metadata": None,
+            "spend_logs_metadata": None,
+        }
+    }
+
+    combined = _get_combined_custom_metadata_from_standard_logging_payload(
+        standard_logging_payload
+    )
+    result = get_custom_labels_from_metadata(combined)
+
+    assert result == {"user_api_key_project_alias": "hotel-recommendations"}

From cca71a07c20e065fe0bd2fa1857cba783946ac93 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrish+github@berri.ai>
Date: Tue, 30 Jun 2026 20:03:59 -0700
Subject: [PATCH 50/51] feat(mcp): add mcp_tool_search virtual tools for large
 tool catalogs (#31777)

* feat(mcp): add tool search virtual tools for large catalogs

When mcp_tool_search_enabled is set on a key's object_permission,
tools/list returns only mcp_tool_search and mcp_tool_call instead of
the full catalog. The LLM searches by keyword then calls discovered
tools by name, avoiding context bloat with 100+ tool deployments.

* fix(mcp): persist mcp_tool_search_enabled and route tool_call by name

The mcp_tool_search_enabled flag existed on the Pydantic models but the
Prisma schema lacked the column, so keys generated with the flag never
persisted it and tools/list kept returning the full catalog. Add the
column across all three schema.prisma copies plus a migration.

handle_mcp_tool_call passed server_name="" into call_tool, which built a
malformed prefixed name ("-<tool>") and failed to resolve the server.
Resolve the caller's allowed servers and dispatch through execute_mcp_tool
instead, matching how the normal /tools/call path routes.

* fix(mcp): filter list_tools to virtual tools on the protocol path

The REST surface (/mcp-rest/tools/list) returned only the two virtual
tools when mcp_tool_search_enabled was set, but the MCP protocol handler
(handle_list_tools, used by real MCP clients over streamable-http/SSE)
still returned the full catalog. Apply the same early return there so an
actual MCP client sees mcp_tool_search and mcp_tool_call instead of every
tool. call_tool was already intercepted on this path.

* fix(mcp): enforce IP + server filtering on virtual tool search/call

Review flagged that the virtual mcp_tool_search/mcp_tool_call path skipped
access controls the normal MCP flow applies. mcp_tool_call resolved allowed
servers from key permissions only, never applying IP filtering, so a caller
on a public IP could invoke a tool on a server marked
available_on_public_internet: false. mcp_tool_search listed the raw catalog
via global_mcp_server_manager.list_tools, exposing tool names/schemas that
/tools/list would hide and ignoring per-key/per-server tool filters.

Route both virtual handlers through the same filtered paths used by the
normal MCP flow: search now calls _list_mcp_tools and call resolves servers
via _get_allowed_mcp_servers, both threaded with the request client IP so
filter_server_ids_by_ip applies. execute_mcp_tool then enforces the server
allowlist and per-key tool permissions. Thread client_ip through
_list_mcp_tools/_get_tools_from_mcp_servers and pass it from the REST and
SSE call sites.

* fix(ci): ruff format server.py and sync dashboard API types

ruff format normalizes the list_tools client_ip changes in server.py, and
schema.d.ts gains the mcp_tool_search_enabled object-permission field so the
generated dashboard types match the proxy OpenAPI spec.

* style(mcp): drop quoted annotations and sort imports

Clears UP037 on the virtual tool handler signatures (redundant with
from __future__ import annotations) and I001 on the list_tools import block.

* refactor(mcp): extract virtual-tool dispatch and host progress capture

Pulls the mcp_tool_search/mcp_tool_call interception and the host
progress-callback setup out of mcp_server_tool_call into helpers, keeping
that handler under the strict cyclomatic-complexity ceiling after the
client_ip threading. No behavior change.

* test(mcp): cover SSE virtual-tool dispatch and host progress helpers

Adds unit tests for _dispatch_virtual_mcp_tool (non-virtual passthrough,
flag-disabled rejection, search/call routing with client_ip),
_capture_host_progress_callback, and the protocol list_tools virtual
early-return, covering the new server.py paths.

* fix(mcp): forward per-request auth headers through virtual tool handlers

The virtual mcp_tool_search/mcp_tool_call path intercepted the request
before the normal header extraction ran, so client-supplied per-request
auth (Authorization for upstream pass-through, x-mcp-auth-<alias>) was
dropped and execute_mcp_tool/_list_mcp_tools received None. Thread
mcp_auth_header, mcp_server_auth_headers, oauth2_headers, and raw_headers
from both the REST and SSE call sites through the handlers so upstream MCP
servers that require pass-through auth can be listed and called.

* fix(mcp): preserve requested server scope in virtual tool calls

A scoped MCP session (/mcp/<server>/ or header-scoped) carries an
mcp_servers scope that the normal call path passes into routing so the
session can only reach that server. The virtual-tool branch dropped it and
resolved with mcp_servers=None, letting a scoped session call mcp_tool_call
for any server the key can access. Thread the context mcp_servers scope
through _dispatch_virtual_mcp_tool into both handlers so search and call
resolve against the same scoped server set.

* fix(mcp): convert virtual tool errors to isError on the protocol path

The virtual-tool dispatch ran before the protocol handler's HTTPException
and guardrail handling, so a rejected virtual call (e.g. an out-of-scope
403 from execute_mcp_tool) raised out of mcp_server_tool_call and broke the
MCP JSON-RPC stream instead of returning an isError CallToolResult. Move
the dispatch inside the same try that wraps call_mcp_tool so virtual-tool
errors get the same isError conversion as normal tool calls.

* fix(mcp): spend-log virtual tool calls on the REST path

The REST virtual-tool branch returned before common_processing_pre_call_logic,
so execute_mcp_tool ran without a litellm_logging_obj and virtual mcp_tool_call
invocations were not spend-logged or guardrail-checked like normal calls. Run
the same pre-call pipeline in the call branch and thread the resulting
litellm_logging_obj through handle_mcp_tool_call into execute_mcp_tool.

* fix(mcp): reject virtual tool call when key has no accessible servers

handle_mcp_tool_call passed an empty allowed_mcp_servers list into
execute_mcp_tool; an unprefixed local tool name then fell through to the
local registry, which has no server permission check, so a key with only
mcp_tool_search_enabled and no server grants could run operator-configured
local tools by name. Reject with 403 before dispatch when no servers are
accessible, matching call_mcp_tool.

* docs(mcp): document virtual tool_search module and parity rule in AGENTS.md

* style(mcp): apply ruff format at repo line-length (120)

* fix(mcp): add mcp_tool_search_enabled to ObjectPermissionDict and customer test fixture

* chore: trigger CI

* fix(mcp): mirror pre-call pipeline, guard imports, coerce top_k, honor include_disabled_tools

- SSE mcp_tool_call now runs common_processing_pre_call_logic so it spend-logs and runs guardrails like the REST path (P1)
- coerce_top_k avoids ValueError on non-integer top_k from clients (both REST and SSE)
- guard mcp.types import in tool_search behind runtime/TYPE_CHECKING per package convention
- admin list with include_disabled_tools returns the real catalog even when mcp_tool_search_enabled is set
---
 .../migration.sql                             |   2 +
 .../litellm_proxy_extras/schema.prisma        |   1 +
 litellm/models/object_permission.py           |   1 +
 .../proxy/_experimental/mcp_server/AGENTS.md  |   6 +
 .../mcp_server/rest_endpoints.py              |  86 +-
 .../proxy/_experimental/mcp_server/server.py  | 228 ++++-
 .../_experimental/mcp_server/tool_search.py   | 157 ++++
 litellm/proxy/_types.py                       |   1 +
 litellm/proxy/schema.prisma                   |   1 +
 litellm/types/object_permission.py            |   1 +
 schema.prisma                                 |   1 +
 .../mcp_server/test_mcp_tool_search.py        | 837 ++++++++++++++++++
 .../test_customer_endpoints.py                |   1 +
 .../test_object_permission_utils.py           |  32 +
 ui/litellm-dashboard/src/lib/http/schema.d.ts |   4 +
 15 files changed, 1321 insertions(+), 38 deletions(-)
 create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260626120000_add_mcp_tool_search_enabled/migration.sql
 create mode 100644 litellm/proxy/_experimental/mcp_server/tool_search.py
 create mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py

diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260626120000_add_mcp_tool_search_enabled/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260626120000_add_mcp_tool_search_enabled/migration.sql
new file mode 100644
index 00000000000..542677426ba
--- /dev/null
+++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260626120000_add_mcp_tool_search_enabled/migration.sql
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN "mcp_tool_search_enabled" BOOLEAN;
diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
index e21c0016491..5351e0a1470 100644
--- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
@@ -279,6 +279,7 @@ model LiteLLM_ObjectPermissionTable {
   blocked_tools         String[]       @default([]) // Tool names blocked for any key/team/user with this permission
   mcp_toolsets          String[]       @default([]) // Toolset IDs granted to this key/team/user
   search_tools          String[]       @default([]) // search_tool_name values this key/team/user may call
+  mcp_tool_search_enabled Boolean?
   teams                 LiteLLM_TeamTable[]
   projects              LiteLLM_ProjectTable[]
   verification_tokens   LiteLLM_VerificationToken[]
diff --git a/litellm/models/object_permission.py b/litellm/models/object_permission.py
index 6c0d100046c..3052a2af459 100644
--- a/litellm/models/object_permission.py
+++ b/litellm/models/object_permission.py
@@ -24,3 +24,4 @@ class LiteLLM_ObjectPermissionTable(LiteLLMPydanticObjectBase):
     mcp_toolsets: Optional[List[str]] = None
     blocked_tools: Optional[List[str]] = []
     search_tools: Optional[List[str]] = []
+    mcp_tool_search_enabled: Optional[bool] = None
diff --git a/litellm/proxy/_experimental/mcp_server/AGENTS.md b/litellm/proxy/_experimental/mcp_server/AGENTS.md
index 8eebc3ea3b3..6e1d121c3be 100644
--- a/litellm/proxy/_experimental/mcp_server/AGENTS.md
+++ b/litellm/proxy/_experimental/mcp_server/AGENTS.md
@@ -41,6 +41,7 @@ litellm/proxy/_experimental/mcp_server/
   sampling_handler.py        # MCP sampling to LiteLLM completion flow
   elicitation_handler.py     # MCP elicitation relay flow
   semantic_tool_filter.py    # semantic filtering of available MCP tools
+  tool_search.py             # opt-in virtual tools (mcp_tool_search + mcp_tool_call) for large catalogs
   guardrail_translation/
     handler.py               # MCP guardrail result translation
   sse_transport.py           # SSE transport implementation
@@ -79,6 +80,11 @@ module materially harder to understand.
   encryption need focused tests for both allowed and rejected paths.
 - Avoid adding comments to new code unless they explain non-obvious security or
   protocol behavior. Prefer clear names and small functions.
+- The virtual tool path (`tool_search.py`, gated by `mcp_tool_search_enabled`)
+  must mirror the normal tool flow: IP filtering, server allowlist, per-key tool
+  permissions, no-accessible-server rejection, per-request auth headers, server
+  scope, error to `isError` conversion, and spend logging. Reuse `_list_mcp_tools`
+  and `execute_mcp_tool` rather than reimplementing any of these checks.
 
 ## Tests
 
diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py
index d30d8af2af2..7ab7eb28147 100644
--- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py
+++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py
@@ -569,6 +569,21 @@ if MCP_AVAILABLE:
                 include_disabled_tools and user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
             )
 
+            if apply_tool_filters and getattr(
+                getattr(user_api_key_dict, "object_permission", None),
+                "mcp_tool_search_enabled",
+                False,
+            ):
+                from litellm.proxy._experimental.mcp_server.tool_search import (
+                    get_virtual_tool_definitions,
+                )
+
+                return {
+                    "tools": get_virtual_tool_definitions(),
+                    "error": None,
+                    "message": "Successfully retrieved tools",
+                }
+
             # Extract auth headers from request
             headers = request.headers
             raw_headers_from_request = dict(headers)
@@ -727,6 +742,74 @@ if MCP_AVAILABLE:
         try:
             data = await request.json()
 
+            tool_name = data.get("name")
+            tool_arguments = data.get("arguments") or {}
+
+            from litellm.proxy._experimental.mcp_server.tool_search import (
+                MCP_TOOL_CALL_TOOL_NAME,
+                MCP_TOOL_SEARCH_TOOL_NAME,
+                coerce_top_k,
+                handle_mcp_tool_call,
+                handle_mcp_tool_search,
+            )
+
+            if tool_name in (MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME):
+                if not getattr(
+                    getattr(user_api_key_dict, "object_permission", None),
+                    "mcp_tool_search_enabled",
+                    False,
+                ):
+                    raise HTTPException(
+                        status_code=403,
+                        detail={
+                            "error": "forbidden",
+                            "message": f"{tool_name} requires mcp_tool_search_enabled on the key",
+                        },
+                    )
+                rest_client_ip = IPAddressUtils.get_mcp_client_ip(request)
+                (
+                    virtual_mcp_auth_header,
+                    virtual_mcp_server_auth_headers,
+                    virtual_raw_headers,
+                ) = _extract_mcp_headers_from_request(request, MCPRequestHandler)
+                virtual_oauth2_headers = MCPRequestHandler._get_oauth2_headers_from_headers(request.headers)
+                if tool_name == MCP_TOOL_SEARCH_TOOL_NAME:
+                    return await handle_mcp_tool_search(
+                        query=tool_arguments.get("query", ""),
+                        top_k=coerce_top_k(tool_arguments.get("top_k", 5)),
+                        user_api_key_dict=user_api_key_dict,
+                        client_ip=rest_client_ip,
+                        mcp_auth_header=virtual_mcp_auth_header,
+                        mcp_server_auth_headers=virtual_mcp_server_auth_headers,
+                        oauth2_headers=virtual_oauth2_headers,
+                        raw_headers=virtual_raw_headers,
+                    )
+                else:  # MCP_TOOL_CALL_TOOL_NAME
+                    # Run the same pre-call pipeline as the normal call path so the
+                    # tool execution is spend-logged and guardrail-checked.
+                    (
+                        _,
+                        virtual_logging_obj,
+                    ) = await ProxyBaseLLMRequestProcessing(data=data).common_processing_pre_call_logic(
+                        request=request,
+                        user_api_key_dict=user_api_key_dict,
+                        proxy_config=proxy_config,
+                        route_type=CallTypes.call_mcp_tool.value,
+                        proxy_logging_obj=proxy_logging_obj,
+                        general_settings=general_settings,
+                    )
+                    return await handle_mcp_tool_call(
+                        tool_name=tool_arguments.get("tool_name", ""),
+                        arguments=tool_arguments.get("arguments") or {},
+                        user_api_key_dict=user_api_key_dict,
+                        client_ip=rest_client_ip,
+                        mcp_auth_header=virtual_mcp_auth_header,
+                        mcp_server_auth_headers=virtual_mcp_server_auth_headers,
+                        oauth2_headers=virtual_oauth2_headers,
+                        raw_headers=virtual_raw_headers,
+                        litellm_logging_obj=virtual_logging_obj,
+                    )
+
             # Validate required parameters early
             server_id = data.get("server_id")
             if not server_id:
@@ -738,7 +821,6 @@ if MCP_AVAILABLE:
                     },
                 )
 
-            tool_name = data.get("name")
             if not tool_name:
                 raise HTTPException(
                     status_code=400,
@@ -748,8 +830,6 @@ if MCP_AVAILABLE:
                     },
                 )
 
-            tool_arguments = data.get("arguments") or {}
-
             proxy_base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
             (
                 data,
diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py
index 607e676524e..a65239b296f 100644
--- a/litellm/proxy/_experimental/mcp_server/server.py
+++ b/litellm/proxy/_experimental/mcp_server/server.py
@@ -10,8 +10,8 @@ import contextvars
 import hashlib
 import json
 import time
-import types
 import traceback
+import types
 import uuid
 from datetime import datetime
 from typing import (
@@ -37,13 +37,17 @@ from starlette.types import Message, Receive, Scope, Send
 from litellm._logging import verbose_logger
 from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.custom_httpx.http_handler import (
+    get_async_httpx_client,
+    httpxSpecialProvider,
+)
 from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
     MCPRequestHandler,
 )
-from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError
 from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
     get_request_base_url,
 )
+from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError
 from litellm.proxy._experimental.mcp_server.mcp_context import (
     _mcp_active_toolset_id,
     _mcp_gateway_initialize_instructions,
@@ -59,10 +63,6 @@ from litellm.proxy._experimental.mcp_server.utils import (
     get_server_prefix,
     iter_known_server_prefixes,
 )
-from litellm.llms.custom_httpx.http_handler import (
-    get_async_httpx_client,
-    httpxSpecialProvider,
-)
 from litellm.proxy._types import (
     ProxyException,
     SpecialMCPServerNames,
@@ -122,9 +122,12 @@ def _write_byok_cred_cache(user_id: str, server_id: str, credential: Optional[st
 # TODO: Make this a util function for litellm client usage
 MCP_AVAILABLE: bool = True
 try:
+    import weakref
+
     from mcp import ReadResourceResult, Resource
     from mcp.server import Server
     from mcp.server.lowlevel.helper_types import ReadResourceContents
+    from mcp.server.session import ServerSession as _McpServerSession
     from mcp.types import (
         BlobResourceContents,
         GetPromptResult,
@@ -132,8 +135,6 @@ try:
         TextResourceContents,
         Tool,
     )
-    from mcp.server.session import ServerSession as _McpServerSession
-    import weakref
 
     # Robust auth lookup keyed by session_object.
     _session_obj_auth_storage: "weakref.WeakKeyDictionary[Any, MCPAuthenticatedUser]" = weakref.WeakKeyDictionary()
@@ -303,14 +304,14 @@ def _proxy_exception_to_http_exception(exc: ProxyException) -> HTTPException:
 
 if MCP_AVAILABLE:
     from mcp.server import Server
-    from mcp.server.lowlevel.server import NotificationOptions
-    from mcp.server.models import InitializationOptions
 
     # Import auth context variables and middleware
     from mcp.server.auth.middleware.auth_context import (
         AuthContextMiddleware,
         auth_context_var,
     )
+    from mcp.server.lowlevel.server import NotificationOptions
+    from mcp.server.models import InitializationOptions
 
     try:
         from mcp.server.streamable_http_manager import StreamableHTTPSessionManager
@@ -664,6 +665,19 @@ if MCP_AVAILABLE:
             verbose_logger.debug(
                 f"MCP list_tools - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}"
             )
+            if getattr(
+                getattr(user_api_key_auth, "object_permission", None),
+                "mcp_tool_search_enabled",
+                False,
+            ):
+                from mcp.types import Tool
+
+                from litellm.proxy._experimental.mcp_server.tool_search import (
+                    get_virtual_tool_definitions,
+                )
+
+                return [Tool(**d) for d in get_virtual_tool_definitions()]
+
             # Get mcp_servers from context variable
             verbose_logger.debug("MCP list_tools - Calling _list_mcp_tools")
             tools = await _list_mcp_tools(
@@ -688,6 +702,150 @@ if MCP_AVAILABLE:
             if _session_reset_token is not None:
                 active_mcp_session_var.reset(_session_reset_token)
 
+    def _capture_host_progress_callback(host_server) -> Optional[Callable]:
+        """Return a progress-forwarding callback bound to the host MCP session.
+
+        Returns ``None`` when the host did not supply a progress token.
+        """
+        try:
+            host_ctx = host_server.request_context
+        except Exception as e:
+            verbose_logger.warning(f"Could not capture host progress context: {e}")
+            return None
+
+        if not (host_ctx and hasattr(host_ctx, "meta") and host_ctx.meta):
+            return None
+        host_token = getattr(host_ctx.meta, "progressToken", None)
+        if not (host_token and hasattr(host_ctx, "session") and host_ctx.session):
+            return None
+        host_session = host_ctx.session
+
+        async def forward_progress(progress: float, total: Optional[float]):
+            """Forward progress notifications from external MCP to Host"""
+            try:
+                await host_session.send_progress_notification(
+                    progress_token=host_token,
+                    progress=progress,
+                    total=total,
+                )
+                verbose_logger.debug(f"Forwarded progress {progress}/{total} to Host")
+            except Exception as e:
+                verbose_logger.error(f"Failed to forward progress to Host: {e}")
+
+        verbose_logger.debug(f"Host progressToken captured: {host_token[:8]}...")
+        return forward_progress
+
+    async def _build_virtual_call_logging_obj(
+        name: str,
+        arguments: dict[str, Any],
+        user_api_key_auth: UserAPIKeyAuth,
+    ) -> Optional[LiteLLMLoggingObj]:
+        """Run the pre-call pipeline (guardrails + logging setup) for a virtual
+        mcp_tool_call so the SSE path spend-logs like the REST path."""
+        from fastapi import Request
+
+        from litellm.proxy.common_request_processing import (
+            ProxyBaseLLMRequestProcessing,
+        )
+        from litellm.proxy.proxy_server import (
+            general_settings,
+            proxy_config,
+            proxy_logging_obj,
+        )
+
+        request = Request(
+            scope={
+                "type": "http",
+                "method": "POST",
+                "path": "/mcp/tools/call",
+                "headers": [(b"content-type", b"application/json")],
+            }
+        )
+        _, virtual_logging_obj = await ProxyBaseLLMRequestProcessing(
+            data={"name": name, "arguments": arguments}
+        ).common_processing_pre_call_logic(
+            request=request,
+            user_api_key_dict=user_api_key_auth,
+            proxy_config=proxy_config,
+            route_type=CallTypes.call_mcp_tool.value,
+            proxy_logging_obj=proxy_logging_obj,
+            general_settings=general_settings,
+        )
+        return virtual_logging_obj
+
+    async def _dispatch_virtual_mcp_tool(
+        name: str,
+        arguments: Optional[dict[str, Any]],
+        user_api_key_auth: Optional[UserAPIKeyAuth],
+        client_ip: Optional[str],
+        mcp_servers: Optional[list[str]] = None,
+        mcp_auth_header: Optional[str] = None,
+        mcp_server_auth_headers: Optional[dict[str, dict[str, str]]] = None,
+        oauth2_headers: Optional[dict[str, str]] = None,
+        raw_headers: Optional[dict[str, str]] = None,
+    ) -> Optional[CallToolResult]:
+        """Handle the mcp_tool_search / mcp_tool_call virtual tools.
+
+        Returns a CallToolResult when ``name`` is a virtual tool, else ``None`` so
+        the caller falls through to normal tool routing.
+        """
+        from litellm.proxy._experimental.mcp_server.tool_search import (
+            MCP_TOOL_CALL_TOOL_NAME,
+            MCP_TOOL_SEARCH_TOOL_NAME,
+            coerce_top_k,
+            handle_mcp_tool_call,
+            handle_mcp_tool_search,
+        )
+
+        if name not in (MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME):
+            return None
+
+        if not getattr(
+            getattr(user_api_key_auth, "object_permission", None),
+            "mcp_tool_search_enabled",
+            False,
+        ):
+            return CallToolResult(
+                content=[
+                    TextContent(
+                        type="text",
+                        text=f"Tool {name} requires mcp_tool_search_enabled on the key",
+                    )
+                ],
+                isError=True,
+            )
+
+        args = arguments or {}
+        if name == MCP_TOOL_SEARCH_TOOL_NAME:
+            return await handle_mcp_tool_search(
+                query=args.get("query", ""),
+                top_k=coerce_top_k(args.get("top_k", 5)),
+                user_api_key_dict=user_api_key_auth,
+                client_ip=client_ip,
+                mcp_servers=mcp_servers,
+                mcp_auth_header=mcp_auth_header,
+                mcp_server_auth_headers=mcp_server_auth_headers,
+                oauth2_headers=oauth2_headers,
+                raw_headers=raw_headers,
+            )
+
+        assert user_api_key_auth is not None  # guaranteed by the flag check above
+        virtual_logging_obj = await _build_virtual_call_logging_obj(
+            name=name, arguments=args, user_api_key_auth=user_api_key_auth
+        )
+        return await handle_mcp_tool_call(
+            tool_name=args.get("tool_name", ""),
+            arguments=args.get("arguments") or {},
+            user_api_key_dict=user_api_key_auth,
+            client_ip=client_ip,
+            mcp_servers=mcp_servers,
+            mcp_auth_header=mcp_auth_header,
+            mcp_server_auth_headers=mcp_server_auth_headers,
+            oauth2_headers=oauth2_headers,
+            raw_headers=raw_headers,
+            litellm_logging_obj=virtual_logging_obj,
+        )
+
     @server.call_tool()
     async def mcp_server_tool_call(name: str, arguments: Dict[str, Any] | None) -> CallToolResult:
         """
@@ -701,11 +859,12 @@ if MCP_AVAILABLE:
             HTTPException: If tool not found or arguments missing
         """
         from fastapi import Request
+        from mcp.server.lowlevel.server import request_ctx
+        from mcp.types import CallToolResult
+
         from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
         from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
         from litellm.proxy.proxy_server import proxy_config
-        from mcp.types import CallToolResult
-        from mcp.server.lowlevel.server import request_ctx
 
         req_ctx = request_ctx.get(None)
         _session_reset_token = None
@@ -730,31 +889,25 @@ if MCP_AVAILABLE:
             )
 
             verbose_logger.debug(f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}")
-            host_progress_callback = None
-            try:
-                host_ctx = server.request_context
-                if host_ctx and hasattr(host_ctx, "meta") and host_ctx.meta:
-                    host_token = getattr(host_ctx.meta, "progressToken", None)
-                    if host_token and hasattr(host_ctx, "session") and host_ctx.session:
-                        host_session = host_ctx.session
 
-                        async def forward_progress(progress: float, total: Optional[float]):
-                            """Forward progress notifications from external MCP to Host"""
-                            try:
-                                await host_session.send_progress_notification(
-                                    progress_token=host_token,
-                                    progress=progress,
-                                    total=total,
-                                )
-                                verbose_logger.debug(f"Forwarded progress {progress}/{total} to Host")
-                            except Exception as e:
-                                verbose_logger.error(f"Failed to forward progress to Host: {e}")
-
-                        host_progress_callback = forward_progress
-                        verbose_logger.debug(f"Host progressToken captured: {host_token[:8]}...")
-            except Exception as e:
-                verbose_logger.warning(f"Could not capture host progress context: {e}")
             try:
+                # Inside this try so virtual-tool errors convert to isError
+                # CallToolResult instead of raising out of the protocol handler.
+                virtual_tool_result = await _dispatch_virtual_mcp_tool(
+                    name=name,
+                    arguments=arguments,
+                    user_api_key_auth=user_api_key_auth,
+                    client_ip=_client_ip,
+                    mcp_servers=mcp_servers,
+                    mcp_auth_header=mcp_auth_header,
+                    mcp_server_auth_headers=mcp_server_auth_headers,
+                    oauth2_headers=oauth2_headers,
+                    raw_headers=raw_headers,
+                )
+                if virtual_tool_result is not None:
+                    return virtual_tool_result
+
+                host_progress_callback = _capture_host_progress_callback(server)
                 # Create a body date for logging
                 body_data = {"name": name, "arguments": arguments}
                 # Set trace/session id from raw_headers so spend logs and logging_obj stay consistent (same as A2A)
@@ -1528,6 +1681,7 @@ if MCP_AVAILABLE:
         log_list_tools_to_spendlogs: bool = False,
         list_tools_log_source: Optional[str] = None,
         litellm_trace_id: Optional[str] = None,
+        client_ip: Optional[str] = None,
     ) -> List[MCPTool]:
         """
         Helper method to fetch tools from MCP servers based on server filtering criteria.
@@ -1615,6 +1769,7 @@ if MCP_AVAILABLE:
             allowed_mcp_servers = await _get_allowed_mcp_servers(
                 user_api_key_auth=user_api_key_auth,
                 mcp_servers=mcp_servers,
+                client_ip=client_ip,
             )
 
             # Pre-fetch OAuth credentials only when at least one server uses OAuth2,
@@ -2024,6 +2179,7 @@ if MCP_AVAILABLE:
         raw_headers: Optional[Dict[str, str]] = None,
         log_list_tools_to_spendlogs: bool = False,
         list_tools_log_source: Optional[str] = None,
+        client_ip: Optional[str] = None,
     ) -> List[MCPTool]:
         """
         List all available MCP tools.
@@ -2033,6 +2189,7 @@ if MCP_AVAILABLE:
             mcp_auth_header: Optional auth header for MCP server (deprecated)
             mcp_servers: Optional list of server names/aliases to filter by
             mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value}
+            client_ip: Client IP for IP-based server access control
 
         Returns:
             List[MCPTool]: Combined list of tools from all accessible servers
@@ -2056,6 +2213,7 @@ if MCP_AVAILABLE:
                 raw_headers=raw_headers,
                 log_list_tools_to_spendlogs=log_list_tools_to_spendlogs,
                 list_tools_log_source=list_tools_log_source,
+                client_ip=client_ip,
             )
             verbose_logger.debug(f"Successfully fetched {len(managed_tools)} tools from managed MCP servers")
         except Exception as e:
diff --git a/litellm/proxy/_experimental/mcp_server/tool_search.py b/litellm/proxy/_experimental/mcp_server/tool_search.py
new file mode 100644
index 00000000000..fa57a2b3eb2
--- /dev/null
+++ b/litellm/proxy/_experimental/mcp_server/tool_search.py
@@ -0,0 +1,157 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, Optional
+
+if TYPE_CHECKING:
+    from mcp.types import CallToolResult
+
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+    from litellm.proxy._types import UserAPIKeyAuth
+
+MCP_TOOL_SEARCH_TOOL_NAME: str = "mcp_tool_search"
+MCP_TOOL_CALL_TOOL_NAME: str = "mcp_tool_call"
+
+
+def coerce_top_k(value: Any, default: int = 5) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def search_tools(query: str, tools: list[dict[str, Any]], top_k: int = 5) -> list[dict[str, Any]]:
+    if not query:
+        return []
+    tokens = query.lower().split()
+
+    def _score(tool: dict[str, Any]) -> int:
+        haystack = (tool.get("name", "") + " " + tool.get("description", "")).lower()
+        return sum(1 for t in tokens if t in haystack)
+
+    scored = ((s, tool) for tool in tools if (s := _score(tool)) > 0)
+    return [tool for _, tool in sorted(scored, key=lambda x: x[0], reverse=True)[:top_k]]
+
+
+def get_virtual_tool_definitions() -> list[dict[str, Any]]:
+    return [
+        {
+            "name": MCP_TOOL_SEARCH_TOOL_NAME,
+            "description": "Search for MCP tools by keyword. Returns top matching tools with names, descriptions, and input schemas.",
+            "inputSchema": {
+                "type": "object",
+                "properties": {
+                    "query": {
+                        "type": "string",
+                        "description": "Keywords to search for in tool names and descriptions.",
+                    },
+                    "top_k": {
+                        "type": "integer",
+                        "description": "Maximum number of results to return.",
+                        "default": 5,
+                    },
+                },
+                "required": ["query"],
+            },
+        },
+        {
+            "name": MCP_TOOL_CALL_TOOL_NAME,
+            "description": "Call an MCP tool by name with the given arguments.",
+            "inputSchema": {
+                "type": "object",
+                "properties": {
+                    "tool_name": {
+                        "type": "string",
+                        "description": "The exact name of the MCP tool to call.",
+                    },
+                    "arguments": {
+                        "type": "object",
+                        "description": "Arguments to pass to the tool.",
+                    },
+                },
+                "required": ["tool_name"],
+            },
+        },
+    ]
+
+
+async def handle_mcp_tool_search(
+    query: str,
+    top_k: int,
+    user_api_key_dict: UserAPIKeyAuth,
+    client_ip: Optional[str] = None,
+    mcp_servers: Optional[list[str]] = None,
+    mcp_auth_header: Optional[str] = None,
+    mcp_server_auth_headers: Optional[dict[str, dict[str, str]]] = None,
+    oauth2_headers: Optional[dict[str, str]] = None,
+    raw_headers: Optional[dict[str, str]] = None,
+) -> CallToolResult:
+    from mcp.types import CallToolResult, TextContent
+
+    from litellm.proxy._experimental.mcp_server.server import _list_mcp_tools
+
+    mcp_tools = await _list_mcp_tools(
+        user_api_key_auth=user_api_key_dict,
+        mcp_servers=mcp_servers,
+        client_ip=client_ip,
+        mcp_auth_header=mcp_auth_header,
+        mcp_server_auth_headers=mcp_server_auth_headers,
+        oauth2_headers=oauth2_headers,
+        raw_headers=raw_headers,
+    )
+    tools = [
+        {
+            "name": t.name,
+            "description": t.description or "",
+            "inputSchema": t.inputSchema,
+        }
+        for t in mcp_tools
+    ]
+    results = search_tools(query, tools, top_k)
+    return CallToolResult(content=[TextContent(type="text", text=json.dumps(results))], isError=False)
+
+
+async def handle_mcp_tool_call(
+    tool_name: str,
+    arguments: dict[str, Any],
+    user_api_key_dict: UserAPIKeyAuth,
+    client_ip: Optional[str] = None,
+    mcp_servers: Optional[list[str]] = None,
+    mcp_auth_header: Optional[str] = None,
+    mcp_server_auth_headers: Optional[dict[str, dict[str, str]]] = None,
+    oauth2_headers: Optional[dict[str, str]] = None,
+    raw_headers: Optional[dict[str, str]] = None,
+    litellm_logging_obj: Optional[LiteLLMLoggingObj] = None,
+) -> CallToolResult:
+    from litellm.proxy._experimental.mcp_server.server import (
+        _get_allowed_mcp_servers,
+        execute_mcp_tool,
+    )
+
+    allowed_mcp_servers = await _get_allowed_mcp_servers(
+        user_api_key_auth=user_api_key_dict,
+        mcp_servers=mcp_servers,
+        client_ip=client_ip,
+    )
+
+    # Reject before dispatch when the key has no accessible servers; otherwise an
+    # unprefixed local tool name would fall through to the local registry in
+    # execute_mcp_tool, which has no server permission check.
+    if not allowed_mcp_servers:
+        from fastapi import HTTPException
+
+        raise HTTPException(status_code=403, detail="User not allowed to call this tool.")
+
+    return await execute_mcp_tool(
+        name=tool_name,
+        arguments=arguments,
+        allowed_mcp_servers=allowed_mcp_servers,
+        start_time=datetime.now(),
+        user_api_key_auth=user_api_key_dict,
+        mcp_auth_header=mcp_auth_header,
+        mcp_server_auth_headers=mcp_server_auth_headers,
+        oauth2_headers=oauth2_headers,
+        raw_headers=raw_headers,
+        litellm_logging_obj=litellm_logging_obj,
+    )
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 643f8d69300..12466b525d6 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -1006,6 +1006,7 @@ class LiteLLM_ObjectPermissionBase(LiteLLMPydanticObjectBase):
     agent_access_groups: Optional[List[str]] = None
     models: Optional[List[str]] = None
     search_tools: Optional[List[str]] = None
+    mcp_tool_search_enabled: Optional[bool] = None
 
 
 from litellm.types.object_permission import (  # noqa: E402
diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma
index e21c0016491..5351e0a1470 100644
--- a/litellm/proxy/schema.prisma
+++ b/litellm/proxy/schema.prisma
@@ -279,6 +279,7 @@ model LiteLLM_ObjectPermissionTable {
   blocked_tools         String[]       @default([]) // Tool names blocked for any key/team/user with this permission
   mcp_toolsets          String[]       @default([]) // Toolset IDs granted to this key/team/user
   search_tools          String[]       @default([]) // search_tool_name values this key/team/user may call
+  mcp_tool_search_enabled Boolean?
   teams                 LiteLLM_TeamTable[]
   projects              LiteLLM_ProjectTable[]
   verification_tokens   LiteLLM_VerificationToken[]
diff --git a/litellm/types/object_permission.py b/litellm/types/object_permission.py
index ff932dccd5d..d0458173fbf 100644
--- a/litellm/types/object_permission.py
+++ b/litellm/types/object_permission.py
@@ -24,3 +24,4 @@ class ObjectPermissionDict(TypedDict, total=False):
     agent_access_groups: Optional[list[str]]
     models: Optional[list[str]]
     search_tools: Optional[list[str]]
+    mcp_tool_search_enabled: Optional[bool]
diff --git a/schema.prisma b/schema.prisma
index e21c0016491..5351e0a1470 100644
--- a/schema.prisma
+++ b/schema.prisma
@@ -279,6 +279,7 @@ model LiteLLM_ObjectPermissionTable {
   blocked_tools         String[]       @default([]) // Tool names blocked for any key/team/user with this permission
   mcp_toolsets          String[]       @default([]) // Toolset IDs granted to this key/team/user
   search_tools          String[]       @default([]) // search_tool_name values this key/team/user may call
+  mcp_tool_search_enabled Boolean?
   teams                 LiteLLM_TeamTable[]
   projects              LiteLLM_ProjectTable[]
   verification_tokens   LiteLLM_VerificationToken[]
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py
new file mode 100644
index 00000000000..9c20808df67
--- /dev/null
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py
@@ -0,0 +1,837 @@
+"""
+Tests for MCP tool search feature.
+
+Covers:
+- search_tools() pure function
+- get_virtual_tool_definitions() shape
+- list_tool_rest_api returns only virtual tools when mcp_tool_search_enabled=True
+- call_tool_rest_api intercepts mcp_tool_search calls
+- call_tool_rest_api intercepts mcp_tool_call calls
+"""
+
+import json
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
+from litellm.proxy._experimental.mcp_server.tool_search import (
+    MCP_TOOL_CALL_TOOL_NAME,
+    MCP_TOOL_SEARCH_TOOL_NAME,
+    coerce_top_k,
+    get_virtual_tool_definitions,
+    search_tools,
+)
+from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
+
+
+def _make_tools(specs: list[tuple[str, str]]) -> list[dict[str, Any]]:
+    return [
+        {
+            "name": name,
+            "description": desc,
+            "inputSchema": {"type": "object", "properties": {}},
+        }
+        for name, desc in specs
+    ]
+
+
+def _make_perm(**kwargs: Any) -> LiteLLM_ObjectPermissionTable:
+    return LiteLLM_ObjectPermissionTable(object_permission_id="test", **kwargs)
+
+
+SAMPLE_TOOLS = _make_tools(
+    [
+        ("github-create_issue", "Create a new issue in a GitHub repository"),
+        ("github-list_repos", "List all repositories for a GitHub user"),
+        ("slack-send_message", "Send a message to a Slack channel"),
+        ("slack-list_channels", "List all Slack channels in a workspace"),
+        ("notion-create_page", "Create a new page in Notion"),
+    ]
+)
+
+
+class TestCoerceTopK:
+    def test_int_passthrough(self) -> None:
+        assert coerce_top_k(3) == 3
+
+    def test_numeric_string_coerced(self) -> None:
+        assert coerce_top_k("7") == 7
+
+    def test_float_truncated(self) -> None:
+        assert coerce_top_k(3.9) == 3
+
+    def test_non_numeric_string_returns_default(self) -> None:
+        assert coerce_top_k("abc") == 5
+
+    def test_none_returns_default(self) -> None:
+        assert coerce_top_k(None) == 5
+
+    def test_custom_default(self) -> None:
+        assert coerce_top_k("nope", default=10) == 10
+
+
+class TestSearchTools:
+    def test_returns_matching_tools(self) -> None:
+        results = search_tools("github issue", SAMPLE_TOOLS)
+        names = [t["name"] for t in results]
+        assert "github-create_issue" in names
+
+    def test_ranks_by_relevance(self) -> None:
+        results = search_tools("github", SAMPLE_TOOLS)
+        names = [t["name"] for t in results]
+        github_positions = [i for i, n in enumerate(names) if n.startswith("github")]
+        other_positions = [i for i, n in enumerate(names) if not n.startswith("github")]
+        assert all(g < o for g in github_positions for o in other_positions)
+
+    def test_top_k_limits_results(self) -> None:
+        results = search_tools("a", SAMPLE_TOOLS, top_k=2)
+        assert len(results) <= 2
+
+    def test_empty_query_returns_empty(self) -> None:
+        assert search_tools("", SAMPLE_TOOLS) == []
+
+    def test_no_match_returns_empty(self) -> None:
+        assert search_tools("xyzzy_nonexistent_zzz", SAMPLE_TOOLS) == []
+
+    def test_matches_description_not_just_name(self) -> None:
+        results = search_tools("channel", SAMPLE_TOOLS)
+        names = [t["name"] for t in results]
+        assert "slack-list_channels" in names
+
+    def test_case_insensitive(self) -> None:
+        lower = [t["name"] for t in search_tools("github", SAMPLE_TOOLS)]
+        upper = [t["name"] for t in search_tools("GITHUB", SAMPLE_TOOLS)]
+        assert lower == upper
+
+    def test_result_tools_have_full_schema(self) -> None:
+        for tool in search_tools("github", SAMPLE_TOOLS):
+            assert "name" in tool
+            assert "description" in tool
+            assert "inputSchema" in tool
+
+
+class TestGetVirtualToolDefinitions:
+    def test_returns_two_tools(self) -> None:
+        assert len(get_virtual_tool_definitions()) == 2
+
+    def test_has_mcp_tool_search(self) -> None:
+        names = [t["name"] for t in get_virtual_tool_definitions()]
+        assert MCP_TOOL_SEARCH_TOOL_NAME in names
+
+    def test_has_mcp_tool_call(self) -> None:
+        names = [t["name"] for t in get_virtual_tool_definitions()]
+        assert MCP_TOOL_CALL_TOOL_NAME in names
+
+    def test_mcp_tool_search_schema_has_query(self) -> None:
+        tools = get_virtual_tool_definitions()
+        search_tool = next(t for t in tools if t["name"] == MCP_TOOL_SEARCH_TOOL_NAME)
+        props = search_tool["inputSchema"]["properties"]
+        assert "query" in props
+        assert search_tool["inputSchema"]["required"] == ["query"]
+
+    def test_mcp_tool_call_schema_has_tool_name_and_arguments(self) -> None:
+        tools = get_virtual_tool_definitions()
+        call_tool = next(t for t in tools if t["name"] == MCP_TOOL_CALL_TOOL_NAME)
+        props = call_tool["inputSchema"]["properties"]
+        assert "tool_name" in props
+        assert "arguments" in props
+        assert "tool_name" in call_tool["inputSchema"]["required"]
+
+    def test_all_tools_have_description(self) -> None:
+        for tool in get_virtual_tool_definitions():
+            assert tool.get("description"), f"{tool['name']} missing description"
+
+    def test_definitions_construct_mcp_protocol_tool(self) -> None:
+        """The MCP protocol list_tools handler builds mcp.types.Tool(**d) from
+        each definition, so the dict keys must stay valid Tool fields."""
+        from mcp.types import Tool
+
+        built = [Tool(**d) for d in get_virtual_tool_definitions()]
+        assert {t.name for t in built} == {
+            MCP_TOOL_SEARCH_TOOL_NAME,
+            MCP_TOOL_CALL_TOOL_NAME,
+        }
+
+
+class TestListToolRestApiWithToolSearch:
+    @pytest.mark.asyncio
+    async def test_returns_only_virtual_tools_when_flag_enabled(self) -> None:
+        from litellm.proxy._experimental.mcp_server.rest_endpoints import router
+
+        user_api_key_dict = UserAPIKeyAuth(
+            api_key="test_key",
+            object_permission=_make_perm(
+                mcp_tool_search_enabled=True,
+                mcp_servers=["github", "slack"],
+            ),
+        )
+
+        mock_request = MagicMock()
+        mock_request.headers = {}
+
+        list_fn = next(
+            r.endpoint
+            for r in router.routes
+            if hasattr(r, "path") and r.path.endswith("/tools/list") and hasattr(r, "methods") and "GET" in r.methods
+        )
+
+        result = await list_fn(
+            request=mock_request,
+            server_id=None,
+            include_disabled_tools=False,
+            user_api_key_dict=user_api_key_dict,
+        )
+
+        assert result["error"] is None
+        tool_names = [t["name"] for t in result["tools"]]
+        assert set(tool_names) == {MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME}
+
+    @pytest.mark.asyncio
+    async def test_returns_full_catalog_when_flag_disabled(self) -> None:
+        from litellm.proxy._experimental.mcp_server.rest_endpoints import router
+
+        user_api_key_dict = UserAPIKeyAuth(
+            api_key="test_key",
+            object_permission=_make_perm(
+                mcp_tool_search_enabled=False,
+                mcp_servers=["github"],
+            ),
+        )
+
+        mock_request = MagicMock()
+        mock_request.headers = {}
+
+        fake_tools = [
+            {
+                "name": "github-create_issue",
+                "description": "Create issue",
+                "inputSchema": {"type": "object"},
+            }
+        ]
+
+        list_fn = next(
+            r.endpoint
+            for r in router.routes
+            if hasattr(r, "path") and r.path.endswith("/tools/list") and hasattr(r, "methods") and "GET" in r.methods
+        )
+
+        with (
+            patch(
+                "litellm.proxy._experimental.mcp_server.rest_endpoints.build_effective_auth_contexts",
+                new_callable=AsyncMock,
+                return_value=[user_api_key_dict],
+            ),
+            patch("litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager") as mock_manager,
+            patch(
+                "litellm.proxy._experimental.mcp_server.rest_endpoints._get_tools_for_single_server",
+                new_callable=AsyncMock,
+                return_value=fake_tools,
+            ),
+            patch("litellm.proxy._experimental.mcp_server.rest_endpoints.IPAddressUtils"),
+            patch(
+                "litellm.proxy._experimental.mcp_server.rest_endpoints._prefetch_user_oauth_creds",
+                new_callable=AsyncMock,
+                return_value={},
+            ),
+            patch(
+                "litellm.proxy._experimental.mcp_server.rest_endpoints._get_oauth2_server_ids",
+                return_value=[],
+            ),
+            patch(
+                "litellm.proxy._experimental.mcp_server.rest_endpoints._get_server_auth_header",
+                return_value=None,
+            ),
+            patch(
+                "litellm.proxy._experimental.mcp_server.rest_endpoints._get_user_oauth_extra_headers",
+                new_callable=AsyncMock,
+                return_value=None,
+            ),
+        ):
+            mock_manager.get_allowed_mcp_servers = AsyncMock(return_value=["github"])
+            mock_manager.filter_server_ids_by_ip_with_info = MagicMock(return_value=(["github"], 0))
+            mock_manager.get_mcp_server_by_id = MagicMock(return_value=MagicMock(name="github", server_id="github"))
+            result = await list_fn(
+                request=mock_request,
+                server_id=None,
+                include_disabled_tools=False,
+                user_api_key_dict=user_api_key_dict,
+            )
+
+        tool_names = [t["name"] for t in result["tools"]]
+        assert MCP_TOOL_SEARCH_TOOL_NAME not in tool_names
+        assert "github-create_issue" in tool_names
+
+    @pytest.mark.asyncio
+    async def test_admin_include_disabled_tools_bypasses_virtual_catalog(self) -> None:
+        """Regression: an admin listing with include_disabled_tools must see the
+        real catalog (to configure allowlists) even when mcp_tool_search_enabled is
+        set, instead of the two virtual tools."""
+        from litellm.proxy._experimental.mcp_server.rest_endpoints import router
+
+        user_api_key_dict = UserAPIKeyAuth(
+            api_key="admin_key",
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+            object_permission=_make_perm(
+                mcp_tool_search_enabled=True,
+                mcp_servers=["github"],
+            ),
+        )
+
+        mock_request = MagicMock()
+        mock_request.headers = {}
+
+        fake_tools = [
+            {
+                "name": "github-create_issue",
+                "description": "Create issue",
+                "inputSchema": {"type": "object"},
+            }
+        ]
+
+        list_fn = next(
+            r.endpoint
+            for r in router.routes
+            if hasattr(r, "path") and r.path.endswith("/tools/list") and hasattr(r, "methods") and "GET" in r.methods
+        )
+
+        with (
+            patch(
+                "litellm.proxy._experimental.mcp_server.rest_endpoints.build_effective_auth_contexts",
+                new_callable=AsyncMock,
+                return_value=[user_api_key_dict],
+            ),
+            patch("litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager") as mock_manager,
+            patch(
+                "litellm.proxy._experimental.mcp_server.rest_endpoints._get_tools_for_single_server",
+                new_callable=AsyncMock,
+                return_value=fake_tools,
+            ),
+            patch("litellm.proxy._experimental.mcp_server.rest_endpoints.IPAddressUtils"),
+            patch(
+                "litellm.proxy._experimental.mcp_server.rest_endpoints._prefetch_user_oauth_creds",
+                new_callable=AsyncMock,
+                return_value={},
+            ),
+            patch(
+                "litellm.proxy._experimental.mcp_server.rest_endpoints._get_oauth2_server_ids",
+                return_value=[],
+            ),
+            patch(
+                "litellm.proxy._experimental.mcp_server.rest_endpoints._get_server_auth_header",
+                return_value=None,
+            ),
+            patch(
+                "litellm.proxy._experimental.mcp_server.rest_endpoints._get_user_oauth_extra_headers",
+                new_callable=AsyncMock,
+                return_value=None,
+            ),
+        ):
+            mock_manager.get_allowed_mcp_servers = AsyncMock(return_value=["github"])
+            mock_manager.filter_server_ids_by_ip_with_info = MagicMock(return_value=(["github"], 0))
+            mock_manager.get_mcp_server_by_id = MagicMock(return_value=MagicMock(name="github", server_id="github"))
+            result = await list_fn(
+                request=mock_request,
+                server_id=None,
+                include_disabled_tools=True,
+                user_api_key_dict=user_api_key_dict,
+            )
+
+        tool_names = [t["name"] for t in result["tools"]]
+        assert MCP_TOOL_SEARCH_TOOL_NAME not in tool_names
+        assert "github-create_issue" in tool_names
+
+
+class TestCallToolRestApiVirtualTools:
+    def _make_request(self, body: dict[str, Any]) -> MagicMock:
+        mock_request = MagicMock()
+        mock_request.json = AsyncMock(return_value=body)
+        mock_request.headers = {}
+        mock_request.url = MagicMock()
+        mock_request.url.path = "/mcp-rest/tools/call"
+        return mock_request
+
+    def _get_call_fn(self) -> Any:
+        from litellm.proxy._experimental.mcp_server.rest_endpoints import router
+
+        return next(
+            r.endpoint
+            for r in router.routes
+            if hasattr(r, "path") and r.path.endswith("/tools/call") and hasattr(r, "methods") and "POST" in r.methods
+        )
+
+    @pytest.mark.asyncio
+    async def test_mcp_tool_search_call_returns_tool_defs(self) -> None:
+        user_api_key_dict = UserAPIKeyAuth(
+            api_key="test_key",
+            object_permission=_make_perm(
+                mcp_tool_search_enabled=True,
+                mcp_servers=["github"],
+            ),
+        )
+
+        request = self._make_request({"name": MCP_TOOL_SEARCH_TOOL_NAME, "arguments": {"query": "create issue"}})
+
+        mock_tool = MagicMock()
+        mock_tool.name = "github-create_issue"
+        mock_tool.description = "Create a GitHub issue"
+        mock_tool.inputSchema = {"type": "object", "properties": {}}
+
+        with patch(
+            "litellm.proxy._experimental.mcp_server.server._list_mcp_tools",
+            new_callable=AsyncMock,
+            return_value=[mock_tool],
+        ):
+            result = await self._get_call_fn()(
+                request=request,
+                user_api_key_dict=user_api_key_dict,
+            )
+
+        assert result.content
+        assert result.content[0].type == "text"
+        returned_tools = json.loads(result.content[0].text)
+        assert isinstance(returned_tools, list)
+        assert any(t["name"] == "github-create_issue" for t in returned_tools)
+
+    @pytest.mark.asyncio
+    async def test_mcp_tool_call_executes_discovered_tool(self) -> None:
+        from mcp.types import CallToolResult, TextContent
+
+        user_api_key_dict = UserAPIKeyAuth(
+            api_key="test_key",
+            object_permission=_make_perm(
+                mcp_tool_search_enabled=True,
+                mcp_servers=["github"],
+            ),
+        )
+
+        request = self._make_request(
+            {
+                "name": MCP_TOOL_CALL_TOOL_NAME,
+                "arguments": {
+                    "tool_name": "github-create_issue",
+                    "arguments": {"title": "bug", "repo": "myrepo"},
+                },
+            }
+        )
+
+        fake_result = CallToolResult(
+            content=[TextContent(type="text", text="Issue created")],
+            isError=False,
+        )
+
+        with (
+            patch(
+                "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+                new_callable=AsyncMock,
+                return_value=[MagicMock()],
+            ),
+            patch(
+                "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool",
+                new_callable=AsyncMock,
+                return_value=fake_result,
+            ) as mock_execute,
+        ):
+            result = await self._get_call_fn()(
+                request=request,
+                user_api_key_dict=user_api_key_dict,
+            )
+
+        mock_execute.assert_awaited_once()
+        assert mock_execute.await_args.kwargs["name"] == "github-create_issue"
+
+        assert result.isError is False
+        assert result.content[0].text == "Issue created"
+
+    @pytest.mark.asyncio
+    async def test_mcp_tool_call_forwards_client_ip_for_ip_filtering(self) -> None:
+        """Regression: the virtual call path must resolve allowed servers with the
+        request's client IP so IP-restricted servers (available_on_public_internet:
+        false) cannot be reached from a public IP."""
+        from mcp.types import CallToolResult, TextContent
+
+        user_api_key_dict = UserAPIKeyAuth(
+            api_key="test_key",
+            object_permission=_make_perm(mcp_tool_search_enabled=True, mcp_servers=["github"]),
+        )
+        request = self._make_request(
+            {
+                "name": MCP_TOOL_CALL_TOOL_NAME,
+                "arguments": {"tool_name": "github-create_issue", "arguments": {}},
+            }
+        )
+
+        fake_result = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False)
+
+        with (
+            patch(
+                "litellm.proxy._experimental.mcp_server.rest_endpoints.IPAddressUtils.get_mcp_client_ip",
+                return_value="203.0.113.7",
+            ),
+            patch(
+                "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+                new_callable=AsyncMock,
+                return_value=[MagicMock()],
+            ) as mock_allowed,
+            patch(
+                "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool",
+                new_callable=AsyncMock,
+                return_value=fake_result,
+            ),
+        ):
+            await self._get_call_fn()(request=request, user_api_key_dict=user_api_key_dict)
+
+        mock_allowed.assert_awaited_once()
+        assert mock_allowed.await_args.kwargs["client_ip"] == "203.0.113.7"
+
+    @pytest.mark.asyncio
+    async def test_mcp_tool_search_forwards_client_ip_for_ip_filtering(self) -> None:
+        """Search must list tools through the IP-filtered catalog, not the raw one."""
+        user_api_key_dict = UserAPIKeyAuth(
+            api_key="test_key",
+            object_permission=_make_perm(mcp_tool_search_enabled=True, mcp_servers=["github"]),
+        )
+        request = self._make_request({"name": MCP_TOOL_SEARCH_TOOL_NAME, "arguments": {"query": "issue"}})
+
+        with (
+            patch(
+                "litellm.proxy._experimental.mcp_server.rest_endpoints.IPAddressUtils.get_mcp_client_ip",
+                return_value="203.0.113.7",
+            ),
+            patch(
+                "litellm.proxy._experimental.mcp_server.server._list_mcp_tools",
+                new_callable=AsyncMock,
+                return_value=[],
+            ) as mock_list,
+        ):
+            await self._get_call_fn()(request=request, user_api_key_dict=user_api_key_dict)
+
+        mock_list.assert_awaited_once()
+        assert mock_list.await_args.kwargs["client_ip"] == "203.0.113.7"
+
+    @pytest.mark.asyncio
+    async def test_mcp_tool_search_requires_flag_enabled(self) -> None:
+        from fastapi import HTTPException
+
+        user_api_key_dict = UserAPIKeyAuth(
+            api_key="test_key",
+            object_permission=_make_perm(mcp_tool_search_enabled=False),
+        )
+
+        request = self._make_request({"name": MCP_TOOL_SEARCH_TOOL_NAME, "arguments": {"query": "create issue"}})
+
+        with pytest.raises(HTTPException) as exc_info:
+            await self._get_call_fn()(
+                request=request,
+                user_api_key_dict=user_api_key_dict,
+            )
+
+        assert exc_info.value.status_code in (400, 403, 404)
+
+
+class TestDispatchVirtualMcpTool:
+    """Covers the SSE/protocol-path interception helper in server.py."""
+
+    @pytest.mark.asyncio
+    async def test_returns_none_for_non_virtual_tool(self) -> None:
+        from litellm.proxy._experimental.mcp_server.server import (
+            _dispatch_virtual_mcp_tool,
+        )
+
+        result = await _dispatch_virtual_mcp_tool(
+            name="github-create_issue",
+            arguments={},
+            user_api_key_auth=UserAPIKeyAuth(api_key="k"),
+            client_ip=None,
+        )
+        assert result is None
+
+    @pytest.mark.asyncio
+    async def test_rejects_when_flag_disabled(self) -> None:
+        from litellm.proxy._experimental.mcp_server.server import (
+            _dispatch_virtual_mcp_tool,
+        )
+
+        uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=False))
+        result = await _dispatch_virtual_mcp_tool(
+            name=MCP_TOOL_SEARCH_TOOL_NAME,
+            arguments={"query": "x"},
+            user_api_key_auth=uak,
+            client_ip=None,
+        )
+        assert result is not None
+        assert result.isError is True
+
+    @pytest.mark.asyncio
+    async def test_routes_search_with_client_ip(self) -> None:
+        from litellm.proxy._experimental.mcp_server import server as srv
+
+        uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True))
+        with patch(
+            "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search",
+            new_callable=AsyncMock,
+            return_value="SEARCH_RESULT",
+        ) as mock_search:
+            result = await srv._dispatch_virtual_mcp_tool(
+                name=MCP_TOOL_SEARCH_TOOL_NAME,
+                arguments={"query": "q", "top_k": 3},
+                user_api_key_auth=uak,
+                client_ip="203.0.113.9",
+            )
+
+        assert result == "SEARCH_RESULT"
+        assert mock_search.await_args.kwargs["client_ip"] == "203.0.113.9"
+        assert mock_search.await_args.kwargs["query"] == "q"
+        assert mock_search.await_args.kwargs["top_k"] == 3
+
+    @pytest.mark.asyncio
+    async def test_routes_call_with_client_ip(self) -> None:
+        from litellm.proxy._experimental.mcp_server import server as srv
+
+        uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True))
+        with patch(
+            "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call",
+            new_callable=AsyncMock,
+            return_value="CALL_RESULT",
+        ) as mock_call:
+            result = await srv._dispatch_virtual_mcp_tool(
+                name=MCP_TOOL_CALL_TOOL_NAME,
+                arguments={"tool_name": "math-add", "arguments": {"a": 1, "b": 2}},
+                user_api_key_auth=uak,
+                client_ip="203.0.113.9",
+                mcp_auth_header="bearer-xyz",
+                mcp_server_auth_headers={"github": {"Authorization": "Bearer gh"}},
+                oauth2_headers={"Authorization": "Bearer oauth"},
+                raw_headers={"x-mcp-auth": "tok"},
+            )
+
+        assert result == "CALL_RESULT"
+        kw = mock_call.await_args.kwargs
+        assert kw["tool_name"] == "math-add"
+        assert kw["client_ip"] == "203.0.113.9"
+        assert kw["mcp_auth_header"] == "bearer-xyz"
+        assert kw["mcp_server_auth_headers"] == {"github": {"Authorization": "Bearer gh"}}
+        assert kw["oauth2_headers"] == {"Authorization": "Bearer oauth"}
+        assert kw["raw_headers"] == {"x-mcp-auth": "tok"}
+
+    @pytest.mark.asyncio
+    async def test_call_builds_and_forwards_logging_obj(self) -> None:
+        """Regression: the SSE dispatch must run the pre-call pipeline and forward
+        the resulting logging object to handle_mcp_tool_call, otherwise mcp_tool_call
+        over /mcp/ skips spend logging and guardrails (unlike the REST path)."""
+        from litellm.proxy._experimental.mcp_server import server as srv
+
+        uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True))
+        sentinel_logging_obj = object()
+        with (
+            patch.object(
+                srv,
+                "_build_virtual_call_logging_obj",
+                new_callable=AsyncMock,
+                return_value=sentinel_logging_obj,
+            ) as mock_build,
+            patch(
+                "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call",
+                new_callable=AsyncMock,
+                return_value="CALL_RESULT",
+            ) as mock_call,
+        ):
+            await srv._dispatch_virtual_mcp_tool(
+                name=MCP_TOOL_CALL_TOOL_NAME,
+                arguments={"tool_name": "math-add", "arguments": {"a": 1}},
+                user_api_key_auth=uak,
+                client_ip=None,
+            )
+
+        assert mock_build.await_count == 1
+        assert mock_call.await_args.kwargs["litellm_logging_obj"] is sentinel_logging_obj
+
+    @pytest.mark.asyncio
+    async def test_search_coerces_non_int_top_k(self) -> None:
+        """Regression: a non-integer top_k from an MCP client must not raise; it
+        falls back to the default instead of ValueError propagating out."""
+        from litellm.proxy._experimental.mcp_server import server as srv
+
+        uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True))
+        with patch(
+            "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search",
+            new_callable=AsyncMock,
+            return_value="SEARCH_RESULT",
+        ) as mock_search:
+            await srv._dispatch_virtual_mcp_tool(
+                name=MCP_TOOL_SEARCH_TOOL_NAME,
+                arguments={"query": "issue", "top_k": "not-a-number"},
+                user_api_key_auth=uak,
+                client_ip=None,
+            )
+
+        assert mock_search.await_args.kwargs["top_k"] == 5
+
+    @pytest.mark.asyncio
+    async def test_call_handler_forwards_auth_headers_to_execute(self) -> None:
+        """Regression: per-request auth headers must reach execute_mcp_tool so
+        upstream MCP servers needing pass-through auth can be called."""
+        from mcp.types import CallToolResult, TextContent
+
+        from litellm.proxy._experimental.mcp_server.tool_search import (
+            handle_mcp_tool_call,
+        )
+
+        uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True))
+        fake = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False)
+        with (
+            patch(
+                "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+                new_callable=AsyncMock,
+                return_value=[MagicMock()],
+            ) as mock_allowed,
+            patch(
+                "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool",
+                new_callable=AsyncMock,
+                return_value=fake,
+            ) as mock_exec,
+        ):
+            sentinel_logging_obj = object()
+            await handle_mcp_tool_call(
+                tool_name="github-create_issue",
+                arguments={},
+                user_api_key_dict=uak,
+                mcp_servers=["github"],
+                mcp_auth_header="bearer-xyz",
+                mcp_server_auth_headers={"github": {"Authorization": "Bearer gh"}},
+                oauth2_headers={"Authorization": "Bearer oauth"},
+                raw_headers={"x-mcp-auth": "tok"},
+                litellm_logging_obj=sentinel_logging_obj,
+            )
+
+        kw = mock_exec.await_args.kwargs
+        assert kw["mcp_auth_header"] == "bearer-xyz"
+        assert kw["mcp_server_auth_headers"] == {"github": {"Authorization": "Bearer gh"}}
+        assert kw["oauth2_headers"] == {"Authorization": "Bearer oauth"}
+        assert kw["raw_headers"] == {"x-mcp-auth": "tok"}
+        # Spend logging: the logging object must reach execute_mcp_tool
+        assert kw["litellm_logging_obj"] is sentinel_logging_obj
+        # Scoped session: the requested mcp_servers scope must reach server resolution
+        assert mock_allowed.await_args.kwargs["mcp_servers"] == ["github"]
+
+    @pytest.mark.asyncio
+    async def test_call_rejected_when_no_accessible_servers(self) -> None:
+        """Regression: a key with no accessible MCP servers must not reach
+        execute_mcp_tool, where an unprefixed local tool name would otherwise
+        run via the local registry without a server permission check."""
+        from fastapi import HTTPException
+
+        from litellm.proxy._experimental.mcp_server.tool_search import (
+            handle_mcp_tool_call,
+        )
+
+        uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True))
+        with (
+            patch(
+                "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
+                new_callable=AsyncMock,
+                return_value=[],
+            ),
+            patch(
+                "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool",
+                new_callable=AsyncMock,
+            ) as mock_exec,
+        ):
+            with pytest.raises(HTTPException) as exc_info:
+                await handle_mcp_tool_call(
+                    tool_name="local_secret_tool",
+                    arguments={},
+                    user_api_key_dict=uak,
+                )
+
+        assert exc_info.value.status_code == 403
+        mock_exec.assert_not_awaited()
+
+
+class TestCaptureHostProgressCallback:
+    """Covers the host progress-forwarding helper extracted from the tool call path."""
+
+    def test_returns_none_when_request_context_unavailable(self) -> None:
+        from litellm.proxy._experimental.mcp_server.server import (
+            _capture_host_progress_callback,
+        )
+
+        class _NoCtx:
+            @property
+            def request_context(self):  # type: ignore[no-untyped-def]
+                raise RuntimeError("no context")
+
+        assert _capture_host_progress_callback(_NoCtx()) is None
+
+    def test_returns_none_when_no_progress_token(self) -> None:
+        from litellm.proxy._experimental.mcp_server.server import (
+            _capture_host_progress_callback,
+        )
+
+        host = MagicMock()
+        host.request_context.meta.progressToken = None
+        assert _capture_host_progress_callback(host) is None
+
+    def test_returns_callable_when_token_present(self) -> None:
+        from litellm.proxy._experimental.mcp_server.server import (
+            _capture_host_progress_callback,
+        )
+
+        host = MagicMock()
+        host.request_context.meta.progressToken = "tok12345"
+        host.request_context.session = MagicMock()
+        assert callable(_capture_host_progress_callback(host))
+
+
+class TestHandleListToolsVirtual:
+    """Covers the protocol list_tools early-return when the flag is enabled."""
+
+    @pytest.mark.asyncio
+    async def test_returns_virtual_tools_when_flag_enabled(self) -> None:
+        from litellm.proxy._experimental.mcp_server import server as srv
+
+        uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True))
+        with patch(
+            "litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context",
+            new_callable=AsyncMock,
+            return_value=(uak, None, None, None, None, None, None),
+        ):
+            tools = await srv.handle_list_tools()
+
+        assert {t.name for t in tools} == {
+            MCP_TOOL_SEARCH_TOOL_NAME,
+            MCP_TOOL_CALL_TOOL_NAME,
+        }
+
+
+class TestMcpServerToolCallErrorHandling:
+    """The protocol tool-call handler must convert virtual-tool errors to an
+    isError CallToolResult instead of letting them raise out of the handler."""
+
+    @pytest.mark.asyncio
+    async def test_virtual_tool_error_returns_iserror_not_raised(self) -> None:
+        from fastapi import HTTPException
+
+        from litellm.proxy._experimental.mcp_server import server as srv
+
+        uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True))
+        with (
+            patch(
+                "litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context",
+                new_callable=AsyncMock,
+                return_value=(uak, None, None, None, None, None, None),
+            ),
+            patch(
+                "litellm.proxy._experimental.mcp_server.server._dispatch_virtual_mcp_tool",
+                new_callable=AsyncMock,
+                side_effect=HTTPException(status_code=403, detail="User not allowed to call this tool"),
+            ),
+        ):
+            result = await srv.mcp_server_tool_call(
+                name=MCP_TOOL_CALL_TOOL_NAME,
+                arguments={"tool_name": "other-server-tool", "arguments": {}},
+            )
+
+        assert result.isError is True
+        assert "User not allowed to call this tool" in result.content[0].text
diff --git a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py
index d4089b23e81..5fbc3c4869b 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py
@@ -718,6 +718,7 @@ _EXPECTED_CUSTOMER = {
         "mcp_toolsets": None,
         "blocked_tools": [],
         "search_tools": [],
+        "mcp_tool_search_enabled": None,
     },
 }
 
diff --git a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py
index 26c8c774812..0981c4239ee 100644
--- a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py
+++ b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py
@@ -87,6 +87,38 @@ async def test_set_object_permission():
     assert result["models"] == ["gpt-4"]
 
 
+@pytest.mark.asyncio
+async def test_set_object_permission_persists_mcp_tool_search_enabled():
+    """
+    Regression: mcp_tool_search_enabled must be carried into the Prisma create
+    payload so it persists to LiteLLM_ObjectPermissionTable. The field was
+    present on the Pydantic models but missing from the create path, so keys
+    generated with mcp_tool_search_enabled=True silently lost the flag.
+    """
+    mock_prisma_client = MagicMock()
+    mock_created_permission = MagicMock()
+    mock_created_permission.object_permission_id = "perm_id"
+    mock_prisma_client.db.litellm_objectpermissiontable.create = AsyncMock(
+        return_value=mock_created_permission
+    )
+
+    data_json = {
+        "object_permission": {
+            "mcp_servers": ["server_a"],
+            "mcp_tool_search_enabled": True,
+        },
+    }
+
+    await _set_object_permission(data_json=data_json, prisma_client=mock_prisma_client)
+
+    created_data = (
+        mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs[
+            "data"
+        ]
+    )
+    assert created_data["mcp_tool_search_enabled"] is True
+
+
 # ---- Tests for _extract_requested_mcp_server_ids ----
 
 
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index f15eaf9ea1f..ddf2040cd04 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -25107,6 +25107,8 @@ export interface components {
             mcp_tool_permissions?: {
                 [key: string]: string[];
             } | null;
+            /** Mcp Tool Search Enabled */
+            mcp_tool_search_enabled?: boolean | null;
             /** Mcp Toolsets */
             mcp_toolsets?: string[] | null;
             /** Models */
@@ -25150,6 +25152,8 @@ export interface components {
             mcp_tool_permissions?: {
                 [key: string]: string[];
             } | null;
+            /** Mcp Tool Search Enabled */
+            mcp_tool_search_enabled?: boolean | null;
             /** Mcp Toolsets */
             mcp_toolsets?: string[] | null;
             /**

From 13b590c8ec8d5c0d69bdd6e6affe51a57976fd4b Mon Sep 17 00:00:00 2001
From: tin-berri <tin@berri.ai>
Date: Tue, 30 Jun 2026 21:23:49 -0700
Subject: [PATCH 51/51] fix(proxy): hydrate MCP server registry from DB on
 startup when store_model_in_db is false (#31775)

MCP servers created through the UI are persisted to the database independent of
store_model_in_db, but the in-memory registry that GET /v1/mcp/server reads was
hydrated from the database only through add_deployment, which runs solely when
store_model_in_db is True. On a DB-backed single-instance proxy with
store_model_in_db unset the registry started empty after a restart, so the MCP
Servers page showed nothing until an add or edit triggered a reload.

Hydrate the registry from the database on startup regardless of store_model_in_db
via a new ProxyConfig.init_mcp_servers_from_db, honoring supported_db_objects.
---
 litellm/proxy/proxy_server.py                 |  7 +++
 tests/test_litellm/proxy/test_proxy_server.py | 60 +++++++++++++++++++
 2 files changed, 67 insertions(+)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 2f6c48a751b..6d64843fab0 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -6314,6 +6314,10 @@ class ProxyConfig:
                 "litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db - {}".format(str(e))
             )
 
+    async def init_mcp_servers_from_db(self) -> None:
+        if self._should_load_db_object(object_type="mcp"):
+            await self._init_mcp_servers_in_db()
+
     async def _init_agents_in_db(self, prisma_client: PrismaClient):
         from litellm.proxy.agent_endpoints.agent_registry import (
             global_agent_registry as AGENT_REGISTRY,
@@ -7561,6 +7565,9 @@ class ProxyStartupEvent:
             )
             await proxy_config.get_credentials(prisma_client=prisma_client)
 
+        if store_model_in_db is not True:
+            await proxy_config.init_mcp_servers_from_db()
+
         await cls._initialize_slack_alerting_jobs(
             scheduler=scheduler,
             general_settings=general_settings,
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index 88d9ad0d968..0d6cd972459 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -754,6 +754,66 @@ async def test_initialize_scheduled_jobs_credentials(monkeypatch):
         assert len(mock_scheduler_calls) > 0
 
 
+@pytest.mark.asyncio
+async def test_initialize_scheduled_jobs_hydrates_mcp_when_store_model_in_db_false(monkeypatch):
+    """
+    Regression (LIT-4128): MCP servers created via the UI are persisted to the DB
+    regardless of store_model_in_db, but the in-memory registry that GET
+    /v1/mcp/server reads is hydrated from the DB only by the store_model_in_db
+    model-sync loop (add_deployment). On a DB-backed proxy with store_model_in_db
+    unset the registry must still be hydrated on startup so previously-added
+    servers survive a restart instead of showing an empty list until a write.
+    """
+    monkeypatch.delenv("DISABLE_PRISMA_SCHEMA_UPDATE", raising=False)
+    monkeypatch.delenv("STORE_MODEL_IN_DB", raising=False)
+    from litellm.proxy.proxy_server import ProxyStartupEvent
+    from litellm.proxy.utils import ProxyLogging
+
+    mock_prisma_client = MagicMock()
+    mock_proxy_logging = MagicMock(spec=ProxyLogging)
+    mock_proxy_logging.slack_alerting_instance = MagicMock()
+    mock_proxy_config = AsyncMock()
+
+    with (
+        patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config),
+        patch("litellm.proxy.proxy_server.store_model_in_db", False),
+    ):
+        await ProxyStartupEvent.initialize_scheduled_background_jobs(
+            general_settings={},
+            prisma_client=mock_prisma_client,
+            proxy_budget_rescheduler_min_time=1,
+            proxy_budget_rescheduler_max_time=2,
+            proxy_batch_write_at=5,
+            proxy_logging_obj=mock_proxy_logging,
+        )
+
+    mock_proxy_config.add_deployment.assert_not_called()
+    mock_proxy_config.init_mcp_servers_from_db.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_init_mcp_servers_from_db_respects_supported_db_objects(monkeypatch):
+    """
+    init_mcp_servers_from_db hydrates MCP from the DB by default but skips it when
+    an explicit supported_db_objects allowlist omits "mcp".
+    """
+    from litellm.proxy.proxy_server import ProxyConfig
+
+    config = ProxyConfig()
+    with patch.object(config, "_init_mcp_servers_in_db", new=AsyncMock()) as mock_init:
+        monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {})
+        await config.init_mcp_servers_from_db()
+        mock_init.assert_awaited_once()
+
+        mock_init.reset_mock()
+        monkeypatch.setattr(
+            "litellm.proxy.proxy_server.general_settings",
+            {"supported_db_objects": ["models"]},
+        )
+        await config.init_mcp_servers_from_db()
+        mock_init.assert_not_awaited()
+
+
 def test_update_config_fields_deep_merge_db_wins():
     from litellm.proxy.proxy_server import ProxyConfig