From 89d503778026b5b1f48d3716b4af1103ce8c1e94 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 12 Aug 2026 21:48:51 -0700 Subject: [PATCH 01/10] test(ui): characterize TokenFlow and JsonViewer Both components are shared by the logs, guardrails-monitor and tool-policies routes and had no test. These assert on rendered text and roles only, so they hold against antd Typography and against its replacement. --- .../LogDetailsDrawer/JsonViewer.test.tsx | 28 +++++++++++++++++++ .../LogDetailsDrawer/TokenFlow.test.tsx | 23 +++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/JsonViewer.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TokenFlow.test.tsx diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/JsonViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/JsonViewer.test.tsx new file mode 100644 index 00000000000..055e400167b --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/JsonViewer.test.tsx @@ -0,0 +1,28 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { JsonViewer } from "./JsonViewer"; + +describe("JsonViewer", () => { + it("should render a placeholder and no tree when the log entry carries no payload", () => { + render(); + + expect(screen.getByText("No data")).toBeInTheDocument(); + expect(screen.queryByRole("tree")).not.toBeInTheDocument(); + }); + + it("should render the payload as a tree exposing its keys", () => { + render(); + + expect(screen.getByRole("tree")).toBeInTheDocument(); + expect(screen.getByText(/model/)).toBeInTheDocument(); + expect(screen.getByText(/stream/)).toBeInTheDocument(); + expect(screen.queryByText("No data")).not.toBeInTheDocument(); + }); + + it("should treat an empty payload as data rather than showing the placeholder", () => { + render(); + + expect(screen.getByRole("tree")).toBeInTheDocument(); + expect(screen.queryByText("No data")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TokenFlow.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TokenFlow.test.tsx new file mode 100644 index 00000000000..f1784bb4009 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TokenFlow.test.tsx @@ -0,0 +1,23 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { TokenFlow } from "./TokenFlow"; + +describe("TokenFlow", () => { + it("should render the total followed by its prompt and completion breakdown", () => { + render(); + + expect(screen.getByText("12 (9 prompt tokens + 3 completion tokens)")).toBeInTheDocument(); + }); + + it("should group large counts with thousands separators", () => { + render(); + + expect(screen.getByText("1,323,579 (1,234,567 prompt tokens + 89,012 completion tokens)")).toBeInTheDocument(); + }); + + it("should fall back to zero for counts the log entry does not carry", () => { + render(); + + expect(screen.getByText("12 (0 prompt tokens + 0 completion tokens)")).toBeInTheDocument(); + }); +}); From 49c697ee895b7509c73526e4f62dd792a9d3abd7 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 12 Aug 2026 22:03:20 -0700 Subject: [PATCH 02/10] refactor(ui): migrate HistoryTree and CollapsibleMessage to shadcn Both log-drawer collapse rows hand-rolled the same pattern: a click handler on a plain div, hover tracked in React state, and a max-height tween on an always-mounted panel. Move both onto the ui/collapsible primitive with lucide chevrons, so the row is a real button that keyboard users can reach and the open state lives in the primitive. CollapsibleContent keeps keepMounted, which preserves the existing contract that panel content stays in the DOM while collapsed. Neither test file is touched: both were already role and text based, and they pass unedited against the new markup. --- ui/litellm-dashboard/eslint-suppressions.json | 10 --- .../LogDetailsDrawer/CollapsibleMessage.tsx | 70 ++++------------- .../LogDetailsDrawer/HistoryTree.tsx | 76 +++++-------------- 3 files changed, 36 insertions(+), 120 deletions(-) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 8a86305b4cb..299d2a6cfdf 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -3858,21 +3858,11 @@ "count": 1 } }, - "src/components/view_logs/LogDetailsDrawer/CollapsibleMessage.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/view_logs/LogDetailsDrawer/DrawerHeader.tsx": { "no-restricted-imports": { "count": 1 } }, - "src/components/view_logs/LogDetailsDrawer/HistoryTree.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/view_logs/LogDetailsDrawer/JsonViewer.tsx": { "no-restricted-imports": { "count": 1 diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/CollapsibleMessage.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/CollapsibleMessage.tsx index 9aa13c3832a..81d2fc20fdf 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/CollapsibleMessage.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/CollapsibleMessage.tsx @@ -4,10 +4,8 @@ */ import { useState } from "react"; -import { Typography } from "antd"; -import { DownOutlined, RightOutlined } from "@ant-design/icons"; - -const { Text } = Typography; +import { ChevronDown, ChevronRight } from "lucide-react"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; interface CollapsibleMessageProps { label: string; @@ -17,7 +15,6 @@ interface CollapsibleMessageProps { export function CollapsibleMessage({ label, content, defaultExpanded = false }: CollapsibleMessageProps) { const [isExpanded, setIsExpanded] = useState(defaultExpanded); - const [isHovered, setIsHovered] = useState(false); const charCount = content?.length || 0; if (!content || charCount === 0) { @@ -25,60 +22,23 @@ export function CollapsibleMessage({ label, content, defaultExpanded = false }: } return ( -
- {/* Clickable Header with hover state */} -
setIsExpanded(!isExpanded)} - onMouseEnter={() => setIsHovered(true)} - onMouseLeave={() => setIsHovered(false)} - style={{ - display: "flex", - alignItems: "center", - gap: 6, - cursor: "pointer", - padding: "4px 0", - borderRadius: 4, - background: isHovered ? "#f5f5f5" : "transparent", - transition: "background 0.15s ease", - marginBottom: isExpanded ? 4 : 0, - }} - > + + {isExpanded ? ( - + ) : ( - + )} - - {label} - - - ({charCount.toLocaleString()} chars) - -
+ {label} + ({charCount.toLocaleString()} chars) + - {/* Content with smooth animation */} -
-
- {content} -
-
-
+ {content} + + ); } diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/HistoryTree.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/HistoryTree.tsx index b6c5048513c..4854ca90fc2 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/HistoryTree.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/HistoryTree.tsx @@ -4,80 +4,46 @@ */ import { useState } from "react"; -import { Typography } from "antd"; -import { DownOutlined, RightOutlined } from "@ant-design/icons"; +import { ChevronDown, ChevronRight } from "lucide-react"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import { ParsedMessage } from "./prettyMessagesTypes"; import { SimpleMessageBlock } from "./SimpleMessageBlock"; -const { Text } = Typography; - interface HistoryTreeProps { messages: ParsedMessage[]; } export function HistoryTree({ messages }: HistoryTreeProps) { const [isExpanded, setIsExpanded] = useState(false); - const [isHovered, setIsHovered] = useState(false); if (messages.length === 0) { return null; } return ( -
- {/* Clickable Header with hover state */} -
setIsExpanded(!isExpanded)} - onMouseEnter={() => setIsHovered(true)} - onMouseLeave={() => setIsHovered(false)} - style={{ - display: "flex", - alignItems: "center", - gap: 6, - cursor: "pointer", - padding: "4px 0", - borderRadius: 4, - background: isHovered ? "#f5f5f5" : "transparent", - transition: "background 0.15s ease", - marginBottom: isExpanded ? 4 : 0, - }} - > + + {isExpanded ? ( - + ) : ( - + )} - + HISTORY ({messages.length} message{messages.length !== 1 ? "s" : ""}) - -
+ + - {/* Expanded Tree Content with smooth animation */} -
-
- {messages.map((msg, index) => ( - - ))} -
-
-
+ + {messages.map((msg, index) => ( + + ))} + + ); } From 214a476c3e196812447060c68e6c7ca54679aab0 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 12 Aug 2026 22:10:39 -0700 Subject: [PATCH 03/10] refactor(ui): migrate TokenFlow and JsonViewer to shadcn Drops the antd Typography import from both. TokenFlow renders inside an antd Descriptions.Item that already sets the colour, font size, line height and wrapping the Text wrapper restated, so a bare span is pixel-identical there. JsonViewer's placeholder moves onto the muted-foreground token. The characterisation tests from the previous commit are unchanged and stay green, which is what shows the markup swap did not move behaviour. --- ui/litellm-dashboard/eslint-suppressions.json | 10 ---------- .../view_logs/LogDetailsDrawer/JsonViewer.tsx | 5 +---- .../view_logs/LogDetailsDrawer/TokenFlow.tsx | 8 ++------ 3 files changed, 3 insertions(+), 20 deletions(-) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 8a86305b4cb..8b067f79af6 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -3873,11 +3873,6 @@ "count": 1 } }, - "src/components/view_logs/LogDetailsDrawer/JsonViewer.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx": { "no-nested-ternary": { "count": 3 @@ -3927,11 +3922,6 @@ "count": 1 } }, - "src/components/view_logs/LogDetailsDrawer/TokenFlow.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/view_logs/LogDetailsDrawer/TruncatedValue.tsx": { "no-restricted-imports": { "count": 1 diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/JsonViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/JsonViewer.tsx index 0a873a7e255..af99eee0d16 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/JsonViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/JsonViewer.tsx @@ -1,10 +1,7 @@ -import { Typography } from "antd"; import { JsonView, defaultStyles } from "react-json-view-lite"; import "react-json-view-lite/dist/index.css"; import { JSON_MAX_HEIGHT, COLOR_BG_LIGHT, SPACING_LARGE } from "./constants"; -const { Text } = Typography; - interface JsonViewerProps { data: any; mode: "formatted"; @@ -15,7 +12,7 @@ interface JsonViewerProps { * Uses an interactive tree component for easy navigation. */ export function JsonViewer({ data }: JsonViewerProps) { - if (!data) return No data; + if (!data) return No data; return (
+ {total.toLocaleString()} ({prompt.toLocaleString()} prompt tokens + {completion.toLocaleString()} completion tokens) - + ); } From c25f927f034803e4613955872bb48f8f6219c1e6 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 12 Aug 2026 22:23:06 -0700 Subject: [PATCH 04/10] test(ui): make the TokenFlow grouping assertion locale-independent The assertion hard-coded en-US separators while toLocaleString follows the host locale, so it failed under de_DE. Building the expected string the same way keeps it deterministic everywhere and still catches a dropped toLocaleString wherever the locale groups at all. --- .../view_logs/LogDetailsDrawer/TokenFlow.test.tsx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TokenFlow.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TokenFlow.test.tsx index f1784bb4009..0e76b92da36 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TokenFlow.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TokenFlow.test.tsx @@ -2,6 +2,8 @@ import { render, screen } from "@testing-library/react"; import { describe, expect, it } from "vitest"; import { TokenFlow } from "./TokenFlow"; +const localised = (count: number) => count.toLocaleString(); + describe("TokenFlow", () => { it("should render the total followed by its prompt and completion breakdown", () => { render(); @@ -9,10 +11,14 @@ describe("TokenFlow", () => { expect(screen.getByText("12 (9 prompt tokens + 3 completion tokens)")).toBeInTheDocument(); }); - it("should group large counts with thousands separators", () => { + it("should group large counts the way the reader's locale does", () => { render(); - expect(screen.getByText("1,323,579 (1,234,567 prompt tokens + 89,012 completion tokens)")).toBeInTheDocument(); + expect( + screen.getByText( + `${localised(1323579)} (${localised(1234567)} prompt tokens + ${localised(89012)} completion tokens)`, + ), + ).toBeInTheDocument(); }); it("should fall back to zero for counts the log entry does not carry", () => { From 9355953414f80166e3564c091e84bacd76482f4c Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 12 Aug 2026 22:27:20 -0700 Subject: [PATCH 05/10] refactor(ui): migrate SimpleMessageBlock and SimpleToolCallBlock to shadcn Replace antd Typography and inline hex styling in the log drawer's message and tool-call blocks with plain elements and design tokens. Both components' existing tests pass unedited before and after. Retires their no-restricted-imports entries from the eslint suppressions baseline. --- ui/litellm-dashboard/eslint-suppressions.json | 10 ----- .../LogDetailsDrawer/SimpleMessageBlock.tsx | 31 +++---------- .../LogDetailsDrawer/SimpleToolCallBlock.tsx | 44 +++++-------------- 3 files changed, 17 insertions(+), 68 deletions(-) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 8a86305b4cb..3675bb540fc 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -3917,16 +3917,6 @@ "count": 1 } }, - "src/components/view_logs/LogDetailsDrawer/SimpleMessageBlock.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/view_logs/LogDetailsDrawer/SimpleToolCallBlock.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/view_logs/LogDetailsDrawer/TokenFlow.tsx": { "no-restricted-imports": { "count": 1 diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SimpleMessageBlock.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SimpleMessageBlock.tsx index 0262f9d7879..2f50981211b 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SimpleMessageBlock.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SimpleMessageBlock.tsx @@ -3,12 +3,10 @@ * Used for messages in tree view and last user message */ -import { Typography } from "antd"; +import { cn } from "@/lib/cva.config"; import { ToolCall } from "./prettyMessagesTypes"; import { SimpleToolCallBlock } from "./SimpleToolCallBlock"; -const { Text } = Typography; - interface SimpleMessageBlockProps { label: string; content?: string; @@ -27,30 +25,15 @@ export function SimpleMessageBlock({ label, content, toolCalls, isCompact = fals } return ( -
- - {label} - +
+ {label} {displayContent && (
{displayContent}
diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SimpleToolCallBlock.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SimpleToolCallBlock.tsx index 42738794e2f..6d69d07682c 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SimpleToolCallBlock.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SimpleToolCallBlock.tsx @@ -3,11 +3,9 @@ * Used in compact/tree views */ -import { Typography } from "antd"; +import { cn } from "@/lib/cva.config"; import { ToolCall } from "./prettyMessagesTypes"; -const { Text } = Typography; - interface SimpleToolCallBlockProps { tool: ToolCall; compact?: boolean; @@ -16,46 +14,24 @@ interface SimpleToolCallBlockProps { export function SimpleToolCallBlock({ tool, compact = false }: SimpleToolCallBlockProps) { return (
{/* Function badge */} -
+
function
- - {tool.name} - + {tool.name} {Object.keys(tool.arguments).length > 0 && (
{Object.entries(tool.arguments).map(([key, value]) => ( -
- - {key}:{" "} - - {JSON.stringify(value)} +
+ {key}: + {JSON.stringify(value)}
))}
From f17143086a8b9a48ec6b839cbe6e4c2e6cd3ccec Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 12 Aug 2026 22:29:57 -0700 Subject: [PATCH 06/10] refactor(ui): migrate TruncatedValue and OutputCard to shadcn TruncatedValue swaps antd Tooltip and Typography for the shadcn Tooltip and the shared CopyButton, so the full value now lives in the page and is clipped with CSS rather than shortened in the text node. OutputCard drops antd Typography for token-styled markup and folds its two duplicated branches into one. Its border and the drawer's mono font stay on the module's existing constants so the card still matches InputCard, which is out of scope here. Both files retire their no-restricted-imports suppression. --- ui/litellm-dashboard/eslint-suppressions.json | 10 --- .../view_logs/LogDetailsDrawer/OutputCard.tsx | 68 +++---------------- .../LogDetailsDrawer/TruncatedValue.tsx | 39 ++++++----- 3 files changed, 30 insertions(+), 87 deletions(-) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 8a86305b4cb..34d8c90e7b3 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -3897,11 +3897,6 @@ "count": 2 } }, - "src/components/view_logs/LogDetailsDrawer/OutputCard.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/view_logs/LogDetailsDrawer/RealtimePrettyView.test.tsx": { "unused-imports/no-unused-imports": { "count": 2 @@ -3932,11 +3927,6 @@ "count": 1 } }, - "src/components/view_logs/LogDetailsDrawer/TruncatedValue.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/view_logs/LogDetailsDrawer/useKeyboardNavigation.ts": { "react-hooks/immutability": { "count": 2 diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/OutputCard.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/OutputCard.tsx index 9362d249874..2b7f26a63e4 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/OutputCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/OutputCard.tsx @@ -4,14 +4,12 @@ */ import { useState } from "react"; -import { Typography } from "antd"; import MessageManager from "@/components/molecules/message_manager"; +import { COLOR_BORDER } from "./constants"; import { ParsedMessage } from "./prettyMessagesTypes"; import { SectionHeader } from "./SectionHeader"; import { SimpleMessageBlock } from "./SimpleMessageBlock"; -const { Text } = Typography; - interface OutputCardProps { message: ParsedMessage | null; completionTokens?: number; @@ -24,55 +22,12 @@ export function OutputCard({ message, completionTokens, outputCost }: OutputCard const handleCopy = () => { if (!message) return; - const content = message.content || ""; - navigator.clipboard.writeText(content); + navigator.clipboard.writeText(message.content || ""); MessageManager.success("Output copied"); }; - if (!message) { - return ( -
- setIsCollapsed(!isCollapsed)} - /> -
-
- - No response data available - -
-
-
- ); - } - return ( -
- {/* Datadog-style Header */} +
setIsCollapsed(!isCollapsed)} /> - {/* Content */}
-
- +
+ {message ? ( + + ) : ( + No response data available + )}
diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TruncatedValue.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TruncatedValue.tsx index b03895808d6..2d4fa7ad7ef 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TruncatedValue.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TruncatedValue.tsx @@ -1,7 +1,6 @@ -import { Typography, Tooltip } from "antd"; -import { DEFAULT_MAX_WIDTH, FONT_FAMILY_MONO, FONT_SIZE_SMALL } from "./constants"; - -const { Text } = Typography; +import CopyButton from "@/components/shared/CopyButton"; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; +import { DEFAULT_MAX_WIDTH, FONT_FAMILY_MONO } from "./constants"; interface TruncatedValueProps { value?: string; @@ -13,23 +12,23 @@ interface TruncatedValueProps { * Useful for displaying long IDs, URLs, or other text that may overflow. */ export function TruncatedValue({ value, maxWidth = DEFAULT_MAX_WIDTH }: TruncatedValueProps) { - if (!value) return -; + if (!value) return -; return ( - - - {value} - - + + + + + {value} + + + + } + /> + {value} + + ); } From 266a35b88c61a7fd97ee96d27df6eca36a2adb62 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 12 Aug 2026 22:39:29 -0700 Subject: [PATCH 07/10] test(ui): cover keyboard activation of the log drawer collapse rows The migration turned each collapse row into a real button, but the existing tests only click, so a regression in Enter or Space activation would still pass. Add one test per component that tabs to the row, expands with Enter and collapses with Space, asserting visibility rather than markup. Both fail against the antd version and pass against the migrated one. --- .../CollapsibleMessage.test.tsx | 14 ++++++++++++++ .../LogDetailsDrawer/HistoryTree.test.tsx | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/CollapsibleMessage.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/CollapsibleMessage.test.tsx index 1477febd967..3aa8d21c31c 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/CollapsibleMessage.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/CollapsibleMessage.test.tsx @@ -37,4 +37,18 @@ describe("CollapsibleMessage", () => { await user.click(screen.getByText("SYSTEM")); expect(screen.getByText("Toggle me")).toBeInTheDocument(); }); + + it("should expand with Enter and collapse with Space from the keyboard", async () => { + const user = userEvent.setup(); + render(); + + expect(screen.getByText("Toggle me")).not.toBeVisible(); + + await user.tab(); + await user.keyboard("{Enter}"); + expect(screen.getByText("Toggle me")).toBeVisible(); + + await user.keyboard(" "); + expect(screen.getByText("Toggle me")).not.toBeVisible(); + }); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/HistoryTree.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/HistoryTree.test.tsx index 10d6bfa0847..0726a9f4b82 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/HistoryTree.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/HistoryTree.test.tsx @@ -41,4 +41,22 @@ describe("HistoryTree", () => { expect(screen.getByText("Hello")).toBeInTheDocument(); expect(screen.getByText("Hi there")).toBeInTheDocument(); }); + + it("should expand with Enter and collapse with Space from the keyboard", async () => { + const user = userEvent.setup(); + const messages: ParsedMessage[] = [ + { role: "user", content: "Hello" }, + { role: "assistant", content: "Hi there" }, + ]; + render(); + + expect(screen.getByText("Hello")).not.toBeVisible(); + + await user.tab(); + await user.keyboard("{Enter}"); + expect(screen.getByText("Hello")).toBeVisible(); + + await user.keyboard(" "); + expect(screen.getByText("Hello")).not.toBeVisible(); + }); }); From 6b5249bcce05b70a57aa1221dcd2dd252106c4cb Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 13 Aug 2026 03:59:26 -0700 Subject: [PATCH 08/10] refactor: replace Any with precise types across responses, proxy, and llms modules Types 28 files with Protocols, TypedDicts, and Pydantic validation in place of Any, cutting basedpyright reportAny by 974 and reportExplicitAny by 261 (1411 errors total across 48 rules), and ratchets the basedpyright, ruff strict, and type discipline budgets down to match --- basedpyright-code-budget.json | 20 +-- .../litellm_completion_bridge/handler.py | 23 +-- litellm/a2a_protocol/main.py | 66 ++++---- .../google_genai/adapters/transformation.py | 61 ++++--- litellm/integrations/rubrik.py | 85 ++++++---- .../handler.py | 38 +++-- .../context_management/editors/compact.py | 74 ++++++--- litellm/llms/azure/common_utils.py | 31 +++- litellm/llms/bedrock/files/transformation.py | 94 +++++++---- .../llms/runwayml/videos/transformation.py | 55 +++++-- .../llms/vertex_ai/files/transformation.py | 71 +++++++-- .../llms/vertex_ai/videos/transformation.py | 71 ++++++--- .../proxy/_experimental/mcp_server/utils.py | 71 ++++++--- .../hiddenlayer/hiddenlayer.py | 32 +++- .../proxy/guardrails/guardrail_registry.py | 38 +++-- .../customer_endpoints.py | 118 ++++++++++---- .../mcp_management_endpoints.py | 5 +- .../tool_management_endpoints.py | 149 ++++++++++++++---- .../usage_endpoints/ai_usage_chat.py | 75 ++++++--- .../user_agent_analytics_endpoints.py | 119 ++++++++++---- litellm/proxy/prompts/prompt_endpoints.py | 144 ++++++++++++----- .../public_endpoints/public_endpoints.py | 83 ++++++++-- litellm/proxy/rag_endpoints/endpoints.py | 28 ++-- .../response_polling/background_streaming.py | 30 ++-- .../responses/file_search/emulated_handler.py | 65 ++++---- .../responses/mcp/mcp_streaming_iterator.py | 17 +- litellm/responses/streaming_iterator.py | 41 ++--- litellm/responses/utils.py | 20 +-- litellm/router_strategy/tag_based_routing.py | 122 +++++++++----- ruff-strict-budget.json | 8 +- type-discipline-budget.json | 8 +- 31 files changed, 1274 insertions(+), 588 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 7e4cc2d6100..e85e3701539 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,9 +1,9 @@ { "reportAny": { - "limit": 23914 + "limit": 22941 }, "reportArgumentType": { - "limit": 2580 + "limit": 2579 }, "reportAssignmentType": { "limit": 323 @@ -24,7 +24,7 @@ "limit": 19 }, "reportExplicitAny": { - "limit": 7573 + "limit": 7312 }, "reportFunctionMemberAccess": { "limit": 7 @@ -54,10 +54,10 @@ "limit": 0 }, "reportMissingParameterType": { - "limit": 5719 + "limit": 5707 }, "reportMissingTypeArgument": { - "limit": 15657 + "limit": 15642 }, "reportMissingTypeStubs": { "limit": 40 @@ -99,22 +99,22 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 44832 + "limit": 44776 }, "reportUnknownLambdaType": { "limit": 113 }, "reportUnknownMemberType": { - "limit": 39269 + "limit": 39237 }, "reportUnknownParameterType": { - "limit": 19988 + "limit": 19969 }, "reportUnknownVariableType": { - "limit": 30923 + "limit": 30881 }, "reportUnnecessaryCast": { - "limit": 118 + "limit": 117 }, "reportUnnecessaryComparison": { "limit": 699 diff --git a/litellm/a2a_protocol/litellm_completion_bridge/handler.py b/litellm/a2a_protocol/litellm_completion_bridge/handler.py index 90a82e8fa28..a62a2b0c724 100644 --- a/litellm/a2a_protocol/litellm_completion_bridge/handler.py +++ b/litellm/a2a_protocol/litellm_completion_bridge/handler.py @@ -10,7 +10,7 @@ A2A Streaming Events (in order): 4. Status update (kind: "status-update") - Final status "completed" with final=true """ -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Callable, Coroutine, Mapping from typing import Any, Final import litellm @@ -54,7 +54,7 @@ class A2ACompletionBridgeHandler: agent_extra_headers: Mapping[str, str] | None, *, stream: bool, - ) -> Mapping[str, Any]: + ) -> Mapping[str, object]: # Extract message from params message: Final = params.get("message", {}) @@ -63,7 +63,7 @@ class A2ACompletionBridgeHandler: # Get completion params custom_llm_provider: Final = litellm_params.get("custom_llm_provider") - model: Final = litellm_params.get("model", "agent") + model: Final[str] = litellm_params.get("model", "agent") # Build full model string if provider specified # Skip prepending if model already starts with the provider prefix @@ -109,13 +109,16 @@ class A2ACompletionBridgeHandler: return completion_params @staticmethod - async def _acompletion(completion_params: Mapping[str, Any]) -> ModelResponse | CustomStreamWrapper: - return await litellm.acompletion(**completion_params) + async def _acompletion(completion_params: Mapping[str, object]) -> ModelResponse | CustomStreamWrapper: + acompletion_fn: Final[Callable[..., Coroutine[object, object, ModelResponse | CustomStreamWrapper]]] = vars( + litellm + )["acompletion"] + return await acompletion_fn(**completion_params) @staticmethod async def handle_non_streaming( request_id: str, - params: dict[str, Any], + params: dict[str, object], litellm_params: dict[str, Any], api_base: str | None = None, agent_extra_headers: dict[str, str] | None = None, @@ -296,8 +299,8 @@ class A2ACompletionBridgeHandler: # Convenience functions that delegate to the class methods async def handle_a2a_completion( request_id: str, - params: dict[str, Any], - litellm_params: dict[str, Any], + params: dict[str, object], + litellm_params: dict[str, object], api_base: str | None = None, agent_extra_headers: dict[str, str] | None = None, ) -> dict[str, object]: @@ -313,8 +316,8 @@ async def handle_a2a_completion( async def handle_a2a_completion_streaming( request_id: str, - params: dict[str, Any], - litellm_params: dict[str, Any], + params: dict[str, object], + litellm_params: dict[str, object], api_base: str | None = None, agent_extra_headers: dict[str, str] | None = None, ) -> AsyncIterator[dict[str, object]]: diff --git a/litellm/a2a_protocol/main.py b/litellm/a2a_protocol/main.py index 322393cd9c4..1c6ebf0b95c 100644 --- a/litellm/a2a_protocol/main.py +++ b/litellm/a2a_protocol/main.py @@ -12,7 +12,8 @@ Provides standalone functions with @client decorator for LiteLLM logging integra import asyncio import datetime import uuid -from collections.abc import AsyncIterator, Coroutine +from collections.abc import AsyncIterator, Coroutine, Mapping +from types import ModuleType from typing import TYPE_CHECKING, Any, Final, Optional, cast import litellm @@ -38,12 +39,15 @@ if TYPE_CHECKING: SendMessageResponse, SendStreamingMessageRequest, SendStreamingMessageResponse, + SendStreamingMessageSuccessResponse, Task, ) + from a2a.types.a2a_pb2 import SendMessageRequest as CoreSendMessageRequest + from a2a.types.a2a_pb2 import StreamResponse as CoreStreamResponse # Runtime imports — requires a2a-sdk>=1.1.0 A2A_SDK_AVAILABLE = False -_a2a_conversions: Any = None +_a2a_conversions: ModuleType | None = None try: from a2a.client import Client, ClientCallContext, ClientConfig, create_client @@ -128,7 +132,7 @@ _A2A_COST_PARAM_KEYS: Final = ("cost_per_query", "input_cost_per_token", "output def _set_litellm_params_on_logging_obj( kwargs: dict[str, Any], - litellm_params: dict[str, Any], + litellm_params: Mapping[str, object], ) -> None: """ Merge the agent's pricing params into model_call_details["litellm_params"] @@ -150,7 +154,7 @@ def _set_litellm_params_on_logging_obj( logging_obj.model_call_details["litellm_params"] = {**existing, **cost_params} -def _get_a2a_model_info(a2a_client: Any, kwargs: dict[str, Any]) -> str: +def _get_a2a_model_info(a2a_client: "A2AClientType", kwargs: dict[str, Any]) -> str: """ Extract agent info and set model/custom_llm_provider for cost tracking. @@ -179,7 +183,7 @@ def _get_a2a_model_info(a2a_client: Any, kwargs: dict[str, Any]) -> str: return agent_name -def _get_a2a_client_agent_card(a2a_client: Any) -> Optional["AgentCard"]: +def _get_a2a_client_agent_card(a2a_client: "A2AClientType") -> Optional["AgentCard"]: agent_card = cast(Optional["AgentCard"], getattr(a2a_client, "_litellm_agent_card", None)) if agent_card is not None: return agent_card @@ -191,9 +195,9 @@ def _get_a2a_client_agent_card(a2a_client: Any) -> Optional["AgentCard"]: async def _send_message_via_completion_bridge( request: "SendMessageRequest", - custom_llm_provider: str, + custom_llm_provider: object, api_base: str | None, - litellm_params: dict[str, Any], + litellm_params: dict[str, object], agent_extra_headers: dict[str, str] | None = None, ) -> LiteLLMSendMessageResponse: """ @@ -224,6 +228,20 @@ def _get_a2a_call_context(a2a_client: "A2AClientType") -> Optional["A2ACallConte return getattr(a2a_client, "_litellm_call_context", None) +def _to_core_send_message_request(request: "SendMessageRequest") -> "CoreSendMessageRequest": + from a2a.compat.v0_3 import conversions + + return conversions.to_core_send_message_request(request) + + +def _to_compat_stream_response( + event: "CoreStreamResponse", request_id: str | int +) -> "SendStreamingMessageSuccessResponse": + from a2a.compat.v0_3 import conversions + + return conversions.to_compat_stream_response(event, request_id=request_id) + + async def _send_message(a2a_client: "A2AClientType", request: "SendMessageRequest") -> "SendMessageResponse": """Send a non-streaming message via a2a-sdk 1.x and return JSON-RPC response.""" if _a2a_conversions is None: @@ -231,17 +249,14 @@ async def _send_message(a2a_client: "A2AClientType", request: "SendMessageReques "The 'a2a' package is required for A2A agent invocation. Install it with: pip install a2a-sdk" ) - pb_request: Final = _a2a_conversions.to_core_send_message_request(request) + pb_request: Final = _to_core_send_message_request(request) last_event = None async for event in a2a_client.send_message(pb_request, context=_get_a2a_call_context(a2a_client)): last_event = event if last_event is None: raise RuntimeError("A2A send_message failed: no response received from agent.") - stream_compat: Final = _a2a_conversions.to_compat_stream_response( - last_event, - request_id=request.id, - ) + stream_compat: Final = _to_compat_stream_response(last_event, request_id=request.id) result: Final = stream_compat.result if not isinstance(result, (Message, Task)): raise RuntimeError( @@ -306,12 +321,9 @@ async def _stream_messages( "The 'a2a' package is required for A2A agent invocation. Install it with: pip install a2a-sdk" ) - pb_request: Final = _a2a_conversions.to_core_send_message_request(request) + pb_request: Final[CoreSendMessageRequest] = _a2a_conversions.to_core_send_message_request(request) async for event in a2a_client.send_message(pb_request, context=_get_a2a_call_context(a2a_client)): - compat_chunk = _a2a_conversions.to_compat_stream_response( - event, - request_id=request.id, - ) + compat_chunk = _to_compat_stream_response(event, request_id=request.id) yield SendStreamingMessageResponse(root=compat_chunk) @@ -368,10 +380,10 @@ async def asend_message( a2a_client: Optional["A2AClientType"] = None, request: Optional["SendMessageRequest"] = None, api_base: str | None = None, - litellm_params: dict[str, Any] | None = None, + litellm_params: dict[str, object] | None = None, agent_id: str | None = None, agent_extra_headers: dict[str, str] | None = None, - **kwargs: Any, + **kwargs: object, ) -> LiteLLMSendMessageResponse: """ Async: Send a message to an A2A agent. @@ -485,7 +497,7 @@ async def asend_message( response: Final = LiteLLMSendMessageResponse.from_a2a_response(a2a_response, request_id=str(request.id)) # Calculate token usage from request and response - response_dict: Final = a2a_response.model_dump(mode="json", exclude_none=True) + response_dict: Final[dict[str, object]] = a2a_response.model_dump(mode="json", exclude_none=True) ( prompt_tokens, completion_tokens, @@ -516,7 +528,7 @@ def send_message( a2a_client: "A2AClientType", request: "SendMessageRequest", **kwargs: Any, -) -> LiteLLMSendMessageResponse | Coroutine[Any, Any, LiteLLMSendMessageResponse]: +) -> LiteLLMSendMessageResponse | Coroutine[object, object, LiteLLMSendMessageResponse]: """ Sync: Send a message to an A2A agent. @@ -545,9 +557,9 @@ def _build_streaming_logging_obj( request: "SendStreamingMessageRequest", agent_name: str, agent_id: str | None, - litellm_params: dict[str, Any] | None, - metadata: dict[str, Any] | None, - proxy_server_request: dict[str, Any] | None, + litellm_params: dict[str, object] | None, + metadata: dict[str, object] | None, + proxy_server_request: dict[str, object] | None, ) -> Logging: """Build logging object for streaming A2A requests.""" start_time: Final = datetime.datetime.now() @@ -588,10 +600,10 @@ async def asend_message_streaming( a2a_client: Optional["A2AClientType"] = None, request: Optional["SendStreamingMessageRequest"] = None, api_base: str | None = None, - litellm_params: dict[str, Any] | None = None, + litellm_params: dict[str, object] | None = None, agent_id: str | None = None, - metadata: dict[str, Any] | None = None, - proxy_server_request: dict[str, Any] | None = None, + metadata: dict[str, object] | None = None, + proxy_server_request: dict[str, object] | None = None, agent_extra_headers: dict[str, str] | None = None, **kwargs: object, ) -> AsyncIterator[Any]: diff --git a/litellm/google_genai/adapters/transformation.py b/litellm/google_genai/adapters/transformation.py index 4f127f476c3..e43e0dfd5f7 100644 --- a/litellm/google_genai/adapters/transformation.py +++ b/litellm/google_genai/adapters/transformation.py @@ -1,6 +1,8 @@ import json from collections.abc import AsyncIterator, Iterator -from typing import Any, Final, cast +from typing import Any, Final, TypedDict, cast + +from typing_extensions import ReadOnly from litellm import verbose_logger from litellm.litellm_core_utils.json_validation_rule import normalize_tool_schema @@ -28,6 +30,19 @@ from litellm.types.utils import ( ) +class _GenAITextPart(TypedDict, total=False): + text: ReadOnly[str] + + +class _GenAISystemInstruction(TypedDict, total=False): + parts: ReadOnly[list[_GenAITextPart]] + + +class _GenAIPart(TypedDict, total=False): + text: ReadOnly[str] + functionCall: ReadOnly[dict[str, object]] + + class GoogleGenAIStreamWrapper(AdapterCompletionStreamWrapper): """ Wrapper for streaming Google GenAI generate_content responses. @@ -36,9 +51,9 @@ class GoogleGenAIStreamWrapper(AdapterCompletionStreamWrapper): sent_first_chunk: bool = False # State tracking for accumulating partial tool calls - accumulated_tool_calls: dict[str, dict[str, Any]] + accumulated_tool_calls: dict[str, dict[str, str]] - def __init__(self, completion_stream: Any): + def __init__(self, completion_stream: object): self.sent_first_chunk = False self.accumulated_tool_calls = {} self._returned_response = False @@ -85,7 +100,7 @@ class GoogleGenAIStreamWrapper(AdapterCompletionStreamWrapper): # After the stream is exhausted, check for any remaining accumulated tool calls if self.accumulated_tool_calls: try: - parts: Final = [] + parts: Final[list[_GenAIPart]] = [] for ( tool_call_index, tool_call_data, @@ -94,7 +109,7 @@ class GoogleGenAIStreamWrapper(AdapterCompletionStreamWrapper): # For tool calls with no arguments, accumulated_args will be "", which is not valid JSON. # We default to an empty JSON object in this case. parsed_args = json.loads(tool_call_data["arguments"] or "{}") - function_call_part = { + function_call_part: _GenAIPart = { "functionCall": { "name": tool_call_data["name"] or "undefined_tool_name", "args": parsed_args, @@ -110,7 +125,7 @@ class GoogleGenAIStreamWrapper(AdapterCompletionStreamWrapper): tool_call_data["arguments"], ) if parts: - final_chunk: Final = { + final_chunk: Final[dict[str, object]] = { "candidates": [ { "content": {"parts": parts, "role": "model"}, @@ -273,9 +288,9 @@ class GoogleGenAIAdapter: def _add_generic_litellm_params_to_request( self, - completion_request_dict: dict[str, Any], + completion_request_dict: dict[str, object], litellm_params: GenericLiteLLMParams | None = None, - ) -> dict: + ) -> dict[str, object]: """Add generic litellm params to request. e.g add api_base, api_key, api_version, etc. Args: @@ -295,7 +310,7 @@ class GoogleGenAIAdapter: def translate_completion_output_params_streaming( self, - completion_stream: Any, + completion_stream: object, ) -> AsyncIterator[bytes] | None: """Transform streaming completion output to Google GenAI format""" google_genai_wrapper: Final = GoogleGenAIStreamWrapper(completion_stream=completion_stream) @@ -307,12 +322,12 @@ class GoogleGenAIAdapter: tools: list[dict[str, Any]], ) -> list[ChatCompletionToolParam]: """Transform Google GenAI tools to OpenAI tools format""" - openai_tools: Final[list[dict[str, Any]]] = [] + openai_tools: Final[list[dict[str, object]]] = [] for tool in tools: if "functionDeclarations" in tool: for func_decl in tool["functionDeclarations"]: - function_chunk: dict[str, Any] = { + function_chunk: dict[str, object] = { "name": func_decl.get("name", ""), } @@ -321,7 +336,7 @@ class GoogleGenAIAdapter: if "parametersJsonSchema" in func_decl: function_chunk["parameters"] = func_decl["parametersJsonSchema"] - openai_tool = {"type": "function", "function": function_chunk} + openai_tool: dict[str, object] = {"type": "function", "function": function_chunk} openai_tools.append(openai_tool) # normalize the tool schemas @@ -345,7 +360,7 @@ class GoogleGenAIAdapter: def _transform_contents_to_messages( self, contents: list[dict[str, Any]], - system_instruction: dict[str, Any] | None = None, + system_instruction: _GenAISystemInstruction | None = None, ) -> list[AllMessageValues]: """Transform Google GenAI contents to OpenAI messages format""" messages: Final[list[AllMessageValues]] = [] @@ -461,7 +476,7 @@ class GoogleGenAIAdapter: def translate_completion_to_generate_content( self, response: ModelResponse, - ) -> dict[str, Any]: + ) -> dict[str, object]: """ Transform litellm completion response to Google GenAI generate_content format @@ -490,7 +505,7 @@ class GoogleGenAIAdapter: parts = [{"text": message_content}] if message_content else [] # Create Google GenAI format response - generate_content_response: Final[dict[str, Any]] = { + generate_content_response: Final[dict[str, object]] = { "candidates": [ { "content": {"parts": parts, "role": "model"}, @@ -524,7 +539,7 @@ class GoogleGenAIAdapter: self, response: ModelResponse | ModelResponseStream, wrapper: GoogleGenAIStreamWrapper, - ) -> dict[str, Any] | None: + ) -> dict[str, object] | None: """ Transform streaming litellm completion chunk to Google GenAI generate_content format @@ -560,7 +575,7 @@ class GoogleGenAIAdapter: return None # Create Google GenAI streaming format response - streaming_chunk: Final[dict[str, Any]] = { + streaming_chunk: Final[dict[str, object]] = { "candidates": [ { "content": {"parts": parts, "role": "model"}, @@ -597,9 +612,9 @@ class GoogleGenAIAdapter: def _transform_openai_message_to_google_genai_parts( self, message: Any, - ) -> list[dict[str, Any]]: + ) -> list[_GenAIPart]: """Transform OpenAI message to Google GenAI parts format""" - parts: Final[list[dict[str, Any]]] = [] + parts: Final[list[_GenAIPart]] = [] # Add text content if present if hasattr(message, "content") and message.content: @@ -614,7 +629,7 @@ class GoogleGenAIAdapter: except json.JSONDecodeError: args = {} - function_call_part = { + function_call_part: _GenAIPart = { "functionCall": { "name": tool_call.function.name or "undefined_tool_name", "args": args, @@ -626,14 +641,14 @@ class GoogleGenAIAdapter: def _transform_openai_delta_to_google_genai_parts_with_accumulation( self, delta: Any, wrapper: GoogleGenAIStreamWrapper - ) -> list[dict[str, Any]]: + ) -> list[_GenAIPart]: """Transforms OpenAI delta to Google GenAI parts, accumulating streaming tool calls.""" # 1. Initialize wrapper state if it doesn't exist if not hasattr(wrapper, "accumulated_tool_calls"): wrapper.accumulated_tool_calls = {} - parts: Final[list[dict[str, Any]]] = [] + parts: Final[list[_GenAIPart]] = [] if hasattr(delta, "content") and delta.content: parts.append({"text": delta.content}) @@ -686,7 +701,7 @@ class GoogleGenAIAdapter: # The part will be created by a later chunk that brings the name. if accumulated_name: # If successful, create the part and clean up - function_call_part = {"functionCall": {"name": accumulated_name, "args": parsed_args}} + function_call_part: _GenAIPart = {"functionCall": {"name": accumulated_name, "args": parsed_args}} parts.append(function_call_part) # Remove the completed tool call from the accumulator diff --git a/litellm/integrations/rubrik.py b/litellm/integrations/rubrik.py index 97e831f5822..a474a11601d 100644 --- a/litellm/integrations/rubrik.py +++ b/litellm/integrations/rubrik.py @@ -6,12 +6,13 @@ import random import time import uuid from collections import Counter -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Optional +from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict import httpx +from typing_extensions import Never, ReadOnly from litellm._logging import verbose_logger from litellm.integrations.custom_batch_logger import CustomBatchLogger @@ -48,7 +49,20 @@ _WEBHOOK_PATH_PROMPT_MODERATION: Final = "/v1/before_prompt/openai/v1" _WEBHOOK_PATH_LOGGING_BATCH: Final = "/v1/litellm/batch" _MAX_QUEUE_SIZE: Final = 10_000 _DROP_WARNING_INTERVAL_SECONDS: Final = 60.0 -_EMPTY_MAPPING: Final[Mapping[str, Any]] = MappingProxyType({}) +_EMPTY_MAPPING: Final[Mapping[str, Never]] = MappingProxyType({}) + + +class _ServiceToolCall(TypedDict): + id: ReadOnly[str] + + +class _ServiceMessage(TypedDict, total=False): + content: ReadOnly[str] + tool_calls: ReadOnly[Sequence[_ServiceToolCall]] + + +class _ServiceChoice(TypedDict, total=False): + message: ReadOnly[_ServiceMessage] class _MalformedToolBlockingResponseError(Exception): @@ -143,7 +157,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): else {"Content-Type": "application/json"} ) - self._periodic_flush_task: asyncio.Task[Any] | None = self._start_periodic_flush_task() + self._periodic_flush_task: asyncio.Task[None] | None = self._start_periodic_flush_task() @classmethod def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]: @@ -191,7 +205,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): params={"timeout": httpx.Timeout(5.0, connect=2.0)}, ) - def _start_periodic_flush_task(self) -> asyncio.Task[Any] | None: + def _start_periodic_flush_task(self) -> asyncio.Task[None] | None: """Start the periodic flush task only when an event loop is already running.""" try: loop: Final = asyncio.get_running_loop() @@ -212,7 +226,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): Closing them here would close the shared connection pool for every other logger instance; let LiteLLM manage their lifecycle instead. """ - task: Final = getattr(self, "_periodic_flush_task", None) + task: Final[asyncio.Task[None] | None] = getattr(self, "_periodic_flush_task", None) if task is not None: task.cancel() @@ -253,7 +267,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): @staticmethod async def _guarded( - coro: Any, + coro: Awaitable[GenericGuardrailAPIInputs], inputs: GenericGuardrailAPIInputs, label: str, ) -> GenericGuardrailAPIInputs: @@ -400,7 +414,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): request_data["_rubrik_logging_obj"] = logging_obj @staticmethod - def _normalize_tool_calls(tool_calls: Any) -> tuple[ChatCompletionMessageToolCall, ...]: + def _normalize_tool_calls(tool_calls: Sequence[object]) -> tuple[ChatCompletionMessageToolCall, ...]: """Convert tool_calls from inputs to ChatCompletionMessageToolCall objects.""" return tuple(RubrikLogger._normalize_tool_call(tc) for tc in tool_calls) @@ -427,7 +441,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): raise TypeError(f"Cannot normalize tool_call of type {type(tc).__name__}: {tc!r}") @staticmethod - def _join_texts(texts: Any) -> str: + def _join_texts(texts: Sequence[str] | None) -> str: """Join response text segments into the single content string the webhook evaluates. Empty when there is no assistant text.""" if not texts: @@ -439,14 +453,14 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): tool_calls: Sequence[ChatCompletionMessageToolCall], content: str, request_id: str | None, - ) -> Mapping[str, Any]: + ) -> Mapping[str, object]: """Build an OpenAI ChatCompletion-format dict (assistant text + tool calls) for the after_completion webhook. ``content`` is sent so the webhook can moderate the response text; ``None`` when the assistant produced no text (tool-call-only response). """ - message: Final[dict[str, Any]] = { + message: Final[dict[str, object]] = { "role": "assistant", "content": content or None, } @@ -467,7 +481,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): } @staticmethod - def _flatten_messages_for_moderation(messages: Any) -> tuple[Mapping[str, Any], ...]: + def _flatten_messages_for_moderation(messages: Sequence[object] | None) -> tuple[Mapping[str, Any], ...]: """Collapse each message's content to a plain string for the webhook. litellm normalizes Anthropic ``/v1/messages`` requests to OpenAI shape, @@ -506,8 +520,8 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): @staticmethod def _build_prompt_moderation_payload( inputs: GenericGuardrailAPIInputs, - request_data: Mapping[str, Any], - ) -> Mapping[str, Any]: + request_data: Mapping[str, object], + ) -> Mapping[str, object]: """Build the bare OpenAI request the before_prompt webhook consumes. Unlike the after_completion envelope, this endpoint takes a raw OpenAI @@ -516,7 +530,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): ``/v1/messages`` requests too. Optional fields are sent only when present so the payload stays clean. """ - payload: Final[dict[str, Any]] = { + payload: Final[dict[str, object]] = { "model": inputs.get("model") or request_data.get("model") or "", "messages": RubrikLogger._flatten_messages_for_moderation(inputs.get("structured_messages")), } @@ -540,8 +554,8 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): @staticmethod def _extract_request_data( call_details: Mapping[str, Any], - request_data: Mapping[str, Any] | None, - ) -> Mapping[str, Any]: + request_data: Mapping[str, object] | None, + ) -> Mapping[str, object]: """Extract original request data from model_call_details for the response moderation service envelope. @@ -576,7 +590,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): } @staticmethod - def _sanitize_proxy_server_request(proxy_server_request: Any) -> Any: + def _sanitize_proxy_server_request(proxy_server_request: object) -> object: """Allowlist only routing fields (``url``, ``method``) when forwarding ``proxy_server_request`` to an external webhook, dropping inbound ``headers`` (Authorization, Cookie, x-api-key, ...) and the raw @@ -586,17 +600,18 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): return {key: proxy_server_request[key] for key in ("url", "method") if key in proxy_server_request} @staticmethod - def _resolve_model(request_data: Mapping[str, Any], call_details: Mapping[str, Any]) -> str: + def _resolve_model(request_data: Mapping[str, object], call_details: Mapping[str, str]) -> str: """Get the model name for the ModifyResponseException.""" response: Final = request_data.get("response") if response and hasattr(response, "model"): - return response.model or "unknown" + response_model: Final[str | None] = getattr(response, "model", None) + return response_model or "unknown" return call_details.get("model", "unknown") # -- Logging hooks --------------------------------------------------------- @staticmethod - def _correlation_id(call_details: Mapping[str, Any], request_data: Mapping[str, Any] | None = None) -> str | None: + def _correlation_id(call_details: Mapping[str, str], request_data: Mapping[str, str] | None = None) -> str | None: """The id that joins a blocked request's two S3 logs by filename: the moderation (``_blocking``) log and the failure (response) log. @@ -610,7 +625,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): return call_details.get("litellm_call_id") or (request_data or _EMPTY_MAPPING).get("litellm_call_id") @classmethod - def _apply_correlation_id(cls, payload: dict[str, Any], source: Mapping[str, Any]) -> None: + def _apply_correlation_id(cls, payload: dict[str, object], source: Mapping[str, str]) -> None: """Pin ``payload["id"]`` to ``litellm_call_id`` in place so this log shares its S3 filename id with the moderation (``_blocking``) and failure logs for the same request -- for every provider. @@ -630,7 +645,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): payload["id"] = correlated @staticmethod - def _prepend_system_prompt(payload: dict[str, Any], source: Mapping[str, Any]) -> None: + def _prepend_system_prompt(payload: dict[str, object], source: Mapping[str, object]) -> None: """Prepend ``source["system"]`` onto ``payload["messages"]``. Builds a NEW messages list rather than mutating ``payload["messages"]`` @@ -658,7 +673,9 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): exc_info=True, ) - async def _prepare_log_payload(self, kwargs: Mapping[str, Any], event_type: str) -> StandardLoggingPayload | None: + async def _prepare_log_payload( + self, kwargs: Mapping[str, object], event_type: str + ) -> StandardLoggingPayload | None: """Shared logic for success logging (sampled).""" if random.random() > self.sampling_rate: verbose_logger.debug("Skipping Rubrik %s logging (sampling_rate=%s)", event_type, self.sampling_rate) @@ -697,7 +714,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): self._dropped_since_warning = 0 self._last_drop_warning_time = now - async def _enqueue_log_event(self, kwargs: Mapping[str, Any], event_type: str): + async def _enqueue_log_event(self, kwargs: Mapping[str, object], event_type: str): try: payload: Final = await self._prepare_log_payload(kwargs, event_type) if payload is None: @@ -862,7 +879,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): base: Final = call_details.get("standard_logging_object") if base is not None: - payload: dict = safe_deep_copy(base) + payload: dict[str, object] = safe_deep_copy(base) else: verbose_logger.debug( "Rubrik: standard_logging_object not yet on model_call_details " @@ -908,7 +925,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): cls, call_details: Mapping[str, Any], user_api_key_dict: "UserAPIKeyAuth", - ) -> dict[str, Any]: + ) -> dict[str, object]: # Convert datetime to a Unix float so json.dumps can serialize it. # httpx's json= parameter uses stdlib json.dumps with no custom encoder. _raw_start: Final = call_details.get("start_time") @@ -996,7 +1013,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): # -- Webhook services ------------------------------------------------------ - async def _post_json(self, endpoint: str, payload: Mapping[str, Any], service_name: str) -> Mapping[str, Any]: + async def _post_json(self, endpoint: str, payload: Mapping[str, object], service_name: str) -> Mapping[str, Any]: """POST ``payload`` to a Rubrik webhook and return its dict response. Raises: @@ -1010,7 +1027,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): headers=self._headers, ) http_response.raise_for_status() - result: Final = http_response.json() + result: Final[object] = http_response.json() if not isinstance(result, dict): raise TypeError( f"{service_name} returned non-dict JSON " @@ -1021,8 +1038,8 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): async def _post_to_response_moderation_endpoint( self, - response_data: Mapping[str, Any], - request_data: Mapping[str, Any], + response_data: Mapping[str, object], + request_data: Mapping[str, object], ) -> Mapping[str, Any]: """Post the ``{request, response}`` envelope to the after_completion webhook and return its (possibly rewritten) response. @@ -1039,7 +1056,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): "Response moderation service", ) - async def _post_to_prompt_moderation_endpoint(self, payload: Mapping[str, Any]) -> Mapping[str, Any]: + async def _post_to_prompt_moderation_endpoint(self, payload: Mapping[str, object]) -> Mapping[str, Any]: """Post a bare OpenAI request to the before_prompt webhook. Returns ``{}`` (passthrough) or a synthetic chat.completion (block). @@ -1054,7 +1071,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): chat.completion whose ``choices[0].message.content`` is the refusal explanation. """ - choices: Final = service_response.get("choices") + choices: Final[Sequence[_ServiceChoice] | None] = service_response.get("choices") if not choices: return None message: Final = choices[0].get("message") or _EMPTY_MAPPING @@ -1086,7 +1103,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): Expects service_response in OpenAI chat completion format: {"choices": [{"message": {"tool_calls": [...], "content": "..."}}]} """ - choices: Final = service_response.get("choices") or () + choices: Final[Sequence[_ServiceChoice]] = service_response.get("choices") or () if not choices: raise _MalformedToolBlockingResponseError("Response moderation service returned empty response") diff --git a/litellm/interactions/litellm_responses_transformation/handler.py b/litellm/interactions/litellm_responses_transformation/handler.py index f826c980ac4..8fee0fcd1b5 100644 --- a/litellm/interactions/litellm_responses_transformation/handler.py +++ b/litellm/interactions/litellm_responses_transformation/handler.py @@ -2,8 +2,8 @@ Handler for transforming interactions API requests to litellm.responses requests. """ -from collections.abc import AsyncIterator, Coroutine, Iterator -from typing import Any, Final, cast +from collections.abc import AsyncIterator, Callable, Coroutine, Iterator +from typing import Any, Final import litellm from litellm.interactions.litellm_responses_transformation.streaming_iterator import ( @@ -37,7 +37,7 @@ class LiteLLMResponsesInteractionsHandler: ) -> ( InteractionsAPIResponse | Iterator[InteractionsAPIStreamingResponse] - | Coroutine[Any, Any, InteractionsAPIResponse | AsyncIterator[InteractionsAPIStreamingResponse]] + | Coroutine[object, object, InteractionsAPIResponse | AsyncIterator[InteractionsAPIStreamingResponse]] ): """ Handle Interactions API request by calling litellm.responses(). @@ -55,13 +55,15 @@ class LiteLLMResponsesInteractionsHandler: InteractionsAPIResponse or streaming iterator """ # Transform interactions request to responses request - responses_request = LiteLLMResponsesInteractionsConfig.transform_interactions_request_to_responses_request( - model=model, - input=input, - optional_params=optional_params, - custom_llm_provider=custom_llm_provider, - stream=stream, - **kwargs, + responses_request: Final = ( + LiteLLMResponsesInteractionsConfig.transform_interactions_request_to_responses_request( + model=model, + input=input, + optional_params=optional_params, + custom_llm_provider=custom_llm_provider, + stream=stream, + **kwargs, + ) ) if _is_async: @@ -76,7 +78,10 @@ class LiteLLMResponsesInteractionsHandler: # Call litellm.responses() # Note: litellm.responses() returns Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator] # but the type checker may see it as a coroutine in some contexts - responses_response: Final = litellm.responses( + responses_fn: Final[Callable[..., ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]] = vars(litellm)[ + "responses" + ] + responses_response: Final = responses_fn( **responses_request, ) @@ -92,8 +97,7 @@ class LiteLLMResponsesInteractionsHandler: ) # At this point, responses_response must be ResponsesAPIResponse (not streaming) - # Cast to satisfy type checker since we've already checked it's not a streaming iterator - responses_api_response: Final = cast(ResponsesAPIResponse, responses_response) + responses_api_response: Final = responses_response # Transform responses response to interactions response return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response( @@ -112,7 +116,10 @@ class LiteLLMResponsesInteractionsHandler: """Async handler for interactions API requests.""" # Call litellm.aresponses() # Note: litellm.aresponses() returns Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator] - responses_response: Final = await litellm.aresponses( + aresponses_fn: Final[ + Callable[..., Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]] + ] = vars(litellm)["aresponses"] + responses_response: Final = await aresponses_fn( **responses_request, ) @@ -128,8 +135,7 @@ class LiteLLMResponsesInteractionsHandler: ) # At this point, responses_response must be ResponsesAPIResponse (not streaming) - # Cast to satisfy type checker since we've already checked it's not a streaming iterator - responses_api_response: Final = cast(ResponsesAPIResponse, responses_response) + responses_api_response: Final = responses_response # Transform responses response to interactions response return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response( diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py index dbeac453791..34d981e2e55 100644 --- a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py @@ -13,8 +13,10 @@ Mirrors Anthropic's native ``compact_20260112`` for non-Anthropic providers: """ import re -from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Any, Final, Literal, NotRequired, Optional, TypedDict, Union, cast + +from typing_extensions import ReadOnly import litellm from litellm._logging import verbose_logger @@ -29,9 +31,8 @@ if TYPE_CHECKING: from litellm.proxy._types import UserAPIKeyAuth from litellm.router import Router from litellm.types.llms.anthropic import ( + AllAnthropicPassThroughMessageValues, AllAnthropicToolsValues, - AnthopicMessagesAssistantMessageParam, - AnthropicMessagesUserMessageParam, ) from litellm.types.llms.openai import ChatCompletionToolParam from litellm.types.utils import ModelResponse @@ -157,11 +158,11 @@ async def _check_summary_model_access( return True key_models: Final = list(getattr(user_api_key_auth, "models", None) or []) - team_id: Final = getattr(user_api_key_auth, "team_id", None) + team_id: Final = user_api_key_auth.team_id team_model_aliases: Final = getattr(user_api_key_auth, "team_model_aliases", None) team_models: Final = list(getattr(user_api_key_auth, "team_models", None) or []) - user_id: Final = getattr(user_api_key_auth, "user_id", None) - project_id: Final = getattr(user_api_key_auth, "project_id", None) + user_id: Final = user_api_key_auth.user_id + project_id: Final = user_api_key_auth.project_id checks: Final[tuple[tuple[Literal["key", "team"], list[str]], ...]] = ( ("key", key_models), @@ -347,7 +348,7 @@ async def _check_summary_model_budget( return False end_user_model_max_budget: Final = getattr(user_api_key_auth, "end_user_model_max_budget", None) - end_user_id: Final = getattr(user_api_key_auth, "end_user_id", None) + end_user_id: Final = user_api_key_auth.end_user_id if isinstance(end_user_model_max_budget, dict) and end_user_model_max_budget and end_user_id is not None: try: await model_max_budget_limiter.is_end_user_within_model_budget( @@ -534,7 +535,7 @@ def _augment_system_with_summary( return [{"type": "text", "text": prefix.rstrip()}, *system] -def _resolve_trigger_tokens(edit_spec: dict[str, object]) -> tuple[int, list[str]]: +def _resolve_trigger_tokens(edit_spec: Mapping[str, object]) -> tuple[int, list[str]]: """Validate and resolve ``trigger.value``. Raises ``AnthropicContextManagementError`` if the explicitly-supplied value @@ -568,7 +569,7 @@ def _resolve_trigger_tokens(edit_spec: dict[str, object]) -> tuple[int, list[str return value, warnings -def _build_summary_prompt(edit_spec: dict[str, object], tools: list[dict[str, object]] | None) -> str: +def _build_summary_prompt(edit_spec: Mapping[str, object], tools: Sequence[Mapping[str, object]] | None) -> str: custom: Final = edit_spec.get("instructions") if isinstance(custom, str) and custom.strip(): return custom @@ -623,7 +624,7 @@ def _count_effective_tokens( try: openai_shape = adapter.translate_anthropic_messages_to_openai( messages=cast( - "list[AnthropicMessagesUserMessageParam | AnthopicMessagesAssistantMessageParam]", + "list[AllAnthropicPassThroughMessageValues]", messages_without_compaction, ) ) @@ -736,7 +737,7 @@ def _extract_summary_text(raw: str | None) -> str | None: def _system_to_openai_message( system: str | list[dict[str, Any]] | None, -) -> dict[str, Any] | None: +) -> dict[str, object] | None: """Translate Anthropic-shaped ``system`` to an OpenAI system message. Accepts a bare string or a list of Anthropic content blocks; returns @@ -773,7 +774,7 @@ def _build_summary_messages( try: openai_messages = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( messages=cast( - "list[AnthropicMessagesUserMessageParam | AnthopicMessagesAssistantMessageParam]", + "list[AllAnthropicPassThroughMessageValues]", stripped, ) ) @@ -809,7 +810,7 @@ def _is_user_message(msg: object) -> bool: return isinstance(msg, dict) and msg.get("role") == "user" -def _append_text_to_content(content: Any, extra_text: str) -> Any: +def _append_text_to_content(content: object, extra_text: str) -> object: """Append ``extra_text`` to an OpenAI-shape message ``content`` field. Handles the two common shapes: ``str`` and ``list`` of content parts. @@ -820,10 +821,29 @@ def _append_text_to_content(content: Any, extra_text: str) -> Any: if isinstance(content, str): return f"{content}\n\n{extra_text}" if isinstance(content, list): - return [*content, {"type": "text", "text": extra_text}] + appended: Final[list[object]] = [*content, {"type": "text", "text": extra_text}] + return appended return [content, {"type": "text", "text": extra_text}] +class _SummaryCallUserKwarg(TypedDict, total=False): + user: ReadOnly[object] + + +class _SummaryCallRegionKwarg(TypedDict, total=False): + allowed_model_region: ReadOnly[str] + + +class _SummaryCallKwargs(TypedDict): + model: ReadOnly[str] + messages: ReadOnly[list[dict[str, object]]] + max_tokens: ReadOnly[int] + timeout: ReadOnly[float] + litellm_metadata: ReadOnly[Mapping[str, object]] + user: NotRequired[ReadOnly[object]] + allowed_model_region: NotRequired[ReadOnly[str]] + + async def _call_summary_model( *, summary_model: str, @@ -860,22 +880,24 @@ async def _call_summary_model( # the parent ``/v1/messages`` request. On timeout the caller catches the # exception and surfaces ``applied_edits[0].error = "summary_call_failed"``, # forwarding the request without compaction rather than hanging. - call_kwargs: Final[dict[str, Any]] = { - "model": summary_model, - "messages": summary_messages, - "max_tokens": max_tokens, - "timeout": COMPACT_SUMMARY_TIMEOUT_SECONDS, - "litellm_metadata": metadata, - } # The end-user id must also travel as the top-level ``user`` kwarg: legacy # limiter hooks and prometheus end-user tracking read it from there rather # than from ``litellm_metadata``, so without it the summary tokens would not # debit the caller's end-user counters. end_user_id: Final = metadata.get("user_api_key_end_user_id") - if end_user_id: - call_kwargs["user"] = end_user_id - if allowed_model_region is not None: - call_kwargs["allowed_model_region"] = allowed_model_region + call_kwargs: Final[_SummaryCallKwargs] = { + "model": summary_model, + "messages": summary_messages, + "max_tokens": max_tokens, + "timeout": COMPACT_SUMMARY_TIMEOUT_SECONDS, + "litellm_metadata": metadata, + **(_SummaryCallUserKwarg(user=end_user_id) if end_user_id else _SummaryCallUserKwarg()), + **( + _SummaryCallRegionKwarg(allowed_model_region=allowed_model_region) + if allowed_model_region is not None + else _SummaryCallRegionKwarg() + ), + } if llm_router is not None and hasattr(llm_router, "acompletion"): return await llm_router.acompletion(**call_kwargs) return await litellm.acompletion(**call_kwargs) diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 1ce83e226e7..b77ba2f9460 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -2,11 +2,12 @@ import asyncio import hashlib import json import os -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import Any, Final, Literal, NamedTuple, cast import httpx from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI +from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -23,6 +24,22 @@ from litellm.utils import _add_path_to_api_base azure_ad_cache: Final = DualCache() +class _AzureAdTokenJson(TypedDict, total=False): + access_token: ReadOnly[str] + expires_in: ReadOnly[int] + + +class _AzureV1ClientParams(TypedDict, total=False, extra_items=object): + base_url: ReadOnly[str] + + +class _AzureGatewayClientParams(TypedDict, total=False, extra_items=object): + api_version: ReadOnly[str] + base_url: ReadOnly[str] + max_retries: ReadOnly[int] + timeout: ReadOnly[float | httpx.Timeout] + + class AzureOpenAIError(BaseLLMException): def __init__( self, @@ -220,7 +237,7 @@ def get_azure_ad_token_from_oidc( message=req_token.text, ) - azure_ad_token_json: Final = req_token.json() + azure_ad_token_json: Final[_AzureAdTokenJson] = req_token.json() azure_ad_token_access_token = azure_ad_token_json.get("access_token", None) azure_ad_token_expires_in: Final = azure_ad_token_json.get("expires_in", None) @@ -486,7 +503,7 @@ class BaseAzureLLM(BaseOpenAILLM): v1_api_key = _async_v1_api_key - v1_params: Final[dict[str, Any]] = { + v1_params: Final[_AzureV1ClientParams] = { "api_key": v1_api_key, "base_url": f"{api_base}/openai/v1/", } @@ -643,7 +660,7 @@ class BaseAzureLLM(BaseOpenAILLM): api_base += "/" api_base += f"{model}" - azure_client_params: Final[dict[str, Any]] = { + azure_client_params: Final[_AzureGatewayClientParams] = { "api_version": api_version, "base_url": f"{api_base}", "http_client": litellm.client_session, @@ -702,7 +719,7 @@ class BaseAzureLLM(BaseOpenAILLM): @staticmethod def _get_base_azure_url( api_base: str | None, - litellm_params: GenericLiteLLMParams | dict[str, Any] | None, + litellm_params: GenericLiteLLMParams | Mapping[str, object] | None, route: Literal["/openai/responses", "/openai/vector_stores"] | str, default_api_version: str | Literal["latest", "preview"] | None = None, ) -> str: @@ -757,7 +774,9 @@ class BaseAzureLLM(BaseOpenAILLM): return False return api_version in {"preview", "latest", "v1"} - def _resolve_env_var(self, litellm_params: dict[str, Any], param_key: str, env_var_key: str) -> str | None: + def _resolve_env_var( + self, litellm_params: Mapping[str, str | None], param_key: str, env_var_key: str + ) -> str | None: """Resolve the environment variable for a given parameter key. The logic here is different from `params.get(key, os.getenv(env_var))` because diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 4ff7323c33f..b50a9ae04d1 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -2,17 +2,18 @@ import base64 import json import os import time -from collections.abc import Iterable, Mapping, MutableMapping +from collections.abc import Iterable, Mapping, MutableMapping, Sequence from functools import cache from itertools import chain from types import MappingProxyType -from typing import Any, Final +from typing import Any, Final, TypeAlias, TypedDict from urllib.parse import unquote import httpx from httpx import Headers, Response from openai.types.file_deleted import FileDeleted from pydantic import BaseModel, ConfigDict, TypeAdapter +from typing_extensions import ReadOnly from litellm._logging import verbose_logger from litellm._uuid import uuid @@ -63,10 +64,39 @@ from ..common_utils import BedrockError, merge_bedrock_aws_request_params, resol S3_SIGNED_GET_HEADERS_PARAM: Final = "_s3_signed_get_headers" -def _frozen_mapping(items: Iterable[tuple[str, Any]]) -> Mapping[str, Any]: +def _frozen_mapping(items: Iterable[tuple[str, object]]) -> Mapping[str, object]: return MappingProxyType(dict(items)) +_EmbeddingBatchInput: TypeAlias = ( + str | int | float | Sequence[str] | Sequence[int] | Sequence[Sequence[int]] | Mapping[str, object] +) + + +class _OpenAIBatchRecordBody(TypedDict, total=False): + model: ReadOnly[str] + prompt: ReadOnly[str | Sequence[str] | Sequence[int] | Sequence[Sequence[int]]] + input: ReadOnly[_EmbeddingBatchInput] + metadata: ReadOnly[Mapping[str, object]] + + +class _OpenAIBatchRecord(TypedDict, total=False): + custom_id: ReadOnly[str] + url: ReadOnly[str] + body: ReadOnly[_OpenAIBatchRecordBody] + + +class _BedrockBatchRecord(TypedDict): + recordId: ReadOnly[str] + modelInput: ReadOnly[Mapping[str, object]] + + +class _S3UploadResponse(TypedDict, total=False): + Key: ReadOnly[str] + Bucket: ReadOnly[str] + ContentLength: ReadOnly[int] + + # JSONL batch records are untyped json, so the `/v1/responses` fields are # validated into their concrete Responses API types before being handed to the # Responses-to-Chat bridge. Both adapters drop keys the Responses API doesn't @@ -231,7 +261,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): def _get_s3_object_name_from_batch_jsonl( self, - openai_jsonl_content: list[dict[str, Any]], + openai_jsonl_content: Sequence[_OpenAIBatchRecord], ) -> str: """ Gets a unique S3 object name for the Bedrock batch processing job @@ -341,7 +371,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): OPENAI_RESPONSES_URL = "/v1/responses" @staticmethod - def _classify_batch_record(openai_jsonl_record: Mapping[str, Any]) -> BedrockBatchRecordKind: + def _classify_batch_record(openai_jsonl_record: _OpenAIBatchRecord) -> BedrockBatchRecordKind: """ Decide which OpenAI endpoint shape an OpenAI batch JSONL line carries. @@ -484,7 +514,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): return value if isinstance(value, str) and value else None @staticmethod - def _coerce_embedding_input_to_string(raw_input: Any, model: str = "") -> str: + def _coerce_embedding_input_to_string(raw_input: _EmbeddingBatchInput | None, model: str = "") -> str: """ Normalize an OpenAI /v1/embeddings `input` field into the single string that Bedrock Titan v2 InvokeModel expects in `inputText`. @@ -541,8 +571,8 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): def _map_openai_embedding_to_bedrock_params( self, - openai_request_body: dict[str, Any], - ) -> dict[str, Any]: + openai_request_body: _OpenAIBatchRecordBody, + ) -> dict[str, object]: """ Transform an OpenAI /v1/embeddings request body into the Bedrock InvokeModel `modelInput` for embedding models that AWS @@ -588,7 +618,9 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): return dict(titan_config._transform_request(input=input_text, inference_params=inference_params)) @staticmethod - def _transform_text_completion_body_to_chat_body(openai_request_body: Mapping[str, Any]) -> Mapping[str, Any]: + def _transform_text_completion_body_to_chat_body( + openai_request_body: _OpenAIBatchRecordBody, + ) -> Mapping[str, object]: """ Rewrite an OpenAI `/v1/completions` batch body as a Chat Completions body. @@ -610,7 +642,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): ) @staticmethod - def _transform_responses_body_to_chat_body(openai_request_body: Mapping[str, Any]) -> Mapping[str, Any]: + def _transform_responses_body_to_chat_body(openai_request_body: _OpenAIBatchRecordBody) -> Mapping[str, object]: """ Rewrite an OpenAI `/v1/responses` batch body as a Chat Completions body. @@ -631,23 +663,25 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): "Batch record for /v1/responses is missing required `input` field: " f"model={openai_request_body.get('model', '')}" ) - chat_body: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( - model=openai_request_body.get("model", ""), - input=_responses_input_adapter().validate_python(responses_input), - responses_api_request=_responses_request_adapter().validate_python( - _frozen_mapping( - (key, value) for key, value in openai_request_body.items() if key not in ("model", "input") - ) - ), - metadata=openai_request_body.get("metadata"), + chat_body: Final[Mapping[str, object]] = ( + LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model=openai_request_body.get("model", ""), + input=_responses_input_adapter().validate_python(responses_input), + responses_api_request=_responses_request_adapter().validate_python( + _frozen_mapping( + (key, value) for key, value in openai_request_body.items() if key not in ("model", "input") + ) + ), + metadata=openai_request_body.get("metadata"), + ) ) return _frozen_mapping((key, value) for key, value in chat_body.items() if key != "tools" or value) @staticmethod def _transform_batch_body_to_chat_body( - openai_request_body: Mapping[str, Any], + openai_request_body: _OpenAIBatchRecordBody, record_kind: BedrockBatchRecordKind, - ) -> Mapping[str, Any]: + ) -> Mapping[str, object]: """ Normalize a non-embedding batch body to the Chat Completions shape the per-provider Bedrock transformations expect. @@ -666,7 +700,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): self, openai_request_body: Mapping[str, Any], provider: str | None = None, - ) -> dict[str, Any]: + ) -> dict[str, object]: """ Transform OpenAI request body to Bedrock-compatible modelInput parameters using existing transformation logic. @@ -677,7 +711,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): """ from litellm.types.utils import LlmProviders - _model: Final = openai_request_body.get("model", "") + _model: Final[str] = openai_request_body.get("model", "") messages: Final = openai_request_body.get("messages", []) optional_params: Final = {k: v for k, v in openai_request_body.items() if k not in ["model", "messages"]} @@ -733,8 +767,8 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): } def _transform_openai_jsonl_content_to_bedrock_jsonl_content( - self, openai_jsonl_content: list[dict[str, Any]] - ) -> list[dict[str, Any]]: + self, openai_jsonl_content: Sequence[_OpenAIBatchRecord] + ) -> list[_BedrockBatchRecord]: """ Transforms OpenAI JSONL content to Bedrock batch format @@ -1026,7 +1060,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): response_headers: Final = raw_response.headers # Extract S3 object information from the response # S3 PUT object returns ETag and other metadata in headers - content_length: Final = response_headers.get("Content-Length", "0") + content_length: Final[str] = response_headers.get("Content-Length", "0") # Use the actual upload URL that was used for the S3 upload upload_url: Final = litellm_params.get("upload_url") @@ -1224,7 +1258,9 @@ class BedrockJsonlFilesTransformation: object_name: Final = self._get_s3_object_name(openai_jsonl_content=openai_jsonl_content) return bedrock_jsonl_string, object_name - def _transform_openai_jsonl_content_to_bedrock_jsonl_content(self, openai_jsonl_content: list[dict[str, Any]]): + def _transform_openai_jsonl_content_to_bedrock_jsonl_content( + self, openai_jsonl_content: Sequence[_OpenAIBatchRecord] + ): """ Delegate to the main BedrockFilesConfig transformation method """ @@ -1233,7 +1269,7 @@ class BedrockJsonlFilesTransformation: def _get_s3_object_name( self, - openai_jsonl_content: list[dict[str, Any]], + openai_jsonl_content: Sequence[_OpenAIBatchRecord], ) -> str: """ Gets a unique S3 object name for the Bedrock batch processing job @@ -1285,7 +1321,7 @@ class BedrockJsonlFilesTransformation: return content def transform_s3_bucket_response_to_openai_file_object( - self, create_file_data: CreateFileRequest, s3_upload_response: dict[str, Any] + self, create_file_data: CreateFileRequest, s3_upload_response: _S3UploadResponse ) -> OpenAIFileObject: """ Transforms S3 Bucket upload file response to OpenAI FileObject diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index 2e0ae30a192..b8e57fa7cc0 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -1,8 +1,10 @@ +from collections.abc import Mapping, Sequence from datetime import datetime -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, Literal import httpx from httpx._types import RequestFiles +from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import RUNWAYML_DEFAULT_API_VERSION @@ -31,6 +33,29 @@ else: LiteLLMLoggingObj = Any +class _RunwayTaskResponse(TypedDict, total=False): + id: ReadOnly[str] + status: ReadOnly[str] + createdAt: ReadOnly[str] + completedAt: ReadOnly[str] + output: ReadOnly[Sequence[str] | str] + failureCode: ReadOnly[str] + failure: ReadOnly[str] + progress: ReadOnly[int] + + +class _VideoObjectData(TypedDict, extra_items=object): + id: ReadOnly[str] + object: ReadOnly[Literal["video"]] + status: ReadOnly[str] + created_at: ReadOnly[int] + + +def _parse_runway_task_response(raw_response: httpx.Response) -> _RunwayTaskResponse: + response_data: Final[_RunwayTaskResponse] = raw_response.json() + return response_data + + class RunwayMLVideoConfig(BaseVideoConfig): """ Configuration class for RunwayML video generation. @@ -78,7 +103,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): - size -> ratio (convert "WIDTHxHEIGHT" to "WIDTH:HEIGHT") - seconds -> duration (convert to integer) """ - mapped_params: Final[dict[str, Any]] = {} + mapped_params: Final[dict[str, object]] = {} # Handle input_reference parameter - map to promptImage if "input_reference" in video_create_optional_params: @@ -180,7 +205,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): } """ # Build the request data - request_data: Final[dict[str, Any]] = { + request_data: Final[dict[str, object]] = { "model": model, "promptText": prompt, } @@ -189,7 +214,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): request_data.update(video_create_optional_request_params) # RunwayML uses JSON body, no files multipart - files_list: Final[list[tuple[str, Any]]] = [] + files_list: Final[RequestFiles] = [] # Append the specific endpoint for video generation full_api_base: Final = f"{api_base}/image_to_video" @@ -216,10 +241,10 @@ class RunwayMLVideoConfig(BaseVideoConfig): We map this to OpenAI VideoObject format. """ - response_data: Final = raw_response.json() + response_data: Final = _parse_runway_task_response(raw_response) # Map RunwayML task response to VideoObject format - video_data: Final[dict[str, Any]] = { + video_data: Final[_VideoObjectData] = { "id": response_data.get("id", ""), "object": "video", "status": self._map_runway_status(response_data.get("status", "pending")), @@ -326,7 +351,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): # Get task status to retrieve video URL url: Final = f"{api_base}/tasks/{encoded_video_id}" - params: Final[dict[str, Any]] = {} + params: Final[dict[str, str]] = {} return url, params @@ -421,7 +446,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - extra_body: dict[str, Any] | None = None, + extra_body: Mapping[str, object] | None = None, ) -> tuple[str, dict]: """ Transform the video remix request for RunwayML API. @@ -448,7 +473,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): after: str | None = None, limit: int | None = None, order: str | None = None, - extra_query: dict[str, Any] | None = None, + extra_query: Mapping[str, object] | None = None, ) -> tuple[str, dict]: """ Transform the video list request for RunwayML API. @@ -484,7 +509,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): # Construct the URL for task cancellation url: Final = f"{api_base}/tasks/{encoded_video_id}/cancel" - data: Final[dict[str, Any]] = {} + data: Final[dict[str, str]] = {} return url, data @@ -494,7 +519,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): logging_obj: LiteLLMLoggingObj, ) -> VideoObject: """Transform the RunwayML video delete/cancel response.""" - response_data: Final = raw_response.json() + response_data: Final = _parse_runway_task_response(raw_response) video_obj: Final = VideoObject( id=response_data.get("id", ""), @@ -524,7 +549,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): url: Final = f"{api_base}/tasks/{encoded_video_id}" # Empty dict for GET request (no body) - data: Final[dict[str, Any]] = {} + data: Final[dict[str, str]] = {} return url, data @@ -537,10 +562,10 @@ class RunwayMLVideoConfig(BaseVideoConfig): """ Transform the RunwayML video status retrieve response. """ - response_data: Final = raw_response.json() + response_data: Final = _parse_runway_task_response(raw_response) # Map RunwayML task response to VideoObject format - video_data: Final[dict[str, Any]] = { + video_data: Final[_VideoObjectData] = { "id": response_data.get("id", ""), "object": "video", "status": self._map_runway_status(response_data.get("status", "pending")), @@ -572,7 +597,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): return video_obj - def transform_video_create_character_request(self, name, video, api_base, litellm_params, headers): + def transform_video_create_character_request(self, name, video: object, api_base, litellm_params, headers): raise NotImplementedError("video create character is not supported for RunwayML") def transform_video_create_character_response(self, raw_response, logging_obj): diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 3538fc5b1a7..3db94211032 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -5,12 +5,13 @@ import json import os import re import time -from collections.abc import Callable, Iterable, Iterator -from typing import Any, Final +from collections.abc import Callable, Iterable, Iterator, Mapping +from typing import Any, Final, TypedDict import httpx from httpx import Headers, Response from openai.types.file_deleted import FileDeleted +from typing_extensions import ReadOnly import litellm from litellm._uuid import uuid @@ -50,6 +51,7 @@ from litellm.types.llms.openai import ( HttpxBinaryResponseContent, OpenAICreateFileRequestOptionalParams, OpenAIFileObject, + OpenAIFilesPurpose, PathLike, ) from litellm.types.llms.vertex_ai import GcsBucketResponse @@ -62,6 +64,46 @@ _GCP_LABEL_VALUE_MAX_LEN: Final = 63 _CUSTOM_ID_RAW_LABEL_PREFIX: Final = "b32_" +class _GcsObjectMetadataJson(TypedDict, total=False): + purpose: ReadOnly[OpenAIFilesPurpose] + + +class _GcsObjectJson(TypedDict, total=False): + id: ReadOnly[str] + name: ReadOnly[str] + size: ReadOnly[str] + timeCreated: ReadOnly[str] + metadata: ReadOnly[_GcsObjectMetadataJson] + + +class _VertexBatchRowRequest(TypedDict, total=False): + labels: ReadOnly[Mapping[str, object]] + + +class _VertexBatchRow(TypedDict, total=False): + request: ReadOnly[_VertexBatchRowRequest] + status: ReadOnly[str] + processed_time: ReadOnly[str] + + +class _OpenAIBatchOutputError(TypedDict): + code: ReadOnly[str] + message: ReadOnly[str] + + +class _OpenAIBatchOutputResponse(TypedDict): + status_code: ReadOnly[int] + request_id: ReadOnly[str] + body: ReadOnly[Mapping[str, object]] + + +class _OpenAIBatchOutputRow(TypedDict): + id: ReadOnly[str] + custom_id: ReadOnly[str] + response: ReadOnly[_OpenAIBatchOutputResponse | None] + error: ReadOnly[_OpenAIBatchOutputError | None] + + def _sanitize_gcp_label_value(value: str) -> str: """ Sanitize a string to meet GCP label value constraints. @@ -106,7 +148,7 @@ def _decode_gcp_label_value_chunks(values: list[str]) -> str | None: return None -def _set_litellm_batch_custom_id_labels(labels: dict[str, str], custom_id: Any) -> None: +def _set_litellm_batch_custom_id_labels(labels: dict[str, str], custom_id: object) -> None: """ Store OpenAI batch custom_id for Vertex batch correlation. @@ -122,7 +164,7 @@ def _set_litellm_batch_custom_id_labels(labels: dict[str, str], custom_id: Any) labels[f"litellm_custom_id_raw_{index}"] = raw_label_chunk -def _get_litellm_batch_custom_id_from_labels(labels: dict[str, Any]) -> str: +def _get_litellm_batch_custom_id_from_labels(labels: Mapping[str, object]) -> str: """Prefer encoded custom_id when present (see _set_litellm_batch_custom_id_labels).""" raw: Final = labels.get("litellm_custom_id_raw") if raw: @@ -186,7 +228,7 @@ def _iter_openai_jsonl_lines(openai_file_content: FileTypes) -> Iterator[str]: ``str.splitlines()`` + ``line.strip()`` for ``\\n`` / ``\\r\\n`` delimited JSONL. """ - content: Any = openai_file_content + content: FileTypes | str = openai_file_content if isinstance(content, tuple): content = content[1] @@ -246,6 +288,11 @@ def _iter_openai_jsonl_entries( yield json.loads(line) +def _parse_vertex_batch_output_row(line: str) -> _VertexBatchRow: + row: Final[_VertexBatchRow] = json.loads(line) + return row + + class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream): """Streams an OpenAI batch JSONL upload as Vertex-wrapped JSONL one row at a time, so the transformed payload is never held in full. @@ -463,7 +510,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): """ Transform VertexAI File upload response into OpenAI-style FileObject """ - response_json: Final = raw_response.json() + response_json: Final[GcsBucketResponse] = raw_response.json() try: response_object: Final = GcsBucketResponse(**response_json) @@ -523,7 +570,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): logging_obj: LiteLLMLoggingObj, litellm_params: dict, ) -> OpenAIFileObject: - response_json: Final = raw_response.json() + response_json: Final[_GcsObjectJson] = raw_response.json() gcs_id = response_json.get("id", "") gcs_id = "/".join(gcs_id.split("/")[:-1]) if gcs_id else "" return OpenAIFileObject( @@ -682,7 +729,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): # discriminating fields. Anything else (e.g. a binary file whose # first line is not valid UTF-8/JSON) raises and falls through to the # passthrough below, leaving the content untouched. - first_row: Final = json.loads(first_line) + first_row: Final = _parse_vertex_batch_output_row(first_line) is_vertex_batch_output: Final = ( "request" in first_row and "response" in first_row @@ -723,7 +770,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): for line in itertools.chain([first_line], lines): try: openai_output = self._transform_single_vertex_batch_output_to_openai( - vertex_output=json.loads(line), + vertex_output=_parse_vertex_batch_output_row(line), vertex_gemini_config=vertex_gemini_config, logging_obj=batch_transform_logging_obj, mock_httpx_response=mock_httpx_response, @@ -742,18 +789,18 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): def _transform_single_vertex_batch_output_to_openai( self, - vertex_output: dict[str, Any], + vertex_output: _VertexBatchRow, vertex_gemini_config: VertexGeminiConfig, logging_obj: Logging, mock_httpx_response: httpx.Response, - ) -> dict[str, Any]: + ) -> _OpenAIBatchOutputRow: """ Transform a single Vertex AI batch output line to OpenAI format. Uses the existing VertexGeminiConfig transformation for the response. """ # Extract custom_id from request labels (prefer raw for OpenAI round-trip) request_data: Final = vertex_output.get("request", {}) - labels: Final = request_data.get("labels", {}) or {} + labels: Final[Mapping[str, object]] = request_data.get("labels", {}) or {} custom_id: Final = _get_litellm_batch_custom_id_from_labels(labels) # Check if there's an error diff --git a/litellm/llms/vertex_ai/videos/transformation.py b/litellm/llms/vertex_ai/videos/transformation.py index d28f5b5b120..16e72e3062d 100644 --- a/litellm/llms/vertex_ai/videos/transformation.py +++ b/litellm/llms/vertex_ai/videos/transformation.py @@ -7,10 +7,12 @@ Based on: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-refer import base64 import time -from typing import TYPE_CHECKING, Any, Final, cast +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any, Final, TypedDict, cast import httpx from httpx._types import RequestFiles +from typing_extensions import ReadOnly from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS from litellm.images.utils import ImageEditRequestUtils @@ -40,11 +42,37 @@ else: BaseLLMException = Any +class _VeoVideo(TypedDict, total=False): + gcsUri: ReadOnly[str] + bytesBase64Encoded: ReadOnly[str] + mimeType: ReadOnly[str] + + +class _VeoOperationResponse(TypedDict, total=False): + videos: ReadOnly[Sequence[_VeoVideo]] + + +class _VeoOperationMetadata(TypedDict, total=False): + createTime: ReadOnly[str] + + +class _VeoOperation(TypedDict, total=False): + name: ReadOnly[str] + done: ReadOnly[bool] + metadata: ReadOnly[_VeoOperationMetadata] + response: ReadOnly[_VeoOperationResponse] + + +def _parse_veo_operation(raw_response: httpx.Response) -> _VeoOperation: + operation: Final[_VeoOperation] = raw_response.json() + return operation + + def _build_vertex_video_usage_from_request_data( request_data: dict[str, Any] | None, -) -> dict[str, Any]: +) -> dict[str, float | str]: """Build usage metadata (duration, resolution) for video cost calculation.""" - usage_data: Final[dict[str, Any]] = {} + usage_data: Final[dict[str, float | str]] = {} if not request_data: return usage_data @@ -125,7 +153,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): video_create_optional_params: VideoCreateOptionalRequestParams, model: str, drop_params: bool, - ) -> dict[str, Any]: + ) -> dict[str, object]: """ Map OpenAI-style parameters to Veo format. @@ -135,7 +163,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): - size → aspectRatio (e.g., "1280x720" → "16:9") - seconds → durationSeconds (defaults to 4 seconds if not provided) """ - mapped_params: Final[dict[str, Any]] = {} + mapped_params: Final[dict[str, object]] = {} # Map input_reference to image (will be processed in transform_video_create_request) if "input_reference" in video_create_optional_params: @@ -289,7 +317,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): } """ # Build instance with prompt - instance_dict: Final[dict[str, Any]] = {"prompt": prompt} + instance_dict: Final[dict[str, object]] = {"prompt": prompt} params_copy: Final = video_create_optional_request_params.copy() # Check if user wants to provide full instance dict @@ -324,13 +352,13 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): # {"parameters": {"parameters": {...}}} ← wrong # {"parameters": {...}} ← correct nested_params: Final = params_copy.pop("parameters", None) - vertex_params: Final[dict[str, Any]] = {} + vertex_params: Final[dict[str, object]] = {} if isinstance(nested_params, dict): vertex_params.update(nested_params) vertex_params.update(params_copy) # Build request data directly (TypedDict doesn't have model_dump) - request_data: Final[dict[str, Any]] = {"instances": [instance_dict]} + request_data: Final[dict[str, object]] = {"instances": [instance_dict]} # Only add parameters if there are any if vertex_params: @@ -363,7 +391,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): - status: "processing" - usage: includes duration_seconds and optional video_resolution for cost calculation """ - response_data: Final = raw_response.json() + response_data: Final = _parse_veo_operation(raw_response) operation_name: Final = response_data.get("name") if not operation_name: @@ -441,7 +469,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): } } """ - response_data: Final = raw_response.json() + response_data: Final = _parse_veo_operation(raw_response) operation_name: Final = response_data.get("name", "") is_done: Final = response_data.get("done", False) @@ -513,7 +541,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): Extracts the base64 encoded video from the response and decodes it to bytes. """ - response_data: Final = raw_response.json() + response_data: Final = _parse_veo_operation(raw_response) if not response_data.get("done", False): raise ValueError( @@ -548,7 +576,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, ) -> tuple[str, dict]: """ Video remix is not supported by Veo API. @@ -574,7 +602,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): after: str | None = None, limit: int | None = None, order: str | None = None, - extra_query: dict[str, Any] | None = None, + extra_query: dict[str, object] | None = None, ) -> tuple[str, dict]: """ Video list is not supported by Veo API. @@ -615,7 +643,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): """Video delete is not supported.""" raise NotImplementedError("Video delete is not supported by Vertex AI Veo.") - def transform_video_create_character_request(self, name, video, api_base, litellm_params, headers): + def transform_video_create_character_request(self, name, video: object, api_base, litellm_params, headers): raise NotImplementedError("video create character is not supported for Vertex AI") def transform_video_create_character_response(self, raw_response, logging_obj): @@ -649,7 +677,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, prefetched_source_data: dict[str, Any] | None = None, ) -> tuple[str, dict]: """ @@ -667,12 +695,13 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): if not prefetched_source_data.get("done", False): raise ValueError("Source video generation is not complete yet. Check the video status before editing.") - videos: Final = prefetched_source_data.get("response", {}).get("videos", []) + source_response: Final[_VeoOperationResponse] = prefetched_source_data.get("response", {}) + videos: Final = source_response.get("videos", []) if not videos: raise ValueError("No videos found in the completed operation. Cannot edit.") source_video: Final = videos[0] - video_input: Final[dict[str, Any]] = {} + video_input: Final[dict[str, str]] = {} if "gcsUri" in source_video: video_input["gcsUri"] = source_video["gcsUri"] elif "bytesBase64Encoded" in source_video: @@ -684,13 +713,13 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): operation_name: Final = extract_original_video_id(video_id) model: Final = self.extract_model_from_operation_name(operation_name) or "" - instance_dict: Final[dict[str, Any]] = {"prompt": prompt, "video": video_input} - request_data: Final[dict[str, Any]] = {"instances": [instance_dict]} + instance_dict: Final[dict[str, object]] = {"prompt": prompt, "video": video_input} + request_data: Final[dict[str, object]] = {"instances": [instance_dict]} if extra_body: extra_body_copy: Final = dict(extra_body) nested_params: Final = extra_body_copy.pop("parameters", None) - vertex_params: Final[dict[str, Any]] = {} + vertex_params: Final[dict[str, object]] = {} if isinstance(nested_params, dict): vertex_params.update(nested_params) vertex_params.update(extra_body_copy) @@ -716,7 +745,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): usage includes duration_seconds and optional video_resolution from the edit request parameters for cost calculation. """ - response_data: Final = raw_response.json() + response_data: Final = _parse_veo_operation(raw_response) operation_name: Final = response_data.get("name") if not operation_name: diff --git a/litellm/proxy/_experimental/mcp_server/utils.py b/litellm/proxy/_experimental/mcp_server/utils.py index e61ede4478c..5534b8b5a3c 100644 --- a/litellm/proxy/_experimental/mcp_server/utils.py +++ b/litellm/proxy/_experimental/mcp_server/utils.py @@ -8,11 +8,33 @@ import json import os import re from collections.abc import Iterable, Iterator, Mapping, MutableMapping, MutableSequence -from typing import Any, Final +from collections.abc import Set as AbstractSet +from typing import Any, Final, Protocol from urllib.parse import quote from litellm.types.mcp_server.mcp_server_manager import MCPServer + +class _McpServerLike(Protocol): + @property + def server_id(self) -> str: ... + @property + def server_name(self) -> str | None: ... + @property + def alias(self) -> str | None: ... + @property + def short_prefix(self) -> str | None: ... + + +class McpServerPayloadLike(Protocol): + alias: str | None + + @property + def server_name(self) -> str | None: ... + @property + def tool_name_to_display_name(self) -> Mapping[str, str] | None: ... + + # Constants # # NOTE: The environment-backed values below are read once, when this module is @@ -102,7 +124,7 @@ def compute_short_server_prefix(server_id: str, attempt: int = 0) -> str: # at the end so the first emitted char comes from the high-order # bits of the digest (which is the position we constrain to be # alphabetic). - chars: Final = [] + chars: Final[list[str]] = [] for position in range(SHORT_MCP_TOOL_PREFIX_LENGTH): is_first_char = position == SHORT_MCP_TOOL_PREFIX_LENGTH - 1 alphabet = _BASE52_ALPHA_ALPHABET if is_first_char else _BASE62_ALPHABET @@ -176,34 +198,34 @@ def lookup_mcp_server_auth_in_headers( MCP_TOOL_ALLOWLIST_ENFORCED_KEY: Final = "tool_allowlist_enforced" -def _parse_mcp_info_dict(mcp_info: Any) -> dict[str, Any] | None: +def _parse_mcp_info_dict(mcp_info: object) -> Mapping[str, object] | None: if mcp_info is None: return None if isinstance(mcp_info, dict): return mcp_info if isinstance(mcp_info, str): try: - parsed: Final = json.loads(mcp_info) + parsed: Final[object] = json.loads(mcp_info) except (ValueError, TypeError): return None return parsed if isinstance(parsed, dict) else None return None -def is_server_tool_allowlist_enforced(mcp_server: Any) -> bool: +def is_server_tool_allowlist_enforced(mcp_server: object) -> bool: mcp_info: Final = _parse_mcp_info_dict(getattr(mcp_server, "mcp_info", None)) if not mcp_info: return False return bool(mcp_info.get(MCP_TOOL_ALLOWLIST_ENFORCED_KEY)) -def server_applies_tool_allowlist(mcp_server: Any) -> bool: +def server_applies_tool_allowlist(mcp_server: object) -> bool: """Whether server-level allowed_tools whitelist filtering is active.""" - allowed_tools: Final = getattr(mcp_server, "allowed_tools", None) or [] + allowed_tools: Final[object] = getattr(mcp_server, "allowed_tools", None) or [] return is_server_tool_allowlist_enforced(mcp_server) or bool(allowed_tools) -def validate_and_normalize_mcp_server_payload(payload: Any) -> None: +def validate_and_normalize_mcp_server_payload(payload: McpServerPayloadLike) -> None: """ Validate and normalize MCP server payload fields (server_name, alias, and tool_name_to_display_name). @@ -233,8 +255,8 @@ def validate_and_normalize_mcp_server_payload(payload: Any) -> None: validate_tool_display_names(payload.tool_name_to_display_name) # Alias normalization and defaulting - alias = getattr(payload, "alias", None) - server_name: Final = getattr(payload, "server_name", None) + alias: str | None = getattr(payload, "alias", None) + server_name: Final[str | None] = getattr(payload, "server_name", None) if not alias and server_name: alias = normalize_server_name(server_name) @@ -257,7 +279,7 @@ def add_server_prefix_to_name(name: str, server_name: str) -> str: ) -def get_server_prefix(server: Any) -> str: +def get_server_prefix(server: object) -> str: """Return the prefix for a server. When the short-prefix mode is enabled (``LITELLM_USE_SHORT_MCP_TOOL_PREFIX``) @@ -270,23 +292,26 @@ def get_server_prefix(server: Any) -> str: alias if present, else server_name, else server_id. """ if is_short_mcp_tool_prefix_enabled(): - cached: Final = getattr(server, "short_prefix", None) + cached: Final[str | None] = getattr(server, "short_prefix", None) if cached: return cached - server_id: Final = getattr(server, "server_id", None) + server_id: Final[str | None] = getattr(server, "server_id", None) if server_id: return compute_short_server_prefix(server_id) - if hasattr(server, "alias") and server.alias: - return server.alias - if hasattr(server, "server_name") and server.server_name: - return server.server_name + alias: Final[str | None] = getattr(server, "alias", None) + if alias: + return alias + server_name: Final[str | None] = getattr(server, "server_name", None) + if server_name: + return server_name if hasattr(server, "server_id"): - return server.server_id + fallback_server_id: Final[str] = getattr(server, "server_id", "") + return fallback_server_id return "" -def iter_known_server_prefixes(server: Any) -> Iterator[str]: +def iter_known_server_prefixes(server: _McpServerLike) -> Iterator[str]: """Yield every prefix form that may appear in tool names for ``server``. Always includes the *current* prefix returned by ``get_server_prefix``. @@ -304,7 +329,7 @@ def iter_known_server_prefixes(server: Any) -> Iterator[str]: yield from _emit(get_server_prefix(server)) yield from _emit(getattr(server, "short_prefix", None)) - server_id: Final = getattr(server, "server_id", None) + server_id: Final[str | None] = getattr(server, "server_id", None) if server_id: try: yield from _emit(compute_short_server_prefix(server_id)) @@ -397,7 +422,7 @@ def match_known_server_prefix(name: str, known_prefixes: Iterable[str]) -> tuple return None -def strip_known_server_prefix(name: str, server: Any | None) -> str: +def strip_known_server_prefix(name: str, server: _McpServerLike | None) -> str: """Strip ``server``'s registered prefix from a prefixed tool/resource name. Unlike :func:`split_server_prefix_from_name`, which guesses the boundary at @@ -420,7 +445,7 @@ def strip_known_server_prefix(name: str, server: Any | None) -> str: def is_tool_name_prefixed( tool_name: str, - known_server_prefixes: set | None = None, + known_server_prefixes: AbstractSet[str] | None = None, ) -> bool: """ Check if tool name has a known MCP server prefix. @@ -640,7 +665,7 @@ def parse_admin_env_vars( if raw is None: continue if hasattr(raw, "model_dump"): - entry = raw.model_dump() + entry: Mapping[str, object] = raw.model_dump() elif isinstance(raw, dict): entry = raw else: diff --git a/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py b/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py index ca84ff47884..7d6fafe141f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py +++ b/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py @@ -11,6 +11,7 @@ import requests from fastapi import HTTPException from httpx import HTTPStatusError from requests.auth import HTTPBasicAuth +from typing_extensions import ReadOnly from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( @@ -55,6 +56,26 @@ class _HiddenlayerResponse(TypedDict, total=False): modified_data: Mapping[str, _HiddenlayerModifiedSide] +class _LoggedCallMetadata(TypedDict, total=False): + headers: ReadOnly[Mapping[str, str]] + + +class _LoggedCallLitellmParams(TypedDict, total=False): + metadata: ReadOnly[_LoggedCallMetadata] + + +class _HiddenlayerOutputMessage(TypedDict, total=False): + content: ReadOnly[str | Sequence[Mapping[str, str]]] + + +class _HiddenlayerChoiceMessage(TypedDict, total=False): + content: ReadOnly[str] + + +class _HiddenlayerChoice(TypedDict, total=False): + message: ReadOnly[_HiddenlayerChoiceMessage] + + def is_saas(host: str) -> bool: """Checks whether the connection is to the SaaS platform""" @@ -155,7 +176,10 @@ class HiddenlayerGuardrail(CustomGuardrail): # from the logger object on the response from the model. headers = request_data.get("proxy_server_request", {}).get("headers", {}) if not headers and logging_obj and logging_obj.model_call_details: - headers = logging_obj.model_call_details.get("litellm_params", {}).get("metadata", {}).get("headers", {}) + logged_litellm_params: Final[_LoggedCallLitellmParams] = logging_obj.model_call_details.get( + "litellm_params", {} + ) + headers = logged_litellm_params.get("metadata", {}).get("headers", {}) hl_request_metadata["requester_id"] = headers.get("hl-requester-id") or "LiteLLM" project_id: Final = headers.get("hl-project-id") @@ -408,7 +432,8 @@ class HiddenlayerGuardrailV2(CustomGuardrail): if input_type == "request": inputs["structured_messages"] = output - for message in output.get("messages", []): + modified_messages: Final[Sequence[_HiddenlayerOutputMessage]] = output.get("messages", []) + for message in modified_messages: content = message.get("content", "") if isinstance(content, list): text_parts = [ @@ -422,7 +447,8 @@ class HiddenlayerGuardrailV2(CustomGuardrail): inputs["texts"] = new_texts elif input_type == "response" and inputs.get("texts"): - inputs["texts"] = [output.get("choices", [{}])[-1].get("message", {}).get("content", "")] + redacted_choices: Final[Sequence[_HiddenlayerChoice]] = output.get("choices", [{}]) + inputs["texts"] = [redacted_choices[-1].get("message", {}).get("content", "")] elif input_type == "response" and inputs.get("tool_calls"): inputs["tool_calls"] = output diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 9f70ed63dcb..5f7374581a2 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -2,9 +2,10 @@ import importlib import os +from collections.abc import Callable, Iterator, Mapping from datetime import datetime, timezone from itertools import chain, count -from typing import Any, Final, Literal, Optional, cast +from typing import Any, Final, Literal, Optional, Protocol, cast from pydantic import ValidationError @@ -59,6 +60,13 @@ from .guardrail_initializers import ( initialize_tool_permission, ) + +class _GuardrailRowLike(Protocol): + @property + def guardrail_id(self) -> str: ... + def __iter__(self) -> Iterator[tuple[str, object]]: ... + + guardrail_initializer_registry: Final = { SupportedGuardrailIntegrations.BEDROCK.value: initialize_bedrock, SupportedGuardrailIntegrations.LAKERA.value: initialize_lakera, @@ -125,7 +133,9 @@ def get_guardrail_initializer_from_hooks(): # Check for guardrail_initializer_registry dictionary if hasattr(module, "guardrail_initializer_registry"): - registry = getattr(module, "guardrail_initializer_registry") + registry: Mapping[str, Callable[..., CustomGuardrail]] | None = getattr( + module, "guardrail_initializer_registry", None + ) if isinstance(registry, dict): discovered_initializers.update(registry) verbose_proxy_logger.debug( @@ -135,7 +145,7 @@ def get_guardrail_initializer_from_hooks(): # Check for standalone initialize_guardrail function (fallback for directory-based guardrails) elif hasattr(module, "initialize_guardrail"): # For directories with just initialize_guardrail, use the directory name as the key - initialize_fn = getattr(module, "initialize_guardrail") + initialize_fn: Callable[..., CustomGuardrail] | None = getattr(module, "initialize_guardrail", None) discovered_initializers[item] = initialize_fn verbose_proxy_logger.debug("Found initialize_guardrail function in %s", module_path) @@ -206,7 +216,9 @@ def get_guardrail_class_from_hooks(): # Check for guardrail_initializer_registry dictionary if hasattr(module, "guardrail_class_registry"): - registry = getattr(module, "guardrail_class_registry") + registry: Mapping[str, type[CustomGuardrail]] | None = getattr( + module, "guardrail_class_registry", None + ) if isinstance(registry, dict): discovered_classes.update(registry) @@ -275,7 +287,7 @@ class GuardrailRegistry: guardrail_info: Final[str] = safe_dumps(guardrail.get("guardrail_info", {})) # Create guardrail in DB - created_guardrail: Final = await GuardrailsRepository(prisma_client).table.create( + created_guardrail: Final[_GuardrailRowLike] = await GuardrailsRepository(prisma_client).table.create( data={ "guardrail_name": guardrail_name, "litellm_params": litellm_params, @@ -321,7 +333,7 @@ class GuardrailRegistry: guardrail_info: Final[str] = safe_dumps(guardrail.get("guardrail_info", {})) # Update in DB - updated_guardrail: Final = await GuardrailsRepository(prisma_client).table.update( + updated_guardrail: Final[_GuardrailRowLike] = await GuardrailsRepository(prisma_client).table.update( where={"guardrail_id": guardrail_id}, data={ "guardrail_name": guardrail_name, @@ -482,7 +494,7 @@ class InMemoryGuardrailHandler: custom_guardrail_callback = initializer(litellm_params, guardrail) elif isinstance(guardrail_type, str) and "." in guardrail_type: custom_guardrail_callback = self.initialize_custom_guardrail( - guardrail=cast(dict, guardrail), + guardrail=guardrail, guardrail_type=guardrail_type, litellm_params=litellm_params, config_file_path=config_file_path, @@ -512,7 +524,7 @@ class InMemoryGuardrailHandler: "skip_tool_message_in_guardrail are enabled together, which excludes every message from " "scanning, so no request content would ever be scanned. Remove one of the two." ) - configured_run_in_parallel: Final = getattr(litellm_params, "run_in_parallel", None) + configured_run_in_parallel: Final[bool | None] = getattr(litellm_params, "run_in_parallel", None) if configured_run_in_parallel is not None: custom_guardrail_callback.run_in_parallel = bool(configured_run_in_parallel) @@ -532,7 +544,7 @@ class InMemoryGuardrailHandler: def initialize_custom_guardrail( self, - guardrail: dict, + guardrail: Guardrail, guardrail_type: str, litellm_params: LitellmParams, config_file_path: str | None = None, @@ -550,7 +562,9 @@ class InMemoryGuardrailHandler: guardrail_type, ) - _guardrail_class: Final = get_instance_fn(guardrail_type, config_file_path=config_file_path) + _guardrail_class: Final[Callable[..., CustomGuardrail]] = get_instance_fn( + guardrail_type, config_file_path=config_file_path + ) mode: Final = litellm_params.mode if mode is None: @@ -683,8 +697,8 @@ class InMemoryGuardrailHandler: @staticmethod def _normalize_litellm_params_for_comparison( - params: Any | None, - ) -> dict[str, Any] | None: + params: LitellmParams | Mapping[str, object] | None, + ) -> Mapping[str, object] | None: """ Render litellm_params to a canonical dict so an in-memory LitellmParams and the raw dict loaded from the DB compare equal when they describe the same diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index bfc70da46ea..6c25f096532 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -10,12 +10,19 @@ All /customer management endpoints """ #### END-USER/CUSTOMER MANAGEMENT #### +from collections.abc import Mapping, Sequence from datetime import datetime, timedelta -from typing import Final +from typing import TYPE_CHECKING, Final, Protocol, TypeVar, overload import fastapi from fastapi import APIRouter, Depends, HTTPException, Request -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter + +if TYPE_CHECKING: + from prisma.models import LiteLLM_BudgetTable as PrismaBudgetRow + from prisma.models import LiteLLM_EndUserTable as PrismaEndUserRow + + from litellm.proxy.utils import PrismaClient import litellm from litellm._logging import verbose_proxy_logger @@ -41,6 +48,54 @@ from litellm.types.proxy.management_endpoints.customer_endpoints import ( UnblockUsersResponse, ) +_RowT_co: Final = TypeVar("_RowT_co", covariant=True) +_STR_OBJECT_DICT: Final = TypeAdapter(dict[str, object]) + +if TYPE_CHECKING: + + class _TableOps(Protocol[_RowT_co]): + async def find_first( + self, + where: Mapping[str, object] | None = None, + include: Mapping[str, bool] | None = None, + ) -> _RowT_co | None: ... + + async def find_many( + self, + where: Mapping[str, object] | None = None, + include: Mapping[str, bool] | None = None, + ) -> Sequence[_RowT_co]: ... + + async def create( + self, + data: Mapping[str, object], + include: Mapping[str, bool] | None = None, + ) -> _RowT_co: ... + + async def update( + self, + where: Mapping[str, object], + data: Mapping[str, object], + include: Mapping[str, bool] | None = None, + ) -> _RowT_co | None: ... + + async def upsert( + self, + where: Mapping[str, object], + data: Mapping[str, Mapping[str, object]], + ) -> _RowT_co: ... + + async def delete_many(self, where: Mapping[str, object]) -> int: ... + + +@overload +def _typed_table(repo: EndUserRepository) -> "_TableOps[PrismaEndUserRow]": ... +@overload +def _typed_table(repo: BudgetRepository) -> "_TableOps[PrismaBudgetRow]": ... +def _typed_table(repo: EndUserRepository | BudgetRepository) -> object: + return repo.table + + router: Final = APIRouter() @@ -89,7 +144,7 @@ async def block_user(data: BlockUsers): records: Final = [] if prisma_client is not None: for id in data.user_ids: - record = await EndUserRepository(prisma_client).table.upsert( + record = await _typed_table(EndUserRepository(prisma_client)).upsert( where={"user_id": id}, data={ "create": {"user_id": id, "blocked": True}, @@ -184,7 +239,7 @@ def new_budget_request(data: NewCustomerRequest) -> BudgetNewRequest | None: budget_kv_pairs[field_name] = value if budget_kv_pairs: - budget_request: Final = BudgetNewRequest(**budget_kv_pairs) + budget_request: Final = BudgetNewRequest.model_validate(budget_kv_pairs) validate_budget_duration(budget_request.budget_duration) if budget_request.budget_reset_at is None and budget_request.budget_duration is not None: budget_request.budget_reset_at = datetime.utcnow() + timedelta( @@ -195,10 +250,10 @@ def new_budget_request(data: NewCustomerRequest) -> BudgetNewRequest | None: async def _handle_customer_object_permission_update( - non_default_values: dict, + non_default_values: dict[str, object], end_user_table_data_typed: LiteLLM_EndUserTable | None, - update_end_user_table_data: dict, - prisma_client, + update_end_user_table_data: dict[str, object], + prisma_client: "PrismaClient", ) -> None: """ Handle object permission updates for customer endpoints. @@ -344,13 +399,13 @@ async def new_end_user( }, ) - new_end_user_obj: dict = {} + new_end_user_obj: dict[str, object] = {} ## CREATE BUDGET ## if set _new_budget: Final = new_budget_request(data) if _new_budget is not None: try: - budget_record: Final = await BudgetRepository(prisma_client).table.create( + budget_record: Final = await _typed_table(BudgetRepository(prisma_client)).create( data={ **_new_budget.model_dump(exclude_unset=True), "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name, @@ -364,16 +419,18 @@ async def new_end_user( elif data.budget_id is not None: new_end_user_obj["budget_id"] = data.budget_id - _user_data: Final = data.dict(exclude_none=True) + _user_data: Final = _STR_OBJECT_DICT.validate_python(data.dict(exclude_none=True)) for k, v in _user_data.items(): if k not in BudgetNewRequest.model_fields: new_end_user_obj[k] = v ## Handle Object Permission - MCP Servers, Vector Stores etc. - new_end_user_obj = await _set_object_permission( - data_json=new_end_user_obj, - prisma_client=prisma_client, + new_end_user_obj = _STR_OBJECT_DICT.validate_python( + await _set_object_permission( + data_json=new_end_user_obj, + prisma_client=prisma_client, + ) ) # Ensure object_permission is not in the data being sent to create @@ -386,7 +443,7 @@ async def new_end_user( new_end_user_obj.pop("object_permission", None) ## WRITE TO DB ## - end_user_record: Final = await EndUserRepository(prisma_client).table.create( + end_user_record: Final = await _typed_table(EndUserRepository(prisma_client)).create( data=new_end_user_obj, include={"litellm_budget_table": True, "object_permission": True}, ) @@ -442,7 +499,7 @@ async def end_user_info( detail={"error": CommonProxyErrors.db_not_connected_error.value}, ) - user_info: Final = await EndUserRepository(prisma_client).table.find_first( + user_info: Final = await _typed_table(EndUserRepository(prisma_client)).find_first( where={"user_id": end_user_id}, include={"litellm_budget_table": True, "object_permission": True}, ) @@ -535,13 +592,13 @@ async def update_end_user( from litellm.proxy.proxy_server import litellm_proxy_admin_name, prisma_client try: - data_json: Final[dict] = data.json() + data_json: Final = _STR_OBJECT_DICT.validate_python(data.json()) # get the row from db if prisma_client is None: raise Exception("Not connected to DB!") # get non default values for key - non_default_values: Final = {} + non_default_values: Final = dict[str, object]() for k, v in data_json.items(): if v is not None and v not in ( [], @@ -551,7 +608,7 @@ async def update_end_user( non_default_values[k] = v ## Get end user table data ## - end_user_table_data: Final = await EndUserRepository(prisma_client).table.find_first( + end_user_table_data: Final = await _typed_table(EndUserRepository(prisma_client)).find_first( where={"user_id": data.user_id}, include={"litellm_budget_table": True} ) @@ -563,14 +620,14 @@ async def update_end_user( param="user_id", ) - end_user_table_data_typed: Final = LiteLLM_EndUserTable(**end_user_table_data.model_dump()) + end_user_table_data_typed: Final = LiteLLM_EndUserTable.model_validate(end_user_table_data.model_dump()) ## Get budget table data ## end_user_budget_table: Final = end_user_table_data_typed.litellm_budget_table ## Get all params for budget table ## - budget_table_data: Final = {} - update_end_user_table_data: Final = {} + budget_table_data: Final = dict[str, object]() + update_end_user_table_data: Final = dict[str, object]() for k, v in non_default_values.items(): # budget_id is for linking to existing budget, not for creating new budget if k == "budget_id": @@ -593,7 +650,7 @@ async def update_end_user( if budget_table_data: if end_user_budget_table is None: ## Create new budget ## - budget_table_data_record = await BudgetRepository(prisma_client).table.create( + budget_table_data_record = await _typed_table(BudgetRepository(prisma_client)).create( data={ **budget_table_data, "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name, @@ -605,7 +662,7 @@ async def update_end_user( update_end_user_table_data["budget_id"] = budget_table_data_record.budget_id else: ## Update existing budget ## - budget_table_data_record = await BudgetRepository(prisma_client).table.update( + budget_table_data_record = await _typed_table(BudgetRepository(prisma_client)).update( where={"budget_id": end_user_budget_table.budget_id}, data=budget_table_data, ) @@ -625,7 +682,7 @@ async def update_end_user( if data.user_id is not None and len(data.user_id) > 0: update_end_user_table_data["user_id"] = data.user_id verbose_proxy_logger.debug("In update customer, user_id condition block.") - response: Final = await EndUserRepository(prisma_client).table.update( + response: Final = await _typed_table(EndUserRepository(prisma_client)).update( where={"user_id": data.user_id}, data=update_end_user_table_data, include={"litellm_budget_table": True, "object_permission": True}, @@ -688,7 +745,7 @@ async def delete_end_user( verbose_proxy_logger.debug("/customer/delete: Received data = %s", data) if data.user_ids is not None and isinstance(data.user_ids, list) and len(data.user_ids) > 0: # First check if all users exist - existing_users: Final = await EndUserRepository(prisma_client).table.find_many( + existing_users: Final = await _typed_table(EndUserRepository(prisma_client)).find_many( where={"user_id": {"in": data.user_ids}} ) existing_user_ids: Final = {user.user_id for user in existing_users} @@ -703,7 +760,7 @@ async def delete_end_user( ) # All users exist, proceed with deletion - response: Final = await EndUserRepository(prisma_client).table.delete_many( + response: Final = await _typed_table(EndUserRepository(prisma_client)).delete_many( where={"user_id": {"in": data.user_ids}} ) verbose_proxy_logger.debug("received response from updating prisma client. response=%s", response) @@ -764,7 +821,7 @@ async def list_end_user( detail={"error": CommonProxyErrors.db_not_connected_error.value}, ) - response: Final = await EndUserRepository(prisma_client).table.find_many( + response: Final = await _typed_table(EndUserRepository(prisma_client)).find_many( include={"litellm_budget_table": True, "object_permission": True} ) @@ -827,11 +884,10 @@ async def get_customer_daily_activity( exclude_end_user_ids_list = exclude_end_user_ids.split(",") if exclude_end_user_ids else None # Fetch organization aliases for metadata - where_condition: Final = {} + where_condition: Final = dict[str, object]() if end_user_ids_list: where_condition["user_id"] = {"in": list(end_user_ids_list)} - end_user_aliases: Final = await EndUserRepository(prisma_client).table.find_many(where=where_condition) - end_user_alias_metadata: Final = {e.user_id: {"alias": e.alias} for e in end_user_aliases} + end_user_aliases: Final = await _typed_table(EndUserRepository(prisma_client)).find_many(where=where_condition) # Query daily activity for organizations return await get_daily_activity( @@ -839,7 +895,7 @@ async def get_customer_daily_activity( table_name="litellm_dailyenduserspend", entity_id_field="end_user_id", entity_id=end_user_ids_list, - entity_metadata_field=end_user_alias_metadata, + entity_metadata_field={e.user_id: {"alias": e.alias} for e in end_user_aliases}, exclude_entity_ids=exclude_end_user_ids_list, start_date=start_date, end_date=end_date, diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index e156e5f0046..c4f5ceaba7a 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -50,6 +50,7 @@ from litellm.constants import LITELLM_PROXY_ADMIN_NAME from litellm.proxy._experimental.mcp_server.utils import ( LITELLM_MCP_SERVER_DESCRIPTION, LITELLM_MCP_SERVER_NAME, + McpServerPayloadLike, build_env_var_setup_url, collect_env_var_references, get_server_prefix, @@ -196,7 +197,7 @@ if MCP_AVAILABLE: server: MCPServer expires_at: datetime - def _validate_mcp_server_name_fields(payload: Any) -> None: + def _validate_mcp_server_name_fields(payload: McpServerPayloadLike) -> None: candidates: Final[list[tuple[str, str | None]]] = [] server_name: Final = getattr(payload, "server_name", None) @@ -223,7 +224,7 @@ if MCP_AVAILABLE: detail={"error": error_messages_text}, ) - def validate_and_normalize_mcp_server_payload(payload: Any) -> None: + def validate_and_normalize_mcp_server_payload(payload: McpServerPayloadLike) -> None: _base_validate_and_normalize_mcp_server_payload(payload) _validate_mcp_server_name_fields(payload) diff --git a/litellm/proxy/management_endpoints/tool_management_endpoints.py b/litellm/proxy/management_endpoints/tool_management_endpoints.py index 0fdedafb2bf..b480d46f185 100644 --- a/litellm/proxy/management_endpoints/tool_management_endpoints.py +++ b/litellm/proxy/management_endpoints/tool_management_endpoints.py @@ -10,13 +10,21 @@ POST /v1/tool/policy - Update the input_policy / output_policy for a """ import uuid +from collections.abc import Mapping, Sequence from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, Annotated, Any, Final +from typing import TYPE_CHECKING, Annotated, Final, Protocol, TypeAlias, TypeVar, overload from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel, Field, TypeAdapter if TYPE_CHECKING: + from prisma.models import LiteLLM_DailyToolSpend as PrismaDailyToolSpendRow + from prisma.models import LiteLLM_ObjectPermissionTable as PrismaObjectPermissionRow + from prisma.models import LiteLLM_SpendLogs as PrismaSpendLogRow + from prisma.models import LiteLLM_SpendLogToolIndex as PrismaSpendLogToolIndexRow + from prisma.models import LiteLLM_TeamTable as PrismaTeamRow + from prisma.models import LiteLLM_VerificationToken as PrismaVerificationTokenRow + from litellm.proxy.utils import PrismaClient from litellm._logging import verbose_proxy_logger @@ -49,6 +57,72 @@ from litellm.types.tool_management import ( ToolUsageLogsResponse, ) +_RowT_co: Final = TypeVar("_RowT_co", covariant=True) + +if TYPE_CHECKING: + + class _TableOps(Protocol[_RowT_co]): + async def find_many( + self, + where: Mapping[str, object] | None = None, + order: Mapping[str, object] | Sequence[Mapping[str, object]] | None = None, + skip: int | None = None, + take: int | None = None, + ) -> Sequence[_RowT_co]: ... + + async def find_unique(self, where: Mapping[str, object]) -> _RowT_co | None: ... + + async def count(self, where: Mapping[str, object] | None = None) -> int: ... + + async def create(self, data: Mapping[str, object]) -> _RowT_co: ... + + async def update_many( + self, + where: Mapping[str, object], + data: Mapping[str, object], + ) -> int: ... + + async def delete(self, where: Mapping[str, object]) -> _RowT_co | None: ... + + async def group_by( + self, + by: Sequence[str], + sum: Mapping[str, bool] | None = None, + where: Mapping[str, object] | None = None, + order: Mapping[str, object] | None = None, + take: int | None = None, + ) -> Sequence[Mapping[str, object]]: ... + + class _SpendLogRow(Protocol): + @property + def messages(self) -> object: ... + @property + def proxy_server_request(self) -> str | Mapping[str, object] | None: ... + + +@overload +def _typed_table(repo: DailyToolSpendRepository) -> "_TableOps[PrismaDailyToolSpendRow]": ... +@overload +def _typed_table(repo: SpendLogToolIndexRepository) -> "_TableOps[PrismaSpendLogToolIndexRow]": ... +@overload +def _typed_table(repo: SpendLogsRepository) -> "_TableOps[PrismaSpendLogRow]": ... +@overload +def _typed_table(repo: VerificationTokenRepository) -> "_TableOps[PrismaVerificationTokenRow]": ... +@overload +def _typed_table(repo: TeamRepository) -> "_TableOps[PrismaTeamRow]": ... +@overload +def _typed_table(repo: ObjectPermissionRepository) -> "_TableOps[PrismaObjectPermissionRow]": ... +def _typed_table( + repo: DailyToolSpendRepository + | SpendLogToolIndexRepository + | SpendLogsRepository + | VerificationTokenRepository + | TeamRepository + | ObjectPermissionRepository, +) -> object: + return repo.table + + router: Final = APIRouter() TOOL_POLICY_OPTIONS: Final = ToolPolicyOptionsResponse( @@ -201,7 +275,7 @@ async def get_tool_spend( end_str: Final = end_day.strftime("%Y-%m-%d") date_window: Final = {"date": {"gte": start_str, "lte": end_str}} - table: Final = DailyToolSpendRepository(prisma_client).table + table: Final = _typed_table(DailyToolSpendRepository(prisma_client)) top_tools: Final = _TOP_TOOL_ROWS.validate_python( await table.group_by( by=["tool_name"], @@ -222,7 +296,7 @@ async def get_tool_spend( for row in top_tools ] - daily_rows: Final = ( + daily_rows: Final[Sequence[PrismaDailyToolSpendRow]] = ( await table.find_many( where={**date_window, "tool_name": {"in": [row.tool_name for row in top_tools]}}, order=[{"date": "asc"}, {"spend": "desc"}], @@ -270,36 +344,43 @@ async def get_tool_detail( raise HTTPException(status_code=500, detail=str(e)) -def _input_snippet_for_tool_log(sl: Any, max_len: int = 200) -> str | None: +_ParsedJson: TypeAlias = dict[str, object] | list[object] | str | int | float | bool | None +_PARSED_JSON: Final[TypeAdapter[_ParsedJson]] = TypeAdapter(_ParsedJson) +_STR_OBJECT_DICT: Final = TypeAdapter(dict[str, object]) + + +def _input_snippet_for_tool_log(sl: "_SpendLogRow | None", max_len: int = 200) -> str | None: """Short snippet from messages or proxy_server_request for tool usage log row.""" if sl is None: return None - messages: Final = getattr(sl, "messages", None) + messages: Final = sl.messages if messages is not None: s = _snippet_str(messages, max_len) if s: return s - psr = getattr(sl, "proxy_server_request", None) + psr = sl.proxy_server_request if not psr: return None if isinstance(psr, str): import json try: - psr = json.loads(psr) + psr = _PARSED_JSON.validate_python(json.loads(psr)) except Exception: return _snippet_str(psr, max_len) if isinstance(psr, dict): msgs = psr.get("messages") - if msgs is None and isinstance(psr.get("body"), dict): - msgs = psr["body"].get("messages") + if msgs is None: + body: Final = psr.get("body") + if isinstance(body, dict): + msgs = _STR_OBJECT_DICT.validate_python(body).get("messages") s = _snippet_str(msgs, max_len) if s: return s return _snippet_str(psr, max_len) -def _snippet_str(text: Any, max_len: int = 200) -> str | None: +def _snippet_str(text: object, max_len: int = 200) -> str | None: if text is None: return None if isinstance(text, str): @@ -344,7 +425,7 @@ async def get_tool_usage_logs( raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) try: - where: Final[dict] = {"tool_name": tool_name} + where: Final[dict[str, object]] = {"tool_name": tool_name} if start_date or end_date: start_time_filter: datetime | None = None end_time_filter: datetime | None = None @@ -363,14 +444,14 @@ async def get_tool_usage_logs( except ValueError: pass if start_time_filter is not None or end_time_filter is not None: - where["start_time"] = {} - if start_time_filter is not None: - where["start_time"]["gte"] = start_time_filter - if end_time_filter is not None: - where["start_time"]["lte"] = end_time_filter + where["start_time"] = { + key: value + for key, value in (("gte", start_time_filter), ("lte", end_time_filter)) + if value is not None + } - total: Final = await SpendLogToolIndexRepository(prisma_client).table.count(where=where) - index_rows: Final = await SpendLogToolIndexRepository(prisma_client).table.find_many( + total: Final = await _typed_table(SpendLogToolIndexRepository(prisma_client)).count(where=where) + index_rows: Final = await _typed_table(SpendLogToolIndexRepository(prisma_client)).find_many( where=where, order={"start_time": "desc"}, skip=(page - 1) * page_size, @@ -380,7 +461,9 @@ async def get_tool_usage_logs( if not request_ids: return ToolUsageLogsResponse(logs=[], total=total, page=page, page_size=page_size) - spend_logs = await SpendLogsRepository(prisma_client).table.find_many(where={"request_id": {"in": request_ids}}) + spend_logs = await _typed_table(SpendLogsRepository(prisma_client)).find_many( + where={"request_id": {"in": request_ids}} + ) log_by_id: Final = {s.request_id: s for s in spend_logs} logs_out: Final[list[ToolUsageLogEntry]] = [] @@ -449,24 +532,24 @@ async def _resolve_key_hash_to_object_permission_id( hashed: Final = key_hash if "sk-" not in (key_hash or "") else hash_token(key_hash) if not hashed: return None - row = await VerificationTokenRepository(prisma_client).table.find_unique(where={"token": hashed}) + row = await _typed_table(VerificationTokenRepository(prisma_client)).find_unique(where={"token": hashed}) if row is None: return None - op_id: Final = getattr(row, "object_permission_id", None) + op_id: Final = row.object_permission_id if op_id: return op_id new_id: Final = str(uuid.uuid4()) - await ObjectPermissionRepository(prisma_client).table.create( + await _typed_table(ObjectPermissionRepository(prisma_client)).create( data={"object_permission_id": new_id, "blocked_tools": []} ) - updated_count: Final = await VerificationTokenRepository(prisma_client).table.update_many( + updated_count: Final = await _typed_table(VerificationTokenRepository(prisma_client)).update_many( where={"token": hashed, "object_permission_id": None}, data={"object_permission_id": new_id}, ) if updated_count == 0: - await ObjectPermissionRepository(prisma_client).table.delete(where={"object_permission_id": new_id}) - row = await VerificationTokenRepository(prisma_client).table.find_unique(where={"token": hashed}) - return getattr(row, "object_permission_id", None) if row else None + await _typed_table(ObjectPermissionRepository(prisma_client)).delete(where={"object_permission_id": new_id}) + row = await _typed_table(VerificationTokenRepository(prisma_client)).find_unique(where={"token": hashed}) + return row.object_permission_id if row else None return new_id @@ -478,24 +561,24 @@ async def _resolve_team_id_to_object_permission_id( if not team_id or not team_id.strip(): return None team_id_clean: Final = team_id.strip() - row = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id_clean}) + row = await _typed_table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id_clean}) if row is None: return None - op_id: Final = getattr(row, "object_permission_id", None) + op_id: Final = row.object_permission_id if op_id: return op_id new_id: Final = str(uuid.uuid4()) - await ObjectPermissionRepository(prisma_client).table.create( + await _typed_table(ObjectPermissionRepository(prisma_client)).create( data={"object_permission_id": new_id, "blocked_tools": []} ) - updated_count: Final = await TeamRepository(prisma_client).table.update_many( + updated_count: Final = await _typed_table(TeamRepository(prisma_client)).update_many( where={"team_id": team_id_clean, "object_permission_id": None}, data={"object_permission_id": new_id}, ) if updated_count == 0: - await ObjectPermissionRepository(prisma_client).table.delete(where={"object_permission_id": new_id}) - row = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id_clean}) - return getattr(row, "object_permission_id", None) if row else None + await _typed_table(ObjectPermissionRepository(prisma_client)).delete(where={"object_permission_id": new_id}) + row = await _typed_table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id_clean}) + return row.object_permission_id if row else None return new_id diff --git a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py index 2e38abddd0f..9d5ddda017a 100644 --- a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py +++ b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py @@ -4,11 +4,11 @@ usage/spend data by querying the aggregated daily activity endpoints. """ import json -from collections.abc import AsyncIterator, Callable +from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence from datetime import date -from typing import Any, Final, Literal, cast +from typing import Any, Final, Literal, Protocol, cast, overload -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_proxy_logger @@ -73,9 +73,36 @@ class SSEErrorEvent(TypedDict): SSEEvent = SSEStatusEvent | SSEToolCallEvent | SSEChunkEvent | SSEDoneEvent | SSEErrorEvent +class _EntityEntry(TypedDict, total=False): + metrics: ReadOnly[Mapping[str, float]] + metadata: ReadOnly[Mapping[str, str]] + + +class _DayDump(TypedDict, total=False): + breakdown: ReadOnly[Mapping[str, Mapping[str, _EntityEntry]]] + + +class _UsageDump(Protocol): + @overload + def get(self, key: Literal["metadata"], default: Mapping[str, float], /) -> Mapping[str, float]: ... + @overload + def get(self, key: Literal["results"], default: Sequence[_DayDump], /) -> Sequence[_DayDump]: ... + + +class _ToolFunctionDef(TypedDict): + name: ReadOnly[str] + description: ReadOnly[str] + parameters: ReadOnly[Mapping[str, object]] + + +class _ToolDef(TypedDict): + type: ReadOnly[str] + function: ReadOnly[_ToolFunctionDef] + + class ToolHandler(TypedDict): - fetch: Callable[..., Any] - summarise: Callable[[dict[str, Any]], str] + fetch: Callable[..., Awaitable[_UsageDump]] + summarise: Callable[[_UsageDump], str] label: str @@ -88,7 +115,7 @@ _DATE_PARAMS: Final = { "end_date": {"type": "string", "description": "End date in YYYY-MM-DD format"}, } -_TOOL_USAGE: Final = { +_TOOL_USAGE: Final[_ToolDef] = { "type": "function", "function": { "name": "get_usage_data", @@ -111,7 +138,7 @@ _TOOL_USAGE: Final = { }, } -_TOOL_TEAM: Final = { +_TOOL_TEAM: Final[_ToolDef] = { "type": "function", "function": { "name": "get_team_usage_data", @@ -133,7 +160,7 @@ _TOOL_TEAM: Final = { }, } -_TOOL_TAG: Final = { +_TOOL_TAG: Final[_ToolDef] = { "type": "function", "function": { "name": "get_tag_usage_data", @@ -159,7 +186,7 @@ TOOLS_BASE: Final = [_TOOL_USAGE] TOOLS_ADMIN: Final = [_TOOL_USAGE, _TOOL_TEAM, _TOOL_TAG] -def get_tools_for_role(is_admin: bool) -> list[dict[str, Any]]: +def get_tools_for_role(is_admin: bool) -> list[_ToolDef]: """Return the tool list appropriate for the user's role.""" return TOOLS_ADMIN if is_admin else TOOLS_BASE @@ -254,7 +281,7 @@ async def _query_activity( ) -async def _fetch_usage_data(start_date: str, end_date: str, user_id: str | None = None) -> dict[str, Any]: +async def _fetch_usage_data(start_date: str, end_date: str, user_id: str | None = None) -> _UsageDump: resp: Final = await _query_activity( TABLE_DAILY_USER_SPEND, ENTITY_FIELD_USER, @@ -266,7 +293,7 @@ async def _fetch_usage_data(start_date: str, end_date: str, user_id: str | None return resp.model_dump(mode="json") -async def _fetch_team_usage_data(start_date: str, end_date: str, team_ids: str | None = None) -> dict[str, Any]: +async def _fetch_team_usage_data(start_date: str, end_date: str, team_ids: str | None = None) -> _UsageDump: resp: Final = await _query_activity( TABLE_DAILY_TEAM_SPEND, ENTITY_FIELD_TEAM, @@ -277,7 +304,7 @@ async def _fetch_team_usage_data(start_date: str, end_date: str, team_ids: str | return resp.model_dump(mode="json") -async def _fetch_tag_usage_data(start_date: str, end_date: str, tags: str | None = None) -> dict[str, Any]: +async def _fetch_tag_usage_data(start_date: str, end_date: str, tags: str | None = None) -> _UsageDump: resp: Final = await _query_activity( TABLE_DAILY_TAG_SPEND, ENTITY_FIELD_TAG, @@ -294,7 +321,7 @@ async def _fetch_tag_usage_data(start_date: str, end_date: str, tags: str | None def _accumulate_breakdown( - results: list[dict[str, Any]], dimension: str, fields: list[str] + results: Sequence[_DayDump], dimension: str, fields: Sequence[str] ) -> dict[str, dict[str, float]]: """Aggregate a single breakdown dimension across days.""" totals: Final[dict[str, dict[str, float]]] = {} @@ -317,7 +344,7 @@ def _ranked_lines( return [fmt(name, vals) for name, vals in sorted(totals.items(), key=lambda x: -x[1].get("spend", 0))[:limit]] -def _summarise_usage_data(data: dict[str, Any]) -> str: +def _summarise_usage_data(data: _UsageDump) -> str: meta: Final = data.get("metadata", {}) results: Final = data.get("results", []) @@ -349,7 +376,7 @@ def _summarise_usage_data(data: dict[str, Any]) -> str: return "\n".join(sections) -def _summarise_entity_data(data: dict[str, Any], entity_label: str) -> str: +def _summarise_entity_data(data: _UsageDump, entity_label: str) -> str: """Summarise team/tag entity usage data.""" results: Final = data.get("results", []) if not results: @@ -409,16 +436,16 @@ def _sse(event: SSEEvent) -> str: def _resolve_fetch_kwargs( fn_name: str, - fn_args: dict[str, str], + fn_args: Mapping[str, str], user_id: str | None, is_admin: bool, -) -> dict[str, Any]: +) -> dict[str, str]: """Build keyword arguments for a tool's fetch function.""" start_date: Final = fn_args.get("start_date", "") end_date: Final = fn_args.get("end_date", "") if not start_date or not end_date: raise ValueError("Missing required start_date or end_date from tool arguments") - kwargs: Final[dict[str, Any]] = {"start_date": start_date, "end_date": end_date} + kwargs: Final[dict[str, str]] = {"start_date": start_date, "end_date": end_date} if fn_name == "get_usage_data": if not is_admin: if user_id is None: @@ -443,7 +470,7 @@ def _resolve_fetch_kwargs( async def _execute_tool_call( handler: ToolHandler, fn_name: str, - fn_args: dict[str, str], + fn_args: Mapping[str, str], user_id: str | None, is_admin: bool, ) -> str: @@ -455,13 +482,13 @@ async def _execute_tool_call( async def _process_tool_call( tc: Any, - chat_messages: list[dict[str, Any]], + chat_messages: list[Mapping[str, object]], user_id: str | None, is_admin: bool, ) -> AsyncIterator[str]: """Execute a single tool call, yielding SSE events for status.""" - fn_name: Final = tc.function.name - fn_args: Final = json.loads(tc.function.arguments) + fn_name: Final[str] = tc.function.name + fn_args: Final[Mapping[str, str]] = json.loads(tc.function.arguments) allowed_names: Final = {t["function"]["name"] for t in get_tools_for_role(is_admin)} handler: Final = TOOL_HANDLERS.get(fn_name) @@ -495,7 +522,7 @@ async def _process_tool_call( chat_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_result}) -async def _stream_final_response(model: str, chat_messages: list[dict[str, Any]]) -> AsyncIterator[str]: +async def _stream_final_response(model: str, chat_messages: list[Mapping[str, object]]) -> AsyncIterator[str]: """Stream the final LLM response after tool results are appended.""" yield _sse({"type": "status", "message": "Analyzing results..."}) @@ -520,7 +547,7 @@ async def stream_usage_ai_chat( """Stream SSE events: status → tool_call → chunk → done.""" resolved_model: Final = (model or "").strip() or DEFAULT_COMPETITOR_DISCOVERY_MODEL truncated: Final = messages[-MAX_CHAT_MESSAGES:] if len(messages) > MAX_CHAT_MESSAGES else messages - chat_messages: Final[list[dict[str, Any]]] = [ + chat_messages: Final[list[Mapping[str, object]]] = [ {"role": "system", "content": _build_system_prompt(is_admin)}, *truncated, ] diff --git a/litellm/proxy/management_endpoints/user_agent_analytics_endpoints.py b/litellm/proxy/management_endpoints/user_agent_analytics_endpoints.py index 09de170d542..0422c72cdb3 100644 --- a/litellm/proxy/management_endpoints/user_agent_analytics_endpoints.py +++ b/litellm/proxy/management_endpoints/user_agent_analytics_endpoints.py @@ -11,11 +11,19 @@ These endpoints use optimized single SQL queries with joins to efficiently calcu user metrics from tag activity data and return time series for dashboard visualization. """ +from collections.abc import Mapping, Sequence from datetime import datetime, timedelta -from typing import Any, Final +from typing import TYPE_CHECKING, Final, Protocol, TypeVar, overload from fastapi import APIRouter, Depends, HTTPException, Query -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter + +if TYPE_CHECKING: + from prisma.models import LiteLLM_DailyTagSpend as PrismaDailyTagSpendRow + from prisma.models import LiteLLM_UserTable as PrismaUserRow + from prisma.models import LiteLLM_VerificationToken as PrismaVerificationTokenRow + + from litellm.proxy.utils import PrismaClient from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -103,6 +111,54 @@ class PerUserAnalyticsResponse(BaseModel): total_pages: int +class _DistinctTagRow(BaseModel): + tag: str + + +class _ActiveUsersRow(BaseModel): + tag: str + active_users: int + date: str + period_start: str | None = None + period_end: str | None = None + + +class _TagSummaryRow(BaseModel): + tag: str + unique_users: int | None = None + total_requests: float | int | str | None = None + successful_requests: float | int | str | None = None + failed_requests: float | int | str | None = None + total_tokens: float | int | str | None = None + total_spend: float | int | str | None = None + + +_DISTINCT_TAG_ROWS: Final = TypeAdapter(list[_DistinctTagRow]) +_ACTIVE_USERS_ROWS: Final = TypeAdapter(list[_ActiveUsersRow]) +_TAG_SUMMARY_ROWS: Final = TypeAdapter(list[_TagSummaryRow]) + +_RowT_co: Final = TypeVar("_RowT_co", covariant=True) + +if TYPE_CHECKING: + + class _TableOps(Protocol[_RowT_co]): + async def find_many(self, where: Mapping[str, object] | None = None) -> Sequence[_RowT_co]: ... + + +@overload +def _typed_table(repo: DailyTagSpendRepository) -> "_TableOps[PrismaDailyTagSpendRow]": ... +@overload +def _typed_table(repo: VerificationTokenRepository) -> "_TableOps[PrismaVerificationTokenRow]": ... +@overload +def _typed_table(repo: UserRepository) -> "_TableOps[PrismaUserRow]": ... +def _typed_table(repo: DailyTagSpendRepository | VerificationTokenRepository | UserRepository) -> object: + return repo.table + + +async def _query_raw(prisma_client: "PrismaClient", sql_query: str, *params: object) -> object: + return await prisma_client.db.query_raw(sql_query, *params) + + @router.get( "/tag/distinct", response_model=DistinctTagsResponse, @@ -141,9 +197,9 @@ async def get_distinct_user_agent_tags( LIMIT {MAX_TAGS} """ - db_response: Final = await prisma_client.db.query_raw(sql_query) + db_response: Final = _DISTINCT_TAG_ROWS.validate_python(await _query_raw(prisma_client, sql_query)) - results: Final = [DistinctTagResponse(tag=row["tag"]) for row in db_response] + results: Final = [DistinctTagResponse(tag=row.tag) for row in db_response] return DistinctTagsResponse(results=results) @@ -231,11 +287,10 @@ async def get_daily_active_users( ORDER BY dts.date DESC, active_users DESC """ - db_response: Final = await prisma_client.db.query_raw(sql_query, *params) + db_response: Final = _ACTIVE_USERS_ROWS.validate_python(await _query_raw(prisma_client, sql_query, *params)) results: Final = [ - TagActiveUsersResponse(tag=row["tag"], active_users=row["active_users"], date=row["date"]) - for row in db_response + TagActiveUsersResponse(tag=row.tag, active_users=row.active_users, date=row.date) for row in db_response ] return ActiveUsersAnalyticsResponse(results=results) @@ -346,15 +401,15 @@ async def get_weekly_active_users( ORDER BY week_offset DESC, active_users DESC """ - db_response: Final = await prisma_client.db.query_raw(sql_query, *params) + db_response: Final = _ACTIVE_USERS_ROWS.validate_python(await _query_raw(prisma_client, sql_query, *params)) results: Final = [ TagActiveUsersResponse( - tag=row["tag"], - active_users=row["active_users"], - date=row["date"], # This will be "Week 1 (Jan 15)", "Week 2 (Jan 8)", etc. - period_start=row["period_start"], - period_end=row["period_end"], + tag=row.tag, + active_users=row.active_users, + date=row.date, # This will be "Week 1 (Jan 15)", "Week 2 (Jan 8)", etc. + period_start=row.period_start, + period_end=row.period_end, ) for row in db_response ] @@ -467,15 +522,15 @@ async def get_monthly_active_users( ORDER BY month_offset DESC, active_users DESC """ - db_response: Final = await prisma_client.db.query_raw(sql_query, *params) + db_response: Final = _ACTIVE_USERS_ROWS.validate_python(await _query_raw(prisma_client, sql_query, *params)) results: Final = [ TagActiveUsersResponse( - tag=row["tag"], - active_users=row["active_users"], - date=row["date"], # This will be "Month 1 (Jan)", "Month 2 (Dec)", etc. - period_start=row["period_start"], - period_end=row["period_end"], + tag=row.tag, + active_users=row.active_users, + date=row.date, # This will be "Month 1 (Jan)", "Month 2 (Dec)", etc. + period_start=row.period_start, + period_end=row.period_end, ) for row in db_response ] @@ -565,17 +620,17 @@ async def get_tag_summary( ORDER BY total_requests DESC """ - db_response: Final = await prisma_client.db.query_raw(sql_query, *params) + db_response: Final = _TAG_SUMMARY_ROWS.validate_python(await _query_raw(prisma_client, sql_query, *params)) results: Final = [ TagSummaryMetrics( - tag=row["tag"], - unique_users=row["unique_users"] or 0, - total_requests=int(row["total_requests"] or 0), - successful_requests=int(row["successful_requests"] or 0), - failed_requests=int(row["failed_requests"] or 0), - total_tokens=int(row["total_tokens"] or 0), - total_spend=float(row["total_spend"] or 0.0), + tag=row.tag, + unique_users=row.unique_users or 0, + total_requests=int(row.total_requests or 0), + successful_requests=int(row.successful_requests or 0), + failed_requests=int(row.failed_requests or 0), + total_tokens=int(row.total_tokens or 0), + total_spend=float(row.total_spend or 0.0), ) for row in db_response ] @@ -648,7 +703,7 @@ async def get_per_user_analytics( start_date: Final = start_dt.strftime("%Y-%m-%d") # Build where clause with date range - where_clause: Final[dict[str, Any]] = {"date": {"gte": start_date, "lte": end_date}} + where_clause: Final[dict[str, object]] = {"date": {"gte": start_date, "lte": end_date}} # Add tag filtering if provided if tag_filters and len(tag_filters) > 0: @@ -657,7 +712,7 @@ async def get_per_user_analytics( where_clause["tag"] = {"contains": tag_filter} # Get all tag records in the date range with optional tag filtering - tag_records: Final = await DailyTagSpendRepository(prisma_client).table.find_many(where=where_clause) + tag_records: Final = await _typed_table(DailyTagSpendRepository(prisma_client)).find_many(where=where_clause) # Get unique api_keys api_keys: Final = set(record.api_key for record in tag_records if record.api_key) @@ -672,7 +727,7 @@ async def get_per_user_analytics( ) # Lookup user_id for each api_key - api_key_records: Final = await VerificationTokenRepository(prisma_client).table.find_many( + api_key_records: Final = await _typed_table(VerificationTokenRepository(prisma_client)).find_many( where={"token": {"in": list(api_keys)}} ) @@ -681,7 +736,9 @@ async def get_per_user_analytics( # Get user emails for the user_ids user_ids: Final = list(set(api_key_to_user_id.values())) - user_records: Final = await UserRepository(prisma_client).table.find_many(where={"user_id": {"in": user_ids}}) + user_records: Final = await _typed_table(UserRepository(prisma_client)).find_many( + where={"user_id": {"in": user_ids}} + ) # Create mapping from user_id to user_email user_id_to_email: Final = {record.user_id: record.user_email for record in user_records} diff --git a/litellm/proxy/prompts/prompt_endpoints.py b/litellm/proxy/prompts/prompt_endpoints.py index d8e9f8dfaee..1d71ea658e4 100644 --- a/litellm/proxy/prompts/prompt_endpoints.py +++ b/litellm/proxy/prompts/prompt_endpoints.py @@ -3,8 +3,10 @@ CRUD ENDPOINTS FOR PROMPTS """ import tempfile +from collections.abc import Awaitable, Mapping, Sequence +from datetime import datetime from pathlib import Path -from typing import Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, Protocol, cast from fastapi import ( APIRouter, @@ -38,9 +40,68 @@ from litellm.types.prompts.init_prompts import ( ) from litellm.types.proxy.prompt_endpoints import TestPromptRequest +if TYPE_CHECKING: + from litellm.proxy.prompts.prompt_registry import InMemoryPromptRegistry + from litellm.proxy.utils import PrismaClient + router: Final = APIRouter() +class _PromptRow(Protocol): + @property + def id(self) -> str: ... + @property + def prompt_id(self) -> str: ... + @property + def version(self) -> int: ... + @property + def environment(self) -> str: ... + @property + def created_by(self) -> str | None: ... + @property + def created_at(self) -> "datetime": ... + @property + def updated_at(self) -> "datetime": ... + @property + def litellm_params(self) -> str | Mapping[str, object]: ... + @property + def prompt_info(self) -> str | Mapping[str, object] | None: ... + + def model_dump(self) -> Mapping[str, object]: ... + + +class _PromptRowData(BaseModel): + prompt_id: str + version: int = 1 + environment: str = "development" + created_by: str | None = None + litellm_params: str | Mapping[str, object] | None = None + prompt_info: str | Mapping[str, object] | None = None + created_at: datetime | None = None + updated_at: datetime | None = None + + +class _PromptTableActions(Protocol): + def find_many( + self, + *, + where: Mapping[str, str | int], + order: Mapping[str, str] = ..., + take: int = ..., + distinct: Sequence[str] = ..., + ) -> Awaitable[Sequence[_PromptRow]]: ... + + def create(self, *, data: Mapping[str, str | int | None]) -> Awaitable[_PromptRow]: ... + + def update(self, *, where: Mapping[str, str | int], data: Mapping[str, str]) -> Awaitable[_PromptRow]: ... + + def delete_many(self, *, where: Mapping[str, str]) -> Awaitable[int]: ... + + +def _prompt_table(prisma_client: "PrismaClient") -> _PromptTableActions: + return PromptRepository(prisma_client).table + + def get_base_prompt_id(prompt_id: str) -> str: """ Extract the base prompt ID by stripping the version suffix if present. @@ -132,7 +193,7 @@ def construct_versioned_prompt_id(prompt_id: str, version: int | None = None) -> return f"{base_id}.v{version}" -def get_latest_version_prompt_id(prompt_id: str, all_prompt_ids: dict[str, Any]) -> str: +def get_latest_version_prompt_id(prompt_id: str, all_prompt_ids: Mapping[str, object]) -> str: """ Find the latest version of a prompt from available prompt IDs. @@ -198,7 +259,9 @@ def get_latest_prompt_versions(prompts: list[PromptSpec]) -> list[PromptSpec]: return list(latest_prompts.values()) -async def get_next_version_for_prompt(prisma_client, prompt_id: str, environment: str = "development") -> int: +async def get_next_version_for_prompt( + prisma_client: "PrismaClient", prompt_id: str, environment: str = "development" +) -> int: """ Get the next version number for a prompt in a specific environment. @@ -210,7 +273,7 @@ async def get_next_version_for_prompt(prisma_client, prompt_id: str, environment Returns: Next version number (1 if no versions exist, max_version + 1 otherwise) """ - existing_prompts: Final = await PromptRepository(prisma_client).table.find_many( + existing_prompts: Final = await _prompt_table(prisma_client).find_many( where={"prompt_id": prompt_id, "environment": environment} ) @@ -221,7 +284,7 @@ async def get_next_version_for_prompt(prisma_client, prompt_id: str, environment return 1 -def create_versioned_prompt_spec(db_prompt) -> PromptSpec: +def create_versioned_prompt_spec(db_prompt: _PromptRow) -> PromptSpec: """ Helper function to create a PromptSpec with versioned prompt_id from a DB prompt entry. @@ -235,38 +298,33 @@ def create_versioned_prompt_spec(db_prompt) -> PromptSpec: from litellm.types.prompts.init_prompts import PromptLiteLLMParams - prompt_dict: Final = db_prompt.model_dump() - base_prompt_id: Final = prompt_dict["prompt_id"] - version: Final = prompt_dict.get("version", 1) - environment: Final = prompt_dict.get("environment", "development") - created_by: Final = prompt_dict.get("created_by") + row: Final = _PromptRowData.model_validate(db_prompt.model_dump()) - # Parse litellm_params - litellm_params_data = prompt_dict.get("litellm_params") - if isinstance(litellm_params_data, str): - litellm_params_data = json.loads(litellm_params_data) - litellm_params: Final = PromptLiteLLMParams(**litellm_params_data) + litellm_params_data: Final = row.litellm_params + litellm_params_dict: Final[Mapping[str, object] | None] = ( + json.loads(litellm_params_data) if isinstance(litellm_params_data, str) else litellm_params_data + ) + litellm_params: Final = PromptLiteLLMParams.model_validate(litellm_params_dict) - # Parse prompt_info - prompt_info_data = prompt_dict.get("prompt_info") + prompt_info_data: Final = row.prompt_info if prompt_info_data: - if isinstance(prompt_info_data, str): - prompt_info_data = json.loads(prompt_info_data) - prompt_info = PromptInfo(**prompt_info_data) + prompt_info_dict: Final[Mapping[str, object]] = ( + json.loads(prompt_info_data) if isinstance(prompt_info_data, str) else prompt_info_data + ) + prompt_info = PromptInfo.model_validate(prompt_info_dict) else: prompt_info = PromptInfo(prompt_type="db") - # Create versioned prompt_id - versioned_prompt_id: Final = f"{base_prompt_id}.v{version}" + versioned_prompt_id: Final = f"{row.prompt_id}.v{row.version}" return PromptSpec( prompt_id=versioned_prompt_id, litellm_params=litellm_params, prompt_info=prompt_info, - created_at=prompt_dict.get("created_at"), - updated_at=prompt_dict.get("updated_at"), - environment=environment, - created_by=created_by, + created_at=row.created_at, + updated_at=row.updated_at, + environment=row.environment, + created_by=row.created_by, ) @@ -431,10 +489,10 @@ async def get_prompt_versions( # Query DB for versions versioned_prompts: Final = [] if prisma_client is not None: - where_clause: Final[dict[str, Any]] = {"prompt_id": base_prompt_id} + where_clause: Final[dict[str, str]] = {"prompt_id": base_prompt_id} if environment: where_clause["environment"] = environment - db_prompts: Final = await PromptRepository(prisma_client).table.find_many( + db_prompts: Final = await _prompt_table(prisma_client).find_many( where=where_clause, order={"version": "desc"}, ) @@ -590,7 +648,7 @@ async def get_prompt_info( # Query all environments this prompt exists in (lightweight: distinct on environment) all_environments: list[str] = [] if prisma_client is not None: - all_prompt_rows: Final = await PromptRepository(prisma_client).table.find_many( + all_prompt_rows: Final = await _prompt_table(prisma_client).find_many( where={"prompt_id": base_prompt_id}, distinct=["environment"], ) @@ -602,13 +660,13 @@ async def get_prompt_info( prompt_spec = None requested_version: Final = get_version_number(prompt_id=prompt_id) if prompt_id != base_prompt_id else None if environment and prisma_client is not None: - where_clause: Final[dict[str, Any]] = { + where_clause: Final[dict[str, str | int]] = { "prompt_id": base_prompt_id, "environment": environment, } if requested_version is not None: where_clause["version"] = requested_version - env_prompts: Final = await PromptRepository(prisma_client).table.find_many( + env_prompts: Final = await _prompt_table(prisma_client).find_many( where=where_clause, order={"version": "desc"}, take=1, @@ -721,7 +779,7 @@ async def create_prompt( ) # Store prompt in db with version - prompt_db_entry: Final = await PromptRepository(prisma_client).table.create( + prompt_db_entry: Final = await _prompt_table(prisma_client).create( data={ "prompt_id": request.prompt_id, "version": new_version, @@ -811,7 +869,7 @@ async def update_prompt( ) # Check if any version of this prompt exists (in any environment) - existing_prompts = await PromptRepository(prisma_client).table.find_many(where={"prompt_id": base_prompt_id}) + existing_prompts = await _prompt_table(prisma_client).find_many(where={"prompt_id": base_prompt_id}) if not existing_prompts: raise HTTPException( @@ -835,7 +893,7 @@ async def update_prompt( ) # Store new version in db - prompt_db_entry: Final = await PromptRepository(prisma_client).table.create( + prompt_db_entry: Final = await _prompt_table(prisma_client).create( data={ "prompt_id": base_prompt_id, "version": new_version, @@ -936,12 +994,12 @@ async def delete_prompt( base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id) # Build delete filter; scope to environment if provided - delete_where: Final[dict[str, Any]] = {"prompt_id": base_prompt_id} + delete_where: Final[dict[str, str]] = {"prompt_id": base_prompt_id} if environment: delete_where["environment"] = environment # Delete versions from the database (scoped to environment if provided) - await PromptRepository(prisma_client).table.delete_many(where=delete_where) + await _prompt_table(prisma_client).delete_many(where=delete_where) # Remove matching prompts from memory — scope to environment if provided if environment: @@ -967,7 +1025,9 @@ async def delete_prompt( raise HTTPException(status_code=500, detail=str(e)) -def _reload_prompt_in_registry(registry: Any, versioned_id: str, updated_prompt_spec: PromptSpec) -> PromptSpec: +def _reload_prompt_in_registry( + registry: "InMemoryPromptRegistry", versioned_id: str, updated_prompt_spec: PromptSpec +) -> PromptSpec: """Remove stale entry and re-initialize the prompt in the in-memory registry.""" if versioned_id in registry.IN_MEMORY_PROMPTS: del registry.IN_MEMORY_PROMPTS[versioned_id] @@ -1033,14 +1093,14 @@ async def patch_prompt( requested_version: Final = get_version_number(prompt_id=prompt_id) if prompt_id != base_prompt_id else None # Build query to find the exact row by composite unique key - find_where: Final[dict[str, Any]] = { + find_where: Final[dict[str, str | int]] = { "prompt_id": base_prompt_id, "environment": env, } if requested_version is not None: find_where["version"] = requested_version - db_rows: Final = await PromptRepository(prisma_client).table.find_many( + db_rows: Final = await _prompt_table(prisma_client).find_many( where=find_where, order={"version": "desc"}, take=1, @@ -1084,7 +1144,7 @@ async def patch_prompt( raise HTTPException(status_code=400, detail="litellm_params cannot be None") # Build update data dict - update_data: Final[dict[str, Any]] = { + update_data: Final[dict[str, str]] = { "litellm_params": updated_litellm_params.model_dump_json(), "prompt_info": updated_prompt_info.model_dump_json(), } @@ -1092,7 +1152,7 @@ async def patch_prompt( update_data["created_by"] = user_api_key_dict.user_id # Update by primary key (id) to target exactly one row - updated_prompt_db_entry: Final = await PromptRepository(prisma_client).table.update( + updated_prompt_db_entry: Final = await _prompt_table(prisma_client).update( where={"id": target_row.id}, data=update_data, ) @@ -1216,7 +1276,7 @@ async def test_prompt( # Use ProxyBaseLLMRequestProcessing to go through all proxy logic base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data) - result: Final = await base_llm_response_processor.base_process_llm_request( + result: Final[object] = await base_llm_response_processor.base_process_llm_request( request=fastapi_request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 79791607b2e..47e30555a4f 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -1,10 +1,12 @@ import json import os import re +from collections.abc import Awaitable, Mapping, Sequence from importlib.resources import files -from typing import Any, Final +from typing import TYPE_CHECKING, Final, Protocol from fastapi import APIRouter, HTTPException, Request +from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -32,14 +34,66 @@ from litellm.types.proxy.public_endpoints.public_endpoints import ( ) from litellm.types.utils import LlmProviders +if TYPE_CHECKING: + from datetime import datetime + router: Final = APIRouter() +class _ProviderSupportEntry(TypedDict, total=False): + display_name: ReadOnly[str] + endpoints: ReadOnly[Mapping[str, bool]] + + +class _ProvidersFile(TypedDict, total=False): + providers: ReadOnly[Mapping[str, _ProviderSupportEntry]] + + +class _EndpointProviderEntry(TypedDict): + slug: ReadOnly[str] + display_name: ReadOnly[str] + + +class _EndpointEntry(TypedDict): + key: ReadOnly[str] + label: ReadOnly[str] + endpoint: ReadOnly[str] + providers: ReadOnly[Sequence[_EndpointProviderEntry]] + + +class _PluginRow(Protocol): + @property + def id(self) -> str: ... + + @property + def name(self) -> str: ... + + @property + def enabled(self) -> bool: ... + + @property + def created_at(self) -> "datetime | None": ... + + @property + def updated_at(self) -> "datetime | None": ... + + @property + def manifest_json(self) -> str | None: ... + + +class _PluginTableActions(Protocol): + def find_many(self, *, where: Mapping[str, bool]) -> Awaitable[Sequence[_PluginRow]]: ... + + +def _plugin_table(prisma_client: object) -> _PluginTableActions: + return ClaudeCodePluginRepository(prisma_client).table + + # --------------------------------------------------------------------------- # /public/endpoints — helpers # --------------------------------------------------------------------------- -_ENDPOINT_METADATA: Final[dict[str, dict[str, str]]] = { +_ENDPOINT_METADATA: Final[Mapping[str, Mapping[str, str]]] = { "chat_completions": {"label": "Chat Completions", "endpoint": "/chat/completions"}, "messages": {"label": "Messages", "endpoint": "/messages"}, "responses": {"label": "Responses", "endpoint": "/responses"}, @@ -108,12 +162,12 @@ def _clean_display_name(raw: str) -> str: return _SLUG_SUFFIX_RE.sub("", raw).strip() -def _build_endpoints(raw: dict[str, Any]) -> list[dict[str, Any]]: +def _build_endpoints(raw: _ProvidersFile) -> list[_EndpointEntry]: """Transform raw provider_endpoints_support_backup.json into the response shape.""" - providers: Final[dict[str, Any]] = raw.get("providers", {}) + providers: Final = raw.get("providers", {}) # Collect endpoint keys in insertion order (union across all providers). - seen: Final[set] = set() + seen: Final[set[str]] = set() all_keys: Final[list[str]] = [] for provider_data in providers.values(): for key in provider_data.get("endpoints", {}): @@ -121,13 +175,13 @@ def _build_endpoints(raw: dict[str, Any]) -> list[dict[str, Any]]: seen.add(key) all_keys.append(key) - result: Final[list[dict[str, Any]]] = [] + result: Final[list[_EndpointEntry]] = [] for key in all_keys: meta = _ENDPOINT_METADATA.get(key) label = meta["label"] if meta else key.replace("_", " ").title() path = meta["endpoint"] if meta else "/" + key.replace("_", "/") - supporting: list[dict[str, str]] = [ + supporting: list[_EndpointProviderEntry] = [ { "slug": slug, "display_name": _clean_display_name(pd.get("display_name", slug)), @@ -140,8 +194,10 @@ def _build_endpoints(raw: dict[str, Any]) -> list[dict[str, Any]]: return result -def _load_endpoints() -> list[dict[str, Any]]: - raw = json.loads(files("litellm").joinpath("provider_endpoints_support_backup.json").read_text(encoding="utf-8")) +def _load_endpoints() -> list[_EndpointEntry]: + raw: Final[_ProvidersFile] = json.loads( + files("litellm").joinpath("provider_endpoints_support_backup.json").read_text(encoding="utf-8") + ) return _build_endpoints(raw) @@ -235,12 +291,7 @@ async def get_mcp_servers(): ) public_mcp_servers: Final = global_mcp_server_manager.get_public_mcp_servers() - return [ - MCPPublicServer( - **server.model_dump(), - ) - for server in public_mcp_servers - ] + return [MCPPublicServer.model_validate(server.model_dump()) for server in public_mcp_servers] @router.get( @@ -259,7 +310,7 @@ async def public_skill_hub(): try: prisma_client: Final = await _get_prisma_client() - plugins: Final = await ClaudeCodePluginRepository(prisma_client).table.find_many(where={"enabled": True}) + plugins: Final = await _plugin_table(prisma_client).find_many(where={"enabled": True}) items: Final = [] for plugin in plugins: raw = plugin.manifest_json or {} diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index e7ae6031e45..9e2b1c9d82d 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -7,7 +7,8 @@ Provides: """ import base64 -from typing import Any, Final +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Final import orjson from fastapi import APIRouter, Depends, HTTPException, Request, Response, status @@ -31,6 +32,9 @@ from litellm.proxy.vector_store_endpoints.utils import ( ) from litellm.repositories.table_repositories import ManagedVectorStoresRepository +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + router: Final = APIRouter() @@ -58,7 +62,7 @@ def _append_payload_to_scan_stack( payload_stack.append((value, next_depth)) -def _collect_vector_store_ids_from_payload(payload: Any) -> set[str]: +def _collect_vector_store_ids_from_payload(payload: object) -> set[str]: vector_store_ids: Final[set[str]] = set() payload_stack: Final = [(payload, 0)] @@ -95,7 +99,7 @@ def _collect_vector_store_ids_from_payload(payload: Any) -> set[str]: async def _authorize_nested_vector_store_ids( - payload: Any, + payload: object, user_api_key_dict: UserAPIKeyAuth, ) -> None: for vector_store_id in sorted(_collect_vector_store_ids_from_payload(payload)): @@ -109,7 +113,7 @@ def _build_file_metadata_entry( response: Any, file_data: tuple[str, bytes, str] | None = None, file_url: str | None = None, -) -> dict[str, Any]: +) -> Mapping[str, str | int | None]: """ Build a file metadata entry for storing in vector_store_metadata. @@ -159,8 +163,8 @@ def _build_file_metadata_entry( async def _save_vector_store_to_db_from_rag_ingest( response: Any, - ingest_options: dict[str, Any], - prisma_client, + ingest_options: Mapping[str, dict[str, str | None]], + prisma_client: "PrismaClient", user_api_key_dict: UserAPIKeyAuth, file_data: tuple[str, bytes, str] | None = None, file_url: str | None = None, @@ -299,9 +303,9 @@ async def parse_rag_ingest_request( headers: Final = _safe_get_request_headers(request) content_type = headers.get("content-type", "") - file_data = None - file_url = None - file_id = None + file_data: tuple[str, bytes, str] | None = None + file_url: str | None = None + file_id: str | None = None ingest_options: dict[str, Any] = {} if "multipart/form-data" in content_type: @@ -315,7 +319,7 @@ async def parse_rag_ingest_request( file_data = (file_obj.filename, file_content, file_obj.content_type) # Parse JSON from 'request' form field (contains full request body as JSON) - request_json_str: Final = form_data.get("request") + request_json_str: Final[str | bytes | None] = form_data.get("request") if request_json_str: request_data: Final = orjson.loads(request_json_str) ingest_options = request_data.get("ingest_options", {}) @@ -382,7 +386,7 @@ async def parse_rag_ingest_request( "api_key", "api_base", } - vector_store_opts: Final = ingest_options.get("vector_store", {}) + vector_store_opts: Final[object] = ingest_options.get("vector_store", {}) if isinstance(vector_store_opts, dict): for field in _BLOCKED_VECTOR_STORE_CREDENTIAL_PARAMS: if field in vector_store_opts: @@ -658,7 +662,7 @@ async def rag_query( ) # Add litellm data - request_data: dict[str, Any] = {} + request_data: dict[str, object] = {} request_data = await add_litellm_data_to_request( data=request_data, request=request, diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index 383ada5a1bc..31ab3596418 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -10,9 +10,10 @@ https://platform.openai.com/docs/api-reference/responses-streaming import asyncio import json -from typing import Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, cast from fastapi import Request, Response +from fastapi.responses import StreamingResponse from litellm._logging import verbose_proxy_logger from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth @@ -20,25 +21,30 @@ from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessin from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler from litellm.types.llms.openai import ResponsesAPIStatus +if TYPE_CHECKING: + from litellm.proxy.proxy_server import ProxyConfig + from litellm.proxy.utils import ProxyLogging + from litellm.router import Router + async def background_streaming_task( polling_id: str, - data: dict, + data, polling_handler: ResponsePollingHandler, request: Request, fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth, - general_settings: dict, - llm_router, - proxy_config, - proxy_logging_obj, + general_settings, + llm_router: "Router | None", + proxy_config: "ProxyConfig", + proxy_logging_obj: "ProxyLogging", select_data_generator, user_model, - user_temperature, - user_request_timeout, - user_max_tokens, - user_api_base, - version, + user_temperature: float | None, + user_request_timeout: float | None, + user_max_tokens: int | None, + user_api_base: str | None, + version: str | None, ): """ Background task to stream response and update cache @@ -69,7 +75,7 @@ async def background_streaming_task( # Make streaming request. # Pre-call checks (rate limits, guardrails, budget) were already run # before polling ID creation, so skip them here to avoid double-counting. - response: Final = await processor.base_process_llm_request( + response: Final[StreamingResponse] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, diff --git a/litellm/responses/file_search/emulated_handler.py b/litellm/responses/file_search/emulated_handler.py index 7854b17a06f..e9e7ae908a5 100644 --- a/litellm/responses/file_search/emulated_handler.py +++ b/litellm/responses/file_search/emulated_handler.py @@ -14,16 +14,19 @@ Flow: import json import time import uuid -from collections.abc import Iterable -from typing import Any, Final, cast +from collections.abc import Iterable, Sequence +from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast from litellm._internal_context import is_internal_call from litellm._logging import verbose_logger from litellm.types.llms.openai import ResponseOutputItem, ResponsesAPIResponse from litellm.types.vector_stores import VectorStoreSearchResult +if TYPE_CHECKING: + from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig + # Keep ToolParam broad so we stay compatible with both dict and Pydantic forms -ToolParam = Any +ToolParam: TypeAlias = object FILE_SEARCH_FUNCTION_NAME: Final = "litellm_file_search" @@ -35,7 +38,7 @@ FILE_SEARCH_FUNCTION_NAME: Final = "litellm_file_search" def should_use_emulated_file_search( tools: Iterable[ToolParam] | None, - provider_config: Any, # BaseResponsesAPIConfig + provider_config: "BaseResponsesAPIConfig | None", ) -> bool: """Return True when there is a file_search tool and the provider can't handle it natively.""" if not tools: @@ -51,7 +54,7 @@ def should_use_emulated_file_search( # --------------------------------------------------------------------------- -def _build_function_tool(vector_store_ids: list[str]) -> dict[str, Any]: +def _build_function_tool(vector_store_ids: list[str]) -> dict[str, object]: """ Create a Responses API function-tool definition that describes file search. The function accepts one or more natural-language queries (like OpenAI's native @@ -96,14 +99,14 @@ def _build_function_tool(vector_store_ids: list[str]) -> dict[str, Any]: def _replace_file_search_tools( tools: Iterable[ToolParam] | None, -) -> tuple[list[dict[str, Any]], list[str]]: +) -> tuple[list[object], list[str]]: """ Replace all file_search tools with a single function tool. Returns: (new_tools_list, all_vector_store_ids) """ - non_file_search: Final[list[dict[str, Any]]] = [] + non_file_search: Final[list[object]] = [] vector_store_ids: Final[list[str]] = [] for tool in tools or []: @@ -172,7 +175,7 @@ async def _run_vector_searches( # --------------------------------------------------------------------------- -def _get_field(result: Any, key: str, default: Any = None) -> Any: +def _get_field(result: object, key: str, default: object = None) -> Any: """Read a field from either a dict/TypedDict or an attribute-based object.""" if isinstance(result, dict): return result.get(key, default) @@ -211,7 +214,7 @@ def _format_search_results_as_tool_output( def _build_search_results_for_include( results: list[VectorStoreSearchResult], -) -> list[dict[str, Any]]: +) -> list[dict[str, object]]: """ Convert VectorStoreSearchResult objects to the format expected in file_search_call.search_results (mirrors OpenAI's include= format). @@ -220,7 +223,7 @@ def _build_search_results_for_include( behaviour of OpenAI's native file_search which surfaces every relevant chunk even when multiple chunks originate from the same document. """ - formatted: Final[list[dict[str, Any]]] = [] + formatted: Final[list[dict[str, object]]] = [] for result in results: file_id = _get_field(result, "file_id") or "" content_items = _get_field(result, "content") or [] @@ -243,7 +246,7 @@ def _build_file_search_call_output( queries: list[str], results: list[VectorStoreSearchResult] | None = None, include_search_results: bool = False, -) -> dict[str, Any]: +) -> dict[str, object]: """Build the file_search_call output item (mirrors OpenAI's format). Args: @@ -268,14 +271,14 @@ def _build_file_search_call_output( def _build_file_citation_annotations( results: list[VectorStoreSearchResult], text: str, -) -> list[dict[str, Any]]: +) -> list[dict[str, object]]: """ Build file_citation annotations for the text. Each result with a file_id gets a citation at the end of the text. """ - annotations: Final[list[dict[str, Any]]] = [] + annotations: Final[list[dict[str, object]]] = [] index: Final = len(text) # cite at end of text block - seen_file_ids: Final[set] = set() + seen_file_ids: Final[set[object]] = set() for result in results: file_id = _get_field(result, "file_id") @@ -298,7 +301,7 @@ def _build_file_citation_annotations( def _build_message_output( response_text: str, results: list[VectorStoreSearchResult], -) -> dict[str, Any]: +) -> dict[str, object]: """Build the message output item with optional file_citation annotations.""" annotations: Final = _build_file_citation_annotations(results, response_text) return { @@ -330,8 +333,8 @@ def _extract_text_from_responses_output(response: ResponsesAPIResponse) -> str: def _synthesize_responses_api_response( original_response: ResponsesAPIResponse, - file_search_call_output: dict[str, Any], - message_output: dict[str, Any], + file_search_call_output: dict[str, object], + message_output: dict[str, object], first_response: ResponsesAPIResponse | None = None, ) -> ResponsesAPIResponse: """ @@ -343,7 +346,7 @@ def _synthesize_responses_api_response( synthesized _hidden_params so that billing callbacks see the total cost of both provider calls that the emulated flow makes. """ - synthesized_output: Final[list[dict[str, Any]]] = [file_search_call_output, message_output] + synthesized_output: Final[list[dict[str, object]]] = [file_search_call_output, message_output] synthesized: Final = ResponsesAPIResponse( id=getattr(original_response, "id", f"resp_{uuid.uuid4().hex}"), object="response", @@ -383,12 +386,12 @@ async def _call_aresponses(input, model, tools, **kwargs): # pragma: no cover def _prepare_emulated_file_search_call( kwargs: dict[str, Any], -) -> tuple[bool, dict[str, Any]]: +) -> tuple[bool, dict[str, object]]: include_items: Final[list[str]] = list(kwargs.get("include") or []) include_search_results: Final = "file_search_call.results" in include_items original_stream: Final = kwargs.get("stream") - updated_kwargs = kwargs + updated_kwargs: dict[str, object] = kwargs if original_stream: verbose_logger.debug( "Streaming is not yet supported for emulated file_search. Disabling stream for this request." @@ -398,7 +401,7 @@ def _prepare_emulated_file_search_call( return include_search_results, updated_kwargs -def _extract_tool_call_fields(tool_call: Any, fallback_call_id: str) -> tuple[str, str]: +def _extract_tool_call_fields(tool_call: object, fallback_call_id: str) -> tuple[str, str]: """Extract (call_id, raw_arguments_string) from a dict or Pydantic tool_call item.""" if isinstance(tool_call, dict): call_id = str(tool_call.get("call_id") or tool_call.get("id") or fallback_call_id) @@ -410,7 +413,7 @@ def _extract_tool_call_fields(tool_call: Any, fallback_call_id: str) -> tuple[st return call_id, raw_args -def _resolve_queries_from_args(args: dict[str, Any], input: Any) -> list[str]: +def _resolve_queries_from_args(args: dict[str, Any], input: object) -> list[str]: """Pull the queries list out of parsed tool-call arguments, with backward-compat fallbacks.""" queries_from_call: Final = args.get("queries") if not queries_from_call: @@ -423,13 +426,13 @@ def _resolve_queries_from_args(args: dict[str, Any], input: Any) -> list[str]: async def _execute_file_search_tool_calls( - file_search_calls: list[Any], + file_search_calls: Sequence[object], all_vs_ids: list[str], - input: Any, + input: object, file_search_call_id: str, -) -> tuple[list[dict[str, Any]], list[str], list[VectorStoreSearchResult]]: +) -> tuple[list[object], list[str], list[VectorStoreSearchResult]]: """Run the vector search for each file_search tool_call and collect results.""" - tool_results: Final[list[dict[str, Any]]] = [] + tool_results: Final[list[object]] = [] all_queries: Final[list[str]] = [] all_results: Final[list[VectorStoreSearchResult]] = [] @@ -465,17 +468,17 @@ async def _execute_file_search_tool_calls( def _build_follow_up_input( - input: Any, + input: object, first_response: ResponsesAPIResponse, - tool_results: list[dict[str, Any]], -) -> list[Any]: + tool_results: list[object], +) -> list[object]: """Assemble the follow-up call input: original messages + first-response output + tool results. Including all output items (text blocks, reasoning, non-file-search calls) ensures providers like Anthropic that emit text before the tool call have complete conversation context. Serializes Pydantic model instances to plain dicts so the transformation layer can call .get(). """ - original_input_items: Final = ( + original_input_items: Final[list[object]] = ( list(input) if isinstance(input, (list, tuple)) else [{"role": "user", "content": str(input)}] ) first_response_output_items: Final[list[Any]] = [] @@ -491,7 +494,7 @@ def _build_follow_up_input( async def aresponses_with_emulated_file_search( - input: Any, + input: object, model: str, tools: Iterable[ToolParam] | None = None, # Pass-through params — forwarded as-is to the underlying aresponses call diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index 186852f91c2..022b9ece32e 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -68,7 +68,7 @@ async def create_mcp_list_tools_events( # Convert tools to dict format for the event _mcp_tools_dict: Final = [ tool.model_dump() - if hasattr(tool, "model_dump") and callable(getattr(tool, "model_dump")) + if hasattr(tool, "model_dump") and callable(getattr(tool, "model_dump", None)) else tool.__dict__ if hasattr(tool, "__dict__") else {"name": getattr(tool, "name", str(tool))} @@ -356,7 +356,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): self.oauth2_headers = MCPRequestHandler._get_oauth2_headers_from_headers(headers_obj) # Also check if headers are provided in tools array (from request body) - tools: Final = self.original_request_params.get("tools") + tools: Final[Sequence[object] | None] = self.original_request_params.get("tools") if tools: for tool in tools: if isinstance(tool, dict) and tool.get("type") == "mcp": @@ -395,7 +395,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): def _make_stream_error_event(self) -> ResponsesAPIStreamingResponse: err: Final = self._stream_error - status_code: Final = getattr(err, "status_code", None) + status_code: Final[object] = getattr(err, "status_code", None) return ErrorEvent( type=ResponsesAPIStreamEvents.ERROR, sequence_number=self._last_sequence_number + 1, @@ -515,7 +515,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): # Capture the response ID from the first event to ensure consistency if self._cached_response_id is None and hasattr(chunk, "response"): - response_obj = getattr(chunk, "response", None) + response_obj: ResponsesAPIResponse | None = getattr(chunk, "response", None) if response_obj and hasattr(response_obj, "id"): self._cached_response_id = response_obj.id verbose_logger.debug("Cached response ID: %s", self._cached_response_id) @@ -559,7 +559,8 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): """Check if this chunk indicates the response is completed""" from litellm.types.llms.openai import ResponsesAPIStreamEvents - return getattr(chunk, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + chunk_type: Final[object] = getattr(chunk, "type", None) + return chunk_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED async def _process_base_iterator_chunk(self) -> ResponsesAPIStreamingResponse: """ @@ -571,14 +572,14 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): chunk: Final = await cast(Any, self.base_iterator).__anext__() if self._cached_response_id is None and hasattr(chunk, "response"): - new_response: Final = getattr(chunk, "response", None) + new_response: Final[ResponsesAPIResponse | None] = getattr(chunk, "response", None) new_response_id: Final = getattr(new_response, "id", None) if new_response is not None else None if new_response_id: self._cached_response_id = new_response_id # Ensure response ID consistency - update chunk if needed if self._cached_response_id and hasattr(chunk, "response"): - response_obj = getattr(chunk, "response", None) + response_obj: ResponsesAPIResponse | None = getattr(chunk, "response", None) if response_obj and hasattr(response_obj, "id"): if response_obj.id != self._cached_response_id: verbose_logger.debug( @@ -605,7 +606,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): from litellm.responses.main import aresponses # Make the initial response API call - but avoid the MCP wrapper - params: Final = self.original_request_params.copy() + params: Final[dict[str, object]] = self.original_request_params.copy() params["stream"] = True # Ensure streaming # Use the pre-fetched all_tools from original_request_params (no re-processing needed) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index d7f6ece5cd1..25e5fcb6976 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -5,7 +5,7 @@ import json import time import traceback import uuid -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from datetime import datetime from functools import lru_cache from types import MappingProxyType @@ -1035,7 +1035,7 @@ class CachedResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): @runtime_checkable class _HasModelDump(Protocol): - def model_dump(self, *, exclude_none: bool = ...) -> Mapping[str, object]: ... + def model_dump(self, *, exclude_none: bool = ...) -> dict[str, object]: ... @runtime_checkable @@ -1043,8 +1043,8 @@ class _HasModelDumpJson(Protocol): def model_dump_json(self, *, exclude_none: bool = ...) -> str: ... -def _dump_response_object(obj: Any) -> dict[str, Any]: - if hasattr(obj, "model_dump"): +def _dump_response_object(obj: object) -> dict[str, Any]: + if isinstance(obj, _HasModelDump): return obj.model_dump() if _is_json_object(obj): return obj @@ -1134,7 +1134,8 @@ def _add_text_like_part_events( delta=text[i : i + chunk_size], ) ) - for annotation_index, annotation in enumerate(part_payload.get("annotations", []) or []): + annotations_payload: Final[Sequence[dict[str, object]]] = part_payload.get("annotations", []) or [] + for annotation_index, annotation in enumerate(annotations_payload): events.append( openai_types.OutputTextAnnotationAddedEvent( type=openai_types.ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED, @@ -1200,7 +1201,8 @@ def _build_synthetic_response_events( ] sequence_number = 0 - for output_index, output_item in enumerate(getattr(transformed, "output", []) or []): + output_items: Final[Sequence[object]] = getattr(transformed, "output", []) or [] + for output_index, output_item in enumerate(output_items): output_item_payload = _dump_response_object(output_item) item_id = str(output_item_payload.get("id") or transformed.id) item_type = output_item_payload.get("type") @@ -1214,7 +1216,8 @@ def _build_synthetic_response_events( ) if item_type == "message": - for content_index, part in enumerate(output_item_payload.get("content", []) or []): + content_parts: Sequence[object] = output_item_payload.get("content", []) or [] + for content_index, part in enumerate(content_parts): part_payload = _dump_response_object(part) events.append( openai_types.ContentPartAddedEvent( @@ -1261,7 +1264,8 @@ def _build_synthetic_response_events( ) ) elif item_type == "reasoning": - for summary_index, summary in enumerate(output_item_payload.get("summary", []) or []): + summaries: Sequence[object] = output_item_payload.get("summary", []) or [] + for summary_index, summary in enumerate(summaries): summary_payload = _dump_response_object(summary) summary_text = str(summary_payload.get("text") or "") for i in range(0, len(summary_text), chunk_size): @@ -1463,7 +1467,8 @@ class ResponsesWebSocketStreaming: # masked response.completed. if self.output_guardrail_callbacks: try: - _evt_type = json.loads(response_str).get("type") + _evt_payload: Mapping[str, object] = json.loads(response_str) + _evt_type = _evt_payload.get("type") except (json.JSONDecodeError, TypeError): _evt_type = None if _evt_type in self._DELTA_EVENT_TYPES or _evt_type in self._OUTPUT_DONE_EVENT_TYPES: @@ -1527,7 +1532,7 @@ class ResponsesWebSocketStreaming: Non-``response.create`` messages are returned unchanged. """ try: - msg_obj: Final = json.loads(message) + msg_obj: Final[dict[str, object]] = json.loads(message) except (json.JSONDecodeError, TypeError): return message @@ -1544,7 +1549,8 @@ class ResponsesWebSocketStreaming: self.request_data["metadata"] = {} modified = model_modified - for cb in self.guardrail_callbacks: + guardrail_cbs: Final[tuple[PresidioGuardrailCallback, ...]] = tuple(self.guardrail_callbacks) + for cb in guardrail_cbs: presidio_config = cb.get_presidio_settings_from_request_data(self.request_data) # response.create carries client text in two shapes: # flat: {"type": "response.create", "input": ..., "instructions": ...} @@ -1655,7 +1661,7 @@ class ResponsesWebSocketStreaming: return response_str try: - evt_obj: Final = json.loads(response_str) + evt_obj: Final[dict[str, object]] = json.loads(response_str) except (json.JSONDecodeError, TypeError): return response_str @@ -2012,7 +2018,7 @@ class ManagedResponsesWebSocketHandler: async def _parse_message(self, raw_message: str) -> dict[str, object] | None: """Parse raw WS text; return the message dict or None (JSON error / ignored type).""" try: - msg_obj: Final = json.loads(raw_message) + msg_obj: Final[dict[str, object]] = json.loads(raw_message) except json.JSONDecodeError: await self._send_error("Invalid JSON in response.create event", "invalid_request_error") return None @@ -2293,11 +2299,10 @@ class ManagedResponsesWebSocketHandler: # reuse the router-resolved self.model; passing the alias raw to # litellm.aresponses fails in get_llm_provider. A genuinely different # provider-prefixed per-frame model is still honored. - requested_model: Final = call_kwargs.pop("model", None) - if requested_model is None or requested_model == self.model_group: - model = self.model - else: - model = requested_model + requested_model: Final[str | None] = call_kwargs.pop("model", None) + model: Final[str] = ( + self.model if requested_model is None or requested_model == self.model_group else requested_model + ) previous_response_id: Final[str | None] = call_kwargs.pop("previous_response_id", None) current_messages: Final = self._input_to_messages(call_kwargs.get("input")) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 1907b5aa447..4b5def790ed 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -93,9 +93,9 @@ class ResponsesAPIRequestUtils: @staticmethod def merge_client_forwarded_headers( - extra_headers: dict[str, Any] | None, + extra_headers: dict[str, object] | None, client_headers: dict[str, str] | None, - ) -> dict[str, Any] | None: + ) -> dict[str, object] | None: """ Merge headers forwarded by the proxy (`headers` kwarg, set when `forward_client_headers_to_llm_api` is enabled) into `extra_headers`. @@ -210,9 +210,9 @@ class ResponsesAPIRequestUtils: valid_keys: Final = get_type_hints(ResponsesAPIOptionalRequestParams).keys() custom_llm_provider: Final = params.pop("custom_llm_provider", None) - special_params: Final = params.pop("kwargs", {}) + special_params: Final[dict[str, object]] = params.pop("kwargs", {}) - additional_drop_params: Final = params.pop("additional_drop_params", None) + additional_drop_params: Final[list[str] | None] = params.pop("additional_drop_params", None) non_default_params: Final = PreProcessNonDefaultParams.base_pre_process_non_default_params( passed_params=params, special_params=special_params, @@ -401,9 +401,9 @@ class ResponsesAPIRequestUtils: @staticmethod def _update_encrypted_content_item_ids_in_response( - response: Union["ResponsesAPIResponse", dict[str, Any]], + response: Union["ResponsesAPIResponse", dict[str, object]], model_id: str | None, - ) -> Union["ResponsesAPIResponse", dict[str, Any]]: + ) -> Union["ResponsesAPIResponse", dict[str, object]]: """Rewrite item IDs for output items that contain ``encrypted_content``. Encodes ``model_id`` into the item ID so that follow-up requests can be @@ -415,7 +415,7 @@ class ResponsesAPIRequestUtils: if not model_id: return response - output: list | None = None + output: object = None if isinstance(response, dict): output = response.get("output") else: @@ -459,7 +459,7 @@ class ResponsesAPIRequestUtils: return response @staticmethod - def _restore_encrypted_content_item_ids_in_input(request_input: Any) -> Any: + def _restore_encrypted_content_item_ids_in_input(request_input: object) -> Any: """Decode litellm-encoded item IDs in request input back to original IDs. Called before forwarding the request to the upstream provider so the @@ -867,7 +867,7 @@ class ResponsesAPIRequestUtils: ) @staticmethod - def collect_container_ids_from_responses_response(response: Any) -> list[str]: + def collect_container_ids_from_responses_response(response: object) -> list[str]: """Return unique container IDs referenced in a Responses API payload.""" if response is None: return [] @@ -953,7 +953,7 @@ class ResponsesAPIRequestUtils: @staticmethod def extract_mcp_headers_from_request( secret_fields: dict[str, Any] | None, - tools: Iterable[Any] | None, + tools: Iterable[object] | None, ) -> tuple[ str | None, dict[str, dict[str, str]] | None, diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index 1120323b4f9..e4ac45df4d5 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -8,9 +8,11 @@ Use this to route requests between Teams """ import re -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal +from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict + +from typing_extensions import ReadOnly from litellm._logging import verbose_logger from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY @@ -25,9 +27,39 @@ else: LitellmRouter = Any +class _TagRoutingLitellmParams(TypedDict, total=False): + tags: ReadOnly[Sequence[str] | None] + tag_regex: ReadOnly[Sequence[str] | None] + + +class _TagRoutingDeployment(TypedDict, total=False): + model_name: ReadOnly[str] + litellm_params: ReadOnly[_TagRoutingLitellmParams] + model_info: ReadOnly[Mapping[str, object] | None] + + +class _TagRoutingMatchStamp(TypedDict): + matched_deployment: ReadOnly[str | None] + matched_via: ReadOnly[str] + matched_value: ReadOnly[str] + request_tags: ReadOnly[Sequence[str]] + user_agent: ReadOnly[str] + + +class _TagRoutingMetadata(TypedDict, total=False): + tags: ReadOnly[Sequence[str] | None] + inherited_tags: ReadOnly[Sequence[str] | None] + user_agent: ReadOnly[str] + tag_routing: ReadOnly[_TagRoutingMatchStamp] + _consumed_request_tags: ReadOnly[object] + + +_EMPTY_MODEL_INFO: Final[Mapping[str, object]] = MappingProxyType({}) + + def _is_valid_deployment_tag_regex( - tag_regexes: list[str], - header_strings: list[str], + tag_regexes: Sequence[str], + header_strings: Sequence[str], ) -> str | None: """ Test compiled regex patterns against "Header-Name: value" strings. @@ -77,11 +109,11 @@ def is_valid_deployment_tag( def _match_deployment( - deployment: Any, - request_tags: list[str] | None, - header_strings: list[str], + deployment: _TagRoutingDeployment, + request_tags: Sequence[str] | None, + header_strings: Sequence[str], match_any: bool, -) -> dict[str, str] | None: +) -> Mapping[str, str] | None: """ Determine whether *deployment* matches the current request. @@ -94,8 +126,8 @@ def _match_deployment( ran and failed, so the regex cannot override strict-tag policy. """ litellm_params: Final = deployment.get("litellm_params", {}) - deployment_tags: Final[list[str] | None] = litellm_params.get("tags") - deployment_tag_regex: Final[list[str] | None] = litellm_params.get("tag_regex") + deployment_tags: Final[Sequence[str] | None] = litellm_params.get("tags") + deployment_tag_regex: Final[Sequence[str] | None] = litellm_params.get("tag_regex") # 1. Exact tag match (existing behaviour). if deployment_tags and request_tags: @@ -166,38 +198,38 @@ def _split_tags(tags: Sequence[str]) -> tuple[tuple[str, ...], list[str], tuple[ def _exclude_deployments( - deployments: Sequence[Any] | Mapping[Any, Any], + deployments: Iterable[_TagRoutingDeployment], excluded_set: frozenset[str], -) -> list[Any]: +) -> list[_TagRoutingDeployment]: if not excluded_set: return list(deployments) return [d for d in deployments if not excluded_set.intersection(d.get("litellm_params", {}).get("tags") or [])] def _require_all_tags( - deployments: Sequence[Any] | Mapping[Any, Any], + deployments: Iterable[_TagRoutingDeployment], required_set: frozenset[str], -) -> tuple[Any, ...]: +) -> tuple[_TagRoutingDeployment, ...]: if not required_set: return tuple(deployments) return tuple(d for d in deployments if required_set.issubset(d.get("litellm_params", {}).get("tags") or [])) def _default_tagged_pool( - deployments: Sequence[Any] | Mapping[Any, Any], -) -> tuple[Any, ...]: + deployments: Iterable[_TagRoutingDeployment], +) -> tuple[_TagRoutingDeployment, ...]: defaults: Final = tuple(d for d in deployments if "default" in (d.get("litellm_params", {}).get("tags") or [])) return defaults if defaults else tuple(deployments) -def _known_tag_values(deployments: Sequence[Any] | Mapping[Any, Any]) -> frozenset[str]: +def _known_tag_values(deployments: Iterable[_TagRoutingDeployment]) -> frozenset[str]: return frozenset( - tag for d in deployments for tag in (d.get("litellm_params", MappingProxyType({})).get("tags") or ()) + tag for d in deployments for tag in (d.get("litellm_params", _TagRoutingLitellmParams()).get("tags") or ()) ) def _unknown_required_tag_hides_an_answer( - healthy_deployments: Sequence[Any] | Mapping[Any, Any], + healthy_deployments: Iterable[_TagRoutingDeployment], excluded_set: frozenset[str], required_set: frozenset[str], routing_confirmed: frozenset[str], @@ -221,23 +253,23 @@ def _unknown_required_tag_hides_an_answer( def _chain_allows_fail_open( - healthy_deployments: Sequence[Any] | Mapping[Any, Any], + healthy_deployments: Iterable[_TagRoutingDeployment], excluded_set: frozenset[str], required_set: frozenset[str], routing_confirmed: frozenset[str], ) -> bool: if _unknown_required_tag_hides_an_answer(healthy_deployments, excluded_set, required_set, routing_confirmed): return False - return any((d.get("model_info") or {}).get("allow_fail_open") is True for d in healthy_deployments) + return any((d.get("model_info") or _EMPTY_MODEL_INFO).get("allow_fail_open") is True for d in healthy_deployments) def _trusted_only_pool( - healthy_deployments: Sequence[Any] | Mapping[Any, Any], + healthy_deployments: Iterable[_TagRoutingDeployment], excluded_set: frozenset[str], required_set: frozenset[str], inherited_excluded_set: frozenset[str] | None, inherited_required_set: frozenset[str] | None, -) -> tuple[Any, ...]: +) -> tuple[_TagRoutingDeployment, ...]: # inherited_*_set is None only when this request carries no origin information # at all (e.g. direct SDK Router usage, bypassing the proxy layer that # populates metadata.inherited_tags) -- treat every constraint as @@ -264,8 +296,8 @@ def _trusted_only_pool( def _resolve_or_fail_open( - pool: Sequence[Any], - healthy_deployments: Sequence[Any] | Mapping[Any, Any], + pool: Sequence[_TagRoutingDeployment], + healthy_deployments: Iterable[_TagRoutingDeployment], excluded_set: frozenset[str], required_set: frozenset[str], inherited_excluded_set: frozenset[str] | None, @@ -273,7 +305,7 @@ def _resolve_or_fail_open( routing_confirmed: frozenset[str], model: str, request_tags: object, -) -> tuple[Any, ...]: +) -> tuple[_TagRoutingDeployment, ...]: if pool: return tuple(pool) if _chain_allows_fail_open(healthy_deployments, excluded_set, required_set, routing_confirmed): @@ -293,7 +325,7 @@ def _resolve_or_fail_open( def _resolve_constraint_only_pool( - healthy_deployments: Sequence[Any] | Mapping[Any, Any], + healthy_deployments: Iterable[_TagRoutingDeployment], excluded_set: frozenset[str], required_set: frozenset[str], inherited_excluded_set: frozenset[str] | None, @@ -301,7 +333,7 @@ def _resolve_constraint_only_pool( routing_confirmed: frozenset[str], model: str, request_tags: object, -) -> tuple[Any, ...]: +) -> tuple[_TagRoutingDeployment, ...]: pool: Final = ( _require_all_tags(_exclude_deployments(healthy_deployments, excluded_set), required_set) if required_set @@ -323,8 +355,8 @@ def _resolve_constraint_only_pool( def _all_deployments_or_fallback( llm_router_instance: LitellmRouter, model: str, - fallback: Sequence[Any] | Mapping[Any, Any], -) -> Sequence[Any] | Mapping[Any, Any]: + fallback: Iterable[_TagRoutingDeployment], +) -> Iterable[_TagRoutingDeployment]: try: return llm_router_instance._get_all_deployments(model_name=model) except Exception: # noqa: BLE001 # fail safe toward today's healthy-only behavior on lookup errors @@ -334,8 +366,8 @@ def _all_deployments_or_fallback( def _chain_tag_filtering_override( llm_router_instance: LitellmRouter, model: str, - healthy_deployments: Sequence[Any] | Mapping[Any, Any], -) -> bool | None: + healthy_deployments: Iterable[_TagRoutingDeployment], +) -> object: # Resolved from every deployment configured for this model group, not just the # ones that survived cooldown/health filtering (async_get_healthy_deployments # filters cooldowns before calling get_deployments_for_tag) -- otherwise the @@ -347,14 +379,14 @@ def _chain_tag_filtering_override( # than crashing the request. all_deployments: Final = _all_deployments_or_fallback(llm_router_instance, model, healthy_deployments) for d in all_deployments: - value = (d.get("model_info") or MappingProxyType({})).get("enable_tag_filtering") + value = (d.get("model_info") or _EMPTY_MODEL_INFO).get("enable_tag_filtering") if value is not None: return value return None def _inherited_constraint_sets( - inherited_tags: object, routing_prefix: str + inherited_tags: Sequence[str] | None, routing_prefix: str ) -> tuple[frozenset[str] | None, frozenset[str] | None]: # None means no origin information is available at all (e.g. this request # bypassed the proxy layer that populates metadata.inherited_tags, as direct @@ -385,15 +417,18 @@ def _tag_known_to_group( if tag_set & routing_confirmed: return True try: - all_deployments: Final = llm_router_instance._get_all_deployments(model_name=model) + all_deployments: Final[Sequence[_TagRoutingDeployment]] = llm_router_instance._get_all_deployments( + model_name=model + ) except Exception: # noqa: BLE001 # fail safe toward "unrecognized" so lookup errors preserve the existing silent-fallback behavior return False return any( - tag_set.intersection(d.get("litellm_params", MappingProxyType({})).get("tags") or ()) for d in all_deployments + tag_set.intersection(d.get("litellm_params", _TagRoutingLitellmParams()).get("tags") or ()) + for d in all_deployments ) -def _request_tags_after_router_consumption(metadata: Mapping[Any, Any], model: str) -> Sequence[str] | None: +def _request_tags_after_router_consumption(metadata: _TagRoutingMetadata, model: str) -> Sequence[str] | None: # The pre-routing hook stamps which tags selected the router it rewrote the request # to: those tags already did their job and must not also constrain deployment choice # inside the routed group. The request's other tags still apply there, on top of the @@ -451,7 +486,8 @@ async def get_deployments_for_tag( verbose_logger.debug("request metadata: %s", request_kwargs.get(metadata_variable_name)) if metadata_variable_name in request_kwargs: - metadata: Final = request_kwargs[metadata_variable_name] + metadata: Final[_TagRoutingMetadata] = request_kwargs[metadata_variable_name] + stampable_metadata: Final[dict[str, object]] = request_kwargs[metadata_variable_name] request_tags: Final = _request_tags_after_router_consumption(metadata, model) match_any: Final = llm_router_instance.tag_filtering_match_any routing_prefix: Final = llm_router_instance.tag_routing_prefix or "" @@ -496,8 +532,8 @@ async def get_deployments_for_tag( request_tags, ) - new_healthy_deployments: Final[list[Any]] = [] - default_deployments: Final[list[Any]] = [] + new_healthy_deployments: Final[list[_TagRoutingDeployment]] = [] + default_deployments: Final[list[_TagRoutingDeployment]] = [] if has_positive_filter: verbose_logger.debug( @@ -523,7 +559,7 @@ async def get_deployments_for_tag( match_result["matched_value"], ) if "tag_routing" not in metadata: - metadata["tag_routing"] = { + stampable_metadata["tag_routing"] = { "matched_deployment": deployment.get("model_name"), "matched_via": match_result["matched_via"], "matched_value": match_result["matched_value"], @@ -568,7 +604,7 @@ async def get_deployments_for_tag( return new_healthy_deployments if len(new_healthy_deployments) > 0 else default_deployments # for Untagged requests use default deployments if set - _default_deployments_with_tags: Final = [] + _default_deployments_with_tags: Final[list[_TagRoutingDeployment]] = [] for deployment in healthy_deployments: if "default" in deployment.get("litellm_params", {}).get("tags", []): _default_deployments_with_tags.append(deployment) @@ -603,7 +639,7 @@ def _tags_in_metadata(metadata: object) -> list[str]: def _get_tags_from_request_kwargs( - request_kwargs: Mapping[Any, Any] | None = None, + request_kwargs: Mapping[str, object] | None = None, metadata_variable_name: Literal["metadata", "litellm_metadata"] | None = None, ) -> list[str]: """ diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index dff010bfd30..17c8f02dfdd 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -1,6 +1,6 @@ { "ANN001": { - "limit": 3058 + "limit": 3046 }, "ANN002": { "limit": 71 @@ -24,7 +24,7 @@ "limit": 133 }, "ANN401": { - "limit": 1384 + "limit": 1342 }, "ASYNC230": { "limit": 11 @@ -39,7 +39,7 @@ "limit": 505 }, "B009": { - "limit": 64 + "limit": 60 }, "B010": { "limit": 190 @@ -234,7 +234,7 @@ "limit": 5 }, "TID251": { - "limit": 1224 + "limit": 1220 }, "TRY002": { "limit": 524 diff --git a/type-discipline-budget.json b/type-discipline-budget.json index a7286d9a89a..0104733bcf3 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 23001 + "limit": 22943 }, "LIT002": { - "limit": 27146 + "limit": 27141 }, "LIT003": { "limit": 269 @@ -15,7 +15,7 @@ "limit": 0 }, "LIT006": { - "limit": 1077 + "limit": 1074 }, "LIT007": { "limit": 0 @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16731 + "limit": 16722 }, "LIT011": { "limit": 5596 From 08ae8648e0b5111c628d6dfe9b78e2800c3673f3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 13 Aug 2026 08:24:23 -0700 Subject: [PATCH 09/10] fix: restore tolerant auth attribute access in compact summary checks The summary model access and budget checks read team_id, user_id, project_id, and end_user_id via getattr with a None default so duck-typed auth objects without those attributes keep working --- basedpyright-code-budget.json | 2 +- .../context_management/editors/compact.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index e85e3701539..521b4315e6e 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 22941 + "limit": 22947 }, "reportArgumentType": { "limit": 2579 diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py index 34d981e2e55..a7c462a8fb0 100644 --- a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py @@ -158,11 +158,11 @@ async def _check_summary_model_access( return True key_models: Final = list(getattr(user_api_key_auth, "models", None) or []) - team_id: Final = user_api_key_auth.team_id + team_id: Final = getattr(user_api_key_auth, "team_id", None) team_model_aliases: Final = getattr(user_api_key_auth, "team_model_aliases", None) team_models: Final = list(getattr(user_api_key_auth, "team_models", None) or []) - user_id: Final = user_api_key_auth.user_id - project_id: Final = user_api_key_auth.project_id + user_id: Final = getattr(user_api_key_auth, "user_id", None) + project_id: Final = getattr(user_api_key_auth, "project_id", None) checks: Final[tuple[tuple[Literal["key", "team"], list[str]], ...]] = ( ("key", key_models), @@ -348,7 +348,7 @@ async def _check_summary_model_budget( return False end_user_model_max_budget: Final = getattr(user_api_key_auth, "end_user_model_max_budget", None) - end_user_id: Final = user_api_key_auth.end_user_id + end_user_id: Final = getattr(user_api_key_auth, "end_user_id", None) if isinstance(end_user_model_max_budget, dict) and end_user_model_max_budget and end_user_id is not None: try: await model_max_budget_limiter.is_end_user_within_model_budget( From d8fda675cc78912ba9eb964d2d2207b8357fd1e6 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Thu, 13 Aug 2026 13:15:45 -0700 Subject: [PATCH 10/10] feat: pre-adoption shadow eval for the auto-router (blind pairwise judge, derived state) (#36587) --- .../migration.sql | 49 ++ .../litellm_proxy_extras/schema.prisma | 38 ++ litellm/integrations/shadow_eval_logger.py | 563 ++++++++++++++++++ .../internal_call_metadata.py | 94 +++ litellm/litellm_core_utils/llm_judge.py | 87 +++ litellm/proxy/db/autorouter_session_rollup.py | 12 +- litellm/proxy/db/db_spend_update_writer.py | 15 +- .../llm_as_a_judge/__init__.py | 83 +-- .../hooks/parallel_request_limiter_v3.py | 8 +- .../auto_router_endpoints.py | 347 ++++++++++- litellm/proxy/proxy_server.py | 3 + litellm/proxy/schema.prisma | 38 ++ .../complexity_router/complexity_router.py | 47 +- .../auto_router_endpoints.py | 114 +++- litellm/types/utils.py | 4 +- schema.prisma | 38 ++ .../integrations/test_shadow_eval_logger.py | 466 +++++++++++++++ .../test_internal_call_metadata.py | 121 ++++ .../litellm_core_utils/test_llm_judge.py | 92 +++ .../db/test_autorouter_session_rollup.py | 7 + .../proxy/db/test_db_spend_update_writer.py | 47 ++ .../hooks/test_parallel_request_limiter_v3.py | 39 ++ .../test_auto_router_endpoints.py | 273 +++++++++ .../proxy/proxy_server/test_lifecycle.py | 8 +- .../router_strategy/test_complexity_router.py | 92 --- ui/litellm-dashboard/src/lib/http/schema.d.ts | 359 +++++++++++ 26 files changed, 2827 insertions(+), 217 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260811172448_add_shadow_eval/migration.sql create mode 100644 litellm/integrations/shadow_eval_logger.py create mode 100644 litellm/litellm_core_utils/internal_call_metadata.py create mode 100644 litellm/litellm_core_utils/llm_judge.py create mode 100644 tests/test_litellm/integrations/test_shadow_eval_logger.py create mode 100644 tests/test_litellm/litellm_core_utils/test_internal_call_metadata.py create mode 100644 tests/test_litellm/litellm_core_utils/test_llm_judge.py diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260811172448_add_shadow_eval/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260811172448_add_shadow_eval/migration.sql new file mode 100644 index 00000000000..26932addb42 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260811172448_add_shadow_eval/migration.sql @@ -0,0 +1,49 @@ +-- CreateTable +CREATE TABLE "LiteLLM_ShadowEvalJob" ( + "id" TEXT NOT NULL, + "api_key_id" TEXT NOT NULL, + "router_name" TEXT NOT NULL, + "judge_model" TEXT NOT NULL, + "shadow_percentage" DOUBLE PRECISION NOT NULL, + "max_turns" INTEGER NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "created_by" TEXT, + "ends_at" TIMESTAMP(3) NOT NULL, + "stopped_at" TIMESTAMP(3), + + CONSTRAINT "LiteLLM_ShadowEvalJob_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "LiteLLM_ShadowEvalAttempt" ( + "id" TEXT NOT NULL, + "job_id" TEXT NOT NULL, + "request_id" TEXT NOT NULL, + "outcome" TEXT NOT NULL, + "tier" TEXT, + "real_model" TEXT, + "shadow_model" TEXT, + "confidence" DOUBLE PRECISION, + "judge_cost" DOUBLE PRECISION NOT NULL DEFAULT 0, + "error" TEXT, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "LiteLLM_ShadowEvalAttempt_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "LiteLLM_ShadowEvalJob_api_key_id_idx" ON "LiteLLM_ShadowEvalJob"("api_key_id"); + +-- CreateIndex +CREATE INDEX "LiteLLM_ShadowEvalJob_created_at_idx" ON "LiteLLM_ShadowEvalJob"("created_at"); + +-- CreateIndex +CREATE INDEX "LiteLLM_ShadowEvalAttempt_job_id_idx" ON "LiteLLM_ShadowEvalAttempt"("job_id"); + + +-- One active job per key, enforced by the database rather than a read-then-create in the +-- start endpoint, which races against a concurrent start on another pod. Partial indexes +-- are not expressible in schema.prisma, so this lives here only. Active means not yet +-- stopped; the start endpoint stamps stopped_at on expired jobs before creating. +CREATE UNIQUE INDEX "LiteLLM_ShadowEvalJob_one_active_per_key" + ON "LiteLLM_ShadowEvalJob"("api_key_id") WHERE "stopped_at" IS NULL; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 854602f5380..79d778fb464 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1450,6 +1450,44 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } +// Shadow eval: pre-adoption evaluation of an auto-router against a key's live traffic. +// A sampled slice of requests is duplicated through the router in a detached task and an +// LLM judge compares real vs shadow responses blind. The job row is immutable config plus +// stopped_at; every count, status, and spend figure is derived from the append-only +// attempt rows, so nothing can disagree across pods or stop races. +model LiteLLM_ShadowEvalJob { + id String @id @default(cuid()) + api_key_id String // hashed virtual key whose traffic is shadowed + router_name String + judge_model String + shadow_percentage Float + max_turns Int // sample budget: judge at most this many turns + created_at DateTime @default(now()) + created_by String? + ends_at DateTime + stopped_at DateTime? + + @@index([api_key_id]) + @@index([created_at]) +} + +// One row per sampled pipeline: a blind verdict (real | shadow | tie) or an error. +model LiteLLM_ShadowEvalAttempt { + id String @id @default(cuid()) + job_id String + request_id String // the judged real request + outcome String // real | shadow | tie | error + tier String? // router's tier for the prompt, when classified + real_model String? + shadow_model String? + confidence Float? + judge_cost Float @default(0) + error String? + created_at DateTime @default(now()) + + @@index([job_id]) +} + // --------------------------------------------------------------------------- // Workflow Run Tracking // diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py new file mode 100644 index 00000000000..c7b89e0e9b0 --- /dev/null +++ b/litellm/integrations/shadow_eval_logger.py @@ -0,0 +1,563 @@ +"""Shadow Eval Logger: samples a shadowed key's successful chat requests, duplicates each +through the auto-router in a detached task, blind-judges real vs shadow, and appends one +``LiteLLM_ShadowEvalAttempt`` row (verdict or error) as the feature's only hot-path write. +Counts, status, and spend derive from those rows at read time, so nothing can disagree +across pods or stop races; the hook reads active jobs through a short-TTL cache.""" + +import asyncio +import hashlib +import random +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from pydantic import BaseModel + +from litellm._logging import verbose_logger +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs +from litellm.litellm_core_utils.internal_call_metadata import sanitized_forwardable_call_metadata +from litellm.litellm_core_utils.llm_judge import ( + default_router_provider, + extract_text_from_content, + judge_acompletion, + parse_json_verdict, +) +from litellm.litellm_core_utils.redact_messages import should_redact_message_logging +from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + from litellm.router import Router + from litellm.types.utils import StandardLoggingPayload + +# A job starting, stopping, or hitting its turn budget propagates to sampling within one +# TTL; the turn budget can overshoot by at most one TTL of in-flight samples per pod. +_JOBS_CACHE_TTL_SECONDS: Final = 10 + +# Concurrent shadow+judge pipelines per pod: a traffic spike turns into skipped samples +# rather than an unbounded task pileup. +_MAX_CONCURRENT_SHADOW_TASKS: Final = 16 + +# Total character budget for the judge's user prompt, however long the conversation and +# the two responses are, so the prompt can never overflow a judge model's context window. +_MAX_JUDGE_RESPONSE_CHARS: Final = 8_000 +_MAX_JUDGE_PROMPT_CHARS: Final = 24_000 + +# The judge answers with a small JSON object; a tighter budget truncates the JSON +# mid-object and the attempt is lost to an error row. +JUDGE_MAX_OUTPUT_TOKENS: Final = 500 + +_MAX_ERROR_CHARS: Final = 500 + +_EMPTY_METADATA: Final[Mapping[str, object]] = MappingProxyType({}) + +_SAMPLED_CALL_TYPES: Final = frozenset({"completion", "acompletion"}) + +PAIRWISE_JUDGE_SYSTEM_PROMPT: Final = """You are an impartial quality judge comparing two responses to the same conversation. + +The responses are labeled A and B in random order. You do not know which system produced which. + +Criteria: correctness, completeness, clarity, conciseness. + +Return ONLY valid JSON in this exact format, no other text: +{ + "preference": "A" | "B" | "tie", + "confidence": <0.0 to 1.0>, + "reasoning": "" +}""" + + +class PairwiseVerdict(BaseModel): + """The judge's blind A/B verdict, validated at the parse boundary.""" + + preference: str = "tie" + confidence: float = 0.0 + + +def _sample_hits(request_id: str, job_id: str, percentage: float) -> bool: + """Deterministically decide whether a request falls in the shadowed slice: hash-based + rather than random so retries sample the same way and pods agree without coordination.""" + digest: Final = hashlib.sha256(f"{job_id}:{request_id}".encode()).digest() + bucket: Final = int.from_bytes(digest[:8], "big") / float(2**64) + return bucket * 100.0 < percentage + + +def _judge_call_cost(response: object) -> float: + """Price a judge call, treating an unmapped judge model as free rather than fatal.""" + import litellm + + try: + return litellm.completion_cost(completion_response=response) or 0.0 + except Exception: # noqa: BLE001 # unmapped judge model: the verdict still counts, cost stays 0 + return 0.0 + + +def _unmask_preference(raw_preference: str, real_is_a: bool) -> str: + """Map the judge's blind A/B/tie verdict back to real/shadow/tie.""" + normalized: Final = raw_preference.strip().lower() + if normalized == "a": + return "real" if real_is_a else "shadow" + if normalized == "b": + return "shadow" if real_is_a else "real" + return "tie" + + +def _judge_user_prompt(conversation: str, response_a: str, response_b: str) -> str: + """The judge prompt under one total character budget: each response is capped, and + the conversation tail gets whatever budget the responses left over.""" + a: Final = response_a[:_MAX_JUDGE_RESPONSE_CHARS] + b: Final = response_b[:_MAX_JUDGE_RESPONSE_CHARS] + conversation_budget: Final = _MAX_JUDGE_PROMPT_CHARS - len(a) - len(b) + return ( + f"Conversation:\n{conversation[-conversation_budget:]}\n\n" + f"Response A:\n{a}\n\n" + f"Response B:\n{b}\n\n" + "Which response is better?" + ) + + +async def _key_or_team_is_over_budget(metadata: Mapping[str, object]) -> bool: + """Whether the shadowed key or its team is over budget, decided by the same owners + the request path uses, so counter keys and thresholds can never drift from auth's. + + Advisory and fail-open: real traffic on an over-budget key is already rejected at + auth (so nothing reaches the success hook), and this gate only closes the race + where the key crosses its budget while a request is in flight. + """ + try: + from litellm.exceptions import BudgetExceededError + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.auth_checks import ( + _team_max_budget_check, + _virtual_key_max_budget_check, + get_team_object, + ) + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + except ImportError: + return False + + auth: Final = metadata.get("user_api_key_auth") + if not isinstance(auth, UserAPIKeyAuth): + return False + try: + await _virtual_key_max_budget_check(valid_token=auth, proxy_logging_obj=proxy_logging_obj) + if auth.team_id: + team: Final = await get_team_object( + team_id=auth.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + check_cache_only=True, + ) + await _team_max_budget_check(team_object=team, valid_token=auth, proxy_logging_obj=proxy_logging_obj) + except BudgetExceededError: + return True + except Exception as e: # noqa: BLE001 # advisory gate: a failed read must not block sampling + verbose_logger.debug("shadow_eval: budget read failed: %s", e) + return False + + +def _request_was_routed_by(request_metadata: Mapping[str, object], router_name: str) -> bool: + """Duplicating a request the shadowed router already served compares the router to + itself: guaranteed ties, judge spend for zero information.""" + decision: Final = request_metadata.get("routing_decision") + if not isinstance(decision, Mapping): + return False + return decision.get("router_model_name") == router_name + + +@dataclass(frozen=True, slots=True) +class _CallFailure: + """A shadow or judge call that produced no usable response. cost carries any judge + spend the failed attempt still billed, so job-level judge_spend never undercounts.""" + + error: str + cost: float = 0.0 + + +@dataclass(frozen=True, slots=True) +class _ShadowResponse: + """A successful shadow call, with what the attempt row records.""" + + text: str + model: str + tier: str | None + + +@dataclass(frozen=True, slots=True) +class _JudgeVerdict: + """A parsed judge verdict, unmasked back to real/shadow/tie.""" + + preference: str + confidence: float + cost: float + + +@dataclass(frozen=True, slots=True) +class ActiveShadowEvalJob: + """One active job as the sampling path needs it: immutable config plus the attempt + count as of the cache fill (the turn budget's staleness is bounded by the cache TTL).""" + + id: str + router_name: str + shadow_percentage: float + judge_model: str + max_turns: int + ends_at: datetime + attempts: int + + +def _as_utc(value: datetime) -> datetime: + return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value + + +_jobs_cache: Final = InMemoryCache(max_size_in_memory=4, default_ttl=_JOBS_CACHE_TTL_SECONDS) +_JOBS_CACHE_KEY: Final = "shadow_eval:active_jobs" + + +class ShadowEvalLogger(CustomLogger): + """Fires blind pairwise shadow evaluations for keys with an active shadow-eval job.""" + + def __init__( + self, + router_provider: Callable[[], "Router | None"] | None = None, + prisma_provider: Callable[[], "PrismaClient | None"] | None = None, + jobs_cache: InMemoryCache | None = None, + ) -> None: + """Providers are callables so the proxy's lazily-initialized globals are resolved + at call time, not at logger construction.""" + self._router_provider = router_provider or default_router_provider + self._prisma_provider = prisma_provider or _default_prisma_provider + self._jobs_cache = jobs_cache or _jobs_cache + self._inflight_shadow_tasks: int = 0 + # Starts per job since the last cache fill, never decremented within a + # generation; the refill absorbs written rows and resets. + self._job_starts: dict[str, int] = {} # mutable-ok: per-generation counter + + async def _active_jobs(self) -> Mapping[str, ActiveShadowEvalJob]: + """Active jobs by api_key_id, cache-first. A DB fault returns empty without + caching, so sampling pauses for that request and the next one retries.""" + cached: Final = await self._jobs_cache.async_get_cache(_JOBS_CACHE_KEY) + if cached is not None: + return cached # pyright: ignore[reportReturnType] # cache stores exactly this mapping shape + prisma: Final = self._prisma_provider() + if prisma is None: + return _EMPTY_JOBS + try: + records: Final = await prisma.db.litellm_shadowevaljob.find_many( + where={ # mutable-ok: Prisma filter + "stopped_at": None, + "ends_at": {"gt": datetime.now(timezone.utc)}, # mutable-ok: Prisma filter + }, + ) + grouped: Final = ( + await prisma.db.litellm_shadowevalattempt.group_by( + by=["job_id"], + count=True, + where={"job_id": {"in": [str(record.id) for record in records]}}, # mutable-ok: Prisma filter + ) + if records + else () + ) + attempt_counts: Final = {str(row["job_id"]): int(row["_count"]["_all"]) for row in grouped or []} + jobs: Final = { + str(record.api_key_id): ActiveShadowEvalJob( + id=str(record.id), + router_name=str(record.router_name), + shadow_percentage=float(record.shadow_percentage), + judge_model=str(record.judge_model), + max_turns=int(record.max_turns), + ends_at=_as_utc(record.ends_at), + attempts=attempt_counts.get(str(record.id), 0), + ) + for record in records or [] + } + await self._jobs_cache.async_set_cache(_JOBS_CACHE_KEY, jobs) + self._job_starts = {} # rebind-ok: new generation, counts absorbed into the fill + return jobs + except Exception as e: # noqa: BLE001 # a DB blip must never break request logging + verbose_logger.debug("shadow_eval: active-job read failed: %s", e) + return _EMPTY_JOBS + + #### hook #### + + async def async_log_success_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: object, + end_time: object, + ) -> None: + try: + payload: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object") # pyright: ignore[reportAssignmentType] # untyped callback kwargs + if payload is None: + return + raw_meta: Final = get_litellm_metadata_from_kwargs(dict(kwargs)) # mutable-ok: helper needs dict + request_metadata: Final = raw_meta if isinstance(raw_meta, Mapping) else _EMPTY_METADATA + if request_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY): + return # internal sub-call (our own shadow/judge, a classifier), not user traffic + # redaction rewrites logged content before callbacks run, so this hook + # only ever sees placeholders for a redacted request + if should_redact_message_logging(dict(kwargs)): # mutable-ok: predicate takes a plain dict + return + metadata: Final = payload.get("metadata") or _EMPTY_METADATA + api_key_hash: Final = metadata.get("user_api_key_hash") + if not api_key_hash: + return + job: Final = (await self._active_jobs()).get(str(api_key_hash)) + if job is None: + return + if datetime.now(timezone.utc) >= job.ends_at: + return + if job.attempts + self._job_starts.get(job.id, 0) >= job.max_turns: + return + request_id: Final = payload.get("id") or "" + if not request_id: + return + if not _sample_hits(request_id, job.id, job.shadow_percentage): + return + if payload.get("call_type") not in _SAMPLED_CALL_TYPES: + return # only known chat-shaped traffic is comparable; unknown or missing types fail closed + if _request_was_routed_by(request_metadata, job.router_name): + return + if self._inflight_shadow_tasks >= _MAX_CONCURRENT_SHADOW_TASKS: + return + raw_messages: Final = kwargs.get("messages") + self._job_starts[job.id] = self._job_starts.get(job.id, 0) + 1 + self._inflight_shadow_tasks += 1 + task: Final = asyncio.create_task( + self._run_shadow_eval( + job=job, + request_id=request_id, + messages=tuple(m for m in raw_messages if isinstance(m, Mapping)) + if isinstance(raw_messages, Sequence) + else (), + response_obj=response_obj, + real_model=payload.get("model") or "", + model_parameters=MappingProxyType( + dict(payload.get("model_parameters") or {}) # mutable-ok: frozen snapshot + ), + parent_metadata=MappingProxyType(dict(request_metadata)), # mutable-ok: frozen snapshot + ) + ) + task.add_done_callback(self._release_shadow_slot) + except Exception as e: # noqa: BLE001 # logging hooks must never fail the request + verbose_logger.debug("shadow_eval: failed to schedule task: %s", e) + + def _release_shadow_slot(self, _task: "asyncio.Task[None]") -> None: + self._inflight_shadow_tasks -= 1 + + #### the detached pipeline: one attempt row per sampled request, verdict or error #### + + async def _run_shadow_eval( + self, + job: ActiveShadowEvalJob, + request_id: str, + messages: Sequence[Mapping[str, object]], + response_obj: object, + real_model: str, + model_parameters: Mapping[str, object], + parent_metadata: Mapping[str, object], + ) -> None: + """Budget gate -> shadow call -> blind judge -> one attempt row. The prisma gate + sits above the dispatch so no provider spend happens without a place to record + the outcome, and the budget read lives here rather than in the success hook.""" + prisma: Final = self._prisma_provider() + try: + if prisma is None: + return + real_text: Final = self._extract_response_text(response_obj) + if not real_text or not messages: + return + if await _key_or_team_is_over_budget(parent_metadata): + return + + shadow: Final = await self._call_router_shadow(job.router_name, messages, model_parameters, parent_metadata) + if isinstance(shadow, _CallFailure): + await self._record_attempt(prisma, job, request_id, outcome="error", error=shadow.error) + return + + verdict: Final = await self._call_judge( + judge_model=job.judge_model, + messages=messages, + real_text=real_text, + shadow_text=shadow.text, + parent_metadata=parent_metadata, + ) + if isinstance(verdict, _CallFailure): + await self._record_attempt( + prisma, + job, + request_id, + outcome="error", + error=verdict.error, + shadow=shadow, + judge_cost=verdict.cost, + ) + return + await self._record_attempt( + prisma, + job, + request_id, + outcome=verdict.preference, + shadow=shadow, + real_model=real_model, + confidence=verdict.confidence, + judge_cost=verdict.cost, + ) + except Exception as e: # noqa: BLE001 # detached task: record what happened, never raise + verbose_logger.debug("shadow_eval: pipeline failed for %s: %s", request_id, e) + await self._record_attempt(prisma, job, request_id, outcome="error", error=f"pipeline error: {e}") + + @staticmethod + async def _record_attempt( + prisma: "PrismaClient | None", + job: ActiveShadowEvalJob, + request_id: str, + *, + outcome: str, + shadow: _ShadowResponse | None = None, + real_model: str = "", + confidence: float | None = None, + judge_cost: float = 0.0, + error: str | None = None, + ) -> None: + if prisma is None: + return + try: + await prisma.db.litellm_shadowevalattempt.create( + data={ # mutable-ok: Prisma payload + "job_id": job.id, + "request_id": request_id, + "outcome": outcome, + "tier": shadow.tier if shadow else None, + "real_model": real_model or None, + "shadow_model": shadow.model if shadow else None, + "confidence": confidence, + "judge_cost": judge_cost, + "error": error[:_MAX_ERROR_CHARS] if error else None, + } + ) + except Exception as e: # noqa: BLE001 # a lost row degrades sample size, nothing can disagree with it + verbose_logger.debug("shadow_eval: attempt write failed for %s: %s", request_id, e) + + async def _call_router_shadow( + self, + router_name: str, + messages: Sequence[Mapping[str, object]], + model_parameters: Mapping[str, object], + parent_metadata: Mapping[str, object], + ) -> "_ShadowResponse | _CallFailure": + """Send the prompt through the auto-router being evaluated. The metadata carries + the shadowed key's identity (spend attribution) and receives the router's routing + decision write-back, read back for tier attribution.""" + router: Final = self._router_provider() + if router is None: + return _CallFailure("no router configured on this pod") + shadow_metadata: Final[dict[str, object]] = ( # mutable-ok: router writes its routing decision back + sanitized_forwardable_call_metadata(parent_metadata, SHADOW_EVAL_ROUTER_CALL_ORIGIN) + ) + shadow_params: Final = { # mutable-ok: splatted as kwargs + k: v for k, v in model_parameters.items() if k not in ("stream", "metadata") + } + try: + response: Final = await router.acompletion( + model=router_name, + messages=messages, # pyright: ignore[reportArgumentType] # snapshot of the SDK's own message dicts + metadata=shadow_metadata, + num_retries=0, + fallbacks=[], # mutable-ok: SDK kwarg; a failed shadow is a recorded error, never a spend multiplier + **shadow_params, + ) + except Exception as e: # noqa: BLE001 # provider errors become error rows, not crashes + verbose_logger.debug("shadow_eval: router call failed: %s", e) + return _CallFailure(f"shadow router call failed: {e}") + text: Final = self._extract_response_text(response) + if not text: + return _CallFailure("shadow router returned an empty response") + raw_decision: Final = shadow_metadata.get("routing_decision") + routing_decision: Final = raw_decision if isinstance(raw_decision, Mapping) else _EMPTY_METADATA + raw_tier: Final = routing_decision.get("tier_label") or routing_decision.get("tier") + return _ShadowResponse( + text=text, + model=str(getattr(response, "model", None) or routing_decision.get("routed_model") or ""), + tier=str(raw_tier) if raw_tier is not None else None, + ) + + async def _call_judge( + self, + judge_model: str, + messages: Sequence[Mapping[str, object]], + real_text: str, + shadow_text: str, + parent_metadata: Mapping[str, object], + ) -> "_JudgeVerdict | _CallFailure": + """Blind pairwise judge with A/B labels randomized to cancel position bias.""" + real_is_a: Final = random.random() < 0.5 + response_a: Final = real_text if real_is_a else shadow_text + response_b: Final = shadow_text if real_is_a else real_text + + conversation: Final = "\n".join( + f"{str(m.get('role', 'user')).upper()}: {extract_text_from_content(m.get('content'))}" + for m in messages + if m.get("content") is not None + ) + judge_metadata: Final = sanitized_forwardable_call_metadata(parent_metadata, SHADOW_EVAL_JUDGE_CALL_ORIGIN) + judge_messages: Final = [ # mutable-ok: SDK takes a list + {"role": "system", "content": PAIRWISE_JUDGE_SYSTEM_PROMPT}, # mutable-ok: SDK message + { + "role": "user", + "content": _judge_user_prompt(conversation, response_a, response_b), + }, # mutable-ok: SDK message + ] + try: + response: Final = await judge_acompletion( + self._router_provider(), + judge_model, + judge_messages, # pyright: ignore[reportArgumentType] # plain SDK message dicts + temperature=0, + max_tokens=JUDGE_MAX_OUTPUT_TOKENS, + metadata=judge_metadata, + ) + except Exception as e: # noqa: BLE001 # judge outages become error rows, not crashes + verbose_logger.debug("shadow_eval: judge call failed: %s", e) + return _CallFailure(f"judge call failed: {e}") + try: + raw: Final = response["choices"][0]["message"]["content"] or "" + verdict: Final = PairwiseVerdict.model_validate(parse_json_verdict(raw)) + except Exception as e: # noqa: BLE001 # malformed verdicts become error rows + verbose_logger.debug("shadow_eval: unparseable judge verdict: %s", e) + return _CallFailure(f"unparseable judge verdict: {e}", cost=_judge_call_cost(response)) + return _JudgeVerdict( + preference=_unmask_preference(verdict.preference, real_is_a), + confidence=max(0.0, min(1.0, verdict.confidence)), + cost=_judge_call_cost(response), + ) + + @staticmethod + def _extract_response_text(response_obj: object) -> str: + """Extract the assistant's text from a ModelResponse-shaped object or dict.""" + try: + content: Final = ( + response_obj["choices"][0]["message"]["content"] + if isinstance(response_obj, Mapping) + else response_obj.choices[0].message.content # pyright: ignore[reportAttributeAccessIssue] # duck-typed ModelResponse + ) + except (AttributeError, KeyError, IndexError, TypeError): + return "" + return extract_text_from_content(content) + + +_EMPTY_JOBS: Final[Mapping[str, ActiveShadowEvalJob]] = MappingProxyType({}) + + +def _default_prisma_provider() -> "PrismaClient | None": + try: + from litellm.proxy.proxy_server import prisma_client + except ImportError: + return None + return prisma_client diff --git a/litellm/litellm_core_utils/internal_call_metadata.py b/litellm/litellm_core_utils/internal_call_metadata.py new file mode 100644 index 00000000000..6815727de69 --- /dev/null +++ b/litellm/litellm_core_utils/internal_call_metadata.py @@ -0,0 +1,94 @@ +"""Metadata a request forwards to the internal LLM sub-calls it triggers. + +Internal features (the auto-router's classifier and embeddings, shadow eval's shadow and +judge calls) bill real provider spend that nobody typed a prompt for. That spend must land +on the same key/team/org/user as the request that caused it, so the sub-call carries the +caller's identity metadata, minus two things that must never be forwarded as-is: + +* ``user_api_key_budget_reservation`` (and the reservation nested inside + ``user_api_key_auth``) belongs to the parent completion. If a sub-call's cost callback + sees it, that callback finalizes the reservation and the parent's own callback then + skips incrementing the key/team budget counters, losing the parent's spend. + ``user_api_key_auth`` itself is kept, sanitized, because model access-group filtering + needs it. +* The sub-call is stamped with ``INTERNAL_CALL_ORIGIN_METADATA_KEY`` so its spend log row + records that it is not traffic the caller sent. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Final + +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.types.utils import InternalCallOrigin + +BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"}) + +_USER_API_KEY_AUTH_KEY: Final = "user_api_key_auth" + +FORWARDABLE_IDENTITY_METADATA_KEYS: Final = frozenset( + { + "user_api_key", + "user_api_key_hash", + "user_api_key_alias", + "user_api_key_team_id", + "user_api_key_org_id", + "user_api_key_user_id", + "user_api_key_end_user_id", + _USER_API_KEY_AUTH_KEY, + } +) +"""The caller-identity subset a detached sub-call needs to be attributed and +budget-checked like the request that spawned it. Everything else on the parent's metadata +(routing decision, guardrail state, logging payload) describes the parent call and would +be a lie on a sub-call that runs after it returned.""" + + +def sanitize_user_api_key_auth(auth: object) -> object: + """Copy of the auth object with its budget reservation removed; the cost callback + falls back to reading the reservation from inside the auth object.""" + if isinstance(auth, dict): + return {k: v for k, v in auth.items() if k != "budget_reservation"} # mutable-ok: SDK metadata value + reservation: Final[object] = getattr(auth, "budget_reservation", None) + model_copy: Final[object] = getattr(auth, "model_copy", None) + if reservation is not None and callable(model_copy): + return model_copy(update={"budget_reservation": None}) # mutable-ok: pydantic update payload + return auth + + +def _sanitized(parent_metadata: Mapping[str, object]) -> dict[str, object]: # mutable-ok: SDK metadata kwarg + return { # mutable-ok: SDK metadata kwarg + k: sanitize_user_api_key_auth(v) if k == _USER_API_KEY_AUTH_KEY else v + for k, v in parent_metadata.items() + if k not in BUDGET_RESERVATION_METADATA_KEYS + } + + +def forwarded_internal_call_metadata( + parent_metadata: Mapping[str, object] | None, + call_origin: InternalCallOrigin, +) -> dict[str, object]: # mutable-ok: SDK metadata kwarg + """Parent metadata, minus its budget reservation, stamped with the sub-call's origin. + + For sub-calls made inside the parent request (classifier, embeddings), where the + parent's full context still describes the call being made. + """ + if not parent_metadata: + return {} # mutable-ok: SDK metadata kwarg + return _sanitized(parent_metadata) | { # mutable-ok: SDK metadata kwarg + INTERNAL_CALL_ORIGIN_METADATA_KEY: call_origin + } + + +def sanitized_forwardable_call_metadata( + parent_metadata: Mapping[str, object], + call_origin: InternalCallOrigin, +) -> dict[str, object]: # mutable-ok: SDK metadata kwarg + """Just the caller's identity, stamped with the sub-call's origin. + + For sub-calls detached from the parent request (shadow eval), which outlive it and + must not inherit per-request state such as its routing decision or logging payload. + """ + identity: Final = {k: v for k, v in parent_metadata.items() if k in FORWARDABLE_IDENTITY_METADATA_KEYS} + return _sanitized(identity) | {INTERNAL_CALL_ORIGIN_METADATA_KEY: call_origin} # mutable-ok: SDK metadata kwarg diff --git a/litellm/litellm_core_utils/llm_judge.py b/litellm/litellm_core_utils/llm_judge.py new file mode 100644 index 00000000000..4ad8d719402 --- /dev/null +++ b/litellm/litellm_core_utils/llm_judge.py @@ -0,0 +1,87 @@ +"""Shared primitives for LLM-judge features (llm_as_a_judge guardrail, shadow eval).""" + +from __future__ import annotations + +import json +import re +from typing import TYPE_CHECKING, Final + +import litellm + +if TYPE_CHECKING: + from litellm import Router + from litellm.types.llms.openai import AllMessageValues + from litellm.types.utils import ModelResponse + +JSON_FENCE_RE: Final = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE) + + +def default_router_provider() -> Router | None: + try: + from litellm.proxy.proxy_server import llm_router + except ImportError: + return None + + return llm_router + + +def parse_json_verdict(raw: str) -> dict[str, object]: # mutable-ok: plain parsed-JSON payload + """Parse a judge's JSON verdict, tolerating markdown fences and surrounding prose.""" + text = raw.strip() # rebind-ok: progressively narrowed to the JSON payload + fenced: Final = JSON_FENCE_RE.search(text) + if fenced is not None: + text = fenced.group(1).strip() # rebind-ok: progressively narrowed to the JSON payload + parsed: object + try: + parsed = json.loads(text) + except json.JSONDecodeError: + start: Final = text.find("{") + end: Final = text.rfind("}") + if start == -1 or end <= start: + raise + parsed = json.loads(text[start : end + 1]) + if not isinstance(parsed, dict): + raise ValueError("judge response is not a JSON object") + return {str(k): v for k, v in parsed.items()} # mutable-ok: plain parsed-JSON payload + + +def extract_text_from_content(content: object) -> str: + """Return plain text from a message content field (str or multimodal list).""" + if isinstance(content, str): + return content + if isinstance(content, list): + return " ".join( + str(part.get("text", "")) for part in content if isinstance(part, dict) and part.get("type") == "text" + ) + return "" + + +def router_resolves_model(router: Router | None, model: str) -> bool: + """Whether the model name resolves through the proxy's router (configured deployment + or model-group alias), the same check the judge dispatch itself makes, so start-time + validation cannot accept a name the call path then fails on.""" + return router is not None and bool(model in router.model_group_alias or router.get_model_list(model_name=model)) + + +async def judge_acompletion( + router: Router | None, + judge_model: str, + messages: list[AllMessageValues], # mutable-ok: the SDK acompletion signature takes a list + **params: object, +) -> ModelResponse: + """Dispatch a judge call through the proxy's router when the judge model is a + configured deployment (DB-stored credentials work), through the SDK for + provider-qualified public names. The router path never retries or falls back: + a failed judge call is the caller's counted failure, not a spend multiplier. + Sampling preferences are advisory: models that removed sampling params (e.g. + claude-sonnet-5) drop them instead of rejecting the judge call.""" + if router_resolves_model(router, judge_model): + return await router.acompletion( # pyright: ignore[reportOptionalMemberAccess] # router_resolves_model implies router is not None + model=judge_model, + messages=messages, + num_retries=0, + fallbacks=[], + drop_params=True, + **params, + ) + return await litellm.acompletion(model=judge_model, messages=messages, num_retries=0, drop_params=True, **params) diff --git a/litellm/proxy/db/autorouter_session_rollup.py b/litellm/proxy/db/autorouter_session_rollup.py index 9732b1d7402..96192b884d8 100644 --- a/litellm/proxy/db/autorouter_session_rollup.py +++ b/litellm/proxy/db/autorouter_session_rollup.py @@ -24,6 +24,7 @@ from itertools import groupby from typing import TYPE_CHECKING, Final, NamedTuple from litellm._logging import verbose_proxy_logger +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.proxy._types import DB_RETRY_SAFE_ERROR_TYPES if TYPE_CHECKING: @@ -180,12 +181,17 @@ def build_autorouter_turn_transaction( The routing_decision record is what says a request was auto-routed at all, so a request without one (including the auto-router's own classifier sub-calls) never - reaches the rollup. Failed requests served nothing and are excluded. Cache facts - are derived from the payload's own usage record through the savings owner, never - handed in beside it. + reaches the rollup. Internal sub-calls that DO carry one (a shadow eval's duplicate + of a request through the router) are excluded by their internal_call_origin stamp: + they are not traffic a user sent, so counting them would manufacture sessions and + savings in the adoption metrics. Failed requests served nothing and are excluded. + Cache facts are derived from the payload's own usage record through the savings + owner, never handed in beside it. """ if payload.get("status") != "success": return None + if metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY): + return None routing_decision: Final = metadata.get("routing_decision") if not isinstance(routing_decision, Mapping) or not routing_decision: return None diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index b0130db232a..b2b72c1cac4 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -21,6 +21,7 @@ from litellm.caching import RedisCache from litellm.constants import ( DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME, DB_SPEND_UPDATE_JOB_NAME, + INTERNAL_CALL_ORIGIN_METADATA_KEY, ) from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.proxy._types import ( @@ -1794,6 +1795,7 @@ class DBSpendUpdateWriter: if call_type: endpoint = ROUTE_ENDPOINT_MAPPING.get(call_type, None) + is_internal_call: Final = bool(_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) cache_read_input_tokens: Final = extract_cache_read_tokens(usage_obj) compression_saved_tokens: Final = extract_compression_saved_tokens(_metadata) savings_spend: Final = compute_savings_spend( @@ -1818,15 +1820,20 @@ class DBSpendUpdateWriter: prompt_tokens=payload["prompt_tokens"], completion_tokens=payload["completion_tokens"], spend=payload["spend"], - api_requests=1, - successful_requests=1 if request_status == "success" else 0, - failed_requests=1 if request_status != "success" else 0, + # Internal sub-calls (auto-router classifier, shadow eval's shadow and + # judge) bill real spend and tokens to the key, but they are not + # requests the caller made: counting them inflates request-volume + # readers, and an auto-router savings figure computed on a shadow + # duplicate credits savings for traffic no user sent. + api_requests=0 if is_internal_call else 1, + successful_requests=1 if not is_internal_call and request_status == "success" else 0, + failed_requests=1 if not is_internal_call and request_status != "success" else 0, cache_read_input_tokens=cache_read_input_tokens, cache_creation_input_tokens=extract_cache_creation_tokens(usage_obj), compression_saved_tokens=compression_saved_tokens, compression_savings_spend=savings_spend.compression, prompt_caching_savings_spend=savings_spend.prompt_caching, - autorouter_savings_spend=savings_spend.autorouter, + autorouter_savings_spend=0.0 if is_internal_call else savings_spend.autorouter, ) return daily_transaction except Exception as e: diff --git a/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py index 1907bb19abf..e3f67f0024b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py @@ -1,16 +1,20 @@ """LLM-as-a-Judge guardrail: uses an LLM to score responses against weighted criteria.""" -import json -import re from collections.abc import Callable from datetime import datetime -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast +from typing import TYPE_CHECKING, Any, Final, Literal, Optional from fastapi import HTTPException import litellm from litellm._logging import verbose_logger from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.llm_judge import ( + default_router_provider, + extract_text_from_content, + judge_acompletion, + parse_json_verdict, +) from litellm.types.guardrails import GuardrailEventHooks, SupportedGuardrailIntegrations from litellm.types.utils import GenericGuardrailAPIInputs, GuardrailStatus @@ -32,50 +36,9 @@ Return ONLY valid JSON in this exact format: _VALID_ON_FAILURE: Final = frozenset({"block", "log"}) - -def _default_router_provider() -> "Router | None": - try: - from litellm.proxy.proxy_server import llm_router - except ImportError: - return None - - return llm_router - - -_JSON_FENCE_RE: Final = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE) - - -def _parse_judge_verdict(raw: str) -> dict[str, Any]: - """Parse the judge's JSON verdict, tolerating markdown fences and surrounding prose.""" - text = raw.strip() - fenced: Final = _JSON_FENCE_RE.search(text) - if fenced is not None: - text = fenced.group(1).strip() - parsed: object - try: - parsed = json.loads(text) - except json.JSONDecodeError: - start: Final = text.find("{") - end: Final = text.rfind("}") - if start == -1 or end <= start: - raise - parsed = json.loads(text[start : end + 1]) - if not isinstance(parsed, dict): - raise ValueError("judge response is not a JSON object") - return cast(dict[str, Any], parsed) # cast-ok: narrowed to dict by the isinstance guard above - - -def _extract_text_from_content(content: Any) -> str: - """Return plain text from a message content field (str or multimodal list).""" - if isinstance(content, str): - return content - if isinstance(content, list): - parts: Final = [] - for part in content: - if isinstance(part, dict) and part.get("type") == "text": - parts.append(part.get("text", "")) - return " ".join(parts) - return "" +_default_router_provider: Final = default_router_provider +_parse_judge_verdict: Final = parse_json_verdict +_extract_text_from_content: Final = extract_text_from_content def _get_litellm_param( @@ -168,25 +131,13 @@ class LLMAsAJudgeGuardrail(CustomGuardrail): "content": _build_judge_prompt(self.criteria, messages, response_text), }, ] - router: Final = self._router_provider() - if router is not None and ( - self.judge_model in router.model_group_alias or router.get_model_list(model_name=self.judge_model) - ): - response = await router.acompletion( - model=self.judge_model, - messages=judge_messages, - response_format={"type": "json_object"}, - temperature=0, - num_retries=0, - fallbacks=[], - ) - else: - response = await litellm.acompletion( - model=self.judge_model, - messages=judge_messages, - response_format={"type": "json_object"}, - temperature=0, - ) + response: Final = await judge_acompletion( + self._router_provider(), + self.judge_model, + judge_messages, + response_format={"type": "json_object"}, + temperature=0, + ) raw: Final = response.choices[0].message.content or "{}" return _parse_judge_verdict(raw) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 3fd2adda480..f62dbec2e85 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -24,7 +24,7 @@ from typing import ( from litellm import DualCache from litellm._logging import verbose_proxy_logger -from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE +from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages, @@ -2991,6 +2991,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): rate_limit_type: Literal["output", "input", "total"], ) -> list[RedisPipelineIncrementOperation]: """Build Redis pipeline increment ops for TPM / parallel-request counters.""" + from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs from litellm.proxy.common_utils.callback_utils import ( get_model_group_from_litellm_kwargs, ) @@ -2998,6 +2999,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # Get metadata from standard_logging_object - this correctly handles both # 'metadata' and 'litellm_metadata' fields from litellm_params standard_logging_object: Final = kwargs.get("standard_logging_object") or {} + request_metadata: Final = get_litellm_metadata_from_kwargs(kwargs) + if request_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY): + # Internal sub-calls bill spend to the caller but are not the caller's + # traffic; charging them here would let background evals eat TPM headroom. + return [] standard_logging_metadata: Final = standard_logging_object.get("metadata") or {} model_group: Final = get_model_group_from_litellm_kwargs(kwargs) diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 8b6aafea751..cb0e8dba62a 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -13,6 +13,7 @@ from pydantic import BaseModel, TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.exceptions import BudgetExceededError +from litellm.litellm_core_utils.llm_judge import router_resolves_model from litellm.proxy._types import ( CommonProxyErrors, LiteLLM_TeamTable, @@ -39,11 +40,16 @@ from litellm.types.management_endpoints.auto_router_endpoints import ( AutoRouterRoutingTestRequest, AutoRouterRoutingTestResponse, RequestComplexityRouterConfig, + ShadowEvalJobResponse, + ShadowEvalResult, + ShadowEvalSlice, + StartShadowEvalRequest, ) if TYPE_CHECKING: from fastapi import APIRouter, Depends, HTTPException, Query, status + from litellm.proxy.utils import PrismaClient from litellm.router import Router else: try: @@ -388,14 +394,7 @@ async def get_auto_router_benchmarks( """ from litellm.proxy.proxy_server import prisma_client - if user_api_key_dict.user_role not in ( - LitellmUserRoles.PROXY_ADMIN, - LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, - ): - raise HTTPException( - status_code=403, - detail="Only proxy admin roles can view auto-router benchmarks across the deployment", - ) + _require_admin_viewer(user_api_key_dict, "view auto-router benchmarks across the deployment") if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) @@ -430,3 +429,335 @@ async def get_auto_router_benchmarks( totals=_benchmark_totals(_summed_agg_row(rows)), groups=groups, ) + + +# --------------------------------------------------------------------------- +# Shadow eval: pre-adoption evaluation of an auto-router against live traffic. +# The job row is immutable config plus stopped_at; status, counts, spend, and errors +# are derived from the append-only attempt rows, so reads here are aggregations +# bounded by each job's max_turns through the attempt table's job_id index. +# --------------------------------------------------------------------------- + + +def _require_admin_viewer(user_api_key_dict: UserAPIKeyAuth, action: str) -> None: + if user_api_key_dict.user_role not in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ): + raise HTTPException(status_code=403, detail=f"Only proxy admin roles can {action}") + + +def _require_admin_writer(user_api_key_dict: UserAPIKeyAuth, action: str) -> None: + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException(status_code=403, detail=f"Only a proxy admin can {action}") + + +def _is_configured_pre_routing_strategy(llm_router: "Router", router_name: str) -> bool: + return any( + router_name in registry + for registry in ( + llm_router.auto_routers, + llm_router.complexity_routers, + llm_router.adaptive_routers, + llm_router.quality_routers, + ) + ) + + +def _validate_judge_model(llm_router: "Router | None", judge_model: str) -> None: + """Reject a judge model the dispatch path cannot resolve, at start rather than as a + silently growing error count once the job is already sampling and billing.""" + if llm_router is not None and _is_configured_pre_routing_strategy(llm_router, judge_model): + raise HTTPException( + status_code=400, + detail=f"judge_model '{judge_model}' is an auto-router; the judge must be a plain model", + ) + if router_resolves_model(llm_router, judge_model): + return + import litellm + + try: + litellm.get_llm_provider(model=judge_model) + except Exception as e: + raise HTTPException( + status_code=400, + detail=( + f"judge_model '{judge_model}' is neither a model configured on this proxy nor a " + "provider-qualified public model name (e.g. 'anthropic/claude-sonnet-5')" + ), + ) from e + + +def _is_unique_violation(error: Exception) -> bool: + """Whether a Prisma create failed on a unique index. One active job per key lives in + a partial unique index (raw SQL in the migration; schema.prisma cannot express partial + indexes), so the read-then-create check above it is advisory: two concurrent starts + pass the read, and the loser must surface as the same 409 rather than a 500.""" + try: + from prisma.errors import UniqueViolationError + except ImportError: + return "unique constraint" in str(error).lower() or "P2002" in str(error) + return isinstance(error, UniqueViolationError) + + +class _AttemptAggRow(BaseModel): + grp: str + turn_count: int + real_wins: int + shadow_wins: int + ties: int + avg_confidence: float | None + + +_ATTEMPT_AGG_ROWS: Final = TypeAdapter(list[_AttemptAggRow]) + +_ATTEMPT_AGG_SELECT: Final = """ + COUNT(*)::int AS turn_count, + COUNT(*) FILTER (WHERE outcome = 'real')::int AS real_wins, + COUNT(*) FILTER (WHERE outcome = 'shadow')::int AS shadow_wins, + COUNT(*) FILTER (WHERE outcome = 'tie')::int AS ties, + AVG(confidence)::float AS avg_confidence +FROM "LiteLLM_ShadowEvalAttempt" +WHERE job_id = $1 AND outcome != 'error' +GROUP BY 1 +""" + +_ATTEMPT_AGG_BY_TIER_SQL: Final = "SELECT COALESCE(tier, 'UNCLASSIFIED') AS grp," + _ATTEMPT_AGG_SELECT +_ATTEMPT_AGG_BY_MODEL_SQL: Final = "SELECT COALESCE(real_model, 'unknown') AS grp," + _ATTEMPT_AGG_SELECT + +_SWEEP_FINISHED_JOBS_SQL: Final = """ +UPDATE "LiteLLM_ShadowEvalJob" j SET stopped_at = NOW() +WHERE j.api_key_id = $1 AND j.stopped_at IS NULL + AND ( + j.ends_at <= NOW() + OR (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_turns + ) +""" + +_ATTEMPT_TOTALS_SQL: Final = """ +SELECT + COUNT(*) FILTER (WHERE outcome != 'error')::int AS judged_count, + COUNT(*) FILTER (WHERE outcome = 'error')::int AS error_count, + COALESCE(SUM(judge_cost), 0)::float AS judge_spend +FROM "LiteLLM_ShadowEvalAttempt" +WHERE job_id = $1 +""" + + +class _AttemptTotalsRow(BaseModel): + judged_count: int + error_count: int + judge_spend: float + + +_ATTEMPT_TOTALS_ROWS: Final = TypeAdapter(list[_AttemptTotalsRow]) + + +def _pct_of(numerator: int, denominator: int) -> float: + return _pct(numerator, denominator) + + +def _slices(rows: Sequence[_AttemptAggRow]) -> tuple[ShadowEvalSlice, ...]: + return tuple( + ShadowEvalSlice( + group=row.grp, + turn_count=row.turn_count, + real_win_rate_pct=_pct_of(row.real_wins, row.turn_count), + shadow_win_rate_pct=_pct_of(row.shadow_wins, row.turn_count), + tie_rate_pct=_pct_of(row.ties, row.turn_count), + avg_judge_confidence=round(row.avg_confidence or 0.0, 3), + ) + for row in sorted(rows, key=lambda r: r.turn_count, reverse=True) + ) + + +async def _shadow_eval_results(prisma_client: "PrismaClient", job_id: str) -> ShadowEvalResult | None: + """Both stratifications of one job's verdicts. Tier answers "where does the router do + well"; current-model answers "which of the models this key uses today would the router + beat". Reads are bounded by the job's own attempts (<= max_turns) via the job_id index.""" + by_tier: Final = _ATTEMPT_AGG_ROWS.validate_python( + await prisma_client.db.query_raw(_ATTEMPT_AGG_BY_TIER_SQL, job_id) or () + ) + if not by_tier: + return None + by_model: Final = _ATTEMPT_AGG_ROWS.validate_python( + await prisma_client.db.query_raw(_ATTEMPT_AGG_BY_MODEL_SQL, job_id) or () + ) + total_turns: Final = sum(r.turn_count for r in by_tier) + return ShadowEvalResult( + by_tier=_slices(by_tier), + by_current_model=_slices(by_model), + overall_shadow_win_rate_pct=_pct_of(sum(r.shadow_wins for r in by_tier), total_turns), + overall_tie_rate_pct=_pct_of(sum(r.ties for r in by_tier), total_turns), + ) + + +@router.post( + "/auto_router/shadow_eval/start", + tags=("auto router",), + dependencies=(Depends(user_api_key_auth),), + response_model=ShadowEvalJobResponse, + status_code=status.HTTP_201_CREATED, +) +async def start_shadow_eval( + data: StartShadowEvalRequest, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> ShadowEvalJobResponse: + """ + Start a pre-adoption shadow eval: duplicate a sampled slice of a key's live traffic + through an auto-router, judge real vs. shadow responses blind, and stratify win rates + by the router's tier classification and by the incumbent model. + + Shadow responses are never served to users. The job samples until it has judged + max_turns turns, reaches the end of its window, or is stopped; sampling changes + propagate to pods within about 10 seconds. Shadow and judge calls bill to the + shadowed key but are excluded from request counts and auto-router adoption metrics. + """ + from litellm.proxy.proxy_server import llm_router, prisma_client + + _require_admin_writer(user_api_key_dict, "start a shadow eval") + if prisma_client is None: + raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) + if llm_router is None or not _is_configured_pre_routing_strategy(llm_router, data.router_name): + raise HTTPException(status_code=400, detail=f"'{data.router_name}' is not a configured auto-router") + _validate_judge_model(llm_router, data.judge_model) + key_row: Final = await prisma_client.db.litellm_verificationtoken.find_unique( + where={"token": data.api_key_id} # mutable-ok: Prisma filter + ) + if key_row is None: + raise HTTPException( + status_code=400, + detail=( + f"api_key_id '{data.api_key_id}' is not a key on this proxy; pass the key's token hash, " + "the value the key list and key info endpoints report" + ), + ) + + # A job that expired or exhausted its turn budget stopped sampling on its own, but + # still holds the one-active-per-key partial unique index until stamped; free it so + # a new eval can start. + await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, data.api_key_id) + active: Final = await prisma_client.db.litellm_shadowevaljob.find_first( + where={"api_key_id": data.api_key_id, "stopped_at": None}, # mutable-ok: Prisma filter + ) + if active is not None: + raise HTTPException( + status_code=409, + detail=f"Key already has an active shadow eval job ({active.id}). Stop it first.", + ) + now: Final = datetime.now(timezone.utc) + try: + job: Final = await prisma_client.db.litellm_shadowevaljob.create( + data={ # mutable-ok: Prisma payload + "api_key_id": data.api_key_id, + "router_name": data.router_name, + "judge_model": data.judge_model, + "shadow_percentage": data.shadow_percentage, + "max_turns": data.max_turns, + "created_by": user_api_key_dict.user_id, + "ends_at": now + timedelta(days=data.duration_days), + } + ) + except Exception as e: + if not _is_unique_violation(e): + raise + raise HTTPException( + status_code=409, + detail="Key already has an active shadow eval job (started concurrently). Stop it first.", + ) from e + return ShadowEvalJobResponse.model_validate(job, from_attributes=True) + + +@router.get( + "/auto_router/shadow_eval", + tags=("auto router",), + dependencies=(Depends(user_api_key_auth),), + response_model=list[ShadowEvalJobResponse], +) +async def list_shadow_eval_jobs( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + api_key_id: Annotated[str | None, Query(description="Filter to jobs shadowing this key")] = None, + limit: Annotated[int, Query(ge=1, le=200, description="Newest jobs to return")] = 50, +) -> tuple[ShadowEvalJobResponse, ...]: + """List shadow eval jobs, newest first. Counts and results ride the detail endpoint only.""" + from litellm.proxy.proxy_server import prisma_client + + _require_admin_viewer(user_api_key_dict, "view shadow evals") + if prisma_client is None: + raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) + records: Final = await prisma_client.db.litellm_shadowevaljob.find_many( + where={"api_key_id": api_key_id} if api_key_id else {}, # mutable-ok: Prisma filter + order={"created_at": "desc"}, # mutable-ok: Prisma order + take=limit, + ) + return tuple(ShadowEvalJobResponse.model_validate(record, from_attributes=True) for record in records or ()) + + +@router.get( + "/auto_router/shadow_eval/{job_id}", + tags=("auto router",), + dependencies=(Depends(user_api_key_auth),), + response_model=ShadowEvalJobResponse, +) +async def get_shadow_eval_job( + job_id: str, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> ShadowEvalJobResponse: + """One job with derived counts, judge spend, latest error, and stratified results.""" + from litellm.proxy.proxy_server import prisma_client + + _require_admin_viewer(user_api_key_dict, "view shadow evals") + if prisma_client is None: + raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) + record: Final = await prisma_client.db.litellm_shadowevaljob.find_unique( + where={"id": job_id} # mutable-ok: Prisma filter + ) + if record is None: + raise HTTPException(status_code=404, detail=f"No shadow eval job {job_id}") + totals: Final = _ATTEMPT_TOTALS_ROWS.validate_python( + await prisma_client.db.query_raw(_ATTEMPT_TOTALS_SQL, job_id) or () + ) + latest_error: Final = await prisma_client.db.litellm_shadowevalattempt.find_first( + where={"job_id": job_id, "outcome": "error"}, # mutable-ok: Prisma filter + order={"created_at": "desc"}, # mutable-ok: Prisma order + ) + return ShadowEvalJobResponse.model_validate(record, from_attributes=True).model_copy( + update={ # mutable-ok: pydantic update payload + "judged_count": totals[0].judged_count if totals else 0, + "error_count": totals[0].error_count if totals else 0, + "judge_spend": round(totals[0].judge_spend, 6) if totals else 0.0, + "last_error": latest_error.error if latest_error else None, + "results": await _shadow_eval_results(prisma_client, job_id), + } + ) + + +@router.post( + "/auto_router/shadow_eval/{job_id}/stop", + tags=("auto router",), + dependencies=(Depends(user_api_key_auth),), + response_model=ShadowEvalJobResponse, +) +async def stop_shadow_eval_job( + job_id: str, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> ShadowEvalJobResponse: + """Stop an active shadow eval job. Attempts are kept; sampling halts within ~10s.""" + from litellm.proxy.proxy_server import prisma_client + + _require_admin_writer(user_api_key_dict, "stop a shadow eval") + if prisma_client is None: + raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) + record: Final = await prisma_client.db.litellm_shadowevaljob.find_unique( + where={"id": job_id} # mutable-ok: Prisma filter + ) + if record is None: + raise HTTPException(status_code=404, detail=f"No shadow eval job {job_id}") + current: Final = ShadowEvalJobResponse.model_validate(record, from_attributes=True) + if current.status != "running": + raise HTTPException(status_code=400, detail=f"Job {job_id} is already {current.status}") + updated: Final = await prisma_client.db.litellm_shadowevaljob.update( + where={"id": job_id}, # mutable-ok: Prisma filter + data={"stopped_at": datetime.now(timezone.utc)}, # mutable-ok: Prisma payload + ) + return ShadowEvalJobResponse.model_validate(updated, from_attributes=True) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 64a91b880cd..968c645ffcd 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2329,8 +2329,11 @@ def load_from_azure_key_vault(use_azure_key_vault: bool = False): def cost_tracking(): global prisma_client if prisma_client is not None: + from litellm.integrations.shadow_eval_logger import ShadowEvalLogger + litellm.logging_callback_manager.add_litellm_callback(_ProxyDBLogger()) litellm.logging_callback_manager.add_litellm_async_success_callback(_ProxyDBLogger()) + litellm.logging_callback_manager.add_litellm_callback(ShadowEvalLogger()) # Bounds authoritative DB re-reads when enforcing a budget against a diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 854602f5380..79d778fb464 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1450,6 +1450,44 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } +// Shadow eval: pre-adoption evaluation of an auto-router against a key's live traffic. +// A sampled slice of requests is duplicated through the router in a detached task and an +// LLM judge compares real vs shadow responses blind. The job row is immutable config plus +// stopped_at; every count, status, and spend figure is derived from the append-only +// attempt rows, so nothing can disagree across pods or stop races. +model LiteLLM_ShadowEvalJob { + id String @id @default(cuid()) + api_key_id String // hashed virtual key whose traffic is shadowed + router_name String + judge_model String + shadow_percentage Float + max_turns Int // sample budget: judge at most this many turns + created_at DateTime @default(now()) + created_by String? + ends_at DateTime + stopped_at DateTime? + + @@index([api_key_id]) + @@index([created_at]) +} + +// One row per sampled pipeline: a blind verdict (real | shadow | tie) or an error. +model LiteLLM_ShadowEvalAttempt { + id String @id @default(cuid()) + job_id String + request_id String // the judged real request + outcome String // real | shadow | tie | error + tier String? // router's tier for the prompt, when classified + real_model String? + shadow_model String? + confidence Float? + judge_cost Float @default(0) + error String? + created_at DateTime @default(now()) + + @@index([job_id]) +} + // --------------------------------------------------------------------------- // Workflow Run Tracking // diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index be466fcc575..9f634acfcdd 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -26,8 +26,9 @@ from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast from pydantic import BaseModel, create_model from litellm._logging import verbose_router_logger -from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, RETURN_RAW_MODEL_NAME_METADATA_KEY +from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal_call_metadata from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.types.utils import ( AUTOROUTER_CLASSIFIER_CALL_ORIGIN, @@ -206,40 +207,6 @@ def _append_custom_keywords(base_keywords: list[str], custom_keywords: list[str] return [*base_keywords, *deduped_custom.values()] -# Metadata keys that carry only the parent request's budget reservation state. These -# must not reach internal sub-calls (classifier, embedding): the reservation belongs to -# the routed completion being decided on, not to the sub-call itself, and forwarding it -# would let the sub-call's cost callback finalize the reservation, causing the routed -# completion's callback to skip incrementing key/team budget counters. -# -# Note: user_api_key_auth itself is intentionally kept; it is required by -# _filter_deployments_by_model_access_groups to scope embedding/classifier model -# selection to the caller's authorized access groups. It is forwarded as a sanitized -# copy with its budget_reservation sub-field removed, because the proxy cost callback -# (_get_budget_reservation_from_metadata) falls back to reading the reservation from -# inside the auth object when the top-level key is absent; forwarding it unsanitized -# would re-create the exact double-finalization this stripping exists to prevent. -_BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"}) - - -def _sanitize_user_api_key_auth(auth: Any) -> Any: - if isinstance(auth, dict): - return {k: v for k, v in auth.items() if k != "budget_reservation"} - if getattr(auth, "budget_reservation", None) is not None and hasattr(auth, "model_copy"): - return auth.model_copy(update={"budget_reservation": None}) - return auth - - -def _classifier_call_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]: - if not metadata: - return {} - return { - k: _sanitize_user_api_key_auth(v) if k == "user_api_key_auth" else v - for k, v in metadata.items() - if k not in _BUDGET_RESERVATION_METADATA_KEYS - } | {INTERNAL_CALL_ORIGIN_METADATA_KEY: AUTOROUTER_CLASSIFIER_CALL_ORIGIN} - - def _parent_session_kwargs(request_kwargs: Mapping[str, Any] | None) -> Mapping[str, Any]: kwargs: Final = request_kwargs or {} return {k: kwargs[k] for k in ("litellm_session_id", "litellm_trace_id") if kwargs.get(k) is not None} @@ -1069,7 +1036,7 @@ class ComplexityRouter(CustomLogger): ) request_metadata = (request_kwargs or {}).get("litellm_metadata") or (request_kwargs or {}).get("metadata") - metadata: Final = _classifier_call_metadata(request_metadata) + metadata: Final = forwarded_internal_call_metadata(request_metadata, AUTOROUTER_CLASSIFIER_CALL_ORIGIN) turn_off_message_logging: Final = _effective_turn_off_message_logging(request_kwargs) labeled_tiers: Final = self.config.labeled_tiers() @@ -1562,8 +1529,12 @@ class ComplexityRouter(CustomLogger): # embedding call. Forwarding it would let the embedding's cost callback finalize the # reservation, so the routed completion's own callback then skips incrementing the # key/team budget. Key/team attribution fields are preserved for spend logging. - metadata: Final = _classifier_call_metadata(request_kwargs.get("metadata")) - litellm_metadata: Final = _classifier_call_metadata(request_kwargs.get("litellm_metadata")) + metadata: Final = forwarded_internal_call_metadata( + request_kwargs.get("metadata"), AUTOROUTER_CLASSIFIER_CALL_ORIGIN + ) + litellm_metadata: Final = forwarded_internal_call_metadata( + request_kwargs.get("litellm_metadata"), AUTOROUTER_CLASSIFIER_CALL_ORIGIN + ) turn_off_message_logging: Final = _effective_turn_off_message_logging(request_kwargs) proxy_server_request: Final = {"body": {"model": self.config.embedding_model, "input": [user_message]}} query_vector: Final = ( diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index 269d9b50414..bf8a3d34098 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -3,9 +3,10 @@ Types for auto-router management endpoints """ from collections.abc import Mapping -from typing import Final +from datetime import datetime, timezone +from typing import Final, Literal, TypeAlias -from pydantic import BaseModel, Field, field_validator +from pydantic import AliasChoices, BaseModel, ConfigDict, Field, computed_field, field_validator from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig from litellm.types.utils import StandardLoggingRoutingDecision @@ -141,3 +142,112 @@ class AutoRouterBenchmarksResponse(BaseModel): routers_in_scope: int totals: AutoRouterBenchmarkTotals groups: tuple[AutoRouterBenchmarkGroup, ...] + + +ShadowEvalStatus: TypeAlias = Literal["running", "completed", "stopped"] + +DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "anthropic/claude-sonnet-5" + + +class StartShadowEvalRequest(BaseModel): + """Start shadowing a key's traffic through an auto-router for blind comparison.""" + + api_key_id: str = Field( + description=( + "The hashed virtual key whose traffic will be shadowed. Shadow evaluation runs ONLY on this " + "key's traffic; requests made with any other key are not sampled." + ) + ) + router_name: str = Field(description="The auto-router config to shadow requests through") + shadow_percentage: float = Field( + ge=0.1, + le=100.0, + description="Percentage of the key's requests to duplicate through the router", + ) + judge_model: str = Field( + default=DEFAULT_SHADOW_EVAL_JUDGE_MODEL, + description=( + "Model used to blindly judge real vs. shadow responses. The judge only compares two answers, so a " + "mid-tier model (Claude Sonnet or GPT-4o class) is the sweet spot: small/nano-class models produce " + "unreliable or malformed verdicts, while frontier reasoning models add cost without changing outcomes." + ), + ) + duration_days: int = Field( + default=7, + ge=1, + le=30, + description="How many days the job samples traffic before completing on its own", + ) + max_turns: int = Field( + default=200, + ge=1, + le=2000, + description=( + "Sample budget: the job judges at most this many turns, then completes. This is also the spend " + "bound; expected judge cost is roughly max_turns times one judge call" + ), + ) + + @field_validator("shadow_percentage") + @classmethod + def _round_percentage(cls, value: float) -> float: + return round(value, 2) + + +class ShadowEvalSlice(BaseModel): + """Judge outcomes for one slice of a job's verdicts (a router tier, or one of the + models the shadowed key currently uses).""" + + group: str + turn_count: int + real_win_rate_pct: float = Field(description="Share of judged turns where the real (control) model won") + shadow_win_rate_pct: float = Field(description="Share of judged turns where the shadowed router's pick won") + tie_rate_pct: float + avg_judge_confidence: float + + +class ShadowEvalResult(BaseModel): + """Stratified results of a shadow-eval job's verdicts so far.""" + + by_tier: tuple[ShadowEvalSlice, ...] + by_current_model: tuple[ShadowEvalSlice, ...] + overall_shadow_win_rate_pct: float + overall_tie_rate_pct: float + + +class ShadowEvalJobResponse(BaseModel): + """A shadow-eval job. Validates directly from the prisma record (job_id reads the + row's id); status is derived from stopped_at and ends_at, never stored, so no writer + anywhere can produce an inconsistent one. Aggregate fields are populated by the + detail endpoint only and stay None on list responses.""" + + model_config = ConfigDict(from_attributes=True, populate_by_name=True) + + job_id: str = Field(validation_alias=AliasChoices("id", "job_id")) + api_key_id: str = Field(description="The hashed virtual key whose traffic this job evaluates, and only that key's") + router_name: str + judge_model: str + shadow_percentage: float + max_turns: int + created_at: datetime + ends_at: datetime + stopped_at: datetime | None = None + + judged_count: int | None = Field(default=None, description="Verdicts recorded; detail endpoint only") + error_count: int | None = Field(default=None, description="Sampled attempts that errored; detail endpoint only") + judge_spend: float | None = Field(default=None, description="Judge cost so far; detail endpoint only") + last_error: str | None = Field(default=None, description="Most recent attempt error; detail endpoint only") + results: ShadowEvalResult | None = Field(default=None, description="Stratified verdicts; detail endpoint only") + + @computed_field + @property + def status(self) -> ShadowEvalStatus: + """A job whose window has passed reads completed even if a later sweep stamped + stopped_at; stopped means sampling ended before the window did.""" + if datetime.now(timezone.utc) >= ( + self.ends_at if self.ends_at.tzinfo else self.ends_at.replace(tzinfo=timezone.utc) + ): + return "completed" + if self.stopped_at is not None: + return "stopped" + return "running" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 354857f8d72..d9ef538d530 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2782,11 +2782,13 @@ RoutingDecisionCause = Literal[ ] -InternalCallOrigin = Literal["autorouter_classifier"] +InternalCallOrigin = Literal["autorouter_classifier", "shadow_eval_router", "shadow_eval_judge"] """Which internal litellm feature originated a billed sub-call, so a spend log row records that it is not traffic the caller sent.""" AUTOROUTER_CLASSIFIER_CALL_ORIGIN: Final[InternalCallOrigin] = "autorouter_classifier" +SHADOW_EVAL_ROUTER_CALL_ORIGIN: Final[InternalCallOrigin] = "shadow_eval_router" +SHADOW_EVAL_JUDGE_CALL_ORIGIN: Final[InternalCallOrigin] = "shadow_eval_judge" class StandardLoggingRoutingDecision(TypedDict, total=False): diff --git a/schema.prisma b/schema.prisma index 854602f5380..79d778fb464 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1450,6 +1450,44 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } +// Shadow eval: pre-adoption evaluation of an auto-router against a key's live traffic. +// A sampled slice of requests is duplicated through the router in a detached task and an +// LLM judge compares real vs shadow responses blind. The job row is immutable config plus +// stopped_at; every count, status, and spend figure is derived from the append-only +// attempt rows, so nothing can disagree across pods or stop races. +model LiteLLM_ShadowEvalJob { + id String @id @default(cuid()) + api_key_id String // hashed virtual key whose traffic is shadowed + router_name String + judge_model String + shadow_percentage Float + max_turns Int // sample budget: judge at most this many turns + created_at DateTime @default(now()) + created_by String? + ends_at DateTime + stopped_at DateTime? + + @@index([api_key_id]) + @@index([created_at]) +} + +// One row per sampled pipeline: a blind verdict (real | shadow | tie) or an error. +model LiteLLM_ShadowEvalAttempt { + id String @id @default(cuid()) + job_id String + request_id String // the judged real request + outcome String // real | shadow | tie | error + tier String? // router's tier for the prompt, when classified + real_model String? + shadow_model String? + confidence Float? + judge_cost Float @default(0) + error String? + created_at DateTime @default(now()) + + @@index([job_id]) +} + // --------------------------------------------------------------------------- // Workflow Run Tracking // diff --git a/tests/test_litellm/integrations/test_shadow_eval_logger.py b/tests/test_litellm/integrations/test_shadow_eval_logger.py new file mode 100644 index 00000000000..e1c56db21af --- /dev/null +++ b/tests/test_litellm/integrations/test_shadow_eval_logger.py @@ -0,0 +1,466 @@ +"""Unit tests for the shadow-eval logger: sampling, unmasking, the hook's skip chain, +the detached pipeline's single attempt-row write, and the cache-first job lookup.""" + +import asyncio +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.integrations.shadow_eval_logger import ( + _MAX_CONCURRENT_SHADOW_TASKS, + _MAX_JUDGE_PROMPT_CHARS, + JUDGE_MAX_OUTPUT_TOKENS, + ActiveShadowEvalJob, + ShadowEvalLogger, + _judge_user_prompt, + _sample_hits, + _unmask_preference, +) +from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN + + +def _job(**overrides) -> ActiveShadowEvalJob: + defaults = dict( + id="job-1", + router_name="my-router", + shadow_percentage=100.0, + judge_model="judge-model", + max_turns=200, + ends_at=datetime.now(timezone.utc) + timedelta(days=1), + attempts=0, + ) + return ActiveShadowEvalJob(**{**defaults, **overrides}) + + +def _prisma(jobs=(), attempt_counts=()) -> MagicMock: + prisma = MagicMock() + prisma.db.litellm_shadowevaljob.find_many = AsyncMock(return_value=list(jobs)) + prisma.db.litellm_shadowevalattempt.group_by = AsyncMock( + return_value=[{"job_id": job_id, "_count": {"_all": count}} for job_id, count in attempt_counts] + ) + prisma.db.litellm_shadowevalattempt.create = AsyncMock() + return prisma + + +def _job_record(job: ActiveShadowEvalJob, api_key_id="key-hash") -> MagicMock: + record = MagicMock() + for field, value in dict( + id=job.id, + api_key_id=api_key_id, + router_name=job.router_name, + shadow_percentage=job.shadow_percentage, + judge_model=job.judge_model, + max_turns=job.max_turns, + ends_at=job.ends_at, + ).items(): + setattr(record, field, value) + return record + + +def _router(shadow_text="shadow answer", judge_json='{"preference": "A", "confidence": 0.9, "reasoning": "x"}'): + """One mock router serving the shadow call first, the judge call second. The shadow + call's metadata receives the routing decision write-back, like the real router.""" + router = MagicMock() + router.model_group_alias = {} + router.get_model_list = MagicMock(return_value=[{"litellm_params": {"model": "openai/gpt-4o-mini"}}]) + + async def acompletion(**kwargs): + if kwargs["model"] == "my-router": + kwargs["metadata"]["routing_decision"] = {"tier_label": "SIMPLE", "routed_model": "cheap-model"} + return {"choices": [{"message": {"content": shadow_text}}], "usage": {"completion_tokens": 5}} + return {"choices": [{"message": {"content": judge_json}}]} + + router.acompletion = MagicMock(side_effect=acompletion) + return router + + +def _logger(router=None, prisma=None, job=None) -> ShadowEvalLogger: + cache = InMemoryCache(max_size_in_memory=4, default_ttl=60) + logger = ShadowEvalLogger( + router_provider=lambda: router, + prisma_provider=lambda: prisma, + jobs_cache=cache, + ) + if job is not None: + cache.set_cache("shadow_eval:active_jobs", {"key-hash": job}) + return logger + + +def _success_kwargs(request_id="req-1", api_key_hash="key-hash", request_metadata=None, call_type="acompletion"): + return { + "standard_logging_object": { + "id": request_id, + "call_type": call_type, + "model": "claude-opus", + "metadata": {"user_api_key_hash": api_key_hash}, + "model_parameters": {"temperature": 0.5, "stream": True}, + }, + "litellm_params": {"metadata": request_metadata or {}}, + "messages": [{"role": "user", "content": "what is 2+2"}], + } + + +RESPONSE = {"choices": [{"message": {"content": "real answer"}}]} + + +async def _drain(logger: ShadowEvalLogger, target: int = 0): + for _ in range(100): + if logger._inflight_shadow_tasks == target: + return + await asyncio.sleep(0.01) + raise AssertionError("shadow tasks never drained") + + +class TestSampling: + def test_boundaries_and_determinism(self): + assert not any(_sample_hits(f"req-{i}", "job", 0.0) for i in range(100)) + assert all(_sample_hits(f"req-{i}", "job", 100.0) for i in range(100)) + assert len({_sample_hits("req-1", "job-1", 50.0) for _ in range(10)}) == 1 + + def test_distribution_close_to_percentage(self): + hits = sum(_sample_hits(f"req-{i}", "job-x", 10.0) for i in range(10_000)) + assert 800 < hits < 1200 + + def test_different_jobs_sample_independently(self): + agreements = sum( + _sample_hits(f"req-{i}", "job-a", 50.0) == _sample_hits(f"req-{i}", "job-b", 50.0) for i in range(1000) + ) + assert 300 < agreements < 700 + + +@pytest.mark.parametrize( + "raw,real_is_a,expected", + [ + ("A", True, "real"), + ("a", True, "real"), + ("A", False, "shadow"), + ("B", True, "shadow"), + ("B", False, "real"), + ("tie", True, "tie"), + ("garbage", True, "tie"), + ("", False, "tie"), + ], +) +def test_unmask_preference(raw, real_is_a, expected): + assert _unmask_preference(raw, real_is_a) == expected + + +def test_judge_prompt_is_bounded_however_large_the_inputs(): + prompt = _judge_user_prompt("c" * 200_000, "a" * 200_000, "b" * 200_000) + assert len(prompt) < _MAX_JUDGE_PROMPT_CHARS + 100 + assert prompt.endswith("Which response is better?") + small = _judge_user_prompt("conv", "alpha", "beta") + assert "conv" in small and "alpha" in small and "beta" in small + + +@pytest.mark.asyncio +class TestSuccessHookSkipChain: + async def test_happy_path_writes_exactly_one_attempt_row(self, monkeypatch: pytest.MonkeyPatch): + import litellm as litellm_module + + monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.005) + prisma = _prisma() + router = _router() + logger = _logger(router=router, prisma=prisma, job=_job()) + + await logger.async_log_success_event(_success_kwargs(), RESPONSE, None, None) + await _drain(logger) + + create = prisma.db.litellm_shadowevalattempt.create + create.assert_awaited_once() + row = create.call_args.kwargs["data"] + assert row["job_id"] == "job-1" + assert row["request_id"] == "req-1" + assert row["outcome"] in ("real", "shadow") + assert row["tier"] == "SIMPLE" + assert row["real_model"] == "claude-opus" + assert row["shadow_model"] == "cheap-model" + assert row["confidence"] == 0.9 + assert row["judge_cost"] == 0.005 + assert row["error"] is None + assert prisma.db.litellm_shadowevaljob.find_many.await_count == 0 + + @pytest.mark.parametrize( + "kwargs_mutation,job_mutation", + [ + ({"request_metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: "shadow_eval_router"}}, {}), + ({"api_key_hash": "other-key"}, {}), + ({"call_type": "aembedding"}, {}), + ({"call_type": None}, {}), + ({"request_metadata": {"routing_decision": {"router_model_name": "my-router"}}}, {}), + ({}, {"ends_at": datetime.now(timezone.utc) - timedelta(seconds=1)}), + ({}, {"attempts": 200}), + ({}, {"attempts": 199, "max_turns": 200, "_starts": 1}), + ], + ids=[ + "internal-origin", + "no-job-for-key", + "non-chat", + "missing-call-type", + "self-shadow", + "past-end", + "turn-budget-reached", + "budget-consumed-by-started-tasks", + ], + ) + async def test_skip_paths_store_nothing(self, kwargs_mutation, job_mutation): + starts = job_mutation.pop("_starts", 0) + prisma = _prisma() + logger = _logger(router=_router(), prisma=prisma, job=_job(**job_mutation)) + logger._job_starts = {"job-1": starts} + + await logger.async_log_success_event(_success_kwargs(**kwargs_mutation), RESPONSE, None, None) + await _drain(logger) + + prisma.db.litellm_shadowevalattempt.create.assert_not_called() + assert logger._job_starts.get("job-1", 0) == starts + + async def test_completed_pipelines_hold_turn_budget_within_a_cache_generation(self): + """A finished pipeline frees its concurrency slot but not its slice of the turn + budget; the budget only reopens when a cache refill absorbs the written rows.""" + prisma = _prisma() + logger = _logger(router=_router(), prisma=prisma, job=_job(attempts=199, max_turns=200)) + + await logger.async_log_success_event(_success_kwargs(request_id="req-1"), RESPONSE, None, None) + await _drain(logger) + await logger.async_log_success_event(_success_kwargs(request_id="req-2"), RESPONSE, None, None) + await _drain(logger) + + assert prisma.db.litellm_shadowevalattempt.create.await_count == 1 + + async def test_v1_messages_surface_forwards_identity_from_litellm_metadata(self): + """/v1/messages stores identity in litellm_params.litellm_metadata, so the hook + resolves the bucket through the shared helper; every surface forwards the same + identity to the shadow and judge calls.""" + prisma = _prisma() + router = _router() + logger = _logger(router=router, prisma=prisma, job=_job()) + + hook_kwargs = _success_kwargs() + hook_kwargs["litellm_params"] = { + "litellm_metadata": {"user_api_key_hash": "key-hash", "user_api_key_team_id": "team-1"} + } + await logger.async_log_success_event(hook_kwargs, RESPONSE, None, None) + await _drain(logger) + + shadow_call = router.acompletion.call_args_list[0].kwargs + assert shadow_call["metadata"]["user_api_key_hash"] == "key-hash" + assert shadow_call["metadata"]["user_api_key_team_id"] == "team-1" + + async def test_redacted_requests_are_never_shadowed(self): + """Redaction rewrites the logged messages before callbacks run, so this hook only + ever sees placeholders for opted-out traffic; the skip uses the redactor's own + predicate, so every redaction source counts.""" + prisma = _prisma() + router = _router() + logger = _logger(router=router, prisma=prisma, job=_job()) + + hook_kwargs = _success_kwargs() + hook_kwargs["standard_callback_dynamic_params"] = {"turn_off_message_logging": True} + await logger.async_log_success_event(hook_kwargs, RESPONSE, None, None) + await _drain(logger) + + router.acompletion.assert_not_called() + prisma.db.litellm_shadowevalattempt.create.assert_not_called() + + async def test_inflight_cap_sheds_instead_of_queueing(self): + prisma = _prisma() + logger = _logger(router=_router(), prisma=prisma, job=_job()) + logger._inflight_shadow_tasks = _MAX_CONCURRENT_SHADOW_TASKS + + await logger.async_log_success_event(_success_kwargs(), RESPONSE, None, None) + + assert logger._inflight_shadow_tasks == _MAX_CONCURRENT_SHADOW_TASKS + prisma.db.litellm_shadowevalattempt.create.assert_not_called() + + +@pytest.mark.asyncio +class TestActiveJobsCache: + async def test_cache_miss_reads_db_once_then_serves_from_cache(self): + job = _job() + prisma = _prisma(jobs=[_job_record(job)], attempt_counts=[("job-1", 7)]) + logger = ShadowEvalLogger( + router_provider=lambda: None, + prisma_provider=lambda: prisma, + jobs_cache=InMemoryCache(max_size_in_memory=4, default_ttl=60), + ) + + first = await logger._active_jobs() + second = await logger._active_jobs() + + assert first["key-hash"].id == "job-1" + assert second["key-hash"].attempts == 7 + assert prisma.db.litellm_shadowevaljob.find_many.await_count == 1 + where = prisma.db.litellm_shadowevaljob.find_many.call_args.kwargs["where"] + assert where["stopped_at"] is None + assert "gt" in where["ends_at"] + count_where = prisma.db.litellm_shadowevalattempt.group_by.call_args.kwargs["where"] + assert count_where == {"job_id": {"in": ["job-1"]}} + + async def test_no_active_jobs_is_cached_too(self): + prisma = _prisma(jobs=[]) + logger = ShadowEvalLogger( + router_provider=lambda: None, + prisma_provider=lambda: prisma, + jobs_cache=InMemoryCache(max_size_in_memory=4, default_ttl=60), + ) + + assert await logger._active_jobs() == {} + assert await logger._active_jobs() == {} + assert prisma.db.litellm_shadowevaljob.find_many.await_count == 1 + prisma.db.litellm_shadowevalattempt.group_by.assert_not_called() + + async def test_db_fault_returns_empty_without_caching_the_fault(self): + prisma = _prisma() + prisma.db.litellm_shadowevaljob.find_many = AsyncMock(side_effect=RuntimeError("db blip")) + logger = ShadowEvalLogger( + router_provider=lambda: None, + prisma_provider=lambda: prisma, + jobs_cache=InMemoryCache(max_size_in_memory=4, default_ttl=60), + ) + + assert await logger._active_jobs() == {} + assert await logger._active_jobs() == {} + assert prisma.db.litellm_shadowevaljob.find_many.await_count == 2 + + async def test_cache_refill_resets_the_starts_counter(self): + job = _job() + prisma = _prisma(jobs=[_job_record(job)], attempt_counts=[("job-1", 7)]) + logger = ShadowEvalLogger( + router_provider=lambda: None, + prisma_provider=lambda: prisma, + jobs_cache=InMemoryCache(max_size_in_memory=4, default_ttl=60), + ) + logger._job_starts = {"job-1": 5} + + await logger._active_jobs() + + assert logger._job_starts == {} + + +@pytest.mark.asyncio +class TestShadowPipeline: + async def test_no_prisma_means_no_provider_spend(self): + router = _router() + logger = _logger(router=router, prisma=None) + + await logger._run_shadow_eval( + job=_job(), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + response_obj=RESPONSE, + real_model="claude-opus", + model_parameters={}, + parent_metadata={}, + ) + + router.acompletion.assert_not_called() + + async def test_over_budget_key_skips_before_any_call(self, monkeypatch: pytest.MonkeyPatch): + """The gate delegates to the auth path's own budget owner, so an over-budget + verdict there (BudgetExceededError) skips the shadow before any provider call.""" + import litellm.proxy.auth.auth_checks as auth_checks + from litellm.exceptions import BudgetExceededError + from litellm.proxy._types import UserAPIKeyAuth + + monkeypatch.setattr( + auth_checks, + "_virtual_key_max_budget_check", + AsyncMock(side_effect=BudgetExceededError(current_cost=11.0, max_budget=10.0)), + ) + router = _router() + prisma = _prisma() + logger = _logger(router=router, prisma=prisma) + + await logger._run_shadow_eval( + job=_job(), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + response_obj=RESPONSE, + real_model="claude-opus", + model_parameters={}, + parent_metadata={"user_api_key_auth": UserAPIKeyAuth(api_key="sk-abc", max_budget=10.0)}, + ) + + router.acompletion.assert_not_called() + prisma.db.litellm_shadowevalattempt.create.assert_not_called() + + @pytest.mark.parametrize( + "router_factory,expected_error,expected_cost", + [ + (lambda: _failing_router(), "provider exploded", 0.0), + (lambda: _router(judge_json="I prefer response A, definitely"), "unparseable judge verdict", 0.007), + ], + ids=["shadow-call-fails", "judge-verdict-unparseable"], + ) + async def test_failures_become_error_rows_and_keep_billed_judge_cost( + self, router_factory, expected_error, expected_cost, monkeypatch: pytest.MonkeyPatch + ): + import litellm as litellm_module + + monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.007) + prisma = _prisma() + logger = _logger(router=router_factory(), prisma=prisma) + + await logger._run_shadow_eval( + job=_job(), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + response_obj=RESPONSE, + real_model="claude-opus", + model_parameters={}, + parent_metadata={}, + ) + + row = prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"] + assert row["outcome"] == "error" + assert expected_error in row["error"] + assert row["confidence"] is None + assert row["judge_cost"] == expected_cost + + async def test_sub_calls_carry_identity_and_origin_but_never_parent_request_state(self): + prisma = _prisma() + router = _router() + logger = _logger(router=router, prisma=prisma) + parent_metadata = { + "user_api_key_hash": "key-hash", + "user_api_key_team_id": "team-1", + "user_api_key_budget_reservation": {"amount": 1.0}, + "routing_decision": {"router_model_name": "other-router"}, + } + + await logger._run_shadow_eval( + job=_job(), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + response_obj=RESPONSE, + real_model="claude-opus", + model_parameters={"stream": True, "temperature": 0.2, "metadata": {"x": 1}}, + parent_metadata=parent_metadata, + ) + + shadow_call = router.acompletion.call_args_list[0].kwargs + judge_call = router.acompletion.call_args_list[1].kwargs + for call in (shadow_call, judge_call): + assert call["num_retries"] == 0 + assert call["fallbacks"] == [] + assert call["metadata"]["user_api_key_hash"] == "key-hash" + assert call["metadata"]["user_api_key_team_id"] == "team-1" + assert "user_api_key_budget_reservation" not in call["metadata"] + assert shadow_call["metadata"][INTERNAL_CALL_ORIGIN_METADATA_KEY] == SHADOW_EVAL_ROUTER_CALL_ORIGIN + assert judge_call["metadata"][INTERNAL_CALL_ORIGIN_METADATA_KEY] == SHADOW_EVAL_JUDGE_CALL_ORIGIN + assert "routing_decision" not in judge_call["metadata"] + assert "stream" not in shadow_call + assert shadow_call["temperature"] == 0.2 + assert judge_call["max_tokens"] == JUDGE_MAX_OUTPUT_TOKENS + + +def _failing_router(): + router = MagicMock() + router.model_group_alias = {} + router.get_model_list = MagicMock(return_value=None) + router.acompletion = AsyncMock(side_effect=RuntimeError("provider exploded")) + return router diff --git a/tests/test_litellm/litellm_core_utils/test_internal_call_metadata.py b/tests/test_litellm/litellm_core_utils/test_internal_call_metadata.py new file mode 100644 index 00000000000..73923dc75a5 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_internal_call_metadata.py @@ -0,0 +1,121 @@ +"""Unit tests for internal-call metadata forwarding: budget-reservation stripping and origin stamping.""" + +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.litellm_core_utils.internal_call_metadata import ( + forwarded_internal_call_metadata, + sanitized_forwardable_call_metadata, +) +from litellm.types.utils import SHADOW_EVAL_ROUTER_CALL_ORIGIN + +PARENT = { + "user_api_key": "sk-hash", + "user_api_key_hash": "sk-hash", + "user_api_key_team_id": "team-1", + "user_api_key_budget_reservation": {"amount": 1.0}, + "user_api_key_auth": {"api_key": "sk-hash", "budget_reservation": {"amount": 1.0}}, + "routing_decision": {"router_model_name": "my-router"}, + "headers": {"x-request-id": "abc"}, +} + + +def test_forwarded_metadata_strips_reservation_everywhere_and_stamps_origin(): + result = forwarded_internal_call_metadata(PARENT, "autorouter_classifier") + + assert result[INTERNAL_CALL_ORIGIN_METADATA_KEY] == "autorouter_classifier" + assert "user_api_key_budget_reservation" not in result + assert result["user_api_key_auth"] == {"api_key": "sk-hash"} + assert result["routing_decision"] == {"router_model_name": "my-router"} + assert PARENT["user_api_key_auth"]["budget_reservation"] is not None + + +def test_forwarded_metadata_empty_parent_stays_unstamped(): + assert forwarded_internal_call_metadata(None, "autorouter_classifier") == {} + assert forwarded_internal_call_metadata({}, "autorouter_classifier") == {} + + +def test_sanitized_forwardable_metadata_keeps_only_identity_and_always_stamps(): + result = sanitized_forwardable_call_metadata(PARENT, SHADOW_EVAL_ROUTER_CALL_ORIGIN) + + assert result[INTERNAL_CALL_ORIGIN_METADATA_KEY] == SHADOW_EVAL_ROUTER_CALL_ORIGIN + assert result["user_api_key"] == "sk-hash" + assert result["user_api_key_team_id"] == "team-1" + assert result["user_api_key_auth"] == {"api_key": "sk-hash"} + assert "routing_decision" not in result + assert "headers" not in result + assert "user_api_key_budget_reservation" not in result + + assert sanitized_forwardable_call_metadata({}, SHADOW_EVAL_ROUTER_CALL_ORIGIN) == { + INTERNAL_CALL_ORIGIN_METADATA_KEY: SHADOW_EVAL_ROUTER_CALL_ORIGIN + } + + +class TestSubCallMetadataSanitization: + """The proxy cost callback must not be able to recover the parent budget reservation + from sub-call metadata, in either of the shapes it knows how to read.""" + + def test_cost_callback_cannot_recover_reservation_from_sanitized_metadata(self): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.hooks.proxy_track_cost_callback import ( + _get_budget_reservation_from_metadata, + ) + + reservation = {"reserved_cost": 1.0} + auth_shapes = ( + {"models": ["gpt-4o"], "budget_reservation": dict(reservation)}, + UserAPIKeyAuth(api_key="sk-abc", budget_reservation=dict(reservation)), + ) + for auth in auth_shapes: + metadata = { + "user_api_key_hash": "hash-abc", + "user_api_key_budget_reservation": dict(reservation), + "user_api_key_auth": auth, + } + assert _get_budget_reservation_from_metadata(metadata) == reservation + + sanitized = forwarded_internal_call_metadata(metadata, "autorouter_classifier") + assert sanitized is not None + assert sanitized["user_api_key_auth"] is not None + assert _get_budget_reservation_from_metadata(sanitized) is None + + def test_classifier_buckets_keep_non_spend_fields_on_a_chat_completions_parent(self): + """Drives the real resolver over the buckets the embedding classifier builds. + + An absent bucket must stay empty rather than carry a lone origin stamp: + get_litellm_metadata_from_kwargs prefers litellm_metadata whenever truthy, so an + origin-only dict would make an empty litellm_metadata win and silently drop + requester_ip_address, tags and spend_logs_metadata from the classifier's row.""" + from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs + + parent = { + "user_api_key": "sk-abc", + "requester_ip_address": "10.0.0.1", + "spend_logs_metadata": {"team_note": "keep me"}, + "tags": ["prod"], + } + resolved = get_litellm_metadata_from_kwargs( + { + "litellm_params": { + "metadata": forwarded_internal_call_metadata(parent, "autorouter_classifier"), + "litellm_metadata": forwarded_internal_call_metadata(None, "autorouter_classifier"), + } + } + ) + assert resolved["internal_call_origin"] == "autorouter_classifier" + assert resolved["requester_ip_address"] == "10.0.0.1" + assert resolved["spend_logs_metadata"] == {"team_note": "keep me"} + assert resolved["tags"] == ["prod"] + + def test_sanitized_auth_keeps_access_group_fields_and_leaves_original_untouched(self): + from litellm.proxy._types import UserAPIKeyAuth + + auth = UserAPIKeyAuth( + api_key="sk-abc", + team_id="team-1", + budget_reservation={"reserved_cost": 1.0}, + ) + sanitized = forwarded_internal_call_metadata({"user_api_key_auth": auth}, "autorouter_classifier") + sanitized_auth = sanitized["user_api_key_auth"] + assert sanitized_auth.budget_reservation is None + assert sanitized_auth.team_id == "team-1" + assert sanitized_auth.api_key == auth.api_key + assert auth.budget_reservation == {"reserved_cost": 1.0} diff --git a/tests/test_litellm/litellm_core_utils/test_llm_judge.py b/tests/test_litellm/litellm_core_utils/test_llm_judge.py new file mode 100644 index 00000000000..5c092caa7c3 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_llm_judge.py @@ -0,0 +1,92 @@ +"""Unit tests for the shared LLM-judge primitives: verdict parsing, router resolution, dispatch.""" + +import json +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.litellm_core_utils.llm_judge import ( + extract_text_from_content, + judge_acompletion, + parse_json_verdict, + router_resolves_model, +) + + +@pytest.mark.parametrize( + "raw,expected", + [ + ('{"preference": "A", "confidence": 0.9}', "A"), + ('Here it is:\n```json\n{"preference": "B"}\n```\nDone.', "B"), + ('```\n{"preference": "tie"}\n```', "tie"), + ('Verdict: {"preference": "A", "confidence": 0.5} final.', "A"), + ], +) +def test_parse_json_verdict_tolerates_fences_and_prose(raw, expected): + assert parse_json_verdict(raw)["preference"] == expected + + +def test_parse_json_verdict_rejects_non_object(): + with pytest.raises(ValueError): + parse_json_verdict('["not", "an", "object"]') + with pytest.raises((json.JSONDecodeError, ValueError)): + parse_json_verdict("no json here at all") + + +@pytest.mark.parametrize( + "content,expected", + [ + ("hello", "hello"), + ([{"type": "text", "text": "a"}, {"type": "image_url", "image_url": {}}, {"type": "text", "text": "b"}], "a b"), + (42, ""), + (None, ""), + ], +) +def test_extract_text_from_content(content, expected): + assert extract_text_from_content(content) == expected + + +def _router(alias=(), deployments=False) -> MagicMock: + router = MagicMock() + router.model_group_alias = dict.fromkeys(alias, "x") + router.get_model_list = MagicMock( + return_value=[{"litellm_params": {"model": "openai/gpt-4o"}}] if deployments else None + ) + router.acompletion = AsyncMock(return_value={"choices": [{"message": {"content": "router answer"}}]}) + return router + + +def test_router_resolves_model_matrix(): + assert router_resolves_model(None, "gpt-4o") is False + assert router_resolves_model(_router(), "gpt-4o") is False + assert router_resolves_model(_router(alias=("gpt-4o",)), "gpt-4o") is True + assert router_resolves_model(_router(deployments=True), "gpt-4o") is True + + +@pytest.mark.asyncio +async def test_judge_acompletion_prefers_router_and_disables_retries(): + router = _router(deployments=True) + response = await judge_acompletion(router, "judge-model", [{"role": "user", "content": "hi"}], temperature=0) + assert response == {"choices": [{"message": {"content": "router answer"}}]} + _, kwargs = router.acompletion.call_args + assert kwargs["num_retries"] == 0 + assert kwargs["fallbacks"] == [] + assert kwargs["temperature"] == 0 + assert kwargs["drop_params"] is True + + +@pytest.mark.asyncio +async def test_judge_acompletion_falls_back_to_sdk_for_unconfigured_model(monkeypatch: pytest.MonkeyPatch): + import litellm as litellm_module + + sdk = AsyncMock(return_value={"choices": [{"message": {"content": "sdk answer"}}]}) + monkeypatch.setattr(litellm_module, "acompletion", sdk) + router = _router() + + response = await judge_acompletion(router, "anthropic/claude-sonnet-5", [{"role": "user", "content": "hi"}]) + + assert response == {"choices": [{"message": {"content": "sdk answer"}}]} + router.acompletion.assert_not_called() + assert sdk.call_args.kwargs["model"] == "anthropic/claude-sonnet-5" + assert sdk.call_args.kwargs["num_retries"] == 0 + assert sdk.call_args.kwargs["drop_params"] is True diff --git a/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py b/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py index 0df11f224a2..95dce1ccb0a 100644 --- a/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py +++ b/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py @@ -279,3 +279,10 @@ def test_every_drain_trigger_reads_the_one_queue_census_owner(): assert queue in owner_source, queue for site in (proxy_utils.update_spend, proxy_utils.update_spend_logs_job, proxy_utils._monitor_spend_logs_queue): assert "_total_queued_spend_transactions" in inspect.getsource(site), site.__name__ + + +def test_internal_call_origin_never_reaches_the_rollup(): + """A shadow eval's duplicate carries a real routing_decision, so the decision-presence + gate alone would count it; the internal_call_origin stamp must exclude it.""" + assert _build(metadata=_metadata(internal_call_origin="shadow_eval_router")) is None + assert _build() is not None diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index 2c426e0f071..ca7d5fcd273 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -2221,3 +2221,50 @@ async def test_commit_spend_updates_to_db_does_not_stamp_key_settings_updated_at assert call_kwargs["where"] == {"token": token} assert set(call_kwargs["data"]) == {"spend", "last_active"} assert call_kwargs["data"]["spend"] == {"increment": response_cost} + + +@pytest.mark.asyncio +async def test_daily_transaction_internal_call_keeps_spend_but_not_request_counts(): + """Internal sub-calls (auto-router classifier, shadow eval's shadow and judge) bill + spend and tokens to the key but are not requests the caller made: api_requests, + successful_requests, and autorouter_savings_spend must all stay zero for them.""" + writer = DBSpendUpdateWriter() + mock_prisma = MagicMock() + mock_prisma.get_request_status = MagicMock(return_value="success") + + def _payload(metadata: dict) -> dict: + return { + "request_id": "req-internal-1", + "user": "test-user", + "startTime": "2026-08-11T00:00:00", + "api_key": "test-key", + "model": "claude-sonnet-5", + "custom_llm_provider": "anthropic", + "model_group": "claude-sonnet-5", + "call_type": "acompletion", + "prompt_tokens": 100, + "completion_tokens": 10, + "spend": 0.05, + "metadata": json.dumps(metadata), + } + + internal = await writer._common_add_spend_log_transaction_to_daily_transaction( + payload=_payload({"internal_call_origin": "shadow_eval_judge"}), + prisma_client=mock_prisma, + type="user", + ) + user_sent = await writer._common_add_spend_log_transaction_to_daily_transaction( + payload=_payload({}), + prisma_client=mock_prisma, + type="user", + ) + + assert internal is not None and user_sent is not None + assert internal["spend"] == 0.05 + assert internal["prompt_tokens"] == 100 + assert internal["api_requests"] == 0 + assert internal["successful_requests"] == 0 + assert internal["failed_requests"] == 0 + assert internal["autorouter_savings_spend"] == 0.0 + assert user_sent["api_requests"] == 1 + assert user_sent["successful_requests"] == 1 diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 076151fcd3b..59434290c48 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -17,6 +17,7 @@ from fastapi import HTTPException import litellm from litellm import Router from litellm.caching.caching import DualCache +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PARALLEL_REQUEST_SLOT_TTL_SECONDS, @@ -5554,3 +5555,41 @@ async def test_configured_estimate_blocks_the_overrun_the_static_floor_admits(mo assert await admitted({}) == 7 assert await admitted({"default_estimated_output_tokens": 3000}) == 2 + + +def test_internal_call_origin_success_ops_are_skipped(): + """Internal sub-calls (auto-router classifier, shadow eval shadow/judge) bill spend + to the caller's key but must not consume its TPM counters: the same kwargs charge + ops without the origin stamp and none with it.""" + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache()) + ) + response = ModelResponse( + id="internal-origin-tpm", + object="chat.completion", + created=int(datetime.now().timestamp()), + model="gpt-4o-mini", + usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), + choices=[], + ) + + def _kwargs(metadata: Dict[str, Any]) -> Dict[str, Any]: + return { + "standard_logging_object": { + "metadata": {"user_api_key_hash": hash_token("sk-internal-origin")} + }, + "litellm_params": {"metadata": metadata}, + "model": "gpt-4o-mini", + } + + charged = handler._build_success_event_pipeline_operations( + kwargs=_kwargs({}), response_obj=response, rate_limit_type="output" + ) + skipped = handler._build_success_event_pipeline_operations( + kwargs=_kwargs({INTERNAL_CALL_ORIGIN_METADATA_KEY: "shadow_eval_judge"}), + response_obj=response, + rate_limit_type="output", + ) + + assert charged + assert skipped == [] diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index 3a995e27697..16e82bc3bda 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -466,3 +466,276 @@ class TestAutoRouterBenchmarks: end_date="2026-08-01", ) assert response.groups[0].tier_turns == expected + + +# --------------------------------------------------------------------------- +# Shadow eval endpoints +# --------------------------------------------------------------------------- + +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock, MagicMock + +from fastapi import HTTPException + +from litellm.proxy.management_endpoints.auto_router_endpoints import ( + get_shadow_eval_job, + list_shadow_eval_jobs, + start_shadow_eval, + stop_shadow_eval_job, +) +from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalJobResponse, StartShadowEvalRequest + +VIEWER = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, api_key="sk-view", user_id="viewer") +NON_ADMIN = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-user", user_id="user") + + +def _shadow_router() -> MagicMock: + router = MagicMock() + router.auto_routers = {} + router.complexity_routers = {"my-router": [MagicMock()]} + router.adaptive_routers = {} + router.quality_routers = {} + router.model_group_alias = {} + router.get_model_list = MagicMock(return_value=None) + return router + + +def _job_record(**overrides: object) -> MagicMock: + """Spec'd like a real prisma row: only the table's columns exist as attributes, so + from_attributes validation falls back to model defaults for everything else.""" + defaults = { + "id": "job-1", + "api_key_id": "key-hash", + "router_name": "my-router", + "judge_model": "anthropic/claude-sonnet-5", + "shadow_percentage": 10.0, + "max_turns": 200, + "created_at": datetime(2026, 8, 11, tzinfo=timezone.utc), + "ends_at": datetime.now(timezone.utc) + timedelta(days=7), + "stopped_at": None, + } + fields = {**defaults, **overrides} + record = MagicMock(spec=list(fields)) + for key, value in fields.items(): + setattr(record, key, value) + return record + + +def _shadow_prisma(active_job=None, agg_rows=None) -> MagicMock: + prisma = MagicMock() + prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=MagicMock()) + prisma.db.execute_raw = AsyncMock(return_value=0) + prisma.db.litellm_shadowevaljob.find_first = AsyncMock(return_value=active_job) + prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=None) + prisma.db.litellm_shadowevaljob.find_many = AsyncMock(return_value=[]) + prisma.db.litellm_shadowevaljob.create = AsyncMock(return_value=_job_record()) + prisma.db.litellm_shadowevaljob.update = AsyncMock( + return_value=_job_record(stopped_at=datetime.now(timezone.utc)) + ) + prisma.db.litellm_shadowevalattempt.find_first = AsyncMock(return_value=None) + + async def query_raw(sql: str, *params: object): + if "FILTER (WHERE outcome != 'error')::int AS judged_count" in sql: + return [{"judged_count": 10, "error_count": 2, "judge_spend": 0.031}] + return agg_rows if agg_rows is not None else [] + + prisma.db.query_raw = AsyncMock(side_effect=query_raw) + return prisma + + +def _start_request(**overrides: object) -> StartShadowEvalRequest: + payload = { + "api_key_id": "key-hash", + "router_name": "my-router", + "shadow_percentage": 10.0, + "judge_model": "anthropic/claude-sonnet-5", + "duration_days": 7, + "max_turns": 200, + } + payload.update(overrides) + return StartShadowEvalRequest.model_validate(payload) + + +@pytest.mark.asyncio +async def test_start_shadow_eval_creates_job_and_frees_expired_or_exhausted_ones(monkeypatch: pytest.MonkeyPatch): + """Expiry and turn-budget exhaustion both end sampling on their own; either must + release the one-active-per-key index so a new eval can start.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + response = await start_shadow_eval(_start_request(), ADMIN) + + assert response.status == "running" + assert response.max_turns == 200 + assert response.judged_count is None + sweep_sql, sweep_key = prisma.db.execute_raw.call_args.args + assert "stopped_at IS NULL" in sweep_sql + assert "ends_at <= NOW()" in sweep_sql + assert ">= j.max_turns" in sweep_sql + assert sweep_key == "key-hash" + create_data = prisma.db.litellm_shadowevaljob.create.call_args.kwargs["data"] + assert create_data["api_key_id"] == "key-hash" + assert create_data["created_by"] == "admin" + assert "status" not in create_data + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "caller,request_overrides,active,expected_status", + [ + (NON_ADMIN, {}, None, 403), + (VIEWER, {}, None, 403), + (ADMIN, {"router_name": "not-a-router"}, None, 400), + (ADMIN, {"judge_model": "not/a real model!"}, None, 400), + (ADMIN, {"judge_model": "my-router"}, None, 400), + (ADMIN, {}, "active", 409), + ], + ids=["non-admin", "view-only", "unknown-router", "unresolvable-judge", "router-as-judge", "already-active"], +) +async def test_start_shadow_eval_rejections( + monkeypatch: pytest.MonkeyPatch, caller, request_overrides, active, expected_status +): + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(active_job=_job_record() if active else None) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(_start_request(**request_overrides), caller) + assert exc.value.status_code == expected_status + + +@pytest.mark.asyncio +async def test_start_shadow_eval_rejects_a_key_this_proxy_does_not_know(monkeypatch: pytest.MonkeyPatch): + """A typo'd api_key_id would otherwise create a job no traffic can ever match.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma() + prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(_start_request(), ADMIN) + assert exc.value.status_code == 400 + assert "not a key on this proxy" in exc.value.detail + + +@pytest.mark.asyncio +async def test_start_shadow_eval_concurrent_unique_violation_is_a_409(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + from prisma.errors import UniqueViolationError + + prisma = _shadow_prisma() + prisma.db.litellm_shadowevaljob.create = AsyncMock( + side_effect=UniqueViolationError(MagicMock(message="unique constraint")) + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(_start_request(), ADMIN) + assert exc.value.status_code == 409 + + +@pytest.mark.asyncio +async def test_get_shadow_eval_job_derives_counts_spend_and_stratified_results(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + tier_rows = [ + {"grp": "SIMPLE", "turn_count": 8, "real_wins": 2, "shadow_wins": 4, "ties": 2, "avg_confidence": 0.8}, + {"grp": "REASONING", "turn_count": 2, "real_wins": 2, "shadow_wins": 0, "ties": 0, "avg_confidence": 0.9}, + ] + prisma = _shadow_prisma(agg_rows=tier_rows) + prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=_job_record()) + prisma.db.litellm_shadowevalattempt.find_first = AsyncMock( + return_value=MagicMock(error="judge call failed: boom") + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + response = await get_shadow_eval_job("job-1", VIEWER) + + assert response.job_id == "job-1" + assert response.status == "running" + assert response.judged_count == 10 + assert response.error_count == 2 + assert response.judge_spend == 0.031 + assert response.last_error == "judge call failed: boom" + assert [s.group for s in response.results.by_tier] == ["SIMPLE", "REASONING"] + assert response.results.by_tier[0].shadow_win_rate_pct == 50.0 + assert response.results.overall_shadow_win_rate_pct == 40.0 + assert response.results.overall_tie_rate_pct == 20.0 + + +@pytest.mark.asyncio +async def test_get_shadow_eval_job_404s_and_gates_on_role(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "prisma_client", _shadow_prisma()) + + with pytest.raises(HTTPException) as missing: + await get_shadow_eval_job("nope", VIEWER) + assert missing.value.status_code == 404 + + with pytest.raises(HTTPException) as forbidden: + await get_shadow_eval_job("job-1", NON_ADMIN) + assert forbidden.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_list_shadow_eval_jobs_returns_derived_status_without_aggregates(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma() + prisma.db.litellm_shadowevaljob.find_many = AsyncMock( + return_value=[ + _job_record(), + _job_record(id="job-2", ends_at=datetime.now(timezone.utc) - timedelta(days=1)), + _job_record(id="job-3", stopped_at=datetime.now(timezone.utc)), + ] + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + + assert [job.status for job in jobs] == ["running", "completed", "stopped"] + swept = ShadowEvalJobResponse.model_validate( + _job_record( + id="job-4", + ends_at=datetime.now(timezone.utc) - timedelta(days=1), + stopped_at=datetime.now(timezone.utc), + ), + from_attributes=True, + ) + assert swept.status == "completed" + assert all(job.judged_count is None and job.results is None for job in jobs) + assert prisma.db.query_raw.await_count == 0 + + +@pytest.mark.asyncio +async def test_stop_shadow_eval_sets_stopped_at_and_rejects_non_running(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma() + prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=_job_record()) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + stopped = await stop_shadow_eval_job("job-1", ADMIN) + assert stopped.status == "stopped" + update = prisma.db.litellm_shadowevaljob.update.call_args.kwargs + assert set(update["data"]) == {"stopped_at"} + + prisma.db.litellm_shadowevaljob.find_unique = AsyncMock( + return_value=_job_record(ends_at=datetime.now(timezone.utc) - timedelta(days=1)) + ) + with pytest.raises(HTTPException) as exc: + await stop_shadow_eval_job("job-1", ADMIN) + assert exc.value.status_code == 400 + + with pytest.raises(HTTPException) as forbidden: + await stop_shadow_eval_job("job-1", VIEWER) + assert forbidden.value.status_code == 403 diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index ad41592db1e..a06e7142122 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -524,8 +524,9 @@ def test_load_from_azure_key_vault_missing_uri_failure_is_swallowed(monkeypatch) # --------------------------------------------------------------------------- -def test_cost_tracking_adds_two_callbacks_when_prisma_set(monkeypatch): +def test_cost_tracking_adds_db_and_shadow_eval_callbacks_when_prisma_set(monkeypatch): import litellm + from litellm.integrations.shadow_eval_logger import ShadowEvalLogger fake_prisma = MagicMock() monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False) @@ -535,16 +536,19 @@ def test_cost_tracking_adds_two_callbacks_when_prisma_set(monkeypatch): before_callbacks = len(litellm.callbacks) before_async = len(litellm._async_success_callback) + cost_tracking() cost_tracking() observed = { "added_to_callbacks": len(litellm.callbacks) - before_callbacks, "added_to_async_success": len(litellm._async_success_callback) - before_async, + "shadow_eval_loggers": sum(isinstance(cb, ShadowEvalLogger) for cb in litellm.callbacks), "prisma_was_set": True, } assert normalize(observed) == { - "added_to_callbacks": 1, + "added_to_callbacks": 2, "added_to_async_success": 1, + "shadow_eval_loggers": 1, "prisma_was_set": True, } diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index e3b0872ee65..4f43567de36 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -3449,98 +3449,6 @@ class TestKeywordOverrideEdgeCases: assert result.model in {"gpt-4o-mini", "gpt-4o", "claude-sonnet-4-20250514", "o1-preview"} -class TestSubCallMetadataSanitization: - """The proxy cost callback must not be able to recover the parent budget reservation - from sub-call metadata, in either of the shapes it knows how to read.""" - - def test_cost_callback_cannot_recover_reservation_from_sanitized_metadata(self): - from litellm.proxy._types import UserAPIKeyAuth - from litellm.proxy.hooks.proxy_track_cost_callback import ( - _get_budget_reservation_from_metadata, - ) - from litellm.router_strategy.complexity_router.complexity_router import ( - _classifier_call_metadata, - ) - - reservation = {"reserved_cost": 1.0} - auth_shapes = ( - {"models": ["gpt-4o"], "budget_reservation": dict(reservation)}, - UserAPIKeyAuth(api_key="sk-abc", budget_reservation=dict(reservation)), - ) - for auth in auth_shapes: - metadata = { - "user_api_key_hash": "hash-abc", - "user_api_key_budget_reservation": dict(reservation), - "user_api_key_auth": auth, - } - assert _get_budget_reservation_from_metadata(metadata) == reservation - - sanitized = _classifier_call_metadata(metadata) - assert sanitized is not None - assert sanitized["user_api_key_auth"] is not None - assert _get_budget_reservation_from_metadata(sanitized) is None - - def test_absent_parent_bucket_stays_empty(self): - """An absent bucket must not be materialized just to carry the origin. - - The embedding path passes both buckets, and get_litellm_metadata_from_kwargs - prefers litellm_metadata whenever it is truthy, backfilling only user_api_key* - keys from metadata. Returning an origin-only dict here would make a chat - completions parent's empty litellm_metadata win and silently drop - requester_ip_address, tags and spend_logs_metadata from the classifier's row.""" - from litellm.router_strategy.complexity_router.complexity_router import ( - _classifier_call_metadata, - ) - - for absent in (None, {}): - assert _classifier_call_metadata(absent) == {} - - def test_classifier_buckets_keep_non_spend_fields_on_a_chat_completions_parent(self): - """Drives the real resolver over the buckets the embedding classifier builds.""" - from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs - from litellm.router_strategy.complexity_router.complexity_router import ( - _classifier_call_metadata, - ) - - parent = { - "user_api_key": "sk-abc", - "requester_ip_address": "10.0.0.1", - "spend_logs_metadata": {"team_note": "keep me"}, - "tags": ["prod"], - } - resolved = get_litellm_metadata_from_kwargs( - { - "litellm_params": { - "metadata": _classifier_call_metadata(parent), - "litellm_metadata": _classifier_call_metadata(None), - } - } - ) - assert resolved["internal_call_origin"] == "autorouter_classifier" - assert resolved["requester_ip_address"] == "10.0.0.1" - assert resolved["spend_logs_metadata"] == {"team_note": "keep me"} - assert resolved["tags"] == ["prod"] - - def test_sanitized_auth_keeps_access_group_fields_and_leaves_original_untouched(self): - from litellm.proxy._types import UserAPIKeyAuth - from litellm.router_strategy.complexity_router.complexity_router import ( - _classifier_call_metadata, - ) - - auth = UserAPIKeyAuth( - api_key="sk-abc", - team_id="team-1", - budget_reservation={"reserved_cost": 1.0}, - ) - sanitized = _classifier_call_metadata({"user_api_key_auth": auth}) - assert sanitized is not None - sanitized_auth = sanitized["user_api_key_auth"] - assert sanitized_auth.budget_reservation is None - assert sanitized_auth.team_id == "team-1" - assert sanitized_auth.api_key == auth.api_key - assert auth.budget_reservation == {"reserved_cost": 1.0} - - class TestRoutingDecisionCauseLogging: """The info log must name what drove each routing decision so an operator can tell a literal keyword match, a semantic keyword match, and the complexity scorer apart. diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 327304dc45f..a2a4c01c25f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -807,6 +807,93 @@ export interface paths { patch?: never; trace?: never; }; + "/auto_router/shadow_eval": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * List Shadow Eval Jobs + * @description List shadow eval jobs, newest first. Counts and results ride the detail endpoint only. + */ + get: operations["list_shadow_eval_jobs_auto_router_shadow_eval_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/auto_router/shadow_eval/start": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Start Shadow Eval + * @description Start a pre-adoption shadow eval: duplicate a sampled slice of a key's live traffic + * through an auto-router, judge real vs. shadow responses blind, and stratify win rates + * by the router's tier classification and by the incumbent model. + * + * Shadow responses are never served to users. The job samples until it has judged + * max_turns turns, reaches the end of its window, or is stopped; sampling changes + * propagate to pods within about 10 seconds. Shadow and judge calls bill to the + * shadowed key but are excluded from request counts and auto-router adoption metrics. + */ + post: operations["start_shadow_eval_auto_router_shadow_eval_start_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/auto_router/shadow_eval/{job_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Shadow Eval Job + * @description One job with derived counts, judge spend, latest error, and stratified results. + */ + get: operations["get_shadow_eval_job_auto_router_shadow_eval__job_id__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/auto_router/shadow_eval/{job_id}/stop": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Stop Shadow Eval Job + * @description Stop an active shadow eval job. Attempts are kept; sampling halts within ~10s. + */ + post: operations["stop_shadow_eval_job_auto_router_shadow_eval__job_id__stop_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/auto_router/test_routing": { parameters: { query?: never; @@ -32557,6 +32644,110 @@ export interface components { /** Timeout */ timeout?: number | null; }; + /** + * ShadowEvalJobResponse + * @description A shadow-eval job. Validates directly from the prisma record (job_id reads the + * row's id); status is derived from stopped_at and ends_at, never stored, so no writer + * anywhere can produce an inconsistent one. Aggregate fields are populated by the + * detail endpoint only and stay None on list responses. + */ + ShadowEvalJobResponse: { + /** + * Api Key Id + * @description The hashed virtual key whose traffic this job evaluates, and only that key's + */ + api_key_id: string; + /** + * Created At + * Format: date-time + */ + created_at: string; + /** + * Ends At + * Format: date-time + */ + ends_at: string; + /** + * Error Count + * @description Sampled attempts that errored; detail endpoint only + */ + error_count?: number | null; + /** Job Id */ + job_id: string; + /** Judge Model */ + judge_model: string; + /** + * Judge Spend + * @description Judge cost so far; detail endpoint only + */ + judge_spend?: number | null; + /** + * Judged Count + * @description Verdicts recorded; detail endpoint only + */ + judged_count?: number | null; + /** + * Last Error + * @description Most recent attempt error; detail endpoint only + */ + last_error?: string | null; + /** Max Turns */ + max_turns: number; + /** @description Stratified verdicts; detail endpoint only */ + results?: components["schemas"]["ShadowEvalResult"] | null; + /** Router Name */ + router_name: string; + /** Shadow Percentage */ + shadow_percentage: number; + /** + * Status + * @description A job whose window has passed reads completed even if a later sweep stamped + * stopped_at; stopped means sampling ended before the window did. + * @enum {string} + */ + readonly status: "running" | "completed" | "stopped"; + /** Stopped At */ + stopped_at?: string | null; + }; + /** + * ShadowEvalResult + * @description Stratified results of a shadow-eval job's verdicts so far. + */ + ShadowEvalResult: { + /** By Current Model */ + by_current_model: components["schemas"]["ShadowEvalSlice"][]; + /** By Tier */ + by_tier: components["schemas"]["ShadowEvalSlice"][]; + /** Overall Shadow Win Rate Pct */ + overall_shadow_win_rate_pct: number; + /** Overall Tie Rate Pct */ + overall_tie_rate_pct: number; + }; + /** + * ShadowEvalSlice + * @description Judge outcomes for one slice of a job's verdicts (a router tier, or one of the + * models the shadowed key currently uses). + */ + ShadowEvalSlice: { + /** Avg Judge Confidence */ + avg_judge_confidence: number; + /** Group */ + group: string; + /** + * Real Win Rate Pct + * @description Share of judged turns where the real (control) model won + */ + real_win_rate_pct: number; + /** + * Shadow Win Rate Pct + * @description Share of judged turns where the shadowed router's pick won + */ + shadow_win_rate_pct: number; + /** Tie Rate Pct */ + tie_rate_pct: number; + /** Turn Count */ + turn_count: number; + }; /** * Skill * @description Represents a skill from the Anthropic Skills API @@ -32730,6 +32921,45 @@ export interface components { /** Simple Medium */ simple_medium: number; }; + /** + * StartShadowEvalRequest + * @description Start shadowing a key's traffic through an auto-router for blind comparison. + */ + StartShadowEvalRequest: { + /** + * Api Key Id + * @description The hashed virtual key whose traffic will be shadowed. Shadow evaluation runs ONLY on this key's traffic; requests made with any other key are not sampled. + */ + api_key_id: string; + /** + * Duration Days + * @description How many days the job samples traffic before completing on its own + * @default 7 + */ + duration_days: number; + /** + * Judge Model + * @description Model used to blindly judge real vs. shadow responses. The judge only compares two answers, so a mid-tier model (Claude Sonnet or GPT-4o class) is the sweet spot: small/nano-class models produce unreliable or malformed verdicts, while frontier reasoning models add cost without changing outcomes. + * @default anthropic/claude-sonnet-5 + */ + judge_model: string; + /** + * Max Turns + * @description Sample budget: the job judges at most this many turns, then completes. This is also the spend bound; expected judge cost is roughly max_turns times one judge call + * @default 200 + */ + max_turns: number; + /** + * Router Name + * @description The auto-router config to shadow requests through + */ + router_name: string; + /** + * Shadow Percentage + * @description Percentage of the key's requests to duplicate through the router + */ + shadow_percentage: number; + }; /** * SuccessfulKeyUpdate * @description Successfully updated key with its updated information @@ -37006,6 +37236,135 @@ export interface operations { }; }; }; + list_shadow_eval_jobs_auto_router_shadow_eval_get: { + parameters: { + query?: { + /** @description Filter to jobs shadowing this key */ + api_key_id?: string | null; + /** @description Newest jobs to return */ + limit?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ShadowEvalJobResponse"][]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + start_shadow_eval_auto_router_shadow_eval_start_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["StartShadowEvalRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ShadowEvalJobResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_shadow_eval_job_auto_router_shadow_eval__job_id__get: { + parameters: { + query?: never; + header?: never; + path: { + job_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ShadowEvalJobResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + stop_shadow_eval_job_auto_router_shadow_eval__job_id__stop_post: { + parameters: { + query?: never; + header?: never; + path: { + job_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ShadowEvalJobResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; preview_auto_router_routing_auto_router_test_routing_post: { parameters: { query?: never;