From ff2718bcfa5266fe6da92891d7a88633a04fd5fd Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 13 Oct 2026 23:13:40 -0700 Subject: [PATCH 01/51] feat(ui): move the upgrade notice into the notifications bell (#46334) * feat(ui): move the upgrade notice into the notifications bell The new version notice was a banner in the same stack as deployment warnings. It now lives in the notifications bell next to Auto Router, each announcement has its own read state in localStorage keyed per item and per release, and the bell shows its unread dot while any item is unread. The upgrade banner is removed from both dashboard shells. * test(ui): cover singular release wording, legacy dismissals, and the public page bell guard * fix(proxy-extras): bound telemetry_reports migration lock waits with SET LOCAL lock_timeout Ports the open fix in #46386 so this branch passes the migration lock_timeout check that main currently fails. * Revert "fix(proxy-extras): bound telemetry_reports migration lock waits with SET LOCAL lock_timeout" This reverts commit b7c1df1a579a8d426e55633ed4b665a15dcf3502. --- .../(dashboard)/hooks/useAnnouncementRead.ts | 37 +++ .../hooks/useHideAutoRouterAnnouncement.ts | 35 --- .../src/app/(dashboard)/layout.test.tsx | 4 - .../src/app/(dashboard)/layout.tsx | 4 +- .../DashboardHeader.integration.test.tsx | 2 +- .../src/components/DashboardHeader.test.tsx | 6 +- .../src/components/DashboardHeader.tsx | 12 +- .../NotificationsBell.test.tsx | 220 +++++++++++++----- .../NotificationsBell/NotificationsBell.tsx | 122 +++++++--- .../src/components/UpgradeBanner.test.tsx | 131 ----------- .../src/components/UpgradeBanner.tsx | 77 ------ .../src/components/navbar.test.tsx | 4 +- .../src/components/navbar.tsx | 2 +- 13 files changed, 309 insertions(+), 347 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/useAnnouncementRead.ts delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/useHideAutoRouterAnnouncement.ts delete mode 100644 ui/litellm-dashboard/src/components/UpgradeBanner.test.tsx delete mode 100644 ui/litellm-dashboard/src/components/UpgradeBanner.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAnnouncementRead.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAnnouncementRead.ts new file mode 100644 index 00000000000..916ccc43017 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAnnouncementRead.ts @@ -0,0 +1,37 @@ +import { useSyncExternalStore } from "react"; +import { + emitLocalStorageChange, + getLocalStorageItem, + LOCAL_STORAGE_EVENT, + setLocalStorageItem, +} from "@/utils/localStorageUtils"; + +function subscribe(callback: () => void) { + window.addEventListener("storage", callback); + window.addEventListener(LOCAL_STORAGE_EVENT, callback); + return () => { + window.removeEventListener("storage", callback); + window.removeEventListener(LOCAL_STORAGE_EVENT, callback); + }; +} + +export function useAnnouncementRead(readKey: string): boolean { + return useSyncExternalStore( + subscribe, + () => getLocalStorageItem(readKey) === "true", + () => false, + ); +} + +export function markAnnouncementRead(readKey: string): void { + setLocalStorageItem(readKey, "true"); + emitLocalStorageChange(readKey); +} + +export function useUnreadAnnouncementCount(readKeys: readonly string[]): number { + return useSyncExternalStore( + subscribe, + () => readKeys.filter((readKey) => getLocalStorageItem(readKey) !== "true").length, + () => readKeys.length, + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useHideAutoRouterAnnouncement.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useHideAutoRouterAnnouncement.ts deleted file mode 100644 index 61b9b3eea7a..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useHideAutoRouterAnnouncement.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { useSyncExternalStore } from "react"; -import { getLocalStorageItem, LOCAL_STORAGE_EVENT } from "@/utils/localStorageUtils"; - -export const HIDE_AUTO_ROUTER_ANNOUNCEMENT_KEY = "litellmHideAutoRouterAnnouncement"; - -function subscribe(callback: () => void) { - const onStorage = (e: StorageEvent) => { - if (e.key === HIDE_AUTO_ROUTER_ANNOUNCEMENT_KEY) { - callback(); - } - }; - - const onCustom = (e: Event) => { - const { key } = (e as CustomEvent).detail; - if (key === HIDE_AUTO_ROUTER_ANNOUNCEMENT_KEY) { - callback(); - } - }; - - window.addEventListener("storage", onStorage); - window.addEventListener(LOCAL_STORAGE_EVENT, onCustom); - - return () => { - window.removeEventListener("storage", onStorage); - window.removeEventListener(LOCAL_STORAGE_EVENT, onCustom); - }; -} - -function getSnapshot() { - return getLocalStorageItem(HIDE_AUTO_ROUTER_ANNOUNCEMENT_KEY) === "true"; -} - -export function useHideAutoRouterAnnouncement() { - return useSyncExternalStore(subscribe, getSnapshot, () => false); -} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx index 489c5a515f3..bd5c6949c6b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx @@ -60,10 +60,6 @@ vi.mock("@/components/UserBanner", () => ({ UserBanner: () => null, })); -vi.mock("@/components/UpgradeBanner", () => ({ - UpgradeBanner: () => null, -})); - vi.mock("@/contexts/ThemeContext", () => ({ ThemeProvider: ({ children }: { children: React.ReactNode }) => <>{children}, })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx index 90401235676..b0494f8415a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx @@ -15,7 +15,6 @@ import { EnvCredentialLoginWarningBanner } from "@/components/EnvCredentialLogin import { LicenseExpiryBanner } from "@/components/LicenseExpiryBanner"; import { UserBanner } from "@/components/UserBanner"; import { LiteAdminFrame } from "@/components/liteadmin/LiteAdmin"; -import { UpgradeBanner } from "@/components/UpgradeBanner"; import { routeSegmentForPathname, uiHref } from "@/utils/uiHref"; import { PluginModeProvider, usePluginMode } from "@/contexts/PluginModeContext"; import { createApiClient } from "@/lib/http/client"; @@ -147,7 +146,6 @@ function DashboardShell({ children }: { children: React.ReactNode }) { -
@@ -184,6 +182,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) {
-
{children} diff --git a/ui/litellm-dashboard/src/components/DashboardHeader.integration.test.tsx b/ui/litellm-dashboard/src/components/DashboardHeader.integration.test.tsx index cb58670c20a..d899d032395 100644 --- a/ui/litellm-dashboard/src/components/DashboardHeader.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/DashboardHeader.integration.test.tsx @@ -59,7 +59,7 @@ async function openTools() { - + , diff --git a/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx b/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx index 6c64d09009c..41d35bd40b7 100644 --- a/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx +++ b/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx @@ -35,7 +35,7 @@ vi.mock("@/components/liteadmin/LiteAdmin", () => ({ default: () =>
- + }> diff --git a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx index 8894f2a714b..e0c428a94a0 100644 --- a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx @@ -1,69 +1,175 @@ import { renderWithProviders, screen } from "../../../../tests/test-utils"; -import { NotificationsBell, AUTO_ROUTER_DOCS_URL } from "./NotificationsBell"; +import { AUTO_ROUTER_DOCS_URL, NotificationsBell, NotificationsBellView } from "./NotificationsBell"; +import type { LatestReleaseInfo } from "@/app/(dashboard)/hooks/latestRelease/useLatestReleaseInfo"; import React from "react"; import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails", () => ({ + useHealthReadinessDetails: vi.fn(), +})); +vi.mock("@/app/(dashboard)/hooks/latestRelease/useLatestReleaseInfo", () => ({ + useLatestReleaseInfo: vi.fn(), +})); + +import { useHealthReadinessDetails } from "@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails"; +import { useLatestReleaseInfo } from "@/app/(dashboard)/hooks/latestRelease/useLatestReleaseInfo"; + +const RELEASE: LatestReleaseInfo = { + version: "1.103.0", + new_features: 12, + bug_fixes: 30, + other_updates: 8, + release_url: "https://github.com/BerriAI/litellm/releases/tag/v1.103.0", +}; + +const bell = (unread: number) => + screen.getByRole("button", { name: unread === 0 ? /^notifications$/i : `Notifications, ${unread} unread` }); + +const announcement = (title: string) => screen.getByRole("article", { name: title }); + +describe("NotificationsBellView", () => { + beforeEach(() => { + localStorage.clear(); + }); + + afterEach(() => { + localStorage.clear(); + }); + + it("lists the new version notice above Auto Router when the proxy is behind the latest release", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(bell(2)); + + const titles = screen.getAllByRole("article").map((item) => item.getAttribute("aria-label")); + expect(titles).toEqual(["LiteLLM v1.103.0 is available", "LiteLLM Auto Router"]); + const upgrade = announcement("LiteLLM v1.103.0 is available"); + expect(upgrade).toHaveTextContent("12 new features, 30 fixes, and 8 other updates"); + expect(upgrade).toHaveTextContent("Your current version is v1.102.0"); + const notes = screen.getByRole("link", { name: /^view release notes$/i }); + expect(notes).toHaveAttribute("href", RELEASE.release_url); + expect(notes).toHaveAttribute("rel", "noopener noreferrer"); + const docs = screen.getByRole("link", { name: /^read the docs$/i }); + expect(docs).toHaveAttribute("href", AUTO_ROUTER_DOCS_URL); + expect(docs).toHaveAttribute("target", "_blank"); + }); + + it("uses singular wording for counts of one", async () => { + const user = userEvent.setup(); + renderWithProviders( + , + ); + await user.click(bell(2)); + expect(announcement("LiteLLM v1.103.0 is available")).toHaveTextContent("1 new feature, 1 fix, and 1 other update"); + }); + + it.each([ + ["the running version is unknown", undefined, RELEASE], + ["the latest release is unavailable", "1.102.0", null], + ["the proxy is up to date", "1.103.0", RELEASE], + ["the proxy is ahead of the latest release", "1.104.0-dev.1", RELEASE], + ])("lists only Auto Router when %s", async (_case, currentVersion, latestRelease) => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(bell(1)); + expect(screen.getAllByRole("article").map((item) => item.getAttribute("aria-label"))).toEqual([ + "LiteLLM Auto Router", + ]); + }); + + it("marks each announcement read on its own and clears the unread dot once all are read", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(bell(2)); + await user.click(screen.getAllByRole("button", { name: /^mark as read$/i })[0]); + + expect(bell(1).querySelector("[data-slot=badge]")).not.toBeNull(); + expect(screen.getAllByRole("button", { name: /^mark as read$/i })).toHaveLength(1); + expect(announcement("LiteLLM Auto Router")).toContainElement( + screen.getByRole("button", { name: /^mark as read$/i }), + ); + await user.click(screen.getByRole("button", { name: /^mark as read$/i })); + + expect(bell(0).querySelector("[data-slot=badge]")).toBeNull(); + expect(screen.getAllByRole("article")).toHaveLength(2); + expect(screen.queryByRole("button", { name: /^mark as read$/i })).not.toBeInTheDocument(); + }); + + it("keeps the new version notice read across remounts for the same release", async () => { + const user = userEvent.setup(); + const first = renderWithProviders(); + await user.click(bell(2)); + await user.click(screen.getAllByRole("button", { name: /^mark as read$/i })[0]); + first.unmount(); + + renderWithProviders(); + expect(bell(1)).toBeInTheDocument(); + }); + + it("shows a newer release as unread after an older release was read", async () => { + const user = userEvent.setup(); + const { rerender } = renderWithProviders( + , + ); + await user.click(bell(2)); + await user.click(screen.getAllByRole("button", { name: /^mark as read$/i })[0]); + expect(bell(1)).toBeInTheDocument(); + + rerender(); + expect(bell(2)).toBeInTheDocument(); + expect(announcement("LiteLLM v1.104.0 is available")).toContainElement( + screen.getAllByRole("button", { name: /^mark as read$/i })[0], + ); + }); + + it("treats a release whose upgrade banner was dismissed as already read", () => { + localStorage.setItem("litellm:upgradeBannerDismissed:1.103.0", "true"); + renderWithProviders(); + expect(bell(1)).toBeInTheDocument(); + }); + + it("keeps Auto Router read when it was read before", async () => { + localStorage.setItem("litellmHideAutoRouterAnnouncement", "true"); + const user = userEvent.setup(); + renderWithProviders(); + await user.click(bell(0)); + expect(announcement("LiteLLM Auto Router")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /^mark as read$/i })).not.toBeInTheDocument(); + }); + + it("syncs sibling bells when one marks an announcement read", async () => { + const user = userEvent.setup(); + renderWithProviders( + <> + + + , + ); + const [bellA] = screen.getAllByRole("button", { name: "Notifications, 2 unread" }); + await user.click(bellA); + await user.click(screen.getAllByRole("button", { name: /^mark as read$/i })[0]); + + expect(screen.getAllByRole("button", { name: "Notifications, 1 unread" })).toHaveLength(2); + }); +}); describe("NotificationsBell", () => { beforeEach(() => { localStorage.clear(); }); - it("should open notifications with Auto Router details and docs link", async () => { + it("builds the new version notice from the proxy version and the latest release", async () => { + vi.mocked(useHealthReadinessDetails).mockReturnValue({ + data: { status: "healthy", litellm_version: "1.102.0" }, + } as ReturnType); + vi.mocked(useLatestReleaseInfo).mockReturnValue({ data: RELEASE } as ReturnType); const user = userEvent.setup(); - renderWithProviders(); - await user.click(screen.getByRole("button", { name: /^notifications$/i })); - expect(screen.getByText(/^LiteLLM Auto Router$/i)).toBeInTheDocument(); - const docsBtn = screen.getByRole("link", { name: /^read the docs$/i }); - expect(docsBtn).toHaveAttribute("href", AUTO_ROUTER_DOCS_URL); - expect(docsBtn).toHaveAttribute("target", "_blank"); - expect(docsBtn).toHaveAttribute("rel", "noopener noreferrer"); - }); - - it("should offer mark as read when announcement is unread", async () => { - const user = userEvent.setup(); - renderWithProviders(); - await user.click(screen.getByRole("button", { name: /^notifications$/i })); - expect(screen.getByRole("button", { name: /^mark as read$/i })).toBeInTheDocument(); - }); - - it("should hide mark as read and persist after marking read", async () => { - const user = userEvent.setup(); - renderWithProviders(); - await user.click(screen.getByRole("button", { name: /^notifications$/i })); - await user.click(screen.getByRole("button", { name: /^mark as read$/i })); - expect(localStorage.getItem("litellmHideAutoRouterAnnouncement")).toBe("true"); - await user.click(screen.getByRole("button", { name: /^notifications$/i })); - expect(screen.queryByRole("button", { name: /^mark as read$/i })).not.toBeInTheDocument(); - }); - - it("should not show mark as read when previously dismissed", async () => { - localStorage.setItem("litellmHideAutoRouterAnnouncement", "true"); - const user = userEvent.setup(); - renderWithProviders(); - await user.click(screen.getByRole("button", { name: /^notifications$/i })); - expect(screen.queryByRole("button", { name: /^mark as read$/i })).not.toBeInTheDocument(); - }); - - it("should sync sibling instances when one is dismissed", async () => { - const user = userEvent.setup(); - renderWithProviders( - <> -
- -
-
- -
- , - ); - - // Both bells start unread → both render the "Mark as read" affordance once opened. - const [bellA, bellB] = screen.getAllByRole("button", { name: /^notifications$/i }); - await user.click(bellA); - await user.click(screen.getByRole("button", { name: /^mark as read$/i })); - - // Dismissing in bell A must also clear bell B without a remount. - await user.click(bellB); - expect(screen.queryByRole("button", { name: /^mark as read$/i })).not.toBeInTheDocument(); + renderWithProviders(); + await user.click(bell(2)); + expect(announcement("LiteLLM v1.103.0 is available")).toHaveTextContent("Your current version is v1.102.0"); }); }); diff --git a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx index 8b9b7d1d0d2..4cb9f851153 100644 --- a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx @@ -1,66 +1,128 @@ "use client"; +import { useHealthReadinessDetails } from "@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails"; import { - HIDE_AUTO_ROUTER_ANNOUNCEMENT_KEY, - useHideAutoRouterAnnouncement, -} from "@/app/(dashboard)/hooks/useHideAutoRouterAnnouncement"; -import { emitLocalStorageChange, setLocalStorageItem } from "@/utils/localStorageUtils"; + type LatestReleaseInfo, + useLatestReleaseInfo, +} from "@/app/(dashboard)/hooks/latestRelease/useLatestReleaseInfo"; +import { + markAnnouncementRead, + useAnnouncementRead, + useUnreadAnnouncementCount, +} from "@/app/(dashboard)/hooks/useAnnouncementRead"; import { Badge } from "@/components/ui/badge"; import { Button, buttonVariants } from "@/components/ui/button"; -import { Popover, PopoverContent, PopoverDescription, PopoverTitle, PopoverTrigger } from "@/components/ui/popover"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { Separator } from "@/components/ui/separator"; import { cn } from "@/lib/cva.config"; +import { isNewerVersion } from "@/utils/versionUtils"; import { Bell } from "lucide-react"; -import React, { useState } from "react"; +import React, { Fragment } from "react"; export const AUTO_ROUTER_DOCS_URL = "https://docs.litellm.ai/docs/proxy/auto_routing"; -export const NotificationsBell: React.FC = () => { - const hidden = useHideAutoRouterAnnouncement(); - const hasUnread = !hidden; - const [open, setOpen] = useState(false); +interface Announcement { + readKey: string; + title: string; + description: string; + linkLabel: string; + href: string; +} - const markDismissed = () => { - setLocalStorageItem(HIDE_AUTO_ROUTER_ANNOUNCEMENT_KEY, "true"); - emitLocalStorageChange(HIDE_AUTO_ROUTER_ANNOUNCEMENT_KEY); - setOpen(false); +const AUTO_ROUTER_ANNOUNCEMENT: Announcement = { + readKey: "litellmHideAutoRouterAnnouncement", + title: "LiteLLM Auto Router", + description: "Route every request to the cheapest model that can handle it, no prompt changes needed.", + linkLabel: "Read the docs", + href: AUTO_ROUTER_DOCS_URL, +}; + +const plural = (count: number, singular: string, pluralForm: string): string => + `${count} ${count === 1 ? singular : pluralForm}`; + +const describeRelease = ({ new_features, bug_fixes, other_updates }: LatestReleaseInfo): string => + [ + plural(new_features, "new feature", "new features"), + plural(bug_fixes, "fix", "fixes"), + `and ${plural(other_updates, "other update", "other updates")}`, + ].join(", "); + +const upgradeAnnouncement = ( + currentVersion: string | null | undefined, + latestRelease: LatestReleaseInfo | null | undefined, +): Announcement | null => { + if (!currentVersion || !latestRelease || !isNewerVersion(currentVersion, latestRelease.version)) { + return null; + } + return { + readKey: `litellm:upgradeBannerDismissed:${latestRelease.version}`, + title: `LiteLLM v${latestRelease.version} is available`, + description: `${describeRelease(latestRelease)}. Your current version is v${currentVersion}`, + linkLabel: "View release notes", + href: latestRelease.release_url, }; +}; - const content = ( -
- LiteLLM Auto Router - - Route every request to the cheapest model that can handle it, no prompt changes needed. - +const AnnouncementItem: React.FC<{ announcement: Announcement }> = ({ announcement }) => { + const read = useAnnouncementRead(announcement.readKey); + return ( +
+ ); +}; + +interface NotificationsBellViewProps { + currentVersion: string | null | undefined; + latestRelease: LatestReleaseInfo | null | undefined; +} + +export const NotificationsBellView: React.FC = ({ currentVersion, latestRelease }) => { + const upgrade = upgradeAnnouncement(currentVersion, latestRelease); + const announcements = upgrade ? [upgrade, AUTO_ROUTER_ANNOUNCEMENT] : [AUTO_ROUTER_ANNOUNCEMENT]; + const unreadCount = useUnreadAnnouncementCount(announcements.map((announcement) => announcement.readKey)); return ( - + 0 ? `Notifications, ${unreadCount} unread` : "Notifications"} > - {hasUnread ? : null} + {unreadCount > 0 ? : null} - {content} + + {announcements.map((announcement, index) => ( + + {index > 0 ? : null} + + + ))} + ); }; + +export const NotificationsBell: React.FC<{ accessToken: string | null }> = ({ accessToken }) => { + const { data: healthData } = useHealthReadinessDetails(accessToken); + const { data: latestRelease } = useLatestReleaseInfo(accessToken); + return ; +}; diff --git a/ui/litellm-dashboard/src/components/UpgradeBanner.test.tsx b/ui/litellm-dashboard/src/components/UpgradeBanner.test.tsx deleted file mode 100644 index 4466597394c..00000000000 --- a/ui/litellm-dashboard/src/components/UpgradeBanner.test.tsx +++ /dev/null @@ -1,131 +0,0 @@ -import React from "react"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { fireEvent, render, screen } from "@testing-library/react"; -import { describeRelease, UpgradeBanner, UpgradeBannerView } from "./UpgradeBanner"; -import type { LatestReleaseInfo } from "@/app/(dashboard)/hooks/latestRelease/useLatestReleaseInfo"; - -vi.mock("@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails", () => ({ - useHealthReadinessDetails: vi.fn(), -})); -vi.mock("@/app/(dashboard)/hooks/latestRelease/useLatestReleaseInfo", () => ({ - useLatestReleaseInfo: vi.fn(), -})); - -import { useHealthReadinessDetails } from "@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails"; -import { useLatestReleaseInfo } from "@/app/(dashboard)/hooks/latestRelease/useLatestReleaseInfo"; - -const RELEASE: LatestReleaseInfo = { - version: "1.103.0", - new_features: 12, - bug_fixes: 30, - other_updates: 8, - release_url: "https://github.com/BerriAI/litellm/releases/tag/v1.103.0", -}; - -describe("describeRelease", () => { - it("lists features, fixes, and other updates in the agreed order", () => { - expect(describeRelease(RELEASE)).toBe("12 new features, 30 fixes, and 8 other updates"); - }); - - it("singularises counts of one", () => { - const singularCounts = { ...RELEASE, new_features: 1, bug_fixes: 1, other_updates: 1 }; - expect(describeRelease(singularCounts)).toBe("1 new feature, 1 fix, and 1 other update"); - }); -}); - -describe("UpgradeBannerView", () => { - beforeEach(() => { - localStorage.clear(); - }); - - afterEach(() => { - localStorage.clear(); - }); - - it("renders nothing while either version is unknown", () => { - const { container } = render(); - expect(container).toBeEmptyDOMElement(); - const { container: noRelease } = render(); - expect(noRelease).toBeEmptyDOMElement(); - }); - - it("renders nothing when the running version is up to date or ahead", () => { - const { container } = render(); - expect(container).toBeEmptyDOMElement(); - const { container: ahead } = render(); - expect(ahead).toBeEmptyDOMElement(); - }); - - it("shows the latest version, the stat line, and the current version when behind", () => { - render(); - const alert = screen.getByRole("status"); - expect(alert).toHaveTextContent("The latest version is v1.103.0: 12 new features, 30 fixes, and 8 other updates"); - expect(alert).toHaveTextContent("Your current version is v1.102.0"); - expect(screen.getByRole("link", { name: "v1.103.0" })).toHaveAttribute("href", RELEASE.release_url); - }); - - it("dismissing hides the banner and keeps it hidden on remount for the same release", () => { - const { unmount } = render(); - fireEvent.click(screen.getByRole("button", { name: "Close" })); - expect(screen.queryByRole("status")).not.toBeInTheDocument(); - unmount(); - - const { container } = render(); - expect(container).toBeEmptyDOMElement(); - }); - - it("reappears once a newer release ships after a dismissal", () => { - const { unmount } = render(); - fireEvent.click(screen.getByRole("button", { name: "Close" })); - unmount(); - - render(); - expect(screen.getByRole("status")).toHaveTextContent("The latest version is v1.104.0"); - }); - - it("shows a newer release after the current one was dismissed without remounting", () => { - const { rerender } = render(); - fireEvent.click(screen.getByRole("button", { name: "Close" })); - expect(screen.queryByRole("status")).not.toBeInTheDocument(); - rerender(); - expect(screen.getByRole("status")).toHaveTextContent("The latest version is v1.104.0"); - }); -}); - -describe("UpgradeBanner", () => { - beforeEach(() => { - localStorage.clear(); - }); - - afterEach(() => { - localStorage.clear(); - }); - - it("feeds both hooks the access token and renders from their data", () => { - const healthReadinessResult = { - data: { litellm_version: "1.102.0" }, - } as Partial> as ReturnType; - const latestReleaseResult = { data: RELEASE } as Partial> as ReturnType< - typeof useLatestReleaseInfo - >; - vi.mocked(useHealthReadinessDetails).mockReturnValue(healthReadinessResult); - vi.mocked(useLatestReleaseInfo).mockReturnValue(latestReleaseResult); - render(); - expect(useHealthReadinessDetails).toHaveBeenCalledWith("token"); - expect(useLatestReleaseInfo).toHaveBeenCalledWith("token"); - expect(screen.getByRole("status")).toHaveTextContent("The latest version is v1.103.0"); - }); - - it("renders nothing when the release endpoint returns null", () => { - const healthReadinessResult = { - data: { litellm_version: "1.102.0" }, - } as Partial> as ReturnType; - const latestReleaseResult = { data: null } as Partial> as ReturnType< - typeof useLatestReleaseInfo - >; - vi.mocked(useHealthReadinessDetails).mockReturnValue(healthReadinessResult); - vi.mocked(useLatestReleaseInfo).mockReturnValue(latestReleaseResult); - const { container } = render(); - expect(container).toBeEmptyDOMElement(); - }); -}); diff --git a/ui/litellm-dashboard/src/components/UpgradeBanner.tsx b/ui/litellm-dashboard/src/components/UpgradeBanner.tsx deleted file mode 100644 index 101a6e3410d..00000000000 --- a/ui/litellm-dashboard/src/components/UpgradeBanner.tsx +++ /dev/null @@ -1,77 +0,0 @@ -"use client"; - -import React, { useState } from "react"; -import { ArrowUpCircle, X } from "lucide-react"; -import { Alert, AlertAction, AlertDescription, AlertTitle } from "@/components/shared/Alert"; -import { Button } from "@/components/ui/button"; -import { useHealthReadinessDetails } from "@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails"; -import { - type LatestReleaseInfo, - useLatestReleaseInfo, -} from "@/app/(dashboard)/hooks/latestRelease/useLatestReleaseInfo"; -import { getLocalStorageItem, setLocalStorageItem } from "@/utils/localStorageUtils"; -import { isNewerVersion } from "@/utils/versionUtils"; - -const DISMISS_KEY_PREFIX = "litellm:upgradeBannerDismissed:"; - -interface UpgradeBannerProps { - accessToken: string | null; -} - -interface UpgradeBannerViewProps { - currentVersion: string | null | undefined; - latestRelease: LatestReleaseInfo | null | undefined; -} - -const plural = (count: number, singular: string, pluralForm: string): string => - `${count} ${count === 1 ? singular : pluralForm}`; - -export const describeRelease = ({ new_features, bug_fixes, other_updates }: LatestReleaseInfo): string => - [ - plural(new_features, "new feature", "new features"), - plural(bug_fixes, "fix", "fixes"), - `and ${plural(other_updates, "other update", "other updates")}`, - ].join(", "); - -export const UpgradeBannerView: React.FC = ({ currentVersion, latestRelease }) => { - const [dismissedVersion, setDismissedVersion] = useState(null); - - if (!currentVersion || !latestRelease || !isNewerVersion(currentVersion, latestRelease.version)) { - return null; - } - - const dismissKey = `${DISMISS_KEY_PREFIX}${latestRelease.version}`; - if (dismissedVersion === latestRelease.version || getLocalStorageItem(dismissKey) === "true") { - return null; - } - - const handleClose = () => { - setLocalStorageItem(dismissKey, "true"); - setDismissedVersion(latestRelease.version); - }; - - return ( - - - - The latest version is{" "} - - v{latestRelease.version} - - : {describeRelease(latestRelease)} - - Your current version is v{currentVersion} - - - - - ); -}; - -export const UpgradeBanner: React.FC = ({ accessToken }) => { - const { data: healthData } = useHealthReadinessDetails(accessToken); - const { data: latestRelease } = useLatestReleaseInfo(accessToken); - return ; -}; diff --git a/ui/litellm-dashboard/src/components/navbar.test.tsx b/ui/litellm-dashboard/src/components/navbar.test.tsx index 9d80a89f83e..b8c87a3145e 100644 --- a/ui/litellm-dashboard/src/components/navbar.test.tsx +++ b/ui/litellm-dashboard/src/components/navbar.test.tsx @@ -143,7 +143,7 @@ describe("Navbar", () => { it("should render without crashing", () => { renderWithProviders(); - expect(screen.getByRole("button", { name: /^notifications$/i })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /^notifications/i })).toBeInTheDocument(); expect(screen.getByText("Docs")).toBeInTheDocument(); expect(screen.getByRole("button", { name: /open account menu/i })).toBeInTheDocument(); }); @@ -266,7 +266,7 @@ describe("Navbar", () => { renderWithProviders(); expect(screen.queryByRole("button", { name: /open account menu/i })).not.toBeInTheDocument(); - expect(screen.queryByRole("button", { name: /^notifications$/i })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /^notifications/i })).not.toBeInTheDocument(); }); it("should handle hide new feature indicators toggle", async () => { diff --git a/ui/litellm-dashboard/src/components/navbar.tsx b/ui/litellm-dashboard/src/components/navbar.tsx index 8bc9b06969e..4e99d8e016f 100644 --- a/ui/litellm-dashboard/src/components/navbar.tsx +++ b/ui/litellm-dashboard/src/components/navbar.tsx @@ -149,7 +149,7 @@ const Navbar: React.FC = ({
- +
From aa879f77a90d470427dd2b49877b03a0de131697 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 13 Oct 2026 23:25:58 -0700 Subject: [PATCH 02/51] feat(proxy): let team admins create keys for team members behind an opt-in setting (#45870) * feat(proxy): let team admins create keys for team members behind an opt-in setting A team admin calling POST /key/generate with a team they administer and the user_id of another member of that team got a 403, and the dashboard only offered the "Another User" owner option to proxy admins Add a create_member_keys entry to team_admin_editable_team_fields. When a proxy admin turns it on, a team admin can create a team key owned by another member of a team they administer, through the API and through the Create Key form. It stays off by default, so nothing changes until a proxy admin opts in With the setting off, the 403 a team admin gets for this call now names the setting to enable instead of saying they can only create keys for themselves * fix(ui): order the member owner picker after the team and drop the viewer from it For a team admin the owner picker depends on the team, so it now sits under the Team field, lists only the other members of that team, and the Team field says why it is required * fix(proxy): keep keys a team admin creates for a member from carrying more than LLM access A key authenticates as its owner, so with create_member_keys on a team admin could create a key owned by a proxy admin who sits in their team and act as a proxy admin with it The member now has to hold the internal_user or internal_user_viewer role, and the key is always an llm_api key, so the team admin never holds a secret that reaches management routes. Asking for a management or read_only key for a member is refused The unit tests now check the permission decision directly instead of replacing generate_key_helper_fn, and the saved key is checked against a real proxy in the team admin gate matrix * style(proxy): sort the team admin field permission imports * refactor(proxy): force the member key type on the request instead of merging a second dict * test(proxy): drive member key creation through the key generation helper The pure helpers were covered but the wiring in _common_key_generation_helper was not. Dropping the owner role lookup, the refusal call, or the user_id handed to common_key_access_checks left every unit test green * fix(proxy): skip the owner role lookup when create_member_keys is off The owner role was read before the verdict checked the setting, so with the setting off a failing user table read surfaced as a 500 instead of the 403 a team admin got before, and every such request paid an extra writer read --------- Co-authored-by: Yuneng Jiang --- .../key_management_endpoints.py | 58 +++++- .../team_admin_field_permissions.py | 100 ++++++++++- .../proxy_setting_endpoints.py | 2 + .../authorization/test_team_admin_gate.py | 40 +++++ .../test_key_management_endpoints.py | 162 +++++++++++++++++ .../test_team_admin_field_permissions.py | 74 +++++++- .../test_proxy_setting_endpoints.py | 22 +++ .../organisms/createKeyOwnerAccess.test.ts | 164 +++++++++++++++++ .../organisms/createKeyOwnerAccess.ts | 59 +++++++ .../create_key_button.integration.test.tsx | 53 ++++++ .../organisms/create_key_button.tsx | 167 ++++++++++++------ .../team/teamAdminEditAccess.test.ts | 1 + .../components/team/teamAdminEditAccess.ts | 3 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 14 files changed, 844 insertions(+), 63 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/organisms/createKeyOwnerAccess.test.ts create mode 100644 ui/litellm-dashboard/src/components/organisms/createKeyOwnerAccess.ts diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index a3f65b7315b..896f12cb0c7 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -112,9 +112,13 @@ from litellm.proxy.management_endpoints.model_management_endpoints import ( # n ) from litellm.proxy.management_endpoints.router_weights import validate_router_settings_weights from litellm.proxy.management_endpoints.team_admin_field_permissions import ( + TEAM_ADMIN_MEMBER_KEY_TYPE, team_admin_key_edit_verdict, team_admin_key_request_or_raise, + team_admin_may_create_member_keys, team_admin_may_edit_member_key_budgets, + team_admin_member_key_creation_or_raise, + team_admin_member_key_creation_verdict, ) from litellm.proxy.management_helpers.access_group_key_sync import ( sync_key_access_group_membership, @@ -1216,6 +1220,37 @@ def _enforce_upperbound_key_params( ) +def _team_admin_requests_member_key( + data: GenerateKeyRequest, + team_table: LiteLLM_TeamTableCachedObj | None, + user_api_key_dict: UserAPIKeyAuth, +) -> bool: + """Whether the caller is a team admin asking for a team key owned by another member of that team.""" + return not ( + user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value + or team_table is None + or data.user_id is None + or data.user_id == user_api_key_dict.user_id + or _get_user_in_team(team_table=team_table, user_id=data.user_id) is None + or not is_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team_table) + ) + + +def _member_key_request(data: GenerateKeyRequest, acting_as_team_admin: bool) -> GenerateKeyRequest: + if not acting_as_team_admin: + return data + return data.model_copy(update={"key_type": TEAM_ADMIN_MEMBER_KEY_TYPE}) + + +async def _stored_user_role(prisma_client: PrismaClient | None, user_id: str | None) -> str | None: + if prisma_client is None or user_id is None: + return None + user_row: Final = await _prisma_table(UserRepository(prisma_client, use_writer=True)).find_unique( + where={"user_id": user_id} + ) + return user_row.user_role if user_row is not None else None + + async def _common_key_generation_helper( data: GenerateKeyRequest, user_api_key_dict: UserAPIKeyAuth, @@ -1230,9 +1265,28 @@ async def _common_key_generation_helper( prisma_client, ) + acting_as_team_admin: Final = _team_admin_requests_member_key( + data=data, + team_table=team_table, + user_api_key_dict=user_api_key_dict, + ) + if acting_as_team_admin: + general_settings: Final = _general_settings() + team_admin_member_key_creation_or_raise( + team_admin_member_key_creation_verdict( + general_settings, + key_type=data.key_type, + owner_role=( + await _stored_user_role(prisma_client, data.user_id) + if team_admin_may_create_member_keys(general_settings) + else None + ), + ) + ) common_key_access_checks( user_api_key_dict=user_api_key_dict, data=data, + user_id=user_api_key_dict.user_id if acting_as_team_admin else None, llm_router=llm_router, premium_user=premium_user, ) @@ -1440,7 +1494,7 @@ async def _common_key_generation_helper( data_json = data.model_dump(exclude_unset=True, exclude_none=True) - data_json = handle_key_type(data, data_json) + data_json = handle_key_type(_member_key_request(data, acting_as_team_admin), data_json) # Re-check allowed_routes after handle_key_type, since key_type can derive # an elevated bucket (e.g. ["management_routes"]) that wasn't present in @@ -2114,7 +2168,7 @@ async def generate_key_fn( - key_alias: Optional[str] - User defined key alias - key: Optional[str] - User defined key value. Must start with 'sk-' and be at least 16 characters long. If not set, a 16-digit unique sk-key is created for you. - team_id: Optional[str] - The team id of the key - - user_id: Optional[str] - The user id of the key + - user_id: Optional[str] - The user id of the key. Non-admin callers may only pass their own id, except a team admin creating a key in a team they administer for another member of that team, when a proxy admin has added `create_member_keys` to `team_admin_editable_team_fields`. That member must have the `internal_user` or `internal_user_viewer` role, and the key is always an `llm_api` key. - agent_id: Optional[str] - The agent id associated with the key. - organization_id: Optional[str] - The organization id of the key. If not set, and team_id is set, the organization id will be the same as the team id. If conflict, an error will be raised. - project_id: Optional[str] - The project id of the key. When set, models and max_budget are validated against the project's limits. diff --git a/litellm/proxy/management_endpoints/team_admin_field_permissions.py b/litellm/proxy/management_endpoints/team_admin_field_permissions.py index dfae1b623f1..54e303f69b9 100644 --- a/litellm/proxy/management_endpoints/team_admin_field_permissions.py +++ b/litellm/proxy/management_endpoints/team_admin_field_permissions.py @@ -1,8 +1,11 @@ """Proxy-wide allow-list of what a team admin may do on the teams they administer: team-settings fields on /team/update, the ``raise_max_budget`` permission that lets a team admin grow the team's ``max_budget`` (never above the organization's budget, never removing the cap), the ``projects`` permission for /project/new and -/project/update, and the ``member_key_budgets`` permission for budget fields on other members' keys via -/key/update.""" +/project/update, the ``member_key_budgets`` permission for budget fields on other members' keys via +/key/update, and the ``create_member_keys`` permission for creating team keys owned by other members via +/key/generate. A key created that way authenticates as its owner, so the owner must be an ordinary internal +user and the key is confined to LLM API routes: the team admin who holds its secret gains no management +authority through it.""" from collections.abc import Mapping from dataclasses import dataclass @@ -19,6 +22,8 @@ from litellm.models.verification_token import LiteLLM_VerificationToken from litellm.proxy._types import ( LiteLLM_ManagementEndpoint_MetadataFields, LiteLLM_ManagementEndpoint_MetadataFields_Premium, + LiteLLMKeyType, + LitellmUserRoles, UpdateKeyRequest, UpdateTeamRequest, ) @@ -30,10 +35,16 @@ SUPPORTED_TEAM_ADMIN_EDITABLE_TEAM_FIELDS: Final[frozenset[str]] = frozenset({"t TEAM_ADMIN_RAISE_MAX_BUDGET_PERMISSION: Final = "raise_max_budget" TEAM_ADMIN_PROJECTS_PERMISSION: Final = "projects" TEAM_ADMIN_MEMBER_KEY_BUDGETS_PERMISSION: Final = "member_key_budgets" +TEAM_ADMIN_CREATE_MEMBER_KEYS_PERMISSION: Final = "create_member_keys" +TEAM_ADMIN_MEMBER_KEY_OWNER_ROLES: Final[frozenset[str]] = frozenset( + {LitellmUserRoles.INTERNAL_USER.value, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value} +) +TEAM_ADMIN_MEMBER_KEY_TYPE: Final = LiteLLMKeyType.LLM_API SUPPORTED_TEAM_ADMIN_PERMISSIONS: Final[frozenset[str]] = SUPPORTED_TEAM_ADMIN_EDITABLE_TEAM_FIELDS | { TEAM_ADMIN_RAISE_MAX_BUDGET_PERMISSION, TEAM_ADMIN_PROJECTS_PERMISSION, TEAM_ADMIN_MEMBER_KEY_BUDGETS_PERMISSION, + TEAM_ADMIN_CREATE_MEMBER_KEYS_PERMISSION, } # spend is deliberately excluded: the stored row lags the live cross-pod counter, so a value-diff gate @@ -110,6 +121,91 @@ def team_admin_may_edit_member_key_budgets(general_settings: Mapping[str, object ) +def team_admin_may_create_member_keys(general_settings: Mapping[str, object]) -> bool: + return TEAM_ADMIN_CREATE_MEMBER_KEYS_PERMISSION in resolve_team_admin_editable_fields( + general_settings, frozenset({TEAM_ADMIN_CREATE_MEMBER_KEYS_PERMISSION}) + ) + + +@dataclass(frozen=True, slots=True) +class TeamAdminMemberKeyCreationAllowed: + kind: Literal["allowed"] = "allowed" + + +@dataclass(frozen=True, slots=True) +class TeamAdminMemberKeyCreationDisabled: + kind: Literal["disabled"] = "disabled" + + +@dataclass(frozen=True, slots=True) +class TeamAdminMemberKeyOwnerNotPermitted: + owner_role: str | None + kind: Literal["owner_not_permitted"] = "owner_not_permitted" + + +@dataclass(frozen=True, slots=True) +class TeamAdminMemberKeyTypeNotPermitted: + key_type: LiteLLMKeyType + kind: Literal["key_type_not_permitted"] = "key_type_not_permitted" + + +TeamAdminMemberKeyCreationVerdict: TypeAlias = ( + TeamAdminMemberKeyCreationAllowed + | TeamAdminMemberKeyCreationDisabled + | TeamAdminMemberKeyOwnerNotPermitted + | TeamAdminMemberKeyTypeNotPermitted +) + + +def team_admin_member_key_creation_verdict( + general_settings: Mapping[str, object], + key_type: LiteLLMKeyType | None, + owner_role: str | None, +) -> TeamAdminMemberKeyCreationVerdict: + """``owner_role`` is the stored proxy role of the member the key is for, None when they have no user row.""" + if not team_admin_may_create_member_keys(general_settings): + return TeamAdminMemberKeyCreationDisabled() + if owner_role not in TEAM_ADMIN_MEMBER_KEY_OWNER_ROLES: + return TeamAdminMemberKeyOwnerNotPermitted(owner_role=owner_role) + if key_type not in (None, LiteLLMKeyType.DEFAULT, TEAM_ADMIN_MEMBER_KEY_TYPE): + return TeamAdminMemberKeyTypeNotPermitted(key_type=key_type) + return TeamAdminMemberKeyCreationAllowed() + + +def team_admin_member_key_creation_or_raise(verdict: TeamAdminMemberKeyCreationVerdict) -> None: + match verdict: + case TeamAdminMemberKeyCreationAllowed(): + return + case TeamAdminMemberKeyCreationDisabled(): + raise HTTPException( + status_code=403, + detail=( + "Team admins on this proxy cannot create keys for other team members. " + f"Ask a proxy admin to enable '{TEAM_ADMIN_CREATE_MEMBER_KEYS_PERMISSION}' " + f"under {_SETTINGS_LOCATION}." + ), + ) + case TeamAdminMemberKeyOwnerNotPermitted(owner_role=owner_role): + raise HTTPException( + status_code=403, + detail=( + "Team admins can only create keys for team members whose proxy role is one of " + f"{sorted(TEAM_ADMIN_MEMBER_KEY_OWNER_ROLES)}, not '{owner_role}'. " + "Ask a proxy admin to create this key." + ), + ) + case TeamAdminMemberKeyTypeNotPermitted(key_type=key_type): + raise HTTPException( + status_code=403, + detail=( + "Keys a team admin creates for another team member are always " + f"'{TEAM_ADMIN_MEMBER_KEY_TYPE.value}' keys, not '{key_type.value}'." + ), + ) + case _: + assert_never(verdict) + + def team_admin_may_raise_max_budget(general_settings: Mapping[str, object]) -> bool: """``raise_max_budget`` only takes effect alongside ``max_budget``: a grant to raise a field the team admin may not edit at all is treated as not granted.""" diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index e73507f8f4e..cbfd249eb82 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -365,6 +365,8 @@ class UISettings(LiteLLMBaseModel): "removes the cap. " "Include 'projects' to let team admins create and update projects for those teams. " "Include 'member_key_budgets' to let team admins update budget fields on keys owned by other members of those teams. " + "Include 'create_member_keys' to let team admins create team keys owned by other members of those teams. " + "Those keys can only call LLM API routes, and the member must be an internal user. " "Empty means team admins cannot edit team settings or manage projects at all. " "Proxy admins and org admins are not affected." ), diff --git a/tests/integration/authorization/test_team_admin_gate.py b/tests/integration/authorization/test_team_admin_gate.py index c3172e27c35..7b4fa52c4e4 100644 --- a/tests/integration/authorization/test_team_admin_gate.py +++ b/tests/integration/authorization/test_team_admin_gate.py @@ -76,6 +76,11 @@ class TeamScenario: def member(self) -> str: return self.scenario.member(self.team_id) + def proxy_admin_member(self) -> str: + user_id: Final = self.scenario.user(user_role="proxy_admin") + self.gateway.post("/team/member_add", {"team_id": self.team_id, "member": {"role": "user", "user_id": user_id}}) + return user_id + def member_key(self) -> str: created: Final = self.gateway.post("/key/generate", {"user_id": self.member(), "team_id": self.team_id}) return string_value(created["key"]) @@ -252,6 +257,18 @@ ROUTES: Final[tuple[Route, ...]] = ( Route("key_generate_team", lambda s: Call("POST", "/key/generate", {"team_id": s.team_id}), team_admin=200, others=400, member=401, cleanup=_delete_key), + Route("key_generate_for_member", + lambda s: Call("POST", "/key/generate", {"team_id": s.team_id, "user_id": s.member()}), + team_admin=403, others=400, member=401, cleanup=_delete_key), + Route("key_generate_for_member_permitted", + lambda s: Call("POST", "/key/generate", {"team_id": s.team_id, "user_id": s.member()}), + team_admin=200, others=400, member=401, permission=("create_member_keys",), cleanup=_delete_key), + Route("key_generate_for_proxy_admin_member_permitted", + lambda s: Call("POST", "/key/generate", {"team_id": s.team_id, "user_id": s.proxy_admin_member()}), + team_admin=403, others=400, member=401, permission=("create_member_keys",), cleanup=_delete_key), + Route("key_generate_management_key_for_member_permitted", + lambda s: Call("POST", "/key/generate", {"team_id": s.team_id, "user_id": s.member(), "key_type": "management"}), + team_admin=403, others=400, member=401, permission=("create_member_keys",), cleanup=_delete_key), Route("service_account_generate", lambda s: Call("POST", "/key/service-account/generate", {"team_id": s.team_id, "key_alias": f"matrix-{uuid.uuid4().hex}"}), team_admin=200, others=400, member=401, cleanup=_delete_key), @@ -452,6 +469,29 @@ def test_status_code(shared: TeamScenario, org_team: TeamScenario, route: Route, route.cleanup(s, object_value(response.json())) +def test_member_key_created_by_a_team_admin_reaches_llm_routes_only(shared: TeamScenario) -> None: + """The team admin holds the secret of a key that authenticates as the member, so it must not carry the + member's management access.""" + with shared.gateway.scenario() as scenario: + s: Final = replace(shared, scenario=scenario) + scenario.cleanups.enter_context(team_admin_permissions(s.gateway, ("create_member_keys",))) + member: Final = s.member() + created: Final = s.gateway.post( + "/key/generate", {"team_id": s.team_id, "user_id": member}, key=s.keys["team_admin"] + ) + token: Final = string_value(created["key"]) + scenario.cleanups.callback(delete_key_if_present, s.gateway, token) + own_key: Final = scenario.key(user_id=member, team_id=s.team_id) + + assert read_rows( + 'SELECT user_id, team_id, allowed_routes FROM "LiteLLM_VerificationToken" WHERE token = %s', + (string_value(created["token"]),), + ) == [{"user_id": member, "team_id": s.team_id, "allowed_routes": ["llm_api_routes"]}] + s.gateway.chat(s.model(), key=token) + assert s.gateway.request("GET", "/user/info", key=own_key).status_code == 200 + assert s.gateway.request("GET", "/user/info", key=token).status_code == 403 + + ADMIN_ONLY_CALLERS: Final[tuple[tuple[str, Caller], ...]] = ( ("shared", "team_admin"), ("shared", "member"), diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index 71334e2321b..1af9def3afb 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -21,6 +21,7 @@ import inspect from litellm.models.project import LiteLLM_ProjectTable from litellm.proxy._types import ( + GenerateKeyResponse, GenerateKeyRequest, KeyManagementRoutes, NewUserRequest, @@ -61,6 +62,8 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( _personal_key_generation_check, _team_key_generation_check, _common_key_generation_helper, + _member_key_request, + _team_admin_requests_member_key, KeyProjectBindingError, _effective_key_after_update, _effective_key_for_generate, @@ -22336,6 +22339,165 @@ class TestTeamAdminMemberKeyBudgetUpdate: assert "member_key_budgets" not in str(exc.value.detail) +_MEMBER_KEY_TEAM = LiteLLM_TeamTableCachedObj( + team_id="team-1", + members_with_roles=[ + Member(user_id="team-admin-1", role="admin"), + Member(user_id="member-1", role="user"), + Member(user_id="member-2", role="user"), + ], +) + + +def test_team_admin_asking_for_a_teammates_team_key_is_a_member_key_request(): + assert _team_admin_requests_member_key( + data=GenerateKeyRequest(user_id="member-1", team_id="team-1"), + team_table=_MEMBER_KEY_TEAM, + user_api_key_dict=UserAPIKeyAuth(user_id="team-admin-1", user_role=LitellmUserRoles.INTERNAL_USER), + ) + + +@pytest.mark.parametrize( + "data, team_table, caller, caller_role", + [ + pytest.param( + GenerateKeyRequest(user_id="member-1", team_id="team-1"), + _MEMBER_KEY_TEAM, + "member-2", + LitellmUserRoles.INTERNAL_USER, + id="ordinary_member_for_another_member", + ), + pytest.param( + GenerateKeyRequest(user_id="outsider", team_id="team-1"), + _MEMBER_KEY_TEAM, + "team-admin-1", + LitellmUserRoles.INTERNAL_USER, + id="team_admin_for_a_user_outside_the_team", + ), + pytest.param( + GenerateKeyRequest(user_id="member-1"), + None, + "team-admin-1", + LitellmUserRoles.INTERNAL_USER, + id="personal_key_for_another_user", + ), + pytest.param( + GenerateKeyRequest(user_id="team-admin-1", team_id="team-1"), + _MEMBER_KEY_TEAM, + "team-admin-1", + LitellmUserRoles.INTERNAL_USER, + id="team_admin_for_themselves", + ), + pytest.param( + GenerateKeyRequest(team_id="team-1"), + _MEMBER_KEY_TEAM, + "team-admin-1", + LitellmUserRoles.INTERNAL_USER, + id="team_key_with_no_owner", + ), + pytest.param( + GenerateKeyRequest(user_id="member-1", team_id="team-1"), + _MEMBER_KEY_TEAM, + "team-admin-1", + LitellmUserRoles.PROXY_ADMIN, + id="proxy_admin_keeps_the_proxy_admin_path", + ), + ], +) +def test_other_key_requests_are_not_member_key_requests(data, team_table, caller, caller_role): + assert not _team_admin_requests_member_key( + data=data, + team_table=team_table, + user_api_key_dict=UserAPIKeyAuth(user_id=caller, user_role=caller_role), + ) + + +@pytest.mark.parametrize("requested", [None, LiteLLMKeyType.DEFAULT, LiteLLMKeyType.LLM_API]) +def test_a_member_key_is_always_saved_as_an_llm_api_key(requested): + from litellm.proxy.management_endpoints.key_management_endpoints import handle_key_type + + request = _member_key_request( + GenerateKeyRequest(user_id="member-1", team_id="team-1", key_type=requested), acting_as_team_admin=True + ) + + assert (request.user_id, request.team_id) == ("member-1", "team-1") + assert handle_key_type(request, {}) == {"key_type": "llm_api", "allowed_routes": ["llm_api_routes"]} + + +def test_other_key_requests_keep_the_key_type_they_asked_for(): + from litellm.proxy.management_endpoints.key_management_endpoints import handle_key_type + + request = _member_key_request(GenerateKeyRequest(key_type=LiteLLMKeyType.MANAGEMENT), acting_as_team_admin=False) + + assert handle_key_type(request, {}) == {"key_type": "management", "allowed_routes": ["management_routes"]} + + +async def _team_admin_generates_member_key( + find_owner: AsyncMock, + generate_key: AsyncMock, + editable_fields: tuple[str, ...] = ("create_member_keys",), +) -> GenerateKeyResponse: + prisma_client: Final = MagicMock() + prisma_client.writer_db.litellm_usertable.find_unique = find_owner + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma_client), + patch("litellm.proxy.proxy_server.llm_router", None), + patch("litellm.proxy.proxy_server.premium_user", False), + patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), + patch( + "litellm.proxy.proxy_server.general_settings", + {"team_admin_editable_team_fields": list(editable_fields)}, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.generate_key_helper_fn", + generate_key, + ), + ): + return await _common_key_generation_helper( + data=GenerateKeyRequest(user_id="member-1", team_id="team-1"), + user_api_key_dict=UserAPIKeyAuth(user_id="team-admin-1", user_role=LitellmUserRoles.INTERNAL_USER), + litellm_changed_by=None, + team_table=_MEMBER_KEY_TEAM, + ) + + +@pytest.mark.asyncio +async def test_team_admin_creates_an_llm_api_key_owned_by_an_internal_user_teammate(): + generate_key: Final = AsyncMock(return_value={"key": "sk-member", "user_id": "member-1", "team_id": "team-1"}) + + await _team_admin_generates_member_key( + AsyncMock(return_value=MagicMock(user_role=LitellmUserRoles.INTERNAL_USER.value)), generate_key + ) + + saved: Final = generate_key.await_args.kwargs + assert (saved["user_id"], saved["team_id"]) == ("member-1", "team-1") + assert (saved["key_type"], saved["allowed_routes"]) == ("llm_api", ["llm_api_routes"]) + + +@pytest.mark.asyncio +async def test_team_admin_cannot_create_a_key_owned_by_a_proxy_admin_teammate(): + generate_key: Final = AsyncMock() + + with pytest.raises(HTTPException) as exc: + await _team_admin_generates_member_key( + AsyncMock(return_value=MagicMock(user_role=LitellmUserRoles.PROXY_ADMIN.value)), generate_key + ) + + assert exc.value.status_code == 403 + generate_key.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_member_key_request_with_the_setting_off_is_refused_without_reading_the_owner(): + find_owner: Final = AsyncMock(side_effect=ConnectionError("db down")) + + with pytest.raises(HTTPException) as exc: + await _team_admin_generates_member_key(find_owner, AsyncMock(), editable_fields=()) + + assert exc.value.status_code == 403 + find_owner.assert_not_awaited() + + @pytest.mark.asyncio async def test_rotate_master_key_reencrypts_guardrail_params(monkeypatch): import json diff --git a/tests/unit/proxy/management_endpoints/test_team_admin_field_permissions.py b/tests/unit/proxy/management_endpoints/test_team_admin_field_permissions.py index a593431523b..730f4640da8 100644 --- a/tests/unit/proxy/management_endpoints/test_team_admin_field_permissions.py +++ b/tests/unit/proxy/management_endpoints/test_team_admin_field_permissions.py @@ -5,7 +5,13 @@ from fastapi import HTTPException from litellm.models.team import BudgetLimitEntry from litellm.models.verification_token import LiteLLM_VerificationToken -from litellm.proxy._types import LiteLLM_ModelTable, LiteLLM_TeamTable, UpdateKeyRequest, UpdateTeamRequest +from litellm.proxy._types import ( + LiteLLM_ModelTable, + LiteLLM_TeamTable, + LiteLLMKeyType, + UpdateKeyRequest, + UpdateTeamRequest, +) from litellm.proxy.management_endpoints.team_admin_field_permissions import ( TeamAdminEditAllowed, TeamAdminEditingDisabled, @@ -18,9 +24,16 @@ from litellm.proxy.management_endpoints.team_admin_field_permissions import ( team_admin_edit_verdict, team_admin_key_edit_verdict, team_admin_key_request_or_raise, + team_admin_may_create_member_keys, team_admin_may_edit_member_key_budgets, team_admin_may_manage_projects, team_admin_may_raise_max_budget, + team_admin_member_key_creation_or_raise, + team_admin_member_key_creation_verdict, + TeamAdminMemberKeyCreationAllowed, + TeamAdminMemberKeyCreationDisabled, + TeamAdminMemberKeyOwnerNotPermitted, + TeamAdminMemberKeyTypeNotPermitted, team_admin_request_or_raise, ) @@ -218,6 +231,65 @@ class TestTeamAdminMayEditMemberKeyBudgets: assert team_admin_may_edit_member_key_budgets({"team_admin_editable_team_fields": raw}) is False +class TestTeamAdminMayCreateMemberKeys: + def test_missing_setting_denies(self): + assert team_admin_may_create_member_keys({}) is False + + def test_other_permissions_do_not_grant(self): + configured = {"team_admin_editable_team_fields": ["tpm_limit", "projects", "member_key_budgets"]} + assert team_admin_may_create_member_keys(configured) is False + + def test_create_member_keys_entry_grants(self): + configured = {"team_admin_editable_team_fields": ["create_member_keys"]} + assert team_admin_may_create_member_keys(configured) is True + + @pytest.mark.parametrize("raw", ["create_member_keys", 7, [1, 2]]) + def test_malformed_setting_denies(self, raw): + assert team_admin_may_create_member_keys({"team_admin_editable_team_fields": raw}) is False + + +_MEMBER_KEYS_ON = {"team_admin_editable_team_fields": ["create_member_keys"]} + + +class TestTeamAdminMemberKeyCreationVerdict: + @pytest.mark.parametrize("owner_role", ["internal_user", "internal_user_viewer"]) + @pytest.mark.parametrize("key_type", [None, LiteLLMKeyType.DEFAULT, LiteLLMKeyType.LLM_API]) + def test_an_ordinary_member_and_an_llm_key_are_allowed(self, owner_role, key_type): + verdict = team_admin_member_key_creation_verdict(_MEMBER_KEYS_ON, key_type=key_type, owner_role=owner_role) + assert verdict == TeamAdminMemberKeyCreationAllowed() + assert team_admin_member_key_creation_or_raise(verdict) is None + + def test_disabled_is_a_403_naming_the_permission_and_where_to_enable_it(self): + verdict = team_admin_member_key_creation_verdict( + {"team_admin_editable_team_fields": ["member_key_budgets"]}, key_type=None, owner_role="internal_user" + ) + assert verdict == TeamAdminMemberKeyCreationDisabled() + with pytest.raises(HTTPException) as exc: + team_admin_member_key_creation_or_raise(verdict) + assert exc.value.status_code == 403 + assert "'create_member_keys'" in exc.value.detail + assert "Settings > UI > Team admin editable fields" in exc.value.detail + + @pytest.mark.parametrize("owner_role", ["proxy_admin", "proxy_admin_viewer", "org_admin", "team", "customer", None]) + def test_an_owner_who_is_not_an_ordinary_internal_user_is_refused(self, owner_role): + """A key authenticates with its owner's role, so a privileged owner would hand that role to the team admin.""" + verdict = team_admin_member_key_creation_verdict(_MEMBER_KEYS_ON, key_type=None, owner_role=owner_role) + assert verdict == TeamAdminMemberKeyOwnerNotPermitted(owner_role=owner_role) + with pytest.raises(HTTPException) as exc: + team_admin_member_key_creation_or_raise(verdict) + assert exc.value.status_code == 403 + assert f"'{owner_role}'" in exc.value.detail + + @pytest.mark.parametrize("key_type", [LiteLLMKeyType.MANAGEMENT, LiteLLMKeyType.READ_ONLY]) + def test_a_key_type_that_reaches_beyond_llm_routes_is_refused(self, key_type): + verdict = team_admin_member_key_creation_verdict(_MEMBER_KEYS_ON, key_type=key_type, owner_role="internal_user") + assert verdict == TeamAdminMemberKeyTypeNotPermitted(key_type=key_type) + with pytest.raises(HTTPException) as exc: + team_admin_member_key_creation_or_raise(verdict) + assert exc.value.status_code == 403 + assert f"'{key_type.value}'" in exc.value.detail + + class TestTeamAdminMayRaiseMaxBudget: def test_missing_setting_denies(self): assert team_admin_may_raise_max_budget({}) is False diff --git a/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index 62a57af2108..6705e5ac8a5 100644 --- a/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -4022,6 +4022,28 @@ class TestTeamAdminEditableTeamFieldsSetting: assert stored["team_admin_editable_team_fields"] == ["member_key_budgets"] assert team_admin_may_edit_member_key_budgets(general_settings) is True + def test_patch_accepts_the_create_member_keys_permission(self, monkeypatch): + from litellm.proxy.management_endpoints.team_admin_field_permissions import ( + team_admin_may_create_member_keys, + ) + + mock_prisma = self._as_proxy_admin(monkeypatch) + general_settings: dict = {} + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", general_settings) + assert team_admin_may_create_member_keys(general_settings) is False + + try: + response = client.patch( + "/update/ui_settings", json={"team_admin_editable_team_fields": ["create_member_keys"]} + ) + finally: + app.dependency_overrides.clear() + + assert response.status_code == 200 + stored = json.loads(mock_prisma.db.litellm_uisettings.upsert.call_args.kwargs["data"]["create"]["ui_settings"]) + assert stored["team_admin_editable_team_fields"] == ["create_member_keys"] + assert team_admin_may_create_member_keys(general_settings) is True + def test_patch_with_an_empty_list_turns_team_admin_editing_off_again(self, monkeypatch): mock_prisma = self._as_proxy_admin(monkeypatch) general_settings: dict = {"team_admin_editable_team_fields": ["tpm_limit"]} diff --git a/ui/litellm-dashboard/src/components/organisms/createKeyOwnerAccess.test.ts b/ui/litellm-dashboard/src/components/organisms/createKeyOwnerAccess.test.ts new file mode 100644 index 00000000000..204a2a8fa86 --- /dev/null +++ b/ui/litellm-dashboard/src/components/organisms/createKeyOwnerAccess.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it } from "vitest"; + +import type { Member } from "../networking"; +import { + administersTeam, + anotherUserScope, + effectiveKeyOwner, + isTeamMemberOwner, + teamMemberOwnerOptions, + teamMemberOwnerPlaceholder, + type TeamMembers, +} from "./createKeyOwnerAccess"; + +const VIEWER = "viewer"; + +const member = (user_id: string | null, role: string, user_email: string | null = null): Member => ({ + user_id, + role, + user_email, +}); + +const administered: TeamMembers & { team_id: string } = { + team_id: "administered", + members_with_roles: [member(VIEWER, "admin"), member("alice", "user", "alice@example.com"), member("bob", "user")], +}; + +const joined: TeamMembers & { team_id: string } = { + team_id: "joined", + members_with_roles: [member(VIEWER, "user"), member("carol", "admin", "carol@example.com")], +}; + +const foreign: TeamMembers & { team_id: string } = { + team_id: "foreign", + members_with_roles: [member("dave", "admin")], +}; + +const viewer = { + userRole: "Internal User", + userId: VIEWER, + teamAdminEditableFields: ["create_member_keys"], + teams: [joined, administered], +}; + +describe("anotherUserScope", () => { + it("lets a team admin pick among team members once the proxy enables create_member_keys", () => { + expect(anotherUserScope(viewer)).toBe("team_members"); + }); + + it.each([ + ["nothing is enabled", []], + ["only other team admin permissions are enabled", ["max_budget", "member_key_budgets", "projects"]], + ])("hides Another User from a team admin when %s", (_label, teamAdminEditableFields) => { + expect(anotherUserScope({ ...viewer, teamAdminEditableFields })).toBe("none"); + }); + + it.each([ + ["only a plain member of their teams", [joined]], + ["an admin of no listed team", [foreign]], + ["in no team", []], + ["still waiting on the team list", null], + ])("hides Another User with the permission on from a user who is %s", (_label, teams) => { + expect(anotherUserScope({ ...viewer, teams })).toBe("none"); + }); + + it("hides Another User when the session carries no user id", () => { + expect(anotherUserScope({ ...viewer, userId: null })).toBe("none"); + }); + + it.each([ + ["with the permission off and no teams", { teamAdminEditableFields: [], teams: [] }], + ["with the permission on and an administered team", {}], + ["before the team list has loaded", { teams: null }], + ])("lets a proxy admin pick any user %s", (_label, overrides) => { + expect(anotherUserScope({ ...viewer, userRole: "Admin", ...overrides })).toBe("any_user"); + }); +}); + +describe("effectiveKeyOwner", () => { + it.each(["any_user", "team_members"] as const)("keeps another_user for a viewer scoped to %s", (scope) => { + expect(effectiveKeyOwner("another_user", scope)).toBe("another_user"); + }); + + it("falls back to you when another_user is requested by a viewer who may not pick one", () => { + expect(effectiveKeyOwner("another_user", "none")).toBe("you"); + }); + + it.each(["you", "service_account", "agent"])( + "leaves %s alone for a viewer who may not pick another user", + (owner) => { + expect(effectiveKeyOwner(owner, "none")).toBe(owner); + }, + ); +}); + +describe("administersTeam", () => { + it("narrows a team list to the teams the viewer administers", () => { + const teams = [joined, administered, foreign]; + + expect(teams.filter((team) => administersTeam(team, VIEWER)).map((team) => team.team_id)).toStrictEqual([ + "administered", + ]); + }); + + it.each([ + ["no team", null], + ["a team with no member list", {}], + ["a team whose member list is null", { members_with_roles: null }], + ])("is false for %s", (_label, team) => { + expect(administersTeam(team, VIEWER)).toBe(false); + }); +}); + +describe("teamMemberOwnerOptions", () => { + it("lists the selected team's other members, labelled by email when the team has one", () => { + expect(teamMemberOwnerOptions(administered, VIEWER)).toStrictEqual([ + { value: "alice", label: "alice@example.com (alice)" }, + { value: "bob", label: "bob" }, + ]); + }); + + it("skips a member row that has no user id", () => { + const team = { + members_with_roles: [member(VIEWER, "admin"), member(null, "user", "pending@example.com"), member("bob", "user")], + }; + + expect(teamMemberOwnerOptions(team, VIEWER)).toStrictEqual([{ value: "bob", label: "bob" }]); + }); + + it.each([ + ["no team is selected", null], + ["the selected team is one the viewer only belongs to", joined], + ["the selected team is one the viewer is not in", foreign], + ])("offers nobody when %s", (_label, team) => { + expect(teamMemberOwnerOptions(team, VIEWER)).toStrictEqual([]); + }); +}); + +describe("teamMemberOwnerPlaceholder", () => { + const soloTeam = { members_with_roles: [member(VIEWER, "admin")] }; + + it.each([ + ["asks for a team before one is selected", null, "Select a team first"], + ["says so when the viewer is the team's only member", soloTeam, "This team has no other members"], + ["asks for a member once the team has someone to pick", administered, "Select a member of the team"], + ])("%s", (_label, team, expected) => { + expect(teamMemberOwnerPlaceholder(teamMemberOwnerOptions(team, VIEWER), team)).toBe(expected); + }); +}); + +describe("isTeamMemberOwner", () => { + it("keeps an owner who belongs to the selected team", () => { + expect(isTeamMemberOwner("alice", administered, VIEWER)).toBe(true); + }); + + it.each([ + ["is in another team only", "carol", administered], + ["was picked before the team was cleared", "alice", null], + ["belongs to a team the viewer does not administer", "carol", joined], + ["is the viewer, who already has the You option", VIEWER, administered], + ["is not set", null, administered], + ])("rejects an owner who %s", (_label, ownerId, team) => { + expect(isTeamMemberOwner(ownerId, team, VIEWER)).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/components/organisms/createKeyOwnerAccess.ts b/ui/litellm-dashboard/src/components/organisms/createKeyOwnerAccess.ts new file mode 100644 index 00000000000..dcbebab7c25 --- /dev/null +++ b/ui/litellm-dashboard/src/components/organisms/createKeyOwnerAccess.ts @@ -0,0 +1,59 @@ +import type { SearchSelectOption } from "@/components/shared/SearchSelect"; +import { TEAM_ADMIN_CREATE_MEMBER_KEYS_PERMISSION } from "@/components/team/teamAdminEditAccess"; +import { isProxyAdminRole, isUserTeamAdminForSingleTeam } from "@/utils/roles"; +import type { Member } from "../networking"; + +export type AnotherUserScope = "any_user" | "team_members" | "none"; + +export interface TeamMembers { + readonly members_with_roles?: Member[] | null; +} + +export interface KeyOwnerViewer { + readonly userRole: string | null; + readonly userId: string | null; + readonly teamAdminEditableFields: readonly string[]; + readonly teams: readonly TeamMembers[] | null; +} + +export const administersTeam = (team: TeamMembers | null, userId: string | null): boolean => + userId !== null && isUserTeamAdminForSingleTeam(team?.members_with_roles ?? null, userId); + +export const anotherUserScope = ({ + userRole, + userId, + teamAdminEditableFields, + teams, +}: KeyOwnerViewer): AnotherUserScope => { + if (isProxyAdminRole(userRole ?? "")) return "any_user"; + if (!teamAdminEditableFields.includes(TEAM_ADMIN_CREATE_MEMBER_KEYS_PERMISSION)) return "none"; + return (teams ?? []).some((team) => administersTeam(team, userId)) ? "team_members" : "none"; +}; + +export const effectiveKeyOwner = (requestedOwner: string, scope: AnotherUserScope): string => + requestedOwner === "another_user" && scope === "none" ? "you" : requestedOwner; + +export const teamMemberOwnerOptions = (team: TeamMembers | null, viewerId: string | null): SearchSelectOption[] => { + if (!administersTeam(team, viewerId)) return []; + return (team?.members_with_roles ?? []).flatMap((member) => + member.user_id && member.user_id !== viewerId + ? [ + { + value: member.user_id, + label: member.user_email ? `${member.user_email} (${member.user_id})` : member.user_id, + }, + ] + : [], + ); +}; + +export const teamMemberOwnerPlaceholder = ( + options: readonly SearchSelectOption[], + team: TeamMembers | null, +): string => { + if (options.length > 0) return "Select a member of the team"; + return team ? "This team has no other members" : "Select a team first"; +}; + +export const isTeamMemberOwner = (ownerId: unknown, team: TeamMembers | null, viewerId: string | null): boolean => + teamMemberOwnerOptions(team, viewerId).some((option) => option.value === ownerId); diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx index 63ed7e14b67..0fbe043c0d1 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx @@ -649,6 +649,59 @@ describe("CreateKey", () => { expect(screen.queryByRole("radio", { name: "Another User" })).not.toBeInTheDocument(); }); + describe("for a team admin", () => { + const administeredTeam = { + team_id: "team-admin-of", + team_alias: "Administered Team", + models: [], + members_with_roles: [ + { user_id: "test-user-id", role: "admin" }, + { user_id: "member-1", role: "user", user_email: "member@example.com" }, + ], + }; + const joinedTeam = { + team_id: "team-member-of", + team_alias: "Joined Team", + models: [], + members_with_roles: [ + { user_id: "test-user-id", role: "user" }, + { user_id: "outsider-1", role: "admin", user_email: "outsider@example.com" }, + ], + }; + + beforeEach(() => { + state.authorized = { ...state.authorized, userRole: "Internal User" }; + state.teams = [joinedTeam, administeredTeam]; + }); + + it("hides the Another User option until the proxy enables create_member_keys", async () => { + state.uiSettings = { team_admin_editable_team_fields: ["max_budget"] }; + await openModal({ teams: state.teams as unknown as Team[] }); + + expect(screen.queryByRole("radio", { name: "Another User" })).not.toBeInTheDocument(); + }); + + it("creates a key for a member of an administered team without the proxy-wide user search", async () => { + state.uiSettings = { team_admin_editable_team_fields: ["create_member_keys"] }; + await openModal({ teams: state.teams as unknown as Team[] }); + await nameTheKey(); + await userEvent.click(screen.getByRole("radio", { name: "Another User" })); + + await userEvent.click(await screen.findByLabelText(/^Team/)); + expect(await screen.findByRole("option", { name: /Administered Team/ })).toBeInTheDocument(); + expect(screen.queryByRole("option", { name: /Joined Team/ })).not.toBeInTheDocument(); + await userEvent.click(screen.getByRole("option", { name: /Administered Team/ })); + + await userEvent.click(await screen.findByLabelText(/^User ID/)); + expect(await screen.findAllByRole("option")).toHaveLength(1); + await userEvent.click(screen.getByRole("option", { name: "member@example.com (member-1)" })); + await submit(); + + expect(await createdPayload()).toMatchObject({ user_id: "member-1", team_id: "team-admin-of" }); + expect(userFilterUICall).not.toHaveBeenCalled(); + }); + }); + it("routes a service account through the service account endpoint and stamps the alias into metadata", async () => { state.teams = [{ team_id: "team-1", team_alias: "Team One", models: [] }]; await openModal({ teams: state.teams as unknown as Team[] }); diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 45245ff95b3..eb8c96a675a 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -6,6 +6,7 @@ import { useTags } from "@/app/(dashboard)/hooks/tags/useTags"; import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import useCan from "@/app/(dashboard)/hooks/useCan"; +import { parseTeamAdminEditableFields } from "@/components/team/teamAdminEditAccess"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { useQueryClient } from "@tanstack/react-query"; import { Button } from "@/components/ui/button"; @@ -80,6 +81,14 @@ import { import CreatedKeyDisplay from "../shared/CreatedKeyDisplay"; import NumericalInput from "../shared/numerical_input"; import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; +import { + administersTeam, + anotherUserScope, + effectiveKeyOwner, + isTeamMemberOwner, + teamMemberOwnerOptions, + teamMemberOwnerPlaceholder, +} from "./createKeyOwnerAccess"; import { buildKeyCreatePayload, type KeyCreateInput } from "./createKeyPayload"; import { simplifyKeyGenerateError } from "./utils"; import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; @@ -248,7 +257,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp const [apiKey, setApiKey] = useState(null); const [userModels, setUserModels] = useState([]); const [modelsToPick, setModelsToPick] = useState([]); - const [keyOwner, setKeyOwner] = useState("you"); + const [requestedKeyOwner, setKeyOwner] = useState("you"); const [hasPrefilled, setHasPrefilled] = useState(false); const [pendingPrefillModels, setPendingPrefillModels] = useState(null); const [guardrailsList, setGuardrailsList] = useState([]); @@ -279,6 +288,13 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp const [agentsList, setAgentsList] = useState<{ agent_id: string; agent_name: string }[]>([]); const [selectedAgentId, setSelectedAgentId] = useState(null); const selectedModels: string[] = (useWatch({ control: form.control, name: "models" }) as string[] | undefined) ?? []; + const teamAdminEditableFields = parseTeamAdminEditableFields(uiSettingsData?.values); + const keyOwnerViewer = { userRole, userId: userID, teamAdminEditableFields, teams }; + const ownerScope = anotherUserScope(keyOwnerViewer); + const keyOwner = effectiveKeyOwner(requestedKeyOwner, ownerScope); + const ownerLimitedToTeamMembers = keyOwner === "another_user" && ownerScope === "team_members"; + const memberOwnerOptions = teamMemberOwnerOptions(selectedCreateKeyTeam, userID); + const ownerRequiresTeam = keyOwner === "service_account" || ownerLimitedToTeamMembers; const handleCancel = () => { setIsModalVisible(false); setApiKey(null); @@ -383,14 +399,8 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp // Apply prefill data if provided if (prefillData) { - // Set key owner (owned_by) - validate that "another_user" is only allowed for Admin if (prefillData.owned_by) { - if (prefillData.owned_by === "another_user" && userRole !== "Admin") { - // Ignore invalid owned_by for non-admin users, fall back to default - setKeyOwner("you"); - } else { - setKeyOwner(prefillData.owned_by); - } + setKeyOwner(prefillData.owned_by); } // Set team - find the team by ID and set it (only if team exists in user's teams) @@ -616,6 +626,23 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp } }; + const changeKeyOwner = (owner: string) => { + setKeyOwner(owner); + const limitsOwnerToTeamMembers = owner === "another_user" && ownerScope === "team_members"; + if (limitsOwnerToTeamMembers && selectedCreateKeyTeam && !administersTeam(selectedCreateKeyTeam, userID)) { + selectTeam(null); + form.setValue("team_id", null); + } + }; + + useEffect(() => { + if (!ownerLimitedToTeamMembers) return; + const owner = form.getValues("user_id"); + if (owner != null && !isTeamMemberOwner(owner, selectedCreateKeyTeam, userID)) { + form.setValue("user_id", null); + } + }, [ownerLimitedToTeamMembers, selectedCreateKeyTeam, userID, form]); + const changeProject = (write: FieldWrite) => (projectId: string | null) => { write(projectId); if (!projectId) { @@ -650,6 +677,68 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp } }; + const memberOwnerPlaceholder = teamMemberOwnerPlaceholder(memberOwnerOptions, selectedCreateKeyTeam); + const requiredTeamHelp = ownerLimitedToTeamMembers + ? "required, the key's owner must be a member of this team" + : "required"; + const teamFieldHelp = ownerRequiresTeam ? requiredTeamHelp : ""; + const ownerField = keyOwner === "another_user" && ( + + User ID{" "} + + + + + } + name="user_id" + className="mt-4" + required + rules={requiredRule( + keyOwner === "another_user", + `Please input the user ID of the user you are assigning the key to`, + )} + > + {(control) => + ownerLimitedToTeamMembers ? ( + + ) : ( +
+
+ + +
+
Search by email or user ID to find users
+
+ ) + } + + ); + return (
{userRole && rolesWithWriteAccess.includes(userRole) && ( @@ -679,7 +768,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp setKeyOwner(String(value))} + onValueChange={(value: unknown) => changeKeyOwner(String(value))} > - {userRole === "Admin" && ( + {ownerScope !== "none" && ( - {keyOwner === "another_user" && ( - - User ID{" "} - - - - - } - name="user_id" - className="mt-4" - required - rules={requiredRule( - keyOwner === "another_user", - `Please input the user ID of the user you are assigning the key to`, - )} - > - {(control) => ( -
-
- - -
-
Search by email or user ID to find users
-
- )} - - )} + {!ownerLimitedToTeamMembers && ownerField} {keyOwner === "agent" && (
@@ -803,9 +849,14 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp } name="team_id" className="mt-4" - required={keyOwner === "service_account"} - rules={requiredRule(keyOwner === "service_account", "Please select a team for the service account")} - help={keyOwner === "service_account" ? "required" : ""} + required={ownerRequiresTeam} + rules={requiredRule( + ownerRequiresTeam, + ownerLimitedToTeamMembers + ? "Please select the team the key's owner belongs to" + : "Please select a team for the service account", + )} + help={teamFieldHelp} > {(control) => ( = ({ team, teams, data, addKey, autoOp disabled={selectedProjectId !== null} organizationId={selectedOrganizationId} onTeamSelect={selectTeam} + filterTeam={ownerLimitedToTeamMembers ? (team) => administersTeam(team, userID) : undefined} /> )} + {ownerLimitedToTeamMembers && ownerField} {enableProjectsUI && ( { ["max_budget", "Max Budget (USD)"], ["raise_max_budget", "Raise the team's max budget"], ["projects", "Create and update projects"], + ["create_member_keys", "Create keys for team members"], ])("names %s the way the team settings form does", (field, label) => { expect(teamAdminFieldLabel(field)).toBe(label); }); diff --git a/ui/litellm-dashboard/src/components/team/teamAdminEditAccess.ts b/ui/litellm-dashboard/src/components/team/teamAdminEditAccess.ts index cd24d6bdff0..1fed045292b 100644 --- a/ui/litellm-dashboard/src/components/team/teamAdminEditAccess.ts +++ b/ui/litellm-dashboard/src/components/team/teamAdminEditAccess.ts @@ -5,6 +5,8 @@ export const TEAM_ADMIN_EDITABLE_TEAM_FIELDS_SETTING = "team_admin_editable_team export const TEAM_ADMIN_RAISE_MAX_BUDGET_PERMISSION = "raise_max_budget"; +export const TEAM_ADMIN_CREATE_MEMBER_KEYS_PERMISSION = "create_member_keys"; + export const TEAM_ADMIN_EDITING_DISABLED_TITLE = "Team admins cannot edit team settings on this proxy"; export const TEAM_ADMIN_EDITING_DISABLED_DESCRIPTION = "Ask a proxy admin to enable fields under Settings > UI > Team admin editable fields."; @@ -57,6 +59,7 @@ const TEAM_ADMIN_FIELD_LABELS: ReadonlyMap = new Map([ [TEAM_ADMIN_RAISE_MAX_BUDGET_PERMISSION, "Raise the team's max budget"], ["projects", "Create and update projects"], ["member_key_budgets", "Update budgets on team members' keys"], + [TEAM_ADMIN_CREATE_MEMBER_KEYS_PERMISSION, "Create keys for team members"], ]); export const teamAdminFieldLabel = (field: string): string => TEAM_ADMIN_FIELD_LABELS.get(field) ?? field; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 3b018db7f9b..98071b81e6f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -8347,7 +8347,7 @@ export interface paths { * - key_alias: Optional[str] - User defined key alias * - key: Optional[str] - User defined key value. Must start with 'sk-' and be at least 16 characters long. If not set, a 16-digit unique sk-key is created for you. * - team_id: Optional[str] - The team id of the key - * - user_id: Optional[str] - The user id of the key + * - user_id: Optional[str] - The user id of the key. Non-admin callers may only pass their own id, except a team admin creating a key in a team they administer for another member of that team, when a proxy admin has added `create_member_keys` to `team_admin_editable_team_fields`. That member must have the `internal_user` or `internal_user_viewer` role, and the key is always an `llm_api` key. * - agent_id: Optional[str] - The agent id associated with the key. * - organization_id: Optional[str] - The organization id of the key. If not set, and team_id is set, the organization id will be the same as the team id. If conflict, an error will be raised. * - project_id: Optional[str] - The project id of the key. When set, models and max_budget are validated against the project's limits. From 0f0c1bc80b64afa387dc06bd6c0761678e4c9447 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:29:20 +0000 Subject: [PATCH 03/51] test(integration): cover teams endpoint contracts (#44705) * test(integration): cover teams endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): address review on teams contract tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): drain shared upstream observations at fixture setup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): prove team block, member and bulk permission contracts at runtime Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): assert email admin adds and whole membership rows for team members Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): drive key update grant and revoke, Z-suffixed temp budget expiry, link LIT-9257 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): close review gaps in teams endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): harden team member endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_team_member_permissions.py | 331 ++++++- .../authorization/test_team_scoped_models.py | 149 ++- .../integration/management/test_team_block.py | 226 +++++ .../management/test_team_member_lifecycle.py | 925 +++++++++++++++++ .../spend/test_team_member_spend.py | 932 +++++++++++++++++- 5 files changed, 2557 insertions(+), 6 deletions(-) create mode 100644 tests/integration/management/test_team_block.py create mode 100644 tests/integration/management/test_team_member_lifecycle.py diff --git a/tests/integration/authorization/test_team_member_permissions.py b/tests/integration/authorization/test_team_member_permissions.py index 24ad98b486d..5d29aceb10c 100644 --- a/tests/integration/authorization/test_team_member_permissions.py +++ b/tests/integration/authorization/test_team_member_permissions.py @@ -1,15 +1,81 @@ +import json +from collections.abc import Iterator from dataclasses import dataclass from hashlib import sha256 from typing import Final import httpx -from integration._support.client import Gateway, Scenario, object_value, string_value -from integration._support.database import read_rows -from pydantic import JsonValue +import pytest +from integration._support.client import ( + Gateway, + Scenario, + delete_key_if_present, + object_value, + string_value, +) +from integration._support.database import read_rows, write_rows +from pydantic import BaseModel, JsonValue PERMISSION_ERROR: Final = "team_member_permission_error" +class _ObservedRequest(BaseModel): + body: dict[str, JsonValue] + + +class _Observations(BaseModel): + requests: list[_ObservedRequest] + + +class _TeamPermissions(BaseModel): + team_id: str + team_member_permissions: list[str] | None = None + + +class _PermissionsList(BaseModel): + team_id: str + team_member_permissions: list[str] + + +class _TeamPermissionRow(BaseModel): + team_id: str + team_member_permissions: list[str] | None = None + + +class _BulkPermissionsUpdate(BaseModel): + message: str + teams_updated: int + permissions_appended: list[str] | None = None + + +class _GeneratedKey(BaseModel): + key: str + + +class _Error(BaseModel): + type: str + + +class _ErrorResponse(BaseModel): + error: _Error + + +class _HTTPErrorDetail(BaseModel): + error: str + + +class _HTTPErrorResponse(BaseModel): + detail: _HTTPErrorDetail + + +@pytest.fixture +def upstream(gateway: Gateway) -> Iterator[httpx.Client]: + with httpx.Client(base_url=gateway.upstream_url, timeout=15, trust_env=False) as client: + response: Final = client.get("/__observations") + assert response.status_code == 200, response.text + yield client + + @dataclass(frozen=True, slots=True) class Member: team_id: str @@ -31,6 +97,15 @@ def _team_key_row(key: str) -> list[dict[str, JsonValue]]: ) +def _update_team_key(proxy: Gateway, member: Member, owner: str) -> httpx.Response: + return proxy.request( + "POST", + "/key/update", + {"key": member.team_key, "team_id": member.team_id, "metadata": {"owner": owner}}, + key=member.member_key, + ) + + def _team_key_count(team_id: str) -> int: return len(read_rows('SELECT token FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team_id,))) @@ -41,6 +116,256 @@ def _refused(response: httpx.Response, status: int, error_type: str | None = Non assert object_value(response.json()["error"])["type"] == error_type, response.text +def _observed_models(upstream: httpx.Client) -> tuple[str, ...]: + response: Final = upstream.get("/__observations") + assert response.status_code == 200, response.text + observations: Final = _Observations.model_validate_json(response.text) + return tuple(string_value(request.body["model"]) for request in observations.requests) + + +def _permissions(gateway: Gateway, team_id: str) -> _PermissionsList: + response: Final = gateway.request("GET", "/team/permissions_list", params={"team_id": team_id}) + assert response.status_code == 200, response.text + return _PermissionsList.model_validate_json(response.text) + + +def _team_permission_rows(team_ids: tuple[str, ...]) -> tuple[_TeamPermissionRow, ...]: + return tuple( + _TeamPermissionRow.model_validate(row) + for row in read_rows( + 'SELECT team_id, team_member_permissions FROM "LiteLLM_TeamTable" WHERE team_id = ANY(%s) ORDER BY team_id', + (list(team_ids),), + ) + ) + + +def _restore_team_permissions(team_id: str, permissions: list[str] | None) -> None: + if permissions is None: + write_rows( + 'UPDATE "LiteLLM_TeamTable" SET team_member_permissions = NULL WHERE team_id = %s', + (team_id,), + ) + return + write_rows( + 'UPDATE "LiteLLM_TeamTable" SET team_member_permissions = ARRAY(SELECT jsonb_array_elements_text(%s::jsonb)) WHERE team_id = %s', + (json.dumps(permissions), team_id), + ) + + +def test_permissions_update_persists_requested_member_permissions(gateway: Gateway, upstream: httpx.Client) -> None: + with gateway.scenario() as scenario: + team: Final = scenario.team() + requested: Final = ["/key/generate", "/key/info"] + updated: Final = gateway.request( + "POST", + "/team/permissions_update", + {"team_id": team, "team_member_permissions": requested}, + ) + assert updated.status_code == 200, updated.text + response: Final = _TeamPermissions.model_validate_json(updated.text) + assert response.team_id == team, updated.text + assert response.team_member_permissions == requested, updated.text + readback: Final = _permissions(gateway, team) + assert readback.team_member_permissions == requested, repr(readback) + assert read_rows( + 'SELECT team_member_permissions FROM "LiteLLM_TeamTable" WHERE team_id = %s', + (team,), + ) == [{"team_member_permissions": requested}], updated.text + observed: Final = _observed_models(upstream) + assert observed == (), repr(observed) + + +def test_permissions_update_grants_and_revokes_key_generate_and_key_update_for_a_warmed_member_on_both_proxies( + gateway: Gateway, peer: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + team: Final = scenario.team() + member: Final = scenario.member(team) + member_key: Final = scenario.key(user_id=member) + team_key: Final = scenario.key(team_id=team, metadata={"owner": "team"}) + holder: Final = Member(team, team_key, member_key) + denied_gateway: Final = gateway.request("POST", "/key/generate", {"team_id": team}, key=member_key) + _refused(denied_gateway, 401, PERMISSION_ERROR) + denied_peer: Final = peer.request("POST", "/key/generate", {"team_id": team}, key=member_key) + _refused(denied_peer, 401, PERMISSION_ERROR) + _refused(_update_team_key(gateway, holder, "member before grant"), 401, PERMISSION_ERROR) + _refused(_update_team_key(peer, holder, "member before grant"), 401, PERMISSION_ERROR) + assert _team_key_row(team_key) == [{"team_id": team, "metadata": {"owner": "team"}}] + + granted_permissions: Final = ["/key/generate", "/key/info", "/key/update"] + granted: Final = gateway.request( + "POST", + "/team/permissions_update", + {"team_id": team, "team_member_permissions": granted_permissions}, + ) + assert granted.status_code == 200, granted.text + added: Final = _permissions(gateway, team) + assert added.team_member_permissions == granted_permissions, repr(added) + assert read_rows( + 'SELECT team_member_permissions FROM "LiteLLM_TeamTable" WHERE team_id = %s', + (team,), + ) == [{"team_member_permissions": granted_permissions}], granted.text + generated_gateway: Final = gateway.request("POST", "/key/generate", {"team_id": team}, key=member_key) + assert generated_gateway.status_code == 200, generated_gateway.text + gateway_key: Final = _GeneratedKey.model_validate_json(generated_gateway.text).key + scenario.cleanups.callback(delete_key_if_present, gateway, gateway_key) + generated_peer: Final = peer.request("POST", "/key/generate", {"team_id": team}, key=member_key) + assert generated_peer.status_code == 200, generated_peer.text + peer_key: Final = _GeneratedKey.model_validate_json(generated_peer.text).key + scenario.cleanups.callback(delete_key_if_present, gateway, peer_key) + updated_gateway: Final = _update_team_key(gateway, holder, "member via gateway") + assert updated_gateway.status_code == 200, updated_gateway.text + assert _team_key_row(team_key) == [{"team_id": team, "metadata": {"owner": "member via gateway"}}] + updated_peer: Final = _update_team_key(peer, holder, "member via peer") + assert updated_peer.status_code == 200, updated_peer.text + assert _team_key_row(team_key) == [{"team_id": team, "metadata": {"owner": "member via peer"}}] + + revoked_permissions: Final = ["/key/info"] + revoked: Final = gateway.request( + "POST", + "/team/permissions_update", + {"team_id": team, "team_member_permissions": revoked_permissions}, + ) + assert revoked.status_code == 200, revoked.text + revoked_readback: Final = _permissions(gateway, team) + assert revoked_readback.team_member_permissions == revoked_permissions, repr(revoked_readback) + assert read_rows( + 'SELECT team_member_permissions FROM "LiteLLM_TeamTable" WHERE team_id = %s', + (team,), + ) == [{"team_member_permissions": revoked_permissions}], revoked.text + _refused(_update_team_key(gateway, holder, "member after revoke"), 401, PERMISSION_ERROR) + _refused(_update_team_key(peer, holder, "member after revoke"), 401, PERMISSION_ERROR) + assert _team_key_row(team_key) == [{"team_id": team, "metadata": {"owner": "member via peer"}}] + revoked_gateway: Final = gateway.request("POST", "/key/generate", {"team_id": team}, key=member_key) + _refused(revoked_gateway, 401, PERMISSION_ERROR) + revoked_peer: Final = peer.request("POST", "/key/generate", {"team_id": team}, key=member_key) + _refused(revoked_peer, 401, PERMISSION_ERROR) + observed: Final = _observed_models(upstream) + assert observed == (), repr(observed) + + +def test_permissions_bulk_update_appends_only_to_the_listed_teams( + gateway: Gateway, peer: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + listed: Final = _member(scenario, ["/key/info"]) + team_two: Final = scenario.team(team_member_permissions=["/key/info", "/key/update"]) + team_three_permission: Final = "/team/daily/activity" + team_three: Final = _member(scenario, [team_three_permission]) + unlisted: Final = _member(scenario, ["/key/info"]) + team_ids: Final = (listed.team_id, team_two, team_three.team_id, unlisted.team_id) + for proxy in (gateway, peer): + for member in (listed, team_three, unlisted): + _refused(_update_team_key(proxy, member, "member before bulk"), 401, PERMISSION_ERROR) + before_non_admin: Final = _team_permission_rows(team_ids) + caller: Final = scenario.user(user_role="internal_user") + caller_key: Final = scenario.key(user_id=caller, allowed_routes=["/team/permissions_bulk_update"]) + denied: Final = gateway.request( + "POST", + "/team/permissions_bulk_update", + {"permissions": ["/key/update"], "team_ids": [listed.team_id, team_two]}, + key=caller_key, + ) + assert denied.status_code == 403, denied.text + assert _HTTPErrorResponse.model_validate_json(denied.text) == _HTTPErrorResponse( + detail=_HTTPErrorDetail(error="Only proxy admins can bulk-update team permissions") + ), denied.text + after_denied: Final = _team_permission_rows(team_ids) + assert after_denied == before_non_admin, repr(after_denied) + + selected: Final = gateway.request( + "POST", + "/team/permissions_bulk_update", + {"permissions": ["/key/update"], "team_ids": [listed.team_id, team_two, team_three.team_id]}, + ) + assert selected.status_code == 200, selected.text + selected_response: Final = _BulkPermissionsUpdate.model_validate_json(selected.text) + assert selected_response == _BulkPermissionsUpdate( + message="Team permissions updated successfully", + teams_updated=2, + permissions_appended=["/key/update"], + ), selected.text + selected_rows: Final = _team_permission_rows(team_ids) + assert tuple(sorted(selected_rows, key=lambda row: row.team_id)) == tuple( + sorted( + ( + _TeamPermissionRow(team_id=listed.team_id, team_member_permissions=["/key/info", "/key/update"]), + _TeamPermissionRow(team_id=team_two, team_member_permissions=["/key/info", "/key/update"]), + _TeamPermissionRow( + team_id=team_three.team_id, + team_member_permissions=["/key/update", team_three_permission], + ), + _TeamPermissionRow(team_id=unlisted.team_id, team_member_permissions=["/key/info"]), + ), + key=lambda row: row.team_id, + ) + ), repr(selected_rows) + listed_readback: Final = _permissions(gateway, listed.team_id) + assert listed_readback.team_member_permissions == ["/key/info", "/key/update"], repr(listed_readback) + team_three_readback: Final = _permissions(gateway, team_three.team_id) + assert team_three_readback.team_member_permissions == ["/key/update", team_three_permission], repr( + team_three_readback + ) + unlisted_readback: Final = _permissions(gateway, unlisted.team_id) + assert unlisted_readback.team_member_permissions == ["/key/info"], repr(unlisted_readback) + + gateway_update: Final = _update_team_key(gateway, listed, "member via gateway") + assert gateway_update.status_code == 200, gateway_update.text + assert _team_key_row(listed.team_key) == [ + {"team_id": listed.team_id, "metadata": {"owner": "member via gateway"}} + ], gateway_update.text + peer_update: Final = _update_team_key(peer, listed, "member via peer") + assert peer_update.status_code == 200, peer_update.text + assert _team_key_row(listed.team_key) == [ + {"team_id": listed.team_id, "metadata": {"owner": "member via peer"}} + ], peer_update.text + for proxy in (gateway, peer): + _refused(_update_team_key(proxy, unlisted, "unlisted member"), 401, PERMISSION_ERROR) + assert _team_key_row(unlisted.team_key) == [{"team_id": unlisted.team_id, "metadata": {"owner": "team"}}] + + all_before: Final = tuple( + _TeamPermissionRow.model_validate(row) + for row in read_rows( + 'SELECT team_id, team_member_permissions FROM "LiteLLM_TeamTable" ORDER BY team_id', + (), + ) + ) + for row in all_before: + if row.team_id not in team_ids and "/key/update" not in (row.team_member_permissions or []): + scenario.cleanups.callback(_restore_team_permissions, row.team_id, row.team_member_permissions) + missing: Final = tuple( + row.team_id for row in all_before if "/key/update" not in (row.team_member_permissions or []) + ) + all_updated: Final = gateway.request( + "POST", + "/team/permissions_bulk_update", + {"permissions": ["/key/update"], "apply_to_all_teams": True}, + ) + assert all_updated.status_code == 200, all_updated.text + all_response: Final = _BulkPermissionsUpdate.model_validate_json(all_updated.text) + assert all_response == _BulkPermissionsUpdate( + message="Team permissions updated successfully", + teams_updated=len(missing), + permissions_appended=["/key/update"], + ), all_updated.text + all_after: Final = tuple( + _TeamPermissionRow.model_validate(row) + for row in read_rows( + 'SELECT team_id, team_member_permissions FROM "LiteLLM_TeamTable" ORDER BY team_id', + (), + ) + ) + assert tuple(row.team_id for row in all_after) == tuple(row.team_id for row in all_before), repr(all_after) + assert all("/key/update" in (row.team_member_permissions or []) for row in all_after), repr(all_after) + unlisted_after_all: Final = _update_team_key(peer, unlisted, "unlisted member after all teams") + assert unlisted_after_all.status_code == 200, unlisted_after_all.text + assert _team_key_row(unlisted.team_key) == [ + {"team_id": unlisted.team_id, "metadata": {"owner": "unlisted member after all teams"}} + ], unlisted_after_all.text + observed: Final = _observed_models(upstream) + assert observed == (), repr(observed) + + def test_default_member_permissions_only_allow_reading_team_keys(gateway: Gateway) -> None: with gateway.scenario() as scenario: member: Final = _member(scenario, None) diff --git a/tests/integration/authorization/test_team_scoped_models.py b/tests/integration/authorization/test_team_scoped_models.py index 1323c1ee004..2982fbdb4eb 100644 --- a/tests/integration/authorization/test_team_scoped_models.py +++ b/tests/integration/authorization/test_team_scoped_models.py @@ -4,8 +4,38 @@ from typing import Final import httpx import pytest -from integration._support.client import Gateway, object_value, string_value -from pydantic import JsonValue +from integration._support.client import Gateway, Scenario, object_value, string_value +from integration._support.database import read_rows +from pydantic import BaseModel, JsonValue + + +class _ObservedRequest(BaseModel): + body: dict[str, JsonValue] + + +class _Observations(BaseModel): + requests: list[_ObservedRequest] + + +class _TeamState(BaseModel): + team_id: str + models: list[str] | None = None + + +class _TeamModelRow(BaseModel): + models: list[str] | None = None + + +class _TeamInfo(BaseModel): + team_info: _TeamState + + +class _ProxyError(BaseModel): + type: str + + +class _ErrorResponse(BaseModel): + error: _ProxyError @pytest.fixture @@ -27,6 +57,13 @@ def _calls_to(observed: list[JsonValue], provider_model: str) -> int: return sum(object_value(object_value(request)["body"]).get("model") == provider_model for request in observed) +def _observed_provider_models(upstream: httpx.Client) -> tuple[str, ...]: + observed: Final = upstream.get("/__observations") + assert observed.status_code == 200, observed.text + response: Final = _Observations.model_validate_json(observed.text) + return tuple(string_value(request.body["model"]) for request in response.requests) + + def _chat(gateway: Gateway, model: str, key: str) -> httpx.Response: return gateway.request( "POST", @@ -36,6 +73,28 @@ def _chat(gateway: Gateway, model: str, key: str) -> httpx.Response: ) +def _team_model(scenario: Scenario) -> tuple[str, str]: + provider_model: Final = f"team-model-{uuid.uuid4().hex}" + return scenario.model(model=f"openai/{provider_model}"), provider_model + + +def _team_info(gateway: Gateway, team_id: str) -> _TeamInfo: + response: Final = gateway.request("GET", "/team/info", params={"team_id": team_id}) + assert response.status_code == 200, response.text + return _TeamInfo.model_validate_json(response.text) + + +def _stored_team_models(team_id: str) -> _TeamModelRow: + rows: Final = read_rows('SELECT models FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team_id,)) + assert len(rows) == 1, repr(rows) + return _TeamModelRow.model_validate(rows[0]) + + +def _model_denied(response: httpx.Response) -> None: + assert response.status_code == 403, response.text + assert _ErrorResponse.model_validate_json(response.text).error.type == "team_model_access_denied", response.text + + def _ids(listing: dict[str, JsonValue]) -> set[JsonValue]: data: Final = listing["data"] assert isinstance(data, list) @@ -112,3 +171,89 @@ def test_team_model_alias_routes_a_team_key_to_its_target( assert unaliased.json()["error"]["type"] == "key_model_access_denied" after_refusal: Final = _observed_requests(upstream) assert _calls_to(after_refusal, provider_model) == 0, after_refusal + + +def test_team_model_add_and_delete_change_what_a_warmed_team_key_can_call( + gateway: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + model_one, provider_one = _team_model(scenario) + model_two, provider_two = _team_model(scenario) + model_three, provider_three = _team_model(scenario) + team: Final = scenario.team(models=[model_one]) + key: Final = scenario.key(team_id=team) + + first: Final = _chat(gateway, model_one, key) + assert first.status_code == 200, first.text + initially_denied: Final = _chat(gateway, model_two, key) + _model_denied(initially_denied) + observed_before_add: Final = _observed_provider_models(upstream) + assert observed_before_add == (provider_one,), repr(observed_before_add) + + added: Final = gateway.request("POST", "/team/model/add", {"team_id": team, "models": [model_two, model_three]}) + assert added.status_code == 200, added.text + added_state: Final = _TeamState.model_validate_json(added.text) + expected_added: Final = sorted((model_one, model_two, model_three)) + assert sorted(added_state.models or ()) == expected_added, added.text + added_info: Final = _team_info(gateway, team) + assert sorted(added_info.team_info.models or ()) == expected_added, repr(added_info) + added_row: Final = _stored_team_models(team) + assert sorted(added_row.models or ()) == expected_added, repr(added_row) + + second: Final = _chat(gateway, model_two, key) + assert second.status_code == 200, second.text + third: Final = _chat(gateway, model_three, key) + assert third.status_code == 200, third.text + deleted: Final = gateway.request("POST", "/team/model/delete", {"team_id": team, "models": [model_two]}) + assert deleted.status_code == 200, deleted.text + deleted_state: Final = _TeamState.model_validate_json(deleted.text) + expected_remaining: Final = sorted((model_one, model_three)) + assert sorted(deleted_state.models or ()) == expected_remaining, deleted.text + deleted_info: Final = _team_info(gateway, team) + assert sorted(deleted_info.team_info.models or ()) == expected_remaining, repr(deleted_info) + deleted_row: Final = _stored_team_models(team) + assert sorted(deleted_row.models or ()) == expected_remaining, repr(deleted_row) + + denied_after_delete: Final = _chat(gateway, model_two, key) + _model_denied(denied_after_delete) + still_allowed_one: Final = _chat(gateway, model_one, key) + assert still_allowed_one.status_code == 200, still_allowed_one.text + still_allowed_three: Final = _chat(gateway, model_three, key) + assert still_allowed_three.status_code == 200, still_allowed_three.text + observed_after_add_and_delete: Final = _observed_provider_models(upstream) + assert observed_before_add + observed_after_add_and_delete == ( + provider_one, + provider_two, + provider_three, + provider_one, + provider_three, + ), repr((observed_before_add, observed_after_add_and_delete)) + + +def test_team_model_add_on_an_unrestricted_team_keeps_every_other_model( + gateway: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + model_one, provider_one = _team_model(scenario) + model_two, provider_two = _team_model(scenario) + team: Final = scenario.team(models=[]) + key: Final = scenario.key(team_id=team) + + before_add: Final = _chat(gateway, model_two, key) + assert before_add.status_code == 200, before_add.text + + added: Final = gateway.request("POST", "/team/model/add", {"team_id": team, "models": [model_one]}) + assert added.status_code == 200, added.text + expected: Final = sorted(("all-proxy-models", model_one)) + added_state: Final = _TeamState.model_validate_json(added.text) + assert sorted(added_state.models or ()) == expected, added.text + added_info: Final = _team_info(gateway, team) + assert sorted(added_info.team_info.models or ()) == expected, repr(added_info) + added_row: Final = _stored_team_models(team) + assert sorted(added_row.models or ()) == expected, repr(added_row) + still_unrestricted: Final = _chat(gateway, model_two, key) + assert still_unrestricted.status_code == 200, still_unrestricted.text + added_model: Final = _chat(gateway, model_one, key) + assert added_model.status_code == 200, added_model.text + observed_after_add: Final = _observed_provider_models(upstream) + assert observed_after_add == (provider_two, provider_two, provider_one), repr(observed_after_add) diff --git a/tests/integration/management/test_team_block.py b/tests/integration/management/test_team_block.py new file mode 100644 index 00000000000..37c9da33c06 --- /dev/null +++ b/tests/integration/management/test_team_block.py @@ -0,0 +1,226 @@ +import uuid +from collections.abc import Iterator +from typing import Final + +import httpx +import pytest +from pydantic import BaseModel, JsonValue + +from tests.integration._support.client import Gateway, string_value +from tests.integration._support.database import read_rows + + +class _ObservedRequest(BaseModel): + body: dict[str, JsonValue] + + +class _Observations(BaseModel): + requests: list[_ObservedRequest] + + +class _TeamState(BaseModel): + team_id: str + blocked: bool + + +class _TeamInfo(BaseModel): + team_info: _TeamState + + +class _ProxyError(BaseModel): + type: str + + +class _ErrorResponse(BaseModel): + error: _ProxyError + + +@pytest.fixture +def upstream(gateway: Gateway) -> Iterator[httpx.Client]: + with httpx.Client(base_url=gateway.upstream_url, timeout=15, trust_env=False) as client: + response: Final = client.get("/__observations") + assert response.status_code == 200, response.text + yield client + + +def _chat(proxy: Gateway, model: str, key: str, text: str) -> httpx.Response: + return proxy.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": text}]}, + key=key, + ) + + +def _models(client: httpx.Client) -> tuple[str, ...]: + response: Final = client.get("/__observations") + assert response.status_code == 200, response.text + observations: Final = _Observations.model_validate_json(response.text) + return tuple(string_value(request.body["model"]) for request in observations.requests) + + +def test_team_block_and_unblock_update_endpoint_info_and_db_state( + gateway: Gateway, peer: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + provider_model: Final = f"team-block-{uuid.uuid4().hex}" + model: Final = scenario.model(model=f"openai/{provider_model}") + team: Final = scenario.team(models=[model]) + key: Final = scenario.key(team_id=team) + + primary_before: Final = _chat(gateway, model, key, "team block primary warm") + assert primary_before.status_code == 200, primary_before.text + peer_before: Final = _chat(peer, model, key, "team block peer warm") + assert peer_before.status_code == 200, peer_before.text + observed_before_block: Final = _models(upstream) + assert observed_before_block == (provider_model, provider_model), repr(observed_before_block) + + blocked_response: Final = gateway.request("POST", "/team/block", {"team_id": team}) + assert blocked_response.status_code == 200, blocked_response.text + blocked: Final = _TeamState.model_validate_json(blocked_response.text) + assert blocked.blocked is True, blocked_response.text + + info_response: Final = gateway.request("GET", "/team/info", params={"team_id": team}) + assert info_response.status_code == 200, info_response.text + info: Final = _TeamInfo.model_validate_json(info_response.text) + assert info.team_info.blocked is True, info_response.text + assert info.team_info.team_id == team, info_response.text + assert read_rows( + 'SELECT blocked FROM "LiteLLM_TeamTable" WHERE team_id = %s', + (team,), + ) == [{"blocked": True}], info_response.text + + cold_key: Final = scenario.key(team_id=team) + cold_denied: Final = _chat(gateway, model, cold_key, "cold team key after block") + assert cold_denied.status_code == 401, cold_denied.text + assert _ErrorResponse.model_validate_json(cold_denied.text).error.type == "auth_error", cold_denied.text + after_cold_denied: Final = _models(upstream) + assert after_cold_denied == (), repr(after_cold_denied) + + unblocked_response: Final = gateway.request("POST", "/team/unblock", {"team_id": team}) + assert unblocked_response.status_code == 200, unblocked_response.text + restored_key: Final = scenario.key(team_id=team) + restored: Final = _chat(gateway, model, restored_key, "new team key after unblock") + assert restored.status_code == 200, restored.text + after_unblock: Final = _models(upstream) + assert after_unblock == (provider_model,), repr(after_unblock) + unblocked: Final = _TeamState.model_validate_json(unblocked_response.text) + assert unblocked.blocked is False, unblocked_response.text + unblocked_info_response: Final = gateway.request("GET", "/team/info", params={"team_id": team}) + assert unblocked_info_response.status_code == 200, unblocked_info_response.text + unblocked_info: Final = _TeamInfo.model_validate_json(unblocked_info_response.text) + assert unblocked_info.team_info.blocked is False, unblocked_info_response.text + assert read_rows( + 'SELECT blocked FROM "LiteLLM_TeamTable" WHERE team_id = %s', + (team,), + ) == [{"blocked": False}], unblocked_info_response.text + + assert observed_before_block + after_cold_denied + after_unblock == ( + provider_model, + provider_model, + provider_model, + ), repr((observed_before_block, after_cold_denied, after_unblock)) + + +def test_unblocking_a_team_restores_warmed_keys_on_both_proxies( + gateway: Gateway, peer: Gateway, upstream: httpx.Client +) -> None: + pytest.skip("BUG: SLP-605 /team/unblock does not restore warmed team-key access after blocked auth is cached") + + with gateway.scenario() as scenario: + provider_model: Final = f"team-block-{uuid.uuid4().hex}" + model: Final = scenario.model(model=f"openai/{provider_model}") + team: Final = scenario.team(models=[model]) + key: Final = scenario.key(team_id=team) + + primary_before: Final = _chat(gateway, model, key, "team unblock primary warm") + assert primary_before.status_code == 200, primary_before.text + peer_before: Final = _chat(peer, model, key, "team unblock peer warm") + assert peer_before.status_code == 200, peer_before.text + observed_before_block: Final = _models(upstream) + assert observed_before_block == (provider_model, provider_model), repr(observed_before_block) + + blocked_response: Final = gateway.request("POST", "/team/block", {"team_id": team}) + assert blocked_response.status_code == 200, blocked_response.text + cold_key: Final = scenario.key(team_id=team) + cold_denied: Final = _chat(gateway, model, cold_key, "cold team key after block") + assert cold_denied.status_code == 401, cold_denied.text + assert _ErrorResponse.model_validate_json(cold_denied.text).error.type == "auth_error", cold_denied.text + after_cold_denied: Final = _models(upstream) + assert after_cold_denied == (), repr(after_cold_denied) + + unblocked_response: Final = gateway.request("POST", "/team/unblock", {"team_id": team}) + assert unblocked_response.status_code == 200, unblocked_response.text + + primary_after: Final = _chat(gateway, model, key, "team unblock primary") + assert primary_after.status_code == 200, primary_after.text + peer_after: Final = _chat(peer, model, key, "team unblock peer") + assert peer_after.status_code == 200, peer_after.text + after_unblock: Final = _models(upstream) + assert after_unblock == (provider_model, provider_model), repr(after_unblock) + + +def test_warmed_team_keys_are_refused_on_both_proxies_after_block( + gateway: Gateway, peer: Gateway, upstream: httpx.Client +) -> None: + pytest.skip("BUG: SLP-605 blocking a team does not refresh warmed team-key auth state") + + with gateway.scenario() as scenario: + provider_model: Final = f"team-block-{uuid.uuid4().hex}" + model: Final = scenario.model(model=f"openai/{provider_model}") + team: Final = scenario.team(models=[model]) + key: Final = scenario.key(team_id=team) + + primary_before: Final = _chat(gateway, model, key, "team block primary warm") + assert primary_before.status_code == 200, primary_before.text + peer_before: Final = _chat(peer, model, key, "team block peer warm") + assert peer_before.status_code == 200, peer_before.text + observed_before_block: Final = _models(upstream) + assert observed_before_block == (provider_model, provider_model), repr(observed_before_block) + + blocked_response: Final = gateway.request("POST", "/team/block", {"team_id": team}) + assert blocked_response.status_code == 200, blocked_response.text + + primary_denied: Final = _chat(gateway, model, key, "blocked team primary") + peer_denied: Final = _chat(peer, model, key, "blocked team peer") + after_block: Final = _models(upstream) + assert (primary_denied.status_code, peer_denied.status_code) == (401, 401), ( + f"primary={primary_denied.text}; peer={peer_denied.text}" + ) + assert _ErrorResponse.model_validate_json(primary_denied.text).error.type == "auth_error", primary_denied.text + assert _ErrorResponse.model_validate_json(peer_denied.text).error.type == "auth_error", peer_denied.text + assert after_block == (), repr(after_block) + assert observed_before_block + after_block == (provider_model, provider_model), repr( + (observed_before_block, after_block) + ) + + +def test_a_new_team_key_is_refused_on_a_peer_that_cached_the_team_before_block( + gateway: Gateway, peer: Gateway, upstream: httpx.Client +) -> None: + pytest.skip("BUG: SLP-605 a peer proxy that cached the team before /team/block admits a never-seen team key") + + with gateway.scenario() as scenario: + provider_model: Final = f"team-block-{uuid.uuid4().hex}" + model: Final = scenario.model(model=f"openai/{provider_model}") + team: Final = scenario.team(models=[model]) + warm_key: Final = scenario.key(team_id=team) + + peer_before: Final = _chat(peer, model, warm_key, "team block peer warm") + assert peer_before.status_code == 200, peer_before.text + observed_before_block: Final = _models(upstream) + assert observed_before_block == (provider_model,), repr(observed_before_block) + + blocked_response: Final = gateway.request("POST", "/team/block", {"team_id": team}) + assert blocked_response.status_code == 200, blocked_response.text + assert read_rows( + 'SELECT blocked FROM "LiteLLM_TeamTable" WHERE team_id = %s', + (team,), + ) == [{"blocked": True}], blocked_response.text + + cold_key: Final = scenario.key(team_id=team) + peer_denied: Final = _chat(peer, model, cold_key, "new team key on peer after block") + after_block: Final = _models(upstream) + assert peer_denied.status_code == 401, peer_denied.text + assert _ErrorResponse.model_validate_json(peer_denied.text).error.type == "auth_error", peer_denied.text + assert after_block == (), repr(after_block) diff --git a/tests/integration/management/test_team_member_lifecycle.py b/tests/integration/management/test_team_member_lifecycle.py new file mode 100644 index 00000000000..f2b8abaa9f1 --- /dev/null +++ b/tests/integration/management/test_team_member_lifecycle.py @@ -0,0 +1,925 @@ +import json +import uuid +from collections.abc import Iterator +from hashlib import sha256 +from typing import Final + +import httpx +import pytest +from pydantic import BaseModel, JsonValue + +from tests.integration._support.client import ( + Gateway, + Scenario, + delete_key_if_present, + string_value, +) +from tests.integration._support.database import read_rows, write_rows + + +class _ObservedRequest(BaseModel): + body: dict[str, JsonValue] + + +class _Observations(BaseModel): + requests: list[_ObservedRequest] + + +class _RosterMember(BaseModel): + role: str + user_id: str | None = None + user_email: str | None = None + + +class _TeamRecord(BaseModel): + team_id: str + models: list[str] | None = None + members_with_roles: list[_RosterMember] + + +class _BudgetDetails(BaseModel): + max_budget: float | None = None + allowed_models: list[str] | None = None + + +class _TeamMembership(BaseModel): + team_id: str + user_id: str + budget_id: str | None = None + spend: float | None = None + total_spend: float | None = None + litellm_budget_table: _BudgetDetails | None = None + + +class _TeamInfo(BaseModel): + team_info: _TeamRecord + team_memberships: list[_TeamMembership] | None = None + + +class _UserRecord(BaseModel): + user_id: str + user_email: str | None = None + teams: list[str] | None = None + + +class _MemberAddResponse(BaseModel): + team_id: str + updated_users: list[_UserRecord] + updated_team_memberships: list[_TeamMembership] + + +class _MemberUpdateResponse(BaseModel): + team_id: str + user_id: str + max_budget_in_team: float | None = None + + +class _GeneratedKey(BaseModel): + key: str + + +class _ProxyError(BaseModel): + type: str + + +class _ErrorResponse(BaseModel): + error: _ProxyError + + +class _HTTPErrorDetail(BaseModel): + error: str + + +class _HTTPErrorResponse(BaseModel): + detail: _HTTPErrorDetail + + +class _BulkAddResult(BaseModel): + user_id: str | None = None + user_email: str | None = None + success: bool + error: str | None = None + + +class _BulkAddResponse(BaseModel): + team_id: str + results: list[_BulkAddResult] + total_requested: int + successful_additions: int + failed_additions: int + + +class _BulkDeleteResult(BaseModel): + user_id: str | None = None + user_email: str | None = None + success: bool + error: str | None = None + + +class _BulkDeleteResponse(BaseModel): + data: tuple[_BulkDeleteResult, ...] + + +@pytest.fixture +def upstream(gateway: Gateway) -> Iterator[httpx.Client]: + with httpx.Client(base_url=gateway.upstream_url, timeout=15, trust_env=False) as client: + response: Final = client.get("/__observations") + assert response.status_code == 200, response.text + yield client + + +def _model(scenario: Scenario) -> tuple[str, str]: + provider_model: Final = f"team-member-{uuid.uuid4().hex}" + return scenario.model(model=f"openai/{provider_model}"), provider_model + + +def _key(scenario: Scenario, **fields: JsonValue) -> str: + response: Final = scenario.gateway.request("POST", "/key/generate", fields) + assert response.status_code == 200, response.text + token: Final = _GeneratedKey.model_validate_json(response.text).key + scenario.cleanups.callback(delete_key_if_present, scenario.gateway, token) + return token + + +def _team_info(gateway: Gateway, team_id: str) -> _TeamInfo: + response: Final = gateway.request("GET", "/team/info", params={"team_id": team_id}) + assert response.status_code == 200, response.text + return _TeamInfo.model_validate_json(response.text) + + +def _models(upstream: httpx.Client) -> tuple[str, ...]: + response: Final = upstream.get("/__observations") + assert response.status_code == 200, response.text + observations: Final = _Observations.model_validate_json(response.text) + return tuple(string_value(request.body["model"]) for request in observations.requests) + + +def _chat(proxy: Gateway, model: str, key: str, text: str) -> httpx.Response: + return proxy.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": text}]}, + key=key, + ) + + +def _refused(response: httpx.Response, status: int, error_type: str) -> None: + assert response.status_code == status, response.text + assert _ErrorResponse.model_validate_json(response.text).error.type == error_type, response.text + + +def _remove_default_user(gateway: Gateway, team_id: str) -> None: + response: Final = gateway.request("POST", "/team/member_delete", {"team_id": team_id, "user_id": "default_user_id"}) + assert response.status_code == 200, response.text + + +def _roster_member(user_id: str, user_email: str | None, role: str = "user") -> dict[str, JsonValue]: + return {"role": role, "user_id": user_id, "user_email": user_email} + + +@pytest.mark.parametrize("identifier", ("user_id", "user_email"), ids=("by-id", "by-email")) +def test_member_delete_revokes_the_removed_members_warmed_key( + gateway: Gateway, upstream: httpx.Client, identifier: str +) -> None: + with gateway.scenario() as scenario: + model, provider_model = _model(scenario) + team: Final = scenario.team(models=[model]) + leaver_email: Final = f"member-leaver-{uuid.uuid4().hex}@integration.test" + leaver: Final = scenario.user(user_email=leaver_email) + _remove_default_user(gateway, team) + stayer: Final = scenario.member(team) + added: Final = gateway.request( + "POST", + "/team/member_add", + { + "team_id": team, + "member": ( + {"role": "user", "user_id": leaver} + if identifier == "user_id" + else {"role": "user", "user_email": leaver_email} + ), + }, + ) + assert added.status_code == 200, added.text + leaver_key: Final = _key(scenario, team_id=team, user_id=leaver) + stayer_key: Final = _key(scenario, team_id=team, user_id=stayer) + leaver_warm: Final = _chat(gateway, model, leaver_key, "warm leaver " + uuid.uuid4().hex) + assert leaver_warm.status_code == 200, leaver_warm.text + stayer_warm: Final = _chat(gateway, model, stayer_key, "warm stayer " + uuid.uuid4().hex) + assert stayer_warm.status_code == 200, stayer_warm.text + observed_before_delete: Final = _models(upstream) + assert observed_before_delete == (provider_model, provider_model), repr(observed_before_delete) + + delete_body: Final = ( + {"team_id": team, "user_id": leaver} + if identifier == "user_id" + else {"team_id": team, "user_email": leaver_email} + ) + deleted: Final = gateway.request("POST", "/team/member_delete", delete_body) + assert deleted.status_code == 200, deleted.text + + info: Final = _team_info(gateway, team) + stayer_roster: Final = [_roster_member(stayer, None)] + assert info.team_info.members_with_roles == [_RosterMember.model_validate(stayer_roster[0])], repr(info) + assert read_rows('SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team,)) == [ + {"members_with_roles": stayer_roster} + ], deleted.text + assert ( + read_rows( + 'SELECT user_id, team_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s AND user_id = %s', + (team, leaver), + ) + == [] + ), deleted.text + assert read_rows( + 'SELECT user_id, team_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s AND user_id = %s', + (team, stayer), + ) == [{"user_id": stayer, "team_id": team}], deleted.text + leaver_hash: Final = sha256(leaver_key.encode()).hexdigest() + assert ( + read_rows( + 'SELECT token FROM "LiteLLM_VerificationToken" WHERE token = %s', + (leaver_hash,), + ) + == [] + ), deleted.text + assert read_rows( + 'SELECT token FROM "LiteLLM_DeletedVerificationToken" WHERE token = %s', + (leaver_hash,), + ) == [{"token": leaver_hash}], deleted.text + + refused: Final = _chat(gateway, model, leaver_key, "deleted leaver " + uuid.uuid4().hex) + _refused(refused, 401, "token_not_found_in_db") + after_refused: Final = _models(upstream) + assert after_refused == (), repr(after_refused) + stayer_after: Final = _chat(gateway, model, stayer_key, "stayer after delete " + uuid.uuid4().hex) + assert stayer_after.status_code == 200, stayer_after.text + observed_after_delete: Final = _models(upstream) + assert observed_before_delete + after_refused + observed_after_delete == ( + provider_model, + provider_model, + provider_model, + ), repr((observed_before_delete, after_refused, observed_after_delete)) + + +def test_member_delete_by_email_removes_a_legacy_email_only_roster_entry( + gateway: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + model, provider_model = _model(scenario) + team: Final = scenario.team(models=[model]) + email: Final = f"legacy-member-{uuid.uuid4().hex}@integration.test" + user: Final = scenario.user(user_email=email) + _remove_default_user(gateway, team) + stayer: Final = scenario.member(team) + added: Final = gateway.request( + "POST", "/team/member_add", {"team_id": team, "member": {"role": "user", "user_id": user}} + ) + assert added.status_code == 200, added.text + leaver_key: Final = _key(scenario, team_id=team, user_id=user) + stayer_key: Final = _key(scenario, team_id=team, user_id=stayer) + leaver_warm: Final = _chat(gateway, model, leaver_key, "warm legacy leaver " + uuid.uuid4().hex) + assert leaver_warm.status_code == 200, leaver_warm.text + observed_before_delete: Final = _models(upstream) + assert observed_before_delete == (provider_model,), repr(observed_before_delete) + + stayer_roster: Final = _roster_member(stayer, None) + legacy_roster: Final = {"role": "user", "user_id": None, "user_email": email} + write_rows( + 'UPDATE "LiteLLM_TeamTable" SET members_with_roles = %s::jsonb WHERE team_id = %s', + (json.dumps([legacy_roster, stayer_roster]), team), + ) + assert read_rows('SELECT teams FROM "LiteLLM_UserTable" WHERE user_id = %s', (user,)) == [{"teams": [team]}], ( + added.text + ) + leaver_hash: Final = sha256(leaver_key.encode()).hexdigest() + + deleted: Final = gateway.request("POST", "/team/member_delete", {"team_id": team, "user_email": email}) + assert deleted.status_code == 200, deleted.text + info: Final = _team_info(gateway, team) + assert info.team_info.members_with_roles == [_RosterMember.model_validate(stayer_roster)], repr(info) + assert read_rows('SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team,)) == [ + {"members_with_roles": [stayer_roster]} + ], deleted.text + assert read_rows('SELECT teams FROM "LiteLLM_UserTable" WHERE user_id = %s', (user,)) == [{"teams": []}], ( + deleted.text + ) + assert read_rows( + 'SELECT user_id, team_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s ORDER BY user_id', + (team,), + ) == [{"user_id": stayer, "team_id": team}], deleted.text + assert read_rows('SELECT token FROM "LiteLLM_VerificationToken" WHERE token = %s', (leaver_hash,)) == [], ( + deleted.text + ) + assert read_rows( + 'SELECT token FROM "LiteLLM_DeletedVerificationToken" WHERE token = %s', + (leaver_hash,), + ) == [{"token": leaver_hash}], deleted.text + + refused: Final = _chat(gateway, model, leaver_key, "deleted legacy leaver " + uuid.uuid4().hex) + _refused(refused, 401, "token_not_found_in_db") + after_refused: Final = _models(upstream) + assert after_refused == (), repr(after_refused) + served: Final = _chat(gateway, model, stayer_key, "stayer after legacy delete " + uuid.uuid4().hex) + assert served.status_code == 200, served.text + observed: Final = _models(upstream) + assert observed_before_delete + after_refused + observed == (provider_model, provider_model), repr( + (observed_before_delete, after_refused, observed) + ) + + +def test_member_delete_by_case_variant_email_removes_member_and_refuses_warmed_key( + gateway: Gateway, upstream: httpx.Client +) -> None: + pytest.skip("BUG: case-variant email does not match the member during team deletion") + + with gateway.scenario() as scenario: + model, provider_model = _model(scenario) + team: Final = scenario.team(models=[model]) + email: Final = f"case-delete-{uuid.uuid4().hex}@integration.test" + user: Final = scenario.user(user_email=email) + _remove_default_user(gateway, team) + stayer: Final = scenario.member(team) + added: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"role": "user", "user_email": email}}, + ) + assert added.status_code == 200, added.text + leaver_key: Final = _key(scenario, team_id=team, user_id=user) + warmed: Final = _chat(gateway, model, leaver_key, f"warm case-variant leaver {uuid.uuid4().hex}") + assert warmed.status_code == 200, warmed.text + observed_before_delete: Final = _models(upstream) + assert observed_before_delete == (provider_model,), repr(observed_before_delete) + + deleted: Final = gateway.request( + "POST", + "/team/member_delete", + {"team_id": team, "user_email": email.upper()}, + ) + assert deleted.status_code == 200, deleted.text + info: Final = _team_info(gateway, team) + stayer_roster: Final = _roster_member(stayer, None) + assert info.team_info.members_with_roles == [_RosterMember.model_validate(stayer_roster)], repr(info) + assert read_rows('SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team,)) == [ + {"members_with_roles": [stayer_roster]} + ], deleted.text + assert read_rows( + 'SELECT user_id, team_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s AND user_id = %s', + (team, user), + ) == [], deleted.text + + refused: Final = _chat(gateway, model, leaver_key, f"deleted case-variant leaver {uuid.uuid4().hex}") + _refused(refused, 401, "token_not_found_in_db") + after_refused: Final = _models(upstream) + assert after_refused == (), repr(after_refused) + assert observed_before_delete + after_refused == (provider_model,), repr( + (observed_before_delete, after_refused) + ) + + +def test_member_update_by_case_variant_email_updates_the_member( + gateway: Gateway, +) -> None: + pytest.skip("BUG: case-variant email does not match the member during team update") + + with gateway.scenario() as scenario: + team: Final = scenario.team() + email: Final = f"case-update-{uuid.uuid4().hex}@integration.test" + user: Final = scenario.user(user_email=email) + _remove_default_user(gateway, team) + added: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"role": "user", "user_email": email}}, + ) + assert added.status_code == 200, added.text + + response: Final = gateway.request( + "POST", + "/team/member_update", + {"team_id": team, "user_email": email.upper(), "max_budget_in_team": 12}, + ) + assert response.status_code == 200, response.text + updated: Final = _MemberUpdateResponse.model_validate_json(response.text) + assert updated.user_id == user, response.text + assert updated.max_budget_in_team == 12, response.text + + membership_rows: Final = read_rows( + 'SELECT user_id, team_id, budget_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s AND user_id = %s', + (team, user), + ) + assert len(membership_rows) == 1, membership_rows + budget_id: Final = membership_rows[0]["budget_id"] + assert isinstance(budget_id, str), membership_rows + budget_rows: Final = read_rows( + 'SELECT budget_id, max_budget FROM "LiteLLM_BudgetTable" WHERE budget_id = %s', + (budget_id,), + ) + assert budget_rows == [{"budget_id": budget_id, "max_budget": 12.0}], budget_rows + + info: Final = _team_info(gateway, team) + assert info.team_memberships is not None, repr(info) + updated_membership: Final = next(member for member in info.team_memberships if member.user_id == user) + assert updated_membership.budget_id == budget_id, repr(updated_membership) + assert updated_membership.litellm_budget_table is not None, repr(updated_membership) + assert updated_membership.litellm_budget_table.max_budget == 12, repr(updated_membership) + + +def test_member_add_by_email_provisions_a_new_user_and_resolves_an_existing_one( + gateway: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + team: Final = scenario.team() + _remove_default_user(gateway, team) + existing_email: Final = f"existing-member-{uuid.uuid4().hex}@integration.test" + existing_user: Final = scenario.user(user_email=existing_email) + existing_email_variant: Final = existing_email.upper() + existing_added: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"role": "user", "user_email": existing_email_variant}}, + ) + assert existing_added.status_code == 200, existing_added.text + existing_response: Final = _MemberAddResponse.model_validate_json(existing_added.text) + assert tuple(user.user_id for user in existing_response.updated_users) == (existing_user,), existing_added.text + assert read_rows( + 'SELECT user_id, user_email FROM "LiteLLM_UserTable" WHERE LOWER(user_email) = LOWER(%s)', + (existing_email,), + ) == [{"user_id": existing_user, "user_email": existing_email}], existing_added.text + + new_email: Final = f"provisioned-member-{uuid.uuid4().hex}@integration.test" + new_added: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"role": "user", "user_email": new_email}}, + ) + assert new_added.status_code == 200, new_added.text + new_response: Final = _MemberAddResponse.model_validate_json(new_added.text) + assert len(new_response.updated_users) == 1, new_added.text + provisioned: Final = new_response.updated_users[0] + assert provisioned.user_email == new_email, new_added.text + scenario.cleanups.callback(scenario.delete_user, provisioned.user_id) + assert read_rows( + 'SELECT user_id, user_email, teams FROM "LiteLLM_UserTable" WHERE user_email = %s', + (new_email,), + ) == [{"user_id": provisioned.user_id, "user_email": new_email, "teams": [team]}], new_added.text + info: Final = _team_info(gateway, team) + assert info.team_info.members_with_roles == [ + _RosterMember(role="user", user_id=existing_user, user_email=existing_email_variant), + _RosterMember(role="user", user_id=provisioned.user_id, user_email=new_email), + ], repr(info) + observed: Final = _models(upstream) + assert observed == (), repr(observed) + + +def test_member_add_list_mixing_id_and_email_applies_budget_and_allowed_models( + gateway: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + model_one, provider_one = _model(scenario) + model_two, provider_two = _model(scenario) + team: Final = scenario.team(models=[model_one, model_two]) + _remove_default_user(gateway, team) + user_one: Final = scenario.user(user_role="internal_user") + email_two: Final = f"mixed-member-{uuid.uuid4().hex}@integration.test" + user_two: Final = scenario.user(user_email=email_two, user_role="internal_user") + added: Final = gateway.request( + "POST", + "/team/member_add", + { + "team_id": team, + "member": [ + {"role": "user", "user_id": user_one}, + {"role": "user", "user_email": email_two}, + ], + "max_budget_in_team": 1, + "allowed_models": [model_one], + }, + ) + assert added.status_code == 200, added.text + response: Final = _MemberAddResponse.model_validate_json(added.text) + assert tuple(user.user_id for user in response.updated_users) == (user_one, user_two), added.text + assert tuple(member.user_id for member in response.updated_team_memberships) == (user_one, user_two), added.text + assert read_rows( + "SELECT tm.user_id, b.max_budget, b.allowed_models " + 'FROM "LiteLLM_TeamMembership" tm ' + 'JOIN "LiteLLM_BudgetTable" b ON b.budget_id = tm.budget_id ' + "WHERE tm.team_id = %s ORDER BY tm.user_id", + (team,), + ) == [ + {"user_id": user_id, "max_budget": 1.0, "allowed_models": [model_one]} + for user_id in sorted((user_one, user_two)) + ], added.text + info: Final = _team_info(gateway, team) + membership_limits: Final = tuple( + sorted( + ( + ( + membership.user_id, + membership.litellm_budget_table.max_budget + if membership.litellm_budget_table is not None + else None, + membership.litellm_budget_table.allowed_models + if membership.litellm_budget_table is not None + else None, + ) + for membership in info.team_memberships or () + ), + key=lambda member: member[0], + ) + ) + assert membership_limits == tuple( + sorted(((user_one, 1, [model_one]), (user_two, 1, [model_one])), key=lambda member: member[0]) + ), added.text + + key_one: Final = _key(scenario, team_id=team, user_id=user_one) + key_two: Final = _key(scenario, team_id=team, user_id=user_two) + allowed_one: Final = _chat(gateway, model_one, key_one, "mixed member one allowed") + assert allowed_one.status_code == 200, allowed_one.text + denied_one: Final = _chat(gateway, model_two, key_one, "mixed member one denied") + _refused(denied_one, 403, "team_model_access_denied") + after_denied_one: Final = _models(upstream) + allowed_two: Final = _chat(gateway, model_one, key_two, "mixed member two allowed") + assert allowed_two.status_code == 200, allowed_two.text + denied_two: Final = _chat(gateway, model_two, key_two, "mixed member two denied") + _refused(denied_two, 403, "team_model_access_denied") + after_denied_two: Final = _models(upstream) + assert after_denied_one == (provider_one,), repr(after_denied_one) + assert after_denied_two == (provider_one,), repr(after_denied_two) + assert provider_two not in after_denied_two, repr(after_denied_two) + assert after_denied_one + after_denied_two == (provider_one, provider_one), repr( + (after_denied_one, after_denied_two) + ) + + +def test_bulk_member_add_adds_every_listed_member_and_reports_failures( + gateway: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + model, provider_model = _model(scenario) + team: Final = scenario.team(models=[model]) + _remove_default_user(gateway, team) + user_one: Final = scenario.user(user_role="internal_user") + user_two: Final = scenario.user(user_role="internal_user") + email_three: Final = f"bulk-member-{uuid.uuid4().hex}@integration.test" + user_three: Final = scenario.user(user_email=email_three, user_role="internal_user") + email_four: Final = f"bulk-admin-{uuid.uuid4().hex}@integration.test" + user_four: Final = scenario.user(user_email=email_four, user_role="internal_user") + email_five: Final = f"bulk-admin-provisioned-{uuid.uuid4().hex}@integration.test" + members: Final = [ + {"role": "user", "user_id": user_one}, + {"role": "user", "user_id": user_two}, + {"role": "user", "user_email": email_three}, + {"role": "admin", "user_email": email_four}, + {"role": "admin", "user_email": email_five}, + ] + added: Final = gateway.request( + "POST", + "/team/bulk_member_add", + {"team_id": team, "members": members, "max_budget_in_team": 1}, + ) + assert added.status_code == 200, added.text + provisioned_rows: Final = read_rows( + 'SELECT user_id FROM "LiteLLM_UserTable" WHERE user_email = %s', + (email_five,), + ) + assert len(provisioned_rows) == 1, added.text + user_five: Final = string_value(provisioned_rows[0]["user_id"]) + scenario.cleanups.callback(scenario.delete_user, user_five) + response: Final = _BulkAddResponse.model_validate_json(added.text) + assert response.team_id == team, added.text + assert response.total_requested == 5, added.text + assert response.successful_additions == 5, added.text + assert response.failed_additions == 0, added.text + assert response.results == [ + _BulkAddResult(user_id=user_one, success=True), + _BulkAddResult(user_id=user_two, success=True), + _BulkAddResult(user_id=user_three, user_email=email_three, success=True), + _BulkAddResult(user_id=user_four, user_email=email_four, success=True), + _BulkAddResult(user_email=email_five, success=True), + ], added.text + assert read_rows( + 'SELECT user_id, team_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s ORDER BY user_id', + (team,), + ) == [ + {"user_id": user_id, "team_id": team} + for user_id in sorted((user_one, user_two, user_three, user_four, user_five)) + ], added.text + assert read_rows( + 'SELECT m.user_id, b.max_budget FROM "LiteLLM_TeamMembership" m ' + 'JOIN "LiteLLM_BudgetTable" b ON b.budget_id = m.budget_id WHERE m.team_id = %s ORDER BY m.user_id', + (team,), + ) == [ + {"user_id": user_id, "max_budget": 1.0} + for user_id in sorted((user_one, user_two, user_three, user_four, user_five)) + ], added.text + assert read_rows( + 'SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id = %s', + (team,), + ) == [ + { + "members_with_roles": [ + _roster_member(user_one, None), + _roster_member(user_two, None), + _roster_member(user_three, email_three), + _roster_member(user_four, email_four, role="admin"), + _roster_member(user_five, email_five, role="admin"), + ] + } + ], added.text + + write_rows( + 'UPDATE "LiteLLM_TeamMembership" SET spend = 2 WHERE team_id = %s AND user_id = %s', + (team, user_three), + ) + keys: Final = tuple(_key(scenario, team_id=team, user_id=user) for user in (user_one, user_two, user_three)) + calls: Final = tuple(_chat(gateway, model, key, f"bulk member {index}") for index, key in enumerate(keys)) + assert tuple(response.status_code for response in calls[:2]) == (200, 200), tuple( + response.text for response in calls + ) + _refused(calls[2], 422, "budget_exceeded") + duplicate: Final = gateway.request( + "POST", + "/team/bulk_member_add", + {"team_id": team, "members": members, "max_budget_in_team": 1}, + ) + assert duplicate.status_code == 200, duplicate.text + failed: Final = _BulkAddResponse.model_validate_json(duplicate.text) + assert failed.total_requested == 5, duplicate.text + assert failed.successful_additions == 0, duplicate.text + assert failed.failed_additions == 5, duplicate.text + assert tuple(result.success for result in failed.results) == (False,) * 5, duplicate.text + assert all(result.error is not None for result in failed.results), duplicate.text + assert read_rows( + 'SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id = %s', + (team,), + ) == [ + { + "members_with_roles": [ + _roster_member(user_one, None), + _roster_member(user_two, None), + _roster_member(user_three, email_three), + _roster_member(user_four, email_four, role="admin"), + _roster_member(user_five, email_five, role="admin"), + ] + } + ], duplicate.text + + admin_personal_key: Final = _key(scenario, user_id=user_four) + provisioned_admin_key: Final = _key(scenario, user_id=user_five) + member_personal_key: Final = _key(scenario, user_id=user_one) + probe: Final = scenario.user() + admin_added: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"role": "user", "user_id": probe}}, + key=admin_personal_key, + ) + assert admin_added.status_code == 200, admin_added.text + probe_five: Final = scenario.user() + provisioned_admin_added: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"role": "user", "user_id": probe_five}}, + key=provisioned_admin_key, + ) + assert provisioned_admin_added.status_code == 200, provisioned_admin_added.text + refused_probe: Final = scenario.user() + member_denied: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"role": "user", "user_id": refused_probe}}, + key=member_personal_key, + ) + assert member_denied.status_code == 403, member_denied.text + denied_response: Final = _HTTPErrorResponse.model_validate_json(member_denied.text) + assert "User not proxy admin OR team admin" in denied_response.detail.error, member_denied.text + assert read_rows( + 'SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id = %s', + (team,), + ) == [ + { + "members_with_roles": [ + _roster_member(user_one, None), + _roster_member(user_two, None), + _roster_member(user_three, email_three), + _roster_member(user_four, email_four, role="admin"), + _roster_member(user_five, email_five, role="admin"), + _roster_member(probe, None), + _roster_member(probe_five, None), + ] + } + ], member_denied.text + observed: Final = _models(upstream) + assert observed == (provider_model, provider_model), repr(observed) + + +def test_bulk_member_add_all_users_is_proxy_admin_only_and_adds_every_user( + gateway: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + team: Final = scenario.team() + _remove_default_user(gateway, team) + caller: Final = scenario.user(user_role="internal_user") + caller_key: Final = _key(scenario, user_id=caller, allowed_routes=["/team/bulk_member_add"]) + denied: Final = gateway.request( + "POST", "/team/bulk_member_add", {"team_id": team, "all_users": True}, key=caller_key + ) + assert denied.status_code == 403, denied.text + assert _HTTPErrorResponse.model_validate_json(denied.text) == _HTTPErrorResponse( + detail=_HTTPErrorDetail( + error="`all_users=true` is restricted to PROXY_ADMIN. Org/team admins must specify explicit member lists." + ) + ), denied.text + assert read_rows('SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team,)) == [ + {"members_with_roles": []} + ], denied.text + assert read_rows('SELECT user_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s', (team,)) == [], denied.text + + users: Final = tuple( + (string_value(row["user_id"]), row["user_email"]) + for row in read_rows('SELECT user_id, user_email FROM "LiteLLM_UserTable" ORDER BY user_id', ()) + ) + assert caller in {user_id for user_id, _ in users}, repr(users) + added: Final = gateway.request("POST", "/team/bulk_member_add", {"team_id": team, "all_users": True}) + assert added.status_code == 200, added.text + response: Final = _BulkAddResponse.model_validate_json(added.text) + assert response.team_id == team, added.text + assert response.total_requested == len(users), added.text + assert response.successful_additions == len(users), added.text + assert response.failed_additions == 0, added.text + assert sorted( + ((result.user_id, result.user_email, result.success) for result in response.results), + key=lambda result: result[0] or "", + ) == [(user_id, user_email, True) for user_id, user_email in users], added.text + assert read_rows( + 'SELECT user_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s ORDER BY user_id', + (team,), + ) == [{"user_id": user_id} for user_id, _ in users], added.text + roster: Final = read_rows('SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team,)) + assert len(roster) == 1, added.text + roster_members: Final = roster[0]["members_with_roles"] + assert isinstance(roster_members, list), repr(roster) + assert sorted( + (_RosterMember.model_validate(member) for member in roster_members), + key=lambda member: member.user_id or "", + ) == [ + _RosterMember.model_validate(_roster_member(user_id, string_value(user_email) if user_email else None)) + for user_id, user_email in users + ], added.text + observed: Final = _models(upstream) + assert observed == (), repr(observed) + + +def test_bulk_delete_removes_exactly_the_named_members_and_revokes_their_access( + gateway: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + model, provider_model = _model(scenario) + team: Final = scenario.team(models=[model]) + _remove_default_user(gateway, team) + admin: Final = scenario.member(team, role="admin") + email_member: Final = f"bulk-delete-member-{uuid.uuid4().hex}@integration.test" + member: Final = scenario.user(user_email=email_member, user_role="internal_user") + added: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"role": "user", "user_email": email_member}}, + ) + assert added.status_code == 200, added.text + stayer: Final = scenario.member(team) + admin_key: Final = _key(scenario, team_id=team, user_id=admin) + member_key: Final = _key(scenario, team_id=team, user_id=member) + stayer_key: Final = _key(scenario, team_id=team, user_id=stayer) + personal_admin_key: Final = _key(scenario, user_id=admin) + admin_warm: Final = _chat(gateway, model, admin_key, "bulk delete warm admin") + assert admin_warm.status_code == 200, admin_warm.text + member_warm: Final = _chat(gateway, model, member_key, "bulk delete warm member") + assert member_warm.status_code == 200, member_warm.text + stayer_warm: Final = _chat(gateway, model, stayer_key, "bulk delete warm stayer") + assert stayer_warm.status_code == 200, stayer_warm.text + observed_before_delete: Final = _models(upstream) + assert observed_before_delete == (provider_model, provider_model, provider_model), repr(observed_before_delete) + + probe_user: Final = scenario.user() + admin_route_before: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"role": "user", "user_id": probe_user}}, + key=personal_admin_key, + ) + assert admin_route_before.status_code == 200, admin_route_before.text + probe_removed: Final = gateway.request("POST", "/team/member_delete", {"team_id": team, "user_id": probe_user}) + assert probe_removed.status_code == 200, probe_removed.text + deleted: Final = gateway.request( + "POST", + f"/management/v1/teams/{team}/members/bulk_delete", + { + "members": [ + {"user_id": admin}, + {"user_email": email_member}, + {"user_id": admin}, + ] + }, + ) + assert deleted.status_code == 200, deleted.text + results: Final = _BulkDeleteResponse.model_validate_json(deleted.text).data + assert results == ( + _BulkDeleteResult(user_id=admin, success=True), + _BulkDeleteResult(user_email=email_member, success=True), + _BulkDeleteResult(user_id=admin, success=False, error="Duplicate member in request"), + ), deleted.text + team_info: Final = _team_info(gateway, team) + assert team_info.team_info.members_with_roles == [ + _RosterMember(role="user", user_id=stayer, user_email=None) + ], repr(team_info) + assert read_rows( + 'SELECT user_id, team_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s ORDER BY user_id', + (team,), + ) == [{"user_id": stayer, "team_id": team}], deleted.text + for removed_key in (admin_key, member_key): + removed_hash: Final = sha256(removed_key.encode()).hexdigest() + assert ( + read_rows( + 'SELECT token FROM "LiteLLM_VerificationToken" WHERE token = %s', + (removed_hash,), + ) + == [] + ), deleted.text + + denied_admin: Final = _chat(gateway, model, admin_key, "deleted admin key") + _refused(denied_admin, 401, "token_not_found_in_db") + observed_after_admin_denied: Final = _models(upstream) + denied_member: Final = _chat(gateway, model, member_key, "deleted member key") + _refused(denied_member, 401, "token_not_found_in_db") + observed_after_member_denied: Final = _models(upstream) + denied_admin_route: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"role": "user", "user_id": probe_user}}, + key=personal_admin_key, + ) + assert denied_admin_route.status_code == 403, denied_admin_route.text + denied_admin_response: Final = _HTTPErrorResponse.model_validate_json(denied_admin_route.text) + assert "User not proxy admin OR team admin" in denied_admin_response.detail.error, denied_admin_route.text + stayer_after: Final = _chat(gateway, model, stayer_key, "stayer after bulk delete") + assert stayer_after.status_code == 200, stayer_after.text + observed_after_stayer: Final = _models(upstream) + assert observed_after_admin_denied == (), repr(observed_after_admin_denied) + assert observed_after_member_denied == (), repr(observed_after_member_denied) + assert ( + observed_before_delete + observed_after_admin_denied + observed_after_member_denied + observed_after_stayer + == ( + provider_model, + provider_model, + provider_model, + provider_model, + ) + ) + + +def test_bulk_delete_by_case_variant_email_removes_member_and_refuses_warmed_key( + gateway: Gateway, upstream: httpx.Client +) -> None: + pytest.skip("BUG: bulk member deletion does not match case-variant member email") + with gateway.scenario() as scenario: + model, provider_model = _model(scenario) + team: Final = scenario.team(models=[model]) + _remove_default_user(gateway, team) + email: Final = f"bulk-case-delete-{uuid.uuid4().hex}@integration.test" + user: Final = scenario.user(user_email=email) + added: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"role": "user", "user_email": email}}, + ) + assert added.status_code == 200, added.text + member_key: Final = _key(scenario, team_id=team, user_id=user) + warmed: Final = _chat(gateway, model, member_key, f"warm bulk case-variant member {uuid.uuid4().hex}") + assert warmed.status_code == 200, warmed.text + observed_before_delete: Final = _models(upstream) + assert observed_before_delete == (provider_model,), repr(observed_before_delete) + + deleted: Final = gateway.request( + "POST", + f"/management/v1/teams/{team}/members/bulk_delete", + {"members": [{"user_email": email.upper()}]}, + ) + assert deleted.status_code == 200, deleted.text + results: Final = _BulkDeleteResponse.model_validate_json(deleted.text).data + assert results == (_BulkDeleteResult(user_email=email.upper(), success=True),), deleted.text + info: Final = _team_info(gateway, team) + assert info.team_info.members_with_roles == [], repr(info) + assert read_rows( + 'SELECT user_id, team_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s AND user_id = %s', + (team, user), + ) == [], deleted.text + + refused: Final = _chat(gateway, model, member_key, f"deleted bulk case-variant member {uuid.uuid4().hex}") + _refused(refused, 401, "token_not_found_in_db") + after_refused: Final = _models(upstream) + assert after_refused == (), repr(after_refused) + assert observed_before_delete + after_refused == (provider_model,), repr( + (observed_before_delete, after_refused) + ) diff --git a/tests/integration/spend/test_team_member_spend.py b/tests/integration/spend/test_team_member_spend.py index cf1dac25793..343a3be1b49 100644 --- a/tests/integration/spend/test_team_member_spend.py +++ b/tests/integration/spend/test_team_member_spend.py @@ -1,14 +1,231 @@ import os import uuid +from collections.abc import Callable, Iterator +from datetime import datetime, timedelta, timezone from typing import Final import httpx import pytest -from integration._support.client import Gateway, eventually, object_value +from integration._support.client import Gateway, Scenario, eventually, object_value, string_value from integration._support.database import read_rows +from pydantic import BaseModel, JsonValue from redis import Redis +def _python_isoformat(value: datetime) -> str: + return value.isoformat() + + +def _javascript_iso_string(value: datetime) -> str: + return value.strftime("%Y-%m-%dT%H:%M:%S.000Z") + + +class _ObservedRequest(BaseModel): + body: dict[str, JsonValue] + + +class _Observations(BaseModel): + requests: list[_ObservedRequest] + + +class _ProxyError(BaseModel): + type: str + + +class _ErrorResponse(BaseModel): + error: _ProxyError + + +class _BudgetRecord(BaseModel): + budget_id: str + max_budget: float | None = None + tpm_limit: int | None = None + rpm_limit: int | None = None + budget_duration: str | None = None + allowed_models: list[str] | None = None + temp_budget_increase: float | None = None + temp_budget_expiry: datetime | None = None + + +class _MemberBudgetState(BaseModel): + user_id: str + team_id: str + budget_id: str | None = None + spend: float | None = None + max_budget: float | None = None + tpm_limit: int | None = None + rpm_limit: int | None = None + budget_duration: str | None = None + allowed_models: list[str] | None = None + temp_budget_increase: float | None = None + temp_budget_expiry: datetime | None = None + + +class _TeamRosterMember(BaseModel): + role: str + user_id: str | None = None + user_email: str | None = None + + +class _TeamState(BaseModel): + team_id: str + models: list[str] | None = None + members_with_roles: tuple[_TeamRosterMember, ...] = () + metadata: dict[str, JsonValue] | None = None + + +class _TeamMembershipInfo(BaseModel): + user_id: str + team_id: str + budget_id: str | None = None + spend: float | None = None + budget_source: str | None = None + litellm_budget_table: _BudgetRecord | None = None + + +class _TeamInfo(BaseModel): + team_info: _TeamState + team_memberships: list[_TeamMembershipInfo] + + +class _MemberMe(BaseModel): + user_id: str + team_id: str + budget_id: str | None = None + role: str | None = None + litellm_budget_table: _BudgetRecord | None = None + + +class _MemberUpdateResponse(BaseModel): + team_id: str + user_id: str + user_email: str | None = None + max_budget_in_team: float | None = None + tpm_limit: int | None = None + rpm_limit: int | None = None + budget_duration: str | None = None + allowed_models: list[str] | None = None + temp_budget_increase: float | None = None + temp_budget_expiry: datetime | None = None + + +class _BulkBudgetResult(BaseModel): + user_id: str | None = None + user_email: str | None = None + success: bool + error: str | None = None + budget_id: str | None = None + max_budget: float | None = None + max_budget_source: str | None = None + tpm_limit: int | None = None + rpm_limit: int | None = None + budget_duration: str | None = None + allowed_models: tuple[str, ...] | None = None + + +class _BulkBudgetUpdateResponse(BaseModel): + data: tuple[_BulkBudgetResult, ...] + + +@pytest.fixture +def upstream(gateway: Gateway) -> Iterator[httpx.Client]: + with httpx.Client(base_url=gateway.upstream_url, timeout=15, trust_env=False) as client: + response: Final = client.get("/__observations") + assert response.status_code == 200, response.text + yield client + + +def _priced_model(scenario: Scenario) -> tuple[str, str]: + provider_model: Final = f"team-member-spend-{uuid.uuid4().hex}" + model: Final = scenario.model( + model=f"openai/{provider_model}", + input_cost_per_token=0.001, + output_cost_per_token=0.002, + ) + return model, provider_model + + +def _free_model(scenario: Scenario) -> tuple[str, str]: + provider_model: Final = f"team-member-free-{uuid.uuid4().hex}" + model: Final = scenario.model( + model=f"openai/{provider_model}", + input_cost_per_token=0, + output_cost_per_token=0, + ) + return model, provider_model + + +def _observed_models(upstream: httpx.Client) -> tuple[str, ...]: + response: Final = upstream.get("/__observations") + assert response.status_code == 200, response.text + observations: Final = _Observations.model_validate_json(response.text) + return tuple(string_value(request.body["model"]) for request in observations.requests) + + +def _chat(gateway: Gateway, model: str, key: str, text: str) -> httpx.Response: + return gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": text}]}, + key=key, + ) + + +def _team_info(gateway: Gateway, team_id: str) -> _TeamInfo: + response: Final = gateway.request("GET", "/team/info", params={"team_id": team_id}) + assert response.status_code == 200, response.text + return _TeamInfo.model_validate_json(response.text) + + +def _member_me(gateway: Gateway, team_id: str, key: str) -> _MemberMe: + response: Final = gateway.request("GET", f"/team/{team_id}/members/me", key=key) + assert response.status_code == 200, response.text + return _MemberMe.model_validate_json(response.text) + + +def _budget_state(team_id: str, user_id: str) -> tuple[_MemberBudgetState, ...]: + return tuple( + _MemberBudgetState.model_validate(row) + for row in read_rows( + "SELECT tm.user_id, tm.team_id, tm.budget_id, tm.spend, b.max_budget, b.tpm_limit, b.rpm_limit, " + "b.budget_duration, b.allowed_models, b.temp_budget_increase, " + "b.temp_budget_expiry::text AS temp_budget_expiry " + 'FROM "LiteLLM_TeamMembership" tm LEFT JOIN "LiteLLM_BudgetTable" b ON b.budget_id = tm.budget_id ' + "WHERE tm.team_id = %s AND tm.user_id = %s", + (team_id, user_id), + ) + ) + + +def _budget_record(budget_id: str) -> tuple[_BudgetRecord, ...]: + return tuple( + _BudgetRecord.model_validate(row) + for row in read_rows( + "SELECT budget_id, max_budget, tpm_limit, rpm_limit, budget_duration, allowed_models, " + 'temp_budget_increase, temp_budget_expiry FROM "LiteLLM_BudgetTable" WHERE budget_id = %s', + (budget_id,), + ) + ) + + +def _team_default_budget_id(team_id: str) -> str: + rows: Final = read_rows('SELECT metadata FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team_id,)) + assert len(rows) == 1, rows + metadata: Final = object_value(rows[0]["metadata"]) + return string_value(metadata["team_member_budget_id"]) + + +def _member_update(gateway: Gateway, body: dict[str, JsonValue]) -> _MemberUpdateResponse: + response: Final = gateway.request("POST", "/team/member_update", body) + assert response.status_code == 200, response.text + return _MemberUpdateResponse.model_validate_json(response.text) + + +def _budget_refused(response: httpx.Response) -> None: + assert response.status_code == 422, response.text + assert _ErrorResponse.model_validate_json(response.text).error.type == "budget_exceeded", response.text + + @pytest.mark.covers("spend.team_member.member_without_budget_gets_membership_row_and_spend") def test_member_added_without_any_budget_is_charged_on_its_membership_row(gateway: Gateway) -> None: with gateway.scenario() as scenario: @@ -97,3 +314,716 @@ def test_member_over_budget_is_blocked_when_redis_counter_reads_stale_low(gatewa assert denied.status_code == 422 and denied.json()["error"]["type"] == "budget_exceeded", denied.text assert upstream.get("/__observations").json()["requests"] == [] assert float(cache.get(counter_key)) == pytest.approx(0.06), denied.text + + +def test_member_update_lowering_the_budget_below_spend_blocks_the_next_call( + gateway: Gateway, peer: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + model, provider_model = _priced_model(scenario) + team: Final = scenario.team(models=[model]) + user: Final = scenario.user() + added: Final = gateway.request( + "POST", "/team/member_add", {"team_id": team, "member": {"user_id": user, "role": "user"}} + ) + assert added.status_code == 200, added.text + initial_budget: Final = _member_update( + gateway, + {"team_id": team, "user_id": user, "max_budget_in_team": 1}, + ) + assert initial_budget == _MemberUpdateResponse(team_id=team, user_id=user, max_budget_in_team=1), initial_budget + key: Final = scenario.key(team_id=team, user_id=user) + personal_key: Final = scenario.key(user_id=user) + + first: Final = _chat(gateway, model, key, f"accrued member spend {uuid.uuid4().hex}") + assert first.status_code == 200, first.text + eventually( + lambda: _budget_state(team, user), + lambda values: len(values) == 1 and values[0].spend is not None and values[0].spend >= 0.06, + seconds=70, + ) + peer_first: Final = _chat(peer, model, key, f"accrued member spend on peer {uuid.uuid4().hex}") + assert peer_first.status_code == 200, peer_first.text + observed_before_update: Final = _observed_models(upstream) + assert observed_before_update == (provider_model, provider_model), repr(observed_before_update) + accrued: Final = eventually( + lambda: _budget_state(team, user), + lambda values: len(values) == 1 and values[0].spend is not None and values[0].spend >= 0.12, + seconds=70, + ) + assert accrued[0].spend == pytest.approx(0.12), accrued + cached_membership: Final = _member_me(gateway, team, key) + assert cached_membership.litellm_budget_table is not None, repr(cached_membership) + assert cached_membership.litellm_budget_table.max_budget == 1, repr(cached_membership) + + updated: Final = _member_update( + gateway, + { + "team_id": team, + "user_id": user, + "max_budget_in_team": 0.0001, + "budget_duration": "30d", + "tpm_limit": 1000, + "rpm_limit": 100, + }, + ) + assert updated == _MemberUpdateResponse( + team_id=team, + user_id=user, + max_budget_in_team=0.0001, + budget_duration="30d", + tpm_limit=1000, + rpm_limit=100, + ), repr(updated) + denied: Final = _chat(gateway, model, key, f"lowered member budget {uuid.uuid4().hex}") + _budget_refused(denied) + peer_denied: Final = _chat(peer, model, key, f"lowered member budget on peer {uuid.uuid4().hex}") + _budget_refused(peer_denied) + observed_after_denial: Final = _observed_models(upstream) + assert observed_after_denial == (), repr(observed_after_denial) + assert observed_before_update + observed_after_denial == (provider_model, provider_model), observed_after_denial + + state: Final = _budget_state(team, user) + assert len(state) == 1, state + assert state[0].user_id == user, state + assert state[0].team_id == team, state + assert state[0].spend == pytest.approx(0.12), state + assert state[0].max_budget == 0.0001, state + assert state[0].tpm_limit == 1000, state + assert state[0].rpm_limit == 100, state + assert state[0].budget_duration == "30d", state + team_info: Final = _team_info(gateway, team) + listed: Final = next(member for member in team_info.team_memberships if member.user_id == user) + assert listed.budget_id == state[0].budget_id, repr(listed) + assert listed.litellm_budget_table is not None, repr(listed) + assert listed.litellm_budget_table.max_budget == 0.0001, repr(listed) + assert listed.litellm_budget_table.tpm_limit == 1000, repr(listed) + assert listed.litellm_budget_table.rpm_limit == 100, repr(listed) + assert listed.litellm_budget_table.budget_duration == "30d", repr(listed) + member_info: Final = _member_me(gateway, team, personal_key) + assert member_info.budget_id == state[0].budget_id, repr(member_info) + assert member_info.litellm_budget_table is not None, repr(member_info) + assert member_info.litellm_budget_table.max_budget == 0.0001, repr(member_info) + assert member_info.litellm_budget_table.tpm_limit == 1000, repr(member_info) + assert member_info.litellm_budget_table.rpm_limit == 100, repr(member_info) + assert member_info.litellm_budget_table.budget_duration == "30d", repr(member_info) + + +@pytest.mark.parametrize( + "spell_expiry", + [ + pytest.param(_python_isoformat, id="python-isoformat"), + pytest.param(_javascript_iso_string, id="javascript-toisostring"), + ], +) +@pytest.mark.parametrize( + "budget_source", + [ + pytest.param("own-budget", id="own-budget"), + pytest.param("team-default", id="team-default"), + ], +) +def test_member_update_temp_budget_increase_readmits_a_blocked_member( + gateway: Gateway, + upstream: httpx.Client, + spell_expiry: Callable[[datetime], str], + budget_source: str, +) -> None: + with gateway.scenario() as scenario: + model, provider_model = _priced_model(scenario) + team: Final = ( + scenario.team(models=[model], team_member_budget=0.05) + if budget_source == "team-default" + else scenario.team(models=[model]) + ) + user: Final = scenario.user() + member_add_body: Final = {"team_id": team, "member": {"user_id": user, "role": "user"}} + added: Final = gateway.request( + "POST", + "/team/member_add", + member_add_body | ({"max_budget_in_team": 0.05} if budget_source == "own-budget" else {}), + ) + assert added.status_code == 200, added.text + default_budget_id: Final = _team_default_budget_id(team) if budget_source == "team-default" else None + default_budget_before: Final = ( + _budget_record(default_budget_id) if default_budget_id is not None else () + ) + if default_budget_id is not None: + assert len(default_budget_before) == 1, repr(default_budget_before) + assert default_budget_before[0].max_budget == 0.05, default_budget_before + assert default_budget_before[0].temp_budget_increase is None, default_budget_before + assert default_budget_before[0].temp_budget_expiry is None, default_budget_before + initial_state: Final = _budget_state(team, user) + assert len(initial_state) == 1, initial_state + if budget_source == "own-budget": + assert initial_state[0].max_budget == 0.05, initial_state + else: + assert default_budget_id is not None + assert initial_state[0].budget_id == default_budget_id, initial_state + key: Final = scenario.key(team_id=team, user_id=user) + + first: Final = _chat(gateway, model, key, f"initial member budget {uuid.uuid4().hex}") + assert first.status_code == 200, first.text + observed_before_block: Final = _observed_models(upstream) + assert observed_before_block == (provider_model,), repr(observed_before_block) + accrued: Final = eventually( + lambda: _budget_state(team, user), + lambda values: len(values) == 1 and values[0].spend is not None and values[0].spend >= 0.06, + seconds=70, + ) + assert accrued[0].spend == pytest.approx(0.06), accrued + blocked: Final = _chat(gateway, model, key, f"exceeded member budget {uuid.uuid4().hex}") + _budget_refused(blocked) + observed_after_block: Final = _observed_models(upstream) + assert observed_after_block == (), repr(observed_after_block) + + expiry: Final = (datetime.now(timezone.utc) + timedelta(days=1)).replace(microsecond=0) + updated: Final = _member_update( + gateway, + { + "team_id": team, + "user_id": user, + "temp_budget_increase": 1.0, + "temp_budget_expiry": spell_expiry(expiry), + }, + ) + assert updated.temp_budget_expiry is not None, repr(updated) + assert updated.model_copy( + update={"temp_budget_expiry": updated.temp_budget_expiry.replace(tzinfo=timezone.utc)} + ) == _MemberUpdateResponse( + team_id=team, + user_id=user, + temp_budget_increase=1.0, + temp_budget_expiry=expiry, + ), repr(updated) + state: Final = _budget_state(team, user) + assert len(state) == 1, state + if budget_source == "own-budget": + assert state[0].max_budget == 0.05, state + else: + assert default_budget_id is not None + assert state[0].budget_id != default_budget_id, state + assert state[0].max_budget is None, state + default_budget_after: Final = _budget_record(default_budget_id) + assert default_budget_after == default_budget_before, repr(default_budget_after) + assert default_budget_after[0].max_budget == 0.05, default_budget_after + assert default_budget_after[0].temp_budget_increase is None, default_budget_after + assert default_budget_after[0].temp_budget_expiry is None, default_budget_after + assert state[0].temp_budget_increase == 1.0, state + assert state[0].temp_budget_expiry is not None, state + assert state[0].temp_budget_expiry.replace(tzinfo=timezone.utc) == expiry, state + team_info: Final = _team_info(gateway, team) + listed: Final = next(member for member in team_info.team_memberships if member.user_id == user) + assert listed.budget_id == state[0].budget_id, repr(listed) + assert listed.litellm_budget_table is not None, repr(listed) + assert listed.litellm_budget_table.budget_id == state[0].budget_id, repr(listed) + assert listed.litellm_budget_table.temp_budget_increase == 1.0, repr(listed) + assert listed.litellm_budget_table.temp_budget_expiry is not None, repr(listed) + assert listed.litellm_budget_table.temp_budget_expiry.replace(tzinfo=timezone.utc) == expiry, repr(listed) + member_info: Final = _member_me(gateway, team, key) + assert member_info.budget_id == state[0].budget_id, repr(member_info) + assert member_info.litellm_budget_table is not None, repr(member_info) + assert member_info.litellm_budget_table.budget_id == state[0].budget_id, repr(member_info) + assert member_info.litellm_budget_table.temp_budget_increase == 1.0, repr(member_info) + assert member_info.litellm_budget_table.temp_budget_expiry is not None, repr(member_info) + assert member_info.litellm_budget_table.temp_budget_expiry.replace(tzinfo=timezone.utc) == expiry, repr( + member_info + ) + + readmitted: Final = _chat(gateway, model, key, f"temporary member budget {uuid.uuid4().hex}") + assert readmitted.status_code == 200, readmitted.text + observed_after_readmission: Final = _observed_models(upstream) + assert observed_after_readmission == (provider_model,), repr(observed_after_readmission) + assert observed_before_block + observed_after_readmission == (provider_model, provider_model), repr( + (observed_before_block, observed_after_readmission) + ) + + +def test_member_rpm_limit_update_changes_runtime_admission(gateway: Gateway, upstream: httpx.Client) -> None: + pytest.skip("BUG: member_update leaves a warmed key using the old RPM limit") + with gateway.scenario() as scenario: + model, provider_model = _free_model(scenario) + team: Final = scenario.team(models=[model]) + user: Final = scenario.user() + added: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"user_id": user, "role": "user"}}, + ) + assert added.status_code == 200, added.text + key: Final = scenario.key(team_id=team, user_id=user) + + warm: Final = _chat(gateway, model, key, f"warm member key before rpm limit {uuid.uuid4().hex}") + assert warm.status_code == 200, warm.text + updated: Final = _member_update(gateway, {"team_id": team, "user_id": user, "rpm_limit": 1}) + assert updated.rpm_limit == 1, repr(updated) + + admitted: Final = _chat(gateway, model, key, f"member rpm admitted request {uuid.uuid4().hex}") + assert admitted.status_code == 200, admitted.text + limited: Final = _chat(gateway, model, key, f"member rpm limited request {uuid.uuid4().hex}") + assert limited.status_code == 429, limited.text + assert _ErrorResponse.model_validate_json(limited.text).error.type == "throttling_error", limited.text + observed: Final = _observed_models(upstream) + assert observed == (provider_model, provider_model), repr(observed) + + +def test_member_rpm_limit_update_changes_runtime_admission_for_cold_key( + gateway: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + model, provider_model = _free_model(scenario) + team: Final = scenario.team(models=[model]) + user: Final = scenario.user() + added: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"user_id": user, "role": "user"}}, + ) + assert added.status_code == 200, added.text + key: Final = scenario.key(team_id=team, user_id=user) + + updated: Final = _member_update(gateway, {"team_id": team, "user_id": user, "rpm_limit": 1}) + assert updated.rpm_limit == 1, repr(updated) + + admitted: Final = _chat(gateway, model, key, f"cold member rpm admitted request {uuid.uuid4().hex}") + assert admitted.status_code == 200, admitted.text + limited: Final = _chat(gateway, model, key, f"cold member rpm limited request {uuid.uuid4().hex}") + assert limited.status_code == 429, limited.text + assert _ErrorResponse.model_validate_json(limited.text).error.type == "throttling_error", limited.text + observed: Final = _observed_models(upstream) + assert observed == (provider_model,), repr(observed) + + +def test_member_tpm_limit_update_changes_runtime_admission(gateway: Gateway, upstream: httpx.Client) -> None: + pytest.skip("BUG: member_update leaves a warmed key using the old TPM limit") + with gateway.scenario() as scenario: + model, provider_model = _free_model(scenario) + team: Final = scenario.team(models=[model]) + user: Final = scenario.user() + added: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"user_id": user, "role": "user"}}, + ) + assert added.status_code == 200, added.text + key: Final = scenario.key(team_id=team, user_id=user) + + warm: Final = _chat(gateway, model, key, f"warm member key before tpm limit {uuid.uuid4().hex}") + assert warm.status_code == 200, warm.text + updated: Final = _member_update(gateway, {"team_id": team, "user_id": user, "tpm_limit": 30}) + assert updated.tpm_limit == 30, repr(updated) + + admitted: Final = _chat(gateway, model, key, f"first member tpm-limited request {uuid.uuid4().hex}") + assert admitted.status_code == 200, admitted.text + limited: Final = _chat(gateway, model, key, f"second member tpm-limited request {uuid.uuid4().hex}") + assert limited.status_code == 429, limited.text + assert _ErrorResponse.model_validate_json(limited.text).error.type == "throttling_error", limited.text + observed: Final = _observed_models(upstream) + assert observed == (provider_model, provider_model), repr(observed) + + +def test_member_tpm_limit_update_changes_runtime_admission_for_cold_key( + gateway: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + model, provider_model = _free_model(scenario) + team: Final = scenario.team(models=[model]) + user: Final = scenario.user() + added: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"user_id": user, "role": "user"}}, + ) + assert added.status_code == 200, added.text + key: Final = scenario.key(team_id=team, user_id=user) + + updated: Final = _member_update(gateway, {"team_id": team, "user_id": user, "tpm_limit": 30}) + assert updated.tpm_limit == 30, repr(updated) + + admitted: Final = _chat(gateway, model, key, f"cold member tpm admitted request {uuid.uuid4().hex}") + assert admitted.status_code == 200, admitted.text + limited: Final = _chat(gateway, model, key, f"cold member tpm limited request {uuid.uuid4().hex}") + assert limited.status_code == 429, limited.text + assert _ErrorResponse.model_validate_json(limited.text).error.type == "throttling_error", limited.text + observed: Final = _observed_models(upstream) + assert observed == (provider_model,), repr(observed) + + +def test_member_update_by_email_sets_models_role_and_forks_the_team_default_budget( + gateway: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + model_one, provider_one = _free_model(scenario) + model_two, provider_two = _free_model(scenario) + team: Final = scenario.team(models=[model_one, model_two], team_member_budget=25) + email_one: Final = f"member-{uuid.uuid4().hex}@example.com" + user_one: Final = scenario.user(user_email=email_one, user_role="internal_user") + user_two: Final = scenario.user(user_role="internal_user") + added: Final = gateway.request( + "POST", + "/team/member_add", + { + "team_id": team, + "member": [ + {"user_email": email_one, "role": "user"}, + {"user_id": user_two, "role": "user"}, + ], + }, + ) + assert added.status_code == 200, added.text + default_budget_id: Final = _team_default_budget_id(team) + default_budget_before: Final = _budget_record(default_budget_id) + assert len(default_budget_before) == 1, repr(default_budget_before) + initial_one: Final = _budget_state(team, user_one) + initial_two: Final = _budget_state(team, user_two) + assert len(initial_one) == 1 and initial_one[0].budget_id == default_budget_id, initial_one + assert len(initial_two) == 1 and initial_two[0].budget_id == default_budget_id, initial_two + + user_one_key: Final = scenario.key(team_id=team, user_id=user_one) + model_scope: Final = _member_update( + gateway, + {"team_id": team, "user_email": email_one, "allowed_models": [model_one]}, + ) + assert model_scope.user_id == user_one, repr(model_scope) + assert model_scope.allowed_models == [model_one], repr(model_scope) + scoped_state: Final = _budget_state(team, user_one) + assert len(scoped_state) == 1, scoped_state + assert scoped_state[0].allowed_models == [model_one], scoped_state + assert scoped_state[0].budget_id != default_budget_id, scoped_state + other_member_state: Final = _budget_state(team, user_two) + assert other_member_state[0].budget_id == default_budget_id, other_member_state + assert _budget_record(default_budget_id) == default_budget_before, repr(default_budget_before) + + allowed: Final = _chat(gateway, model_one, user_one_key, f"email allowed model {uuid.uuid4().hex}") + assert allowed.status_code == 200, allowed.text + denied: Final = _chat(gateway, model_two, user_one_key, f"email restricted model {uuid.uuid4().hex}") + assert denied.status_code == 403, denied.text + assert _ErrorResponse.model_validate_json(denied.text).error.type == "team_model_access_denied", denied.text + observed_before_clear: Final = _observed_models(upstream) + assert observed_before_clear == (provider_one,), repr(observed_before_clear) + + cleared_models: Final = _member_update( + gateway, + {"team_id": team, "user_email": email_one, "allowed_models": []}, + ) + assert cleared_models.user_id == user_one, repr(cleared_models) + assert cleared_models.allowed_models == [], repr(cleared_models) + cleared_state: Final = _budget_state(team, user_one) + assert len(cleared_state) == 1, cleared_state + assert cleared_state[0].allowed_models == [], cleared_state + unrestricted: Final = _chat(gateway, model_two, user_one_key, f"email cleared model scope {uuid.uuid4().hex}") + assert unrestricted.status_code == 200, unrestricted.text + observed_after_clear: Final = _observed_models(upstream) + assert observed_after_clear == (provider_two,), repr(observed_after_clear) + + budget_update: Final = _member_update( + gateway, + {"team_id": team, "user_email": email_one, "max_budget_in_team": 12}, + ) + assert budget_update.user_id == user_one, repr(budget_update) + assert budget_update.max_budget_in_team == 12, repr(budget_update) + user_one_state: Final = _budget_state(team, user_one) + user_two_state: Final = _budget_state(team, user_two) + assert len(user_one_state) == 1 and user_one_state[0].budget_id != default_budget_id, user_one_state + assert user_one_state[0].max_budget == 12, user_one_state + assert user_two_state == initial_two, user_two_state + assert _budget_record(default_budget_id) == default_budget_before, repr(default_budget_before) + team_info: Final = _team_info(gateway, team) + listed_one: Final = next(member for member in team_info.team_memberships if member.user_id == user_one) + assert listed_one.litellm_budget_table is not None, repr(listed_one) + assert listed_one.litellm_budget_table.max_budget == 12, repr(listed_one) + assert listed_one.litellm_budget_table.allowed_models == [], repr(listed_one) + member_info: Final = _member_me(gateway, team, user_one_key) + assert member_info.litellm_budget_table is not None, repr(member_info) + assert member_info.litellm_budget_table.max_budget == 12, repr(member_info) + assert member_info.litellm_budget_table.allowed_models == [], repr(member_info) + roster_member: Final = next( + member for member in team_info.team_info.members_with_roles if member.user_id == user_one + ) + assert roster_member.role == "user", repr(roster_member) + + new_member: Final = scenario.user() + user_key: Final = scenario.key(user_id=user_one) + denied_admin_action: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"user_id": new_member, "role": "user"}}, + key=user_key, + ) + assert denied_admin_action.status_code == 403, denied_admin_action.text + role_update: Final = _member_update( + gateway, + {"team_id": team, "user_email": email_one, "role": "admin"}, + ) + assert role_update.user_id == user_one, repr(role_update) + role_info: Final = _team_info(gateway, team) + admin_roster: Final = next( + member for member in role_info.team_info.members_with_roles if member.user_id == user_one + ) + assert admin_roster.role == "admin", repr(admin_roster) + roster_rows: Final = read_rows( + 'SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id = %s', + (team,), + ) + assert len(roster_rows) == 1, roster_rows + database_roster: Final = _TeamState.model_validate( + {"team_id": team, "members_with_roles": roster_rows[0]["members_with_roles"]} + ).members_with_roles + assert next(member for member in database_roster if member.user_id == user_one).role == "admin", repr( + database_roster + ) + allowed_admin_action: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"user_id": new_member, "role": "user"}}, + key=user_key, + ) + assert allowed_admin_action.status_code == 200, allowed_admin_action.text + assert observed_before_clear + observed_after_clear == (provider_one, provider_two), repr( + (observed_before_clear, observed_after_clear) + ) + + +def test_bulk_update_is_a_merge_patch_per_member(gateway: Gateway, upstream: httpx.Client) -> None: + with gateway.scenario() as scenario: + model_one, provider_one = _free_model(scenario) + model_two, _ = _free_model(scenario) + spend_model, provider_spend = _priced_model(scenario) + team: Final = scenario.team(models=[model_one, model_two, spend_model], team_member_budget=25) + user_one: Final = scenario.user() + email_two: Final = f"bulk-member-{uuid.uuid4().hex}@example.com" + user_two: Final = scenario.user(user_email=email_two, user_role="internal_user") + added: Final = gateway.request( + "POST", + "/team/member_add", + { + "team_id": team, + "member": [ + {"user_id": user_one, "role": "user"}, + {"user_email": email_two, "role": "user"}, + ], + }, + ) + assert added.status_code == 200, added.text + default_budget_id: Final = _team_default_budget_id(team) + default_budget_before: Final = _budget_record(default_budget_id) + key_one: Final = scenario.key(team_id=team, user_id=user_one) + key_two: Final = scenario.key(team_id=team, user_id=user_two) + personal_key_one: Final = scenario.key(user_id=user_one) + + first_chat: Final = _chat(gateway, spend_model, key_one, f"bulk member spend {uuid.uuid4().hex}") + assert first_chat.status_code == 200, first_chat.text + observed_before_update: Final = _observed_models(upstream) + assert observed_before_update == (provider_spend,), repr(observed_before_update) + accrued: Final = eventually( + lambda: _budget_state(team, user_one), + lambda values: len(values) == 1 and values[0].spend is not None and values[0].spend >= 0.06, + seconds=70, + ) + assert accrued[0].spend == pytest.approx(0.06), accrued + + update: Final = gateway.request( + "POST", + f"/management/v1/teams/{team}/members/bulk_update", + { + "members": [ + {"user_id": user_one, "max_budget_in_team": 0.0001, "rpm_limit": 7}, + {"user_email": email_two, "rpm_limit": 5, "allowed_models": [model_one]}, + ] + }, + ) + assert update.status_code == 200, update.text + update_response: Final = _BulkBudgetUpdateResponse.model_validate_json(update.text) + state_one: Final = _budget_state(team, user_one) + state_two: Final = _budget_state(team, user_two) + assert len(state_one) == 1 and len(state_two) == 1, (state_one, state_two) + assert state_one[0].budget_id is not None, state_one + assert state_two[0].budget_id is not None, state_two + assert state_one[0].budget_id != default_budget_id, state_one + assert state_two[0].budget_id != default_budget_id, state_two + assert update_response.data == ( + _BulkBudgetResult( + user_id=user_one, + success=True, + budget_id=state_one[0].budget_id, + max_budget=0.0001, + max_budget_source="member", + rpm_limit=7, + allowed_models=tuple(state_one[0].allowed_models or ()), + ), + _BulkBudgetResult( + user_id=user_two, + user_email=email_two, + success=True, + budget_id=state_two[0].budget_id, + max_budget=25, + max_budget_source="member", + rpm_limit=5, + allowed_models=(model_one,), + ), + ), update.text + assert state_one[0].max_budget == 0.0001 and state_one[0].rpm_limit == 7, state_one + assert state_two[0].max_budget == 25 and state_two[0].rpm_limit == 5, state_two + assert state_two[0].allowed_models == [model_one], state_two + assert _budget_record(default_budget_id) == default_budget_before, repr(default_budget_before) + info: Final = _team_info(gateway, team) + info_one: Final = next(member for member in info.team_memberships if member.user_id == user_one) + assert info_one.litellm_budget_table is not None, repr(info_one) + assert info_one.litellm_budget_table.max_budget == 0.0001, repr(info_one) + assert info_one.litellm_budget_table.rpm_limit == 7, repr(info_one) + me_one: Final = _member_me(gateway, team, personal_key_one) + assert me_one.litellm_budget_table is not None, repr(me_one) + assert me_one.litellm_budget_table.max_budget == 0.0001, repr(me_one) + assert me_one.litellm_budget_table.rpm_limit == 7, repr(me_one) + + over_budget: Final = _chat(gateway, spend_model, key_one, f"bulk member blocked {uuid.uuid4().hex}") + _budget_refused(over_budget) + observed_after_over_budget: Final = _observed_models(upstream) + assert observed_after_over_budget == (), repr(observed_after_over_budget) + allowed_two: Final = _chat(gateway, model_one, key_two, f"bulk member scoped allowed {uuid.uuid4().hex}") + assert allowed_two.status_code == 200, allowed_two.text + denied_two: Final = _chat(gateway, model_two, key_two, f"bulk member scoped denied {uuid.uuid4().hex}") + assert denied_two.status_code == 403, denied_two.text + assert _ErrorResponse.model_validate_json(denied_two.text).error.type == "team_model_access_denied", ( + denied_two.text + ) + observed_after_first_patch: Final = _observed_models(upstream) + assert observed_after_first_patch == (provider_one,), repr(observed_after_first_patch) + + cleared_max: Final = gateway.request( + "POST", + f"/management/v1/teams/{team}/members/bulk_update", + {"members": [{"user_id": user_one, "max_budget_in_team": None}]}, + ) + assert cleared_max.status_code == 200, cleared_max.text + cleared_max_response: Final = _BulkBudgetUpdateResponse.model_validate_json(cleared_max.text) + cleared_max_state: Final = _budget_state(team, user_one) + assert len(cleared_max_state) == 1, cleared_max_state + assert cleared_max_response.data == ( + _BulkBudgetResult( + user_id=user_one, + success=True, + budget_id=cleared_max_state[0].budget_id, + max_budget=25, + max_budget_source="team_default", + rpm_limit=7, + allowed_models=tuple(cleared_max_state[0].allowed_models or ()), + ), + ), cleared_max.text + assert cleared_max_state[0].budget_id == state_one[0].budget_id, cleared_max_state + assert cleared_max_state[0].max_budget is None, cleared_max_state + assert cleared_max_state[0].rpm_limit == 7, cleared_max_state + assert _budget_record(default_budget_id) == default_budget_before, repr(default_budget_before) + me_after_max_clear: Final = _member_me(gateway, team, personal_key_one) + assert me_after_max_clear.litellm_budget_table is not None, repr(me_after_max_clear) + assert me_after_max_clear.litellm_budget_table.max_budget is None, repr(me_after_max_clear) + assert me_after_max_clear.litellm_budget_table.rpm_limit == 7, repr(me_after_max_clear) + admitted_again: Final = _chat(gateway, spend_model, key_one, f"bulk member readmitted {uuid.uuid4().hex}") + assert admitted_again.status_code == 200, admitted_again.text + observed_after_max_clear: Final = _observed_models(upstream) + assert observed_after_max_clear == (provider_spend,), repr(observed_after_max_clear) + + assert observed_before_update + observed_after_first_patch + observed_after_max_clear == ( + provider_spend, + provider_one, + provider_spend, + ), repr((observed_before_update, observed_after_first_patch, observed_after_max_clear)) + + +def test_bulk_update_restores_team_default_when_last_member_limit_is_cleared( + gateway: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + model, provider_model = _priced_model(scenario) + team: Final = scenario.team(models=[model], team_member_budget=0.0001) + user: Final = scenario.user() + added: Final = gateway.request( + "POST", + "/team/member_add", + {"team_id": team, "member": {"user_id": user, "role": "user"}}, + ) + assert added.status_code == 200, added.text + default_budget_id: Final = _team_default_budget_id(team) + default_budget_before: Final = _budget_record(default_budget_id) + initial_state: Final = _budget_state(team, user) + assert len(initial_state) == 1 and initial_state[0].budget_id == default_budget_id, initial_state + key: Final = scenario.key(team_id=team, user_id=user, models=[model]) + increased: Final = gateway.request( + "POST", + f"/management/v1/teams/{team}/members/bulk_update", + {"members": [{"user_id": user, "max_budget_in_team": 5}]}, + ) + assert increased.status_code == 200, increased.text + increased_response: Final = _BulkBudgetUpdateResponse.model_validate_json(increased.text) + increased_state: Final = _budget_state(team, user) + assert len(increased_state) == 1, increased_state + assert increased_state[0].budget_id is not None and increased_state[0].budget_id != default_budget_id, ( + increased_state + ) + assert increased_response.data == ( + _BulkBudgetResult( + user_id=user, + success=True, + budget_id=increased_state[0].budget_id, + max_budget=5, + max_budget_source="member", + allowed_models=tuple(increased_state[0].allowed_models or ()), + ), + ), increased.text + first_chat: Final = _chat(gateway, model, key, f"member raised limit {uuid.uuid4().hex}") + assert first_chat.status_code == 200, first_chat.text + first_observations: Final = _observed_models(upstream) + assert first_observations == (provider_model,), repr(first_observations) + accrued: Final = eventually( + lambda: _budget_state(team, user), + lambda values: len(values) == 1 and values[0].spend is not None and values[0].spend >= 0.06, + seconds=70, + ) + assert accrued[0].spend == pytest.approx(0.06), accrued + second_chat: Final = _chat(gateway, model, key, f"member still under limit {uuid.uuid4().hex}") + assert second_chat.status_code == 200, second_chat.text + second_observations: Final = _observed_models(upstream) + assert second_observations == (provider_model,), repr(second_observations) + cleared: Final = gateway.request( + "POST", + f"/management/v1/teams/{team}/members/bulk_update", + {"members": [{"user_id": user, "max_budget_in_team": None}]}, + ) + assert cleared.status_code == 200, cleared.text + cleared_response: Final = _BulkBudgetUpdateResponse.model_validate_json(cleared.text) + final_state: Final = _budget_state(team, user) + assert len(final_state) == 1, final_state + assert final_state[0].budget_id is None, final_state + assert cleared_response.data == ( + _BulkBudgetResult( + user_id=user, + success=True, + budget_id=None, + max_budget=0.0001, + max_budget_source="team_default", + tpm_limit=None, + rpm_limit=None, + budget_duration=None, + allowed_models=None, + ), + ), cleared.text + assert read_rows( + 'SELECT budget_id FROM "LiteLLM_TeamMembership" WHERE team_id=%s AND user_id=%s', + (team, user), + ) == [{"budget_id": None}], cleared.text + assert _budget_record(default_budget_id) == default_budget_before, repr(default_budget_before) + refused: Final = _chat(gateway, model, key, f"member default cap reached {uuid.uuid4().hex}") + _budget_refused(refused) + observed_after_refusal: Final = _observed_models(upstream) + assert observed_after_refusal == (), repr(observed_after_refusal) + assert first_observations + second_observations == (provider_model, provider_model), repr( + (first_observations, second_observations) + ) + + +@pytest.fixture(autouse=True) +def _drain_unobserved_requests(gateway: Gateway) -> Iterator[None]: + yield + with httpx.Client(base_url=gateway.upstream_url, timeout=15, trust_env=False) as client: + response: Final = client.get("/__observations") + assert response.status_code == 200, response.text From e226016aee27d249d88bea15396a42ac30c8ad2d Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:34:22 +0000 Subject: [PATCH 04/51] test(integration): cover health_routing endpoint contracts (#44744) * test(integration): cover health_routing endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): tighten health_routing contracts per review Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): close health_routing review gaps Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(integration): format observed routing fallback cells Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): close health_routing final review findings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): resolve health_routing review findings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management/test_health_routing.py | 267 ++++ .../management/test_model_health_check.py | 1092 ++++++++++++++++- .../routing/test_auto_router_management.py | 636 ++++++++++ .../routing/test_observed_routing.py | 588 ++++++++- .../streaming/test_stream_contracts.py | 384 +++++- 5 files changed, 2944 insertions(+), 23 deletions(-) create mode 100644 tests/integration/management/test_health_routing.py create mode 100644 tests/integration/routing/test_auto_router_management.py diff --git a/tests/integration/management/test_health_routing.py b/tests/integration/management/test_health_routing.py new file mode 100644 index 00000000000..5939d066201 --- /dev/null +++ b/tests/integration/management/test_health_routing.py @@ -0,0 +1,267 @@ +import hashlib +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, eventually +from integration._support.database import scratch_database +from integration._support.database_relay import database_relay, held_statement_relay +from integration._support.process import owned_proxy + +_READINESS_PROBE_QUERY: Final = b"SELECT 1" + + +def _health_routing_config(directory: Path, general_settings: dict[str, object]) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["general_settings"] = {**config.get("general_settings", {}), **general_settings} + path: Final = directory / "health-routing.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +def _plaintext_database_url(database_url: str) -> str: + parts: Final = urlsplit(database_url) + parameters: Final = tuple( + (name, value) for name, value in parse_qsl(parts.query, keep_blank_values=True) if name != "sslmode" + ) + (("sslmode", "disable"),) + return urlunsplit(parts._replace(query=urlencode(parameters))) + + +def _assert_liveness_and_readiness(gateway: Gateway, *, status_code: int, body: dict[str, str]) -> None: + for path in ("/health/liveliness", "/health/liveness"): + response: Final = gateway.client.get(path) + assert response.status_code == status_code, response.text + assert response.json() == ("I'm alive!" if status_code == 200 else body), response.text + readiness: Final = gateway.client.get("/health/readiness") + assert readiness.status_code == status_code, readiness.text + assert readiness.json() == body, readiness.text + + +@pytest.mark.timeout(180) +def test_public_readiness_reports_database_outage_and_allow_unavailable_setting( + gateway: Gateway, tmp_path: Path +) -> None: + with scratch_database() as database_url: + with database_relay(database_url, _READINESS_PROBE_QUERY) as (relay, relayed_url): + with owned_proxy( + gateway, + tmp_path, + { + "DATABASE_URL": _plaintext_database_url(relayed_url), + "PRISMA_HEALTH_WATCHDOG_ENABLED": "false", + }, + workers=1, + ) as candidate: + ready_before_outage: Final = candidate.client.get("/health/readiness") + assert ready_before_outage.status_code == 200, ready_before_outage.text + assert ready_before_outage.json() == {"status": "healthy", "db": "connected"}, ready_before_outage.text + relay.arm() + disconnected_after_outage: Final = eventually( + lambda: candidate.client.get("/health/readiness"), + lambda response: ( + relay.tripped.is_set() and response.json() == {"status": "healthy", "db": "disconnected"} + ), + seconds=45, + return_last_on_timeout=True, + ) + assert relay.tripped.is_set(), "readiness database probe did not reach the relay" + assert disconnected_after_outage.status_code == 503, disconnected_after_outage.text + assert disconnected_after_outage.json() == { + "status": "healthy", + "db": "disconnected", + }, disconnected_after_outage.text + + allow_unavailable_config: Final = _health_routing_config( + tmp_path, + {"allow_requests_on_db_unavailable": True}, + ) + with database_relay(database_url, _READINESS_PROBE_QUERY) as (relay, relayed_url): + with owned_proxy( + gateway, + tmp_path, + { + "DATABASE_URL": _plaintext_database_url(relayed_url), + "PRISMA_HEALTH_WATCHDOG_ENABLED": "false", + }, + config=allow_unavailable_config, + workers=1, + ) as candidate: + ready_before_outage_with_allow: Final = candidate.client.get("/health/readiness") + assert ready_before_outage_with_allow.status_code == 200, ready_before_outage_with_allow.text + assert ready_before_outage_with_allow.json() == {"status": "healthy", "db": "connected"}, ( + ready_before_outage_with_allow.text + ) + relay.arm() + disconnected_with_allow: Final = eventually( + lambda: candidate.client.get("/health/readiness"), + lambda response: ( + relay.tripped.is_set() and response.json() == {"status": "healthy", "db": "disconnected"} + ), + seconds=45, + return_last_on_timeout=True, + ) + assert relay.tripped.is_set(), "allow-unavailable readiness probe did not reach the relay" + assert disconnected_with_allow.status_code == 200, disconnected_with_allow.text + assert disconnected_with_allow.json() == { + "status": "healthy", + "db": "disconnected", + }, disconnected_with_allow.text + + +@pytest.mark.timeout(300) +def test_public_readiness_reports_stalled_database_lookups(gateway: Gateway, tmp_path: Path) -> None: + with scratch_database() as database_url: + for allow_unavailable in (False, True): + unknown_key = "sk-" + uuid.uuid4().hex + lookup_hash = hashlib.sha256(unknown_key.encode()).hexdigest().encode() + config = ( + _health_routing_config(tmp_path, {"allow_requests_on_db_unavailable": True}) + if allow_unavailable + else None + ) + with held_statement_relay(database_url, lookup_hash) as (relay, relayed_url): + overrides = { + "DATABASE_URL": _plaintext_database_url(relayed_url), + "PRISMA_HEALTH_WATCHDOG_ENABLED": "false", + "PROXY_DB_LOOKUP_DEADLINE_SECONDS": "1", + "PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS": "15", + } + with owned_proxy( + gateway, + tmp_path, + overrides, + config=config, + workers=1, + ) as candidate: + expected_request_status = 400 if allow_unavailable else 503 + unknown_model_message = ( + "/chat/completions: Invalid model name passed in model=unconfigured-stalled-readiness-model. " + "Call `/v1/models` to view available models for your key." + ) + expected_request_body = ( + { + "error": { + "message": unknown_model_message, + "type": "invalid_request_error", + "param": None, + "code": "400", + "provider_specific_fields": {"error": unknown_model_message}, + } + } + if allow_unavailable + else { + "error": { + "message": ( + "Service Unavailable, the authentication database is temporarily unreachable. " + "Please retry shortly." + ), + "type": "no_db_connection", + "param": "None", + "code": "503", + } + } + ) + with ThreadPoolExecutor(max_workers=1) as executor: + pending = executor.submit( + candidate.request, + "POST", + "/v1/chat/completions", + { + "model": "unconfigured-stalled-readiness-model", + "messages": [{"role": "user", "content": "stalled lookup control"}], + }, + key=unknown_key, + ) + assert relay.held.wait(30), "unknown-key lookup did not reach the held statement relay" + expected_status = 200 if allow_unavailable else 503 + stalled = eventually( + lambda: candidate.client.get("/health/readiness"), + lambda response, expected_status=expected_status: ( + response.status_code == expected_status + and response.json() == {"status": "healthy", "db": "stalled"} + ), + seconds=5, + return_last_on_timeout=True, + ) + assert stalled.status_code == expected_status, stalled.text + assert stalled.json() == {"status": "healthy", "db": "stalled"}, stalled.text + + relay.release() + recovered = eventually( + lambda: candidate.client.get("/health/readiness"), + lambda response: ( + response.status_code == 200 + and response.json() == {"status": "healthy", "db": "connected"} + ), + seconds=30, + return_last_on_timeout=True, + ) + assert recovered.status_code == 200, recovered.text + assert recovered.json() == {"status": "healthy", "db": "connected"}, recovered.text + + rejected = pending.result(timeout=60) + assert (rejected.status_code, JSON_OBJECT.validate_json(rejected.content)) == ( + expected_request_status, + expected_request_body, + ), rejected.text + + +@pytest.mark.parametrize("token_source", ("config", "environment")) +def test_drain_endpoint_requires_token_and_changes_health_state_only_after_authorization( + token_source: str, gateway: Gateway, tmp_path: Path +) -> None: + disabled: Final = gateway.client.get("/health/drain") + assert disabled.status_code == 404, disabled.text + assert disabled.json() == {"detail": "Not Found"}, disabled.text + + token: Final = "synthetic-health-drain-token" + general_settings: Final = { + "enable_drain_endpoint": True, + **({"drain_endpoint_token": token} if token_source == "config" else {}), + } + config: Final = _health_routing_config(tmp_path, general_settings) + overrides: Final = {"DRAIN_ENDPOINT_TOKEN": token} if token_source == "environment" else {} + remove_environment: Final = ("DRAIN_ENDPOINT_TOKEN",) if token_source == "config" else () + with owned_proxy( + gateway, + tmp_path, + overrides, + config=config, + workers=1, + remove_environment=remove_environment, + ) as candidate: + _assert_liveness_and_readiness( + candidate, + status_code=200, + body={"status": "healthy", "db": "connected"}, + ) + + missing: Final = candidate.client.get("/health/drain") + assert missing.status_code == 401, missing.text + assert missing.json() == {"detail": "Invalid or missing X-Drain-Token"}, missing.text + wrong: Final = candidate.client.get( + "/health/drain", + headers={"X-Drain-Token": "synthetic-wrong-token"}, + ) + assert wrong.status_code == 401, wrong.text + assert wrong.json() == {"detail": "Invalid or missing X-Drain-Token"}, wrong.text + _assert_liveness_and_readiness( + candidate, + status_code=200, + body={"status": "healthy", "db": "connected"}, + ) + + drained: Final = candidate.client.get( + "/health/drain", + headers={"X-Drain-Token": token}, + ) + assert drained.status_code == 200, drained.text + assert drained.json() == {"status": "drained", "drained_requests": 0}, drained.text + for path in ("/health/liveliness", "/health/liveness", "/health/readiness"): + response: Final = candidate.client.get(path) + assert response.status_code == 503, response.text + assert response.json() == {"status": "shutting_down"}, response.text diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 8d77c7d1094..2b9ec658d87 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -1,11 +1,20 @@ +import json import os import uuid +from email.message import Message +from email.parser import BytesParser +from email.policy import HTTP +from pathlib import Path from typing import Final import httpx import pytest -from integration._support.client import Gateway, object_value, string_value +import yaml +from integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, object_value, string_value +from integration._support.database import scratch_database +from integration._support.process import owned_proxy from integration._support.upstream import ScenarioHandle, delete_scenario, register_scenario +from integration._support.wire import Reply, Request, Wire, wire_server from integration.cost_calculation.cost_tracking_case import JsonResponse from pydantic import JsonValue @@ -30,7 +39,12 @@ _VLLM_PROBE_REPLY: Final = JsonResponse( body={ "model": "Qwen/Qwen3-0.6B", "answers": { - "reachable": {"type": "choice", "choice": "yes", "confidence": 1.0, "probabilities": {"yes": 1.0, "no": 0.0}} + "reachable": { + "type": "choice", + "choice": "yes", + "confidence": 1.0, + "probabilities": {"yes": 1.0, "no": 0.0}, + } }, "usage": {"input_tokens": 10, "output_tokens": 1}, }, @@ -47,6 +61,37 @@ _CONFIGURED_STATE: Final[dict[str, JsonValue]] = {"ticket": "health probe"} _CONFIGURED_QUESTIONS: Final[dict[str, JsonValue]] = { "alive": {"type": "choice", "criteria": {"yes": "the service answers", "no": "the service is down"}} } +_HEALTH_CHAT_MESSAGES: Final = ( + [{"role": "user", "content": "Hey how's it going?"}], + [{"role": "user", "content": "What's 1 + 1?"}], +) + + +def _multipart_parts(request: Request) -> tuple[Message, ...]: + envelope: Final = f"content-type: {request.headers['content-type']}\r\n\r\n".encode() + request.body + parsed: Final = BytesParser(policy=HTTP).parsebytes(envelope) + assert parsed.is_multipart(), request.headers["content-type"] + return tuple(parsed.iter_parts()) + + +def _multipart_text_fields(parts: tuple[Message, ...]) -> dict[str, str]: + return { + string_value(part.get_param("name", header="content-disposition")): part.get_payload(decode=True).decode() + for part in parts + if part.get_filename() is None + } + + +def _multipart_file_fields(parts: tuple[Message, ...]) -> dict[str, tuple[str | None, str | None, bytes]]: + return { + string_value(part.get_param("name", header="content-disposition")): ( + part.get_filename(), + part.get_content_type(), + part.get_payload(decode=True), + ) + for part in parts + if part.get_filename() is not None + } def test_health_check_of_a_model_added_through_the_api_calls_its_upstream_and_reports_it_healthy( @@ -84,6 +129,133 @@ def _health_report(gateway: Gateway, model: str) -> dict[str, JsonValue]: return health.json() +def _background_health_reply(request: Request, *, model: str, api_key: str) -> Reply: + if request.method == "GET": + assert request.target == "/v1/models", request.target + assert request.headers["authorization"] == f"Bearer {api_key}", request.headers + return Reply( + body=json.dumps( + { + "object": "list", + "data": [{"id": model, "object": "model", "created": 1700000000, "owned_by": "integration"}], + } + ).encode() + ) + return _health_chat_reply(request, api_key=api_key, model=model) + + +def _assert_background_health_requests(wire: Wire, *, model: str, api_key: str) -> None: + requests: Final = wire.drain() + assert len(requests) == 1, requests + assert ( + requests[0].method, + requests[0].target, + requests[0].headers["authorization"], + JSON_OBJECT.validate_json(requests[0].body), + ) in ( + ( + "POST", + "/v1/chat/completions", + f"Bearer {api_key}", + {"model": model, "messages": _HEALTH_CHAT_MESSAGES[0], "max_tokens": 16}, + ), + ( + "POST", + "/v1/chat/completions", + f"Bearer {api_key}", + {"model": model, "messages": _HEALTH_CHAT_MESSAGES[1], "max_tokens": 16}, + ), + ), requests[0].body.decode() + + +def test_background_health_results_are_scoped_to_the_key_models(gateway: Gateway, tmp_path: Path) -> None: + public_a: Final = f"background-health-a-{uuid.uuid4().hex}" + public_b: Final = f"background-health-b-{uuid.uuid4().hex}" + provider_model_a: Final = f"openai/background-health-a-{uuid.uuid4().hex}" + provider_model_b: Final = f"openai/background-health-b-{uuid.uuid4().hex}" + wire_model_a: Final = provider_model_a.removeprefix("openai/") + wire_model_b: Final = provider_model_b.removeprefix("openai/") + model_id_a: Final = f"background-health-a-{uuid.uuid4().hex}" + model_id_b: Final = f"background-health-b-{uuid.uuid4().hex}" + api_key_a: Final = "synthetic-background-health-a" + api_key_b: Final = "synthetic-background-health-b" + + with ( + scratch_database() as database_url, + wire_server(lambda request: _background_health_reply(request, model=wire_model_a, api_key=api_key_a)) as wire_a, + wire_server(lambda request: _background_health_reply(request, model=wire_model_b, api_key=api_key_b)) as wire_b, + ): + base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config: Final = { + **base, + "general_settings": { + **base["general_settings"], + "background_health_checks": True, + "health_check_interval": 3600, + }, + "model_list": [ + { + "model_name": public_a, + "litellm_params": { + "model": provider_model_a, + "api_base": f"{wire_a.url}/v1", + "api_key": api_key_a, + }, + "model_info": {"id": model_id_a}, + }, + { + "model_name": public_b, + "litellm_params": { + "model": provider_model_b, + "api_base": f"{wire_b.url}/v1", + "api_key": api_key_b, + }, + "model_info": {"id": model_id_b}, + }, + ], + } + config_path: Final = tmp_path / "background_health.yaml" + config_path.write_text(yaml.safe_dump(config)) + + with owned_proxy(gateway, tmp_path, {"DATABASE_URL": database_url}, config=config_path) as candidate: + with candidate.scenario() as scenario: + key: Final = scenario.key(models=[public_a]) + scoped_health: Final = eventually( + lambda: candidate.request("GET", "/health", key=key), + lambda response: ( + response.status_code == 200 + and any( + endpoint.get("model_id") == model_id_a + for endpoint in response.json().get("healthy_endpoints", []) + ) + ), + seconds=70, + return_last_on_timeout=True, + ) + assert scoped_health.status_code == 200, scoped_health.text + assert JSON_OBJECT.validate_json(scoped_health.content) == { + "healthy_endpoints": [{"model": provider_model_a, "model_id": model_id_a}], + "unhealthy_endpoints": [], + "healthy_count": 1, + "unhealthy_count": 0, + }, scoped_health.text + + admin_health: Final = candidate.request("GET", "/health") + assert admin_health.status_code == 200, admin_health.text + assert JSON_OBJECT.validate_json(admin_health.content) == { + "healthy_endpoints": [ + {"model": provider_model_a, "api_base": f"{wire_a.url}/v1", "model_id": model_id_a}, + {"model": provider_model_b, "api_base": f"{wire_b.url}/v1", "model_id": model_id_b}, + ], + "unhealthy_endpoints": [], + "healthy_count": 2, + "unhealthy_count": 0, + }, admin_health.text + + _assert_background_health_requests(wire_a, model=wire_model_a, api_key=api_key_a) + _assert_background_health_requests(wire_b, model=wire_model_b, api_key=api_key_b) + + def _probes_sent_to(gateway: Gateway, handle: ScenarioHandle) -> list[tuple[str, JsonValue]]: with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream: requests: Final = upstream.get("/__observations").json()["requests"] @@ -161,6 +333,918 @@ def test_decisions_mode_health_check_of_the_self_hosted_strands_model_resolves_t ] +def _health_chat_reply(request: Request, *, api_key: str, model: str, status: int = 200) -> Reply: + assert request.method == "POST" + assert request.target == "/v1/chat/completions" + assert request.headers["authorization"] == f"Bearer {api_key}" + body: Final = JSON_OBJECT.validate_json(request.body) + assert body in ( + {"model": model, "messages": _HEALTH_CHAT_MESSAGES[0], "max_tokens": 16}, + {"model": model, "messages": _HEALTH_CHAT_MESSAGES[1], "max_tokens": 16}, + ), request.body.decode() + if status != 200: + return Reply(status=status, body=b'{"error":{"message":"synthetic health failure"}}') + return Reply( + body=json.dumps( + { + "id": "chatcmpl-health", + "object": "chat.completion", + "created": 1700000000, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "healthy"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ).encode(), + ) + + +def _create_team_health_deployment( + gateway: Gateway, + scenario: Scenario, + *, + team_id: str, + public_model_name: str, + upstream_url: str, + api_key: str, + access_group: str | None = None, + api_version: str | None = None, + aws_bedrock_runtime_endpoint: str | None = None, +) -> str: + created: Final = gateway.post( + "/model/new", + { + "model_name": public_model_name, + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": api_key, + "api_base": f"{upstream_url}/v1", + **({"api_version": api_version} if api_version is not None else {}), + **( + {"aws_bedrock_runtime_endpoint": aws_bedrock_runtime_endpoint} + if aws_bedrock_runtime_endpoint is not None + else {} + ), + }, + "model_info": { + "team_id": team_id, + **({"access_groups": [access_group]} if access_group is not None else {}), + }, + }, + ) + model_info: Final = object_value(created["model_info"]) + identity: Final = string_value(model_info["id"]) + assert model_info["team_id"] == team_id, created + assert model_info["team_public_model_name"] == public_model_name, created + scenario.cleanups.callback(scenario.delete_model, identity) + return identity + + +def _health_failure_raw_request(endpoint: dict[str, JsonValue], upstream_url: str) -> dict[str, JsonValue]: + raw_request: Final = object_value(endpoint["raw_request_typed_dict"]) + raw_request_body: Final = object_value(raw_request["raw_request_body"]) + expected_request_bodies: Final = tuple( + { + "model": "gpt-4o-mini", + "messages": messages, + "max_tokens": 16, + "extra_body": {}, + } + for messages in _HEALTH_CHAT_MESSAGES + ) + assert raw_request == { + "raw_request_api_base": f"{upstream_url}/v1/", + "raw_request_body": raw_request_body, + "raw_request_headers": {"Authorization": "Be****ey"}, + "error": None, + }, raw_request + assert raw_request_body in expected_request_bodies, raw_request + return raw_request + + +def test_team_scoped_health_probes_access_group_targets_and_hides_routing_fields(gateway: Gateway) -> None: + with ( + wire_server(lambda request: _health_chat_reply(request, api_key="team-a-key", model="gpt-4o-mini")) as healthy, + wire_server( + lambda request: _health_chat_reply(request, api_key="team-a-key", model="gpt-4o-mini", status=500) + ) as unhealthy, + wire_server( + lambda request: _health_chat_reply(request, api_key="team-b-key", model="gpt-4o-mini") + ) as team_b_wire, + gateway.scenario() as scenario, + ): + public_a: Final = f"team-a-{uuid.uuid4().hex}" + public_b: Final = f"team-b-{uuid.uuid4().hex}" + access_group: Final = f"health-group-{uuid.uuid4().hex}" + team_a: Final = scenario.team(models=[public_a, public_b, access_group]) + team_b_id: Final = scenario.team(models=[public_b]) + user_id: Final = scenario.member(team_a) + healthy_id: Final = _create_team_health_deployment( + gateway, + scenario, + team_id=team_a, + public_model_name=public_a, + upstream_url=healthy.url, + api_key="team-a-key", + access_group=access_group, + api_version="2024-10-21", + aws_bedrock_runtime_endpoint=f"{healthy.url}/runtime", + ) + unhealthy_id: Final = _create_team_health_deployment( + gateway, + scenario, + team_id=team_a, + public_model_name=public_a, + upstream_url=unhealthy.url, + api_key="team-a-key", + access_group=access_group, + api_version="2024-10-21", + aws_bedrock_runtime_endpoint=f"{unhealthy.url}/runtime", + ) + team_b_model_id: Final = _create_team_health_deployment( + gateway, + scenario, + team_id=team_b_id, + public_model_name=public_b, + upstream_url=team_b_wire.url, + api_key="team-b-key", + ) + team_key: Final = scenario.key( + team_id=team_a, + user_id=user_id, + models=[public_a, public_b], + ) + access_group_key: Final = scenario.key( + team_id=team_a, + user_id=user_id, + models=[access_group], + ) + + by_other_team_name: Final = gateway.request("GET", "/health", key=team_key, params={"model": public_b}) + assert by_other_team_name.status_code == 403, by_other_team_name.text + assert by_other_team_name.json() == { + "detail": {"error": f"key not allowed to health-check model {public_b}"} + }, by_other_team_name.text + by_other_team_id: Final = gateway.request("GET", "/health", key=team_key, params={"model_id": team_b_model_id}) + assert by_other_team_id.status_code == 403, by_other_team_id.text + assert by_other_team_id.json() == { + "detail": {"error": f"key not allowed to health-check model_id {team_b_model_id}"} + }, by_other_team_id.text + assert team_b_wire.drain() == () + + public_model_report: Final = gateway.request("GET", "/health", key=team_key, params={"model": public_a}) + assert public_model_report.status_code == 200, public_model_report.text + public_model_body: Final = public_model_report.json() + healthy_endpoint: Final = { + "model": "openai/gpt-4o-mini", + "model_id": healthy_id, + } + public_model_unhealthy_endpoint: Final = object_value(public_model_body["unhealthy_endpoints"][0]) + public_model_error: Final = string_value(public_model_unhealthy_endpoint["error"]) + assert "synthetic health failure" in public_model_error, public_model_report.text + for endpoint in (*public_model_body["healthy_endpoints"], *public_model_body["unhealthy_endpoints"]): + assert { + "api_base", + "api_version", + "aws_bedrock_runtime_endpoint", + }.isdisjoint(endpoint), public_model_report.text + assert { + key: value for key, value in public_model_unhealthy_endpoint.items() if key != "raw_request_typed_dict" + } == { + "model": "openai/gpt-4o-mini", + "model_id": unhealthy_id, + "error": public_model_error, + "exception_status": 500, + }, public_model_report.text + assert public_model_body == { + "healthy_endpoints": [healthy_endpoint], + "unhealthy_endpoints": [public_model_unhealthy_endpoint], + "healthy_count": 1, + "unhealthy_count": 1, + }, public_model_report.text + assert len(healthy.drain()) == 1 + assert len(unhealthy.drain()) == 3 + + expanded_group_report: Final = gateway.request("GET", "/health", key=access_group_key) + assert expanded_group_report.status_code == 200, expanded_group_report.text + expanded_group_body: Final = expanded_group_report.json() + expanded_group_unhealthy_endpoint: Final = object_value(expanded_group_body["unhealthy_endpoints"][0]) + expanded_group_error: Final = string_value(expanded_group_unhealthy_endpoint["error"]) + assert "synthetic health failure" in expanded_group_error, expanded_group_report.text + assert { + key: value for key, value in expanded_group_unhealthy_endpoint.items() if key != "raw_request_typed_dict" + } == { + "model": "openai/gpt-4o-mini", + "model_id": unhealthy_id, + "error": expanded_group_error, + "exception_status": 500, + }, expanded_group_report.text + assert expanded_group_body == { + "healthy_endpoints": [healthy_endpoint], + "unhealthy_endpoints": [expanded_group_unhealthy_endpoint], + "healthy_count": 1, + "unhealthy_count": 1, + }, expanded_group_report.text + for endpoint in (*expanded_group_body["healthy_endpoints"], *expanded_group_body["unhealthy_endpoints"]): + assert { + "api_base", + "api_version", + "aws_bedrock_runtime_endpoint", + }.isdisjoint(endpoint), expanded_group_report.text + assert len(healthy.drain()) == 1 + assert len(unhealthy.drain()) == 3 + assert team_b_wire.drain() == () + + targeted: Final = gateway.request( + "GET", + "/health", + key=team_key, + params={"model": public_b, "model_id": unhealthy_id}, + ) + assert targeted.status_code == 503, targeted.text + targeted_body: Final = targeted.json() + targeted_endpoint: Final = object_value(targeted_body["unhealthy_endpoints"][0]) + targeted_error: Final = string_value(targeted_endpoint["error"]) + assert "synthetic health failure" in targeted_error, targeted.text + assert {key: value for key, value in targeted_endpoint.items() if key != "raw_request_typed_dict"} == { + "model": "openai/gpt-4o-mini", + "model_id": unhealthy_id, + "error": targeted_error, + "exception_status": 500, + }, targeted.text + assert targeted_body == { + "healthy_endpoints": [], + "unhealthy_endpoints": [targeted_endpoint], + "healthy_count": 0, + "unhealthy_count": 1, + }, targeted.text + assert healthy.drain() == () + assert len(unhealthy.drain()) == 3 + assert team_b_wire.drain() == () + + admin: Final = gateway.request( + "GET", + "/health", + params={"model": public_b, "model_id": unhealthy_id}, + ) + assert admin.status_code == 503, admin.text + admin_body: Final = admin.json() + admin_endpoint: Final = object_value(admin_body["unhealthy_endpoints"][0]) + admin_error: Final = string_value(admin_endpoint["error"]) + admin_raw_request: Final = _health_failure_raw_request(admin_endpoint, unhealthy.url) + assert "synthetic health failure" in admin_error, admin.text + assert admin_body == { + "healthy_endpoints": [], + "unhealthy_endpoints": [ + { + "model": "openai/gpt-4o-mini", + "api_base": f"{unhealthy.url}/v1", + "api_version": "2024-10-21", + "aws_bedrock_runtime_endpoint": f"{unhealthy.url}/runtime", + "model_id": unhealthy_id, + "error": admin_error, + "raw_request_typed_dict": admin_raw_request, + "exception_status": 500, + } + ], + "healthy_count": 0, + "unhealthy_count": 1, + }, admin.text + assert len(unhealthy.drain()) == 3 + + unrestricted_team: Final = scenario.team(models=[]) + unrestricted_user: Final = scenario.member(unrestricted_team) + unrestricted_key: Final = scenario.key( + team_id=unrestricted_team, + user_id=unrestricted_user, + models=[], + ) + shared_public_name: Final = f"team-shared-{uuid.uuid4().hex}" + unrestricted_deployment_id: Final = _create_team_health_deployment( + gateway, + scenario, + team_id=unrestricted_team, + public_model_name=shared_public_name, + upstream_url=healthy.url, + api_key="team-a-key", + ) + _create_team_health_deployment( + gateway, + scenario, + team_id=team_b_id, + public_model_name=shared_public_name, + upstream_url=team_b_wire.url, + api_key="team-b-key", + ) + refused_unrestricted: Final = gateway.request( + "GET", + "/health", + key=unrestricted_key, + params={"model_id": team_b_model_id}, + ) + assert refused_unrestricted.status_code == 403, refused_unrestricted.text + assert refused_unrestricted.json() == { + "detail": {"error": f"key not allowed to health-check model_id {team_b_model_id}"} + }, refused_unrestricted.text + assert team_b_wire.drain() == () + + unrestricted_report: Final = gateway.request( + "GET", + "/health", + key=unrestricted_key, + params={"model": shared_public_name}, + ) + assert unrestricted_report.status_code == 200, unrestricted_report.text + unrestricted_body: Final = unrestricted_report.json() + assert unrestricted_body == { + "healthy_endpoints": [ + { + "model": "openai/gpt-4o-mini", + "model_id": unrestricted_deployment_id, + } + ], + "unhealthy_endpoints": [], + "healthy_count": 1, + "unhealthy_count": 0, + }, unrestricted_report.text + endpoint_ids: Final = tuple( + endpoint["model_id"] + for endpoint in (*unrestricted_body["healthy_endpoints"], *unrestricted_body["unhealthy_endpoints"]) + ) + assert team_b_model_id not in endpoint_ids, unrestricted_report.text + for endpoint in (*unrestricted_body["healthy_endpoints"], *unrestricted_body["unhealthy_endpoints"]): + assert { + "api_base", + "api_version", + "aws_bedrock_runtime_endpoint", + }.isdisjoint(endpoint), unrestricted_report.text + assert len(healthy.drain()) == 1 + assert team_b_wire.drain() == () + + +def test_team_scoped_health_hides_raw_request_upstream_details_from_non_admin_keys(gateway: Gateway) -> None: + pytest.skip( + "BUG: GET /health returns raw_request_typed_dict with the upstream raw_request_api_base " + "and request headers to a non-admin team key, though api_base is stripped" + ) + with ( + wire_server( + lambda request: _health_chat_reply(request, api_key="team-a-key", model="gpt-4o-mini", status=500) + ) as unhealthy, + gateway.scenario() as scenario, + ): + public_a: Final = f"team-a-{uuid.uuid4().hex}" + team_a: Final = scenario.team(models=[public_a]) + user_id: Final = scenario.member(team_a) + unhealthy_id: Final = _create_team_health_deployment( + gateway, + scenario, + team_id=team_a, + public_model_name=public_a, + upstream_url=unhealthy.url, + api_key="team-a-key", + ) + team_key: Final = scenario.key(team_id=team_a, user_id=user_id, models=[public_a]) + + non_admin: Final = gateway.request("GET", "/health", key=team_key, params={"model": public_a}) + assert non_admin.status_code == 503, non_admin.text + non_admin_endpoint: Final = object_value(non_admin.json()["unhealthy_endpoints"][0]) + non_admin_error: Final = string_value(non_admin_endpoint["error"]) + assert "synthetic health failure" in non_admin_error, non_admin.text + assert non_admin_endpoint == { + "model": "openai/gpt-4o-mini", + "model_id": unhealthy_id, + "error": non_admin_error, + "exception_status": 500, + }, non_admin.text + + admin: Final = gateway.request("GET", "/health", params={"model": public_a}) + assert admin.status_code == 503, admin.text + admin_endpoint: Final = object_value(admin.json()["unhealthy_endpoints"][0]) + admin_raw_request: Final = _health_failure_raw_request(admin_endpoint, unhealthy.url) + admin_error: Final = string_value(admin_endpoint["error"]) + assert "synthetic health failure" in admin_error, admin.text + assert admin_endpoint == { + "model": "openai/gpt-4o-mini", + "api_base": f"{unhealthy.url}/v1", + "model_id": unhealthy_id, + "error": admin_error, + "raw_request_typed_dict": admin_raw_request, + "exception_status": 500, + }, admin.text + + +def _delete_health_credential_if_present(gateway: Gateway, credential_name: str) -> None: + response: Final = gateway.request("DELETE", f"/credentials/{credential_name}") + assert response.status_code in (200, 404), response.text + + +def _health_connection_credential(gateway: Gateway, credential_name: str, api_key: str) -> None: + created: Final = gateway.request( + "POST", + "/credentials", + { + "credential_name": credential_name, + "credential_values": {"api_key": api_key}, + "credential_info": {}, + }, + ) + assert created.status_code == 200, created.text + + +def _create_health_connection_deployment( + gateway: Gateway, + scenario: Scenario, + *, + public_model_name: str, + provider_model: str, + api_base: str, + api_key: str, +) -> str: + created: Final = gateway.post( + "/model/new", + { + "model_name": public_model_name, + "litellm_params": { + "model": f"openai/{provider_model}", + "api_base": api_base, + "api_key": api_key, + }, + }, + ) + model_info: Final = object_value(created["model_info"]) + model_id: Final = string_value(model_info["id"]) + scenario.cleanups.callback(scenario.delete_model, model_id) + return model_id + + +def _health_connection_response( + request: Request, + model: str, + *, + validate_request_body: bool = True, + expected_temperature: float | None = None, + expected_api_key: str = "synthetic-health-credential", +) -> Reply: + response_body: Final = { + "/v1/chat/completions": { + "id": "chatcmpl-connection", + "object": "chat.completion", + "created": 1700000000, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "healthy"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + "/v1/embeddings": { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": model, + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + }, + "/v1/images/generations": {"created": 1700000000, "data": [{"b64_json": "c3ludGhldGljLWltYWdl"}]}, + "/v1/audio/transcriptions": {"text": "healthy"}, + "/v1/responses": { + "id": "resp-health", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": model, + "output": [ + { + "id": "msg-health", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "healthy", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + "/v1/messages": { + "id": "msg-health", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "healthy"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + } + assert request.method == "POST" + assert request.target in response_body, request.target + if validate_request_body: + if request.target == "/v1/audio/transcriptions": + parts: Final = _multipart_parts(request) + assert _multipart_text_fields(parts) == { + "model": model, + "response_format": "verbose_json", + }, request.target + files: Final = _multipart_file_fields(parts) + assert tuple(files) == ("file",), files + filename, content_type, audio_bytes = files["file"] + assert filename == "audio_health_check.wav", files + assert content_type == "audio/x-wav", files + expected_audio: Final = ( + Path(__file__).resolve().parents[3] + / "litellm" + / "litellm_core_utils" + / "audio_utils" + / "audio_health_check.wav" + ).read_bytes() + assert audio_bytes == expected_audio, files + else: + body: Final = JSON_OBJECT.validate_json(request.body) + if request.target == "/v1/chat/completions": + assert body in ( + { + "model": model, + "messages": _HEALTH_CHAT_MESSAGES[0], + "max_tokens": 16, + **({"temperature": expected_temperature} if expected_temperature is not None else {}), + }, + { + "model": model, + "messages": _HEALTH_CHAT_MESSAGES[1], + "max_tokens": 16, + **({"temperature": expected_temperature} if expected_temperature is not None else {}), + }, + ), request.body.decode() + elif request.target == "/v1/embeddings": + assert body == { + "model": model, + "input": ["test from litellm"], + }, request.body.decode() + elif request.target == "/v1/images/generations": + assert body == {"model": model, "prompt": "test from litellm"}, request.body.decode() + elif request.target == "/v1/responses": + assert body == {"model": model, "input": "test from litellm"}, request.body.decode() + else: + assert body in ( + { + "model": model, + "messages": _HEALTH_CHAT_MESSAGES[0], + "max_tokens": 16, + "stream": False, + }, + { + "model": model, + "messages": _HEALTH_CHAT_MESSAGES[1], + "max_tokens": 16, + "stream": False, + }, + ), request.body.decode() + if request.target == "/v1/messages": + assert request.headers["x-api-key"] == "synthetic-health-credential", request.headers + assert request.headers["anthropic-version"] == "2023-06-01", request.headers + assert "authorization" not in request.headers, request.headers + else: + assert request.headers["authorization"] == f"Bearer {expected_api_key}", request.headers + return Reply(body=json.dumps(response_body[request.target]).encode()) + + +def test_health_test_connection_modes_use_stored_credentials_and_reject_environment_references( + gateway: Gateway, +) -> None: + with gateway.scenario() as scenario: + for mode, provider in ( + ("chat", "openai"), + ("responses", "openai"), + ("anthropic_messages", "anthropic"), + ): + model = f"{provider}/health-{mode}-{uuid.uuid4().hex}" + provider_model = model.split("/", maxsplit=1)[1] + with wire_server( + lambda request, expected_model=provider_model: _health_connection_response(request, expected_model) + ) as wire: + mode_credential_name = f"health-{uuid.uuid4().hex}" + _health_connection_credential(gateway, mode_credential_name, "synthetic-health-credential") + scenario.cleanups.callback(_delete_health_credential_if_present, gateway, mode_credential_name) + base = wire.url if mode == "anthropic_messages" else f"{wire.url}/v1" + response = gateway.request( + "POST", + "/health/test_connection", + { + "litellm_params": { + "model": model, + "api_base": base, + "litellm_credential_name": mode_credential_name, + }, + "mode": mode, + }, + ) + assert response.status_code == 200, response.text + assert JSON_OBJECT.validate_json(response.content) == { + "status": "success", + "result": {"model": model, "api_base": base}, + }, response.text + requests = wire.drain() + assert len(requests) == 1, requests + assert ( + requests[0].target + == { + "chat": "/v1/chat/completions", + "embedding": "/v1/embeddings", + "image_generation": "/v1/images/generations", + "audio_transcription": "/v1/audio/transcriptions", + "responses": "/v1/responses", + "anthropic_messages": "/v1/messages", + }[mode] + ), requests + + configured_provider_model: Final = f"health-credential-{uuid.uuid4().hex}" + configured_model: Final = f"openai/{configured_provider_model}" + with ( + wire_server( + lambda request: _health_connection_response( + request, configured_provider_model, expected_temperature=0.2 + ) + ) as submitted_wire, + wire_server(lambda request: Reply(status=500)) as configured_wire, + ): + configured_credential_name: Final = f"health-submitted-{uuid.uuid4().hex}" + _health_connection_credential(gateway, configured_credential_name, "synthetic-health-credential") + scenario.cleanups.callback(_delete_health_credential_if_present, gateway, configured_credential_name) + scenario.model( + model=configured_model, + api_base=f"{configured_wire.url}/v1", + api_key="synthetic-config-credential", + temperature=0.9, + ) + submitted_base: Final = f"{submitted_wire.url}/v1" + submitted: Final = gateway.request( + "POST", + "/health/test_connection", + { + "litellm_params": { + "model": configured_model, + "api_base": submitted_base, + "litellm_credential_name": configured_credential_name, + "temperature": 0.2, + }, + "model_info": {"mode": "chat"}, + "mode": "chat", + }, + ) + assert submitted.status_code == 200, submitted.text + assert JSON_OBJECT.validate_json(submitted.content) == { + "status": "success", + "result": {"model": configured_model, "api_base": submitted_base}, + }, submitted.text + submitted_requests: Final = submitted_wire.drain() + assert len(submitted_requests) == 1, submitted.text + assert submitted_requests[0].headers["authorization"] == "Bearer synthetic-health-credential", ( + submitted.text + ) + assert JSON_OBJECT.validate_json(submitted_requests[0].body) in ( + { + "model": configured_provider_model, + "messages": _HEALTH_CHAT_MESSAGES[0], + "max_tokens": 16, + "temperature": 0.2, + }, + { + "model": configured_provider_model, + "messages": _HEALTH_CHAT_MESSAGES[1], + "max_tokens": 16, + "temperature": 0.2, + }, + ), submitted.text + assert configured_wire.drain() == () + + rejected: Final = gateway.request( + "POST", + "/health/test_connection", + { + "litellm_params": { + "model": configured_model, + "api_base": submitted_base, + "api_key": "os.environ/X", + }, + "model_info": {"mode": "chat"}, + "mode": "chat", + }, + ) + assert rejected.status_code == 400, rejected.text + assert JSON_OBJECT.validate_json(rejected.content) == { + "detail": {"error": "Environment variable references are not permitted in request parameters."} + }, rejected.text + assert submitted_wire.drain() == () + assert configured_wire.drain() == () + + rejected_model_info: Final = gateway.request( + "POST", + "/health/test_connection", + { + "litellm_params": { + "model": configured_model, + "api_base": submitted_base, + }, + "model_info": {"mode": "chat", "base_model": "os.environ/X"}, + "mode": "chat", + }, + ) + assert rejected_model_info.status_code == 400, rejected_model_info.text + assert JSON_OBJECT.validate_json(rejected_model_info.content) == { + "detail": {"error": "Environment variable references are not permitted in request parameters."} + }, rejected_model_info.text + assert submitted_wire.drain() == () + assert configured_wire.drain() == () + + +def test_health_test_connection_uses_model_info_id_for_duplicate_provider_models(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + provider_model: Final = f"health-duplicate-{uuid.uuid4().hex}" + public_model_name: Final = f"health-duplicate-public-{uuid.uuid4().hex}" + api_key_a: Final = "synthetic-health-credential-a" + api_key_b: Final = "synthetic-health-credential-b" + with ( + wire_server( + lambda request: _health_connection_response(request, provider_model, expected_api_key=api_key_a) + ) as wire_a, + wire_server( + lambda request: _health_connection_response(request, provider_model, expected_api_key=api_key_b) + ) as wire_b, + ): + api_base_a: Final = f"{wire_a.url}/v1" + api_base_b: Final = f"{wire_b.url}/v1" + model_id_a: Final = _create_health_connection_deployment( + gateway, + scenario, + public_model_name=public_model_name, + provider_model=provider_model, + api_base=api_base_a, + api_key=api_key_a, + ) + model_id_b: Final = _create_health_connection_deployment( + gateway, + scenario, + public_model_name=public_model_name, + provider_model=provider_model, + api_base=api_base_b, + api_key=api_key_b, + ) + model: Final = f"openai/{provider_model}" + request_body: Final = {"litellm_params": {"model": model}} + + response_a: Final = gateway.request( + "POST", + "/health/test_connection", + {**request_body, "model_info": {"id": model_id_a}, "mode": "chat"}, + ) + assert response_a.status_code == 200, response_a.text + assert JSON_OBJECT.validate_json(response_a.content) == { + "status": "success", + "result": {"model": model, "api_base": api_base_a}, + }, response_a.text + requests_a: Final = wire_a.drain() + assert len(requests_a) == 1, response_a.text + assert ( + requests_a[0].method, + requests_a[0].target, + requests_a[0].headers["authorization"], + JSON_OBJECT.validate_json(requests_a[0].body), + ) in ( + ( + "POST", + "/v1/chat/completions", + f"Bearer {api_key_a}", + {"model": provider_model, "messages": _HEALTH_CHAT_MESSAGES[0], "max_tokens": 16}, + ), + ( + "POST", + "/v1/chat/completions", + f"Bearer {api_key_a}", + {"model": provider_model, "messages": _HEALTH_CHAT_MESSAGES[1], "max_tokens": 16}, + ), + ), requests_a[0].body.decode() + assert wire_b.drain() == () + + response_b: Final = gateway.request( + "POST", + "/health/test_connection", + {**request_body, "model_info": {"id": model_id_b}, "mode": "chat"}, + ) + assert response_b.status_code == 200, response_b.text + assert JSON_OBJECT.validate_json(response_b.content) == { + "status": "success", + "result": {"model": model, "api_base": api_base_b}, + }, response_b.text + requests_b: Final = wire_b.drain() + assert len(requests_b) == 1, response_b.text + assert ( + requests_b[0].method, + requests_b[0].target, + requests_b[0].headers["authorization"], + JSON_OBJECT.validate_json(requests_b[0].body), + ) in ( + ( + "POST", + "/v1/chat/completions", + f"Bearer {api_key_b}", + {"model": provider_model, "messages": _HEALTH_CHAT_MESSAGES[0], "max_tokens": 16}, + ), + ( + "POST", + "/v1/chat/completions", + f"Bearer {api_key_b}", + {"model": provider_model, "messages": _HEALTH_CHAT_MESSAGES[1], "max_tokens": 16}, + ), + ), requests_b[0].body.decode() + assert wire_a.drain() == () + + +@pytest.mark.parametrize( + ("mode", "expected_path"), + ( + ("embedding", "/v1/embeddings"), + ("image_generation", "/v1/images/generations"), + ("audio_transcription", "/v1/audio/transcriptions"), + ), +) +def test_health_test_connection_non_chat_modes_do_not_receive_chat_max_tokens( + gateway: Gateway, mode: str, expected_path: str +) -> None: + pytest.skip( + "BUG: /health/test_connection sends chat max_tokens 16 to embedding, " + "image_generation and audio_transcription probes" + ) + with gateway.scenario() as scenario: + provider_model: Final = f"health-{mode}-{uuid.uuid4().hex[:8]}" + model: Final = f"openai/{provider_model}" + with wire_server( + lambda request, expected_model=provider_model: _health_connection_response( + request, expected_model, validate_request_body=False + ) + ) as wire: + credential_name: Final = f"health-{uuid.uuid4().hex}" + _health_connection_credential(gateway, credential_name, "synthetic-health-credential") + scenario.cleanups.callback(_delete_health_credential_if_present, gateway, credential_name) + api_base: Final = f"{wire.url}/v1" + response: Final = gateway.request( + "POST", + "/health/test_connection", + { + "litellm_params": { + "model": model, + "api_base": api_base, + "litellm_credential_name": credential_name, + }, + "mode": mode, + }, + ) + assert response.status_code == 200, response.text + assert response.json() == { + "status": "success", + "result": {"model": model, "api_base": api_base}, + }, response.text + + requests: Final = wire.drain() + assert len(requests) == 1, response.text + request: Final = requests[0] + assert request.target == expected_path, response.text + if mode == "embedding": + assert JSON_OBJECT.validate_json(request.body) == { + "model": provider_model, + "input": ["test from litellm"], + }, response.text + elif mode == "image_generation": + assert JSON_OBJECT.validate_json(request.body) == { + "model": provider_model, + "prompt": "test from litellm", + }, response.text + else: + parts: Final = _multipart_parts(request) + assert _multipart_text_fields(parts) == { + "model": provider_model, + "response_format": "verbose_json", + }, response.text + files: Final = _multipart_file_fields(parts) + assert tuple(files) == ("file",), response.text + assert files["file"] == ( + "audio_health_check.wav", + "audio/x-wav", + ( + Path(__file__).resolve().parents[3] + / "litellm" + / "litellm_core_utils" + / "audio_utils" + / "audio_health_check.wav" + ).read_bytes(), + ), response.text + + def test_evaluation_mode_health_check_of_hosted_vllm_sends_a_choice_probe(gateway: Gateway) -> None: with gateway.scenario() as scenario: handle: Final = register_scenario(f"health-decisions-{uuid.uuid4().hex[:12]}", _VLLM_PROBE_REPLY) @@ -243,7 +1327,9 @@ def test_evaluation_mode_health_check_of_databricks_ai_decide_resolves_the_mode_ gateway: Gateway, ) -> None: with gateway.scenario() as scenario: - handle: Final = register_scenario(f"health-decisions-{uuid.uuid4().hex[:12]}", _DATABRICKS_AI_DECIDE_PROBE_REPLY) + handle: Final = register_scenario( + f"health-decisions-{uuid.uuid4().hex[:12]}", _DATABRICKS_AI_DECIDE_PROBE_REPLY + ) scenario.cleanups.callback(delete_scenario, handle) model: Final = scenario.model( model="databricks/ai_decide", diff --git a/tests/integration/routing/test_auto_router_management.py b/tests/integration/routing/test_auto_router_management.py new file mode 100644 index 00000000000..7d46ac4e702 --- /dev/null +++ b/tests/integration/routing/test_auto_router_management.py @@ -0,0 +1,636 @@ +import hashlib +import json +import uuid +from collections.abc import Callable +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, eventually, object_value, string_value +from integration._support.database import read_rows, scratch_database +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +JSON_ARRAY: Final = TypeAdapter(list[JsonValue]) + + +def _assistant_content(response: httpx.Response) -> str: + body: Final = JSON_OBJECT.validate_json(response.content) + choices: Final = JSON_ARRAY.validate_python(body["choices"]) + assert len(choices) == 1, response.text + choice: Final = object_value(choices[0]) + message: Final = object_value(choice["message"]) + return string_value(message["content"]) + + +def _tier_peer(model: str, prompts: tuple[str, ...]) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=b'{"object":"list","data":[]}') + assert request.method == "POST", request.method + if request.target == "/v1/responses": + return Reply( + body=json.dumps( + { + "id": "resp-" + uuid.uuid4().hex[:8], + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": model, + "output": [ + { + "id": "msg-" + uuid.uuid4().hex[:8], + "type": "message", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "synthetic tier response", + "annotations": [], + } + ], + } + ], + "usage": {"input_tokens": 9, "output_tokens": 3, "total_tokens": 12}, + } + ).encode() + ) + assert request.target == "/v1/chat/completions", request.target + body: Final = JSON_OBJECT.validate_json(request.body) + expected_bodies: Final = tuple( + {"model": model, "messages": [{"role": "user", "content": prompt}]} for prompt in prompts + ) + assert body in expected_bodies, request.body.decode() + return Reply( + body=json.dumps( + { + "id": "chatcmpl-" + uuid.uuid4().hex[:8], + "object": "chat.completion", + "created": 1700000000, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "synthetic tier response"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 3, "total_tokens": 12}, + } + ).encode() + ) + + return respond + + +def _deployment( + alias: str, + model: str, + wire: Wire, + input_cost_per_token: float, + output_cost_per_token: float, +) -> dict[str, JsonValue]: + return { + "model_name": alias, + "litellm_params": { + "model": f"openai/{model}", + "api_base": f"{wire.url}/v1", + "api_key": "synthetic-auto-router-key", + }, + "model_info": { + "id": f"deployment-{alias}", + "input_cost_per_token": input_cost_per_token, + "output_cost_per_token": output_cost_per_token, + }, + } + + +def _session_response( + gateway: Gateway, + key: str, + session_id: str, + database_url: str | None = None, + stored_session_id: str | None = None, + seconds: float = 70, + return_last_on_timeout: bool = False, +) -> httpx.Response: + if database_url is None or stored_session_id is None: + return eventually( + lambda: gateway.request( + "GET", + "/auto_router/session", + params={"session_id": session_id}, + key=key, + ), + lambda response: response.status_code == 200, + seconds=seconds, + return_last_on_timeout=return_last_on_timeout, + ) + + key_hash: Final = hashlib.sha256(key.encode()).hexdigest() + + def read_response_state() -> tuple[httpx.Response, bool]: + rows: Final = read_rows( + 'SELECT session_id FROM "LiteLLM_AutoRouterSession" WHERE api_key = %s AND session_id = %s', + (key_hash, stored_session_id), + database_url=database_url, + ) + response: Final = gateway.request( + "GET", + "/auto_router/session", + params={"session_id": session_id}, + key=key, + ) + return response, bool(rows) + + response_state: Final = eventually( + read_response_state, + lambda state: state[0].status_code == 200 or state[1], + seconds=seconds, + return_last_on_timeout=return_last_on_timeout, + ) + return response_state[0] + + +def _assert_rollup_row( + api_key: str, + database_url: str, + router_name: str, + expected_session_id: str, +) -> None: + key_hash: Final = hashlib.sha256(api_key.encode()).hexdigest() + rows: Final = read_rows( + "SELECT session_id, router_name, router_type, last_model, turns, spend, saved_spend, " + "savings_estimated_turns, savings_estimated_actual_spend, savings_estimated_saved_spend, " + "savings_estimated_baseline_models, baseline_models, tier_turns " + 'FROM "LiteLLM_AutoRouterSession" WHERE api_key = %s AND session_id = %s AND router_name = %s', + (key_hash, expected_session_id, router_name), + database_url=database_url, + ) + assert len(rows) == 1, rows + assert rows[0] == { + "session_id": expected_session_id, + "router_name": router_name, + "router_type": "complexity", + "last_model": "openai/integration-simple", + "turns": 1, + "spend": pytest.approx(0.000015), + "saved_spend": pytest.approx(0.000135), + "savings_estimated_turns": 1, + "savings_estimated_actual_spend": pytest.approx(0.000015), + "savings_estimated_saved_spend": pytest.approx(0.000135), + "savings_estimated_baseline_models": {"openai/integration-baseline": 1}, + "baseline_models": {"openai/integration-baseline": 1}, + "tier_turns": {"SIMPLE": 1}, + } + + +def _rollup_row(api_key: str, database_url: str, router_name: str, session_id: str) -> dict[str, object]: + key_hash: Final = hashlib.sha256(api_key.encode()).hexdigest() + rows: Final = read_rows( + "SELECT session_id, router_name, router_type, last_model, turns, spend, saved_spend, " + "savings_estimated_turns, savings_estimated_actual_spend, savings_estimated_saved_spend, " + "savings_estimated_baseline_models, baseline_models, tier_turns " + 'FROM "LiteLLM_AutoRouterSession" WHERE api_key = %s AND session_id = %s AND router_name = %s', + (key_hash, session_id, router_name), + database_url=database_url, + ) + assert len(rows) == 1, rows + return rows[0] + + +def _session_contract(session_id: str, router_name: str) -> dict[str, object]: + return { + "session_id": session_id, + "router_name": router_name, + "router_type": "complexity", + "turns": 1, + "last_model": "openai/integration-simple", + "spend": pytest.approx(0.000015), + "savings_estimated_turns": 1, + "savings_estimated_actual_spend": pytest.approx(0.000015), + "saved_spend": pytest.approx(0.000135), + "baseline_spend": pytest.approx(0.00015), + "savings_estimated_baseline_spend": pytest.approx(0.00015), + "baseline_model": "openai/integration-baseline", + "baseline_models": {"openai/integration-baseline": 1}, + } + + +@pytest.mark.timeout(600) +def test_auto_router_session_tracks_key_scoped_spend_and_bounds_long_session_ids( + gateway: Gateway, + tmp_path: Path, +) -> None: + normal_session_id: Final = str(uuid.uuid4()) + messages_session_id: Final = str(uuid.uuid4()) + responses_session_id: Final = str(uuid.uuid4()) + codex_session_id: Final = str(uuid.uuid4()) + codex_thread_id: Final = str(uuid.uuid4()) + opencode_session_id: Final = str(uuid.uuid4()) + thread_only_session_id: Final = str(uuid.uuid4()) + long_session_id: Final = "s" * 300 + router_name: Final = "auto-router-" + uuid.uuid4().hex[:8] + normal_prompt: Final = f"integration-auto-simple {normal_session_id}" + messages_prompt: Final = f"integration-auto-simple {messages_session_id}" + messages_second_prompt: Final = f"integration-auto-medium {messages_session_id}" + responses_prompt: Final = f"integration-auto-simple {responses_session_id}" + codex_prompt: Final = f"integration-auto-simple {codex_session_id}" + opencode_prompt: Final = f"integration-auto-simple {opencode_session_id}" + thread_only_prompt: Final = f"integration-auto-simple {thread_only_session_id}" + long_prompt: Final = f"integration-auto-simple {long_session_id}" + messages_upstream_body: Final = { + "model": "integration-simple", + "input": [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": messages_prompt}], + } + ], + "include": ["reasoning.encrypted_content"], + "max_output_tokens": 32, + } + messages_second_upstream_body: Final = { + "model": "integration-medium", + "input": [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": messages_second_prompt}], + } + ], + "include": ["reasoning.encrypted_content"], + "max_output_tokens": 32, + } + responses_upstream_body: Final = { + "model": "integration-simple", + "input": responses_prompt, + } + + with ( + scratch_database() as database_url, + wire_server( + _tier_peer( + "integration-simple", + (normal_prompt, codex_prompt, opencode_prompt, thread_only_prompt, long_prompt), + ) + ) as simple, + wire_server(_tier_peer("integration-medium", ())) as medium, + wire_server(_tier_peer("integration-complex", (normal_prompt,))) as complex_tier, + wire_server(_tier_peer("integration-baseline", (normal_prompt,))) as baseline, + ): + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["model_list"] = [ + _deployment("integration-simple", "integration-simple", simple, 0.000001, 0.000002), + _deployment("integration-medium", "integration-medium", medium, 0.000002, 0.000004), + _deployment("integration-complex", "integration-complex", complex_tier, 0.000004, 0.000008), + _deployment("integration-baseline", "integration-baseline", baseline, 0.00001, 0.00002), + { + "model_name": router_name, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "classifier_type": "heuristic", + "keyword_tier_rules": [ + {"keywords": ["integration-auto-simple"], "tier": "SIMPLE"}, + {"keywords": ["integration-auto-medium"], "tier": "MEDIUM"}, + ], + "tiers": { + "SIMPLE": "integration-simple", + "MEDIUM": "integration-medium", + "COMPLEX": "integration-complex", + "REASONING": "integration-baseline", + }, + }, + }, + }, + ] + config["router_settings"] = {**config["router_settings"], "num_retries": 0, "disable_cooldowns": True} + config_path: Final = tmp_path / "auto_router_management.yaml" + config_path.write_text(yaml.safe_dump(config)) + + with owned_proxy(gateway, tmp_path, {"DATABASE_URL": database_url}, config=config_path, workers=1) as candidate: + with candidate.scenario() as scenario: + key: Final = scenario.key() + other_key: Final = scenario.key() + response: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": router_name, "messages": [{"role": "user", "content": normal_prompt}]}, + key=key, + headers={"x-claude-code-session-id": normal_session_id}, + ) + assert response.status_code == 200, response.text + assert _assistant_content(response) == "synthetic tier response", response.text + + session: Final = _session_response(candidate, key, normal_session_id) + assert JSON_OBJECT.validate_json(session.content) == { + "session_id": normal_session_id, + "router_name": router_name, + "router_type": "complexity", + "turns": 1, + "last_model": "openai/integration-simple", + "spend": pytest.approx(0.000015), + "savings_estimated_turns": 1, + "savings_estimated_actual_spend": pytest.approx(0.000015), + "saved_spend": pytest.approx(0.000135), + "baseline_spend": pytest.approx(0.00015), + "savings_estimated_baseline_spend": pytest.approx(0.00015), + "baseline_model": "openai/integration-baseline", + "baseline_models": {"openai/integration-baseline": 1}, + }, session.text + _assert_rollup_row(key, database_url, router_name, normal_session_id) + + hidden: Final = candidate.request( + "GET", + "/auto_router/session", + params={"session_id": normal_session_id}, + key=other_key, + ) + assert hidden.status_code == 404, hidden.text + assert JSON_OBJECT.validate_json(hidden.content) == { + "detail": f"No auto-routed turns recorded for session {normal_session_id!r} under this key" + }, hidden.text + + messages_response: Final = candidate.client.post( + "/v1/messages", + json={ + "model": router_name, + "max_tokens": 32, + "messages": [{"role": "user", "content": messages_prompt}], + }, + headers={ + "x-api-key": key, + "anthropic-version": "2023-06-01", + "x-claude-code-session-id": messages_session_id, + }, + ) + assert messages_response.status_code == 200, messages_response.text + assert JSON_OBJECT.validate_json(messages_response.content)["content"] == [ + {"type": "text", "text": "synthetic tier response"} + ], messages_response.text + + messages_second_response: Final = candidate.client.post( + "/v1/messages", + json={ + "model": router_name, + "max_tokens": 32, + "messages": [{"role": "user", "content": messages_second_prompt}], + }, + headers={ + "x-api-key": key, + "anthropic-version": "2023-06-01", + "x-claude-code-session-id": messages_session_id, + }, + ) + assert messages_second_response.status_code == 200, messages_second_response.text + assert JSON_OBJECT.validate_json(messages_second_response.content)["content"] == [ + {"type": "text", "text": "synthetic tier response"} + ], messages_second_response.text + + messages_session: Final = _session_response(candidate, key, messages_session_id) + assert JSON_OBJECT.validate_json(messages_session.content) == { + "session_id": messages_session_id, + "router_name": router_name, + "router_type": "complexity", + "turns": 2, + "last_model": "openai/integration-medium", + "spend": pytest.approx(0.000045), + "savings_estimated_turns": 2, + "savings_estimated_actual_spend": pytest.approx(0.000045), + "saved_spend": pytest.approx(0.000255), + "baseline_spend": pytest.approx(0.0003), + "savings_estimated_baseline_spend": pytest.approx(0.0003), + "baseline_model": "openai/integration-baseline", + "baseline_models": {"openai/integration-baseline": 2}, + }, messages_session.text + assert _rollup_row(key, database_url, router_name, messages_session_id) == { + "session_id": messages_session_id, + "router_name": router_name, + "router_type": "complexity", + "last_model": "openai/integration-medium", + "turns": 2, + "spend": pytest.approx(0.000045), + "saved_spend": pytest.approx(0.000255), + "savings_estimated_turns": 2, + "savings_estimated_actual_spend": pytest.approx(0.000045), + "savings_estimated_saved_spend": pytest.approx(0.000255), + "savings_estimated_baseline_models": {"openai/integration-baseline": 2}, + "baseline_models": {"openai/integration-baseline": 2}, + "tier_turns": {"SIMPLE": 1, "MEDIUM": 1}, + } + + responses_response: Final = candidate.client.post( + "/v1/responses", + json={"model": router_name, "input": responses_prompt}, + headers={ + "Authorization": f"Bearer {key}", + "user-agent": "codex_cli_rs/0.50.0", + "session-id": responses_session_id, + "thread-id": str(uuid.uuid4()), + }, + ) + assert responses_response.status_code == 200, responses_response.text + responses_output: Final = JSON_ARRAY.validate_python( + JSON_OBJECT.validate_json(responses_response.content)["output"] + ) + assert len(responses_output) == 1, responses_response.text + assert object_value(responses_output[0])["content"] == [ + { + "type": "output_text", + "text": "synthetic tier response", + "annotations": [], + "logprobs": None, + } + ], responses_response.text + responses_session: Final = _session_response(candidate, key, responses_session_id) + assert JSON_OBJECT.validate_json(responses_session.content) == _session_contract( + responses_session_id, router_name + ), responses_session.text + _assert_rollup_row(key, database_url, router_name, responses_session_id) + + codex_response: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": router_name, "messages": [{"role": "user", "content": codex_prompt}]}, + key=key, + headers={ + "user-agent": "codex_cli_rs/0.50.0", + "session-id": codex_session_id, + "thread-id": codex_thread_id, + }, + ) + assert codex_response.status_code == 200, codex_response.text + assert _assistant_content(codex_response) == "synthetic tier response", codex_response.text + codex_session: Final = _session_response(candidate, key, codex_session_id) + assert JSON_OBJECT.validate_json(codex_session.content) == _session_contract( + codex_session_id, router_name + ), codex_session.text + _assert_rollup_row(key, database_url, router_name, codex_session_id) + codex_thread: Final = candidate.request( + "GET", + "/auto_router/session", + params={"session_id": codex_thread_id}, + key=key, + ) + assert codex_thread.status_code == 404, codex_thread.text + assert JSON_OBJECT.validate_json(codex_thread.content) == { + "detail": f"No auto-routed turns recorded for session {codex_thread_id!r} under this key" + }, codex_thread.text + codex_hidden: Final = candidate.request( + "GET", + "/auto_router/session", + params={"session_id": codex_session_id}, + key=other_key, + ) + assert codex_hidden.status_code == 404, codex_hidden.text + assert JSON_OBJECT.validate_json(codex_hidden.content) == { + "detail": f"No auto-routed turns recorded for session {codex_session_id!r} under this key" + }, codex_hidden.text + + opencode_response: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": router_name, "messages": [{"role": "user", "content": opencode_prompt}]}, + key=key, + headers={ + "x-session-id": opencode_session_id, + "user-agent": "opencode/0.15.0", + }, + ) + assert opencode_response.status_code == 200, opencode_response.text + assert _assistant_content(opencode_response) == "synthetic tier response", opencode_response.text + opencode_session: Final = _session_response( + candidate, + key, + opencode_session_id, + return_last_on_timeout=True, + ) + assert JSON_OBJECT.validate_json(opencode_session.content) == _session_contract( + opencode_session_id, router_name + ), opencode_session.text + _assert_rollup_row(key, database_url, router_name, opencode_session_id) + opencode_hidden: Final = candidate.request( + "GET", + "/auto_router/session", + params={"session_id": opencode_session_id}, + key=other_key, + ) + assert opencode_hidden.status_code == 404, opencode_hidden.text + assert JSON_OBJECT.validate_json(opencode_hidden.content) == { + "detail": f"No auto-routed turns recorded for session {opencode_session_id!r} under this key" + }, opencode_hidden.text + + thread_only_response: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": router_name, "messages": [{"role": "user", "content": thread_only_prompt}]}, + key=key, + headers={ + "user-agent": "codex_cli_rs/0.50.0", + "thread-id": thread_only_session_id, + }, + ) + assert thread_only_response.status_code == 200, thread_only_response.text + assert _assistant_content(thread_only_response) == "synthetic tier response", thread_only_response.text + thread_only_session: Final = _session_response(candidate, key, thread_only_session_id) + assert JSON_OBJECT.validate_json(thread_only_session.content) == _session_contract( + thread_only_session_id, router_name + ), thread_only_session.text + _assert_rollup_row(key, database_url, router_name, thread_only_session_id) + thread_only_hidden: Final = candidate.request( + "GET", + "/auto_router/session", + params={"session_id": thread_only_session_id}, + key=other_key, + ) + assert thread_only_hidden.status_code == 404, thread_only_hidden.text + assert JSON_OBJECT.validate_json(thread_only_hidden.content) == { + "detail": f"No auto-routed turns recorded for session {thread_only_session_id!r} under this key" + }, thread_only_hidden.text + + long_response: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": router_name, "messages": [{"role": "user", "content": long_prompt}]}, + key=key, + headers={"x-litellm-session-id": long_session_id}, + ) + assert long_response.status_code == 200, long_response.text + assert _assistant_content(long_response) == "synthetic tier response", long_response.text + + bounded_session_id: Final = "sha256:" + hashlib.sha256(long_session_id.encode()).hexdigest() + long_session: Final = _session_response( + candidate, + key, + long_session_id, + database_url, + bounded_session_id, + seconds=70, + return_last_on_timeout=True, + ) + assert JSON_OBJECT.validate_json(long_session.content) == { + "session_id": long_session_id, + "router_name": router_name, + "router_type": "complexity", + "turns": 1, + "last_model": "openai/integration-simple", + "spend": pytest.approx(0.000015), + "savings_estimated_turns": 1, + "savings_estimated_actual_spend": pytest.approx(0.000015), + "saved_spend": pytest.approx(0.000135), + "baseline_spend": pytest.approx(0.00015), + "savings_estimated_baseline_spend": pytest.approx(0.00015), + "baseline_model": "openai/integration-baseline", + "baseline_models": {"openai/integration-baseline": 1}, + }, long_session.text + _assert_rollup_row(key, database_url, router_name, bounded_session_id) + + simple_calls: Final = tuple(request for request in simple.drain() if request.method == "POST") + assert tuple( + ( + request.method, + request.target, + json.loads(request.body), + ) + for request in simple_calls + ) == ( + ( + "POST", + "/v1/chat/completions", + {"model": "integration-simple", "messages": [{"role": "user", "content": normal_prompt}]}, + ), + ("POST", "/v1/responses", messages_upstream_body), + ("POST", "/v1/responses", responses_upstream_body), + ( + "POST", + "/v1/chat/completions", + {"model": "integration-simple", "messages": [{"role": "user", "content": codex_prompt}]}, + ), + ( + "POST", + "/v1/chat/completions", + {"model": "integration-simple", "messages": [{"role": "user", "content": opencode_prompt}]}, + ), + ( + "POST", + "/v1/chat/completions", + {"model": "integration-simple", "messages": [{"role": "user", "content": thread_only_prompt}]}, + ), + ( + "POST", + "/v1/chat/completions", + {"model": "integration-simple", "messages": [{"role": "user", "content": long_prompt}]}, + ), + ), response.text + medium_calls: Final = tuple(request for request in medium.drain() if request.method == "POST") + assert tuple((request.method, request.target, json.loads(request.body)) for request in medium_calls) == ( + ("POST", "/v1/responses", messages_second_upstream_body), + ), response.text + assert tuple(request for request in complex_tier.drain() if request.method == "POST") == (), response.text + assert tuple(request for request in baseline.drain() if request.method == "POST") == (), response.text diff --git a/tests/integration/routing/test_observed_routing.py b/tests/integration/routing/test_observed_routing.py index d7398b05fd4..d9ac0652a0d 100644 --- a/tests/integration/routing/test_observed_routing.py +++ b/tests/integration/routing/test_observed_routing.py @@ -1,19 +1,28 @@ import json import uuid +from collections.abc import Callable from pathlib import Path +from queue import SimpleQueue from typing import Final import httpx import pytest import yaml - -from integration._support.client import Gateway, object_value +from integration._support.client import JSON_OBJECT, Gateway, object_value +from integration._support.process import owned_proxy from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue -@pytest.mark.covers("other.routing.retries.several_attempts_reach_success_without_hidden_retries", "other.routing.errors.nonretryable_and_exhausted_failures_remain_errors") +@pytest.mark.covers( + "other.routing.retries.several_attempts_reach_success_without_hidden_retries", + "other.routing.errors.nonretryable_and_exhausted_failures_remain_errors", +) def test_retry_counts_and_public_errors_match_actual_provider_attempts(gateway: Gateway) -> None: - with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, gateway.scenario() as scenario: + with ( + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + gateway.scenario() as scenario, + ): original: Final = object_value(gateway.get("/router/settings")["current_values"])["num_retries"] provider_model: Final = "errors-" + uuid.uuid4().hex model: Final = scenario.model(model=f"openai/{provider_model}", input_cost_per_token=0, output_cost_per_token=0) @@ -25,12 +34,23 @@ def test_retry_counts_and_public_errors_match_actual_provider_attempts(gateway: scenario.cleanups.callback(remove) try: - for index, (retries, statuses, status, attempts) in enumerate(((2, [500, 500, 200], 200, 3), (2, [400, 200], 400, 1), (1, [429, 429, 200], 429, 2), (1, [500, 500, 200], 500, 2))): + for index, (retries, statuses, status, attempts) in enumerate( + ( + (2, [500, 500, 200], 200, 3), + (2, [400, 200], 400, 1), + (1, [429, 429, 200], 429, 2), + (1, [500, 500, 200], 500, 2), + ) + ): gateway.post("/config/update", {"router_settings": {"num_retries": retries}}) assert object_value(gateway.get("/router/settings")["current_values"])["num_retries"] == retries upstream.post(f"/__scripts/{provider_model}", json={"statuses": statuses}).raise_for_status() upstream.get("/__observations").raise_for_status() - response: Final = gateway.request("POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": f"{provider_model} {index}"}]}) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": f"{provider_model} {index}"}]}, + ) assert response.status_code == status, response.text requests: Final = upstream.get("/__observations").json()["requests"] assert len(requests) == attempts @@ -42,9 +62,12 @@ def test_retry_counts_and_public_errors_match_actual_provider_attempts(gateway: error: Final = response.json()["error"] assert isinstance(error["message"], str) and "Controlled provider failure" in error["message"] assert str(error["code"]) == str(status) - assert error["type"] == {400: "invalid_request_error", 429: "throttling_error", 500: "internal_server_error"}[status] + assert ( + error["type"] + == {400: "invalid_request_error", 429: "throttling_error", 500: "internal_server_error"}[status] + ) assert error["param"] is None - assert "Traceback" not in response.text and "File \"" not in response.text + assert "Traceback" not in response.text and 'File "' not in response.text finally: gateway.post("/config/update", {"router_settings": {"num_retries": original}}) assert object_value(gateway.get("/router/settings")["current_values"])["num_retries"] == original @@ -58,22 +81,74 @@ def test_loaded_fallback_selects_expected_deployment_and_keeps_response_identity model: Final = json.loads(request.body)["model"] assert model in {"primary-wire", "fallback-wire", "unrelated-wire"} if model == "primary-wire": - return Reply(status=500, body=b'{"error":{"message":"synthetic primary unavailable","type":"api_error","code":"500"}}') - return Reply(body=json.dumps({"id": "response-" + model, "object": "chat.completion", "created": 1, "model": model, "choices": [{"index": 0, "message": {"role": "assistant", "content": "served " + model}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15}}).encode()) + return Reply( + status=500, + body=b'{"error":{"message":"synthetic primary unavailable","type":"api_error","code":"500"}}', + ) + return Reply( + body=json.dumps( + { + "id": "response-" + model, + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "served " + model}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15}, + } + ).encode() + ) with wire_server(respond) as wire: path: Final = tmp_path / "fallback.yaml" - path.write_text(yaml.safe_dump({"model_list": [{"model_name": alias, "litellm_params": {"model": "openai/" + upstream, "api_key": "synthetic-routing-key", "api_base": wire.url + "/v1"}} for alias, upstream in (("primary", "primary-wire"), ("fallback", "fallback-wire"), ("unrelated", "unrelated-wire"))], "router_settings": {"num_retries": 0, "disable_cooldowns": True, "fallbacks": [{"primary": ["fallback"]}]}})) + path.write_text( + yaml.safe_dump( + { + "model_list": [ + { + "model_name": alias, + "litellm_params": { + "model": "openai/" + upstream, + "api_key": "synthetic-routing-key", + "api_base": wire.url + "/v1", + }, + } + for alias, upstream in ( + ("primary", "primary-wire"), + ("fallback", "fallback-wire"), + ("unrelated", "unrelated-wire"), + ) + ], + "router_settings": { + "num_retries": 0, + "disable_cooldowns": True, + "fallbacks": [{"primary": ["fallback"]}], + }, + } + ) + ) loaded: Final = yaml.safe_load(path.read_text()) router: Final = Router(model_list=loaded["model_list"], **loaded["router_settings"]) try: - result: Final = router.completion(model="primary", messages=[{"role": "user", "content": "fallback control"}]) + result: Final = router.completion( + model="primary", messages=[{"role": "user", "content": "fallback control"}] + ) assert result.id == "response-fallback-wire" assert result.choices[0].message.content == "served fallback-wire" assert result.choices[0].finish_reason == "stop" assert result.usage.prompt_tokens == 11 and result.usage.completion_tokens == 4 - assert tuple(json.loads(request.body)["model"] for request in wire.drain()) == ("primary-wire", "fallback-wire") - control: Final = router.completion(model="unrelated", messages=[{"role": "user", "content": "independent route"}]) + assert tuple(json.loads(request.body)["model"] for request in wire.drain()) == ( + "primary-wire", + "fallback-wire", + ) + control: Final = router.completion( + model="unrelated", messages=[{"role": "user", "content": "independent route"}] + ) assert control.id == "response-unrelated-wire" assert tuple(json.loads(request.body)["model"] for request in wire.drain()) == ("unrelated-wire",) finally: @@ -82,17 +157,494 @@ def test_loaded_fallback_selects_expected_deployment_and_keeps_response_identity @pytest.mark.covers("other.routing.alias_update.persisted_target_changes_only_selected_route") def test_saved_deployment_target_update_changes_wire_and_preserves_control(gateway: Gateway) -> None: - with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, gateway.scenario() as scenario: + with ( + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + gateway.scenario() as scenario, + ): prefix: Final = "target-" + uuid.uuid4().hex - model: Final = scenario.model(model="openai/" + prefix + "-first", input_cost_per_token=0, output_cost_per_token=0) - other: Final = scenario.model(model="openai/" + prefix + "-control", input_cost_per_token=0, output_cost_per_token=0) + model: Final = scenario.model( + model="openai/" + prefix + "-first", input_cost_per_token=0, output_cost_per_token=0 + ) + other: Final = scenario.model( + model="openai/" + prefix + "-control", input_cost_per_token=0, output_cost_per_token=0 + ) target: Final = next(entry for entry in gateway.get("/model/info")["data"] if entry["model_name"] == model) for generation, suffix in enumerate(("first", "second")): if generation: - response: Final = gateway.request("PATCH", f"/model/{target['model_info']['id']}/update", {"litellm_params": {"model": "openai/" + prefix + "-second"}}) + response: Final = gateway.request( + "PATCH", + f"/model/{target['model_info']['id']}/update", + {"litellm_params": {"model": "openai/" + prefix + "-second"}}, + ) assert response.status_code == 200, response.text upstream.get("/__observations").raise_for_status() for alias in (model, other): assert gateway.chat(alias, text=f"{prefix} generation {generation}")["usage"]["total_tokens"] == 40 requests: Final = upstream.get("/__observations").json()["requests"] assert [request["body"]["model"] for request in requests] == [prefix + "-" + suffix, prefix + "-control"] + + +@pytest.mark.parametrize( + ( + "case", + "primary_error", + "body_flag", + "expected_sequence", + "expected_model", + "expected_status", + "expected_attempted_fallbacks", + "stream_request", + "key_metadata", + ), + ( + ("context", "context", None, ("primary-up", "big-up"), "big-up", 200, "1", False, False), + ("content-policy", "content-policy", None, ("primary-up", "safe-up"), "safe-up", 200, "1", False, False), + ("server-error", "server-error", None, ("primary-up", "general-up"), "general-up", 200, "1", False, False), + ("disabled-context", "context", "disable_fallbacks", ("primary-up",), "primary-up", 400, None, False, False), + ( + "disabled-server-error", + "server-error", + "disable_fallbacks", + ("primary-up",), + "primary-up", + 500, + None, + False, + False, + ), + ( + "disabled-mid-stream", + "mid-stream", + "disable_fallbacks", + ("primary-up",), + "primary-up", + 200, + "0", + True, + False, + ), + ("key-metadata-disabled", "context", None, ("primary-up",), "primary-up", 400, None, False, True), + ("stream-context", "context", None, ("primary-up", "big-up"), "big-up", 200, "1", True, False), + ( + "stream-mid-stream", + "mid-stream", + None, + ("primary-up", "general-up"), + "general-up", + 200, + "0", + True, + False, + ), + ("mock-context", None, "mock_testing_context_fallbacks", ("big-up",), "big-up", 200, "1", False, False), + ( + "mock-content-policy", + None, + "mock_testing_content_policy_fallbacks", + ("safe-up",), + "safe-up", + 200, + "1", + False, + False, + ), + ("mock-server-error", None, "mock_testing_fallbacks", ("general-up",), "general-up", 200, "1", False, False), + ("ordered-context", "context", None, ("primary-up", "big-up"), "big-up", 200, "1", False, False), + ( + "ordered-content-policy", + "content-policy", + None, + ("primary-up", "safe-up"), + "safe-up", + 200, + "1", + False, + False, + ), + ( + "ordered-server-error", + "server-error", + None, + ("primary-up", "primary-tier2-up"), + "primary-tier2-up", + 200, + "1", + False, + False, + ), + ("body-context", "context", "body_fallbacks", ("primary-up", "big-up"), "big-up", 200, "1", False, False), + ( + "body-content-policy", + "content-policy", + "body_fallbacks", + ("primary-up", "safe-up"), + "safe-up", + 200, + "1", + False, + False, + ), + ), +) +def test_health_routing_fallbacks_preserve_error_specific_targets_and_request_controls( + gateway: Gateway, + tmp_path: Path, + case: str, + primary_error: str | None, + body_flag: str | None, + expected_sequence: tuple[str, ...], + expected_model: str, + expected_status: int, + expected_attempted_fallbacks: str | None, + stream_request: bool, + key_metadata: bool, +) -> None: + prompt: Final = f"health-routing-{case}-{uuid.uuid4().hex}" + call_order: Final = SimpleQueue[str]() + + def respond(model: str) -> Reply: + if stream_request: + served_chunk: Final = ( + b"data: " + + json.dumps( + { + "id": "response-" + model, + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "served " + model}, + "finish_reason": "stop", + } + ], + } + ).encode() + + b"\n\n" + ) + usage_chunk: Final = ( + b"data: " + + json.dumps( + { + "id": "response-" + model, + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [], + "usage": {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15}, + } + ).encode() + + b"\n\n" + ) + return Reply( + content_type="text/event-stream", + chunks=(served_chunk, usage_chunk, b"data: [DONE]\n\n"), + ) + return Reply( + body=json.dumps( + { + "id": "response-" + model, + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "served " + model}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15}, + } + ).encode() + ) + + ordered: Final = case.startswith("ordered-") + body_fallbacks: Final = case.startswith("body-") + + def handler(model: str) -> Callable[[Request], Reply]: + def handle(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=b'{"object":"list","data":[]}') + actual_body: Final = json.loads(request.body) + assert request.target == "/v1/chat/completions", request.target + assert actual_body == { + "model": model, + "messages": [{"role": "user", "content": prompt}], + **({"stream": True, "stream_options": {"include_usage": True}} if stream_request else {}), + }, request.body.decode() + call_order.put(model) + if model == "primary-up" and primary_error == "context": + return Reply( + status=400, + body=b'{"error":{"message":"This model\'s maximum context length is 4096 tokens","type":"invalid_request_error","code":"context_length_exceeded"}}', + ) + if model == "primary-up" and primary_error == "content-policy": + return Reply( + status=400, + body=b'{"error":{"message":"synthetic content policy rejection","type":"invalid_request_error","code":"content_policy_violation"}}', + ) + if model == "primary-up" and primary_error == "server-error": + return Reply( + status=500, + body=b'{"error":{"message":"synthetic upstream failure","type":"api_error","code":"500"}}', + ) + if model == "primary-up" and primary_error == "mid-stream": + empty_first: Final = ( + b"data: " + + json.dumps( + { + "id": "response-" + model, + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": ""}, + "finish_reason": None, + } + ], + } + ).encode() + + b"\n\n" + ) + return Reply( + content_type="text/event-stream", + chunks=(empty_first, b":" + b"x" * 4_000_000 + b"\n\n", empty_first), + abort_after=2, + ) + return respond(model) + + return handle + + with ( + wire_server(handler("primary-up")) as primary, + wire_server(handler("primary-tier2-up")) as primary_tier2, + wire_server(handler("big-up")) as big, + wire_server(handler("safe-up")) as safe, + wire_server(handler("general-up")) as general, + ): + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["model_list"] = [ + *( + ( + { + "model_name": "primary", + "litellm_params": { + "model": "openai/primary-up", + "api_base": primary.url + "/v1", + "api_key": "synthetic-health-routing-key", + "order": 1, + }, + }, + { + "model_name": "primary", + "litellm_params": { + "model": "openai/primary-tier2-up", + "api_base": primary_tier2.url + "/v1", + "api_key": "synthetic-health-routing-key", + "order": 2, + }, + }, + ) + if ordered + else ( + { + "model_name": "primary", + "litellm_params": { + "model": "openai/primary-up", + "api_base": primary.url + "/v1", + "api_key": "synthetic-health-routing-key", + }, + }, + ) + ), + *( + { + "model_name": alias, + "litellm_params": { + "model": f"openai/{upstream_model}", + "api_base": wire.url + "/v1", + "api_key": "synthetic-health-routing-key", + }, + } + for alias, upstream_model, wire in ( + ("big", "big-up", big), + ("safe", "safe-up", safe), + ("general", "general-up", general), + ) + ), + ] + config["router_settings"] = { + "num_retries": 0, + "disable_cooldowns": True, + **( + {} + if body_fallbacks + else { + "context_window_fallbacks": [{"primary": ["big"]}], + "content_policy_fallbacks": [{"primary": ["safe"]}], + } + ), + "fallbacks": [{"primary": ["general"]}], + } + config["general_settings"] = { + **config.get("general_settings", {}), + "dangerously_allow_mock_testing_request_params": True, + } + path: Final = tmp_path / f"health-routing-{case}.yaml" + path.write_text(yaml.safe_dump(config)) + with gateway.scenario() as scenario, owned_proxy(gateway, tmp_path, {}, config=path) as candidate: + request_key: Final = scenario.key(metadata={"disable_fallbacks": True}) if key_metadata else candidate.key + request_body: Final[dict[str, JsonValue]] = { + "model": "primary", + "messages": [{"role": "user", "content": prompt}], + **({"stream": True} if stream_request else {}), + **( + {body_flag: True} + if body_flag + in ( + "disable_fallbacks", + "mock_testing_context_fallbacks", + "mock_testing_content_policy_fallbacks", + "mock_testing_fallbacks", + ) + else {} + ), + **( + { + "context_window_fallbacks": [{"primary": ["big"]}], + "content_policy_fallbacks": [{"primary": ["safe"]}], + } + if body_fallbacks + else {} + ), + } + if stream_request: + with candidate.client.stream( + "POST", + "/v1/chat/completions", + json=request_body, + headers={"Authorization": f"Bearer {request_key}"}, + ) as response: + response_text: Final = response.read().decode() + response_status: Final = response.status_code + response_headers: Final = response.headers + assert response_status == expected_status, response_text + assert response_headers.get("x-litellm-attempted-fallbacks") == expected_attempted_fallbacks, ( + response_text + ) + assert tuple(call_order.get_nowait() for _ in range(call_order.qsize())) == expected_sequence, ( + response_text + ) + event_lines: Final = tuple( + line.removeprefix("data:") for line in response_text.splitlines() if line.startswith("data:") + ) + events: Final = tuple( + JSON_OBJECT.validate_json(event) for event in event_lines if event.strip() != "[DONE]" + ) + if expected_status == 200 and not (primary_error == "mid-stream" and body_flag == "disable_fallbacks"): + assert tuple({**event, "created": 0} if "created" in event else event for event in events) == ( + *( + ( + { + "id": "response-primary-up", + "object": "chat.completion.chunk", + "created": 0, + "model": "primary", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": ""}}], + }, + ) + if primary_error == "mid-stream" + else () + ), + { + "id": "response-" + expected_model, + "object": "chat.completion.chunk", + "created": 0, + "model": expected_model.removesuffix("-up"), + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "served " + expected_model}, + } + ], + }, + { + "id": "response-" + expected_model, + "object": "chat.completion.chunk", + "created": 0, + "model": expected_model.removesuffix("-up"), + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + ), response_text + return + assert len(events) == 2, response_text + first_event: Final = events[0] + assert ({**first_event, "created": 0} if "created" in first_event else first_event) == { + "id": "response-primary-up", + "object": "chat.completion.chunk", + "created": 0, + "model": "primary", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": ""}}], + }, response_text + assert {key: events[1]["error"][key] for key in events[1]["error"] if key != "message"} == { + "type": None, + "param": None, + "code": "500", + }, response_text + message: Final = events[1]["error"]["message"] + assert isinstance(message, str), response_text + assert message.startswith("litellm.APIConnectionError: "), response_text + else: + non_stream_response: Final = candidate.request( + "POST", "/v1/chat/completions", request_body, key=request_key + ) + assert non_stream_response.status_code == expected_status, non_stream_response.text + assert ( + non_stream_response.headers.get("x-litellm-attempted-fallbacks") == expected_attempted_fallbacks + ), non_stream_response.text + if expected_status == 400: + assert JSON_OBJECT.validate_json(non_stream_response.content) == { + "error": { + "message": ( + "litellm.ContextWindowExceededError: litellm.BadRequestError: " + "ContextWindowExceededError: OpenAIException - This model's maximum context length " + "is 4096 tokens" + ), + "type": "invalid_request_error", + "param": None, + "code": "400", + } + }, non_stream_response.text + elif expected_status == 500: + assert JSON_OBJECT.validate_json(non_stream_response.content) == { + "error": { + "message": "litellm.InternalServerError: InternalServerError: OpenAIException - " + "synthetic upstream failure", + "type": "internal_server_error", + "param": None, + "code": "500", + } + }, non_stream_response.text + else: + assert JSON_OBJECT.validate_json(non_stream_response.content) == { + "id": "response-" + expected_model, + "object": "chat.completion", + "created": 1, + "model": expected_model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "served " + expected_model}, + "finish_reason": "stop", + "provider_specific_fields": {}, + } + ], + "usage": {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15}, + }, non_stream_response.text + assert tuple(call_order.get_nowait() for _ in range(call_order.qsize())) == expected_sequence, ( + non_stream_response.text + ) diff --git a/tests/integration/streaming/test_stream_contracts.py b/tests/integration/streaming/test_stream_contracts.py index bd89b869ef2..3f8243033d7 100644 --- a/tests/integration/streaming/test_stream_contracts.py +++ b/tests/integration/streaming/test_stream_contracts.py @@ -2,18 +2,23 @@ import asyncio import json import threading import uuid +from collections.abc import Callable from pathlib import Path +from queue import SimpleQueue from typing import Final import pytest import yaml from hypothesis import Phase, example, given, settings from hypothesis import strategies as st -from integration._support.client import Gateway, eventually +from integration._support.client import Gateway, eventually, object_value, string_value from integration._support.database import read_rows from integration._support.process import owned_proxy -from integration._support.wire import Reply, wire_server +from integration._support.wire import Reply, Request, wire_server from openai import OpenAI +from pydantic import JsonValue, TypeAdapter + +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) def frame(identity: str, delta: dict, *, finish: str | None = None) -> bytes: @@ -165,6 +170,13 @@ def test_proxy_stream_usage_visibility_keeps_exact_persisted_charge(gateway: Gat model: Final = scenario.model( api_base=wire.url + "/v1", input_cost_per_token=0.001, output_cost_per_token=0.002 ) + eventually( + lambda: tuple( + object_value(entry).get("model_name") for entry in gateway.get("/model/info")["data"] + ), + lambda model_names, model=model: model in model_names, + seconds=60, + ) with OpenAI( api_key=gateway.key, base_url=str(gateway.client.base_url), timeout=5, max_retries=0 ) as client: @@ -434,6 +446,100 @@ def sse_data_lines(text: str) -> tuple[str, ...]: return tuple(line.removeprefix("data: ") for line in text.splitlines() if line.startswith("data: ")) +def _native_anthropic_message(model: str, identity: str, text: str) -> dict[str, object]: + return { + "id": identity, + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": text}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 11, "output_tokens": 4}, + } + + +def _native_responses_response(model: str, identity: str, text: str) -> dict[str, object]: + return { + "id": identity, + "object": "response", + "created_at": 1, + "status": "completed", + "model": model, + "output": [ + { + "id": "msg_" + identity, + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + ], + "usage": {"input_tokens": 11, "output_tokens": 4, "total_tokens": 15}, + } + + +def _native_sse_event(event: dict[str, object]) -> bytes: + return f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() + + +def _native_anthropic_stream(model: str, identity: str, text: str) -> tuple[bytes, ...]: + return ( + _native_sse_event( + { + "type": "message_start", + "message": { + **_native_anthropic_message(model, identity, ""), + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 11, "output_tokens": 0}, + }, + } + ), + _native_sse_event({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}), + _native_sse_event( + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": text}, + } + ), + _native_sse_event({"type": "content_block_stop", "index": 0}), + _native_sse_event( + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 4}, + } + ), + _native_sse_event({"type": "message_stop"}), + ) + + +def _native_responses_stream(model: str, identity: str, text: str) -> tuple[bytes, ...]: + response: Final = _native_responses_response(model, identity, text) + return ( + _native_sse_event( + { + "type": "response.created", + "sequence_number": 0, + "response": {**response, "status": "in_progress", "output": []}, + } + ), + _native_sse_event( + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_" + identity, + "output_index": 0, + "content_index": 0, + "delta": text, + } + ), + _native_sse_event({"type": "response.completed", "sequence_number": 2, "response": response}), + ) + + @pytest.mark.covers("other.streaming.usage.provider_cost_object_completes_stream_and_bills_total_cost") def test_perplexity_stream_with_cost_breakdown_object_completes_and_bills_total_cost(gateway: Gateway) -> None: identity: Final = "stream-cost-object-" + uuid.uuid4().hex @@ -594,6 +700,280 @@ def test_primary_stream_with_empty_first_chunk_then_disconnect_falls_back_and_bi assert float(rows[0]["spend"]) == pytest.approx(0.019), rows +@pytest.mark.parametrize( + ("route", "failure_mode"), + ( + ("messages", "non-streaming"), + ("messages", "before-first-byte"), + ("messages", "mid-stream"), + ("messages", "request-body-fallbacks"), + ("responses", "non-streaming"), + ("responses", "before-first-byte"), + ("responses", "mid-stream"), + ("responses", "request-body-fallbacks"), + ), +) +def test_native_messages_and_responses_routes_fall_back_and_bill_the_fallback( + gateway: Gateway, + tmp_path: Path, + route: str, + failure_mode: str, +) -> None: + identity: Final = f"native-fallback-{route}-{failure_mode}-{uuid.uuid4().hex}" + is_messages: Final = route == "messages" + path: Final = "/v1/messages" if is_messages else "/v1/responses" + primary_model: Final = "claude-sonnet-4-5-20250929" if is_messages else "gpt-5.6" + fallback_model: Final = "claude-opus-4-5-20251101" if is_messages else "gpt-5.6" + stream: Final = failure_mode in ("before-first-byte", "mid-stream") + client_body: Final = ( + { + "model": "primary", + "max_tokens": 12, + "messages": [{"role": "user", "content": identity}], + "stream": stream, + **({"fallbacks": ["fallback"]} if failure_mode == "request-body-fallbacks" else {}), + } + if is_messages + else { + "model": "primary", + "input": identity, + "stream": stream, + **({"fallbacks": ["fallback"]} if failure_mode == "request-body-fallbacks" else {}), + } + ) + primary_body: Final = ( + { + "model": primary_model, + "max_tokens": 12, + "messages": [{"role": "user", "content": identity}], + "stream": stream, + } + if is_messages + else {"model": primary_model, "input": identity, "stream": stream} + ) + fallback_body: Final = {**primary_body, "model": fallback_model} + call_order: Final[SimpleQueue[str]] = SimpleQueue() + received: Final[SimpleQueue[tuple[str, Request]]] = SimpleQueue() + fallback_text: Final = f"fallback response {identity}" + primary_text: Final = f"primary response {identity}" + + def response_for(model: str, text: str) -> Reply: + if is_messages: + if stream: + return Reply(content_type="text/event-stream", chunks=_native_anthropic_stream(model, identity, text)) + return Reply(body=json.dumps(_native_anthropic_message(model, identity, text)).encode()) + if stream: + return Reply(content_type="text/event-stream", chunks=_native_responses_stream(model, identity, text)) + return Reply(body=json.dumps(_native_responses_response(model, identity, text)).encode()) + + def upstream(name: str, model: str) -> Callable[[Request], Reply]: + def handle(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=b'{"object":"list","data":[]}') + assert request.method == "POST" and request.target == path, (request.method, request.target) + expected_body: Final = primary_body if name == "primary" else fallback_body + assert json.loads(request.body) == expected_body, request.body.decode() + if is_messages: + assert request.headers.get("x-api-key") == "synthetic-native-fallback-key", request.headers + assert request.headers.get("anthropic-version") == "2023-06-01", request.headers + assert "authorization" not in request.headers, request.headers + else: + assert request.headers.get("authorization") == "Bearer synthetic-native-fallback-key", request.headers + call_order.put(name) + received.put((name, request)) + if name == "primary" and failure_mode == "mid-stream": + primary_events: Final = ( + _native_anthropic_stream(primary_model, identity + "-primary", primary_text) + if is_messages + else _native_responses_stream(primary_model, identity + "-primary", primary_text) + ) + first_event: Final = primary_events[0] + second_event: Final = ( + primary_events[1] + if is_messages + else _native_sse_event( + { + "type": "response.in_progress", + "sequence_number": 1, + "response": { + **_native_responses_response(primary_model, identity + "-primary", primary_text), + "status": "in_progress", + "output": [], + }, + } + ) + ) + return Reply( + content_type="text/event-stream", + chunks=(first_event, second_event, first_event), + abort_after=2, + pause_between_chunks=0.05, + ) + if name == "primary" and failure_mode in ( + "non-streaming", + "before-first-byte", + "request-body-fallbacks", + ): + return Reply( + status=500, + body=b'{"error":{"message":"synthetic primary failure","type":"server_error","code":"internal_error"}}', + ) + return response_for(model, fallback_text if name == "fallback" else primary_text) + + return handle + + with ( + wire_server(upstream("primary", primary_model)) as primary, + wire_server(upstream("fallback", fallback_model)) as fallback, + ): + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["model_list"] = [ + { + "model_name": model_group, + "litellm_params": { + "model": f"{'anthropic' if is_messages else 'openai'}/{model}", + "api_key": "synthetic-native-fallback-key", + "api_base": server.url if is_messages else server.url + "/v1", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + }, + } + for model_group, model, server in ( + ("primary", primary_model, primary), + ("fallback", fallback_model, fallback), + ) + ] + config["router_settings"] = { + "num_retries": 0, + "disable_cooldowns": True, + "fallbacks": [] if failure_mode == "request-body-fallbacks" else [{"primary": ["fallback"]}], + } + proxy_config: Final = tmp_path / f"native-{route}-{failure_mode}.yaml" + proxy_config.write_text(yaml.safe_dump(config)) + with owned_proxy(gateway, tmp_path, {}, config=proxy_config) as candidate: + request_headers: Final = { + "x-litellm-call-id": identity, + **( + {"x-api-key": candidate.key, "anthropic-version": "2023-06-01"} + if is_messages + else {"Authorization": f"Bearer {candidate.key}"} + ), + } + with candidate.client.stream("POST", path, json=client_body, headers=request_headers) as response: + response_text: Final = response.read().decode() + assert response.status_code == 200, response_text + assert response.headers.get("x-litellm-attempted-fallbacks") == "1", response_text + assert tuple(call_order.get_nowait() for _ in range(call_order.qsize())) == ("primary", "fallback") + assert tuple( + (name, request.method, request.target, json.loads(request.body)) + for name, request in (received.get_nowait() for _ in range(received.qsize())) + ) == (("primary", "POST", path, primary_body), ("fallback", "POST", path, fallback_body)) + if stream: + event_lines: Final = tuple(line for line in sse_data_lines(response_text) if line != "[DONE]") + events: Final = tuple(JSON_OBJECT.validate_json(line) for line in event_lines) + event_types: Final = tuple(event["type"] for event in events) + expected_types: Final = ( + ( + "message_start", + "content_block_start", + "content_block_delta", + "content_block_stop", + "message_delta", + "message_stop", + ) + if is_messages + else ("response.created", "response.output_text.delta", "response.completed") + ) + assert event_types.count("message_start" if is_messages else "response.created") == 1, response_text + assert event_types.count("message_stop" if is_messages else "response.completed") == 1, response_text + assert event_types[-1] == ("message_stop" if is_messages else "response.completed"), response_text + assert event_types == expected_types, response_text + output_text: Final = ( + "".join( + string_value(object_value(event["delta"])["text"]) + for event in events + if event["type"] == "content_block_delta" + ) + if is_messages + else "".join( + string_value(event["delta"]) + for event in events + if event["type"] == "response.output_text.delta" + ) + ) + assert output_text == fallback_text, response_text + else: + response_body: Final = JSON_OBJECT.validate_json(response.content) + expected_response: Final = ( + _native_anthropic_message(fallback_model, identity, fallback_text) + if is_messages + else { + "id": response_body["id"], + "created_at": 1, + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": None, + "model": fallback_model, + "object": "response", + "output": [ + { + "id": "msg_" + identity, + "content": [ + { + "annotations": [], + "text": fallback_text, + "type": "output_text", + "logprobs": None, + } + ], + "role": "assistant", + "status": "completed", + "type": "message", + "phase": None, + } + ], + "parallel_tool_calls": None, + "temperature": None, + "tool_choice": None, + "tools": None, + "top_p": None, + "max_output_tokens": None, + "previous_response_id": None, + "reasoning": None, + "status": "completed", + "text": None, + "truncation": None, + "usage": { + "input_tokens": 11, + "input_tokens_details": None, + "output_tokens": 4, + "output_tokens_details": None, + "total_tokens": 15, + "cost": None, + }, + "user": None, + "store": None, + } + ) + assert response_body == expected_response, response_text + if not is_messages: + assert response_body["id"].startswith("resp_"), response_text + rows: Final = eventually( + lambda: read_rows( + "SELECT model, model_group, spend, prompt_tokens, completion_tokens, status " + 'FROM "LiteLLM_SpendLogs" WHERE request_id=%s OR litellm_call_id=%s', + (identity, identity), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert rows[0]["model_group"] == "fallback", rows + assert rows[0]["status"] == "success", rows + assert (rows[0]["prompt_tokens"], rows[0]["completion_tokens"]) == (11, 4), rows + assert float(rows[0]["spend"]) == pytest.approx(0.019), rows + + @pytest.mark.covers("other.streaming.failure.truncated_transport_raises_and_control_recovers") def test_truncated_http_stream_is_an_error_and_next_stream_succeeds() -> None: import litellm From 0ca9f8185cef25bc714cb924456029989bbc13ba Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:57:41 +0000 Subject: [PATCH 05/51] test(integration): cover files_batches endpoint contracts (#44703) * test(integration): cover files_batches endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): split raw cloud-storage delete test by caller Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): drop mutable polling state in content download test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): dispatch request spellings without rebinding Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): import assert_never from typing_extensions for python 3.10 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): tighten files_batches wire and access assertions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): tidy files_batches access tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover repeated target models, upload expiry, batch aliases Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): prove files streaming path, SDK model denial and admin GCS delete Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): give the streaming head the wire barrier's full budget Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: strengthen files and batches wire coverage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: drop duplicate parametrize from the model-routed upload helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: pass the SDK base url in the skipped repeated-plain upload test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): allow batch output metadata lookup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: avoid draining file content requests before wait Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../providers/test_files_batches_wire.py | 882 ++++++++++++++++++ .../security/test_files_batches_access.py | 851 +++++++++++++++++ 2 files changed, 1733 insertions(+) create mode 100644 tests/integration/providers/test_files_batches_wire.py create mode 100644 tests/integration/security/test_files_batches_access.py diff --git a/tests/integration/providers/test_files_batches_wire.py b/tests/integration/providers/test_files_batches_wire.py new file mode 100644 index 00000000000..45ee40652a5 --- /dev/null +++ b/tests/integration/providers/test_files_batches_wire.py @@ -0,0 +1,882 @@ +import base64 +import json +import re +import threading +import uuid +from collections import deque +from collections.abc import Callable, Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor +from email.message import Message +from email.parser import BytesParser +from email.policy import HTTP +from functools import partial +from queue import Queue +from typing import Final, Literal, cast +from urllib.parse import urlsplit + +import httpx +import pytest +from integration._support.client import Gateway, eventually, object_value, string_value +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, Wire, wire_server +from openai import OpenAI +from pydantic import BaseModel, JsonValue, TypeAdapter +from typing_extensions import assert_never + +from litellm.proxy.openai_files_endpoints.common_utils import encode_file_id_with_model + +INPUT_FILENAME: Final = "in.jsonl" +OUTPUT_BYTES: Final = b'{"custom_id": "r1", "result": {"response": {"body": "ok"}}}\n' +STREAM_CHUNK_BYTES: Final = 1024 * 1024 +_HEAD_OPEN: Final = b'{"custom_id": "r0", "padding": "' +_HEAD_CLOSE: Final = b'"}\n' +OUTPUT_HEAD: Final = _HEAD_OPEN + b"h" * (STREAM_CHUNK_BYTES - len(_HEAD_OPEN) - len(_HEAD_CLOSE)) + _HEAD_CLOSE +MANAGED_PREFIX: Final = "litellm_proxy:" +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +MANAGED_FILE_ROW: Final = ( + 'SELECT model_mappings, flat_model_file_ids FROM "LiteLLM_ManagedFileTable" WHERE unified_file_id = %s' +) +_BatchUploadSpelling = Literal["sdk_list", "repeated_bracket", "indexed", "comma_joined", "repeated_plain"] +_ModelRoutedSpelling = Literal[ + "sdk_extra_body", + "sdk_proxy_fields", + "multipart_field", + "query_param", + "model_header", + "multipart_proxy_fields", + "sdk_expires_after", + "multipart_expires_after", +] +_ContentRoute = Literal["v1_encoded", "files_encoded", "files_raw_query", "v1_raw_header"] +_BatchCreateSpelling = Literal["encoded_input", "body_model", "query_model", "header_model", "sanitized_metadata"] + + +def _jsonl(model: str) -> bytes: + line: Final = { + "custom_id": "r1", + "method": "POST", + "url": "/v1/chat/completions", + "body": {"model": model, "messages": [{"role": "user", "content": "one"}]}, + } + return json.dumps(line, separators=(",", ":")).encode() + b"\n" + + +def _rewritten_jsonl(model: str) -> bytes: + line: Final = { + "custom_id": "r1", + "method": "POST", + "url": "/v1/chat/completions", + "body": {"model": model, "messages": [{"role": "user", "content": "one"}]}, + } + return json.dumps(line).encode() + + +class _FileObject(BaseModel): + id: str + object: str + bytes: int + created_at: int + filename: str + purpose: str + + +class _Batch(BaseModel): + id: str + object: str + input_file_id: str | None = None + output_file_id: str | None = None + error_file_id: str | None = None + status: str + + +def _json(response: httpx.Response) -> dict[str, JsonValue]: + return JSON_OBJECT.validate_json(response.content) + + +def _json_reply(body: Mapping[str, JsonValue], status: int = 200) -> Reply: + return Reply(status=status, body=json.dumps(body).encode()) + + +def _file_object(file_id: str, byte_count: int, filename: str = INPUT_FILENAME) -> dict[str, JsonValue]: + return { + "id": file_id, + "object": "file", + "bytes": byte_count, + "created_at": 1700000000, + "filename": filename, + "purpose": "batch", + "status": "processed", + } + + +def _expected_file_object(file_id: str, byte_count: int, filename: str = INPUT_FILENAME) -> _FileObject: + return _FileObject( + id=file_id, + object="file", + bytes=byte_count, + created_at=1700000000, + filename=filename, + purpose="batch", + ) + + +def _batch_object( + batch_id: str, input_file_id: str, status: str, output_file_id: str | None = None, error_file_id: str | None = None +) -> dict[str, JsonValue]: + return { + "id": batch_id, + "object": "batch", + "endpoint": "/v1/chat/completions", + "input_file_id": input_file_id, + "output_file_id": output_file_id, + "error_file_id": error_file_id, + "status": status, + "created_at": 1700000000, + "completion_window": "24h", + } + + +def _multipart_parts(request: Request) -> tuple[Message, ...]: + envelope: Final = f"content-type: {request.headers['content-type']}\r\n\r\n".encode() + request.body + parsed: Final = BytesParser(policy=HTTP).parsebytes(envelope) + assert parsed.is_multipart(), request.headers["content-type"] + return tuple(parsed.iter_parts()) + + +def _text_fields(parts: tuple[Message, ...]) -> dict[str, str]: + return { + part.get_param("name", header="content-disposition"): part.get_payload(decode=True).decode() + for part in parts + if part.get_filename() is None + } + + +def _file_fields(parts: tuple[Message, ...]) -> dict[str, tuple[str, str, bytes]]: + return { + part.get_param("name", header="content-disposition") or "": ( + part.get_filename() or "", + part.get_content_type(), + part.get_payload(decode=True) or b"", + ) + for part in parts + if part.get_filename() is not None + } + + +def _drained_other_than_model_list_probes(wire: Wire) -> tuple[Request, ...]: + return tuple( + request for request in wire.drain() if urlsplit(request.target).path not in {"/v1/models", "/openai/models"} + ) + + +def _bearer(request: Request) -> str: + return request.headers.get("authorization", request.headers.get("api-key", "")).removeprefix("Bearer ") + + +def _sdk_base_url(gateway: Gateway) -> str: + return str(gateway.client.base_url).rstrip("/") + "/v1" + + +def _decoded_unified(managed_file_id: str) -> str: + decoded: Final = base64.urlsafe_b64decode(managed_file_id + "=" * (-len(managed_file_id) % 4)).decode() + assert decoded.startswith(MANAGED_PREFIX), decoded + return decoded + + +def _models_over_a_fresh_connection(gateway: Gateway, _: int) -> frozenset[str]: + with httpx.Client(base_url=gateway.client.base_url, timeout=15, trust_env=False) as client: + listed: Final = client.get("/v1/models", headers={"Authorization": f"Bearer {gateway.key}"}) + assert listed.status_code == 200, listed.text + data: Final = _json(listed)["data"] + assert isinstance(data, list), listed.text + return frozenset(string_value(object_value(entry)["id"]) for entry in data) + + +def _every_worker_serves(gateway: Gateway, *models: str) -> bool: + with ThreadPoolExecutor(max_workers=16) as pool: + rounds: Final = tuple( + tuple(pool.map(partial(_models_over_a_fresh_connection, gateway), range(16))) for _ in range(2) + ) + return all(model in seen for model in models for round_ in rounds for seen in round_) + + +def _wait_until_every_worker_serves(gateway: Gateway, *models: str) -> None: + eventually(lambda: _every_worker_serves(gateway, *models), lambda served: served, seconds=90) + + +def _deployment_ids(gateway: Gateway, *model_names: str) -> dict[str, str]: + entries: Final = gateway.get("/model/info")["data"] + assert isinstance(entries, list) + by_name: Final = { + string_value(entry["model_name"]): string_value(object_value(entry["model_info"])["id"]) + for entry in entries + if isinstance(entry, dict) and entry.get("model_name") in model_names + } + return by_name + + +def _multiplexed_openai_backends( + created_batch: Mapping[str, JsonValue] | None = None, + content_gate: threading.Event | None = None, +) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + raw_path: Final = urlsplit(request.target).path + path: Final = "/v1/" + raw_path.removeprefix("/openai/") if raw_path.startswith("/openai/") else raw_path + if request.method == "POST" and path == "/v1/files": + file_part: Final = _file_fields(_multipart_parts(request))["file"] + return _json_reply(_file_object("file-" + _bearer(request), len(file_part[2]), file_part[0])) + if request.method == "POST" and path == "/v1/batches": + if created_batch is not None: + return _json_reply(created_batch) + return _json_reply(_batch_object("batch-" + _bearer(request), "file-pending", "in_progress")) + if request.method == "POST" and re.fullmatch(r"/v1/batches/[^/]+/cancel", path): + return _json_reply(_batch_object("batch-" + _bearer(request), "file-pending", "cancelled")) + if request.method == "GET" and path.startswith("/v1/batches/"): + return _json_reply( + _batch_object( + path.rsplit("/", 1)[1], "file-in", "completed", output_file_id="file-out-" + _bearer(request) + ) + ) + if request.method == "GET" and path.startswith("/v1/files/") and path.endswith("/content"): + if content_gate is not None: + return Reply( + content_type="application/octet-stream", + chunks=(OUTPUT_HEAD, OUTPUT_BYTES), + gate_after_first=content_gate, + ) + return Reply(body=OUTPUT_BYTES, content_type="application/octet-stream") + if request.method == "GET" and path.startswith("/v1/files/"): + return _json_reply(_file_object(path.rsplit("/", 1)[1], len(_jsonl("uploaded-model")))) + if request.method == "DELETE" and path.startswith("/v1/files/"): + return _json_reply({"id": path.rsplit("/", 1)[1], "object": "file", "deleted": True}) + return _json_reply({"error": {"message": f"unscripted {request.method} {request.target}"}}, 404) + + return respond + + +def _managed_upload( + gateway: Gateway, spelling: _BatchUploadSpelling, fields: Mapping[str, object], sdk_base_url: str +) -> _FileObject: + key: Final = gateway.key + match spelling: + case "sdk_list": + with OpenAI(base_url=sdk_base_url, api_key=key, max_retries=0) as client: + response: Final = client.files.with_raw_response.create( + file=(INPUT_FILENAME, _jsonl("uploaded-model"), "application/jsonl"), + purpose="batch", + extra_body=fields, + ) + assert response.status_code == 200, response.text + return _FileObject.model_validate_json(response.content) + case "repeated_bracket" | "indexed" | "repeated_plain": + posted: Final = gateway.client.post( + "/v1/files", + data={"purpose": "batch", **fields}, + files={"file": (INPUT_FILENAME, _jsonl("uploaded-model"), "application/jsonl")}, + headers={"Authorization": f"Bearer {key}"}, + ) + assert posted.status_code == 200, posted.text + return _FileObject.model_validate_json(posted.content) + case "comma_joined": + created: Final = gateway.request_multipart( + "/v1/files", + {"purpose": "batch", **fields}, + {"file": (INPUT_FILENAME, _jsonl("uploaded-model"), "application/jsonl")}, + ) + assert created.status_code == 200, created.text + return _FileObject.model_validate_json(created.content) + case _: + assert_never(spelling) + + +def _assert_upload_reaches_both_target_models( + gateway: Gateway, spelling: _BatchUploadSpelling, sdk_base_url: str +) -> None: + key_a: Final = "provider-key-a-" + uuid.uuid4().hex[:8] + key_b: Final = "provider-key-b-" + uuid.uuid4().hex[:8] + with wire_server(_multiplexed_openai_backends()) as wire, gateway.scenario() as scenario: + model_a: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key=key_a) + model_b: Final = scenario.model(model="openai/gpt-4.1-mini", api_base=wire.url + "/v1", api_key=key_b) + _wait_until_every_worker_serves(gateway, model_a, model_b) + fields: Final[Mapping[str, object]] = { + "sdk_list": {"target_model_names": [model_a, model_b]}, + "repeated_bracket": {"target_model_names[]": [model_a, model_b]}, + "indexed": {"target_model_names[0]": model_a, "target_model_names[1]": model_b}, + "comma_joined": {"target_model_names": f"{model_a},{model_b}"}, + "repeated_plain": {"target_model_names": [model_a, model_b]}, + }[spelling] + managed_file: Final = _managed_upload(gateway, spelling, fields, sdk_base_url) + assert managed_file.created_at > 0, managed_file.model_dump() + managed: Final = managed_file.id + + requests: Final = _drained_other_than_model_list_probes(wire) + assert sorted((r.method, r.target, r.headers["authorization"]) for r in requests) == [ + ("POST", "/v1/files", f"Bearer {key_a}"), + ("POST", "/v1/files", f"Bearer {key_b}"), + ], requests + uploads: Final = {request.headers["authorization"]: request for request in requests} + for bearer, underlying in ((key_a, "gpt-4o-mini"), (key_b, "gpt-4.1-mini")): + parts: Final = _multipart_parts(uploads[f"Bearer {bearer}"]) + assert len(parts) == 2, [part.get_param("name", header="content-disposition") for part in parts] + assert _text_fields(parts) == {"purpose": "batch"} + file_part: Final = _file_fields(parts)["file"] + assert (file_part[0], file_part[2]) == ("modified_file.jsonl", _rewritten_jsonl(underlying)), file_part + + managed_file_part: Final = _file_fields(_multipart_parts(uploads[f"Bearer {key_a}"]))["file"] + expected_file: Final = _expected_file_object( + managed_file.id, + len(managed_file_part[2]), + managed_file_part[0], + ) + assert managed_file == expected_file, managed_file.model_dump() + + decoded: Final = _decoded_unified(managed) + assert f"target_model_names,{model_a},{model_b}" in decoded, decoded + deployment_ids: Final = _deployment_ids(gateway, model_a, model_b) + rows: Final = eventually( + lambda: read_rows(MANAGED_FILE_ROW, (managed,)), + lambda found: len(found) == 1, + ) + assert rows[0]["model_mappings"] == { + deployment_ids[model_a]: f"file-{key_a}", + deployment_ids[model_b]: f"file-{key_b}", + }, rows + assert sorted(rows[0]["flat_model_file_ids"]) == sorted([f"file-{key_a}", f"file-{key_b}"]), rows + + readback: Final = gateway.request("GET", f"/v1/files/{managed}") + assert readback.status_code == 200, readback.text + readback_file: Final = _FileObject.model_validate_json(readback.content) + assert readback_file == expected_file, readback.text + assert _drained_other_than_model_list_probes(wire) == () + + +@pytest.mark.parametrize( + "spelling", + ["sdk_list", "repeated_bracket", "indexed", "comma_joined"], + ids=["sdk-list-extra-body", "multipart-repeated-bracket", "multipart-indexed", "multipart-comma-joined"], +) +def test_batch_upload_to_two_target_models_reaches_each_deployment( + gateway: Gateway, spelling: _BatchUploadSpelling +) -> None: + _assert_upload_reaches_both_target_models(gateway, spelling, _sdk_base_url(gateway)) + + +@pytest.mark.parametrize( + "route", + ["/files", "/openai/v1/files"], + ids=["files-alias", "openai-v1-files-alias"], +) +def test_batch_upload_to_two_target_models_through_a_route_alias_reaches_each_deployment( + gateway: Gateway, route: str +) -> None: + sdk_base_url: Final = str(gateway.client.base_url).rstrip("/") + route.removesuffix("/files") + _assert_upload_reaches_both_target_models(gateway, "sdk_list", sdk_base_url) + + +def test_batch_upload_with_repeated_plain_target_model_names_reaches_each_deployment(gateway: Gateway) -> None: + pytest.skip("BUG: repeated plain target_model_names keeps only the last value") + _assert_upload_reaches_both_target_models(gateway, "repeated_plain", _sdk_base_url(gateway)) + + +def test_batch_upload_to_two_target_models_keeps_the_jsonl_content_type(gateway: Gateway) -> None: + pytest.skip( + "BUG: managed multi-target upload sends the rewritten batch file as application/octet-stream instead of " + "application/jsonl" + ) + key_a: Final = "provider-key-a-" + uuid.uuid4().hex[:8] + key_b: Final = "provider-key-b-" + uuid.uuid4().hex[:8] + with wire_server(_multiplexed_openai_backends()) as wire, gateway.scenario() as scenario: + model_a: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key=key_a) + model_b: Final = scenario.model(model="openai/gpt-4.1-mini", api_base=wire.url + "/v1", api_key=key_b) + _wait_until_every_worker_serves(gateway, model_a, model_b) + _managed_upload( + gateway, + "repeated_bracket", + {"target_model_names[]": [model_a, model_b]}, + _sdk_base_url(gateway), + ) + uploads: Final = { + request.headers["authorization"]: request for request in _drained_other_than_model_list_probes(wire) + } + content_types: Final = { + f"Bearer {key}": _file_fields(_multipart_parts(uploads[f"Bearer {key}"]))["file"][1] + for key in (key_a, key_b) + } + assert content_types == { + f"Bearer {key_a}": "application/jsonl", + f"Bearer {key_b}": "application/jsonl", + }, content_types + + +def _model_routed_upload( + gateway: Gateway, spelling: _ModelRoutedSpelling, alias: str, uploaded: bytes, sdk_base_url: str +) -> _FileObject: + match spelling: + case "sdk_extra_body": + with OpenAI(base_url=sdk_base_url, api_key=gateway.key, max_retries=0) as client: + response: Final = client.files.with_raw_response.create( + file=(INPUT_FILENAME, uploaded, "application/jsonl"), + purpose="batch", + extra_body={"model": alias}, + ) + assert response.status_code == 200, response.text + return _FileObject.model_validate_json(response.content) + case "sdk_proxy_fields": + with OpenAI(base_url=sdk_base_url, api_key=gateway.key, max_retries=0) as client: + response: Final = client.files.with_raw_response.create( + file=(INPUT_FILENAME, uploaded, "application/jsonl"), + purpose="batch", + extra_body={ + "model": alias, + "custom_llm_provider": "openai", + "target_model_names": alias, + }, + ) + assert response.status_code == 200, response.text + return _FileObject.model_validate_json(response.content) + case "multipart_field": + created: Final = gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "model": alias}, + {"file": (INPUT_FILENAME, uploaded, "application/jsonl")}, + ) + assert created.status_code == 200, created.text + return _FileObject.model_validate_json(created.content) + case "query_param": + queried: Final = gateway.client.post( + "/v1/files", + params={"model": alias}, + data={"purpose": "batch"}, + files={"file": (INPUT_FILENAME, uploaded, "application/jsonl")}, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) + assert queried.status_code == 200, queried.text + return _FileObject.model_validate_json(queried.content) + case "model_header": + headed: Final = gateway.client.post( + "/v1/files", + data={"purpose": "batch"}, + files={"file": (INPUT_FILENAME, uploaded, "application/jsonl")}, + headers={"Authorization": f"Bearer {gateway.key}", "x-litellm-model": alias}, + ) + assert headed.status_code == 200, headed.text + return _FileObject.model_validate_json(headed.content) + case "multipart_proxy_fields": + posted: Final = gateway.client.post( + "/v1/files", + data={ + "purpose": "batch", + "model": alias, + "target_model_names": alias, + "custom_llm_provider": "openai", + "litellm_metadata[tag]": "x", + }, + files={"file": (INPUT_FILENAME, uploaded, "application/jsonl")}, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) + assert posted.status_code == 200, posted.text + return _FileObject.model_validate_json(posted.content) + case "sdk_expires_after": + with OpenAI(base_url=sdk_base_url, api_key=gateway.key, max_retries=0) as client: + response: Final = client.files.with_raw_response.create( + file=(INPUT_FILENAME, uploaded, "application/jsonl"), + purpose="batch", + expires_after={"anchor": "created_at", "seconds": 2592000}, + extra_body={"model": alias}, + ) + assert response.status_code == 200, response.text + return _FileObject.model_validate_json(response.content) + case "multipart_expires_after": + created: Final = gateway.request_multipart( + "/v1/files", + { + "purpose": "batch", + "model": alias, + "expires_after[anchor]": "created_at", + "expires_after[seconds]": "2592000", + }, + {"file": (INPUT_FILENAME, uploaded, "application/jsonl")}, + ) + assert created.status_code == 200, created.text + return _FileObject.model_validate_json(created.content) + case _: + assert_never(spelling) + + +def _assert_model_routed_upload_reaches_one_deployment_and_the_encoded_id_reads_back( + gateway: Gateway, spelling: _ModelRoutedSpelling, sdk_base_url: str +) -> None: + key: Final = "provider-key-" + uuid.uuid4().hex[:8] + uploaded: Final = _jsonl("uploaded-model") + with wire_server(_multiplexed_openai_backends()) as wire, gateway.scenario() as scenario: + alias: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key=key) + _wait_until_every_worker_serves(gateway, alias) + uploaded_file: Final = _model_routed_upload(gateway, spelling, alias, uploaded, sdk_base_url) + + expected_encoded: Final = encode_file_id_with_model(file_id=f"file-{key}", model=alias) + assert uploaded_file == _expected_file_object(expected_encoded, len(uploaded)), uploaded_file.model_dump() + encoded: Final = uploaded_file.id + (upload,) = _drained_other_than_model_list_probes(wire) + assert (upload.method, upload.target) == ("POST", "/v1/files"), upload.target + assert upload.headers["authorization"] == f"Bearer {key}", upload.headers + parts: Final = _multipart_parts(upload) + expected_text: Final = { + "sdk_extra_body": {"purpose": "batch"}, + "multipart_field": {"purpose": "batch"}, + "query_param": {"purpose": "batch"}, + "model_header": {"purpose": "batch"}, + "multipart_proxy_fields": {"purpose": "batch"}, + "sdk_proxy_fields": {"purpose": "batch"}, + "sdk_expires_after": { + "purpose": "batch", + "expires_after[anchor]": "created_at", + "expires_after[seconds]": "2592000", + }, + "multipart_expires_after": { + "purpose": "batch", + "expires_after[anchor]": "created_at", + "expires_after[seconds]": "2592000", + }, + }[spelling] + assert len(parts) == len(expected_text) + 1, [ + part.get_param("name", header="content-disposition") for part in parts + ] + assert _text_fields(parts) == expected_text + assert _file_fields(parts) == {"file": (INPUT_FILENAME, "application/jsonl", uploaded)} + + readback: Final = gateway.request("GET", f"/v1/files/{encoded}") + assert readback.status_code == 200, readback.text + readback_file: Final = _FileObject.model_validate_json(readback.content) + assert readback_file == _expected_file_object(encoded, len(uploaded)), readback.text + (fetched,) = _drained_other_than_model_list_probes(wire) + assert (fetched.method, fetched.target) == ("GET", f"/v1/files/file-{key}"), fetched.target + assert fetched.headers["authorization"] == f"Bearer {key}", fetched.headers + assert fetched.body == b"" + + +@pytest.mark.parametrize( + "spelling", + [ + "sdk_extra_body", + "sdk_proxy_fields", + "multipart_field", + "query_param", + "model_header", + "multipart_proxy_fields", + "sdk_expires_after", + "multipart_expires_after", + ], + ids=[ + "sdk-extra-body-model", + "sdk-extra-body-with-proxy-fields", + "multipart-model-field", + "query-model", + "x-litellm-model-header", + "multipart-model-with-proxy-fields", + "sdk-expires-after", + "multipart-bracketed-expires-after", + ], +) +def test_model_routed_upload_reaches_one_deployment_and_the_encoded_id_reads_back( + gateway: Gateway, spelling: _ModelRoutedSpelling +) -> None: + _assert_model_routed_upload_reaches_one_deployment_and_the_encoded_id_reads_back( + gateway, spelling, _sdk_base_url(gateway) + ) + + +@pytest.mark.parametrize( + "route", + ["/files", "/openai/v1/files"], + ids=["files-alias", "openai-v1-files-alias"], +) +def test_model_routed_upload_through_a_route_alias_reaches_one_deployment( + gateway: Gateway, route: str +) -> None: + sdk_base_url: Final = str(gateway.client.base_url).rstrip("/") + route.removesuffix("/files") + _assert_model_routed_upload_reaches_one_deployment_and_the_encoded_id_reads_back( + gateway, "sdk_extra_body", sdk_base_url + ) + + +AZURE_API_VERSION: Final = "2024-10-21" + + +def _encoded_output_file_id(gateway: Gateway, alias: str, bearer_key: str) -> tuple[str, str]: + uploaded_bytes: Final = _jsonl("uploaded-model") + uploaded: Final = gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "model": alias}, + {"file": (INPUT_FILENAME, uploaded_bytes, "application/jsonl")}, + ) + assert uploaded.status_code == 200, uploaded.text + encoded_file: Final = encode_file_id_with_model(file_id=f"file-{bearer_key}", model=alias) + uploaded_file: Final = _FileObject.model_validate_json(uploaded.content) + assert uploaded_file == _expected_file_object(encoded_file, len(uploaded_bytes)), uploaded.text + encoded_input: Final = uploaded_file.id + created: Final = gateway.request( + "POST", + "/v1/batches", + {"input_file_id": encoded_input, "endpoint": "/v1/chat/completions", "completion_window": "24h"}, + ) + assert created.status_code == 200, created.text + encoded_batch: Final = string_value(_json(created)["id"]) + retrieved: Final = gateway.request("GET", f"/v1/batches/{encoded_batch}") + assert retrieved.status_code == 200, retrieved.text + batch: Final = _Batch.model_validate_json(retrieved.content) + assert batch.output_file_id is not None, retrieved.text + return batch.output_file_id, f"file-out-{bearer_key}" + + +def _content_request( + gateway: Gateway, route: _ContentRoute, encoded_output: str, raw_output: str, alias: str +) -> httpx.Request: + authorization: Final = {"Authorization": f"Bearer {gateway.key}"} + match route: + case "v1_encoded": + return gateway.client.build_request("GET", f"/v1/files/{encoded_output}/content", headers=authorization) + case "files_encoded": + return gateway.client.build_request("GET", f"/files/{encoded_output}/content", headers=authorization) + case "files_raw_query": + return gateway.client.build_request( + "GET", f"/files/{raw_output}/content", params={"model": alias}, headers=authorization + ) + case "v1_raw_header": + return gateway.client.build_request( + "GET", f"/v1/files/{raw_output}/content", headers={**authorization, "x-litellm-model": alias} + ) + case _: + assert_never(route) + + +def _open_and_read_head( + client: httpx.Client, request: httpx.Request +) -> tuple[httpx.Response, Iterator[bytes], bytes]: + response: Final = client.send(request, stream=True) + chunks: Final = response.iter_bytes() + return response, chunks, next(chunks, b"") + + +def _gated_download(gateway: Gateway, request: httpx.Request, gate: threading.Event) -> tuple[int, bytes, bytes]: + with ThreadPoolExecutor(max_workers=1) as reader: + head_read: Final = reader.submit(_open_and_read_head, gateway.client, request) + try: + head_arrived: Final = eventually(head_read.done, bool, seconds=4.5, return_last_on_timeout=True) + finally: + gate.set() + response, chunks, head = head_read.result() + try: + rest: Final = b"".join(chunks) + finally: + response.close() + assert head_arrived, ( + f"client got no content bytes while the upstream held the rest of the file: {response.status_code}" + ) + return response.status_code, head, rest + + +@pytest.mark.parametrize("provider", ["openai", "azure"], ids=["openai-streaming", "azure-non-streaming"]) +@pytest.mark.parametrize( + "route", + ["v1_encoded", "files_encoded", "files_raw_query", "v1_raw_header"], + ids=["v1-files-encoded", "files-encoded", "files-raw-model-query", "v1-files-raw-model-header"], +) +def test_batch_output_file_content_downloads_through_the_model_encoded_id( + gateway: Gateway, provider: str, route: _ContentRoute +) -> None: + key: Final = f"provider-key-{provider}-" + uuid.uuid4().hex[:8] + gate: Final = threading.Event() + gate.set() + received: Final[Queue[Request]] = Queue() + backend: Final = _multiplexed_openai_backends(content_gate=gate) + + def respond(request: Request) -> Reply: + received.put(request) + return backend(request) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + deployment: Final = ( + {"model": "openai/gpt-4o-mini", "api_base": wire.url + "/v1", "api_key": key} + if provider == "openai" + else { + "model": "azure/gpt-4o-mini", + "api_base": wire.url, + "api_version": AZURE_API_VERSION, + "api_key": key, + } + ) + alias: Final = scenario.model(**deployment) + _wait_until_every_worker_serves(gateway, alias) + encoded_output, raw_output = _encoded_output_file_id(gateway, alias, key) + assert encoded_output == encode_file_id_with_model(file_id=raw_output, model=alias), encoded_output + + eventually( + lambda: tuple(cast(deque[Request], received.queue)), + lambda pending: any(urlsplit(request.target).path.endswith("/content") for request in pending), + ) + settled: Final = _drained_other_than_model_list_probes(wire) + base_prefix: Final = "/v1/" if provider == "openai" else "/openai/" + api_suffix: Final = "" if provider == "openai" else f"?api-version={AZURE_API_VERSION}" + expected_content_target: Final = f"{base_prefix}files/{raw_output}/content{api_suffix}" + expected_requests: Final = ( + ("POST", f"{base_prefix}files{api_suffix}"), + ("POST", f"{base_prefix}batches{api_suffix}"), + ("GET", f"{base_prefix}batches/batch-{key}{api_suffix}"), + ("GET", expected_content_target), + ) + observed_requests: Final = tuple((r.method, r.target) for r in settled) + expected_metadata_request: Final = ("GET", f"{base_prefix}files/{raw_output}{api_suffix}") + assert observed_requests.count(expected_metadata_request) <= 1, observed_requests + assert tuple(request for request in observed_requests if request != expected_metadata_request) == ( + expected_requests + ), observed_requests + + if provider == "openai": + gate.clear() + status, head, rest = _gated_download( + gateway, _content_request(gateway, route, encoded_output, raw_output, alias), gate + ) + assert status == 200, (head + rest).decode() + assert head + rest == OUTPUT_HEAD + OUTPUT_BYTES, f"{len(head)} head bytes, {len(rest)} tail bytes" + (content_request,) = _drained_other_than_model_list_probes(wire) + assert (content_request.method, content_request.target) == ("GET", expected_content_target), ( + content_request.target + ) + assert content_request.body == b"", content_request.body + if provider == "openai": + assert content_request.headers["authorization"] == f"Bearer {key}", content_request.headers + else: + assert content_request.headers["api-key"] == key, content_request.headers + + +def _encoded_batch_input( + gateway: Gateway, spelling: _BatchCreateSpelling, alias: str, wire: Wire, raw_input: str, bearer_key: str +) -> str: + if spelling == "encoded_input": + uploaded_bytes: Final = _jsonl("uploaded-model") + uploaded: Final = gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "model": alias}, + {"file": (INPUT_FILENAME, uploaded_bytes, "application/jsonl")}, + ) + assert uploaded.status_code == 200, uploaded.text + encoded_file: Final = encode_file_id_with_model(file_id=f"file-{bearer_key}", model=alias) + uploaded_file: Final = _FileObject.model_validate_json(uploaded.content) + assert uploaded_file == _expected_file_object(encoded_file, len(uploaded_bytes)), uploaded.text + wire.drain() + return uploaded_file.id + return encode_file_id_with_model(file_id=raw_input, model=alias) + + +def _created_batch( + gateway: Gateway, + spelling: _BatchCreateSpelling, + encoded_input: str, + raw_input: str, + alias: str, + sdk_base_url: str, +) -> _Batch: + if spelling in {"query_model", "header_model"}: + created: Final = gateway.request( + "POST", + "/v1/batches", + { + "input_file_id": raw_input, + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "metadata": {"job": "x"}, + }, + params={"model": alias} if spelling == "query_model" else None, + headers={"x-litellm-model": alias} if spelling == "header_model" else None, + ) + assert created.status_code == 200, created.text + return _Batch.model_validate_json(created.content) + if spelling == "sanitized_metadata": + created: Final = gateway.request( + "POST", + "/v1/batches", + { + "input_file_id": raw_input, + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "metadata": {"job": "x", "attempt": 2}, + "model": alias, + }, + ) + assert created.status_code == 200, created.text + return _Batch.model_validate_json(created.content) + with OpenAI(base_url=sdk_base_url, api_key=gateway.key, max_retries=0) as client: + extra: Final = {} if spelling == "encoded_input" else {"extra_body": {"model": alias}} + response: Final = client.batches.with_raw_response.create( + input_file_id=encoded_input if spelling == "encoded_input" else raw_input, + endpoint="/v1/chat/completions", + completion_window="24h", + metadata={"job": "x"}, + **extra, + ) + assert response.status_code == 200, response.text + return _Batch.model_validate_json(response.content) + + +def _assert_create_batch_reaches_one_deployment_and_returns_model_encoded_ids( + gateway: Gateway, spelling: _BatchCreateSpelling, sdk_base_url: str +) -> None: + key: Final = "provider-key-" + uuid.uuid4().hex[:8] + raw_input: Final = f"file-in-{key}" + raw_batch: Final = f"batch-{key}" + created_batch: Final = _batch_object( + raw_batch, raw_input, "validating", output_file_id="file-out", error_file_id="file-err" + ) + with wire_server(_multiplexed_openai_backends(created_batch)) as wire, gateway.scenario() as scenario: + alias: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key=key) + _wait_until_every_worker_serves(gateway, alias) + encoded_input: Final = _encoded_batch_input(gateway, spelling, alias, wire, raw_input, key) + created_batch_payload: Final = _created_batch( + gateway, spelling, encoded_input, raw_input, alias, sdk_base_url + ) + + expected_raw_input: Final = f"file-{key}" if spelling == "encoded_input" else raw_input + expected_input: Final = ( + encoded_input if spelling == "encoded_input" else encode_file_id_with_model(file_id=raw_input, model=alias) + ) + assert created_batch_payload.id == encode_file_id_with_model(file_id=raw_batch, model=alias, id_type="batch"), ( + created_batch_payload.id + ) + assert created_batch_payload.output_file_id == encode_file_id_with_model(file_id="file-out", model=alias), ( + created_batch_payload.output_file_id + ) + assert created_batch_payload.error_file_id == encode_file_id_with_model(file_id="file-err", model=alias), ( + created_batch_payload.error_file_id + ) + assert created_batch_payload.input_file_id == expected_input, created_batch_payload.input_file_id + + requests: Final = _drained_other_than_model_list_probes(wire) + assert [(r.method, r.target) for r in requests] == [("POST", "/v1/batches")], requests + (create_request,) = requests + assert create_request.headers["authorization"] == f"Bearer {key}", create_request.headers + expected_body: Final = { + "input_file_id": expected_raw_input, + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "metadata": {"job": "x"}, + } + assert JSON_OBJECT.validate_json(create_request.body) == expected_body, create_request.body + + +@pytest.mark.parametrize( + "spelling", + ["encoded_input", "body_model", "query_model", "header_model", "sanitized_metadata"], + ids=["encoded-input-id", "body-model", "query-model", "x-litellm-model-header", "non-string-metadata-dropped"], +) +def test_create_batch_reaches_one_deployment_and_returns_model_encoded_ids( + gateway: Gateway, spelling: _BatchCreateSpelling +) -> None: + _assert_create_batch_reaches_one_deployment_and_returns_model_encoded_ids( + gateway, spelling, _sdk_base_url(gateway) + ) + + +@pytest.mark.parametrize("route", ["/batches", "/openai/v1/batches"], ids=["batches-alias", "openai-v1-batches-alias"]) +def test_create_batch_through_a_route_alias_reaches_one_deployment(gateway: Gateway, route: str) -> None: + sdk_base_url: Final = str(gateway.client.base_url).rstrip("/") + route.removesuffix("/batches") + _assert_create_batch_reaches_one_deployment_and_returns_model_encoded_ids(gateway, "encoded_input", sdk_base_url) diff --git a/tests/integration/security/test_files_batches_access.py b/tests/integration/security/test_files_batches_access.py new file mode 100644 index 00000000000..7b6fd4652cf --- /dev/null +++ b/tests/integration/security/test_files_batches_access.py @@ -0,0 +1,851 @@ +import base64 +import contextlib +import hashlib +import json +import socket +import socketserver +import threading +import uuid +from collections.abc import Callable, Generator +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from queue import SimpleQueue +from typing import Final, Literal +from urllib.parse import quote, urlsplit + +import httpx +import pytest +from botocore.auth import S3SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import Credentials +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from integration._support.client import Gateway, Scenario, eventually, object_value, string_value +from integration._support.database import read_rows +from integration._support.process import owned_proxy +from integration._support.tls import server_context, write_self_signed_cert +from integration._support.wire import Reply, Request, Wire, wire_server +from openai import OpenAI, PermissionDeniedError +from pydantic import BaseModel, JsonValue, TypeAdapter +from typing_extensions import assert_never + +from litellm.proxy.openai_files_endpoints.common_utils import encode_file_id_with_model + +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +MANAGED_FILE_ROW: Final = 'SELECT model_mappings, flat_model_file_ids, created_by, team_id FROM "LiteLLM_ManagedFileTable" WHERE unified_file_id = %s' +INPUT_FILENAME: Final = "in.jsonl" +MANAGED_PREFIX: Final = "litellm_proxy:" +BUCKET: Final = "integration-delete-bucket" +VERTEX_PROJECT: Final = "files-batches-delete-project" +VERTEX_LOCATION: Final = "us-central1" +VERTEX_MODEL: Final = "vertex_ai/gemini-2.5-flash" +GCS_HOST: Final = "storage.googleapis.com" +GCS_AUTHORITY: Final = f"{GCS_HOST}:443" +VERTEX_ACCESS_TOKEN: Final = "ya29.scripted-delete-token" +AWS_ACCESS_KEY_ID: Final = "AKIAIOSFODNN7EXAMPLE" +AWS_SECRET_ACCESS_KEY: Final = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +AWS_REGION: Final = "us-east-1" +_ForbiddenSurface = Literal[ + "upload_sdk_extra_body", + "upload_form", + "upload_query", + "upload_header", + "batch_encoded_input", + "batch_body_model", + "batch_query_model", + "batch_header_model", + "file_get_encoded", + "file_get_query", + "file_get_header", + "file_content_encoded", + "file_content_query", + "file_content_header", + "batch_get_encoded", + "batch_get_query", + "batch_get_header", +] + + +def _jsonl(model: str) -> bytes: + line: Final = { + "custom_id": "r1", + "method": "POST", + "url": "/v1/chat/completions", + "body": {"model": model, "messages": [{"role": "user", "content": "one"}]}, + } + return json.dumps(line, separators=(",", ":")).encode() + b"\n" + + +class _FileObject(BaseModel): + id: str + object: str + purpose: str + + +class _Batch(BaseModel): + id: str + object: str + status: str + input_file_id: str | None = None + + +class _ErrorEnvelope(BaseModel): + error: dict[str, JsonValue] + + +class _FileDeleted(BaseModel): + id: str + object: str + deleted: bool + + +def _json(response: httpx.Response) -> dict[str, JsonValue]: + return JSON_OBJECT.validate_json(response.content) + + +def _json_reply(body: dict[str, JsonValue], status: int = 200) -> Reply: + return Reply(status=status, body=json.dumps(body).encode()) + + +def _drained_other_than_model_list_probes(wire: Wire) -> tuple[Request, ...]: + return tuple( + request for request in wire.drain() if urlsplit(request.target).path not in {"/v1/models", "/openai/models"} + ) + + +def _bearer(request: Request) -> str: + return request.headers.get("authorization", request.headers.get("api-key", "")).removeprefix("Bearer ") + + +def _file_object(file_id: str) -> dict[str, JsonValue]: + return { + "id": file_id, + "object": "file", + "bytes": 123, + "created_at": 1700000000, + "filename": INPUT_FILENAME, + "purpose": "batch", + "status": "processed", + } + + +def _batch_object(batch_id: str, status: str) -> dict[str, JsonValue]: + return { + "id": batch_id, + "object": "batch", + "endpoint": "/v1/chat/completions", + "input_file_id": "file-pending", + "status": status, + "created_at": 1700000000, + "completion_window": "24h", + } + + +def _openai_backend() -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + path: Final = urlsplit(request.target).path + if request.method == "POST" and path == "/v1/files": + return _json_reply(_file_object("file-" + _bearer(request))) + if request.method == "POST" and path == "/v1/batches": + return _json_reply(_batch_object("batch-" + _bearer(request), "in_progress")) + if request.method == "POST" and path.endswith("/cancel"): + return _json_reply(_batch_object(path.rsplit("/", 2)[1], "cancelled")) + if request.method == "GET" and path.startswith("/v1/batches/"): + return _json_reply(_batch_object(path.rsplit("/", 1)[1], "in_progress")) + if request.method == "GET" and path.startswith("/v1/files/") and path.endswith("/content"): + return Reply(body=b"file-content\n", content_type="application/octet-stream") + if request.method == "GET" and path.startswith("/v1/files/"): + return _json_reply(_file_object(path.rsplit("/", 1)[1])) + if request.method == "DELETE" and path.startswith("/v1/files/"): + return _json_reply({"id": path.rsplit("/", 1)[1], "object": "file", "deleted": True}) + return _json_reply({"error": {"message": f"unscripted {request.method} {request.target}"}}, 404) + + return respond + + +def _decoded_unified(managed_file_id: str) -> str: + decoded: Final = base64.urlsafe_b64decode(managed_file_id + "=" * (-len(managed_file_id) % 4)).decode() + assert decoded.startswith(MANAGED_PREFIX), decoded + return decoded + + +def _models_over_a_fresh_connection(gateway: Gateway, _: int) -> frozenset[str]: + with httpx.Client(base_url=gateway.client.base_url, timeout=15, trust_env=False) as client: + listed: Final = client.get("/v1/models", headers={"Authorization": f"Bearer {gateway.key}"}) + assert listed.status_code == 200, listed.text + data: Final = _json(listed)["data"] + assert isinstance(data, list), listed.text + return frozenset(string_value(object_value(entry)["id"]) for entry in data) + + +def _every_worker_serves(gateway: Gateway, *models: str) -> bool: + with ThreadPoolExecutor(max_workers=16) as pool: + rounds: Final = tuple( + tuple(pool.map(partial(_models_over_a_fresh_connection, gateway), range(16))) for _ in range(2) + ) + return all(model in seen for model in models for round_ in rounds for seen in round_) + + +def _wait_until_every_worker_serves(gateway: Gateway, *models: str) -> None: + eventually(lambda: _every_worker_serves(gateway, *models), lambda served: served, seconds=90) + + +@dataclass(frozen=True, slots=True) +class _Member: + team: str + user: str + key: str + + +def _member(scenario: Scenario, *models: str) -> _Member: + team: Final = scenario.team(models=list(models)) + user: Final = scenario.member(team) + return _Member(team, user, scenario.key(team_id=team, user_id=user)) + + +def _error_message(response: httpx.Response, status: int) -> str: + assert response.status_code == status, response.text + error: Final = _ErrorEnvelope.model_validate_json(response.content) + return str(error.error["message"]) + + +def test_stranger_cannot_read_download_or_delete_another_teams_managed_file(gateway: Gateway) -> None: + key: Final = "provider-key-" + uuid.uuid4().hex[:8] + with wire_server(_openai_backend()) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key=key) + _wait_until_every_worker_serves(gateway, model) + owner: Final = _member(scenario, model) + stranger: Final = _member(scenario, model) + + uploaded: Final = gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "target_model_names": model}, + {"file": (INPUT_FILENAME, _jsonl(model), "application/jsonl")}, + key=owner.key, + ) + assert uploaded.status_code == 200, uploaded.text + managed: Final = string_value(_json(uploaded)["id"]) + wire.drain() + row: Final = eventually(lambda: read_rows(MANAGED_FILE_ROW, (managed,)), lambda rows: len(rows) == 1)[0] + + denied: Final = ( + gateway.request("GET", f"/v1/files/{managed}", key=stranger.key), + gateway.request("GET", f"/v1/files/{managed}/content", key=stranger.key), + gateway.request("DELETE", f"/v1/files/{managed}", key=stranger.key), + ) + for response in denied: + assert ( + _error_message(response, 403) == f"User {stranger.user} does not have access to the file {managed}" + ), response.text + assert _drained_other_than_model_list_probes(wire) == (), "stranger calls must not reach the provider" + assert read_rows(MANAGED_FILE_ROW, (managed,)) == [row], read_rows(MANAGED_FILE_ROW, (managed,)) + + readback: Final = gateway.request("GET", f"/v1/files/{managed}", key=owner.key) + assert readback.status_code == 200, readback.text + assert string_value(_json(readback)["id"]) == managed, readback.text + assert _drained_other_than_model_list_probes(wire) == () + + +def test_stranger_cannot_retrieve_or_cancel_another_teams_managed_batch(gateway: Gateway) -> None: + key: Final = "provider-key-" + uuid.uuid4().hex[:8] + with wire_server(_openai_backend()) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key=key) + _wait_until_every_worker_serves(gateway, model) + owner: Final = _member(scenario, model) + stranger: Final = _member(scenario, model) + + uploaded: Final = gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "target_model_names": model}, + {"file": (INPUT_FILENAME, _jsonl(model), "application/jsonl")}, + key=owner.key, + ) + assert uploaded.status_code == 200, uploaded.text + managed_input: Final = string_value(_json(uploaded)["id"]) + created: Final = gateway.request( + "POST", + "/v1/batches", + {"input_file_id": managed_input, "endpoint": "/v1/chat/completions", "completion_window": "24h"}, + key=owner.key, + ) + assert created.status_code == 200, created.text + unified_batch: Final = string_value(_json(created)["id"]) + wire.drain() + + expected_message: Final = f"User {stranger.user} does not have access to the object {unified_batch}" + stranger_get: Final = gateway.request("GET", f"/v1/batches/{unified_batch}", key=stranger.key) + assert _error_message(stranger_get, 403) == expected_message, stranger_get.text + stranger_cancel: Final = gateway.request("POST", f"/v1/batches/{unified_batch}/cancel", {}, key=stranger.key) + assert _error_message(stranger_cancel, 403) == expected_message, stranger_cancel.text + assert _drained_other_than_model_list_probes(wire) == (), "stranger batch calls must not reach the provider" + + owner_cancel: Final = gateway.request("POST", f"/v1/batches/{unified_batch}/cancel", {}, key=owner.key) + assert owner_cancel.status_code == 200, owner_cancel.text + assert string_value(_json(owner_cancel)["id"]) == unified_batch, owner_cancel.text + (cancel_request,) = _drained_other_than_model_list_probes(wire) + assert (cancel_request.method, cancel_request.target) == ("POST", f"/v1/batches/batch-{key}/cancel"), ( + cancel_request.target + ) + assert cancel_request.headers["authorization"] == f"Bearer {key}", cancel_request.headers + + +def _sdk_upload_refusal(gateway: Gateway, forbidden: str, caller_key: str) -> httpx.Response: + sdk_base_url: Final = str(gateway.client.base_url).rstrip("/") + "/v1" + with OpenAI(base_url=sdk_base_url, api_key=caller_key, max_retries=0) as client: + try: + uploaded: Final = client.files.with_raw_response.create( + file=(INPUT_FILENAME, _jsonl(forbidden), "application/jsonl"), + purpose="batch", + extra_body={"model": forbidden}, + ) + except PermissionDeniedError as refused: + return refused.response + raise AssertionError(f"SDK upload with a forbidden model was accepted: {uploaded.status_code} {uploaded.text}") + + +def _forbidden_surface_request( + gateway: Gateway, + surface: _ForbiddenSurface, + forbidden: str, + caller_key: str, + raw_file: str, + raw_batch: str, + encoded_file: str, + encoded_batch: str, +) -> httpx.Response: + match surface: + case "upload_sdk_extra_body": + return _sdk_upload_refusal(gateway, forbidden, caller_key) + case "upload_form": + return gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "model": forbidden}, + {"file": (INPUT_FILENAME, _jsonl(forbidden), "application/jsonl")}, + key=caller_key, + ) + case "upload_query": + return gateway.client.post( + "/v1/files", + params={"model": forbidden}, + data={"purpose": "batch"}, + files={"file": (INPUT_FILENAME, _jsonl(forbidden), "application/jsonl")}, + headers={"Authorization": f"Bearer {caller_key}"}, + ) + case "upload_header": + return gateway.client.post( + "/v1/files", + data={"purpose": "batch"}, + files={"file": (INPUT_FILENAME, _jsonl(forbidden), "application/jsonl")}, + headers={"Authorization": f"Bearer {caller_key}", "x-litellm-model": forbidden}, + ) + case "batch_encoded_input": + return gateway.request( + "POST", + "/v1/batches", + { + "input_file_id": encoded_file, + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + }, + key=caller_key, + ) + case "batch_body_model": + return gateway.request( + "POST", + "/v1/batches", + { + "input_file_id": raw_file, + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "model": forbidden, + }, + key=caller_key, + ) + case "batch_query_model": + return gateway.request( + "POST", + "/v1/batches", + {"input_file_id": raw_file, "endpoint": "/v1/chat/completions", "completion_window": "24h"}, + key=caller_key, + params={"model": forbidden}, + ) + case "batch_header_model": + return gateway.request( + "POST", + "/v1/batches", + {"input_file_id": raw_file, "endpoint": "/v1/chat/completions", "completion_window": "24h"}, + key=caller_key, + headers={"x-litellm-model": forbidden}, + ) + case "file_get_encoded": + return gateway.request("GET", f"/v1/files/{encoded_file}", key=caller_key) + case "file_get_query": + return gateway.request("GET", f"/v1/files/{raw_file}", key=caller_key, params={"model": forbidden}) + case "file_get_header": + return gateway.request( + "GET", f"/v1/files/{raw_file}", key=caller_key, headers={"x-litellm-model": forbidden} + ) + case "file_content_encoded": + return gateway.request("GET", f"/v1/files/{encoded_file}/content", key=caller_key) + case "file_content_query": + return gateway.request( + "GET", f"/v1/files/{raw_file}/content", key=caller_key, params={"model": forbidden} + ) + case "file_content_header": + return gateway.request( + "GET", f"/v1/files/{raw_file}/content", key=caller_key, headers={"x-litellm-model": forbidden} + ) + case "batch_get_encoded": + return gateway.request("GET", f"/v1/batches/{encoded_batch}", key=caller_key) + case "batch_get_query": + return gateway.request("GET", f"/v1/batches/{raw_batch}", key=caller_key, params={"model": forbidden}) + case "batch_get_header": + return gateway.request( + "GET", f"/v1/batches/{raw_batch}", key=caller_key, headers={"x-litellm-model": forbidden} + ) + case _: + assert_never(surface) + + +@pytest.mark.parametrize( + "surface", + [ + "upload_sdk_extra_body", + "upload_form", + "upload_query", + "upload_header", + "batch_encoded_input", + "batch_body_model", + "batch_query_model", + "batch_header_model", + "file_get_encoded", + "file_get_query", + "file_get_header", + "file_content_encoded", + "file_content_query", + "file_content_header", + "batch_get_encoded", + "batch_get_query", + "batch_get_header", + ], + ids=[ + "upload-sdk-extra-body-model", + "upload-form-model", + "upload-query-model", + "upload-header-model", + "batch-create-encoded-input", + "batch-create-body-model", + "batch-create-query-model", + "batch-create-model-header", + "file-retrieve-encoded-id", + "file-retrieve-model-query", + "file-retrieve-model-header", + "file-content-encoded-id", + "file-content-model-query", + "file-content-model-header", + "batch-retrieve-encoded-id", + "batch-retrieve-model-query", + "batch-retrieve-model-header", + ], +) +def test_a_key_granted_one_model_cannot_spend_another_deployments_files_or_batches( + gateway: Gateway, surface: _ForbiddenSurface +) -> None: + key_allowed: Final = "provider-key-allowed-" + uuid.uuid4().hex[:8] + key_forbidden: Final = "provider-key-forbidden-" + uuid.uuid4().hex[:8] + with wire_server(_openai_backend()) as wire, gateway.scenario() as scenario: + allowed: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key=key_allowed) + forbidden: Final = scenario.model(model="openai/gpt-4.1-mini", api_base=wire.url + "/v1", api_key=key_forbidden) + _wait_until_every_worker_serves(gateway, allowed, forbidden) + team: Final = scenario.team(models=[allowed, forbidden]) + user: Final = scenario.member(team) + caller_key: Final = scenario.key(team_id=team, user_id=user, models=[allowed]) + caller: Final = _Member(team, user, caller_key) + provider_client: Final = httpx.Client(base_url=wire.url, timeout=15, trust_env=False) + with provider_client: + seeded_file: Final = provider_client.post( + "/v1/files", + data={"purpose": "batch"}, + files={"file": (INPUT_FILENAME, _jsonl(forbidden), "application/jsonl")}, + headers={"Authorization": f"Bearer {key_forbidden}"}, + ) + assert seeded_file.status_code == 200, seeded_file.text + raw_file_object: Final = _FileObject.model_validate_json(seeded_file.content) + assert raw_file_object.id == f"file-{key_forbidden}", seeded_file.text + seeded_batch: Final = provider_client.post( + "/v1/batches", + json={ + "input_file_id": raw_file_object.id, + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + }, + headers={"Authorization": f"Bearer {key_forbidden}"}, + ) + assert seeded_batch.status_code == 200, seeded_batch.text + raw_batch_object: Final = _Batch.model_validate_json(seeded_batch.content) + assert raw_batch_object.id == f"batch-{key_forbidden}", seeded_batch.text + wire.drain() + + encoded_forbidden_file: Final = encode_file_id_with_model(file_id=raw_file_object.id, model=forbidden) + encoded_forbidden_batch: Final = encode_file_id_with_model( + file_id=raw_batch_object.id, model=forbidden, id_type="batch" + ) + response: Final = _forbidden_surface_request( + gateway, + surface, + forbidden, + caller.key, + raw_file_object.id, + raw_batch_object.id, + encoded_forbidden_file, + encoded_forbidden_batch, + ) + provider_requests: Final = _drained_other_than_model_list_probes(wire) + assert response.status_code == 403, f"{response.text}; provider requests={provider_requests}" + assert _error_message(response, 403) == ( + f"The requested model '{forbidden}' is not available for this API key, or the model name is invalid. " + "Check the models available to you and try again." + ), response.text + assert provider_requests == (), f"{surface} reached the provider: {provider_requests}" + + control: Final = gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "model": allowed}, + {"file": (INPUT_FILENAME, _jsonl(allowed), "application/jsonl")}, + key=caller.key, + ) + assert control.status_code == 200, control.text + (control_request,) = _drained_other_than_model_list_probes(wire) + assert control_request.headers["authorization"] == f"Bearer {key_allowed}", control_request.headers + + +def test_indexed_target_model_upload_cannot_use_a_forbidden_deployment(gateway: Gateway) -> None: + pytest.skip("BUG: target_model_names[0] upload bypasses the key's model restriction and reaches the deployment") + key_allowed: Final = "provider-key-allowed-" + uuid.uuid4().hex[:8] + key_forbidden: Final = "provider-key-forbidden-" + uuid.uuid4().hex[:8] + with wire_server(_openai_backend()) as wire, gateway.scenario() as scenario: + allowed: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key=key_allowed) + forbidden: Final = scenario.model(model="openai/gpt-4.1-mini", api_base=wire.url + "/v1", api_key=key_forbidden) + _wait_until_every_worker_serves(gateway, allowed, forbidden) + team: Final = scenario.team(models=[allowed, forbidden]) + user: Final = scenario.member(team) + caller_key: Final = scenario.key(team_id=team, user_id=user, models=[allowed]) + + response: Final = gateway.client.post( + "/v1/files", + data={"purpose": "batch", "target_model_names[0]": forbidden}, + files={"file": (INPUT_FILENAME, _jsonl(forbidden), "application/jsonl")}, + headers={"Authorization": f"Bearer {caller_key}"}, + ) + provider_requests: Final = _drained_other_than_model_list_probes(wire) + assert response.status_code == 403, f"{response.text}; provider requests={provider_requests}" + assert _error_message(response, 403) == ( + f"The requested model '{forbidden}' is not available for this API key, or the model name is invalid. " + "Check the models available to you and try again." + ), response.text + assert provider_requests == (), f"indexed target-model upload reached the provider: {provider_requests}" + + +def test_repeated_target_model_upload_cannot_use_a_forbidden_deployment(gateway: Gateway) -> None: + pytest.skip("BUG: repeated target_model_names[] upload bypasses the key's model restriction and reaches the deployment") + key_allowed: Final = "provider-key-allowed-" + uuid.uuid4().hex[:8] + key_forbidden: Final = "provider-key-forbidden-" + uuid.uuid4().hex[:8] + with wire_server(_openai_backend()) as wire, gateway.scenario() as scenario: + allowed: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key=key_allowed) + forbidden: Final = scenario.model(model="openai/gpt-4.1-mini", api_base=wire.url + "/v1", api_key=key_forbidden) + _wait_until_every_worker_serves(gateway, allowed, forbidden) + team: Final = scenario.team(models=[allowed, forbidden]) + user: Final = scenario.member(team) + caller_key: Final = scenario.key(team_id=team, user_id=user, models=[allowed]) + + response: Final = gateway.client.post( + "/v1/files", + data={"purpose": "batch", "target_model_names[]": [allowed, forbidden]}, + files={"file": (INPUT_FILENAME, _jsonl(forbidden), "application/jsonl")}, + headers={"Authorization": f"Bearer {caller_key}"}, + ) + provider_requests: Final = _drained_other_than_model_list_probes(wire) + assert response.status_code == 403, f"{response.text}; provider requests={provider_requests}" + assert _error_message(response, 403) == ( + f"The requested model '{forbidden}' is not available for this API key, or the model name is invalid. " + "Check the models available to you and try again." + ), response.text + assert provider_requests == (), f"repeated target-model upload reached the provider: {provider_requests}" + + +def test_plain_target_model_upload_cannot_use_a_forbidden_deployment(gateway: Gateway) -> None: + key_allowed: Final = "provider-key-allowed-" + uuid.uuid4().hex[:8] + key_forbidden: Final = "provider-key-forbidden-" + uuid.uuid4().hex[:8] + with wire_server(_openai_backend()) as wire, gateway.scenario() as scenario: + allowed: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key=key_allowed) + forbidden: Final = scenario.model(model="openai/gpt-4.1-mini", api_base=wire.url + "/v1", api_key=key_forbidden) + _wait_until_every_worker_serves(gateway, allowed, forbidden) + team: Final = scenario.team(models=[allowed, forbidden]) + user: Final = scenario.member(team) + caller_key: Final = scenario.key(team_id=team, user_id=user, models=[allowed]) + + response: Final = gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "target_model_names": forbidden}, + {"file": (INPUT_FILENAME, _jsonl(forbidden), "application/jsonl")}, + key=caller_key, + ) + provider_requests: Final = _drained_other_than_model_list_probes(wire) + assert response.status_code == 403, f"{response.text}; provider requests={provider_requests}" + assert _error_message(response, 403) == ( + f"The requested model '{forbidden}' is not available for this API key, or the model name is invalid. " + "Check the models available to you and try again." + ), response.text + assert provider_requests == (), f"plain target-model upload reached the provider: {provider_requests}" + + +def _s3_backend() -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.method in {"DELETE", "GET"}: + return Reply(status=204, body=b"") + return _json_reply({"error": {"message": f"unscripted {request.method} {request.target}"}}, 404) + + return respond + + +def _bedrock_model(scenario: Scenario, wire: Wire) -> str: + return scenario.model( + model="bedrock/anthropic.claude-3-haiku-20240307-v1:0", + api_key=None, + api_base=None, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + aws_region_name=AWS_REGION, + s3_bucket_name=BUCKET, + s3_endpoint_url=wire.url, + ) + + +def _vertex_credentials(token_url: str) -> str: + private_key: Final = ( + rsa.generate_private_key(public_exponent=65537, key_size=2048) + .private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + .decode() + ) + return json.dumps( + { + "type": "service_account", + "project_id": VERTEX_PROJECT, + "private_key_id": "scripted", + "private_key": private_key, + "client_email": f"scripted@{VERTEX_PROJECT}.iam.gserviceaccount.com", + "client_id": "0", + "auth_uri": f"{token_url}/_oauth/authorize", + "token_uri": f"{token_url}/_oauth/token", + } + ) + + +def _vertex_token_backend(request: Request) -> Reply: + if request.method == "POST" and request.target == "/_oauth/token": + return Reply(status=400, body=b'{"error":"invalid_grant"}', content_type="application/json") + return _json_reply({"error": {"message": f"unscripted {request.method} {request.target}"}}, 404) + + +def test_non_admin_cannot_delete_a_raw_s3_uri(gateway: Gateway) -> None: + with wire_server(_s3_backend()) as wire, gateway.scenario() as scenario: + model: Final = _bedrock_model(scenario, wire) + _wait_until_every_worker_serves(gateway, model) + caller_user: Final = scenario.user(user_role="internal_user") + caller_key: Final = scenario.key(user_id=caller_user, models=[model]) + object_uri: Final = f"s3://{BUCKET}/litellm-bedrock-files-obj.jsonl" + provider_path: Final = "/bedrock/v1/files/" + quote(object_uri, safe="") + + denied: Final = gateway.request("DELETE", provider_path, key=caller_key, params={"model": model}) + assert denied.status_code == 403, denied.text + assert _error_message(denied, 403) == ( + "Raw cloud storage file ids can only be deleted by a proxy admin key. " + "Use the LiteLLM managed file id returned when the file was created." + ), denied.text + assert _drained_other_than_model_list_probes(wire) == (), ( + "non-admin cloud-storage delete reached the storage backend" + ) + + +def test_non_admin_cannot_delete_a_raw_gcs_uri_before_vertex_auth_or_storage(gateway: Gateway) -> None: + with ( + wire_server(_vertex_token_backend) as token_wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model( + model=VERTEX_MODEL, + api_key=None, + vertex_project=VERTEX_PROJECT, + vertex_location=VERTEX_LOCATION, + vertex_credentials=_vertex_credentials(token_wire.url), + gcs_bucket_name=BUCKET, + ) + _wait_until_every_worker_serves(gateway, model) + caller_user: Final = scenario.user(user_role="internal_user") + caller_key: Final = scenario.key(user_id=caller_user, models=[model]) + object_uri: Final = f"gs://{BUCKET}/litellm-vertex-files/litellm-vertex-files-obj.jsonl" + token_wire.drain() + + denied: Final = gateway.request( + "DELETE", + "/vertex_ai/v1/files/" + quote(object_uri, safe=""), + key=caller_key, + params={"model": model}, + ) + token_requests: Final = token_wire.drain() + assert denied.status_code == 403, f"{denied.text}; token requests={token_requests}" + assert _error_message(denied, 403) == ( + "Raw cloud storage file ids can only be deleted by a proxy admin key. " + "Use the LiteLLM managed file id returned when the file was created." + ), denied.text + assert token_requests == (), "non-admin raw GCS delete reached the Vertex token endpoint" + + +def test_proxy_admin_deletes_a_raw_s3_uri_with_one_signed_delete(gateway: Gateway) -> None: + with wire_server(_s3_backend()) as wire, gateway.scenario() as scenario: + model: Final = _bedrock_model(scenario, wire) + _wait_until_every_worker_serves(gateway, model) + object_uri: Final = f"s3://{BUCKET}/litellm-bedrock-files-obj.jsonl" + encoded_path: Final = "/v1/files/" + quote(object_uri, safe="") + + deleted: Final = gateway.request("DELETE", encoded_path, params={"model": model}) + assert deleted.status_code == 200, deleted.text + parsed: Final = _FileDeleted.model_validate_json(deleted.content) + assert parsed.deleted is True and parsed.id == object_uri, deleted.text + (request,) = _drained_other_than_model_list_probes(wire) + assert request.target == f"/{BUCKET}/litellm-bedrock-files-obj.jsonl", request.target + request_date: Final = request.headers["x-amz-date"] + content_hash: Final = hashlib.sha256(b"").hexdigest() + assert request.headers["x-amz-content-sha256"] == content_hash, request.headers + aws_request: Final = AWSRequest( + method="DELETE", + url=wire.url + request.target, + headers={ + "host": request.headers["host"], + "x-amz-date": request_date, + "x-amz-content-sha256": content_hash, + }, + ) + aws_request.context["timestamp"] = request_date + auth: Final = S3SigV4Auth(Credentials(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY), "s3", AWS_REGION) + signature: Final = auth.signature( + auth.string_to_sign(aws_request, auth.canonical_request(aws_request)), aws_request + ) + expected_authorization: Final = ( + f"AWS4-HMAC-SHA256 Credential={AWS_ACCESS_KEY_ID}/{request_date[:8]}/{AWS_REGION}/s3/aws4_request, " + f"SignedHeaders={auth.signed_headers(auth.headers_to_sign(aws_request))}, Signature={signature}" + ) + assert request.headers["authorization"] == expected_authorization, request.headers["authorization"] + assert request.method == "DELETE", request.method + + +def _granting_vertex_token_backend(request: Request) -> Reply: + if request.method == "POST" and request.target == "/_oauth/token": + return _json_reply({"access_token": VERTEX_ACCESS_TOKEN, "expires_in": 3600, "token_type": "Bearer"}) + return _json_reply({"error": {"message": f"unscripted {request.method} {request.target}"}}, 404) + + +def _gcs_backend(request: Request) -> Reply: + if request.method == "DELETE": + return Reply(status=204, body=b"") + return _json_reply({"error": {"message": f"unscripted {request.method} {request.target}"}}, 404) + + +def _pipe(source: socket.socket, sink: socket.socket) -> None: + with contextlib.suppress(OSError): + for chunk in iter(lambda: source.recv(65536), b""): + sink.sendall(chunk) + with contextlib.suppress(OSError): + sink.shutdown(socket.SHUT_WR) + + +@dataclass(frozen=True, slots=True) +class _ConnectTunnel: + url: str + authorities: SimpleQueue[str] + + +@contextmanager +def _gcs_tunnel(destination: Wire) -> Generator[_ConnectTunnel, None, None]: + authorities: Final[SimpleQueue[str]] = SimpleQueue() + destination_port: Final = int(destination.url.rsplit(":", 1)[1]) + + class Tunnel(socketserver.StreamRequestHandler): + rbufsize = 0 + request: socket.socket + + def handle(self) -> None: + authority: Final = self.rfile.readline().decode().split()[1] + while self.rfile.readline() not in (b"\r\n", b""): + pass + authorities.put(authority) + if authority != GCS_AUTHORITY: + self.wfile.write(b"HTTP/1.1 403 Forbidden\r\ncontent-length: 0\r\n\r\n") + return + self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n") + self.request.settimeout(10) + with socket.create_connection(("127.0.0.1", destination_port), timeout=10) as upstream: + outbound: Final = threading.Thread(target=_pipe, args=(self.request, upstream)) + outbound.start() + _pipe(upstream, self.request) + outbound.join(timeout=12) + + with socketserver.ThreadingTCPServer(("127.0.0.1", 0), Tunnel) as server: + thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}) + thread.start() + try: + yield _ConnectTunnel(f"http://127.0.0.1:{server.server_address[1]}", authorities) + finally: + server.shutdown() + thread.join(timeout=6) + + +def _queued(queue: SimpleQueue[str]) -> tuple[str, ...]: + return tuple(queue.get_nowait() for _ in range(queue.qsize())) + + +@pytest.mark.timeout(180) +def test_proxy_admin_deletes_a_raw_gcs_uri_with_one_authorized_storage_delete( + gateway: Gateway, tmp_path: Path +) -> None: + cert_file, key_file = write_self_signed_cert(tmp_path, (GCS_HOST,)) + object_name: Final = f"litellm-vertex-files/admin-{uuid.uuid4().hex}.jsonl" + object_uri: Final = f"gs://{BUCKET}/{object_name}" + with ( + wire_server(_granting_vertex_token_backend) as token_wire, + wire_server(_gcs_backend, tls=server_context(cert_file, key_file)) as storage, + _gcs_tunnel(storage) as tunnel, + owned_proxy(gateway, tmp_path, {"HTTPS_PROXY": tunnel.url, "SSL_VERIFY": "False"}) as candidate, + candidate.scenario() as scenario, + ): + model: Final = scenario.model( + model=VERTEX_MODEL, + api_key=None, + vertex_project=VERTEX_PROJECT, + vertex_location=VERTEX_LOCATION, + vertex_credentials=_vertex_credentials(token_wire.url), + gcs_bucket_name=BUCKET, + ) + token_wire.drain() + + deleted: Final = candidate.request( + "DELETE", "/v1/files/" + quote(object_uri, safe=""), params={"model": model} + ) + storage_requests: Final = storage.drain() + assert deleted.status_code == 200, f"{deleted.text}; storage requests={storage_requests}" + parsed: Final = _FileDeleted.model_validate_json(deleted.content) + assert parsed.deleted is True and parsed.id == object_uri, deleted.text + assert [(r.method, r.target) for r in storage_requests] == [ + ("DELETE", f"/storage/v1/b/{BUCKET}/o/{quote(object_name, safe='')}") + ], storage_requests + assert storage_requests[0].headers["authorization"] == f"Bearer {VERTEX_ACCESS_TOKEN}", ( + storage_requests[0].headers + ) + assert _queued(tunnel.authorities) == (GCS_AUTHORITY,), "raw GCS delete took an unexpected route" From 60ee96688529aa7aca7103fc5bb311a27266614c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:57:54 +0000 Subject: [PATCH 06/51] test(integration): cover anthropic_messages endpoint contracts (#44704) * test(integration): cover anthropic_messages endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): close anthropic_messages review gaps Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): prove azure beta filtering and the safeguards allowlist Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): compare the forwarded anthropic-beta token-for-token and pin the truncated stream bug Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: strengthen endpoint wire contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: follow count_tokens upstream error relay from main Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: follow device-code refresh token response from main Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_claude_code_gateway_oauth.py | 520 ++++++++++++++++ .../test_anthropic_count_tokens_wire.py | 240 ++++++++ .../anthropic/test_anthropic_gateway_wire.py | 359 +++++++++++ .../anthropic/test_anthropic_wire.py | 568 ++++++++++++++++++ .../test_anthropic_skills_create_wire.py | 69 +++ .../test_azure_ai_claude_messages_wire.py | 323 ++++++++++ .../test_vertex_partner_messages_wire.py | 193 ++++++ 7 files changed, 2272 insertions(+) create mode 100644 tests/integration/authorization/test_claude_code_gateway_oauth.py create mode 100644 tests/integration/messages_endpoint/providers/anthropic/test_anthropic_count_tokens_wire.py create mode 100644 tests/integration/messages_endpoint/providers/anthropic/test_anthropic_gateway_wire.py create mode 100644 tests/integration/providers/test_anthropic_skills_create_wire.py create mode 100644 tests/integration/providers/test_azure_ai_claude_messages_wire.py create mode 100644 tests/integration/providers/test_vertex_partner_messages_wire.py diff --git a/tests/integration/authorization/test_claude_code_gateway_oauth.py b/tests/integration/authorization/test_claude_code_gateway_oauth.py new file mode 100644 index 00000000000..acab042bfc9 --- /dev/null +++ b/tests/integration/authorization/test_claude_code_gateway_oauth.py @@ -0,0 +1,520 @@ +import asyncio +import base64 +import hashlib +import html +import json +import os +import re +import uuid +from collections import deque +from collections.abc import Iterator +from pathlib import Path +from typing import Final, Literal +from urllib.parse import parse_qs, urlencode, urlsplit + +import httpx +import pytest +import yaml +from anthropic.types import Message +from integration._support import claude_code as cc +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.database import read_rows +from integration._support.process import owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +_CLIENT_ID: Final = "scripted-cli-client" +_CLIENT_SECRET: Final = "scripted-cli-secret" +_AUTHORIZATION_CODE: Final = "scripted-auth-code" +_IDP_ACCESS_TOKEN: Final = "scripted-idp-access-token" +_USER_ID: Final = "gateway-sso-user" +_USER_EMAIL: Final = "gateway-sso-user@example.com" +_BACKEND: Final = "claude-sonnet-4-6" +_API_KEY: Final = "synthetic-anthropic-key" +_EXPECTED_BETA: Final = ",".join( + sorted(set((*cc.CLI_BETA.split(","), "context-management-2025-06-27", "prompt-caching-scope-2026-01-05"))) +) +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MESSAGE: Final = TypeAdapter(Message) +_BROWSER_TOKEN: Final = re.compile(r'name="browser_complete_token" value="([^"]+)"') + + +class _DeviceAuthorization(BaseModel): + device_code: str + user_code: str + verification_uri: str + verification_uri_complete: str | None = None + expires_in: int + interval: int + + +class _AccessToken(BaseModel): + model_config = ConfigDict(frozen=True) + + access_token: str + expires_in: int + token_type: Literal["Bearer"] + refresh_token: str + user_id: str + team_id: str | None + + +def _owned_config(path: Path) -> Path: + config: Final = _JSON_OBJECT.validate_python( + yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + ) + general_settings: Final = _JSON_OBJECT.validate_python(config.get("general_settings", {})) + path.write_text( + yaml.safe_dump( + { + **config, + "general_settings": {**general_settings, "enable_claude_code_gateway": True}, + } + ) + ) + return path + + +def _identity_provider(request: Request) -> Reply: + parsed_target: Final = urlsplit(request.target) + if request.method == "GET" and parsed_target.path == "/authorize": + query: Final = parse_qs(parsed_target.query) + assert query["client_id"] == [_CLIENT_ID], query + assert query["response_type"] == ["code"], query + assert query["scope"] == ["openid email profile"], query + assert query["code_challenge_method"] == ["S256"], query + assert len(query["code_challenge"][0]) == 43, query + callback: Final = query["redirect_uri"][0] + assert urlsplit(callback).path == "/sso/callback", query + state: Final = query["state"][0] + assert state.startswith("litellm-session-token:"), query + location: Final = f"{callback}?{urlencode({'code': _AUTHORIZATION_CODE, 'state': state})}" + return Reply(status=302, headers={"location": location}) + + if request.method == "POST" and parsed_target.path == "/token": + form: Final = parse_qs(request.body.decode()) + encoded_credentials: Final = base64.b64encode(f"{_CLIENT_ID}:{_CLIENT_SECRET}".encode()).decode() + assert request.headers["authorization"] == f"Basic {encoded_credentials}", request.headers + assert request.headers["content-type"] == "application/x-www-form-urlencoded", request.headers + assert set(form) == {"code", "code_verifier", "grant_type", "redirect_uri"}, form + assert form["grant_type"] == ["authorization_code"], form + assert form["code"] == [_AUTHORIZATION_CODE], form + assert len(form["code_verifier"][0]) == 43, form + assert urlsplit(form["redirect_uri"][0]).path == "/sso/callback", form + return Reply(body=json.dumps({"access_token": _IDP_ACCESS_TOKEN, "token_type": "Bearer"}).encode()) + + if request.method == "GET" and parsed_target.path == "/userinfo": + assert request.headers["authorization"] == f"Bearer {_IDP_ACCESS_TOKEN}", request.headers + return Reply(body=json.dumps({"sub": _USER_ID, "email": _USER_EMAIL}).encode()) + + raise AssertionError(f"unexpected scripted IdP request: {request.method} {request.target}") + + +_REPLY_CONTENT: Final = ({"type": "text", "text": "authorized"},) +_REPLY_USAGE: Final = {"input_tokens": 5, "output_tokens": 1} + + +def _anthropic_reply(request: Request, expected_body: dict[str, JsonValue], message_id: str) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages", request.target + assert request.headers["x-api-key"] == _API_KEY, request.headers + assert request.headers["anthropic-version"] == "2023-06-01", request.headers + assert request.headers["anthropic-beta"] == _EXPECTED_BETA, request.headers + assert "authorization" not in request.headers, request.headers + body: Final = _JSON_OBJECT.validate_json(request.body) + streamed: Final = body.get("stream") is True + assert body == {**expected_body, "stream": streamed}, request.body + if streamed: + return Reply( + chunks=cc.message_stream(message_id, _BACKEND, _REPLY_CONTENT, _REPLY_USAGE), + content_type="text/event-stream", + ) + return Reply(body=cc.message_reply(message_id, _BACKEND, _REPLY_CONTENT, _REPLY_USAGE)) + + +@pytest.fixture(scope="module") +def identity_provider() -> Iterator[Wire]: + with wire_server(_identity_provider) as wire: + yield wire + + +@pytest.fixture(scope="module") +def enabled_gateway(identity_provider: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + assert os.environ.get("LITELLM_LICENSE"), "LITELLM_LICENSE is required for the generic SSO integration test" + with gateway_from_environment() as base_gateway: + directory: Final = tmp_path_factory.mktemp("claude-code-gateway-oauth") + config: Final = _owned_config(directory / "gateway-oauth.yaml") + overrides: Final = { + "GENERIC_CLIENT_ID": _CLIENT_ID, + "GENERIC_CLIENT_SECRET": _CLIENT_SECRET, + "GENERIC_AUTHORIZATION_ENDPOINT": f"{identity_provider.url}/authorize", + "GENERIC_TOKEN_ENDPOINT": f"{identity_provider.url}/token", + "GENERIC_USERINFO_ENDPOINT": f"{identity_provider.url}/userinfo", + "GENERIC_USER_ID_ATTRIBUTE": "sub", + "GENERIC_USER_EMAIL_ATTRIBUTE": "email", + "GENERIC_CLIENT_USE_PKCE": "true", + "GENERIC_INCLUDE_CLIENT_ID": "false", + } + with owned_proxy_process( + base_gateway, + directory, + overrides, + config=config, + workers=2, + remove_environment=("GOOGLE_CLIENT_ID", "MICROSOFT_CLIENT_ID", "PROXY_BASE_URL"), + ) as owned: + yield owned.gateway + + +@pytest.mark.timeout(300) +def test_device_authorization_polls_complete_sso_and_mints_a_model_restricted_bearer( + enabled_gateway: Gateway, identity_provider: Wire +) -> None: + request_template: Final = { + **cc.claude_code_request("oauth-model-restriction"), + "model": "", + "stream": False, + } + expected_body: Final = {**request_template, "model": _BACKEND} + run_id: Final = uuid.uuid4().hex + message_ids: Final = (f"msg_oauth_{run_id}_a", f"msg_oauth_{run_id}_a_stream", f"msg_oauth_{run_id}_c") + pending_message_ids: Final = deque(message_ids) + + with enabled_gateway.scenario() as scenario: + with wire_server( + lambda request: _anthropic_reply(request, expected_body, pending_message_ids.popleft()) + ) as upstream: + model_a: Final = scenario.model(model=f"anthropic/{_BACKEND}", api_base=upstream.url, api_key=_API_KEY) + model_b: Final = scenario.model(model=f"anthropic/{_BACKEND}", api_base=upstream.url, api_key=_API_KEY) + model_c: Final = scenario.model(model=f"anthropic/{_BACKEND}", api_base=upstream.url, api_key=_API_KEY) + user_id: Final = scenario.user( + user_id=_USER_ID, + user_email=_USER_EMAIL, + user_role="internal_user", + models=[model_a, model_b], + ) + team_id: Final = scenario.team(models=[model_a, model_c]) + enabled_gateway.post( + "/team/member_add", {"team_id": team_id, "member": {"role": "user", "user_id": user_id}} + ) + + authorization_response: Final = enabled_gateway.client.post( + "/claude_code_gateway/oauth/device_authorization", + data={"client_id": _CLIENT_ID}, + ) + assert authorization_response.status_code == 200, authorization_response.text + authorization: Final = _DeviceAuthorization.model_validate_json(authorization_response.content) + authorization_body: Final = _JSON_OBJECT.validate_json(authorization_response.content) + assert authorization_body == authorization.model_dump(exclude_none=True), authorization_response.text + assert authorization.expires_in > 0 and authorization.interval > 0, authorization_response.text + login_id, separator, poll_secret = authorization.device_code.partition(".") + assert separator == "." and login_id.startswith("cli-") and poll_secret, authorization_response.text + + verification_parts: Final = urlsplit(authorization.verification_uri) + assert verification_parts.path == "/sso/key/generate", authorization.verification_uri + verification_query: Final = parse_qs(verification_parts.query) + assert verification_query == {"source": ["litellm-cli"], "key": [login_id]}, authorization.verification_uri + + pending: Final = enabled_gateway.client.post( + "/claude_code_gateway/oauth/token", + data={ + "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + "device_code": authorization.device_code, + "client_id": _CLIENT_ID, + }, + ) + assert pending.status_code == 400, pending.text + pending_body: Final = _JSON_OBJECT.validate_json(pending.content) + assert pending_body == {"error": "authorization_pending"}, pending.text + + wrong_secret: Final = enabled_gateway.client.post( + "/claude_code_gateway/oauth/token", + data={ + "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + "device_code": f"{login_id}.wrong-poll-secret", + "client_id": _CLIENT_ID, + }, + ) + assert wrong_secret.status_code == 400, wrong_secret.text + assert _JSON_OBJECT.validate_json(wrong_secret.content) == {"error": "expired_token"}, wrong_secret.text + assert "authorization_pending" not in wrong_secret.text, wrong_secret.text + + verification_response: Final = enabled_gateway.client.get( + authorization.verification_uri, + follow_redirects=False, + ) + assert verification_response.status_code == 303, verification_response.text + assert "litellm_oauth_state" in verification_response.headers.get("set-cookie", ""), ( + verification_response.headers, + ) + browser_state: Final = enabled_gateway.client.cookies.get("litellm_oauth_state") + assert browser_state, verification_response.headers + + identity_provider_response: Final = enabled_gateway.client.get( + verification_response.headers["location"], + follow_redirects=False, + ) + assert identity_provider_response.status_code == 302, identity_provider_response.text + callback_location: Final = identity_provider_response.headers["location"] + callback_parts: Final = urlsplit(callback_location) + assert callback_parts.path == "/sso/callback", callback_location + callback_query: Final = parse_qs(callback_parts.query) + assert callback_query["state"] == [browser_state], callback_location + + callback_response: Final = enabled_gateway.client.get(callback_location, follow_redirects=False) + assert callback_response.status_code == 200, callback_response.text + browser_token_match: Final = _BROWSER_TOKEN.search(callback_response.text) + assert browser_token_match is not None, callback_response.text + browser_complete_token: Final = html.unescape(browser_token_match.group(1)) + + still_pending: Final = enabled_gateway.client.post( + "/claude_code_gateway/oauth/token", + data={ + "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + "device_code": authorization.device_code, + "client_id": _CLIENT_ID, + }, + ) + assert still_pending.status_code == 400, still_pending.text + assert _JSON_OBJECT.validate_json(still_pending.content) == {"error": "authorization_pending"}, ( + still_pending.text, + ) + + wrong_code_completion: Final = enabled_gateway.client.post( + f"/sso/cli/complete/{login_id}", + data={"user_code": "WRONG-CODE", "browser_complete_token": browser_complete_token}, + ) + assert wrong_code_completion.status_code == 400, wrong_code_completion.text + assert _JSON_OBJECT.validate_json(wrong_code_completion.content) == { + "detail": "Invalid verification code" + }, wrong_code_completion.text + + pending_after_wrong_code: Final = enabled_gateway.client.post( + "/claude_code_gateway/oauth/token", + data={ + "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + "device_code": authorization.device_code, + "client_id": _CLIENT_ID, + }, + ) + assert pending_after_wrong_code.status_code == 400, pending_after_wrong_code.text + assert _JSON_OBJECT.validate_json(pending_after_wrong_code.content) == {"error": "authorization_pending"}, ( + pending_after_wrong_code.text + ) + + verification_complete: Final = enabled_gateway.client.post( + f"/sso/cli/complete/{login_id}", + data={"user_code": authorization.user_code, "browser_complete_token": browser_complete_token}, + ) + assert verification_complete.status_code == 200, verification_complete.text + + async def poll_device_code() -> tuple[httpx.Response, ...]: + async with httpx.AsyncClient( + base_url=str(enabled_gateway.client.base_url), + timeout=30, + trust_env=False, + ) as client: + return tuple( + await asyncio.gather( + *( + client.post( + "/claude_code_gateway/oauth/token", + data={ + "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + "device_code": authorization.device_code, + "client_id": _CLIENT_ID, + }, + ) + for _ in range(8) + ) + ) + ) + + poll_responses: Final = asyncio.run(poll_device_code()) + successful_polls: Final = tuple(response for response in poll_responses if response.status_code == 200) + assert len(successful_polls) == 1, ( + f"Expected exactly one successful device_code poll, got {len(successful_polls)} " + f"with statuses {tuple(response.status_code for response in poll_responses)}" + ) + token_response: Final = successful_polls[0] + losing_polls: Final = tuple(response for response in poll_responses if response is not token_response) + assert len(losing_polls) == 7, losing_polls + for losing_poll in losing_polls: + assert losing_poll.status_code == 400, losing_poll.text + assert _JSON_OBJECT.validate_json(losing_poll.content) == {"error": "expired_token"}, losing_poll.text + + assert token_response.status_code == 200, token_response.text + access_token: Final = _AccessToken.model_validate_json(token_response.content) + access_token_body: Final = _JSON_OBJECT.validate_json(token_response.content) + assert access_token_body == access_token.model_dump(), token_response.text + assert set(access_token_body) == { + "access_token", + "expires_in", + "token_type", + "refresh_token", + "user_id", + "team_id", + }, token_response.text + assert access_token.token_type == "Bearer" and access_token.expires_in > 0, token_response.text + assert access_token.user_id == user_id, token_response.text + assert access_token.team_id == team_id, token_response.text + + second_poll: Final = enabled_gateway.client.post( + "/claude_code_gateway/oauth/token", + data={ + "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + "device_code": authorization.device_code, + "client_id": _CLIENT_ID, + }, + ) + assert second_poll.status_code == 400, second_poll.text + assert _JSON_OBJECT.validate_json(second_poll.content) == {"error": "expired_token"}, second_poll.text + + unsupported: Final = enabled_gateway.client.post( + "/claude_code_gateway/oauth/token", + data={"grant_type": "urn:unsupported"}, + ) + assert unsupported.status_code == 400, unsupported.text + assert _JSON_OBJECT.validate_json(unsupported.content) == { + "error": "unsupported_grant_type", + "error_description": "Unsupported grant_type: urn:unsupported", + }, unsupported.text + + refresh: Final = enabled_gateway.client.post( + "/claude_code_gateway/oauth/token", + data={"grant_type": "refresh_token"}, + ) + assert refresh.status_code == 400, refresh.text + assert _JSON_OBJECT.validate_json(refresh.content) == { + "error": "invalid_request", + "error_description": "refresh_token is required", + }, refresh.text + + bearer_headers: Final = { + "authorization": f"Bearer {access_token.access_token}", + "anthropic-version": "2023-06-01", + "anthropic-beta": cc.CLI_BETA, + "x-claude-code-session-id": f"oauth-{user_id}", + } + request_body: Final = {**request_template, "model": model_a} + expected_message: Final = _MESSAGE.validate_python( + { + **_JSON_OBJECT.validate_json( + cc.message_reply(message_ids[0], _BACKEND, _REPLY_CONTENT, _REPLY_USAGE) + ), + "model": model_a, + } + ) + model_a_response: Final = enabled_gateway.client.post( + "/claude_code_gateway/v1/messages", json=request_body, headers=bearer_headers + ) + assert model_a_response.status_code == 200, model_a_response.text + parsed_message: Final = _MESSAGE.validate_json(model_a_response.content) + assert parsed_message.model_dump(mode="json") == expected_message.model_dump(mode="json"), ( + model_a_response.text, + ) + + model_a_stream: Final = enabled_gateway.client.post( + "/claude_code_gateway/v1/messages?beta=true", + json={**request_body, "stream": True}, + headers=bearer_headers, + ) + assert model_a_stream.status_code == 200, model_a_stream.text + expected_stream_events: Final = tuple( + ( + event, + {**data, "message": {**_JSON_OBJECT.validate_python(data["message"]), "model": model_a}} + if event == "message_start" + else data, + ) + for event, data in cc.sse_events( + b"".join(cc.message_stream(message_ids[1], _BACKEND, _REPLY_CONTENT, _REPLY_USAGE)).decode() + ) + ) + assert cc.sse_events(model_a_stream.text) == expected_stream_events, model_a_stream.text + + upstream_requests: Final = upstream.drain() + assert len(upstream_requests) == 2, upstream_requests + assert tuple( + (request.method, request.target, _JSON_OBJECT.validate_json(request.body)) + for request in upstream_requests + ) == ( + ("POST", "/v1/messages", expected_body), + ("POST", "/v1/messages", {**expected_body, "stream": True}), + ), upstream_requests + + model_b_response: Final = enabled_gateway.client.post( + "/claude_code_gateway/v1/messages", json={**request_body, "model": model_b}, headers=bearer_headers + ) + assert model_b_response.status_code == 403, model_b_response.text + assert _JSON_OBJECT.validate_json(model_b_response.content) == { + "error": { + "message": ( + f"The requested model '{model_b}' is not available for this API key, or the model name is " + "invalid. Check the models available to you and try again." + ), + "type": "team_model_access_denied", + "param": "model", + "code": "403", + } + }, model_b_response.text + assert upstream.drain() == () + + model_c_response: Final = enabled_gateway.client.post( + "/claude_code_gateway/v1/messages", json={**request_body, "model": model_c}, headers=bearer_headers + ) + assert model_c_response.status_code == 200, model_c_response.text + assert _MESSAGE.validate_json(model_c_response.content).model_dump(mode="json") == ( + expected_message.model_copy(update={"id": message_ids[2], "model": model_c}).model_dump(mode="json") + ), model_c_response.text + assert tuple( + (request.method, request.target, _JSON_OBJECT.validate_json(request.body)) + for request in upstream.drain() + ) == (("POST", "/v1/messages", expected_body),) + + spend_rows: Final = eventually( + lambda: read_rows( + 'SELECT request_id, "user", team_id, model_group, call_type FROM "LiteLLM_SpendLogs" ' + "WHERE team_id=%s AND status='success' ORDER BY \"startTime\"", + (team_id,), + ), + lambda rows: len(rows) == 3, + seconds=70, + return_last_on_timeout=True, + ) + assert spend_rows == [ + { + "request_id": message_id, + "user": _USER_ID, + "team_id": team_id, + "model_group": model, + "call_type": "anthropic_messages", + } + for message_id, model in zip(message_ids, (model_a, model_a, model_c), strict=True) + ], spend_rows + idp_requests: Final = identity_provider.drain() + assert len(idp_requests) == 3, idp_requests + (authorization_request, token_request, userinfo_request) = idp_requests + authorization_query: Final = parse_qs(urlsplit(authorization_request.target).query) + assert authorization_request.method == "GET" and urlsplit(authorization_request.target).path == "/authorize" + assert set(authorization_query) == { + "client_id", + "code_challenge", + "code_challenge_method", + "redirect_uri", + "response_type", + "scope", + "state", + }, authorization_query + assert authorization_query["client_id"] == [_CLIENT_ID], authorization_query + assert authorization_query["response_type"] == ["code"], authorization_query + assert authorization_query["scope"] == ["openid email profile"], authorization_query + assert authorization_query["code_challenge_method"] == ["S256"], authorization_query + token_form: Final = parse_qs(token_request.body.decode()) + verifier: Final = token_form["code_verifier"][0] + expected_challenge: Final = base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()) + assert authorization_query["code_challenge"] == [expected_challenge.rstrip(b"=").decode()], token_request + assert token_request.method == "POST" and urlsplit(token_request.target).path == "/token", token_request + assert token_request.headers["content-type"] == "application/x-www-form-urlencoded", token_request.headers + assert set(token_form) == {"code", "code_verifier", "grant_type", "redirect_uri"}, token_form + assert userinfo_request.method == "GET" and urlsplit(userinfo_request.target).path == "/userinfo" + assert user_id == _USER_ID diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_count_tokens_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_count_tokens_wire.py new file mode 100644 index 00000000000..92512a304bc --- /dev/null +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_count_tokens_wire.py @@ -0,0 +1,240 @@ +import json +from collections.abc import Callable +from typing import Final + +import anthropic +import pytest +from anthropic.types import MessageTokensCount +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "claude-sonnet-4-6" +_API_KEY: Final = "synthetic-anthropic-key" +_COUNT_BETA: Final = "token-counting-2024-11-01" +_CLIENT_BETA: Final = "client-future-beta-2099-01-01" +_MERGED_BETA: Final = f"{_CLIENT_BETA},{_COUNT_BETA}" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MESSAGES: Final[list[JsonValue]] = [{"role": "user", "content": "Count this synthetic request."}] +_SYSTEM: Final[list[JsonValue]] = [{"type": "text", "text": "Count the supplied message and tool."}] +_TOOLS: Final[list[JsonValue]] = [ + { + "name": "lookup", + "description": "Look up a synthetic record.", + "input_schema": { + "type": "object", + "properties": {"record_id": {"type": "string"}}, + "required": ["record_id"], + }, + } +] +_COUNTED_INPUT_TOKENS: Final = 73 +_UPSTREAM_ERROR: Final = { + "type": "error", + "error": {"type": "invalid_request_error", "message": "The scripted count request was rejected."}, +} + + +def _client(gateway: Gateway, key: str) -> anthropic.Anthropic: + return anthropic.Anthropic( + api_key=key, + base_url=str(gateway.client.base_url), + max_retries=0, + http_client=gateway.client, + ) + + +def _count_peer( + expected_body: dict[str, JsonValue], result: Reply, expected_beta: str = _COUNT_BETA +) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages/count_tokens", request.target + assert request.headers["x-api-key"] == _API_KEY, request.headers + assert request.headers["anthropic-version"] == "2023-06-01", request.headers + assert request.headers["anthropic-beta"] == expected_beta, request.headers + assert _JSON_OBJECT.validate_json(request.body) == expected_body, request.body + return result + + return respond + + +def _sdk_count(client: anthropic.Anthropic, model: str, spelling: str, beta: str) -> MessageTokensCount: + if spelling == "beta": + beta_result: Final = client.beta.messages.count_tokens( + model=model, + messages=_MESSAGES, + system=_SYSTEM, + tools=_TOOLS, + betas=[beta], + ) + return MessageTokensCount.model_validate(beta_result.model_dump(exclude_none=True)) + return client.messages.count_tokens( + model=model, + messages=_MESSAGES, + system=_SYSTEM, + tools=_TOOLS, + extra_headers={"anthropic-beta": beta}, + ) + + +@pytest.mark.parametrize("spelling", ["messages", "beta"], ids=["messages_count_tokens", "beta_messages_count_tokens"]) +def test_anthropic_count_tokens_forwards_body_and_refuses_a_key_without_model_access( + gateway: Gateway, spelling: str +) -> None: + with gateway.scenario() as scenario: + with wire_server( + _count_peer( + {"model": _BACKEND, "messages": _MESSAGES, "system": _SYSTEM, "tools": _TOOLS}, + Reply(body=json.dumps({"input_tokens": _COUNTED_INPUT_TOKENS}).encode()), + ) + ) as wire: + model: Final = scenario.model( + model=f"anthropic/{_BACKEND}", + api_base=wire.url, + api_key=_API_KEY, + num_retries=0, + ) + restricted_model: Final = scenario.model( + model="anthropic/claude-haiku-4-5-20251001", + api_base=wire.url, + api_key=_API_KEY, + num_retries=0, + ) + restricted_key: Final = scenario.key(models=[restricted_model]) + + client: Final = _client(gateway, gateway.key) + result: Final = _sdk_count(client, model, spelling, _COUNT_BETA) + assert result.model_dump() == {"input_tokens": _COUNTED_INPUT_TOKENS} + requests: Final = wire.drain() + assert len(requests) == 1, requests + + restricted_client: Final = _client(gateway, restricted_key) + with pytest.raises(anthropic.PermissionDeniedError) as raised: + _sdk_count(restricted_client, model, spelling, _COUNT_BETA) + assert raised.value.status_code == 403, raised.value.response.text + assert model in raised.value.message, raised.value.message + assert wire.drain() == () + + +def test_anthropic_count_tokens_relays_an_upstream_server_error(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + with wire_server( + _count_peer( + {"model": _BACKEND, "messages": _MESSAGES, "system": _SYSTEM, "tools": _TOOLS}, + Reply(status=500, body=json.dumps(_UPSTREAM_ERROR).encode()), + ) + ) as wire: + model: Final = scenario.model( + model=f"anthropic/{_BACKEND}", + api_base=wire.url, + api_key=_API_KEY, + num_retries=0, + ) + client: Final = _client(gateway, gateway.key) + with pytest.raises(anthropic.InternalServerError) as raised: + client.messages.count_tokens( + model=model, + messages=_MESSAGES, + system=_SYSTEM, + tools=_TOOLS, + extra_headers={"anthropic-beta": _COUNT_BETA}, + ) + assert raised.value.status_code == 500 + assert "The scripted count request was rejected." in raised.value.message + assert len(wire.drain()) == 1 + + +def test_anthropic_count_tokens_relays_an_upstream_client_error(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + with wire_server( + _count_peer( + {"model": _BACKEND, "messages": _MESSAGES, "system": _SYSTEM, "tools": _TOOLS}, + Reply(status=400, body=json.dumps(_UPSTREAM_ERROR).encode()), + ) + ) as wire: + model: Final = scenario.model( + model=f"anthropic/{_BACKEND}", + api_base=wire.url, + api_key=_API_KEY, + num_retries=0, + ) + client: Final = _client(gateway, gateway.key) + with pytest.raises(anthropic.BadRequestError) as raised: + client.messages.count_tokens( + model=model, + messages=_MESSAGES, + system=_SYSTEM, + tools=_TOOLS, + extra_headers={"anthropic-beta": _COUNT_BETA}, + ) + assert raised.value.status_code == 400 + assert "The scripted count request was rejected." in raised.value.message + assert len(wire.drain()) == 1 + + +def test_anthropic_count_tokens_preserves_thinking_tool_choice_and_output_config(gateway: Gateway) -> None: + pytest.skip( + "BUG: /v1/messages/count_tokens drops thinking, tool_choice and output_config before the provider count call" + ) + + thinking: Final = {"type": "enabled", "budget_tokens": 1024} + tool_choice: Final = {"type": "auto", "disable_parallel_tool_use": True} + output_config: Final = {"effort": "medium"} + expected_body: Final = { + "model": _BACKEND, + "messages": _MESSAGES, + "system": _SYSTEM, + "tools": _TOOLS, + "thinking": thinking, + "tool_choice": tool_choice, + "output_config": output_config, + } + + with gateway.scenario() as scenario: + with wire_server( + _count_peer(expected_body, Reply(body=json.dumps({"input_tokens": _COUNTED_INPUT_TOKENS}).encode())) + ) as wire: + model: Final = scenario.model( + model=f"anthropic/{_BACKEND}", + api_base=wire.url, + api_key=_API_KEY, + num_retries=0, + ) + client: Final = _client(gateway, gateway.key) + result: Final = client.messages.count_tokens( + model=model, + messages=_MESSAGES, + system=_SYSTEM, + tools=_TOOLS, + thinking=thinking, + tool_choice=tool_choice, + output_config=output_config, + extra_headers={"anthropic-beta": _COUNT_BETA}, + ) + assert result.input_tokens == _COUNTED_INPUT_TOKENS + assert len(wire.drain()) == 1 + + +@pytest.mark.parametrize("spelling", ["messages", "beta"], ids=["messages_count_tokens", "beta_messages_count_tokens"]) +def test_anthropic_count_tokens_merges_the_client_beta_with_the_token_counting_beta( + gateway: Gateway, spelling: str +) -> None: + pytest.skip("BUG: /v1/messages/count_tokens drops the client's anthropic-beta before the provider count call") + + with gateway.scenario() as scenario: + with wire_server( + _count_peer( + {"model": _BACKEND, "messages": _MESSAGES, "system": _SYSTEM, "tools": _TOOLS}, + Reply(body=json.dumps({"input_tokens": _COUNTED_INPUT_TOKENS}).encode()), + _MERGED_BETA, + ) + ) as wire: + model: Final = scenario.model( + model=f"anthropic/{_BACKEND}", + api_base=wire.url, + api_key=_API_KEY, + num_retries=0, + ) + result: Final = _sdk_count(_client(gateway, gateway.key), model, spelling, _CLIENT_BETA) + assert result.model_dump() == {"input_tokens": _COUNTED_INPUT_TOKENS} + assert len(wire.drain()) == 1 diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_gateway_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_gateway_wire.py new file mode 100644 index 00000000000..8a79d5c6a7a --- /dev/null +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_gateway_wire.py @@ -0,0 +1,359 @@ +import hashlib +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import pytest +import yaml +from anthropic.types import Message +from integration._support import claude_code as cc +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.database import read_rows +from integration._support.process import owned_proxy_process +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "claude-sonnet-4-6" +_API_KEY: Final = "synthetic-anthropic-key" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MESSAGE: Final = TypeAdapter(Message) + + +def _expected_stream_events( + stream_reply: tuple[bytes, ...], model: str +) -> tuple[tuple[str, dict[str, JsonValue]], ...]: + return tuple( + ( + event, + { + **data, + **( + { + "message": { + **_JSON_OBJECT.validate_python(data["message"]), + "model": model, + } + } + if event == "message_start" + else {} + ), + }, + ) + for event, data in cc.sse_events(b"".join(stream_reply).decode()) + ) + + +def _owned_config(path: Path) -> Path: + config: Final = _JSON_OBJECT.validate_python( + yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + ) + general_settings: Final = _JSON_OBJECT.validate_python(config.get("general_settings", {})) + litellm_settings: Final = _JSON_OBJECT.validate_python(config.get("litellm_settings", {})) + path.write_text( + yaml.safe_dump( + { + **config, + "general_settings": {**general_settings, "enable_claude_code_gateway": True}, + "litellm_settings": {**litellm_settings, "cache": False}, + } + ) + ) + return path + + +def _cli_bearer_headers(key: str) -> dict[str, str]: + return { + **{name: value for name, value in cc.cli_headers(key).items() if name != "x-api-key"}, + "authorization": f"Bearer {key}", + } + + +@pytest.fixture(scope="module") +def enabled_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + with gateway_from_environment() as base_gateway: + directory: Final = tmp_path_factory.mktemp("anthropic-gateway") + config: Final = _owned_config(directory / "gateway.yaml") + with owned_proxy_process(base_gateway, directory, {}, config=config, workers=2) as owned: + yield owned.gateway + + +@pytest.mark.parametrize( + "path", + ["/claude_code_gateway/v1/messages?beta=true", "/claude_code_gateway/v1/messages/count_tokens?beta=true"], + ids=["messages", "count_tokens"], +) +def test_shared_gateway_route_is_disabled_without_reaching_provider(gateway: Gateway, path: str) -> None: + def respond(request: Request) -> Reply: + raise AssertionError(f"disabled gateway must not reach the provider: {request.target}") + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.client.post( + path, + json={"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "disabled control"}]}, + headers=_cli_bearer_headers(gateway.key), + ) + assert response.status_code == 404, response.text + assert _JSON_OBJECT.validate_json(response.content) == {"detail": "Claude Code gateway is not enabled"}, ( + response.text + ) + assert wire.drain() == () + + +def _restricted_model_responses(enabled_gateway: Gateway) -> tuple[int, str, int, str, str, tuple[Request, ...]]: + def respond(request: Request) -> Reply: + raise AssertionError(f"restricted model must not reach the provider: {request.target}") + + with enabled_gateway.scenario() as scenario, wire_server(respond) as wire: + model_a: Final = scenario.model(model=f"anthropic/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + model_b: Final = scenario.model(model=f"anthropic/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + key: Final = scenario.key(models=[model_a]) + request_body: Final = {**cc.claude_code_request("restricted-model"), "model": model_b} + gateway_response: Final = enabled_gateway.client.post( + "/claude_code_gateway/v1/messages?beta=true", + json=request_body, + headers=_cli_bearer_headers(key), + ) + messages_response: Final = enabled_gateway.client.post( + "/v1/messages?beta=true", + json=request_body, + headers=cc.cli_headers(key), + ) + return ( + gateway_response.status_code, + gateway_response.text, + messages_response.status_code, + messages_response.text, + model_b, + wire.drain(), + ) + + +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +@pytest.mark.timeout(300) +def test_enabled_gateway_matches_native_messages_route(enabled_gateway: Gateway, stream: bool) -> None: + identity: Final = "msg_gateway_" + ("stream" if stream else "json") + usage: Final = {"input_tokens": 6, "output_tokens": 1} + content: Final = ({"type": "text", "text": "PONG"},) + request_body: Final = { + **cc.claude_code_request(f"cache-bust-{identity}"), + "model": "", + "stream": stream, + } + expected_stream: Final = cc.message_stream(identity, _BACKEND, content, usage) + expected_provider_body: Final = {**request_body, "model": _BACKEND} + client_betas: Final = sorted(cc.CLI_BETA.split(",")) + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages", request.target + assert request.headers["x-api-key"] == _API_KEY, request.headers + assert request.headers["anthropic-version"] == "2023-06-01", request.headers + assert sorted(request.headers["anthropic-beta"].split(",")) == client_betas, request.headers + assert "authorization" not in request.headers, request.headers + assert _JSON_OBJECT.validate_json(request.body) == expected_provider_body, request.body + if stream: + return Reply(chunks=expected_stream, content_type="text/event-stream") + return Reply(body=cc.message_reply(identity, _BACKEND, content, usage)) + + with enabled_gateway.scenario() as scenario, wire_server(respond) as wire: + model_a: Final = scenario.model(model=f"anthropic/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + key: Final = scenario.key(models=[model_a]) + expected_message: Final = _MESSAGE.validate_python( + {**_JSON_OBJECT.validate_json(cc.message_reply(identity, _BACKEND, content, usage)), "model": model_a} + ) + expected_stream_events: Final = _expected_stream_events(expected_stream, model_a) + body: Final = {**request_body, "model": model_a} + headers: Final = _cli_bearer_headers(key) + gateway_response: Final = enabled_gateway.client.post( + "/claude_code_gateway/v1/messages?beta=true", + json=body, + headers=headers, + ) + assert gateway_response.status_code == 200, gateway_response.text + gateway_requests: Final = wire.drain() + assert len(gateway_requests) == 1, gateway_requests + + messages_response: Final = enabled_gateway.client.post( + "/v1/messages?beta=true", + json=body, + headers=headers, + ) + assert messages_response.status_code == 200, messages_response.text + messages_requests: Final = wire.drain() + assert len(messages_requests) == 1, messages_requests + if stream: + gateway_events: Final = cc.sse_events(gateway_response.text) + messages_events: Final = cc.sse_events(messages_response.text) + assert gateway_events == messages_events, (gateway_response.text, messages_response.text) + assert gateway_events == expected_stream_events, (gateway_response.text, messages_response.text) + else: + gateway_message: Final = _MESSAGE.validate_json(gateway_response.content) + messages_message: Final = _MESSAGE.validate_json(messages_response.content) + assert gateway_message.model_dump(mode="json") == expected_message.model_dump(mode="json"), ( + gateway_response.text, + ) + assert messages_message.model_dump(mode="json") == expected_message.model_dump(mode="json"), ( + messages_response.text, + ) + + (gateway_request,) = gateway_requests + (messages_request,) = messages_requests + assert gateway_request.method == messages_request.method == "POST" + assert gateway_request.target == messages_request.target == "/v1/messages" + assert gateway_request.headers["x-api-key"] == messages_request.headers["x-api-key"] == _API_KEY + assert ( + gateway_request.headers["anthropic-version"] + == messages_request.headers["anthropic-version"] + == "2023-06-01" + ) + assert gateway_request.headers["anthropic-beta"] == messages_request.headers["anthropic-beta"] + assert sorted(gateway_request.headers["anthropic-beta"].split(",")) == client_betas + assert _JSON_OBJECT.validate_json(gateway_request.body) == expected_provider_body, gateway_request.body + assert _JSON_OBJECT.validate_json(messages_request.body) == expected_provider_body, messages_request.body + + +@pytest.mark.timeout(300) +def test_enabled_gateway_count_tokens_alias_matches_native_count_tokens_route(enabled_gateway: Gateway) -> None: + messages: Final = [{"role": "user", "content": "Count this gateway request."}] + system: Final = [{"type": "text", "text": "Count the gateway message and tool."}] + tools: Final = [ + { + "name": "lookup", + "description": "Look up a synthetic record.", + "input_schema": {"type": "object", "properties": {"record_id": {"type": "string"}}}, + } + ] + expected_provider_body: Final = {"model": _BACKEND, "messages": messages, "system": system, "tools": tools} + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages/count_tokens", request.target + assert request.headers["x-api-key"] == _API_KEY, request.headers + assert request.headers["anthropic-version"] == "2023-06-01", request.headers + assert request.headers["anthropic-beta"] == "token-counting-2024-11-01", request.headers + assert "authorization" not in request.headers, request.headers + assert _JSON_OBJECT.validate_json(request.body) == expected_provider_body, request.body + return Reply(body=b'{"input_tokens": 41}') + + with enabled_gateway.scenario() as scenario, wire_server(respond) as wire: + model: Final = scenario.model(model=f"anthropic/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + restricted_model: Final = scenario.model(model=f"anthropic/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + key: Final = scenario.key(models=[model]) + body: Final = {"model": model, "messages": messages, "system": system, "tools": tools} + headers: Final = {**_cli_bearer_headers(key), "anthropic-beta": "token-counting-2024-11-01"} + gateway_response: Final = enabled_gateway.client.post( + "/claude_code_gateway/v1/messages/count_tokens?beta=true", json=body, headers=headers + ) + assert gateway_response.status_code == 200, gateway_response.text + assert _JSON_OBJECT.validate_json(gateway_response.content) == {"input_tokens": 41}, gateway_response.text + native_response: Final = enabled_gateway.client.post( + "/v1/messages/count_tokens?beta=true", json=body, headers=headers + ) + assert native_response.status_code == 200, native_response.text + assert native_response.content == gateway_response.content, (native_response.text, gateway_response.text) + assert tuple((request.method, request.target) for request in wire.drain()) == ( + ("POST", "/v1/messages/count_tokens"), + ("POST", "/v1/messages/count_tokens"), + ) + + restricted_response: Final = enabled_gateway.client.post( + "/claude_code_gateway/v1/messages/count_tokens?beta=true", + json={**body, "model": restricted_model}, + headers=headers, + ) + assert restricted_response.status_code == 403, restricted_response.text + assert _JSON_OBJECT.validate_json(restricted_response.content) == { + "error": { + "message": ( + f"The requested model '{restricted_model}' is not available for this API key, or the model name " + "is invalid. Check the models available to you and try again." + ), + "type": "key_model_access_denied", + "param": "model", + "code": "403", + } + }, restricted_response.text + assert wire.drain() == () + + +@pytest.mark.timeout(300) +def test_enabled_gateway_refuses_model_outside_virtual_key(enabled_gateway: Gateway) -> None: + result: Final = _restricted_model_responses(enabled_gateway) + expected_body: Final = { + "error": { + "message": ( + f"The requested model '{result[4]}' is not available for this API key, or the model name is invalid. " + "Check the models available to you and try again." + ), + "type": "key_model_access_denied", + "param": "model", + "code": "403", + } + } + assert result[0] == result[2] == 403, (result[1], result[3]) + assert _JSON_OBJECT.validate_json(result[1]) == expected_body, result[1] + assert _JSON_OBJECT.validate_json(result[3]) == expected_body, result[3] + assert result[1] == result[3], (result[1], result[3]) + assert result[5] == () + + +@pytest.mark.timeout(300) +def test_enabled_gateway_attributes_spend_to_virtual_key(enabled_gateway: Gateway) -> None: + gateway_identity: Final = f"msg_gateway_spend_gateway_{uuid.uuid4().hex}" + native_identity: Final = f"msg_gateway_spend_native_{uuid.uuid4().hex}" + response_ids: Final = iter((gateway_identity, native_identity)) + content: Final = ({"type": "text", "text": "PONG"},) + usage: Final = {"input_tokens": 6, "output_tokens": 1} + request_body: Final = {**cc.claude_code_request("cache-bust-msg-gateway-spend"), "model": "", "stream": False} + expected_provider_body: Final = {**request_body, "model": _BACKEND} + client_betas: Final = sorted(cc.CLI_BETA.split(",")) + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages", request.target + assert request.headers["x-api-key"] == _API_KEY, request.headers + assert request.headers["anthropic-version"] == "2023-06-01", request.headers + assert sorted(request.headers["anthropic-beta"].split(",")) == client_betas, request.headers + assert "authorization" not in request.headers, request.headers + assert _JSON_OBJECT.validate_json(request.body) == expected_provider_body, request.body + return Reply(body=cc.message_reply(next(response_ids), _BACKEND, content, usage)) + + with enabled_gateway.scenario() as scenario, wire_server(respond) as wire: + model: Final = scenario.model(model=f"anthropic/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + key: Final = scenario.key(models=[model]) + native_key: Final = scenario.key(models=[model]) + body: Final = {**request_body, "model": model} + response: Final = enabled_gateway.client.post( + "/claude_code_gateway/v1/messages?beta=true", + json=body, + headers=_cli_bearer_headers(key), + ) + assert response.status_code == 200, response.text + native_response: Final = enabled_gateway.client.post( + "/v1/messages?beta=true", + json=body, + headers=cc.cli_headers(native_key), + ) + assert native_response.status_code == 200, native_response.text + requests: Final = wire.drain() + assert tuple(request.target for request in requests) == ("/v1/messages", "/v1/messages"), requests + gateway_key_hash: Final = hashlib.sha256(key.encode()).hexdigest() + native_key_hash: Final = hashlib.sha256(native_key.encode()).hexdigest() + spend_rows: Final = eventually( + lambda: ( + read_rows( + 'SELECT api_key, model_group FROM "LiteLLM_SpendLogs" WHERE api_key=%s', + (gateway_key_hash,), + ), + read_rows( + 'SELECT api_key, model_group FROM "LiteLLM_SpendLogs" WHERE api_key=%s', + (native_key_hash,), + ), + ), + lambda rows: len(rows[0]) == 1 and len(rows[1]) == 1, + seconds=70, + return_last_on_timeout=True, + ) + assert spend_rows == ( + [{"api_key": gateway_key_hash, "model_group": model}], + [{"api_key": native_key_hash, "model_group": model}], + ), spend_rows diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_wire.py index 7e9c5be227a..dcfd1d19ac2 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_wire.py @@ -1,12 +1,20 @@ +import base64 import json import time import uuid from typing import Final +import anthropic import pytest +from anthropic.types import Message +from integration._support import claude_code as cc from integration._support.client import Gateway, eventually, object_value from integration._support.database import read_rows from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MESSAGE: Final = TypeAdapter(Message) @pytest.mark.covers( @@ -204,3 +212,563 @@ def test_anthropic_messages_slow_upstream_is_cut_off_at_the_deployment_request_t assert response.status_code == 408, response.text assert "Timeout" in response.json()["error"]["message"], response.text assert eventually(wire.drain, lambda requests: len(requests) == 1, seconds=5, return_last_on_timeout=True) + + +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +def test_anthropic_messages_preserves_cli_beta_safeguards_and_result(gateway: Gateway, stream: bool) -> None: + identity: Final = "anthropic-safeguards-" + uuid.uuid4().hex + beta: Final = cc.CLI_BETA + ",made-up-future-beta-2099-01-01" + safeguards: Final = {"mode": "strict", "policies": ["sensitive-data"]} + safeguard_results: Final = {"status": "allowed", "policy": "sensitive-data"} + system: Final = [{"type": "text", "text": "Synthetic policy.", "cache_control": {"type": "ephemeral"}}] + tools: Final = [ + { + "name": "lookup", + "description": "Look up a synthetic record.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}, + }, + { + "name": "update", + "description": "Update a synthetic record.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}, + }, + ] + response_body: Final = { + "id": identity, + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "Record checked."}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 6, "output_tokens": 3}, + "safeguard_results": safeguard_results, + } + stream_reply: Final = ( + cc.sse_frame( + "message_start", + { + "type": "message_start", + "message": { + **response_body, + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 6, "output_tokens": 0}, + }, + }, + ), + cc.sse_frame( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + cc.sse_frame( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Record checked."}}, + ), + cc.sse_frame("content_block_stop", {"type": "content_block_stop", "index": 0}), + cc.sse_frame( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 3}, + }, + ), + cc.sse_frame("message_stop", {"type": "message_stop"}), + ) + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages", request.target + assert request.headers["x-api-key"] == "synthetic-anthropic-key", request.headers + assert sorted(request.headers["anthropic-beta"].split(",")) == sorted(beta.split(",")), request.headers + expected_body: Final = { + "model": "claude-sonnet-4-6", + "max_tokens": 32, + "messages": [{"role": "user", "content": "Check the synthetic record."}], + "system": system, + "tools": tools, + "metadata": {"user_id": cc.METADATA_USER_ID}, + "safeguards": safeguards, + "stream": stream, + } + assert _JSON_OBJECT.validate_json(request.body) == expected_body, request.body + if not stream: + return Reply(body=json.dumps(response_body).encode()) + return Reply(chunks=stream_reply, content_type="text/event-stream") + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="anthropic/claude-sonnet-4-6", + api_base=wire.url, + api_key="synthetic-anthropic-key", + ) + client_body: Final = { + "model": model, + "max_tokens": 32, + "messages": [{"role": "user", "content": "Check the synthetic record."}], + "system": system, + "tools": tools, + "metadata": {"user_id": cc.METADATA_USER_ID}, + "safeguards": safeguards, + "stream": stream, + } + response: Final = gateway.client.post( + "/v1/messages?beta=true", + json=client_body, + headers=cc.cli_headers(gateway.key, beta=beta), + ) + assert response.status_code == 200, response.text + (sent,) = wire.drain() + assert sent.target == "/v1/messages", sent.target + + if stream: + expected_stream_events: Final = tuple( + ( + event, + { + **data, + **( + { + "message": { + **_JSON_OBJECT.validate_python(data["message"]), + "model": model, + } + } + if event == "message_start" + else {} + ), + }, + ) + for event, data in cc.sse_events(b"".join(stream_reply).decode()) + ) + assert cc.sse_events(response.text) == expected_stream_events, response.text + + typed_message: Final = ( + _MESSAGE.validate_python( + next(data["message"] for event, data in cc.sse_events(response.text) if event == "message_start") + ) + if stream + else _MESSAGE.validate_json(response.content) + ) + assert typed_message.model_extra == {"safeguard_results": safeguard_results}, response.text + + +@pytest.mark.parametrize( + ("status", "error_type", "error_class", "litellm_error"), + [ + pytest.param(429, "rate_limit_error", anthropic.RateLimitError, "RateLimitError", id="rate_limit"), + pytest.param(400, "invalid_request_error", anthropic.BadRequestError, "BadRequestError", id="invalid_request"), + ], +) +def test_anthropic_messages_preserves_upstream_error_status_and_body( + gateway: Gateway, + status: int, + error_type: str, + error_class: type[anthropic.APIStatusError], + litellm_error: str, +) -> None: + message: Final = f"synthetic {error_type}" + expected_error: Final = {"type": "error", "error": {"type": error_type, "message": message}} + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages", request.target + assert request.headers["x-api-key"] == "synthetic-anthropic-key", request.headers + assert _JSON_OBJECT.validate_json(request.body) == { + "model": "claude-sonnet-4-6", + "max_tokens": 16, + "messages": [{"role": "user", "content": "Trigger the scripted error."}], + "stream": False, + }, request.body + return Reply( + status=status, + body=json.dumps(expected_error).encode(), + headers={"retry-after": "7"} if status == 429 else {}, + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="anthropic/claude-sonnet-4-6", + api_base=wire.url, + api_key="synthetic-anthropic-key", + ) + with ( + anthropic.Anthropic( + base_url=str(gateway.client.base_url), + api_key=gateway.key, + max_retries=0, + ) as client, + pytest.raises(error_class) as caught, + ): + client.messages.create( + model=model, + max_tokens=16, + messages=[{"role": "user", "content": "Trigger the scripted error."}], + ) + assert caught.value.status_code == status, caught.value.response.text + assert caught.value.response.json() == { + "type": "error", + "error": { + "type": error_type, + "message": ( + f"litellm.{litellm_error}: AnthropicException - {json.dumps(expected_error)}\n\n" + f"LiteLLM: model group '{model}' failed with the error above. No fallback was attempted." + ), + }, + }, caught.value.response.text + assert len(wire.drain()) == 1 + + +def test_anthropic_messages_preserves_upstream_rate_limit_retry_after(gateway: Gateway) -> None: + pytest.skip("BUG: /v1/messages drops the upstream retry-after header on 429") + + message: Final = "synthetic rate_limit_error" + upstream_error: Final = { + "type": "error", + "error": {"type": "rate_limit_error", "message": message}, + } + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages", request.target + assert request.headers["x-api-key"] == "synthetic-anthropic-key", request.headers + assert _JSON_OBJECT.validate_json(request.body) == { + "model": "claude-sonnet-4-6", + "max_tokens": 16, + "messages": [{"role": "user", "content": "Trigger the scripted error."}], + "stream": False, + }, request.body + return Reply(status=429, body=json.dumps(upstream_error).encode(), headers={"retry-after": "7"}) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="anthropic/claude-sonnet-4-6", + api_base=wire.url, + api_key="synthetic-anthropic-key", + ) + with ( + anthropic.Anthropic( + base_url=str(gateway.client.base_url), + api_key=gateway.key, + max_retries=0, + ) as client, + pytest.raises(anthropic.RateLimitError) as caught, + ): + client.messages.create( + model=model, + max_tokens=16, + messages=[{"role": "user", "content": "Trigger the scripted error."}], + ) + assert caught.value.status_code == 429, caught.value.response.text + assert caught.value.response.headers.get("retry-after") == "7", caught.value.response.text + assert caught.value.response.json() == { + "type": "error", + "error": { + "type": "rate_limit_error", + "message": ( + f"litellm.RateLimitError: AnthropicException - {json.dumps(upstream_error)}\n\n" + f"LiteLLM: model group '{model}' failed with the error above. No fallback was attempted." + ), + }, + }, caught.value.response.text + assert len(wire.drain()) == 1 + + +def test_anthropic_messages_preserves_upstream_overloaded_error(gateway: Gateway) -> None: + pytest.skip("BUG: /v1/messages maps an upstream 529 overloaded_error to HTTP 500") + + upstream_error: Final = { + "type": "error", + "error": {"type": "overloaded_error", "message": "synthetic overloaded error"}, + } + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages", request.target + assert request.headers["x-api-key"] == "synthetic-anthropic-key", request.headers + assert _JSON_OBJECT.validate_json(request.body) == { + "model": "claude-sonnet-4-6", + "max_tokens": 16, + "messages": [{"role": "user", "content": "Trigger the scripted overload."}], + "stream": False, + }, request.body + return Reply(status=529, body=json.dumps(upstream_error).encode()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="anthropic/claude-sonnet-4-6", + api_base=wire.url, + api_key="synthetic-anthropic-key", + ) + with ( + anthropic.Anthropic( + base_url=str(gateway.client.base_url), + api_key=gateway.key, + max_retries=0, + ) as client, + pytest.raises(anthropic.InternalServerError) as caught, + ): + client.messages.create( + model=model, + max_tokens=16, + messages=[{"role": "user", "content": "Trigger the scripted overload."}], + ) + assert caught.value.status_code == 529, caught.value.response.text + response_body: Final = _JSON_OBJECT.validate_json(caught.value.response.content) + assert response_body == upstream_error, caught.value.response.text + assert len(wire.drain()) == 1 + + +def test_anthropic_messages_stream_surfaces_upstream_error_event(gateway: Gateway) -> None: + error: Final = { + "type": "error", + "error": {"type": "overloaded_error", "message": "synthetic stream overload"}, + } + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages", request.target + assert request.headers["x-api-key"] == "synthetic-anthropic-key", request.headers + assert _JSON_OBJECT.validate_json(request.body) == { + "model": "claude-sonnet-4-6", + "max_tokens": 16, + "messages": [{"role": "user", "content": "Trigger the scripted stream error."}], + "stream": True, + }, request.body + return Reply( + chunks=( + cc.sse_frame( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_stream_error", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 5, "output_tokens": 0}, + }, + }, + ), + cc.sse_frame("error", error), + ), + content_type="text/event-stream", + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="anthropic/claude-sonnet-4-6", + api_base=wire.url, + api_key="synthetic-anthropic-key", + ) + response: Final = gateway.client.post( + "/v1/messages", + json={ + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": "Trigger the scripted stream error."}], + "stream": True, + }, + headers=cc.cli_headers(gateway.key), + ) + assert response.status_code == 200, response.text + assert cc.sse_events(response.text) == ( + ( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_stream_error", + "type": "message", + "role": "assistant", + "model": model, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 5, "output_tokens": 0}, + }, + }, + ), + ("error", error), + ), response.text + assert len(wire.drain()) == 1 + + +def test_anthropic_messages_stream_ending_without_message_stop_surfaces_an_error_event(gateway: Gateway) -> None: + pytest.skip("BUG: /v1/messages relays an upstream stream that closes before message_stop as a clean 200") + message_start: Final = { + "type": "message_start", + "message": { + "id": "msg_stream_truncated", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 5, "output_tokens": 0}, + }, + } + block_start: Final = {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}} + block_delta: Final = {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Partial"}} + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages", request.target + assert request.headers["x-api-key"] == "synthetic-anthropic-key", request.headers + assert _JSON_OBJECT.validate_json(request.body) == { + "model": "claude-sonnet-4-6", + "max_tokens": 16, + "messages": [{"role": "user", "content": "Trigger the scripted truncated stream."}], + "stream": True, + }, request.body + return Reply( + chunks=( + cc.sse_frame("message_start", message_start), + cc.sse_frame("content_block_start", block_start), + cc.sse_frame("content_block_delta", block_delta), + ), + content_type="text/event-stream", + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="anthropic/claude-sonnet-4-6", + api_base=wire.url, + api_key="synthetic-anthropic-key", + ) + response: Final = gateway.client.post( + "/v1/messages", + json={ + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": "Trigger the scripted truncated stream."}], + "stream": True, + }, + headers=cc.cli_headers(gateway.key), + ) + assert response.status_code == 200, response.text + assert cc.sse_events(response.text) == ( + ("message_start", {**message_start, "message": {**message_start["message"], "model": model}}), + ("content_block_start", block_start), + ("content_block_delta", block_delta), + ( + "error", + { + "type": "error", + "error": { + "type": "api_error", + "message": "Provider stream ended before emitting a message_stop event; the response is " + "incomplete and any partial content (e.g. tool_use input JSON) may be truncated.", + }, + }, + ), + ), response.text + assert len(wire.drain()) == 1 + + +def test_anthropic_messages_preserves_image_document_and_tool_result_blocks(gateway: Gateway) -> None: + image_data: Final = base64.b64encode(b"synthetic png bytes").decode() + pdf_data: Final = base64.b64encode(b"%PDF-1.7 synthetic document").decode() + messages: Final = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Inspect these attachments."}, + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": image_data}, + }, + {"type": "image", "source": {"type": "url", "url": "https://example.invalid/synthetic.png"}}, + { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_data}, + }, + ], + }, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "lookup-one", "name": "lookup", "input": {"id": "one"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "lookup-one", + "is_error": True, + "content": [ + {"type": "text", "text": "Synthetic lookup failed."}, + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": image_data}, + }, + ], + } + ], + }, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "lookup-two", "name": "lookup", "input": {"id": "two"}}], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "lookup-two", "content": "Synthetic lookup succeeded."}, + {"type": "text", "text": "Continue with the first result."}, + ], + }, + ] + content: Final = [ + {"type": "text", "text": "The requested blocks were preserved."}, + {"type": "tool_use", "id": "lookup-three", "name": "lookup", "input": {"id": "three"}}, + {"type": "thinking", "thinking": "The history contains two tool calls.", "signature": "synthetic-signature"}, + ] + expected_response: Final = { + "id": "msg_anthropic_blocks", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": content, + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 20, "output_tokens": 12}, + } + tools: Final = [ + { + "name": "lookup", + "description": "Look up a synthetic record.", + "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}, + } + ] + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages", request.target + assert request.headers["x-api-key"] == "synthetic-anthropic-key", request.headers + assert _JSON_OBJECT.validate_json(request.body) == { + "model": "claude-sonnet-4-6", + "max_tokens": 128, + "messages": messages, + "stream": False, + "tools": tools, + }, request.body + return Reply(body=json.dumps(expected_response).encode()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="anthropic/claude-sonnet-4-6", + api_base=wire.url, + api_key="synthetic-anthropic-key", + ) + with anthropic.Anthropic( + base_url=str(gateway.client.base_url), + api_key=gateway.key, + max_retries=0, + ) as client: + response: Final = client.messages.create( + model=model, + max_tokens=128, + messages=messages, + tools=tools, + ) + expected_message: Final = _MESSAGE.validate_python({**expected_response, "model": model}) + assert response.model_dump(mode="json") == expected_message.model_dump(mode="json"), response + assert len(wire.drain()) == 1 diff --git a/tests/integration/providers/test_anthropic_skills_create_wire.py b/tests/integration/providers/test_anthropic_skills_create_wire.py new file mode 100644 index 00000000000..1499c278ffc --- /dev/null +++ b/tests/integration/providers/test_anthropic_skills_create_wire.py @@ -0,0 +1,69 @@ +import json +from email.message import Message +from email.parser import BytesParser +from email.policy import HTTP +from typing import Final + +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server + +from litellm.types.llms.anthropic_skills import Skill + +_API_KEY: Final = "synthetic-anthropic-key" +_TITLE: Final = "Synthetic Skill" +_SKILL_MD: Final = b"---\nname: synthetic-skill\ndescription: A synthetic skill.\n---\nUse the helper.\n" +_HELPER: Final = b"print('synthetic helper')\n" +_UPSTREAM_SKILL: Final = { + "id": "skill_synthetic_01", + "type": "skill", + "display_title": _TITLE, + "latest_version": "1759178010641129", + "source": "custom", + "created_at": "2025-10-02T00:00:00Z", + "updated_at": "2025-10-02T00:00:00Z", +} + + +def _multipart_parts(request: Request) -> tuple[Message, ...]: + envelope: Final = f"content-type: {request.headers['content-type']}\r\n\r\n".encode() + request.body + parsed: Final = BytesParser(policy=HTTP).parsebytes(envelope) + assert parsed.is_multipart(), request.headers["content-type"] + return tuple(parsed.iter_parts()) + + +def _part_summary(part: Message) -> tuple[str, str | None, bytes]: + return ( + str(part.get_param("name", header="content-disposition")), + part.get_filename(), + bytes(part.get_payload(decode=True)), + ) + + +def test_anthropic_skill_create_forwards_repeated_files_as_file_parts(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/skills", request.target + assert request.headers["x-api-key"] == _API_KEY, request.headers + assert request.headers["anthropic-version"] == "2023-06-01", request.headers + assert request.headers["anthropic-beta"] == "skills-2025-10-02", request.headers + assert "authorization" not in request.headers, request.headers + assert tuple(_part_summary(part) for part in _multipart_parts(request)) == ( + ("display_title", None, _TITLE.encode()), + ("files[]", "synthetic-skill/SKILL.md", _SKILL_MD), + ("files[]", "synthetic-skill/helper.py", _HELPER), + ), request.body + return Reply(body=json.dumps(_UPSTREAM_SKILL).encode()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model="anthropic/claude-sonnet-4-6", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.client.post( + "/v1/skills?beta=true", + data={"display_title": _TITLE}, + files=[ + ("files[]", ("synthetic-skill/SKILL.md", _SKILL_MD, "text/markdown")), + ("files[]", ("synthetic-skill/helper.py", _HELPER, "text/x-python")), + ], + headers={"Authorization": f"Bearer {gateway.key}", "x-litellm-model": model}, + ) + assert response.status_code == 200, response.text + assert Skill.model_validate_json(response.content) == Skill.model_validate(_UPSTREAM_SKILL), response.text + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/skills")] diff --git a/tests/integration/providers/test_azure_ai_claude_messages_wire.py b/tests/integration/providers/test_azure_ai_claude_messages_wire.py new file mode 100644 index 00000000000..c150de7520b --- /dev/null +++ b/tests/integration/providers/test_azure_ai_claude_messages_wire.py @@ -0,0 +1,323 @@ +from typing import Final + +import anthropic +import pytest +from anthropic.types import Message +from integration._support import claude_code as cc +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "claude-sonnet-4-6" +_API_KEY: Final = "synthetic-azure-ai-key" +_SUPPORTED_BETA: Final = "interleaved-thinking-2025-05-14" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MESSAGE: Final = TypeAdapter(Message) +_MESSAGES: Final[list[JsonValue]] = [{"role": "user", "content": "Read the synthetic record."}] +_SYSTEM: Final = [{"type": "text", "text": "Use the supplied tool when relevant."}] +_TOOLS: Final[list[JsonValue]] = [ + { + "name": "lookup", + "description": "Look up a synthetic record.", + "input_schema": { + "type": "object", + "properties": {"record_id": {"type": "string"}}, + "required": ["record_id"], + }, + } +] +_CONTENT: Final[tuple[dict[str, JsonValue], ...]] = ({"type": "text", "text": "The record is ready."},) +_USAGE: Final[dict[str, JsonValue]] = {"input_tokens": 8, "output_tokens": 4} + + +def _without_none(value: JsonValue) -> JsonValue: + if isinstance(value, dict): + return {key: _without_none(item) for key, item in value.items() if item is not None} + if isinstance(value, list): + return [_without_none(item) for item in value] + return value + + +@pytest.mark.parametrize("base_suffix", ["", "/anthropic"], ids=["base", "anthropic_base"]) +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +def test_azure_ai_claude_messages_preserves_target_auth_beta_and_typed_response( + gateway: Gateway, base_suffix: str, stream: bool +) -> None: + identity: Final = f"msg_azure_claude_{base_suffix.strip('/') or 'base'}_{stream}" + api_path: Final = "/anthropic/v1/messages" + reply_body: Final = cc.message_reply(identity, _BACKEND, _CONTENT, _USAGE) + stream_reply: Final = cc.message_stream(identity, _BACKEND, _CONTENT, _USAGE) + request_body_template: Final = { + "max_tokens": 1024, + "messages": _MESSAGES, + "model": "", + "stream": stream, + "system": _SYSTEM, + "tools": _TOOLS, + } + expected_provider_body: Final = {**request_body_template, "model": _BACKEND} + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == api_path, request.target + assert request.headers["x-api-key"] == _API_KEY, request.headers + assert request.headers["anthropic-version"] == "2023-06-01", request.headers + assert request.headers.get("anthropic-beta") is None, request.headers + assert "authorization" not in request.headers and "api-key" not in request.headers, request.headers + assert _JSON_OBJECT.validate_json(request.body) == expected_provider_body, request.body + if stream: + return Reply(content_type="text/event-stream", chunks=stream_reply) + return Reply(body=reply_body) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"azure_ai/{_BACKEND}", + api_base=f"{wire.url}{base_suffix}", + api_key=_API_KEY, + num_retries=0, + ) + with anthropic.Anthropic( + api_key=gateway.key, + base_url=str(gateway.client.base_url), + max_retries=0, + ) as client: + if stream: + stream_response: Final = client.messages.create( + model=model, + max_tokens=1024, + messages=_MESSAGES, + system=_SYSTEM, + tools=_TOOLS, + stream=True, + extra_headers={"anthropic-beta": "made-up-future-beta-2099-01-01"}, + ) + events: Final = tuple(stream_response) + expected_events: Final = tuple( + _JSON_OBJECT.validate_python( + _without_none( + { + **data, + **( + { + "message": { + **_JSON_OBJECT.validate_python(data["message"]), + "model": model, + } + } + if event_name == "message_start" + else {} + ), + } + ) + ) + for event_name, data in cc.sse_events(b"".join(stream_reply).decode()) + ) + actual_events: Final = tuple( + _JSON_OBJECT.validate_python(event.model_dump(mode="json", exclude_none=True)) for event in events + ) + assert actual_events == expected_events, actual_events + started_message: Final = next(event.message for event in events if event.type == "message_start") + assert started_message.id == identity and started_message.model == model, actual_events + else: + response: Final = client.messages.create( + model=model, + max_tokens=1024, + messages=_MESSAGES, + system=_SYSTEM, + tools=_TOOLS, + stream=False, + extra_headers={"anthropic-beta": "made-up-future-beta-2099-01-01"}, + ) + parsed_message: Final = _MESSAGE.validate_python(response) + expected_message: Final = _MESSAGE.validate_python( + {**_JSON_OBJECT.validate_json(reply_body), "model": model} + ) + assert parsed_message.model_dump(mode="json") == expected_message.model_dump(mode="json"), (response,) + + requests: Final = wire.drain() + assert len(requests) == 1, requests + provider_request: Final = requests[0] + assert provider_request.method == "POST" and provider_request.target == api_path, provider_request.target + assert provider_request.headers["x-api-key"] == _API_KEY, provider_request.headers + assert provider_request.headers["anthropic-version"] == "2023-06-01", provider_request.headers + assert provider_request.headers.get("anthropic-beta") is None, provider_request.headers + assert "authorization" not in provider_request.headers and "api-key" not in provider_request.headers + assert _JSON_OBJECT.validate_json(provider_request.body) == expected_provider_body, provider_request.body + + +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +def test_azure_ai_claude_messages_filters_auto_injected_betas_by_azure_support(gateway: Gateway, stream: bool) -> None: + identity: Final = f"msg_azure_claude_beta_filter_{stream}" + api_path: Final = "/anthropic/v1/messages" + reply_body: Final = cc.message_reply(identity, _BACKEND, _CONTENT, _USAGE) + stream_reply: Final = cc.message_stream(identity, _BACKEND, _CONTENT, _USAGE) + output_config: Final = { + "format": { + "type": "json_schema", + "schema": { + "type": "object", + "properties": {"record_id": {"type": "string"}}, + "required": ["record_id"], + }, + } + } + context_management: Final = {"edits": [{"type": "compact_20260112"}]} + request_body_template: Final = { + "max_tokens": 1024, + "messages": _MESSAGES, + "model": "", + "stream": stream, + "system": _SYSTEM, + "tools": _TOOLS, + "output_config": output_config, + "context_management": context_management, + } + expected_provider_body: Final = {**request_body_template, "model": _BACKEND} + expected_beta: Final = "structured-outputs-2025-11-13" + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == api_path, request.target + assert request.headers["x-api-key"] == _API_KEY, request.headers + assert request.headers["anthropic-version"] == "2023-06-01", request.headers + assert request.headers.get("anthropic-beta") == expected_beta, request.headers + assert "authorization" not in request.headers and "api-key" not in request.headers, request.headers + assert _JSON_OBJECT.validate_json(request.body) == expected_provider_body, request.body + if stream: + return Reply(content_type="text/event-stream", chunks=stream_reply) + return Reply(body=reply_body) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"azure_ai/{_BACKEND}", + api_base=wire.url, + api_key=_API_KEY, + num_retries=0, + ) + with anthropic.Anthropic( + api_key=gateway.key, + base_url=str(gateway.client.base_url), + max_retries=0, + ) as client: + if stream: + stream_response: Final = client.messages.create( + model=model, + max_tokens=1024, + messages=_MESSAGES, + system=_SYSTEM, + tools=_TOOLS, + stream=True, + output_config=output_config, + extra_body={"context_management": context_management}, + ) + events: Final = tuple(stream_response) + expected_events: Final = tuple( + _JSON_OBJECT.validate_python( + _without_none( + { + **data, + **( + { + "message": { + **_JSON_OBJECT.validate_python(data["message"]), + "model": model, + } + } + if event_name == "message_start" + else {} + ), + } + ) + ) + for event_name, data in cc.sse_events(b"".join(stream_reply).decode()) + ) + actual_events: Final = tuple( + _JSON_OBJECT.validate_python(event.model_dump(mode="json", exclude_none=True)) for event in events + ) + assert actual_events == expected_events, actual_events + started_message: Final = next(event.message for event in events if event.type == "message_start") + assert started_message.id == identity and started_message.model == model, actual_events + else: + response: Final = client.messages.create( + model=model, + max_tokens=1024, + messages=_MESSAGES, + system=_SYSTEM, + tools=_TOOLS, + stream=False, + output_config=output_config, + extra_body={"context_management": context_management}, + ) + parsed_message: Final = _MESSAGE.validate_python(response) + expected_message: Final = _MESSAGE.validate_python( + {**_JSON_OBJECT.validate_json(reply_body), "model": model} + ) + assert parsed_message.model_dump(mode="json") == expected_message.model_dump(mode="json"), (response,) + + requests: Final = wire.drain() + assert len(requests) == 1, requests + provider_request: Final = requests[0] + assert provider_request.method == "POST" and provider_request.target == api_path, provider_request.target + assert provider_request.headers["x-api-key"] == _API_KEY, provider_request.headers + assert provider_request.headers["anthropic-version"] == "2023-06-01", provider_request.headers + assert provider_request.headers.get("anthropic-beta") == expected_beta, provider_request.headers + assert "authorization" not in provider_request.headers and "api-key" not in provider_request.headers + assert _JSON_OBJECT.validate_json(provider_request.body) == expected_provider_body, provider_request.body + + +def test_azure_ai_claude_messages_forwards_supported_beta(gateway: Gateway) -> None: + pytest.skip(f"BUG: Azure AI Claude drops the supported beta header {_SUPPORTED_BETA}") + + identity: Final = "msg_azure_claude_supported_beta" + reply_body: Final = cc.message_reply(identity, _BACKEND, _CONTENT, _USAGE) + request_body: Final = { + "max_tokens": 1024, + "messages": _MESSAGES, + "model": "", + "stream": False, + "system": _SYSTEM, + "tools": _TOOLS, + } + expected_provider_body: Final = {**request_body, "model": _BACKEND} + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/anthropic/v1/messages", request.target + assert request.headers["x-api-key"] == _API_KEY, request.headers + assert request.headers["anthropic-version"] == "2023-06-01", request.headers + assert request.headers.get("anthropic-beta") == _SUPPORTED_BETA, request.headers + assert "authorization" not in request.headers and "api-key" not in request.headers, request.headers + assert _JSON_OBJECT.validate_json(request.body) == expected_provider_body, request.body + return Reply(body=reply_body) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"azure_ai/{_BACKEND}", + api_base=wire.url, + api_key=_API_KEY, + num_retries=0, + ) + with anthropic.Anthropic( + api_key=gateway.key, + base_url=str(gateway.client.base_url), + max_retries=0, + ) as client: + response: Final = client.messages.create( + model=model, + max_tokens=1024, + messages=_MESSAGES, + system=_SYSTEM, + tools=_TOOLS, + stream=False, + extra_headers={"anthropic-beta": f"{_SUPPORTED_BETA},made-up-future-beta-2099-01-01"}, + ) + parsed_message: Final = _MESSAGE.validate_python(response) + expected_message: Final = _MESSAGE.validate_python( + {**_JSON_OBJECT.validate_json(reply_body), "model": model} + ) + assert parsed_message.model_dump(mode="json") == expected_message.model_dump(mode="json"), response + + requests: Final = wire.drain() + assert len(requests) == 1, requests + provider_request: Final = requests[0] + assert provider_request.method == "POST" and provider_request.target == "/anthropic/v1/messages" + assert provider_request.headers["x-api-key"] == _API_KEY, provider_request.headers + assert provider_request.headers.get("anthropic-beta") == _SUPPORTED_BETA, provider_request.headers + assert _JSON_OBJECT.validate_json(provider_request.body) == expected_provider_body, provider_request.body diff --git a/tests/integration/providers/test_vertex_partner_messages_wire.py b/tests/integration/providers/test_vertex_partner_messages_wire.py new file mode 100644 index 00000000000..63629e11939 --- /dev/null +++ b/tests/integration/providers/test_vertex_partner_messages_wire.py @@ -0,0 +1,193 @@ +import json +from collections.abc import Callable +from typing import Final +from urllib.parse import parse_qs + +import anthropic +import pytest +from anthropic.types import Message +from integration._support import claude_code as cc +from integration._support.client import Gateway, Scenario +from integration._support.vertex import service_account_json +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "claude-sonnet-4-6" +_PROJECT: Final = "scripted-project" +_LOCATION: Final = "us-east5" +_MODEL_TARGET: Final = ( + f"/v1/projects/{_PROJECT}/locations/{_LOCATION}/publishers/anthropic/models/{_BACKEND}:rawPredict" +) +_STREAM_TARGET: Final = ( + f"/v1/projects/{_PROJECT}/locations/{_LOCATION}/publishers/anthropic/models/{_BACKEND}:streamRawPredict?alt=sse" +) +_VERTEX_TOKEN: Final = "scripted-vertex-access-token" +_SUPPORTED_BETA: Final = "context-management-2025-06-27" +_BETA_HEADER: Final = f"{_SUPPORTED_BETA},made-up-future-beta-2099-01-01" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MESSAGE: Final = TypeAdapter(Message) +_SYSTEM: Final[list[JsonValue]] = [{"type": "text", "text": "Answer briefly and use the supplied tool when relevant."}] +_MESSAGES: Final[list[JsonValue]] = [{"role": "user", "content": "Look up a synthetic record."}] +_TOOLS: Final[list[JsonValue]] = [ + { + "name": "lookup", + "description": "Look up a synthetic record.", + "input_schema": { + "type": "object", + "properties": {"record_id": {"type": "string"}}, + "required": ["record_id"], + }, + } +] +_THINKING: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 1024} +_CONTENT: Final[tuple[dict[str, JsonValue], ...]] = ({"type": "text", "text": "The record is ready."},) +_USAGE: Final[dict[str, JsonValue]] = {"input_tokens": 8, "output_tokens": 4} + + +def _without_none(value: JsonValue) -> JsonValue: + if isinstance(value, dict): + return {key: _without_none(item) for key, item in value.items() if item is not None} + if isinstance(value, list): + return [_without_none(item) for item in value] + return value + + +def _deployment(scenario: Scenario, api_base: str) -> str: + return scenario.model( + model=f"vertex_ai/{_BACKEND}", + api_base=api_base, + api_key=None, + vertex_project=_PROJECT, + vertex_location=_LOCATION, + vertex_credentials=service_account_json(_PROJECT, api_base), + num_retries=0, + ) + + +def _peer( + expected_body: dict[str, JsonValue], + target: str, + reply_body: bytes, + stream_reply: tuple[bytes, ...], +) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.target == "/_oauth/token": + form: Final = parse_qs(request.body.decode()) + assert request.method == "POST", request.method + assert request.headers["content-type"] == "application/x-www-form-urlencoded", request.headers + assert set(form) == {"assertion", "grant_type"}, form + assert form["grant_type"] == ["urn:ietf:params:oauth:grant-type:jwt-bearer"], form + assert form["assertion"], form + return Reply( + body=json.dumps({"access_token": _VERTEX_TOKEN, "token_type": "Bearer", "expires_in": 3600}).encode() + ) + + assert request.method == "POST" and request.target == target, request.target + assert request.headers["authorization"] == f"Bearer {_VERTEX_TOKEN}", request.headers + assert request.headers["anthropic-beta"] == _SUPPORTED_BETA, request.headers + assert _JSON_OBJECT.validate_json(request.body) == expected_body, request.body + if target == _STREAM_TARGET: + return Reply(content_type="text/event-stream", chunks=stream_reply) + return Reply(body=reply_body) + + return respond + + +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +def test_vertex_partner_messages_uses_raw_predict_contract_and_typed_response(gateway: Gateway, stream: bool) -> None: + identity: Final = f"msg_vertex_partner_{stream}" + reply_body: Final = cc.message_reply(identity, _BACKEND, _CONTENT, _USAGE) + stream_reply: Final = cc.message_stream(identity, _BACKEND, _CONTENT, _USAGE) + + with gateway.scenario() as scenario: + with wire_server( + _peer( + { + "max_tokens": 1024, + "messages": _MESSAGES, + "system": _SYSTEM, + "tools": _TOOLS, + "thinking": _THINKING, + "stream": stream, + "anthropic_version": "vertex-2023-10-16", + }, + _STREAM_TARGET if stream else _MODEL_TARGET, + reply_body, + stream_reply, + ) + ) as wire: + model: Final = _deployment(scenario, wire.url) + with anthropic.Anthropic( + api_key=gateway.key, + base_url=str(gateway.client.base_url), + max_retries=0, + ) as client: + if stream: + stream_response: Final = client.messages.create( + model=model, + max_tokens=1024, + messages=_MESSAGES, + system=_SYSTEM, + tools=_TOOLS, + thinking=_THINKING, + stream=True, + extra_headers={"anthropic-beta": _BETA_HEADER}, + ) + events: Final = tuple(stream_response) + expected_events: Final = tuple( + _JSON_OBJECT.validate_python( + _without_none( + { + **data, + **( + { + "message": { + **_JSON_OBJECT.validate_python(data["message"]), + "model": model, + } + } + if event_name == "message_start" + else {} + ), + } + ) + ) + for event_name, data in cc.sse_events(b"".join(stream_reply).decode()) + ) + actual_events: Final = tuple( + _JSON_OBJECT.validate_python(event.model_dump(mode="json", exclude_none=True)) + for event in events + ) + assert actual_events == expected_events, actual_events + started_message: Final = next(event.message for event in events if event.type == "message_start") + assert started_message.id == identity and started_message.model == model, actual_events + else: + response: Final = client.messages.create( + model=model, + max_tokens=1024, + messages=_MESSAGES, + system=_SYSTEM, + tools=_TOOLS, + thinking=_THINKING, + stream=False, + extra_headers={"anthropic-beta": _BETA_HEADER}, + ) + parsed_message: Final = _MESSAGE.validate_python(response) + expected_message: Final = _MESSAGE.validate_python( + {**_JSON_OBJECT.validate_json(reply_body), "model": model} + ) + assert parsed_message.model_dump(mode="json") == expected_message.model_dump(mode="json"), ( + response, + ) + + requests: Final = wire.drain() + assert tuple(request.target for request in requests) == ( + "/_oauth/token", + _STREAM_TARGET if stream else _MODEL_TARGET, + ), requests + token_request: Final = requests[0] + assert token_request.method == "POST", token_request + assert token_request.headers["content-type"] == "application/x-www-form-urlencoded", token_request.headers + model_request: Final = requests[1] + assert model_request.method == "POST", model_request + assert model_request.headers["authorization"] == f"Bearer {_VERTEX_TOKEN}", model_request.headers From 8822e086ec5650e0ae485a60db272a35c35f4196 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:58:22 +0000 Subject: [PATCH 07/51] test(integration): cover budgets_spend endpoint contracts (#44736) * test(integration): cover budgets_spend endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover budgets_spend endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover budgets_spend endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover budgets_spend endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): harden budgets_spend contracts per review Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): make spend report window midnight safe and drop redundant comments Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): pin organization spend report scope Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): close budgets and spend review findings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): drop streamed cost header absence assertion Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(spend): cover streamed spend on linked budget tiers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(spend): wait for async spend logs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management/test_budget_updates.py | 184 ++++++++ .../pricing/test_configured_prices.py | 99 ++++ .../pricing/test_cost_margin_discount.py | 320 +++++++++++++ .../spend/test_end_user_budget_enforcement.py | 438 ++++++++++++++++++ .../spend/test_global_spend_reset.py | 159 +++++++ .../spend/test_key_budget_lockout.py | 186 +++++++- .../spend/test_scoped_spend_reports.py | 228 +++++++++ .../spend/test_spend_log_read_scope.py | 92 +++- 8 files changed, 1703 insertions(+), 3 deletions(-) create mode 100644 tests/integration/pricing/test_cost_margin_discount.py create mode 100644 tests/integration/spend/test_end_user_budget_enforcement.py create mode 100644 tests/integration/spend/test_global_spend_reset.py create mode 100644 tests/integration/spend/test_scoped_spend_reports.py diff --git a/tests/integration/management/test_budget_updates.py b/tests/integration/management/test_budget_updates.py index 25d525bcaaa..7a79e35d2af 100644 --- a/tests/integration/management/test_budget_updates.py +++ b/tests/integration/management/test_budget_updates.py @@ -1,10 +1,13 @@ from datetime import datetime, timedelta, timezone +from pathlib import Path from typing import Final import pytest +from pydantic import JsonValue, TypeAdapter from tests.integration._support.client import Gateway, string_value from tests.integration._support.database import read_rows +from tests.integration._support.process import owned_proxy def _persisted_reset_at(budget_id: str) -> datetime: @@ -27,3 +30,184 @@ def test_shortening_budget_duration_moves_reset_at_onto_the_new_schedule(gateway updated: Final = _persisted_reset_at(budget_id) assert updated < ten_day_reset_at, f"{updated} not before {ten_day_reset_at}" assert before < updated <= before + timedelta(days=1, minutes=5), f"{updated} not within 1d of {before}" + + +BUDGET_COLUMNS: Final = ( + "budget_id, max_budget, soft_budget, max_parallel_requests, tpm_limit, rpm_limit, tpd_limit, model_max_budget, " + "budget_duration, allowed_models, temp_budget_increase, temp_budget_expiry, created_by, updated_by" +) +VOLATILE_INFO_FIELDS: Final = frozenset({"created_at", "updated_at", "budget_reset_at"}) +UNLOADED_RELATIONS: Final[dict[str, JsonValue]] = dict.fromkeys( + ( + "organization", + "projects", + "keys", + "end_users", + "tags", + "model_access_groups", + "team_membership", + "organization_membership", + ) +) + + +def _budget_row(budget_id: str) -> dict[str, JsonValue]: + rows: Final = read_rows( + f'SELECT {BUDGET_COLUMNS}, budget_reset_at::text AS reset_at FROM "LiteLLM_BudgetTable" WHERE budget_id = %s', + (budget_id,), + ) + assert len(rows) == 1, rows + return rows[0] + + +def _budget_info(gateway: Gateway, budget_id: str) -> dict[str, JsonValue]: + response: Final = gateway.request("POST", "/budget/info", {"budgets": [budget_id]}) + assert response.status_code == 200, response.text + entries: Final = TypeAdapter(list[dict[str, JsonValue]]).validate_json(response.content) + assert len(entries) == 1, entries + return entries[0] + + +def _reset_at(value: JsonValue) -> datetime | None: + if value is None: + return None + parsed: Final = datetime.fromisoformat(string_value(value).replace("Z", "+00:00")) + return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc) + + +def _assert_budget( + gateway: Gateway, + budget_id: str, + expected: dict[str, JsonValue], + reset_after: datetime | None, + reset_within: timedelta, +) -> None: + row: Final = _budget_row(budget_id) + reset_at: Final = _reset_at(row.pop("reset_at")) + assert row == expected, row + info: Final = _budget_info(gateway, budget_id) + assert {name: value for name, value in info.items() if name not in VOLATILE_INFO_FIELDS} == { + **expected, + **UNLOADED_RELATIONS, + }, info + assert VOLATILE_INFO_FIELDS <= info.keys(), info + assert _reset_at(info["budget_reset_at"]) == reset_at, info + if reset_after is None: + assert reset_at is None, row + else: + assert reset_at is not None and reset_after < reset_at <= reset_after + reset_within, (reset_at, reset_after) + + +def _update(gateway: Gateway, body: dict[str, JsonValue]) -> None: + response: Final = gateway.request("POST", "/budget/update", body) + assert response.status_code == 200, response.text + + +def test_partial_update_keeps_sibling_budget_fields(gateway: Gateway) -> None: + model_budget: Final[dict[str, JsonValue]] = {"openai/gpt-4o-mini": {"max_budget": 1, "budget_duration": "1d"}} + with gateway.scenario() as scenario: + created_at: Final = datetime.now(timezone.utc) + budget_id: Final = scenario.budget( + max_budget=10.0, + soft_budget=5.0, + tpm_limit=1000, + rpm_limit=60, + tpd_limit=100000, + model_max_budget=model_budget, + budget_duration="30d", + ) + creator: Final = string_value(_budget_row(budget_id)["created_by"]) + expected: Final[dict[str, JsonValue]] = { + "budget_id": budget_id, + "max_budget": 10.0, + "soft_budget": 5.0, + "max_parallel_requests": None, + "tpm_limit": 1000, + "rpm_limit": 60, + "tpd_limit": 100000, + "model_max_budget": {"openai/gpt-4o-mini": {"max_budget": 1.0, "budget_duration": "1d"}}, + "budget_duration": "30d", + "allowed_models": [], + "temp_budget_increase": None, + "temp_budget_expiry": None, + "created_by": creator, + "updated_by": creator, + } + _assert_budget(gateway, budget_id, expected, created_at, timedelta(days=30, minutes=5)) + + before_partial: Final = datetime.now(timezone.utc) + _update(gateway, {"budget_id": budget_id, "max_budget": 20.0}) + expected["max_budget"] = 20.0 + _assert_budget(gateway, budget_id, expected, created_at, timedelta(days=30, minutes=5)) + + _update( + gateway, + { + "budget_id": budget_id, + "model_max_budget": {"openai/gpt-4o-mini": {"max_budget": 2, "budget_duration": "1d"}}, + }, + ) + expected["model_max_budget"] = {"openai/gpt-4o-mini": {"max_budget": 2.0, "budget_duration": "1d"}} + _assert_budget(gateway, budget_id, expected, created_at, timedelta(days=30, minutes=5)) + + _update(gateway, {"budget_id": budget_id, "soft_budget": None}) + expected["soft_budget"] = None + _assert_budget(gateway, budget_id, expected, created_at, timedelta(days=30, minutes=5)) + + before_hourly: Final = datetime.now(timezone.utc) + _update(gateway, {"budget_id": budget_id, "budget_duration": "1h"}) + expected["budget_duration"] = "1h" + _assert_budget(gateway, budget_id, expected, before_hourly, timedelta(hours=1, minutes=5)) + assert before_partial <= before_hourly + + _update(gateway, {"budget_id": budget_id, "budget_duration": None}) + expected["budget_duration"] = None + _assert_budget(gateway, budget_id, expected, None, timedelta()) + + pinned: Final = datetime(2030, 1, 1, tzinfo=timezone.utc) + _update(gateway, {"budget_id": budget_id, "budget_duration": None, "budget_reset_at": pinned.isoformat()}) + _assert_budget(gateway, budget_id, expected, pinned - timedelta(microseconds=1), timedelta(microseconds=1)) + + rejected: Final = gateway.request( + "POST", + "/budget/update", + {"budget_id": budget_id, "model_max_budget": {"openai/gpt-4o-mini": {"max_budget": "lots"}}}, + ) + assert rejected.status_code == 422, rejected.text + assert rejected.json() == { + "detail": [ + { + "type": "float_parsing", + "loc": ["body", "model_max_budget", "openai/gpt-4o-mini", "max_budget"], + "msg": "Input should be a valid number, unable to parse string as a number", + } + ] + }, rejected.text + _assert_budget(gateway, budget_id, expected, pinned - timedelta(microseconds=1), timedelta(microseconds=1)) + + +def test_model_max_budget_update_is_refused_by_the_handler_without_a_license(gateway: Gateway, tmp_path: Path) -> None: + model_budget: Final[dict[str, JsonValue]] = {"openai/gpt-4o-mini": {"max_budget": 1, "budget_duration": "1d"}} + with gateway.scenario() as scenario: + budget_id: Final = scenario.budget(max_budget=10.0, model_max_budget=model_budget) + before: Final = _budget_row(budget_id) + with owned_proxy(gateway, tmp_path, {}, remove_environment=("LITELLM_LICENSE",)) as unlicensed: + rejected: Final = unlicensed.request( + "POST", + "/budget/update", + {"budget_id": budget_id, "model_max_budget": {"openai/gpt-4o-mini": {"max_budget": 2}}}, + ) + assert rejected.status_code == 400, rejected.text + assert rejected.json() == { + "detail": { + "error": "Invalid model_max_budget: You must have an enterprise license to set model_max_budget. " + "You must be a LiteLLM Enterprise user to use this feature. If you have a license please set " + "`LITELLM_LICENSE` in your env. Get a 7 day trial key here: https://www.litellm.ai/enterprise#trial. " + "\nPricing: https://www.litellm.ai/#pricing. Example of valid model_max_budget: " + "https://docs.litellm.ai/docs/proxy/users" + } + }, rejected.text + assert _budget_row(budget_id) == before + served: Final = unlicensed.request("POST", "/budget/update", {"budget_id": budget_id, "max_budget": 20.0}) + assert served.status_code == 200, served.text + assert _budget_row(budget_id) == {**before, "max_budget": 20.0} diff --git a/tests/integration/pricing/test_configured_prices.py b/tests/integration/pricing/test_configured_prices.py index 93290a404fc..b7eaadc05f9 100644 --- a/tests/integration/pricing/test_configured_prices.py +++ b/tests/integration/pricing/test_configured_prices.py @@ -7,6 +7,7 @@ from hashlib import sha256 from pathlib import Path from typing import Final +import httpx import pytest import yaml from pydantic import JsonValue @@ -358,3 +359,101 @@ def test_loaded_router_preserves_cached_defaults_during_real_requests(gateway: G assert info["output_cost_per_token"] == (0.0 if alias == aliases[2] else 0.0000006) finally: router.reset() + + +def _bounded_chat(gateway: Gateway, model: str, key: str, content: str) -> httpx.Response: + return gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "max_tokens": 20, "messages": [{"role": "user", "content": content}]}, + key=key, + ) + + +def _observed(upstream: httpx.Client) -> list[JsonValue]: + response: Final = upstream.get("/__observations") + response.raise_for_status() + requests: Final = object_value(response.json())["requests"] + assert isinstance(requests, list), response.text + return requests + + +def _upstream_chat(upstream_model: str, content: str) -> dict[str, JsonValue]: + return { + "path": "/v1/chat/completions", + "authorization": "Bearer integration-provider-key", + "body": {"model": upstream_model, "max_tokens": 20, "messages": [{"role": "user", "content": content}]}, + "method": "POST", + "api_key": "", + } + + +def _block_unpriced(gateway: Gateway, enabled: bool) -> httpx.Response: + return gateway.request("PATCH", "/config/block_requests_for_models_without_pricing", {"enabled": enabled}) + + +def test_block_requests_for_models_without_pricing_refuses_unpriced_deployments(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + original: Final = gateway.get("/config/block_requests_for_models_without_pricing")["enabled"] + scenario.cleanups.callback(_block_unpriced, gateway, bool(original)) + enabled: Final = _block_unpriced(gateway, True) + assert enabled.status_code == 200, enabled.text + assert enabled.json() == {"enabled": True}, enabled.text + assert gateway.get("/config/block_requests_for_models_without_pricing") == {"enabled": True} + settings_rows: Final = read_rows( + 'SELECT param_value FROM "LiteLLM_Config" WHERE param_name = %s', ("litellm_settings",) + ) + settings: Final = object_value(settings_rows[0]["param_value"]) + assert settings["block_requests_for_models_without_pricing"] is True, settings + + unpriced_upstream_model: Final = f"integration-unpriced-{uuid.uuid4().hex}" + unpriced: Final = scenario.model(model=f"openai/{unpriced_upstream_model}") + priced: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[unpriced, priced]) + entries: Final = gateway.get("/model/info")["data"] + assert isinstance(entries, list) + unpriced_info: Final = next( + object_value(entry) for entry in entries if object_value(entry)["model_name"] == unpriced + ) + unpriced_model_info: Final = object_value(unpriced_info["model_info"]) + assert unpriced_model_info["input_cost_per_token"] is None, unpriced_model_info + assert unpriced_model_info["output_cost_per_token"] is None, unpriced_model_info + + _observed(upstream) + blocked: Final = _bounded_chat(gateway, unpriced, key, f"unpriced {uuid.uuid4().hex}") + assert blocked.status_code == 403, blocked.text + assert blocked.json() == { + "error": { + "message": f"Model '{unpriced}' has no pricing in the cost map, so litellm cannot price the request. " + "Requests for unpriced models are blocked because 'block_requests_for_models_without_pricing' is " + "enabled. Add pricing (input_cost_per_token/output_cost_per_token) to allow the request.", + "type": "model_cost_map_missing", + "param": "model", + "code": "403", + } + }, blocked.text + assert _observed(upstream) == [], blocked.text + + priced_content: Final = f"priced {uuid.uuid4().hex}" + served: Final = _bounded_chat(gateway, priced, key, priced_content) + assert served.status_code == 200, served.text + assert float(served.headers["x-litellm-response-cost"]) == pytest.approx(0.06), served.text + assert _observed(upstream) == [_upstream_chat("gpt-4o-mini", priced_content)], served.text + + disabled: Final = _block_unpriced(gateway, False) + assert disabled.status_code == 200, disabled.text + assert disabled.json() == {"enabled": False}, disabled.text + assert gateway.get("/config/block_requests_for_models_without_pricing") == {"enabled": False} + settings_rows: Final = read_rows( + 'SELECT param_value FROM "LiteLLM_Config" WHERE param_name = %s', ("litellm_settings",) + ) + disabled_settings: Final = object_value(settings_rows[0]["param_value"]) + assert disabled_settings["block_requests_for_models_without_pricing"] is False, disabled_settings + _observed(upstream) + unpriced_content: Final = f"unpriced {uuid.uuid4().hex}" + now_served: Final = _bounded_chat(gateway, unpriced, key, unpriced_content) + assert now_served.status_code == 200, now_served.text + assert _observed(upstream) == [_upstream_chat(unpriced_upstream_model, unpriced_content)], now_served.text diff --git a/tests/integration/pricing/test_cost_margin_discount.py b/tests/integration/pricing/test_cost_margin_discount.py new file mode 100644 index 00000000000..bfd2002f871 --- /dev/null +++ b/tests/integration/pricing/test_cost_margin_discount.py @@ -0,0 +1,320 @@ +import json +import uuid +from collections.abc import Mapping +from typing import Final + +import httpx +import pytest +from integration._support.client import Gateway, eventually, object_value, string_value +from integration._support.database import read_rows +from pydantic import BaseModel, JsonValue + + +class CostEstimate(BaseModel): + model: str + cost_per_request: float + input_cost_per_request: float + output_cost_per_request: float + margin_cost_per_request: float + + +def _settings_row() -> dict[str, JsonValue]: + rows: Final = read_rows('SELECT param_value FROM "LiteLLM_Config" WHERE param_name = %s', ("litellm_settings",)) + assert len(rows) == 1, rows + value: Final = rows[0]["param_value"] + return object_value(json.loads(value) if isinstance(value, str) else value) + + +def _patch(gateway: Gateway, path: str, body: Mapping[str, JsonValue], message: str) -> dict[str, JsonValue]: + response: Final = gateway.request("PATCH", path, body) + assert response.status_code == 200, response.text + parsed: Final = object_value(response.json()) + assert parsed == {"message": message, "status": "success", "values": dict(body)}, response.text + return parsed + + +class ChatChunk(BaseModel): + id: str + object: str + + +def _observed(upstream: httpx.Client) -> list[JsonValue]: + response: Final = upstream.get("/__observations") + response.raise_for_status() + requests: Final = object_value(response.json())["requests"] + assert isinstance(requests, list), response.text + return requests + + +def _upstream_chat(content: str, streamed: bool) -> dict[str, JsonValue]: + return { + "path": "/v1/chat/completions", + "authorization": "Bearer integration-provider-key", + "body": { + "model": "gpt-4o-mini", + "max_tokens": 20, + "messages": [{"role": "user", "content": content}], + **({"stream": True, "stream_options": {"include_usage": True}} if streamed else {}), + }, + "method": "POST", + "api_key": "", + } + + +def _chat_cost(gateway: Gateway, upstream: httpx.Client, model: str, key: str, expected: float) -> str: + content: Final = f"margin {uuid.uuid4().hex}" + _observed(upstream) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "max_tokens": 20, "messages": [{"role": "user", "content": content}]}, + key=key, + ) + assert response.status_code == 200, response.text + request_id: Final = string_value(object_value(response.json())["id"]) + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected), response.text + assert _observed(upstream) == [_upstream_chat(content, streamed=False)], response.text + return request_id + + +def _streamed_chat(gateway: Gateway, upstream: httpx.Client, model: str, key: str) -> str: + content: Final = f"margin stream {uuid.uuid4().hex}" + _observed(upstream) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "max_tokens": 20, "stream": True, "messages": [{"role": "user", "content": content}]}, + key=key, + ) + assert response.status_code == 200, response.text + events: Final = [line.removeprefix("data: ") for line in response.text.splitlines() if line.startswith("data: ")] + assert events[-1] == "[DONE]", response.text + chunks: Final = [ChatChunk.model_validate_json(event) for event in events[:-1]] + assert {chunk.object for chunk in chunks} == {"chat.completion.chunk"}, response.text + assert len({chunk.id for chunk in chunks}) == 1, response.text + assert _observed(upstream) == [_upstream_chat(content, streamed=True)], response.text + return chunks[0].id + + +def _assert_logged_spends(expectations: tuple[tuple[str, float], ...]) -> None: + placeholders: Final = ", ".join("%s" for _ in expectations) + request_ids: Final = tuple(request_id for request_id, _ in expectations) + rows: Final = eventually( + lambda: read_rows( + f'SELECT request_id, spend FROM "LiteLLM_SpendLogs" WHERE request_id IN ({placeholders})', request_ids + ), + lambda values: len(values) == len(expectations), + seconds=40, + ) + actual: Final = {string_value(row["request_id"]): float(str(row["spend"])) for row in rows} + assert actual == pytest.approx(dict(expectations)), rows + + +def _estimated(gateway: Gateway, model: str) -> CostEstimate: + response: Final = gateway.request( + "POST", "/cost/estimate", {"model": model, "input_tokens": 20, "output_tokens": 20} + ) + assert response.status_code == 200, response.text + return CostEstimate.model_validate(response.json()) + + +def test_discount_and_margin_reprice_a_request_and_survive_rejection(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + original_discount: Final = gateway.get("/config/cost_discount_config")["values"] + original_margin: Final = gateway.get("/config/cost_margin_config")["values"] + + def restore() -> None: + _patch( + gateway, + "/config/cost_discount_config", + original_discount, + "Cost discount configuration updated successfully", + ) + _patch( + gateway, + "/config/cost_margin_config", + original_margin, + "Cost margin configuration updated successfully", + ) + + scenario.cleanups.callback(restore) + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model]) + + _patch(gateway, "/config/cost_discount_config", {}, "Cost discount configuration updated successfully") + _patch( + gateway, + "/config/cost_margin_config", + {}, + "Cost margin configuration updated successfully", + ) + base_request_id: Final = _chat_cost(gateway, upstream, model, key, 0.06) + + _patch( + gateway, + "/config/cost_discount_config", + {"openai": 0.05}, + "Cost discount configuration updated successfully", + ) + discounted_request_id: Final = _chat_cost(gateway, upstream, model, key, 0.057) + + _patch( + gateway, + "/config/cost_margin_config", + {"openai": 0.05}, + "Cost margin configuration updated successfully", + ) + bare_margin_request_id: Final = _chat_cost(gateway, upstream, model, key, 0.06 * 0.95 * 1.05) + + _patch( + gateway, + "/config/cost_margin_config", + {"openai": {"fixed_amount": 0.001}}, + "Cost margin configuration updated successfully", + ) + fixed_margin_request_id: Final = _chat_cost(gateway, upstream, model, key, 0.057 + 0.001) + + _patch( + gateway, + "/config/cost_margin_config", + {"openai": {"percentage": 0.1, "fixed_amount": 0.001}, "global": 0.05}, + "Cost margin configuration updated successfully", + ) + provider_margin_request_id: Final = _chat_cost(gateway, upstream, model, key, 0.057 * 1.1 + 0.001) + streamed_margin_request_id: Final = _streamed_chat(gateway, upstream, model, key) + + assert gateway.get("/config/cost_discount_config")["values"] == {"openai": 0.05} + assert gateway.get("/config/cost_margin_config")["values"] == { + "openai": {"percentage": 0.1, "fixed_amount": 0.001}, + "global": 0.05, + } + persisted: Final = _settings_row() + assert persisted["cost_discount_config"] == {"openai": 0.05}, persisted + assert persisted["cost_margin_config"] == { + "openai": {"percentage": 0.1, "fixed_amount": 0.001}, + "global": 0.05, + }, persisted + + expected: Final = 0.057 * 1.1 + 0.001 + estimate: Final = _estimated(gateway, model) + assert estimate.model == model + assert estimate.cost_per_request == pytest.approx(expected), estimate + assert estimate.input_cost_per_request == pytest.approx(0.02), estimate + assert estimate.output_cost_per_request == pytest.approx(0.04), estimate + discounted_subtotal: Final = estimate.cost_per_request - estimate.margin_cost_per_request + assert ( + estimate.input_cost_per_request + estimate.output_cost_per_request - discounted_subtotal + == pytest.approx(0.06 * 0.05) + ), estimate + assert estimate.margin_cost_per_request == pytest.approx(0.06 * 0.95 * 0.1 + 0.001), estimate + + _patch(gateway, "/config/cost_discount_config", {}, "Cost discount configuration updated successfully") + no_discount_request_id: Final = _chat_cost(gateway, upstream, model, key, 0.06 * 1.1 + 0.001) + + _patch( + gateway, + "/config/cost_margin_config", + {"global": 1}, + "Cost margin configuration updated successfully", + ) + global_margin_request_id: Final = _chat_cost(gateway, upstream, model, key, 0.12) + streamed_global_request_id: Final = _streamed_chat(gateway, upstream, model, key) + + _assert_logged_spends( + ( + (base_request_id, 0.06), + (discounted_request_id, 0.057), + (bare_margin_request_id, 0.06 * 0.95 * 1.05), + (fixed_margin_request_id, 0.057 + 0.001), + (provider_margin_request_id, 0.057 * 1.1 + 0.001), + (streamed_margin_request_id, 0.057 * 1.1 + 0.001), + (no_discount_request_id, 0.06 * 1.1 + 0.001), + (global_margin_request_id, 0.12), + (streamed_global_request_id, 0.12), + ) + ) + + rejections: Final = ( + ( + "/config/cost_margin_config", + {"not_a_provider": 0.1}, + { + "detail": { + "error": "Invalid provider(s): not_a_provider. Must be valid LiteLLM providers or 'global'. " + "See https://docs.litellm.ai/docs/providers for the full list." + } + }, + ), + ( + "/config/cost_discount_config", + {"openai": 1.5}, + {"detail": "Discount for openai must be between 0 and 1 (0% to 100%)"}, + ), + ( + "/config/cost_discount_config", + {"global": 0.1}, + { + "detail": { + "error": "Invalid provider(s): global. Must be valid LiteLLM providers. " + "See https://docs.litellm.ai/docs/providers for the full list." + } + }, + ), + ) + for path, body, error in rejections: + rejected = gateway.request("PATCH", path, body) + assert rejected.status_code == 400, rejected.text + assert rejected.json() == error, rejected.text + assert gateway.get("/config/cost_discount_config")["values"] == {} + assert gateway.get("/config/cost_margin_config")["values"] == {"global": 1} + final_settings: Final = _settings_row() + assert final_settings["cost_discount_config"] == {}, final_settings + assert final_settings["cost_margin_config"] == {"global": 1}, final_settings + + +def test_streamed_response_cost_header_matches_billed_spend(gateway: Gateway) -> None: + pytest.skip( + "BUG: a streamed chat with a fixed cost margin returns x-litellm-response-cost equal to the fixed amount " + "(computed before usage is known) while LiteLLM_SpendLogs bills the full margined cost" + ) + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + original_margin: Final = gateway.get("/config/cost_margin_config")["values"] + scenario.cleanups.callback( + _patch, + gateway, + "/config/cost_margin_config", + original_margin, + "Cost margin configuration updated successfully", + ) + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model]) + _patch( + gateway, + "/config/cost_margin_config", + {"openai": {"percentage": 0.1, "fixed_amount": 0.001}}, + "Cost margin configuration updated successfully", + ) + content: Final = f"margin stream header {uuid.uuid4().hex}" + _observed(upstream) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "max_tokens": 20, "stream": True, "messages": [{"role": "user", "content": content}]}, + key=key, + ) + assert response.status_code == 200, response.text + events: Final = [ + line.removeprefix("data: ") for line in response.text.splitlines() if line.startswith("data: ") + ] + assert events[-1] == "[DONE]", response.text + request_id: Final = ChatChunk.model_validate_json(events[0]).id + assert _observed(upstream) == [_upstream_chat(content, streamed=True)], response.text + _assert_logged_spends(((request_id, 0.06 * 1.1 + 0.001),)) + advertised: Final = response.headers.get("x-litellm-response-cost") + assert advertised is None or float(advertised) == pytest.approx(0.06 * 1.1 + 0.001), dict(response.headers) diff --git a/tests/integration/spend/test_end_user_budget_enforcement.py b/tests/integration/spend/test_end_user_budget_enforcement.py new file mode 100644 index 00000000000..de0bfcd5194 --- /dev/null +++ b/tests/integration/spend/test_end_user_budget_enforcement.py @@ -0,0 +1,438 @@ +import json +import re +import uuid +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Final + +import httpx +import pytest +from integration._support.client import Gateway, Scenario, eventually, object_value, string_value +from integration._support.database import read_rows +from integration._support.upstream import delete_scenario, register_scenario +from integration.cost_calculation.cost_tracking_case import JsonResponse +from pydantic import BaseModel, JsonValue + +PROVIDER_AUTHORIZATION: Final = "Bearer integration-provider-key" +CANNED_CONTENT: Final = "Hello! This is a mock response from the fake OpenAI endpoint." +RESPONSES_REPLY: Final = JsonResponse( + content_type="application/json", + body={ + "id": "resp_$UNIQUE_ID", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "type": "message", + "id": "msg_$UNIQUE_ID", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 20, "output_tokens": 20, "total_tokens": 40}, + }, +) + + +class ChatCompletion(BaseModel): + id: str + object: str + + +class ChatChunkDelta(BaseModel): + content: str | None = None + + +class ChatChunkChoice(BaseModel): + delta: ChatChunkDelta + + +class ChatChunk(BaseModel): + object: str + choices: list[ChatChunkChoice] + + +class ResponsesResult(BaseModel): + id: str + object: str + status: str + + +class AnthropicMessage(BaseModel): + type: str + role: str + content: list[dict[str, JsonValue]] + + +@dataclass(frozen=True, slots=True) +class Spelling: + route: str + fields: Callable[[str], dict[str, JsonValue]] + headers: Callable[[str], dict[str, str]] + upstream_fields: Callable[[str], dict[str, JsonValue]] + stream: bool = False + + +def _no_fields(_: str) -> dict[str, JsonValue]: + return {} + + +def _no_headers(_: str) -> dict[str, str]: + return {} + + +SPELLINGS: Final[Mapping[str, Spelling]] = { + "user": Spelling("/v1/chat/completions", lambda user: {"user": user}, _no_headers, lambda user: {"user": user}), + "x-litellm-customer-id": Spelling( + "/v1/chat/completions", _no_fields, lambda user: {"x-litellm-customer-id": user}, _no_fields + ), + "x-litellm-end-user-id": Spelling( + "/v1/chat/completions", _no_fields, lambda user: {"x-litellm-end-user-id": user}, _no_fields + ), + "litellm_metadata": Spelling( + "/v1/chat/completions", lambda user: {"litellm_metadata": {"user": user}}, _no_headers, _no_fields + ), + "litellm_metadata_json": Spelling( + "/v1/chat/completions", + lambda user: {"litellm_metadata": json.dumps({"user": user})}, + _no_headers, + _no_fields, + ), + "metadata": Spelling("/v1/chat/completions", lambda user: {"metadata": {"user_id": user}}, _no_headers, _no_fields), + "metadata_json": Spelling( + "/v1/chat/completions", lambda user: {"metadata": json.dumps({"user_id": user})}, _no_headers, _no_fields + ), + "safety_identifier": Spelling( + "/v1/chat/completions", + lambda user: {"safety_identifier": user}, + _no_headers, + lambda user: {"safety_identifier": user}, + ), + "user_stream": Spelling( + "/v1/chat/completions", + lambda user: {"user": user, "stream": True}, + _no_headers, + lambda user: {"user": user, "stream": True, "stream_options": {"include_usage": True}}, + stream=True, + ), + "litellm_metadata_stream": Spelling( + "/v1/chat/completions", + lambda user: {"litellm_metadata": {"user": user}, "stream": True}, + _no_headers, + lambda _: {"stream": True, "stream_options": {"include_usage": True}}, + stream=True, + ), + "responses_user": Spelling("/v1/responses", lambda user: {"user": user}, _no_headers, lambda user: {"user": user}), + "responses_safety_identifier": Spelling( + "/v1/responses", + lambda user: {"safety_identifier": user}, + _no_headers, + lambda user: {"safety_identifier": user}, + ), + "messages_metadata_user_id": Spelling( + "/v1/messages", lambda user: {"metadata": {"user_id": user}}, _no_headers, lambda user: {"user": user} + ), +} + + +@dataclass(frozen=True, slots=True) +class Deployment: + model: str + upstream_prefix: str + + +def _deployment(scenario: Scenario, spelling: Spelling) -> Deployment: + if spelling.route == "/v1/chat/completions": + return Deployment(scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002), "") + scenario_id: Final = f"enduser{uuid.uuid4().hex[:12]}" + handle: Final = register_scenario(scenario_id, RESPONSES_REPLY) + scenario.cleanups.callback(delete_scenario, handle) + model: Final = scenario.model( + api_base=f"{handle.api_base()}/v1", input_cost_per_token=0.001, output_cost_per_token=0.002 + ) + return Deployment(model, f"/{scenario_id}") + + +def _client_body(spelling: Spelling, model: str, user_id: str, content: str) -> dict[str, JsonValue]: + match spelling.route: + case "/v1/chat/completions": + base: dict[str, JsonValue] = { + "model": model, + "max_tokens": 20, + "messages": [{"role": "user", "content": content}], + } + case "/v1/responses": + base = {"model": model, "max_output_tokens": 20, "input": content} + case _: + base = {"model": model, "max_tokens": 20, "messages": [{"role": "user", "content": content}]} + return {**base, **spelling.fields(user_id)} + + +def _upstream_request(spelling: Spelling, prefix: str, user_id: str, content: str) -> dict[str, JsonValue]: + match spelling.route: + case "/v1/chat/completions": + path = "/v1/chat/completions" + body: dict[str, JsonValue] = { + "model": "gpt-4o-mini", + "max_tokens": 20, + "messages": [{"role": "user", "content": content}], + } + case "/v1/responses": + path = f"{prefix}/v1/responses" + body = {"model": "gpt-4o-mini", "input": content, "max_output_tokens": 20} + case _: + path = f"{prefix}/v1/responses" + body = { + "model": "gpt-4o-mini", + "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": content}]}], + "include": ["reasoning.encrypted_content"], + "max_output_tokens": 20, + } + return { + "path": path, + "authorization": PROVIDER_AUTHORIZATION, + "body": {**body, **spelling.upstream_fields(user_id)}, + "method": "POST", + "api_key": "", + } + + +def _send( + gateway: Gateway, spelling: Spelling, deployment: Deployment, key: str, user_id: str, content: str +) -> httpx.Response: + return gateway.request( + "POST", + spelling.route, + _client_body(spelling, deployment.model, user_id, content), + key=key, + headers=spelling.headers(user_id), + ) + + +def _assert_served(spelling: Spelling, response: httpx.Response) -> None: + assert response.status_code == 200, response.text + if spelling.stream: + events: Final = [ + line.removeprefix("data: ") for line in response.text.splitlines() if line.startswith("data: ") + ] + assert events[-1] == "[DONE]", response.text + chunks: Final = [ChatChunk.model_validate_json(event) for event in events[:-1]] + assert {chunk.object for chunk in chunks} == {"chat.completion.chunk"}, response.text + assert "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices) == CANNED_CONTENT + return + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(0.06), dict(response.headers) + match spelling.route: + case "/v1/chat/completions": + assert ChatCompletion.model_validate_json(response.content).object == "chat.completion", response.text + case "/v1/responses": + parsed: Final = ResponsesResult.model_validate_json(response.content) + assert (parsed.object, parsed.status) == ("response", "completed"), response.text + case _: + message: Final = AnthropicMessage.model_validate_json(response.content) + assert (message.type, message.role, message.content) == ( + "message", + "assistant", + [{"type": "text", "text": "ok"}], + ), response.text + + +def _delete_customer(gateway: Gateway, user_id: str) -> None: + if not _end_user_spend(user_id): + return + response: Final = gateway.request("POST", "/customer/delete", {"user_ids": [user_id]}) + assert response.status_code == 200, response.text + + +def _end_user_spend(user_id: str) -> list[dict[str, JsonValue]]: + return read_rows('SELECT spend FROM "LiteLLM_EndUserTable" WHERE user_id = %s', (user_id,)) + + +def _observed(upstream: httpx.Client) -> list[JsonValue]: + response: Final = upstream.get("/__observations") + response.raise_for_status() + requests: Final = object_value(response.json())["requests"] + assert isinstance(requests, list), response.text + return requests + + +def _denial_error(denied: httpx.Response) -> dict[str, JsonValue]: + assert denied.status_code == 422, denied.text + return object_value(object_value(denied.json())["error"]) + + +def _prove_end_user_budget_lockout( + gateway: Gateway, + upstream: httpx.Client, + spelling: Spelling, + deployment: Deployment, + key: str, + end_user_id: str, + other_end_user_id: str, + expected_budget: Mapping[str, JsonValue], + info_path: str = "/customer/info", +) -> None: + served_content: Final = f"served {end_user_id}" + _observed(upstream) + served: Final = _send(gateway, spelling, deployment, key, end_user_id, served_content) + _assert_served(spelling, served) + assert _observed(upstream) == [ + _upstream_request(spelling, deployment.upstream_prefix, end_user_id, served_content) + ], served.text + eventually( + lambda: _end_user_spend(end_user_id), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) > 0.05, + seconds=30, + ) + persisted_spend: Final = float(str(_end_user_spend(end_user_id)[0]["spend"])) + assert persisted_spend == pytest.approx(0.06), persisted_spend + customer_info: Final = gateway.request("GET", info_path, params={"end_user_id": end_user_id}) + assert customer_info.status_code == 200, customer_info.text + customer: Final = object_value(customer_info.json()) + assert customer["user_id"] == end_user_id, customer_info.text + assert float(str(customer["spend"])) == pytest.approx(persisted_spend), customer_info.text + assert customer["budget_id"] == expected_budget["budget_id"], customer_info.text + budget: Final = customer["litellm_budget_table"] + expected_max_budget: Final = expected_budget["max_budget"] + if expected_max_budget is None: + assert budget is None, customer_info.text + else: + assert float(str(object_value(budget)["max_budget"])) == float(str(expected_max_budget)), customer_info.text + + _observed(upstream) + denied: Final = _send(gateway, spelling, deployment, key, end_user_id, f"denied {end_user_id}") + assert _observed(upstream) == [], denied.text + error: Final = _denial_error(denied) + assert error["type"] == "budget_exceeded", denied.text + matched: Final = re.fullmatch( + rf"ExceededBudget: End User={re.escape(end_user_id)} over budget\. " + rf"Spend=([-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?), Budget=0\.05", + string_value(error["message"]), + ) + assert matched is not None, denied.text + assert float(matched.group(1)) == pytest.approx(persisted_spend), denied.text + + other_content: Final = f"served {other_end_user_id}" + other: Final = _send(gateway, spelling, deployment, key, other_end_user_id, other_content) + _assert_served(spelling, other) + assert _observed(upstream) == [ + _upstream_request(spelling, deployment.upstream_prefix, other_end_user_id, other_content) + ], other.text + other_spend: Final = eventually( + lambda: _end_user_spend(other_end_user_id), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) > 0.05, + seconds=30, + ) + assert float(str(other_spend[0]["spend"])) == pytest.approx(0.06), other_spend + + +@pytest.mark.parametrize("spelling", tuple(SPELLINGS)) +def test_customer_max_budget_refuses_second_request(gateway: Gateway, spelling: str) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + variant: Final = SPELLINGS[spelling] + deployment: Final = _deployment(scenario, variant) + key: Final = scenario.key(models=[deployment.model]) + end_user_id: Final = f"end-user-{uuid.uuid4().hex}" + created: Final = gateway.post("/customer/new", {"user_id": end_user_id, "max_budget": 0.05}) + assert created["user_id"] == end_user_id + scenario.cleanups.callback(_delete_customer, gateway, end_user_id) + other_end_user_id: Final = f"end-user-{uuid.uuid4().hex}" + scenario.cleanups.callback(_delete_customer, gateway, other_end_user_id) + _prove_end_user_budget_lockout( + gateway, + upstream, + variant, + deployment, + key, + end_user_id, + other_end_user_id, + {"budget_id": string_value(created["budget_id"]), "max_budget": 0.05}, + ) + + +def test_end_user_alias_routes_create_and_read_a_budgeted_customer(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + variant: Final = SPELLINGS["user"] + deployment: Final = _deployment(scenario, variant) + key: Final = scenario.key(models=[deployment.model]) + end_user_id: Final = f"end-user-{uuid.uuid4().hex}" + created: Final = gateway.post("/end_user/new", {"user_id": end_user_id, "max_budget": 0.05}) + assert created["user_id"] == end_user_id, created + assert float(str(object_value(created["litellm_budget_table"])["max_budget"])) == 0.05, created + scenario.cleanups.callback(_delete_customer, gateway, end_user_id) + other_end_user_id: Final = f"end-user-{uuid.uuid4().hex}" + scenario.cleanups.callback(_delete_customer, gateway, other_end_user_id) + _prove_end_user_budget_lockout( + gateway, + upstream, + variant, + deployment, + key, + end_user_id, + other_end_user_id, + {"budget_id": string_value(created["budget_id"]), "max_budget": 0.05}, + info_path="/end_user/info", + ) + + +@pytest.mark.parametrize("spelling", ("litellm_metadata", "litellm_metadata_stream")) +def test_customer_tier_budget_refuses_second_request(gateway: Gateway, spelling: str) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + variant: Final = SPELLINGS[spelling] + deployment: Final = _deployment(scenario, variant) + tier: Final = scenario.budget(max_budget=0.05) + key: Final = scenario.key(models=[deployment.model]) + end_user_id: Final = f"end-user-{uuid.uuid4().hex}" + created: Final = gateway.post("/customer/new", {"user_id": end_user_id, "budget_id": tier}) + assert created["user_id"] == end_user_id + scenario.cleanups.callback(_delete_customer, gateway, end_user_id) + other_end_user_id: Final = f"end-user-{uuid.uuid4().hex}" + scenario.cleanups.callback(_delete_customer, gateway, other_end_user_id) + _prove_end_user_budget_lockout( + gateway, + upstream, + variant, + deployment, + key, + end_user_id, + other_end_user_id, + {"budget_id": tier, "max_budget": 0.05}, + ) + + +def test_key_end_user_budget_id_refuses_second_request(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + variant: Final = SPELLINGS["metadata"] + deployment: Final = _deployment(scenario, variant) + tier: Final = scenario.budget(max_budget=0.05) + key: Final = scenario.key(models=[deployment.model], metadata={"end_user_budget_id": tier}) + key_info: Final = object_value(gateway.get("/key/info", {"key": key})["info"]) + assert object_value(key_info["metadata"]) == {"end_user_budget_id": tier}, key_info + end_user_id: Final = f"end-user-{uuid.uuid4().hex}" + other_end_user_id: Final = f"end-user-{uuid.uuid4().hex}" + scenario.cleanups.callback(_delete_customer, gateway, end_user_id) + scenario.cleanups.callback(_delete_customer, gateway, other_end_user_id) + _prove_end_user_budget_lockout( + gateway, + upstream, + variant, + deployment, + key, + end_user_id, + other_end_user_id, + {"budget_id": None, "max_budget": None}, + ) diff --git a/tests/integration/spend/test_global_spend_reset.py b/tests/integration/spend/test_global_spend_reset.py new file mode 100644 index 00000000000..fad5c14175a --- /dev/null +++ b/tests/integration/spend/test_global_spend_reset.py @@ -0,0 +1,159 @@ +import uuid +from hashlib import sha256 +from typing import Final + +import httpx +import pytest +from integration._support.client import Gateway, eventually, object_value +from integration._support.database import read_rows +from pydantic import JsonValue + + +def _chat(gateway: Gateway, model: str, key: str, content: str | None = None) -> httpx.Response: + return gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "max_tokens": 20, + "messages": [{"role": "user", "content": content or f"reset {uuid.uuid4().hex}"}], + }, + key=key, + ) + + +def _observed(upstream: httpx.Client) -> list[JsonValue]: + response: Final = upstream.get("/__observations") + response.raise_for_status() + requests: Final = object_value(response.json())["requests"] + assert isinstance(requests, list), response.text + return requests + + +def _upstream_chat(content: str) -> dict[str, JsonValue]: + return { + "path": "/v1/chat/completions", + "authorization": "Bearer integration-provider-key", + "body": {"model": "gpt-4o-mini", "max_tokens": 20, "messages": [{"role": "user", "content": content}]}, + "method": "POST", + "api_key": "", + } + + +def _token_spend(key: str) -> list[dict[str, object]]: + return read_rows( + 'SELECT spend FROM "LiteLLM_VerificationToken" WHERE token = %s', (sha256(key.encode()).hexdigest(),) + ) # fmt: skip + + +def _team_spend(team_id: str) -> list[dict[str, object]]: + return read_rows('SELECT spend FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team_id,)) + + +def _spend_log_totals(keys: tuple[str, ...]) -> list[dict[str, object]]: + digests: Final = tuple(sha256(key.encode()).hexdigest() for key in keys) + return read_rows( + 'SELECT COUNT(*) AS rows, COALESCE(SUM(spend), 0) AS total_spend FROM "LiteLLM_SpendLogs" ' + "WHERE api_key = ANY(%s::text[])", + ("{" + ",".join(digests) + "}",), + ) + + +def _seed_spend(gateway: Gateway, model: str, team_id: str, key_a: str, key_b: str) -> None: + assert _chat(gateway, model, key_a).status_code == 200 + assert _chat(gateway, model, key_b).status_code == 200 + for key in (key_a, key_b): + eventually( + lambda key=key: _token_spend(key), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) > 0, + seconds=70, + ) + eventually( + lambda: _team_spend(team_id), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) > 0, + seconds=70, + ) + + +def test_global_spend_reset_zeroes_token_and_team_spend_but_keeps_logs(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + team_id: Final = scenario.team() + key_a: Final = scenario.key(team_id=team_id, models=[model], max_budget=0.06) + key_b: Final = scenario.key(team_id=team_id, models=[model]) + internal_user: Final = scenario.user(user_role="internal_user") + internal_key: Final = scenario.key(user_id=internal_user, models=[model]) + _seed_spend(gateway, model, team_id, key_a, key_b) + before: Final = eventually( + lambda: _spend_log_totals((key_a, key_b)), + lambda rows: ( + len(rows) == 1 + and int(str(rows[0]["rows"])) == 2 + and float(str(rows[0]["total_spend"])) == pytest.approx(0.12) + ), + seconds=70, + ) + assert float(str(before[0]["total_spend"])) == pytest.approx(0.12), before + + refused: Final = gateway.request("POST", "/global/spend/reset", key=internal_key) + assert refused.status_code == 401, refused.text + assert refused.json() == { + "error": { + "message": "Authentication Error, Tried to access route=/global/spend/reset, which is only for MASTER KEY", + "type": "auth_error", + "param": "None", + "code": "401", + } + }, refused.text + assert float(str(_token_spend(key_a)[0]["spend"])) == pytest.approx(0.06), _token_spend(key_a) + assert float(str(_token_spend(key_b)[0]["spend"])) == pytest.approx(0.06), _token_spend(key_b) + assert float(str(_team_spend(team_id)[0]["spend"])) == pytest.approx(0.12), _team_spend(team_id) + + _observed(upstream) + denied: Final = _chat(gateway, model, key_a) + assert denied.status_code == 422, denied.text + assert object_value(denied.json()["error"])["type"] == "budget_exceeded", denied.text + assert _observed(upstream) == [], denied.text + + reset: Final = gateway.request("POST", "/global/spend/reset") + assert reset.status_code == 200, reset.text + assert reset.json() == { + "message": "Spend for all API Keys and Teams reset successfully", + "status": "success", + }, reset.text + for key in (key_a, key_b): + rows: Final = _token_spend(key) + assert float(str(rows[0]["spend"])) == 0, rows + assert float(str(_team_spend(team_id)[0]["spend"])) == 0 + assert _spend_log_totals((key_a, key_b)) == before + + +def test_reset_exhausted_key_is_served_again(gateway: Gateway) -> None: + pytest.skip( + "BUG: /global/spend/reset zeroes DB spend but leaves Redis spend:key counters, " + "so an exhausted key stays refused with budget_exceeded" + ) + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model], max_budget=0.06) + assert _chat(gateway, model, key).status_code == 200 + eventually( + lambda: _token_spend(key), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) >= 0.06, + seconds=70, + ) + denied: Final = _chat(gateway, model, key) + assert denied.status_code == 422, denied.text + reset: Final = gateway.request("POST", "/global/spend/reset") + assert reset.status_code == 200, reset.text + content: Final = f"restored {uuid.uuid4().hex}" + _observed(upstream) + restored: Final = _chat(gateway, model, key, content) + assert restored.status_code == 200, restored.text + assert _observed(upstream) == [_upstream_chat(content)], restored.text diff --git a/tests/integration/spend/test_key_budget_lockout.py b/tests/integration/spend/test_key_budget_lockout.py index 02831cd59cd..7108978f6e5 100644 --- a/tests/integration/spend/test_key_budget_lockout.py +++ b/tests/integration/spend/test_key_budget_lockout.py @@ -6,6 +6,8 @@ import httpx import pytest from integration._support.client import Gateway, eventually, object_value from integration._support.database import read_rows +from openai.types.chat import ChatCompletionChunk +from pydantic import JsonValue def test_an_exhausted_key_is_refused_inference_but_can_still_read_its_own_info(gateway: Gateway) -> None: @@ -43,19 +45,46 @@ def test_an_exhausted_key_is_refused_inference_but_can_still_read_its_own_info(g assert own["max_budget"] == 0.06 -def _bounded_chat(gateway: Gateway, model: str, key: str) -> httpx.Response: +def _bounded_chat(gateway: Gateway, model: str, key: str, content: str | None = None) -> httpx.Response: return gateway.request( "POST", "/v1/chat/completions", { "model": model, "max_tokens": 20, - "messages": [{"role": "user", "content": f"key recovery {uuid.uuid4().hex}"}], + "messages": [{"role": "user", "content": content or f"key recovery {uuid.uuid4().hex}"}], }, key=key, ) +def _observed(upstream: httpx.Client) -> list[JsonValue]: + response: Final = upstream.get("/__observations") + response.raise_for_status() + requests: Final = object_value(response.json())["requests"] + assert isinstance(requests, list), response.text + return requests + + +def _upstream_chat(content: str) -> dict[str, JsonValue]: + return { + "path": "/v1/chat/completions", + "authorization": "Bearer integration-provider-key", + "body": {"model": "gpt-4o-mini", "max_tokens": 20, "messages": [{"role": "user", "content": content}]}, + "method": "POST", + "api_key": "", + } + + +def _served_chat(gateway: Gateway, upstream: httpx.Client, model: str, key: str) -> None: + content: Final = f"tier served {uuid.uuid4().hex}" + _observed(upstream) + served: Final = _bounded_chat(gateway, model, key, content) + assert served.status_code == 200, served.text + assert float(served.headers["x-litellm-response-cost"]) == 0.06, dict(served.headers) + assert _observed(upstream) == [_upstream_chat(content)], served.text + + def test_raising_a_spent_keys_budget_restores_serving(gateway: Gateway) -> None: with ( gateway.scenario() as scenario, @@ -82,3 +111,156 @@ def test_raising_a_spent_keys_budget_restores_serving(gateway: Gateway) -> None: served: Final = tuple(_bounded_chat(gateway, model, key) for _ in range(3)) assert [response.status_code for response in served] == [200, 200, 200], [response.text for response in served] assert len(upstream.get("/__observations").json()["requests"]) == 3 + + +def _denied_with_budget_exceeded(gateway: Gateway, model: str, key: str) -> None: + denied: Final = _bounded_chat(gateway, model, key) + assert denied.status_code == 422, denied.text + assert object_value(denied.json()["error"])["type"] == "budget_exceeded" + + +@pytest.mark.timeout(149) +def test_raising_a_spent_tier_budget_restores_serving(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + budget_id: Final = scenario.budget(budget_id=f"tier-{uuid.uuid4().hex}", max_budget=0.06, budget_duration="30d") + key: Final = scenario.key(models=[model], budget_id=budget_id) + info: Final = object_value(gateway.get("/key/info", {"key": key})["info"]) + assert info["budget_id"] == budget_id, info + tier_rows: Final = read_rows( + 'SELECT max_budget, budget_duration FROM "LiteLLM_BudgetTable" WHERE budget_id = %s', (budget_id,) + ) + assert len(tier_rows) == 1 + assert float(str(tier_rows[0]["max_budget"])) == 0.06 + assert tier_rows[0]["budget_duration"] == "30d" + _served_chat(gateway, upstream, model, key) + eventually( + lambda: read_rows( + 'SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', (sha256(key.encode()).hexdigest(),) + ), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) >= 0.06, + seconds=70, + ) + upstream.get("/__observations").raise_for_status() + _denied_with_budget_exceeded(gateway, model, key) + assert upstream.get("/__observations").json()["requests"] == [] + gateway.post("/budget/update", {"budget_id": budget_id, "max_budget": 1.0}) + updated_rows: Final = read_rows( + 'SELECT max_budget FROM "LiteLLM_BudgetTable" WHERE budget_id = %s', (budget_id,) + ) + assert float(str(updated_rows[0]["max_budget"])) == 1.0, updated_rows + restored_content: Final = f"tier restored {uuid.uuid4().hex}" + served: Final = eventually( + lambda: _bounded_chat(gateway, model, key, restored_content), + lambda response: response.status_code == 200, + seconds=125, + ) + assert served.status_code == 200, served.text + assert _observed(upstream) == [_upstream_chat(restored_content)], served.text + + +@pytest.mark.timeout(149) +def test_lowering_a_tier_budget_below_spend_blocks_the_key(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + budget_id: Final = scenario.budget( + budget_id=f"tier-{uuid.uuid4().hex}", max_budget=1000.0, budget_duration="30d" + ) + key: Final = scenario.key(models=[model], budget_id=budget_id) + _served_chat(gateway, upstream, model, key) + eventually( + lambda: read_rows( + 'SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', (sha256(key.encode()).hexdigest(),) + ), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) >= 0.06, + seconds=30, + ) + upstream.get("/__observations").raise_for_status() + probe: Final = gateway.request("POST", "/v1/chat/completions", {"model": model}, key=key) + assert probe.status_code == 400, probe.text + assert object_value(probe.json()["error"]) == { + "message": "/chat/completions: Missing required parameter: 'messages'.", + "type": "invalid_request_error", + "param": "messages", + "code": "400", + }, probe.text + assert upstream.get("/__observations").json()["requests"] == [] + gateway.post("/budget/update", {"budget_id": budget_id, "max_budget": 0.01}) + updated_rows: Final = read_rows( + 'SELECT max_budget FROM "LiteLLM_BudgetTable" WHERE budget_id = %s', (budget_id,) + ) + assert len(updated_rows) == 1 + assert float(str(updated_rows[0]["max_budget"])) == 0.01, updated_rows + upstream.get("/__observations").raise_for_status() + lowered_probe: Final = eventually( + lambda: gateway.request("POST", "/v1/chat/completions", {"model": model}, key=key), + lambda response: ( + response.status_code == 422 and object_value(response.json()["error"])["type"] == "budget_exceeded" + ), + seconds=125, + ) + assert "Max budget: 0.01" in lowered_probe.text, lowered_probe.text + denied: Final = _bounded_chat(gateway, model, key) + assert denied.status_code == 422, denied.text + assert object_value(denied.json()["error"])["type"] == "budget_exceeded", denied.text + assert "Max budget: 0.01" in denied.text, denied.text + assert upstream.get("/__observations").json()["requests"] == [] + + +@pytest.mark.timeout(149) +def test_key_linked_to_a_tier_by_update_is_refused_once_streamed_spend_exhausts_it(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + budget_id: Final = scenario.budget(budget_id=f"tier-{uuid.uuid4().hex}", max_budget=0.06, budget_duration="30d") + key: Final = scenario.key(models=[model]) + gateway.post("/key/update", {"key": key, "budget_id": budget_id}) + info: Final = object_value(gateway.get("/key/info", {"key": key})["info"]) + assert info["budget_id"] == budget_id, info + + content: Final = f"tier streamed {uuid.uuid4().hex}" + body: Final = { + "model": model, + "max_tokens": 20, + "messages": [{"role": "user", "content": content}], + "stream": True, + } + _observed(upstream) + served: Final = gateway.request("POST", "/v1/chat/completions", body, key=key) + assert served.status_code == 200, served.text + events: Final = tuple( + line.removeprefix("data: ") for line in served.text.splitlines() if line.startswith("data: ") + ) + assert events[-1] == "[DONE]", served.text + chunks: Final = tuple(ChatCompletionChunk.model_validate_json(event) for event in events[:-1]) + assert {chunk.object for chunk in chunks} == {"chat.completion.chunk"}, served.text + assert "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices), served.text + observations: Final = _observed(upstream) + assert len(observations) == 1, observations + upstream_body: Final = object_value(object_value(observations[0])["body"]) + assert upstream_body["stream"] is True, observations + + denied: Final = eventually( + lambda: gateway.request("POST", "/v1/chat/completions", body, key=key), + lambda response: ( + response.status_code == 422 and object_value(response.json()["error"])["type"] == "budget_exceeded" + ), + seconds=125, + ) + assert denied.status_code == 422, denied.text + assert object_value(denied.json()["error"])["type"] == "budget_exceeded", denied.text + assert "Max budget: 0.06" in denied.text, denied.text + _observed(upstream) + fresh_denied: Final = gateway.request("POST", "/v1/chat/completions", body, key=key) + assert fresh_denied.status_code == 422, fresh_denied.text + assert object_value(fresh_denied.json()["error"])["type"] == "budget_exceeded", fresh_denied.text + assert "Max budget: 0.06" in fresh_denied.text, fresh_denied.text + assert _observed(upstream) == [] diff --git a/tests/integration/spend/test_scoped_spend_reports.py b/tests/integration/spend/test_scoped_spend_reports.py new file mode 100644 index 00000000000..229e5ac8a8b --- /dev/null +++ b/tests/integration/spend/test_scoped_spend_reports.py @@ -0,0 +1,228 @@ +import uuid +from collections.abc import Mapping +from datetime import date, datetime, time, timedelta +from hashlib import sha256 +from typing import Final + +import httpx +import pytest +from integration._support.client import Gateway, eventually, object_value +from integration._support.database import read_rows +from pydantic import BaseModel + + +class ModelDetail(BaseModel): + model: str + total_cost: float + total_input_tokens: float + total_output_tokens: float + team_id: str | None = None + + +class ReportRow(BaseModel): + api_key: str + total_cost: float + total_input_tokens: float + total_output_tokens: float + model_details: list[ModelDetail] + + +def _chat(gateway: Gateway, model: str, key: str) -> None: + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "max_tokens": 20, "messages": [{"role": "user", "content": f"report {uuid.uuid4().hex}"}]}, + key=key, + ) + assert response.status_code == 200, response.text + + +def _report( + gateway: Gateway, path: str, *, key: str | None = None, params: Mapping[str, str] | None = None +) -> httpx.Response: + return gateway.request("GET", path, key=key, params=params) + + +def _rows( + gateway: Gateway, path: str, *, key: str | None = None, params: Mapping[str, str] | None = None +) -> list[ReportRow]: + response: Final = _report(gateway, path, key=key, params=params) + assert response.status_code == 200, response.text + return [ReportRow.model_validate(row) for row in response.json()] + + +def _logged(scope_column: str, scope_value: str, start: datetime, end: datetime) -> list[dict[str, object]]: + return read_rows( + f'SELECT api_key, model, spend, prompt_tokens, completion_tokens, team_id FROM "LiteLLM_SpendLogs" ' + f'WHERE "startTime" >= %s AND "startTime" < %s AND {scope_column} = %s', + (start.isoformat(), end.isoformat(), scope_value), + ) + + +def _logged_keys(keys: tuple[str, ...], start: datetime, end: datetime) -> list[dict[str, object]]: + placeholders: Final = ", ".join("%s" for _ in keys) + return read_rows( + 'SELECT api_key, model, spend, prompt_tokens, completion_tokens, team_id FROM "LiteLLM_SpendLogs" ' + f'WHERE "startTime" >= %s AND "startTime" < %s AND api_key IN ({placeholders})', + (start.isoformat(), end.isoformat(), *keys), + ) + + +def _assert_report_matches_db(report: list[ReportRow], logged: list[dict[str, object]]) -> None: + assert len(report) == len({row["api_key"] for row in logged}), (report, logged) + by_key: Final[dict[str, dict[str, list[float]]]] = {} + for row in logged: + bucket: Final = by_key.setdefault(str(row["api_key"]), {}) + model_bucket: Final = bucket.setdefault(str(row["model"]), [0.0, 0.0, 0.0]) + model_bucket[0] += float(str(row["spend"])) + model_bucket[1] += int(str(row["prompt_tokens"])) + model_bucket[2] += int(str(row["completion_tokens"])) + for entry in report: + assert entry.api_key in by_key, (entry, logged) + models: Final = by_key[entry.api_key] + assert entry.total_cost == pytest.approx(sum(v[0] for v in models.values())), entry + assert entry.total_input_tokens == pytest.approx(sum(v[1] for v in models.values())), entry + assert entry.total_output_tokens == pytest.approx(sum(v[2] for v in models.values())), entry + details: Final = {detail.model: detail for detail in entry.model_details} + assert set(details) == set(models), (entry, logged) + for model, detail in details.items(): + cost, prompt, completion = models[model] + assert detail.total_cost == pytest.approx(cost), detail + assert detail.total_input_tokens == pytest.approx(prompt), detail + assert detail.total_output_tokens == pytest.approx(completion), detail + + +def test_scoped_spend_reports_match_sql_and_enforce_caller_scope(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + org1: Final = scenario.organization() + org2: Final = scenario.organization() + team1: Final = scenario.team(organization_id=org1) + team2: Final = scenario.team(organization_id=org2) + user1: Final = scenario.member(team1, role="user") + user2: Final = scenario.member(team2, role="user") + org_admin: Final = scenario.org_member(org1, "org_admin") + k1: Final = scenario.key(user_id=user1, team_id=team1, models=[model]) + k2: Final = scenario.key(user_id=user2, team_id=team2, models=[model]) + direct_org_key: Final = scenario.key(organization_id=org2, models=[model]) + org_admin_key: Final = scenario.key(user_id=org_admin, organization_id=org1, models=[model]) + org_admin_info: Final = object_value(gateway.get("/key/info", {"key": org_admin_key})["info"]) + assert org_admin_info["organization_id"] == org1, org_admin_info + _chat(gateway, model, k1) + _chat(gateway, model, k2) + _chat(gateway, model, direct_org_key) + k1_hash: Final = sha256(k1.encode()).hexdigest() + k2_hash: Final = sha256(k2.encode()).hexdigest() + direct_org_key_hash: Final = sha256(direct_org_key.encode()).hexdigest() + logged: Final = eventually( + lambda: read_rows( + "SELECT api_key, to_char(\"startTime\", 'YYYY-MM-DD') AS day " + 'FROM "LiteLLM_SpendLogs" WHERE api_key IN (%s, %s, %s)', + (k1_hash, k2_hash, direct_org_key_hash), + ), + lambda rows: len(rows) == 3, + seconds=30, + ) + dates: Final = {date.fromisoformat(str(row["day"])) for row in logged} + start_date: Final[date] = min(dates) + end_date: Final[date] = max(dates) + start: Final = datetime.combine(start_date, time.min) + end: Final = datetime.combine(end_date, time.min) + timedelta(days=1) + window: Final = {"start_date": start_date.isoformat(), "end_date": end_date.isoformat()} + + _assert_report_matches_db( + _rows(gateway, "/key/spend/report", key=k1, params=window), _logged("api_key", k1_hash, start, end) + ) + _assert_report_matches_db( + _rows(gateway, "/user/spend/report", key=k1, params=window), _logged('"user"', user1, start, end) + ) + _assert_report_matches_db( + _rows(gateway, "/team/spend/report", key=k1, params=window), _logged("team_id", team1, start, end) + ) + org1_report: Final = _rows(gateway, "/organization/spend/report", key=org_admin_key, params=window) + _assert_report_matches_db(org1_report, _logged_keys((k1_hash,), start, end)) + refused_org2: Final = _report( + gateway, + "/organization/spend/report", + key=org_admin_key, + params={**window, "organization_id": org2}, + ) + assert refused_org2.status_code == 403, refused_org2.text + assert refused_org2.json() == {"detail": "You do not have access to this organization"}, refused_org2.text + + own_org: Final = _report(gateway, "/organization/spend/report", key=k1, params=window) + assert own_org.status_code == 403, own_org.text + assert own_org.json() == {"detail": "You do not have access to this organization"}, own_org.text + + assert _rows(gateway, "/key/spend/report", key=k1, params={**window, "api_key": k1}) == _rows( + gateway, "/key/spend/report", key=k1, params={**window, "api_key": k1_hash} + ) + + for path, param, value, detail in ( + ( + "/key/spend/report", + "api_key", + k2_hash, + "Not authorized to view spend for a api_key other than your own", + ), + ( + "/user/spend/report", + "internal_user_id", + user2, + "Not authorized to view spend for a internal_user_id other than your own", + ), + ( + "/team/spend/report", + "team_id", + team2, + "Not authorized to view spend for a team_id other than your own", + ), + ("/organization/spend/report", "organization_id", org2, "You do not have access to this organization"), + ): + refused: Final = _report(gateway, path, key=k1, params={**window, param: value}) + assert refused.status_code == 403, refused.text + assert refused.json() == {"detail": detail}, refused.text + + _assert_report_matches_db( + _rows(gateway, "/key/spend/report", params={**window, "api_key": k2_hash}), + _logged("api_key", k2_hash, start, end), + ) + _assert_report_matches_db( + _rows(gateway, "/user/spend/report", params={**window, "internal_user_id": user2}), + _logged('"user"', user2, start, end), + ) + _assert_report_matches_db( + _rows(gateway, "/team/spend/report", params={**window, "team_id": team2}), + _logged("team_id", team2, start, end), + ) + org_report: Final = _rows(gateway, "/organization/spend/report", params={**window, "organization_id": org2}) + _assert_report_matches_db(org_report, _logged_keys((k2_hash, direct_org_key_hash), start, end)) + assert {detail.team_id for entry in org_report for detail in entry.model_details} == {team2, ""}, org_report + + +@pytest.mark.parametrize( + "path", + ( + "/key/spend/report", + "/user/spend/report", + "/team/spend/report", + "/organization/spend/report", + ), +) +@pytest.mark.parametrize( + ("date_kind", "params", "detail"), + ( + ("missing", None, "Please provide start_date and end_date"), + ( + "malformed", + {"start_date": "2026-13-45", "end_date": "2026-01-01"}, + "start_date and end_date must be in YYYY-MM-DD format", + ), + ), +) +def test_spend_report_rejects_missing_and_malformed_dates( + gateway: Gateway, path: str, date_kind: str, params: Mapping[str, str] | None, detail: str +) -> None: + response: Final = _report(gateway, path, params=params) + assert response.status_code == 400, response.text + assert response.json() == {"detail": detail}, response.text diff --git a/tests/integration/spend/test_spend_log_read_scope.py b/tests/integration/spend/test_spend_log_read_scope.py index f9034371e35..8a208d61aed 100644 --- a/tests/integration/spend/test_spend_log_read_scope.py +++ b/tests/integration/spend/test_spend_log_read_scope.py @@ -3,13 +3,15 @@ import uuid from collections.abc import AsyncIterator from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from hashlib import sha256 from typing import Final from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit import psycopg import pytest import pytest_asyncio -from integration._support.client import Gateway +from integration._support.client import Gateway, eventually, object_value, string_value +from integration._support.database import read_rows from prisma import Prisma from psycopg import sql from psycopg.types.json import Jsonb @@ -224,3 +226,91 @@ def test_spend_log_routes_preserve_user_and_permitted_team_access( admin: Final = gateway.request("GET", f"/spend/logs/ui/{rows[2].request_id}") assert admin.status_code == 200, admin.text assert admin.json()["messages"] == [{"role": "user", "content": rows[2].request_id + " payload"}], admin.text + + +def _chat_request_id(gateway: Gateway, model: str, key: str) -> str: + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "max_tokens": 20, "messages": [{"role": "user", "content": f"scope {uuid.uuid4().hex}"}]}, + key=key, + ) + assert response.status_code == 200, response.text + body: Final = TypeAdapter(dict[str, object]).validate_python(response.json()) + return str(body["id"]) + + +def _legacy_spend_rows(gateway: Gateway, key: str, params: dict[str, str]) -> list[dict[str, object]]: + response: Final = gateway.request("GET", "/spend/logs", key=key, params=params) + assert response.status_code == 200, response.text + return TypeAdapter(list[dict[str, object]]).validate_python(response.json()) + + +def test_legacy_spend_logs_clamp_internal_user_filters_to_their_own_user(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + user_a: Final = scenario.user(user_role="internal_user") + user_b: Final = scenario.user(user_role="internal_user") + key_a: Final = scenario.key(user_id=user_a, models=[model]) + key_b: Final = scenario.key(user_id=user_b, models=[model]) + request_a: Final = _chat_request_id(gateway, model, key_a) + request_b: Final = _chat_request_id(gateway, model, key_b) + for request_id in (request_a, request_b): + eventually( + lambda request_id=request_id: read_rows( + 'SELECT request_id, "user" FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (request_id,) + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + + filtered: Final = _legacy_spend_rows(gateway, key_a, {"user_id": user_b}) + assert filtered != [], filtered + assert {string_value(object_value(row)["user"]) for row in filtered} == {user_a}, filtered + assert request_b not in {string_value(object_value(row)["request_id"]) for row in filtered} + assert request_a in {string_value(object_value(row)["request_id"]) for row in filtered} + + by_key: Final = _legacy_spend_rows(gateway, key_a, {"api_key": key_b}) + assert by_key == [], by_key + hashed_key_b: Final = sha256(key_b.encode()).hexdigest() + by_hashed_key: Final = _legacy_spend_rows(gateway, key_a, {"api_key": hashed_key_b}) + assert by_hashed_key == [], by_hashed_key + + by_request: Final = _legacy_spend_rows(gateway, key_a, {"request_id": request_b}) + assert by_request == [], by_request + + now: Final = datetime.now(timezone.utc) + window: Final = { + "start_date": (now - timedelta(days=1)).strftime("%Y-%m-%d"), + "end_date": (now + timedelta(days=1)).strftime("%Y-%m-%d"), + } + summarized: Final = _legacy_spend_rows(gateway, key_a, {**window, "summarize": "true"}) + assert user_b not in {name for row in summarized for name in object_value(object_value(row)["users"])}, ( + summarized + ) + assert sum(float(str(object_value(row)["spend"])) for row in summarized) == pytest.approx(0.06), summarized + unsummarized: Final = _legacy_spend_rows(gateway, key_a, {**window, "summarize": "false"}) + assert unsummarized != [], unsummarized + assert {string_value(object_value(row)["user"]) for row in unsummarized} == {user_a}, unsummarized + + master: Final = _legacy_spend_rows(gateway, gateway.key, {"user_id": user_b, "request_id": request_b}) + assert [string_value(object_value(row)["request_id"]) for row in master] == [request_b], master + for api_key in (key_b, hashed_key_b): + admin_by_key = _legacy_spend_rows(gateway, gateway.key, {"api_key": api_key}) + assert [ + (string_value(object_value(row)["request_id"]), object_value(row)["user"]) for row in admin_by_key + ] == [(request_b, user_b)], admin_by_key + summarized_by_key: Final = _legacy_spend_rows(gateway, key_a, {**window, "summarize": "true", "api_key": key_b}) + assert sum(float(str(object_value(row)["spend"])) for row in summarized_by_key) == 0, summarized_by_key + admin_summarized: Final = _legacy_spend_rows( + gateway, gateway.key, {**window, "summarize": "true", "api_key": key_b} + ) + assert sum(float(str(object_value(row)["spend"])) for row in admin_summarized) == pytest.approx(0.06), ( + admin_summarized + ) + admin_users: Final = { + name: float(str(spend)) + for row in admin_summarized + for name, spend in object_value(object_value(row)["users"]).items() + } + assert admin_users == pytest.approx({user_b: 0.06}), admin_summarized From 5261a2cc06320a9ff513b86adff3e7949fb3c7f9 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:59:15 +0000 Subject: [PATCH 08/51] test(integration): cover responses endpoint contracts (#44733) * test(integration): cover responses endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): tighten responses contract tests per review Isolate team-only response-id ownership with a same-user different-team key, send every denial through the OpenAI SDK on all three aliases, make the previous_response_id affinity check deterministic, assert the whole cancel response, prove the Cursor stream text, -fast suffixes and include_obfuscation strip, add SDK compact cases, restore router pre-call checks after the affinity test, and inline the root-level responses_test_support helpers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): wait for every proxy worker before routing to new deployments Restore the cross-worker convergence wait dropped when the shared helper was inlined, so the affinity and retrieve-routing tests cannot pass before the alternative deployment is loaded on a multi-worker gateway Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover Responses ownership and wire contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): address responses contract review Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): expect the provider model in the conversation passthrough body Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): expand responses endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): restore streamed response retrieval Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): wait for proxy workers after deployment setup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): track readiness per proxy worker Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integration/_support/worker_readiness.py | 64 ++ .../test_responses_id_ownership.py | 487 +++++++++++++++ .../providers/test_cursor_responses_wire.py | 450 ++++++++++++++ .../providers/test_responses_compact_wire.py | 367 +++++++++++ .../providers/test_responses_core_wire.py | 406 ++++++++++++ .../test_responses_lifecycle_wire.py | 583 ++++++++++++++++++ 6 files changed, 2357 insertions(+) create mode 100644 tests/integration/_support/worker_readiness.py create mode 100644 tests/integration/authorization/test_responses_id_ownership.py create mode 100644 tests/integration/providers/test_cursor_responses_wire.py create mode 100644 tests/integration/providers/test_responses_compact_wire.py create mode 100644 tests/integration/providers/test_responses_core_wire.py create mode 100644 tests/integration/providers/test_responses_lifecycle_wire.py diff --git a/tests/integration/_support/worker_readiness.py b/tests/integration/_support/worker_readiness.py new file mode 100644 index 00000000000..ba11d2ba4bf --- /dev/null +++ b/tests/integration/_support/worker_readiness.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import time +from collections.abc import Iterator, Mapping +from itertools import accumulate +from types import MappingProxyType +from typing import Final + +import httpx + +from integration._support.client import Gateway, JSON_OBJECT, object_value +from litellm.constants import PROXY_CONFIG_RELOAD_INTERVAL_SECONDS + + +def _record_latest_readback( + latest_by_worker: Mapping[int, int], + observation: tuple[int, int], +) -> Mapping[int, int]: + worker_pid, count = observation + return MappingProxyType({**latest_by_worker, worker_pid: count}) + + +def wait_for_worker_readiness( + gateway: Gateway, + group: str, + deployments: int, + workers: int, +) -> None: + deadline: Final = time.monotonic() + (30 if workers == 1 else PROXY_CONFIG_RELOAD_INTERVAL_SECONDS * 2 + 30) + headers: Final = {"Authorization": f"Bearer {gateway.key}"} + + def readbacks() -> Iterator[tuple[int, int]]: + while time.monotonic() < deadline: + timeout: Final = deadline - time.monotonic() + with httpx.Client( + base_url=str(gateway.client.base_url), + timeout=timeout, + trust_env=False, + verify=False, + ) as client: + debug_response: Final = client.get("/debug/memory/summary", headers=headers) + response: Final = client.get( + "/model/info", + headers=headers, + ) + assert debug_response.status_code == 200, debug_response.text + assert response.status_code == 200, response.text + assert debug_response.extensions.get("network_stream") is not None + assert debug_response.extensions.get("network_stream") is response.extensions.get("network_stream") + debug_body: Final = JSON_OBJECT.validate_json(debug_response.content) + worker_pid: Final = debug_body.get("worker_pid") + assert isinstance(worker_pid, int), debug_body + body: Final = JSON_OBJECT.validate_json(response.content) + entries: Final = body.get("data") + assert isinstance(entries, list), body + count: Final = sum(object_value(entry).get("model_name") == group for entry in entries) + yield worker_pid, count + + initial_counts: Final[Mapping[int, int]] = MappingProxyType({}) + latest_counts: Final = accumulate(readbacks(), _record_latest_readback, initial=initial_counts) + for latest_by_worker in latest_counts: + if len(latest_by_worker) >= workers and all(count == deployments for count in latest_by_worker.values()): + return + raise AssertionError(f"Expected {deployments} deployments on {workers} workers before the readiness deadline") diff --git a/tests/integration/authorization/test_responses_id_ownership.py b/tests/integration/authorization/test_responses_id_ownership.py new file mode 100644 index 00000000000..9e13b45e385 --- /dev/null +++ b/tests/integration/authorization/test_responses_id_ownership.py @@ -0,0 +1,487 @@ +from __future__ import annotations + +import json +import os +import uuid +from collections.abc import Callable +from pathlib import Path +from typing import Final, Literal +from urllib.parse import urlsplit + +import httpx +import openai +import pytest +from integration._support.client import Gateway, Scenario, object_value +from integration._support.process import owned_proxy +from integration._support.responses_vendor import same_response +from integration._support.wire import Reply, Request, Wire, wire_server +from integration._support.worker_readiness import wait_for_worker_readiness +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +_MODEL: Final = "gpt-5" +_API_KEY: Final = "synthetic-responses-key" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_PROXY_WORKERS: Final = max(1, int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1"))) +_ALIASES: Final = ("/v1", "", "/openai/v1") +_DISABLE_HINT: Final = ( + "To disable this security feature, set general_settings::disable_responses_id_security to True " + "in the config.yaml file." +) +_OTHER_USER_DETAIL: Final = ( + f"Forbidden. The response id is not associated with the user, who this key belongs to. {_DISABLE_HINT}" +) +_OTHER_TEAM_DETAIL: Final = ( + f"Forbidden. The response id is not associated with the team, who this key belongs to. {_DISABLE_HINT}" +) +_UNMANAGED_DETAIL: Final = ( + "Forbidden. This response id was not issued by this proxy, so the proxy cannot tell who owns it. " + "To let keys address responses this proxy did not issue, set " + "general_settings::allow_unmanaged_response_ids to True in the config.yaml file." +) + + +class _DeleteResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + object: Literal["response"] + deleted: bool + + +def _deployments_in_group(gateway: Gateway, group: str) -> int: + entries: Final = gateway.get("/model/info")["data"] + assert isinstance(entries, list), entries + return sum(object_value(entry).get("model_name") == group for entry in entries) + + +def _wait_for_workers(gateway: Gateway, group: str, deployments: int) -> None: + wait_for_worker_readiness(gateway, group, deployments, _PROXY_WORKERS) + + +def _model_discovery_reply() -> Reply: + return Reply( + body=json.dumps( + {"object": "list", "data": [{"id": _MODEL, "object": "model", "created": 1, "owned_by": "openai"}]} + ).encode() + ) + + +def _contract_requests(wire: Wire) -> tuple[Request, ...]: + requests: Final = wire.drain() + discovery: Final = tuple(request for request in requests if request.target == "/v1/models") + assert all(request.method == "GET" and request.body == b"" for request in discovery), requests + return tuple(request for request in requests if request.target != "/v1/models") + + +def _response(identity: str, status: str = "completed") -> bytes: + return json.dumps( + { + "id": identity, + "object": "response", + "created_at": 1, + "status": status, + "model": _MODEL, + "output": [], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + } + ).encode() + + +def _deployment(scenario: Scenario, wire: Wire) -> str: + model: Final = scenario.model( + model=f"openai/{_MODEL}", + api_key=_API_KEY, + api_base=f"{wire.url}/v1", + ) + _wait_for_workers(scenario.gateway, model, 1) + return model + + +def _key(scenario: Scenario, team_id: str, user_id: str, model: str) -> str: + return scenario.key(team_id=team_id, user_id=user_id, models=[model]) + + +def _client(gateway: Gateway, key: str, prefix: str) -> openai.OpenAI: + return openai.OpenAI(base_url=f"{gateway.client.base_url}{prefix}", api_key=key, max_retries=0) + + +def _error(response: httpx.Response, detail: str) -> None: + assert response.status_code == 403, response.text + assert response.json() == { + "error": { + "message": detail, + "type": "permission_error", + "param": None, + "code": "403", + } + }, response.text + + +def _refused(call: Callable[[], object], detail: str) -> None: + with pytest.raises(openai.PermissionDeniedError) as refusal: + call() + _error(refusal.value.response, detail) + + +def _every_id_route(client: openai.OpenAI, model: str, response_id: str) -> tuple[Callable[[], object], ...]: + return ( + lambda: client.responses.retrieve(response_id), + lambda: client.responses.delete(response_id), + lambda: client.responses.cancel(response_id), + lambda: client.responses.input_items.list(response_id), + lambda: client.responses.create(model=model, input="cross-tenant follow-up", previous_response_id=response_id), + ) + + +@pytest.mark.parametrize("prefix", _ALIASES) +def test_response_ids_are_scoped_to_the_issuing_user_and_team(gateway: Gateway, prefix: str) -> None: + raw_id: Final = f"resp_owned_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return _model_discovery_reply() + body: Final = _JSON_OBJECT.validate_json(request.body) if request.body else {} + if request.method == "POST" and request.target == "/v1/responses": + if body.get("input") == "create owned response": + return Reply(body=_response(raw_id)) + return Reply(body=_response(raw_id)) + if request.method == "GET" and urlsplit(request.target).path.endswith("/input_items"): + return Reply(body=b'{"data":[],"has_more":false,"object":"list"}') + if request.method == "GET": + return Reply(body=_response(raw_id)) + if request.method == "DELETE": + return Reply(body=json.dumps({"id": raw_id, "object": "response", "deleted": True}).encode()) + if request.method == "POST" and request.target.endswith("/cancel"): + return Reply(body=_response(raw_id, "cancelled")) + raise AssertionError(f"Unexpected upstream request: {request.method} {request.target}") + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + team_a: Final = scenario.team(models=[model]) + team_b: Final = scenario.team(models=[model]) + user_a: Final = scenario.member(team_a) + user_b: Final = scenario.member(team_b) + gateway.post("/team/member_add", {"team_id": team_b, "member": {"role": "user", "user_id": user_a}}) + client_a: Final = _client(gateway, _key(scenario, team_a, user_a, model), prefix) + other_user_client: Final = _client(gateway, _key(scenario, team_b, user_b, model), prefix) + other_team_client: Final = _client(gateway, _key(scenario, team_b, user_a, model), prefix) + created: Final = client_a.responses.create(model=model, input="create owned response") + client_id: Final = created.id + assert client_id.startswith("resp_") and client_id != raw_id, created.model_dump_json() + created_requests: Final = _contract_requests(wire) + assert [(request.method, request.target, request.body) for request in created_requests] == [ + ("POST", "/v1/responses", created_requests[0].body) + ], created_requests + assert _JSON_OBJECT.validate_json(created_requests[0].body) == { + "model": _MODEL, + "input": "create owned response", + }, created_requests[0].body + + for call in _every_id_route(other_user_client, model, client_id): + _refused(call, _OTHER_USER_DETAIL) + for call in _every_id_route(other_team_client, model, client_id): + _refused(call, _OTHER_TEAM_DETAIL) + assert _contract_requests(wire) == () + + retrieved_response: Final = client_a.responses.with_raw_response.retrieve(client_id) + retrieved: Final = retrieved_response.parse() + assert retrieved.id != raw_id and same_response(retrieved.id, client_id), retrieved_response.text + assert retrieved.model_dump(mode="json", exclude_none=True) == { + "id": retrieved.id, + "object": "response", + "created_at": 1.0, + "status": "completed", + "model": _MODEL, + "output": [], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + }, retrieved_response.text + deleted: Final = client_a.responses.with_raw_response.delete(client_id) + assert deleted.status_code == 200, deleted.text + deleted_response: Final = _DeleteResponse.model_validate_json(deleted.content) + assert deleted_response.model_dump(exclude={"id"}) == {"object": "response", "deleted": True}, deleted.text + assert same_response(deleted_response.id, client_id), deleted.text + cancelled_response: Final = client_a.responses.with_raw_response.cancel(client_id) + cancelled: Final = cancelled_response.parse() + assert cancelled.status == "cancelled", cancelled_response.text + assert cancelled.id != raw_id and same_response(cancelled.id, client_id), cancelled_response.text + assert cancelled.model_dump(mode="json", exclude_none=True) == { + "id": cancelled.id, + "object": "response", + "created_at": 1.0, + "status": "cancelled", + "model": _MODEL, + "output": [], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + }, cancelled_response.text + page_response: Final = client_a.responses.input_items.with_raw_response.list(client_id) + page: Final = page_response.parse() + assert page.model_dump(mode="json", exclude_none=True) == { + "data": [], + "has_more": False, + "object": "list", + }, page_response.text + followed: Final = client_a.responses.create( + model=model, + input="owned follow-up", + previous_response_id=client_id, + ) + assert followed.id != raw_id and same_response(followed.id, client_id), followed.model_dump_json() + requests: Final = _contract_requests(wire) + assert [(request.method, urlsplit(request.target).path, request.body) for request in requests] == [ + ("GET", f"/v1/responses/{raw_id}", b""), + ("DELETE", f"/v1/responses/{raw_id}", b""), + ("POST", f"/v1/responses/{raw_id}/cancel", b"{}"), + ("GET", f"/v1/responses/{raw_id}/input_items", b""), + ("POST", "/v1/responses", requests[-1].body), + ], requests + assert _JSON_OBJECT.validate_json(requests[-1].body) == { + "input": "owned follow-up", + "model": _MODEL, + "previous_response_id": raw_id, + }, requests[-1].body + + unmanaged_id: Final = f"resp_vendor_{uuid.uuid4().hex}" + for call in _every_id_route(client_a, model, unmanaged_id): + _refused(call, _UNMANAGED_DETAIL) + assert _contract_requests(wire) == () + + +@pytest.mark.parametrize("prefix", _ALIASES) +def test_compact_previous_response_id_is_scoped_to_the_issuing_user_and_team( + gateway: Gateway, + prefix: str, +) -> None: + pytest.skip( + "BUG: POST /v1/responses/compact does not check previous_response_id ownership; " + "another team's id and unmanaged ids reach the provider" + ) + raw_id: Final = f"resp_compact_owned_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return _model_discovery_reply() + if request.method == "POST" and request.target == "/v1/responses": + return Reply(body=_response(raw_id)) + return Reply(body=_response(raw_id)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + team_a: Final = scenario.team(models=[model]) + team_b: Final = scenario.team(models=[model]) + user_a: Final = scenario.member(team_a) + user_b: Final = scenario.member(team_b) + gateway.post("/team/member_add", {"team_id": team_b, "member": {"role": "user", "user_id": user_a}}) + client_a: Final = _client(gateway, _key(scenario, team_a, user_a, model), prefix) + other_user_client: Final = _client(gateway, _key(scenario, team_b, user_b, model), prefix) + other_team_client: Final = _client(gateway, _key(scenario, team_b, user_a, model), prefix) + created: Final = client_a.responses.create(model=model, input="create compact response") + client_id: Final = created.id + assert client_id != raw_id, created.model_dump_json() + assert len(_contract_requests(wire)) == 1 + _refused( + lambda: other_user_client.responses.compact( + model=model, + previous_response_id=client_id, + input=[ + {"role": "user", "content": [{"type": "input_text", "text": "compact someone else's response"}]} + ], + ), + _OTHER_USER_DETAIL, + ) + _refused( + lambda: other_team_client.responses.compact( + model=model, + previous_response_id=client_id, + input=[ + {"role": "user", "content": [{"type": "input_text", "text": "compact someone else's response"}]} + ], + ), + _OTHER_TEAM_DETAIL, + ) + unmanaged_id: Final = f"resp_vendor_{uuid.uuid4().hex}" + _refused( + lambda: client_a.responses.compact( + model=model, + previous_response_id=unmanaged_id, + input=[ + {"role": "user", "content": [{"type": "input_text", "text": "compact someone else's response"}]} + ], + ), + _UNMANAGED_DETAIL, + ) + assert _contract_requests(wire) == () + + +def test_allow_unmanaged_response_ids_forward_the_vendor_id_on_create( + gateway: Gateway, + tmp_path: Path, +) -> None: + raw_id: Final = f"resp_vendor_{uuid.uuid4().hex}" + raw_followup_id: Final = f"resp_followup_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return _model_discovery_reply() + return Reply(body=_response(raw_followup_id)) + + config: Final = tmp_path / "allow_unmanaged_responses.yaml" + config.write_text("general_settings:\n allow_unmanaged_response_ids: true\n") + with wire_server(respond) as wire, wire_server(respond) as decoy, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + team: Final = scenario.team(models=[model]) + user: Final = scenario.member(team) + key: Final = _key(scenario, team, user, model) + with owned_proxy( + gateway, + tmp_path, + { + "INTEGRATION_PROXY_WORKERS": "1", + "OPENAI_BASE_URL": f"{decoy.url}/v1", + "OPENAI_API_KEY": "synthetic-env-fallback-key", + }, + config=config, + ) as allowed_gateway: + client: Final = _client(allowed_gateway, key, "/v1") + followed: Final = client.responses.create( + model=model, + input="allowed vendor follow-up", + previous_response_id=raw_id, + ) + assert followed.model_dump(mode="json", exclude_none=True) == { + "id": followed.id, + "object": "response", + "created_at": 1.0, + "status": "completed", + "model": model, + "output": [], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + }, followed.model_dump_json() + retrieved_response: Final = client.responses.with_raw_response.retrieve(followed.id) + retrieved: Final = retrieved_response.parse() + assert same_response(retrieved.id, followed.id), retrieved_response.text + assert retrieved.model_dump(mode="json", exclude_none=True) == { + "id": retrieved.id, + "object": "response", + "created_at": 1.0, + "status": "completed", + "model": _MODEL, + "output": [], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + }, retrieved_response.text + requests: Final = _contract_requests(wire) + assert decoy.drain() == (), "The environment fallback deployment received a request" + assert [(request.method, request.target, request.body) for request in requests] == [ + ("POST", "/v1/responses", requests[0].body), + ("GET", f"/v1/responses/{raw_followup_id}", b""), + ], requests + assert all(request.headers["authorization"] == f"Bearer {_API_KEY}" for request in requests), requests + assert _JSON_OBJECT.validate_json(requests[0].body) == { + "model": _MODEL, + "input": "allowed vendor follow-up", + "previous_response_id": raw_id, + }, requests[0].body + + +@pytest.mark.parametrize("route", ("retrieve", "delete", "cancel", "input_items")) +def test_allow_unmanaged_response_id_lifecycle_route_reaches_provider( + gateway: Gateway, + tmp_path: Path, + route: Literal["retrieve", "delete", "cancel", "input_items"], +) -> None: + pytest.skip( + "BUG: unmanaged response-id lifecycle routes select model=None and return 400 instead of reaching OpenAI" + ) + raw_id: Final = f"resp_vendor_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return _model_discovery_reply() + if request.method == "GET" and request.target == f"/v1/responses/{raw_id}/input_items": + return Reply(body=b'{"data":[],"has_more":false,"object":"list"}') + if request.method == "DELETE": + return Reply(body=json.dumps({"id": raw_id, "object": "response", "deleted": True}).encode()) + if request.method == "POST" and request.target.endswith("/cancel"): + return Reply(body=_response(raw_id, "cancelled")) + return Reply(body=_response(raw_id)) + + config: Final = tmp_path / "allow_unmanaged_responses.yaml" + config.write_text("general_settings:\n allow_unmanaged_response_ids: true\n") + with wire_server(respond) as wire, wire_server(respond) as decoy, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + team: Final = scenario.team(models=[model]) + user: Final = scenario.member(team) + key: Final = _key(scenario, team, user, model) + with owned_proxy( + gateway, + tmp_path, + { + "INTEGRATION_PROXY_WORKERS": "1", + "OPENAI_BASE_URL": f"{decoy.url}/v1", + "OPENAI_API_KEY": "synthetic-env-fallback-key", + }, + config=config, + ) as allowed_gateway: + client: Final = _client(allowed_gateway, key, "/v1") + expected_request: Final = { + "retrieve": ("GET", f"/v1/responses/{raw_id}", b""), + "delete": ("DELETE", f"/v1/responses/{raw_id}", b""), + "cancel": ("POST", f"/v1/responses/{raw_id}/cancel", b"{}"), + "input_items": ("GET", f"/v1/responses/{raw_id}/input_items", b""), + }[route] + match route: + case "retrieve": + retrieved: Final = client.responses.retrieve(raw_id) + assert retrieved.model_dump(mode="json", exclude_none=True) == { + "id": raw_id, + "object": "response", + "created_at": 1.0, + "status": "completed", + "model": _MODEL, + "output": [], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + }, retrieved.model_dump_json() + case "delete": + deleted: Final = client.responses.delete(raw_id) + assert deleted.model_dump(mode="json", exclude_none=True) == { + "id": raw_id, + "object": "response", + "deleted": True, + }, deleted.model_dump_json() + case "cancel": + cancelled: Final = client.responses.cancel(raw_id) + assert cancelled.model_dump(mode="json", exclude_none=True) == { + "id": raw_id, + "object": "response", + "created_at": 1.0, + "status": "cancelled", + "model": _MODEL, + "output": [], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + }, cancelled.model_dump_json() + case "input_items": + page: Final = client.responses.input_items.list(raw_id) + assert page.model_dump(mode="json", exclude_none=True) == { + "data": [], + "has_more": False, + "object": "list", + }, page.model_dump_json() + requests: Final = _contract_requests(wire) + assert tuple((request.method, request.target, request.body) for request in requests) == (expected_request,), ( + requests + ) + assert all(request.headers["authorization"] == f"Bearer {_API_KEY}" for request in requests), requests + assert decoy.drain() == (), "The environment fallback deployment received a request" diff --git a/tests/integration/providers/test_cursor_responses_wire.py b/tests/integration/providers/test_cursor_responses_wire.py new file mode 100644 index 00000000000..cf766c9105a --- /dev/null +++ b/tests/integration/providers/test_cursor_responses_wire.py @@ -0,0 +1,450 @@ +from __future__ import annotations + +import json +import os +import uuid +from typing import Final + +import pytest +from integration._support.client import Gateway, Scenario +from integration._support.responses_vendor import same_response +from integration._support.wire import Reply, Request, Wire, wire_server +from integration._support.worker_readiness import wait_for_worker_readiness +from pydantic import JsonValue, TypeAdapter + +_MODEL: Final = "gpt-5" +_API_KEY: Final = "synthetic-cursor-key" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_PROXY_WORKERS: Final = max(1, int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1"))) +_THINKING_LEVELS: Final = ("none", "minimal", "low", "medium", "high", "xhigh", "max") +_CHAT_THINKING_LEVELS: Final = ("none", "minimal", "low", "medium", "high", "max") +_RESPONSES_SUFFIXES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = tuple( + (f"-thinking-{level}", {"reasoning": {"effort": level}}) for level in _THINKING_LEVELS +) + ( + ("-fast", {}), + ("-thinking-low-fast", {"reasoning": {"effort": "low"}}), + ("-thinking-high-fast", {"reasoning": {"effort": "high"}}), +) +_CHAT_SUFFIXES: Final[tuple[tuple[str, str], ...]] = (("", ""),) + tuple( + (f"-thinking-{level}", level) for level in _CHAT_THINKING_LEVELS +) +_PROMPT: Final = "Inspect the requested change." +_INPUT: Final = [{"role": "user", "content": [{"type": "input_text", "text": _PROMPT}]}] +_CUSTOM_TOOL: Final = { + "type": "custom", + "name": "ApplyPatch", + "description": "Apply a patch", + "format": {"type": "grammar", "syntax": "lark", "definition": 'start: "ok"'}, +} +_FUNCTION_TOOL: Final = { + "type": "function", + "name": "lookup", + "description": "Look up a record", + "parameters": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}, + "strict": True, +} + + +def model_discovery_reply(model: str) -> Reply: + return Reply( + body=json.dumps( + {"object": "list", "data": [{"id": model, "object": "model", "created": 1, "owned_by": "openai"}]} + ).encode() + ) + + +def drain_contract_requests(wire: Wire) -> tuple[Request, ...]: + requests: Final = wire.drain() + discovery: Final = tuple(request for request in requests if request.target == "/v1/models") + assert all(request.method == "GET" and request.body == b"" for request in discovery), requests + return tuple(request for request in requests if request.target != "/v1/models") + + +def _response(identity: str) -> dict[str, JsonValue]: + return { + "id": identity, + "object": "response", + "created_at": 1, + "status": "completed", + "model": _MODEL, + "output": [ + { + "id": "msg_cursor", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Patch inspected.", "annotations": []}], + } + ], + "usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7}, + } + + +def _stream(identity: str) -> tuple[bytes, ...]: + response: Final = _response(identity) + return tuple( + f"data: {json.dumps(event)}\n\n".encode() + for event in ( + {"type": "response.created", "response": {**response, "status": "in_progress", "output": []}}, + { + "type": "response.output_text.delta", + "item_id": "msg_cursor", + "output_index": 0, + "content_index": 0, + "delta": "Patch inspected.", + }, + {"type": "response.completed", "response": response}, + ) + ) + (b"data: [DONE]\n\n",) + + +def _chat_completion(stream: bool) -> bytes | tuple[bytes, ...]: + if stream: + return ( + b'data: {"id":"chat_cursor_1","object":"chat.completion.chunk","created":1,"model":"gpt-5","choices":[{"index":0,"delta":{"content":"Patch inspected."},"finish_reason":null}]}\n\n', + b'data: {"id":"chat_cursor_1","object":"chat.completion.chunk","created":1,"model":"gpt-5","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n', + b"data: [DONE]\n\n", + ) + return json.dumps( + { + "id": "chat_cursor_1", + "object": "chat.completion", + "created": 1, + "model": _MODEL, + "system_fingerprint": "fp_cursor", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Patch inspected."}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}, + } + ).encode() + + +def _deployment(scenario: Scenario, wire: Wire) -> str: + model: Final = scenario.model( + model=f"openai/{_MODEL}", + api_key=_API_KEY, + api_base=f"{wire.url}/v1", + ) + wait_for_worker_readiness(scenario.gateway, model, 1, _PROXY_WORKERS) + return model + + +@pytest.mark.parametrize( + ("suffix", "reasoning"), + _RESPONSES_SUFFIXES, +) +@pytest.mark.parametrize("stream", (False, True)) +def test_cursor_responses_input_uses_responses_wire_and_chat_output( + gateway: Gateway, + stream: bool, + suffix: str, + reasoning: dict[str, JsonValue], +) -> None: + raw_id: Final = f"resp_cursor_{uuid.uuid4().hex}" + with gateway.scenario() as scenario: + team_alias: Final = f"integration-{uuid.uuid4().hex}" + team: Final = scenario.team(team_alias=team_alias) + user: Final = scenario.member(team) + key: Final = scenario.key(team_id=team, user_id=user) + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + assert request.method == "POST" + assert request.headers["authorization"] == f"Bearer {_API_KEY}", dict(request.headers) + if request.target == "/v1/chat/completions": + completion: Final = _chat_completion(stream) + return Reply( + content_type="text/event-stream" if stream else "application/json", + chunks=completion if isinstance(completion, tuple) else None, + body=completion if isinstance(completion, bytes) else b"", + ) + assert request.target == "/v1/responses", request.target + body: Final = _JSON_OBJECT.validate_json(request.body) + assert {key: value for key, value in body.items() if key != "metadata"} == { + "model": _MODEL, + "input": _INPUT, + **reasoning, + "tools": [_CUSTOM_TOOL], + "stream": stream, + }, request.body + if stream: + return Reply(content_type="text/event-stream", chunks=_stream(raw_id)) + return Reply(body=json.dumps(_response(raw_id)).encode()) + + with wire_server(respond) as wire: + model: Final = _deployment(scenario, wire) + response: Final = gateway.request( + "POST", + "/cursor/chat/completions", + { + "model": f"{model}{suffix}", + "input": _INPUT, + "messages": [], + "tools": [_CUSTOM_TOOL], + "stream_options": {"include_usage": True, "include_obfuscation": False}, + "stream": stream, + }, + key=key, + ) + assert response.status_code == 200, response.text + if stream: + events: Final = tuple( + _JSON_OBJECT.validate_json(line.removeprefix("data: ")) + for line in response.text.splitlines() + if line.startswith("data: ") and line != "data: [DONE]" + ) + assert events, response.text + assert all(event.get("object") == "chat.completion.chunk" for event in events), response.text + assert all(not str(event.get("type", "")).startswith("response.") for event in events), response.text + assert [{k: v for k, v in event.items() if k not in ("id", "created")} for event in events] == [ + { + "object": "chat.completion.chunk", + "model": model, + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Patch inspected."}}], + }, + { + "object": "chat.completion.chunk", + "model": model, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + ], response.text + else: + payload: Final = _JSON_OBJECT.validate_json(response.content) + identity: Final = payload.get("id") + created: Final = payload.get("created") + assert isinstance(identity, str) and same_response(identity, raw_id), response.text + assert isinstance(created, int) and created > 0, response.text + assert payload == { + "id": identity, + "created": created, + "model": model, + "object": "chat.completion", + "system_fingerprint": None, + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Patch inspected.", + "role": "assistant", + "tool_calls": None, + "function_call": None, + "provider_specific_fields": None, + }, + } + ], + "usage": { + "completion_tokens": 3, + "prompt_tokens": 4, + "total_tokens": 7, + "completion_tokens_details": None, + "prompt_tokens_details": None, + }, + }, response.text + assert [request.target for request in drain_contract_requests(wire)] == ["/v1/responses"] + + +def test_cursor_responses_body_carries_no_proxy_metadata(gateway: Gateway) -> None: + pytest.skip( + "BUG: /cursor/chat/completions forwards proxy-internal metadata (key hash, team, user, client ip, user agent) " + "to the provider in the Responses body" + ) + raw_id: Final = f"resp_cursor_metadata_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + assert request.method == "POST" and request.target == "/v1/responses", request.target + return Reply(body=json.dumps(_response(raw_id)).encode()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + response: Final = gateway.request( + "POST", + "/cursor/chat/completions", + { + "model": f"{model}-thinking-high", + "input": _INPUT, + "messages": [], + "tools": [_CUSTOM_TOOL], + }, + ) + assert response.status_code == 200, response.text + requests: Final = drain_contract_requests(wire) + assert len(requests) == 1, requests + body: Final = _JSON_OBJECT.validate_json(requests[0].body) + assert body == { + "model": _MODEL, + "input": _INPUT, + "reasoning": {"effort": "high"}, + "tools": [_CUSTOM_TOOL], + }, requests[0].body + + +def test_cursor_stream_reports_usage_when_the_client_asks_for_it(gateway: Gateway) -> None: + pytest.skip( + "BUG: /cursor/chat/completions with stream_options.include_usage streams no usage chunk; " + "the Responses usage from response.completed never reaches the chat client" + ) + raw_id: Final = f"resp_cursor_usage_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + assert (request.method, request.target) == ("POST", "/v1/responses"), request.target + return Reply(content_type="text/event-stream", chunks=_stream(raw_id)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + response: Final = gateway.request( + "POST", + "/cursor/chat/completions", + { + "model": model, + "input": _INPUT, + "stream_options": {"include_usage": True}, + "stream": True, + }, + ) + assert response.status_code == 200, response.text + events: Final = tuple( + _JSON_OBJECT.validate_json(line.removeprefix("data: ")) + for line in response.text.splitlines() + if line.startswith("data: ") and line != "data: [DONE]" + ) + assert [event["usage"] for event in events if event.get("usage")] == [ + {"completion_tokens": 3, "prompt_tokens": 4, "total_tokens": 7} + ], response.text + assert [request.target for request in drain_contract_requests(wire)] == ["/v1/responses"] + + +@pytest.mark.parametrize(("suffix", "reasoning_effort"), _CHAT_SUFFIXES) +@pytest.mark.parametrize("stream", (False, True)) +def test_cursor_messages_body_uses_chat_completions_wire( + gateway: Gateway, + stream: bool, + suffix: str, + reasoning_effort: str, +) -> None: + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + completion: Final = _chat_completion(stream) + return Reply( + content_type="text/event-stream" if stream else "application/json", + chunks=completion if isinstance(completion, tuple) else None, + body=completion if isinstance(completion, bytes) else b"", + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + team: Final = scenario.team(models=[model]) + user: Final = scenario.member(team) + key: Final = scenario.key(team_id=team, user_id=user, models=[model]) + response: Final = gateway.request( + "POST", + "/cursor/chat/completions", + { + "model": f"{model}{suffix}", + "messages": [{"role": "user", "content": "genuine chat request"}], + "tools": [_FUNCTION_TOOL, _CUSTOM_TOOL], + "tool_choice": {"type": "function", "name": "lookup"}, + "stream": stream, + }, + key=key, + ) + assert response.status_code == 200, response.text + if stream: + events: Final = tuple( + _JSON_OBJECT.validate_json(line.removeprefix("data: ")) + for line in response.text.splitlines() + if line.startswith("data: ") and line != "data: [DONE]" + ) + assert [ + {key: value for key, value in event.items() if key not in ("id", "created")} for event in events + ] == [ + { + "object": "chat.completion.chunk", + "model": model, + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Patch inspected."}}], + }, + { + "object": "chat.completion.chunk", + "model": model, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + ], response.text + else: + payload: Final = _JSON_OBJECT.validate_json(response.content) + identity: Final = payload.get("id") + created: Final = payload.get("created") + assert isinstance(identity, str) and same_response(identity, "chat_cursor_1"), response.text + assert isinstance(created, int) and created > 0, response.text + assert payload == { + "id": identity, + "created": created, + "model": model, + "object": "chat.completion", + "system_fingerprint": "fp_cursor", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Patch inspected.", + "role": "assistant", + "provider_specific_fields": {"refusal": None}, + }, + "provider_specific_fields": {}, + } + ], + "usage": { + "completion_tokens": 3, + "prompt_tokens": 4, + "total_tokens": 7, + }, + }, response.text + requests: Final = drain_contract_requests(wire) + assert [(request.method, request.target) for request in requests] == [("POST", "/v1/chat/completions")], ( + requests + ) + assert _JSON_OBJECT.validate_json(requests[0].body) == { + "model": _MODEL, + "messages": [{"role": "user", "content": "genuine chat request"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": "Look up a record", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + "strict": True, + }, + }, + { + "type": "custom", + "custom": { + "name": "ApplyPatch", + "description": "Apply a patch", + "format": { + "type": "grammar", + "grammar": {"syntax": "lark", "definition": 'start: "ok"'}, + }, + }, + }, + ], + "tool_choice": {"type": "function", "function": {"name": "lookup"}}, + **({"stream": True} if stream else {}), + **({"stream_options": {"include_usage": True}} if stream else {}), + **({"reasoning_effort": reasoning_effort} if reasoning_effort else {}), + }, requests[0].body diff --git a/tests/integration/providers/test_responses_compact_wire.py b/tests/integration/providers/test_responses_compact_wire.py new file mode 100644 index 00000000000..bc1c1ba2a7a --- /dev/null +++ b/tests/integration/providers/test_responses_compact_wire.py @@ -0,0 +1,367 @@ +from __future__ import annotations + +import json +import os +import uuid +from typing import Final + +import openai +import pytest +from integration._support.client import Gateway, Scenario +from integration._support.responses_vendor import same_response +from integration._support.wire import Reply, Request, Wire, wire_server +from integration._support.worker_readiness import wait_for_worker_readiness +from openai.types.responses import CompactedResponse +from pydantic import JsonValue, TypeAdapter + +_MODEL: Final = "gpt-5" +_API_KEY: Final = "synthetic-responses-key" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_PROXY_WORKERS: Final = max(1, int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1"))) +_ALIASES: Final = ("/v1/responses/compact", "/responses/compact", "/openai/v1/responses/compact") +_FUNCTION_TOOL: Final = { + "type": "function", + "name": "lookup", + "description": "Look up a record", + "parameters": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}, + "strict": True, +} + + +def model_discovery_reply(model: str) -> Reply: + return Reply( + body=json.dumps( + {"object": "list", "data": [{"id": model, "object": "model", "created": 1, "owned_by": "openai"}]} + ).encode() + ) + + +def drain_contract_requests(wire: Wire) -> tuple[Request, ...]: + requests: Final = wire.drain() + discovery: Final = tuple(request for request in requests if request.target == "/v1/models") + assert all(request.method == "GET" and request.body == b"" for request in discovery), requests + return tuple(request for request in requests if request.target != "/v1/models") + + +def _deployment(scenario: Scenario, wire: Wire) -> str: + model: Final = scenario.model( + model=f"openai/{_MODEL}", + api_key=_API_KEY, + api_base=f"{wire.url}/v1", + ) + wait_for_worker_readiness(scenario.gateway, model, 1, _PROXY_WORKERS) + return model + + +def test_responses_compact_routes_forward_complete_request_and_typed_items(gateway: Gateway) -> None: + reasoning_item: Final = { + "id": f"rs_{uuid.uuid4().hex}", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Prior reasoning"}], + "encrypted_content": "synthetic-encrypted-content", + } + function_output_item: Final = { + "type": "function_call_output", + "call_id": "call_previous", + "output": "The record is ready.", + } + requests: Final = ( + { + "input": "Compact this conversation.", + }, + { + "input": [reasoning_item, function_output_item], + "instructions": "Preserve the latest decision.", + "tools": [_FUNCTION_TOOL], + }, + ) + output_items: Final = [ + { + "id": f"rs_compact_{uuid.uuid4().hex}", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Compacted state"}], + "encrypted_content": "synthetic-compact-state", + }, + { + "id": "fc_output_compacted", + "type": "function_call_output", + "call_id": "call_previous", + "output": "The record is ready.", + "status": "completed", + }, + ] + completed_id: Final = f"resp_compact_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + assert request.method == "POST" + assert request.headers["authorization"] == f"Bearer {_API_KEY}", dict(request.headers) + body: Final = _JSON_OBJECT.validate_json(request.body) + compact_params: Final = ( + { + "model": _MODEL, + **requests[0], + } + if body.get("input") == requests[0]["input"] + else { + "model": _MODEL, + **requests[1], + } + ) + assert body == compact_params, request.body + return Reply( + body=json.dumps( + { + "id": completed_id, + "created_at": 1, + "object": "response.compaction", + "output": output_items, + "usage": { + "input_tokens": 7, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 3, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 10, + }, + } + ).encode() + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + for alias in _ALIASES: + for request_body in requests: + response: Final = gateway.request( + "POST", + alias, + {"model": model, **request_body}, + ) + assert response.status_code == 200, response.text + compacted: Final = CompactedResponse.model_validate_json(response.content) + assert compacted.id != completed_id, compacted.model_dump_json() + assert same_response(compacted.id, completed_id), compacted.model_dump_json() + assert compacted.model_dump(mode="json") == { + "id": compacted.id, + "created_at": 1, + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": None, + "model": model, + "object": "response.compaction", + "output": [ + { + "id": output_items[0]["id"], + "summary": [{"text": "Compacted state", "type": "summary_text"}], + "type": "reasoning", + "content": None, + "encrypted_content": "synthetic-compact-state", + "status": None, + }, + { + "id": "fc_output_compacted", + "call_id": "call_previous", + "output": "The record is ready.", + "status": "completed", + "type": "function_call_output", + "created_by": None, + }, + ], + "parallel_tool_calls": None, + "temperature": None, + "tool_choice": None, + "tools": None, + "top_p": None, + "max_output_tokens": None, + "previous_response_id": None, + "reasoning": None, + "status": None, + "text": None, + "truncation": None, + "usage": { + "input_tokens": 7, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": None, + "video_tokens": None, + }, + "output_tokens": 3, + "output_tokens_details": { + "audio_tokens": None, + "reasoning_tokens": 0, + "text_tokens": None, + }, + "total_tokens": 10, + "cost": None, + }, + "user": None, + "store": None, + }, response.text + assert [request.target for request in drain_contract_requests(wire)] == ["/v1/responses/compact"] * 6 + + +@pytest.mark.parametrize("prefix", ("/v1", "", "/openai/v1")) +def test_sdk_compact_reaches_the_compact_endpoint_with_its_full_body(gateway: Gateway, prefix: str) -> None: + completed_id: Final = f"resp_compact_sdk_{uuid.uuid4().hex}" + input_items: Final = [ + {"role": "user", "content": [{"type": "input_text", "text": "Summarize the patch so far."}]}, + { + "id": f"rs_{uuid.uuid4().hex}", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Prior reasoning"}], + "encrypted_content": "synthetic-encrypted-content", + }, + {"type": "function_call_output", "call_id": "call_previous", "output": "The record is ready."}, + ] + output_item: Final = { + "id": f"rs_compact_{uuid.uuid4().hex}", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Compacted state"}], + "encrypted_content": "synthetic-compact-state", + } + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + assert (request.method, request.target) == ("POST", "/v1/responses/compact"), request.target + assert request.headers["authorization"] == f"Bearer {_API_KEY}", dict(request.headers) + assert _JSON_OBJECT.validate_json(request.body) == { + "model": _MODEL, + "input": input_items, + "instructions": "Preserve the latest decision.", + }, request.body + return Reply( + body=json.dumps( + { + "id": completed_id, + "created_at": 1, + "object": "response.compaction", + "output": [output_item], + "usage": { + "input_tokens": 7, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 3, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 10, + }, + } + ).encode() + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + client: Final = openai.OpenAI(base_url=f"{gateway.client.base_url}{prefix}", api_key=gateway.key, max_retries=0) + compacted: Final = client.responses.compact( + model=model, + input=input_items, + instructions="Preserve the latest decision.", + ) + assert compacted.id != completed_id and same_response(compacted.id, completed_id), compacted.model_dump_json() + assert compacted.model_dump(mode="json", exclude_none=True) == { + "id": compacted.id, + "created_at": 1, + "model": model, + "object": "response.compaction", + "output": [output_item], + "usage": { + "input_tokens": 7, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 3, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 10, + }, + }, compacted.model_dump_json() + assert [(request.method, request.target) for request in drain_contract_requests(wire)] == [ + ("POST", "/v1/responses/compact") + ] + + +@pytest.mark.parametrize("prefix", ("/v1", "", "/openai/v1")) +def test_compact_previous_response_id_reaches_the_provider_as_the_raw_id( + gateway: Gateway, + prefix: str, +) -> None: + pytest.skip("BUG: compact forwards the proxy-encrypted previous_response_id instead of the raw provider id") + raw_id: Final = f"resp_compact_prev_{uuid.uuid4().hex}" + completed_id: Final = f"resp_compact_result_{uuid.uuid4().hex}" + input_items: Final = [{"role": "user", "content": [{"type": "input_text", "text": "Summarize the patch so far."}]}] + output_item: Final = { + "id": f"rs_compact_{uuid.uuid4().hex}", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Compacted patch"}], + "encrypted_content": "synthetic-compact-state", + } + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + if request.method == "POST" and request.target == "/v1/responses": + return Reply( + body=json.dumps( + { + "id": raw_id, + "object": "response", + "created_at": 1, + "status": "completed", + "model": _MODEL, + "output": [], + } + ).encode() + ) + return Reply( + body=json.dumps( + { + "id": completed_id, + "created_at": 1, + "object": "response.compaction", + "output": [output_item], + "usage": { + "input_tokens": 7, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 3, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 10, + }, + } + ).encode() + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + client: Final = openai.OpenAI( + base_url=f"{gateway.client.base_url}{prefix}", + api_key=gateway.key, + max_retries=0, + ) + created: Final = client.responses.create(model=model, input="create before compact") + assert created.id != raw_id, created.model_dump_json() + compact_response: Final = client.responses.with_raw_response.compact( + model=model, + previous_response_id=created.id, + input=input_items, + ) + compacted: Final = compact_response.parse() + assert compacted.object == "response.compaction", compact_response.http_response.text + assert compacted.created_at == 1, compact_response.http_response.text + assert same_response(compacted.id, completed_id), compact_response.http_response.text + assert len(compacted.output) == 1, compact_response.http_response.text + assert compacted.output[0].type == "reasoning", compact_response.http_response.text + assert compacted.output[0].summary[0].text == "Compacted patch", compact_response.http_response.text + requests: Final = drain_contract_requests(wire) + assert [(request.method, request.target) for request in requests] == [ + ("POST", "/v1/responses"), + ("POST", "/v1/responses/compact"), + ], requests + assert _JSON_OBJECT.validate_json(requests[0].body) == { + "model": _MODEL, + "input": "create before compact", + }, requests[0].body + assert _JSON_OBJECT.validate_json(requests[1].body) == { + "model": _MODEL, + "previous_response_id": raw_id, + "input": input_items, + }, compact_response.http_response.text diff --git a/tests/integration/providers/test_responses_core_wire.py b/tests/integration/providers/test_responses_core_wire.py new file mode 100644 index 00000000000..791b2b8783f --- /dev/null +++ b/tests/integration/providers/test_responses_core_wire.py @@ -0,0 +1,406 @@ +from __future__ import annotations + +import json +import os +import uuid +from typing import Final + +import openai +import pytest +from integration._support.client import Gateway, Scenario, object_value, string_value +from integration._support.wire import Reply, Request, Wire, wire_server +from integration._support.worker_readiness import wait_for_worker_readiness +from openai.types.responses import ResponseCompletedEvent +from pydantic import JsonValue, TypeAdapter + +_MODEL: Final = "gpt-5" +_API_KEY: Final = "synthetic-responses-key" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_PROXY_WORKERS: Final = max(1, int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1"))) +_ALIASES: Final = ("/v1", "", "/openai/v1") + + +def _deployments_in_group(gateway: Gateway, group: str) -> int: + entries: Final = gateway.get("/model/info")["data"] + assert isinstance(entries, list), entries + return sum(object_value(entry).get("model_name") == group for entry in entries) + + +def _wait_for_workers(gateway: Gateway, group: str, deployments: int) -> None: + wait_for_worker_readiness(gateway, group, deployments, _PROXY_WORKERS) + + +def model_discovery_reply(model: str) -> Reply: + return Reply( + body=json.dumps( + {"object": "list", "data": [{"id": model, "object": "model", "created": 1, "owned_by": "openai"}]} + ).encode() + ) + + +def drain_contract_requests(wire: Wire) -> tuple[Request, ...]: + requests: Final = wire.drain() + discovery: Final = tuple(request for request in requests if request.target == "/v1/models") + assert all(request.method == "GET" and request.body == b"" for request in discovery), requests + return tuple(request for request in requests if request.target != "/v1/models") + + +def _response(identity: str, status: str = "completed") -> bytes: + return json.dumps( + { + "id": identity, + "object": "response", + "created_at": 1, + "status": status, + "model": _MODEL, + "output": [], + } + ).encode() + + +def _stream(identity: str) -> tuple[bytes, ...]: + return tuple( + f"data: {json.dumps(event)}\n\n".encode() + for event in ( + {"type": "response.created", "response": json.loads(_response(identity, "in_progress"))}, + {"type": "response.completed", "response": json.loads(_response(identity))}, + ) + ) + (b"data: [DONE]\n\n",) + + +def _register_in_group(scenario: Scenario, wire: Wire, group: str, order: int | None = None) -> str: + existing: Final = _deployments_in_group(scenario.gateway, group) + created: Final = scenario.gateway.post( + "/model/new", + { + "model_name": group, + "litellm_params": { + "model": f"openai/{_MODEL}", + "api_key": _API_KEY, + "api_base": f"{wire.url}/v1", + **({} if order is None else {"order": order}), + }, + "model_info": {}, + }, + ) + model_id: Final = string_value(object_value(created["model_info"])["id"]) + scenario.cleanups.callback(scenario.delete_model, model_id) + _wait_for_workers(scenario.gateway, group, existing + 1) + return group + + +def _register(scenario: Scenario, wire: Wire, group: str | None = None) -> str: + if group is not None: + return _register_in_group(scenario, wire, group) + model: Final = scenario.model( + model=f"openai/{_MODEL}", + api_key=_API_KEY, + api_base=f"{wire.url}/v1", + ) + _wait_for_workers(scenario.gateway, model, 1) + return model + + +def _enable_responses_affinity(scenario: Scenario) -> None: + gateway: Final = scenario.gateway + settings: Final = object_value(gateway.get("/router/settings")["current_values"]) + current: Final = settings.get("optional_pre_call_checks") + original: Final = list(current) if isinstance(current, list) else [] + scenario.cleanups.callback( + gateway.post, + "/config/update", + {"router_settings": {"optional_pre_call_checks": original}}, + ) + gateway.post( + "/config/update", + {"router_settings": {"optional_pre_call_checks": [*original, "responses_api_deployment_check"]}}, + ) + + +@pytest.mark.parametrize("prefix", _ALIASES) +@pytest.mark.parametrize("stream", (False, True)) +def test_previous_response_id_uses_the_raw_id_and_affinity( + gateway: Gateway, + prefix: str, + stream: bool, +) -> None: + first_raw_id: Final = f"resp_turn_one_{uuid.uuid4().hex}" + second_raw_id: Final = f"resp_turn_two_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + assert request.method == "POST" + assert request.target == "/v1/responses" + assert request.headers["authorization"] == f"Bearer {_API_KEY}", dict(request.headers) + body: Final = _JSON_OBJECT.validate_json(request.body) + if body["input"] == "turn one": + assert body == {"model": _MODEL, "input": "turn one"}, request.body + return Reply(body=_response(first_raw_id)) + if stream: + return Reply(content_type="text/event-stream", chunks=_stream(second_raw_id)) + return Reply(body=_response(second_raw_id)) + + def unused(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + assert request.method == "POST" + assert request.target == "/v1/responses" + assert request.headers["authorization"] == f"Bearer {_API_KEY}", dict(request.headers) + if stream: + return Reply(content_type="text/event-stream", chunks=_stream(second_raw_id)) + return Reply(body=_response(second_raw_id)) + + with wire_server(respond) as first_wire, wire_server(unused) as second_wire, gateway.scenario() as scenario: + model: Final = f"integration-{uuid.uuid4().hex}" + _enable_responses_affinity(scenario) + assert _register(scenario, first_wire, model) == model + client: Final = openai.OpenAI( + base_url=f"{gateway.client.base_url}{prefix}", + api_key=gateway.key, + max_retries=0, + ) + first: Final = client.responses.create(model=model, input="turn one") + assert first.id.startswith("resp_") and first.id != first_raw_id, first.model_dump_json() + _register_in_group(scenario, second_wire, model, order=1) + second_result: Final = client.responses.create( + model=model, + input="turn two", + previous_response_id=first.id, + stream=stream, + ) + if stream: + events: Final = list(second_result) + completed: Final = events[-1] + assert isinstance(completed, ResponseCompletedEvent), events + assert completed.response.id != second_raw_id, completed.model_dump_json() + assert completed.response.output == [], completed.model_dump_json() + else: + assert second_result.id != second_raw_id, second_result.model_dump_json() + assert second_result.output == [], second_result.model_dump_json() + assert drain_contract_requests(second_wire) == (), "previous_response_id did not preserve deployment affinity" + requests: Final = drain_contract_requests(first_wire) + assert len(requests) == 2, requests + assert [_JSON_OBJECT.validate_json(request.body) for request in requests] == [ + {"model": _MODEL, "input": "turn one"}, + { + "model": _MODEL, + "input": "turn two", + "previous_response_id": first_raw_id, + "stream": stream, + }, + ], [request.body for request in requests] + + +@pytest.mark.parametrize("prefix", _ALIASES) +def test_codex_responses_request_preserves_all_supported_fields(gateway: Gateway, prefix: str) -> None: + response_id: Final = f"resp_codex_{uuid.uuid4().hex}" + input_items: Final = [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "Inspect the requested change."}], + }, + { + "type": "reasoning", + "id": f"rs_{uuid.uuid4().hex}", + "summary": [{"type": "summary_text", "text": "Plan the inspection"}], + "encrypted_content": f"gAAAA-codex-encrypted-{uuid.uuid4().hex}", + }, + { + "type": "function_call", + "call_id": "call_lookup", + "name": "lookup", + "arguments": '{"query": "diff"}', + }, + { + "type": "function_call_output", + "call_id": "call_lookup", + "output": '{"rows": 1}', + }, + { + "type": "custom_tool_call", + "call_id": "call_patch", + "name": "ApplyPatch", + "input": "*** Begin Patch\n*** End Patch", + }, + { + "type": "custom_tool_call_output", + "call_id": "call_patch", + "output": "Done", + }, + ] + function_tool: Final = { + "type": "function", + "name": "lookup", + "description": "Look up a record", + "parameters": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}, + "strict": True, + } + custom_tool: Final = { + "type": "custom", + "name": "ApplyPatch", + "description": "Apply a patch", + "format": {"type": "grammar", "syntax": "lark", "definition": 'start: "ok"'}, + } + web_search_tool: Final = {"type": "web_search"} + expected_body: Final = { + "model": _MODEL, + "input": input_items, + "stream": True, + "include": ["reasoning.encrypted_content"], + "store": False, + "prompt_cache_key": "cache-codex-contract", + "reasoning": {"effort": "high", "summary": "auto"}, + "text": { + "verbosity": "low", + "format": { + "type": "json_schema", + "name": "patch_summary", + "strict": True, + "schema": { + "type": "object", + "properties": {"summary": {"type": "string"}}, + "required": ["summary"], + "additionalProperties": False, + }, + }, + }, + "tools": [function_tool, custom_tool, web_search_tool], + "tool_choice": "auto", + "parallel_tool_calls": True, + "instructions": "Return concise structured output.", + } + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + assert request.method == "POST" + assert request.target == "/v1/responses" + assert request.headers["authorization"] == f"Bearer {_API_KEY}", dict(request.headers) + _JSON_OBJECT.validate_json(request.body) + return Reply(content_type="text/event-stream", chunks=_stream(response_id)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _register(scenario, wire) + client: Final = openai.OpenAI( + base_url=f"{gateway.client.base_url}{prefix}", + api_key=gateway.key, + max_retries=0, + ) + stream: Final = client.responses.create( + model=model, + input=input_items, + stream=True, + include=["reasoning.encrypted_content"], + store=False, + prompt_cache_key="cache-codex-contract", + reasoning={"effort": "high", "summary": "auto"}, + text={ + "verbosity": "low", + "format": { + "type": "json_schema", + "name": "patch_summary", + "strict": True, + "schema": { + "type": "object", + "properties": {"summary": {"type": "string"}}, + "required": ["summary"], + "additionalProperties": False, + }, + }, + }, + tools=[function_tool, custom_tool, web_search_tool], + tool_choice="auto", + parallel_tool_calls=True, + instructions="Return concise structured output.", + ) + events: Final = list(stream) + completed: Final = events[-1] + assert isinstance(completed, ResponseCompletedEvent), events + assert completed.response.status == "completed", completed.model_dump_json() + assert completed.response.output == [], completed.model_dump_json() + requests: Final = drain_contract_requests(wire) + assert len(requests) == 1, requests + assert _JSON_OBJECT.validate_json(requests[0].body) == expected_body, requests[0].body + assert [request.target for request in requests] == ["/v1/responses"] + + +@pytest.mark.parametrize("prefix", _ALIASES) +@pytest.mark.parametrize("case", ("mixed", "many")) +def test_responses_metadata_reaches_the_provider_as_typed( + gateway: Gateway, + prefix: str, + case: str, +) -> None: + metadata: Final = ( + { + "string_value": "keep this", + "integer_value": 7, + "boolean_value": True, + "nested_value": {"key": "drop this"}, + "second_string": "keep this too", + } + if case == "mixed" + else {f"key_{index:02d}": f"value_{index:02d}" for index in range(17)} + ) + expected_metadata: Final = dict( + tuple((key, value) for key, value in metadata.items() if isinstance(value, str))[:16] + ) + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + assert (request.method, request.target) == ("POST", "/v1/responses"), request.target + return Reply(body=_response(f"resp_metadata_{uuid.uuid4().hex}")) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _register(scenario, wire) + response: Final = gateway.request( + "POST", + f"{prefix}/responses", + { + "model": model, + "input": "metadata propagation", + "metadata": metadata, + }, + ) + assert response.status_code == 200, response.text + requests: Final = drain_contract_requests(wire) + assert len(requests) == 1, requests + assert _JSON_OBJECT.validate_json(requests[0].body) == { + "model": _MODEL, + "input": "metadata propagation", + "metadata": expected_metadata, + }, requests[0].body + + +def test_conversation_is_forwarded_as_a_separate_responses_parameter(gateway: Gateway) -> None: + pytest.skip("BUG: POST /v1/responses drops the conversation param; upstream receives the body without it") + conversation: Final = {"id": f"conv_{uuid.uuid4().hex}"} + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + return Reply(body=_response(f"resp_conversation_{uuid.uuid4().hex}")) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _register(scenario, wire) + client: Final = openai.OpenAI( + base_url=f"{gateway.client.base_url}/v1", + api_key=gateway.key, + max_retries=0, + ) + response: Final = client.responses.create( + model=model, + input="continue this conversation", + conversation=conversation, + ) + assert response.object == "response", response.model_dump_json() + requests: Final = drain_contract_requests(wire) + assert [(request.method, request.target) for request in requests] == [("POST", "/v1/responses")], requests + assert _JSON_OBJECT.validate_json(requests[0].body) == { + "model": _MODEL, + "input": "continue this conversation", + "conversation": conversation, + }, requests[0].body diff --git a/tests/integration/providers/test_responses_lifecycle_wire.py b/tests/integration/providers/test_responses_lifecycle_wire.py new file mode 100644 index 00000000000..6509c8ced96 --- /dev/null +++ b/tests/integration/providers/test_responses_lifecycle_wire.py @@ -0,0 +1,583 @@ +from __future__ import annotations + +import json +import os +import uuid +from typing import Final, Literal +from urllib.parse import parse_qsl, urlsplit + +import openai +import pytest +from integration._support.client import Gateway, Scenario, eventually, object_value, string_value +from integration._support.responses_vendor import same_response +from integration._support.wire import Reply, Request, Wire, wire_server +from integration._support.worker_readiness import wait_for_worker_readiness +from openai.types.responses import Response, ResponseStreamEvent +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +_MODEL: Final = "gpt-5" +_API_KEY: Final = "synthetic-responses-key" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_PROXY_WORKERS: Final = max(1, int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1"))) +_ALIASES: Final = ("/v1", "", "/openai/v1") + + +class _DeleteResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + object: Literal["response"] + deleted: bool + + +def _deployments_in_group(gateway: Gateway, group: str) -> int: + entries: Final = gateway.get("/model/info")["data"] + assert isinstance(entries, list), entries + return sum(object_value(entry).get("model_name") == group for entry in entries) + + +def _wait_for_workers(gateway: Gateway, group: str, deployments: int) -> None: + wait_for_worker_readiness(gateway, group, deployments, _PROXY_WORKERS) + + +def model_discovery_reply(model: str) -> Reply: + return Reply( + body=json.dumps( + {"object": "list", "data": [{"id": model, "object": "model", "created": 1, "owned_by": "openai"}]} + ).encode() + ) + + +def drain_contract_requests(wire: Wire) -> tuple[Request, ...]: + requests: Final = wire.drain() + discovery: Final = tuple(request for request in requests if request.target == "/v1/models") + assert all(request.method == "GET" and request.body == b"" for request in discovery), requests + return tuple(request for request in requests if request.target != "/v1/models") + + +def _retrieve_or_missing_deployment(client: openai.OpenAI, response_id: str) -> Response | openai.BadRequestError: + try: + return client.responses.retrieve(response_id) + except openai.BadRequestError as refusal: + if "no healthy deployments" not in str(refusal).lower(): + raise + return refusal + + +def _response(identity: str, status: str = "completed") -> bytes: + return json.dumps( + { + "id": identity, + "object": "response", + "created_at": 1, + "status": status, + "model": _MODEL, + "output": [], + } + ).encode() + + +def _stream(identity: str) -> tuple[bytes, ...]: + return tuple( + f"data: {json.dumps(event)}\n\n".encode() + for event in ( + {"type": "response.created", "response": json.loads(_response(identity, "in_progress"))}, + {"type": "response.completed", "response": json.loads(_response(identity))}, + ) + ) + (b"data: [DONE]\n\n",) + + +def _deployment(scenario: Scenario, wire: Wire, api_key: str = _API_KEY) -> str: + model: Final = scenario.model( + model=f"openai/{_MODEL}", + api_key=api_key, + api_base=f"{wire.url}/v1", + ) + _wait_for_workers(scenario.gateway, model, 1) + return model + + +def _stream_response_events( + client: openai.OpenAI, response_id: str +) -> tuple[tuple[ResponseStreamEvent, ...], str | None]: + try: + return ( + tuple( + client.responses.retrieve( + response_id, + stream=True, + starting_after=3, + include=["reasoning.encrypted_content"], + ) + ), + None, + ) + except openai.InternalServerError as error: + return (), error.response.text + + +def _register_preferred_in_group(scenario: Scenario, wire: Wire, group: str, api_key: str) -> None: + existing: Final = _deployments_in_group(scenario.gateway, group) + created: Final = scenario.gateway.post( + "/model/new", + { + "model_name": group, + "litellm_params": { + "model": f"openai/{_MODEL}", + "api_key": api_key, + "api_base": f"{wire.url}/v1", + "order": 1, + }, + "model_info": {}, + }, + ) + deployment_id: Final = string_value(object_value(created["model_info"])["id"]) + scenario.cleanups.callback(scenario.delete_model, deployment_id) + _wait_for_workers(scenario.gateway, group, existing + 1) + + +def _client(gateway: Gateway, key: str, prefix: str) -> openai.OpenAI: + return openai.OpenAI( + base_url=f"{gateway.client.base_url}{prefix}", + api_key=key, + max_retries=0, + ) + + +@pytest.mark.parametrize("prefix", _ALIASES) +def test_retrieve_routes_to_the_deployment_that_created_the_response( + gateway: Gateway, + prefix: str, +) -> None: + raw_id: Final = f"resp_affinity_{uuid.uuid4().hex}" + wire_b_key: Final = "synthetic-responses-key-b" + wire_a_key: Final = "synthetic-responses-key-a" + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + if request.method == "POST": + return Reply(body=_response(raw_id)) + return Reply(body=_response(raw_id)) + + with wire_server(respond) as wire_b, wire_server(respond) as wire_a, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire_b, wire_b_key) + client: Final = _client(gateway, gateway.key, prefix) + created: Final = client.responses.create(model=model, input="create for affinity") + client_id: Final = created.id + assert client_id.startswith("resp_") and client_id != raw_id, created.model_dump_json() + _register_preferred_in_group(scenario, wire_a, model, wire_a_key) + retrieved: Final = client.responses.retrieve(client_id) + assert retrieved.id != raw_id, retrieved.model_dump_json() + assert same_response(retrieved.id, client_id), retrieved.model_dump_json() + assert retrieved.model_dump(mode="json", exclude_none=True) == { + "id": retrieved.id, + "object": "response", + "created_at": 1.0, + "status": "completed", + "model": _MODEL, + "output": [], + }, retrieved.model_dump_json() + requests: Final = drain_contract_requests(wire_b) + assert [(request.method, request.target, request.body) for request in requests] == [ + ("POST", "/v1/responses", requests[0].body), + ("GET", f"/v1/responses/{raw_id}", b""), + ], requests + assert _JSON_OBJECT.validate_json(requests[0].body) == { + "model": _MODEL, + "input": "create for affinity", + }, requests[0].body + assert all(request.headers["authorization"] == f"Bearer {wire_b_key}" for request in requests), requests + assert requests[1].body == b"", requests[1] + assert drain_contract_requests(wire_a) == (), ( + f"{prefix or '/responses'} did not preserve response deployment affinity" + ) + + +@pytest.mark.parametrize("prefix", _ALIASES) +def test_retrieve_returns_the_id_the_client_holds(gateway: Gateway, prefix: str) -> None: + pytest.skip( + "BUG: GET /v1/responses/{id} returns a re-encrypted id that differs byte-for-byte from the id the client sent" + ) + raw_id: Final = f"resp_retrieve_id_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + assert request.headers["authorization"] == f"Bearer {_API_KEY}", dict(request.headers) + if request.method == "POST": + assert request.target == "/v1/responses" + assert _JSON_OBJECT.validate_json(request.body) == {"model": _MODEL, "input": "retrieve client id"}, ( + request.body + ) + return Reply(body=_response(raw_id)) + assert request.method == "GET" + assert request.target == f"/v1/responses/{raw_id}", request.target + assert request.body == b"" + return Reply(body=_response(raw_id)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + client: Final = _client(gateway, gateway.key, prefix) + created: Final = client.responses.create(model=model, input="retrieve client id") + client_id: Final = created.id + retrieved: Final = client.responses.retrieve(client_id) + requests: Final = drain_contract_requests(wire) + assert [(request.method, request.target, request.body) for request in requests] == [ + ("POST", "/v1/responses", requests[0].body), + ("GET", f"/v1/responses/{raw_id}", b""), + ], requests + assert _JSON_OBJECT.validate_json(requests[0].body) == { + "model": _MODEL, + "input": "retrieve client id", + }, requests[0].body + assert retrieved.id == client_id, (retrieved.model_dump_json(), requests) + + +@pytest.mark.parametrize("prefix", _ALIASES) +def test_streamed_response_ids_are_stable_and_retrievable_on_each_alias( + gateway: Gateway, + prefix: str, +) -> None: + raw_id: Final = f"resp_stream_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + assert request.headers["authorization"] == f"Bearer {_API_KEY}", dict(request.headers) + if request.method == "POST": + assert request.target == "/v1/responses" + assert _JSON_OBJECT.validate_json(request.body) == { + "model": _MODEL, + "input": "stream lifecycle", + "stream": True, + }, request.body + return Reply(content_type="text/event-stream", chunks=_stream(raw_id)) + return Reply(body=_response(raw_id)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + team: Final = scenario.team(models=[model]) + user: Final = scenario.member(team) + key: Final = scenario.key(team_id=team, user_id=user, models=[model]) + client: Final = _client(gateway, key, prefix) + stream: Final = client.responses.create(model=model, input="stream lifecycle", stream=True) + events: Final = list(stream) + response_ids: Final = tuple(event.response.id for event in events if hasattr(event, "response")) + assert response_ids and all(response_id == response_ids[0] for response_id in response_ids), events + streamed_id: Final = response_ids[0] + assert streamed_id.startswith("resp_") and streamed_id != raw_id, events + retrieved: Final = eventually( + lambda: _retrieve_or_missing_deployment(client, streamed_id), + lambda outcome: not isinstance(outcome, openai.BadRequestError), + seconds=60, + ) + assert isinstance(retrieved, Response), retrieved + assert same_response(retrieved.id, streamed_id), retrieved.model_dump_json() + assert retrieved.model_dump(mode="json", exclude_none=True) == { + "id": retrieved.id, + "object": "response", + "created_at": 1.0, + "status": "completed", + "model": _MODEL, + "output": [], + }, retrieved.model_dump_json() + requests: Final = drain_contract_requests(wire) + assert [(request.method, urlsplit(request.target).path, request.body) for request in requests] == [ + ("POST", "/v1/responses", requests[0].body), + ("GET", f"/v1/responses/{raw_id}", b""), + ], requests + assert _JSON_OBJECT.validate_json(requests[0].body) == { + "model": _MODEL, + "input": "stream lifecycle", + "stream": True, + }, requests[0].body + assert all(request.headers["authorization"] == f"Bearer {_API_KEY}" for request in requests), requests + + +@pytest.mark.parametrize("prefix", _ALIASES) +def test_retrieve_query_reaches_the_provider(gateway: Gateway, prefix: str) -> None: + pytest.skip( + "BUG: GET /v1/responses/{id} drops include and starting_after; streamed GET returns 500 for provider SSE" + ) + normal_raw_id: Final = f"resp_query_normal_{uuid.uuid4().hex}" + stream_raw_id: Final = f"resp_query_stream_{uuid.uuid4().hex}" + normal_include: Final = ["message.output_text.logprobs", "reasoning.encrypted_content"] + stream_include: Final = ["reasoning.encrypted_content"] + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + if request.method == "POST" and urlsplit(request.target).path == "/v1/responses": + body: Final = _JSON_OBJECT.validate_json(request.body) + identity: Final = stream_raw_id if body.get("input") == "query retrieve stream" else normal_raw_id + return Reply(body=_response(identity)) + if urlsplit(request.target).path == f"/v1/responses/{stream_raw_id}": + return Reply(content_type="text/event-stream", chunks=_stream(stream_raw_id)) + return Reply(body=_response(normal_raw_id)) + + with wire_server(respond) as wire: + sdk_client: Final = openai.OpenAI(base_url=f"{wire.url}/v1", api_key=_API_KEY, max_retries=0) + sdk_client.responses.retrieve(normal_raw_id, include=normal_include) + tuple( + sdk_client.responses.retrieve( + stream_raw_id, + stream=True, + starting_after=3, + include=stream_include, + ) + ) + sdk_requests: Final = drain_contract_requests(wire) + assert [request.method for request in sdk_requests] == ["GET", "GET"], sdk_requests + expected_queries: Final = tuple( + tuple(sorted(parse_qsl(urlsplit(request.target).query, keep_blank_values=True))) for request in sdk_requests + ) + + with gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + client: Final = _client(gateway, gateway.key, prefix) + normal_created: Final = client.responses.create(model=model, input="query retrieve normal") + stream_created: Final = client.responses.create(model=model, input="query retrieve stream") + retrieved_response: Final = client.responses.with_raw_response.retrieve( + normal_created.id, + include=normal_include, + ) + retrieved: Final = retrieved_response.parse() + assert retrieved.id != normal_raw_id and same_response(retrieved.id, normal_created.id), ( + retrieved_response.text + ) + assert retrieved.model_dump(mode="json", exclude_none=True) == { + "id": retrieved.id, + "object": "response", + "created_at": 1.0, + "status": "completed", + "model": _MODEL, + "output": [], + }, retrieved_response.text + stream_result: Final = _stream_response_events(client, stream_created.id) + stream_events: Final = stream_result[0] + stream_error_text: Final = stream_result[1] + if stream_error_text is not None: + requests: Final = drain_contract_requests(wire) + assert [(request.method, urlsplit(request.target).path, request.body) for request in requests[2:]] == [ + ("GET", f"/v1/responses/{normal_raw_id}", b""), + ("GET", f"/v1/responses/{stream_raw_id}", b""), + ], stream_error_text + actual_queries: Final = tuple( + tuple(sorted(parse_qsl(urlsplit(request.target).query, keep_blank_values=True))) + for request in requests[2:] + ) + assert actual_queries == expected_queries, ( + requests[2:], + actual_queries, + expected_queries, + stream_error_text, + ) + pytest.fail(f"Streaming retrieval returned an API error: {stream_error_text}") + completed_events: Final = tuple(event for event in stream_events if event.type == "response.completed") + assert len(completed_events) == 1, stream_events + completed: Final = completed_events[0] + assert completed.response.id != stream_raw_id and same_response(completed.response.id, stream_created.id), ( + stream_events + ) + assert completed.model_dump(mode="json", exclude_none=True) == { + "type": "response.completed", + "response": { + "id": completed.response.id, + "object": "response", + "created_at": 1.0, + "status": "completed", + "model": _MODEL, + "output": [], + }, + }, stream_events + requests: Final = drain_contract_requests(wire) + assert [(request.method, urlsplit(request.target).path, request.body) for request in requests] == [ + ("POST", "/v1/responses", requests[0].body), + ("POST", "/v1/responses", requests[1].body), + ("GET", f"/v1/responses/{normal_raw_id}", b""), + ("GET", f"/v1/responses/{stream_raw_id}", b""), + ], requests + assert _JSON_OBJECT.validate_json(requests[0].body) == { + "model": _MODEL, + "input": "query retrieve normal", + }, requests[0].body + assert _JSON_OBJECT.validate_json(requests[1].body) == { + "model": _MODEL, + "input": "query retrieve stream", + }, requests[1].body + actual_queries: Final = tuple( + tuple(sorted(parse_qsl(urlsplit(request.target).query, keep_blank_values=True))) + for request in requests[2:] + ) + assert actual_queries == expected_queries, requests + + +@pytest.mark.parametrize("prefix", _ALIASES) +def test_delete_forwards_the_provider_id_once_and_reports_deletion( + gateway: Gateway, + prefix: str, +) -> None: + raw_id: Final = f"resp_delete_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + assert request.headers["authorization"] == f"Bearer {_API_KEY}", dict(request.headers) + if request.method == "POST": + assert request.target == "/v1/responses" + assert _JSON_OBJECT.validate_json(request.body) == {"model": _MODEL, "input": "delete lifecycle"}, ( + request.body + ) + return Reply(body=_response(raw_id)) + assert request.method == "DELETE" + assert request.body == b"" + return Reply(body=json.dumps({"id": raw_id, "object": "response", "deleted": True}).encode()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + client: Final = _client(gateway, gateway.key, prefix) + created: Final = client.responses.create(model=model, input="delete lifecycle") + deleted: Final = client.responses.with_raw_response.delete(created.id) + assert deleted.status_code == 200, deleted.text + parsed: Final = _DeleteResponse.model_validate_json(deleted.content) + assert same_response(parsed.id, created.id), deleted.text + assert parsed.model_dump(mode="json", exclude={"id"}) == { + "object": "response", + "deleted": True, + }, deleted.text + requests: Final = drain_contract_requests(wire) + assert [(request.method, request.target, request.body) for request in requests] == [ + ("POST", "/v1/responses", requests[0].body), + ("DELETE", f"/v1/responses/{raw_id}", b""), + ], requests + + +@pytest.mark.parametrize("prefix", _ALIASES) +def test_delete_returns_the_client_held_id_not_the_provider_id( + gateway: Gateway, + prefix: str, +) -> None: + pytest.skip("BUG: DELETE /v1/responses/{id} returns the raw provider id instead of the proxy-issued id") + raw_id: Final = f"resp_delete_id_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + assert request.headers["authorization"] == f"Bearer {_API_KEY}", dict(request.headers) + if request.method == "POST": + assert request.target == "/v1/responses" + assert _JSON_OBJECT.validate_json(request.body) == {"model": _MODEL, "input": "delete client id"}, ( + request.body + ) + return Reply(body=_response(raw_id)) + assert request.method == "DELETE" + assert request.target == f"/v1/responses/{raw_id}", request.target + assert request.body == b"" + return Reply(body=json.dumps({"id": raw_id, "object": "response", "deleted": True}).encode()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + client: Final = _client(gateway, gateway.key, prefix) + created: Final = client.responses.create(model=model, input="delete client id") + deleted: Final = client.responses.with_raw_response.delete(created.id) + assert deleted.status_code == 200, deleted.text + parsed: Final = _DeleteResponse.model_validate_json(deleted.content) + requests: Final = drain_contract_requests(wire) + assert [(request.method, request.target, request.body) for request in requests] == [ + ("POST", "/v1/responses", requests[0].body), + ("DELETE", f"/v1/responses/{raw_id}", b""), + ], requests + assert _JSON_OBJECT.validate_json(requests[0].body) == { + "model": _MODEL, + "input": "delete client id", + }, requests[0].body + assert parsed.id != raw_id, (deleted.text, requests) + assert parsed.model_dump(mode="json") == { + "id": created.id, + "object": "response", + "deleted": True, + }, deleted.text + + +@pytest.mark.parametrize("prefix", _ALIASES) +def test_cancel_forwards_provider_id_and_returns_cancelled_response( + gateway: Gateway, + prefix: str, +) -> None: + raw_id: Final = f"resp_cancel_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + assert request.headers["authorization"] == f"Bearer {_API_KEY}", dict(request.headers) + if request.method == "POST" and request.target == "/v1/responses": + assert _JSON_OBJECT.validate_json(request.body) == { + "model": _MODEL, + "input": "cancel lifecycle", + "background": True, + }, request.body + return Reply(body=_response(raw_id, "queued")) + assert request.body in (b"", b"{}") + return Reply(body=_response(raw_id, "cancelled")) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + client: Final = _client(gateway, gateway.key, prefix) + created: Final = client.responses.create(model=model, input="cancel lifecycle", background=True) + cancelled: Final = client.responses.cancel(created.id) + assert cancelled.id != raw_id and same_response(cancelled.id, created.id), cancelled.model_dump_json() + assert cancelled.model_dump(mode="json", exclude_none=True) == { + "id": cancelled.id, + "object": "response", + "created_at": 1.0, + "status": "cancelled", + "model": _MODEL, + "output": [], + }, cancelled.model_dump_json() + requests: Final = drain_contract_requests(wire) + assert [(request.method, request.target) for request in requests] == [ + ("POST", "/v1/responses"), + ("POST", f"/v1/responses/{raw_id}/cancel"), + ], requests + assert requests[1].body == b"{}", requests[1] + + +@pytest.mark.parametrize("prefix", _ALIASES) +def test_input_items_pagination_query_reaches_the_provider(gateway: Gateway, prefix: str) -> None: + pytest.skip( + "BUG: GET /v1/responses/{id}/input_items drops pagination and include query params; " + "upstream receives limit=20&order=desc without after or include" + ) + raw_id: Final = f"resp_input_items_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + if request.method == "POST": + return Reply(body=_response(raw_id)) + return Reply(body=b'{"data":[],"has_more":false,"object":"list"}') + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + client: Final = _client(gateway, gateway.key, prefix) + created: Final = client.responses.create(model=model, input="list response input items") + page: Final = client.responses.input_items.list( + created.id, + limit=2, + order="asc", + after="item_after", + include=["message.output_text.logprobs"], + ) + assert page.data == [], page.model_dump_json() + requests: Final = drain_contract_requests(wire) + assert [(request.method, request.target) for request in requests] == [ + ("POST", "/v1/responses"), + ( + "GET", + f"/v1/responses/{raw_id}/input_items?limit=2&order=asc&after=item_after" + "&include=message.output_text.logprobs", + ), + ], requests + assert all(request.headers["authorization"] == f"Bearer {_API_KEY}" for request in requests), requests + assert requests[1].body == b"", requests[1] From 595cf066cd4c4850572a2147577e7c6099d6e8a7 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:00:23 +0000 Subject: [PATCH 09/51] test(integration): cover config_utils endpoint contracts (#44737) * test(integration): cover config_utils endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): prove config_utils contracts on the peer, Vault AppRole and cert logins, and admin-view checks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): assert no Vault traffic after config override delete Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): prove the Vault cert login presents the client certificate over TLS Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover the blank plugin_key dashboard edit and the env-only CyberArk readback Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(tests): ruff-sort imports and reformat config_utils integration tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): check plaintext Vault traffic in the cert phase with one drain Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): probe CyberArk runtime state after DELETE with a secret-resolving chat Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): barrier the Langfuse absence check on the shared export channel Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): send the dashboard's blanked CyberArk field and check row, readback and runtime env Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../compatibility/test_proxy_utils.py | 221 ++++ .../configuration/test_effective_settings.py | 798 +++++++++++- .../security/test_config_overrides.py | 1103 +++++++++++++++++ 3 files changed, 2119 insertions(+), 3 deletions(-) create mode 100644 tests/integration/compatibility/test_proxy_utils.py create mode 100644 tests/integration/security/test_config_overrides.py diff --git a/tests/integration/compatibility/test_proxy_utils.py b/tests/integration/compatibility/test_proxy_utils.py new file mode 100644 index 00000000000..6a2cf9c800f --- /dev/null +++ b/tests/integration/compatibility/test_proxy_utils.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +from typing import Final + +import httpx +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import BaseModel, JsonValue + +_TOOL_PARAMETERS: Final = { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], +} +_FUNCTION_TOOL: Final = { + "type": "function", + "function": { + "name": "integration_lookup", + "description": "Look up a synthetic value", + "parameters": _TOOL_PARAMETERS, + }, +} +_FAKE_API_KEY: Final = "my-fake-api-key" +_OPENAI_API_BASE: Final = "https://api.openai.com/v1/" +_ANTHROPIC_API_BASE: Final = "https://api.anthropic.com/v1/messages" + + +class _RawRequestResponse(BaseModel): + raw_request_api_base: str | None = None + raw_request_body: dict[str, JsonValue] | None = None + raw_request_headers: dict[str, str] | None = None + error: str | None = None + + +def _transform_request( + gateway: Gateway, + key: str, + model: str, + stream: bool, +) -> tuple[httpx.Response, _RawRequestResponse]: + request_body: Final = { + "model": model, + "messages": [{"role": "user", "content": "integration transform request"}], + "tools": [_FUNCTION_TOOL], + "tool_choice": "auto", + "max_tokens": 17, + "stream": stream, + } + response: Final = gateway.request( + "POST", + "/utils/transform_request", + {"call_type": "completion", "request_body": request_body}, + key=key, + ) + assert response.status_code == 200, response.text + parsed: Final = _RawRequestResponse.model_validate_json(response.content) + assert parsed.error is None, response.text + return response, parsed + + +def test_transform_request_returns_exact_provider_request_and_rejects_unsafe_bodies(gateway: Gateway) -> None: + openai_body: Final = { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "integration transform request"}], + "max_tokens": 17, + "tools": [_FUNCTION_TOOL], + "tool_choice": "auto", + "extra_body": {}, + "stream": True, + "stream_options": {"include_usage": True}, + } + openai_non_streaming_body: Final = { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "integration transform request"}], + "max_tokens": 17, + "tools": [_FUNCTION_TOOL], + "tool_choice": "auto", + "extra_body": {}, + } + anthropic_body: Final = { + "model": "claude-opus-4-8", + "messages": [{"role": "user", "content": [{"type": "text", "text": "integration transform request"}]}], + "max_tokens": 17, + "tools": [ + { + "type": "custom", + "name": "integration_lookup", + "description": "Look up a synthetic value", + "input_schema": _TOOL_PARAMETERS, + } + ], + "tool_choice": {"type": "auto"}, + } + + def no_destination_traffic(request: Request) -> Reply: + return Reply() + + with gateway.scenario() as scenario, wire_server(no_destination_traffic) as destination: + key: Final = scenario.key() + openai_stream_result: Final = _transform_request(gateway, key, "openai/gpt-4o-mini", True) + openai_stream_response, openai_stream = openai_stream_result + assert openai_stream.raw_request_api_base == _OPENAI_API_BASE, openai_stream_response.text + assert openai_stream.raw_request_body == openai_body, openai_stream_response.text + assert openai_stream.raw_request_headers == {"Authorization": "Be****ey"}, openai_stream_response.text + assert _FAKE_API_KEY not in openai_stream_response.text, "Transform response exposed its synthetic credential" + + openai_non_stream_result: Final = _transform_request(gateway, key, "openai/gpt-4o-mini", False) + openai_non_stream_response, openai_non_stream = openai_non_stream_result + assert openai_non_stream.raw_request_api_base == _OPENAI_API_BASE, openai_non_stream_response.text + assert openai_non_stream.raw_request_body == openai_non_streaming_body, openai_non_stream_response.text + assert _FAKE_API_KEY not in openai_non_stream_response.text, ( + "Transform response exposed its synthetic credential" + ) + + anthropic_result: Final = _transform_request(gateway, key, "anthropic/claude-opus-4-8", False) + anthropic_response, anthropic = anthropic_result + assert anthropic.raw_request_api_base == _ANTHROPIC_API_BASE, anthropic_response.text + assert anthropic.raw_request_body == anthropic_body, anthropic_response.text + assert anthropic.raw_request_headers == { + "anthropic-version": "2023-06-01", + "accept": "application/json", + "content-type": "application/json", + "x-api-key": "my****ey", + }, anthropic_response.text + assert _FAKE_API_KEY not in anthropic_response.text, "Transform response exposed its synthetic credential" + + unsafe_requests: Final = ( + ( + "api_base", + destination.url, + "Rejected Request: api_base is not allowed in request body. Clientside passthrough requires " + "explicit admin opt-in via either `general_settings.allow_client_side_credentials = true` " + "(proxy-wide) or `configurable_clientside_auth_params` on the deployment in your proxy " + "config.yaml. Relevant Issue: " + "https://huntr.com/bounties/4001e1a2-7b7a-4776-a3ae-e6692ec3d997", + ), + ( + "base_url", + destination.url, + "Rejected Request: base_url is not allowed in request body. Clientside passthrough requires " + "explicit admin opt-in via either `general_settings.allow_client_side_credentials = true` " + "(proxy-wide) or `configurable_clientside_auth_params` on the deployment in your proxy " + "config.yaml. Relevant Issue: " + "https://huntr.com/bounties/4001e1a2-7b7a-4776-a3ae-e6692ec3d997", + ), + ( + "model_list", + [{"model_name": "unsafe", "litellm_params": {"api_base": destination.url}}], + "Rejected Request: model_list is not allowed in the request body.", + ), + ) + for field, value, error in unsafe_requests: + rejected: Final = gateway.request( + "POST", + "/utils/transform_request", + { + "call_type": "completion", + "request_body": { + "model": "openai/gpt-4o-mini", + "messages": [{"role": "user", "content": "unsafe transform request"}], + field: value, + }, + }, + key=key, + ) + assert rejected.status_code == 400, rejected.text + assert rejected.json() == {"detail": {"error": error}}, rejected.text + assert _FAKE_API_KEY not in rejected.text, "Unsafe response exposed its synthetic credential" + assert destination.drain() == () + assert destination.drain() == () + + +def test_transform_request_reports_masked_authorization_for_non_streaming_openai(gateway: Gateway) -> None: + pytest.skip("BUG: /utils/transform_request returns empty raw_request_headers for a non-streaming openai completion") + with gateway.scenario() as scenario: + key: Final = scenario.key() + response, parsed = _transform_request(gateway, key, "openai/gpt-4o-mini", False) + assert parsed.raw_request_api_base == _OPENAI_API_BASE, response.text + assert parsed.raw_request_body == { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "integration transform request"}], + "max_tokens": 17, + "tools": [_FUNCTION_TOOL], + "tool_choice": "auto", + "extra_body": {}, + }, response.text + assert parsed.raw_request_headers == {"Authorization": "Be****ey"}, response.text + assert _FAKE_API_KEY not in response.text, "Transform response exposed its synthetic credential" + + +def test_transform_request_reports_stream_flag_for_streaming_anthropic(gateway: Gateway) -> None: + pytest.skip( + "BUG: /utils/transform_request omits stream: true from the anthropic raw_request_body when stream is true" + ) + with gateway.scenario() as scenario: + key: Final = scenario.key() + response, parsed = _transform_request(gateway, key, "anthropic/claude-opus-4-8", True) + assert parsed.raw_request_api_base == _ANTHROPIC_API_BASE, response.text + assert parsed.raw_request_headers == { + "anthropic-version": "2023-06-01", + "accept": "application/json", + "content-type": "application/json", + "x-api-key": "my****ey", + }, response.text + assert parsed.raw_request_body == { + "model": "claude-opus-4-8", + "messages": [{"role": "user", "content": [{"type": "text", "text": "integration transform request"}]}], + "max_tokens": 17, + "tools": [ + { + "type": "custom", + "name": "integration_lookup", + "description": "Look up a synthetic value", + "input_schema": _TOOL_PARAMETERS, + } + ], + "tool_choice": {"type": "auto"}, + "stream": True, + }, response.text + assert _FAKE_API_KEY not in response.text, "Transform response exposed its synthetic credential" diff --git a/tests/integration/configuration/test_effective_settings.py b/tests/integration/configuration/test_effective_settings.py index 52f010843e3..7f6313e209e 100644 --- a/tests/integration/configuration/test_effective_settings.py +++ b/tests/integration/configuration/test_effective_settings.py @@ -1,15 +1,24 @@ import json +import os import uuid +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path from typing import Final import httpx import pytest -from pydantic import BaseModel, JsonValue +from pydantic import BaseModel, JsonValue, TypeAdapter -from tests.integration._support.client import Gateway, object_value, string_value -from tests.integration._support.database import read_rows +from tests.integration._support.client import Gateway, eventually, object_value, string_value +from tests.integration._support.database import read_rows, scratch_database +from tests.integration._support.process import owned_proxy from tests.integration._support.wire import Reply, Request, Wire, wire_server +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + def model_identity(gateway: Gateway, alias: str) -> str: entries: Final = gateway.get("/model/info")["data"] @@ -18,6 +27,789 @@ def model_identity(gateway: Gateway, alias: str) -> str: return string_value(object_value(entry["model_info"])["id"]) +@dataclass(frozen=True, slots=True) +class _ConfigCallbackRig: + first: Gateway + peer: Gateway + database_url: str + generic: Wire + langfuse: Wire + provider: Wire + + +@contextmanager +def _config_callback_rig( + gateway: Gateway, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + provider: Callable[[Request], Reply], + *, + generic_flush_interval: str, +) -> Iterator[_ConfigCallbackRig]: + def langfuse(request: Request) -> Reply: + if request.method == "GET": + assert request.target.startswith("/api/public/projects"), request.target + return Reply(body=b'{"data":[{"id":"integration-project","name":"integration"}]}') + assert request.method == "POST" + assert request.target.startswith("/api/public/otel/v1/traces"), request.target + return Reply(content_type="application/x-protobuf") + + def generic(request: Request) -> Reply: + assert request.method == "POST" + assert request.headers["authorization"] == "Bearer integration-generic-sink-secret" + return Reply() + + with ( + scratch_database() as database_url, + wire_server(provider) as provider_wire, + wire_server(langfuse) as langfuse_wire, + wire_server(generic) as generic_wire, + ): + monkeypatch.setenv("DATABASE_URL", database_url) + environment: Final = { + "DATABASE_URL": database_url, + "DEFAULT_FLUSH_INTERVAL_SECONDS": generic_flush_interval, + "LANGFUSE_FLUSH_INTERVAL": "1", + "LITELLM_STORE_AUDIT_LOGS": "True", + } + removed: Final = ( + "DATABASE_URL_READ_REPLICA", + "GENERIC_LOGGER_ENDPOINT", + "GENERIC_LOGGER_HEADERS", + "LANGFUSE_HOST", + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", + ) + with ( + owned_proxy(gateway, tmp_path, environment, remove_environment=removed) as first, + owned_proxy(gateway, tmp_path, environment, remove_environment=removed) as peer, + ): + yield _ConfigCallbackRig(first, peer, database_url, generic_wire, langfuse_wire, provider_wire) + + +def _config_section(database_url: str, section: str) -> dict[str, JsonValue]: + rows: Final = read_rows( + 'SELECT param_value FROM "LiteLLM_Config" WHERE param_name = %s', + (section,), + database_url=database_url, + ) + assert len(rows) == 1, rows + return object_value(rows[0]["param_value"]) + + +def _callback_entries(response: httpx.Response) -> dict[str, dict[str, JsonValue]]: + body: Final = object_value(response.json()) + callbacks: Final = TypeAdapter(list[dict[str, JsonValue]]).validate_python(body["callbacks"]) + return {string_value(callback["name"]): callback for callback in callbacks} + + +def _callback_entries_match(response: httpx.Response, expected: dict[str, dict[str, JsonValue]]) -> bool: + entries: Final = _callback_entries(response) + return all(entries.get(name) == callback for name, callback in expected.items()) + + +def _completion_reply(text: str) -> Reply: + return Reply( + body=json.dumps( + { + "id": "chatcmpl-" + text, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15}, + } + ).encode() + ) + + +def test_multi_section_config_update_merges_sent_keys_normalizes_callbacks_and_applies_on_the_peer( + gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + marker: Final = "config-update-" + uuid.uuid4().hex + provider_secret: Final = "provider-secret-" + marker + langfuse_public_key: Final = "public-key-" + marker + langfuse_secret_key: Final = "secret-key-" + marker + provider_requests: Final[list[Request]] = [] # mutable-ok: request count distinguishes the retry + + def provider(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/v1/chat/completions" + assert request.headers["authorization"] == f"Bearer {provider_secret}" + body: Final = _JSON_OBJECT.validate_json(request.body) + content: Final = object_value(body["messages"][0])["content"] + assert body == { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": content}], + }, request.body + assert isinstance(content, str) + retry_attempts: Final = sum(marker + "-retry" in prior.body.decode() for prior in provider_requests) + provider_requests.append(request) + if marker + "-retry" in content and retry_attempts == 0: + return Reply( + status=500, + body=b'{"error":{"message":"retry control","type":"server_error","code":"retry_control"}}', + ) + return _completion_reply(content) + + with ( + _config_callback_rig(gateway, tmp_path, monkeypatch, provider, generic_flush_interval="1") as rig, + rig.first.scenario() as scenario, + ): + seed: Final = rig.first.request( + "POST", + "/config/update", + { + "general_settings": {"alerting_threshold": 321}, + "litellm_settings": {"success_callback": ["generic_api"], "drop_params": True}, + "environment_variables": { + "GENERIC_LOGGER_ENDPOINT": rig.generic.url, + "GENERIC_LOGGER_HEADERS": "Authorization=Bearer integration-generic-sink-secret", + }, + "router_settings": {"allowed_fails": 7}, + }, + ) + assert seed.status_code == 200, seed.text + assert seed.json() == {"message": "Config updated successfully"}, seed.text + update: Final = rig.first.request( + "POST", + "/config/update", + { + "general_settings": {"store_prompts_in_spend_logs": True}, + "litellm_settings": {"success_callback": ["Langfuse"]}, + "environment_variables": { + "LANGFUSE_HOST": rig.langfuse.url, + "LANGFUSE_PUBLIC_KEY": langfuse_public_key, + "LANGFUSE_SECRET_KEY": langfuse_secret_key, + }, + "router_settings": {"num_retries": 1}, + }, + ) + assert update.status_code == 200, update.text + assert update.json() == {"message": "Config updated successfully"}, update.text + assert _config_section(rig.database_url, "general_settings") == { + "alerting_threshold": 321, + "store_prompts_in_spend_logs": True, + } + litellm_settings: Final = _config_section(rig.database_url, "litellm_settings") + assert litellm_settings["drop_params"] is True + success_callbacks: Final = TypeAdapter(list[str]).validate_python(litellm_settings["success_callback"]) + assert sorted(success_callbacks) == ["generic_api", "langfuse"] + assert _config_section(rig.database_url, "router_settings") == {"allowed_fails": 7, "num_retries": 1} + environment_values: Final = _config_section(rig.database_url, "environment_variables") + plain_environment: Final = { + "GENERIC_LOGGER_ENDPOINT": rig.generic.url, + "GENERIC_LOGGER_HEADERS": "Authorization=Bearer integration-generic-sink-secret", + "LANGFUSE_HOST": rig.langfuse.url, + "LANGFUSE_PUBLIC_KEY": langfuse_public_key, + "LANGFUSE_SECRET_KEY": langfuse_secret_key, + } + assert set(environment_values) == set(plain_environment), environment_values + assert all( + isinstance(environment_values[name], str) and environment_values[name] != value + for name, value in plain_environment.items() + ), environment_values + assert all(value not in json.dumps(environment_values) for value in plain_environment.values()) + + expected_callbacks: Final[dict[str, dict[str, JsonValue]]] = { + "generic_api": { + "name": "generic_api", + "variables": { + "GENERIC_LOGGER_ENDPOINT": rig.generic.url, + "GENERIC_LOGGER_HEADERS": "Authorization=Bearer integration-generic-sink-secret", + }, + "type": "success", + }, + "langfuse": { + "name": "langfuse", + "variables": { + "LANGFUSE_HOST": rig.langfuse.url, + "LANGFUSE_PUBLIC_KEY": langfuse_public_key, + "LANGFUSE_SECRET_KEY": langfuse_secret_key, + }, + "type": "success", + }, + } + callbacks_first: Final = rig.first.request("GET", "/get/config/callbacks") + assert callbacks_first.status_code == 200, callbacks_first.text + assert _callback_entries_match(callbacks_first, expected_callbacks), callbacks_first.text + callbacks_peer: Final = eventually( + lambda: rig.peer.request("GET", "/get/config/callbacks"), + lambda response: response.status_code == 200 and _callback_entries_match(response, expected_callbacks), + seconds=20, + ) + assert _callback_entries_match(callbacks_peer, expected_callbacks), callbacks_peer.text + + model: Final = scenario.model(api_base=rig.provider.url + "/v1", api_key=provider_secret) + api_key: Final = scenario.key(models=[model]) + chat: Final = rig.peer.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": marker}], "stream": False}, + key=api_key, + ) + assert chat.status_code == 200, chat.text + assert chat.json()["choices"][0]["message"]["content"] == marker + generic_batches: Final[list[Request]] = [] # mutable-ok: the sink queue is drained while waiting + langfuse_batches: Final[list[Request]] = [] # mutable-ok: the sink queue is drained while waiting + + def collect_generic() -> tuple[Request, ...]: + generic_batches.extend(rig.generic.drain()) + return tuple(generic_batches) + + def collect_langfuse() -> tuple[Request, ...]: + langfuse_batches.extend(rig.langfuse.drain()) + return tuple(langfuse_batches) + + generic_deliveries: Final = eventually( + collect_generic, + lambda requests: marker.encode() in b"".join(request.body for request in requests), + seconds=15, + ) + langfuse_deliveries: Final = eventually( + collect_langfuse, + lambda requests: marker.encode() in b"".join(request.body for request in requests), + seconds=15, + ) + assert any(marker.encode() in request.body for request in generic_deliveries) + assert any(marker.encode() in request.body for request in langfuse_deliveries) + + retry: Final = rig.peer.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": marker + "-retry"}], "stream": False}, + key=api_key, + ) + assert retry.status_code == 200, retry.text + assert sum(marker + "-retry" in request.body.decode() for request in provider_requests) == 2 + + +def test_deleted_callback_case_variant_stops_delivery_on_both_workers( + gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + assert os.environ.get("LITELLM_LICENSE"), "LITELLM_LICENSE must be set: audit logs are an enterprise feature" + marker: Final = "callback-delete-" + uuid.uuid4().hex + provider_secret: Final = "provider-secret-" + marker + + def provider(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/v1/chat/completions" + assert request.headers["authorization"] == f"Bearer {provider_secret}" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body == { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": object_value(body["messages"][0])["content"]}], + }, request.body + return _completion_reply(string_value(object_value(body["messages"][0])["content"])) + + with ( + _config_callback_rig(gateway, tmp_path, monkeypatch, provider, generic_flush_interval="3") as rig, + rig.first.scenario() as scenario, + ): + seed: Final = rig.first.request( + "POST", + "/config/update", + { + "litellm_settings": { + "success_callback": ["langfuse", "generic_api"], + "DEFAULT_FLUSH_INTERVAL_SECONDS": 3, + }, + "environment_variables": { + "GENERIC_LOGGER_ENDPOINT": rig.generic.url, + "GENERIC_LOGGER_HEADERS": "Authorization=Bearer integration-generic-sink-secret", + "LANGFUSE_HOST": rig.langfuse.url, + "LANGFUSE_PUBLIC_KEY": "public-key-" + marker, + "LANGFUSE_SECRET_KEY": "secret-key-" + marker, + }, + }, + ) + assert seed.status_code == 200, seed.text + assert seed.json() == {"message": "Config updated successfully"}, seed.text + expected_callbacks: Final[dict[str, dict[str, JsonValue]]] = { + "generic_api": { + "name": "generic_api", + "variables": { + "GENERIC_LOGGER_ENDPOINT": rig.generic.url, + "GENERIC_LOGGER_HEADERS": "Authorization=Bearer integration-generic-sink-secret", + }, + "type": "success", + }, + "langfuse": { + "name": "langfuse", + "variables": { + "LANGFUSE_HOST": rig.langfuse.url, + "LANGFUSE_PUBLIC_KEY": "public-key-" + marker, + "LANGFUSE_SECRET_KEY": "secret-key-" + marker, + }, + "type": "success", + }, + } + callbacks_first_before: Final = rig.first.request("GET", "/get/config/callbacks") + assert callbacks_first_before.status_code == 200, callbacks_first_before.text + assert _callback_entries_match(callbacks_first_before, expected_callbacks), callbacks_first_before.text + callbacks_peer_before: Final = eventually( + lambda: rig.peer.request("GET", "/get/config/callbacks"), + lambda response: response.status_code == 200 and _callback_entries_match(response, expected_callbacks), + seconds=20, + ) + assert _callback_entries_match(callbacks_peer_before, expected_callbacks), callbacks_peer_before.text + model: Final = scenario.model(api_base=rig.provider.url + "/v1", api_key=provider_secret) + api_key: Final = scenario.key(models=[model]) + + for worker, proxy in (("first", rig.first), ("peer", rig.peer)): + positive: Final = proxy.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": marker + "-" + worker}], "stream": False}, + key=api_key, + ) + assert positive.status_code == 200, positive.text + + positive_generic: Final[list[Request]] = [] # mutable-ok: preserve batches while the async sinks flush + positive_langfuse: Final[list[Request]] = [] # mutable-ok: preserve batches while the async sinks flush + + def collect_positive_generic() -> tuple[Request, ...]: + positive_generic.extend(rig.generic.drain()) + return tuple(positive_generic) + + def collect_positive_langfuse() -> tuple[Request, ...]: + positive_langfuse.extend(rig.langfuse.drain()) + return tuple(positive_langfuse) + + positive_markers: Final = (marker + "-first", marker + "-peer") + generic_before: Final = eventually( + collect_positive_generic, + lambda requests: all( + value.encode() in b"".join(request.body for request in requests) for value in positive_markers + ), + seconds=20, + ) + langfuse_before: Final = eventually( + collect_positive_langfuse, + lambda requests: all( + value.encode() in b"".join(request.body for request in requests) for value in positive_markers + ), + seconds=20, + ) + assert all( + marker_value.encode() in b"".join(request.body for request in generic_before) + for marker_value in positive_markers + ) + assert all( + marker_value.encode() in b"".join(request.body for request in langfuse_before) + for marker_value in positive_markers + ) + + deleted: Final = rig.first.request("POST", "/config/callback/delete", {"callback_name": "Langfuse"}) + assert deleted.status_code == 200, deleted.text + deleted_body: Final = object_value(deleted.json()) + assert set(deleted_body) == {"message", "removed_callback", "remaining_callbacks", "deleted_at"}, deleted.text + assert deleted_body["message"] == "Successfully deleted callback: langfuse", deleted.text + assert deleted_body["removed_callback"] == "langfuse", deleted.text + assert deleted_body["remaining_callbacks"] == ["generic_api"], deleted.text + datetime.fromisoformat(string_value(deleted_body["deleted_at"])) + assert _config_section(rig.database_url, "litellm_settings") == { + "DEFAULT_FLUSH_INTERVAL_SECONDS": 3, + "success_callback": ["generic_api"], + } + expected_remaining_callbacks: Final = {"generic_api": expected_callbacks["generic_api"]} + callbacks_first: Final = rig.first.request("GET", "/get/config/callbacks") + assert callbacks_first.status_code == 200, callbacks_first.text + assert _callback_entries_match(callbacks_first, expected_remaining_callbacks), callbacks_first.text + assert "langfuse" not in _callback_entries(callbacks_first), callbacks_first.text + callbacks_peer: Final = eventually( + lambda: rig.peer.request("GET", "/get/config/callbacks"), + lambda response: ( + response.status_code == 200 + and _callback_entries_match(response, expected_remaining_callbacks) + and "langfuse" not in _callback_entries(response) + ), + seconds=20, + ) + assert _callback_entries_match(callbacks_peer, expected_remaining_callbacks), callbacks_peer.text + assert "langfuse" not in _callback_entries(callbacks_peer), callbacks_peer.text + + audit: Final = eventually( + lambda: read_rows( + 'SELECT action, table_name, object_id, before_value, updated_values FROM "LiteLLM_AuditLog" ' + "WHERE table_name = %s AND object_id = %s ORDER BY updated_at DESC LIMIT 1", + ("LiteLLM_Config", "litellm_settings"), + database_url=rig.database_url, + ), + lambda rows: bool(rows), + seconds=10, + ) + assert len(audit) == 1, audit + audit_row: Final = object_value(audit[0]) + assert set(audit_row) == {"action", "table_name", "object_id", "before_value", "updated_values"}, audit_row + assert audit_row["action"] == "deleted", audit_row + assert audit_row["table_name"] == "LiteLLM_Config", audit_row + assert audit_row["object_id"] == "litellm_settings", audit_row + before_value: Final = object_value(audit_row["before_value"]) + assert set(before_value) == {"success_callback"}, audit_row + before_callbacks: Final = TypeAdapter(list[str]).validate_python(before_value["success_callback"]) + assert sorted(before_callbacks) == ["generic_api", "langfuse"], audit_row + assert audit_row["updated_values"] == {"success_callback": ["generic_api"]}, audit_row + + absence_markers: Final = (marker + "-after-first", marker + "-after-peer") + for worker, proxy, absence_marker in zip( + ("first", "peer"), (rig.first, rig.peer), absence_markers, strict=True + ): + absent: Final = proxy.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": absence_marker}], "stream": False}, + key=api_key, + ) + assert absent.status_code == 200, absent.text + + generic_after: Final[list[Request]] = [] # mutable-ok: preserve batches while the async sink flushes + + def collect_after_generic() -> tuple[Request, ...]: + generic_after.extend(rig.generic.drain()) + return tuple(generic_after) + + generic_deliveries: Final = eventually( + collect_after_generic, + lambda requests: all( + value.encode() in b"".join(request.body for request in requests) for value in absence_markers + ), + seconds=25, + ) + assert all( + value.encode() in b"".join(request.body for request in generic_deliveries) for value in absence_markers + ) + langfuse_after: Final[list[Request]] = [] # mutable-ok: preserve batches while the async sink flushes + + def collect_after_langfuse() -> tuple[Request, ...]: + langfuse_after.extend(rig.langfuse.drain()) + return tuple(langfuse_after) + + collect_after_langfuse() + readded: Final = rig.first.request( + "POST", + "/config/update", + { + "litellm_settings": {"success_callback": ["langfuse"]}, + "environment_variables": { + "LANGFUSE_HOST": rig.langfuse.url, + "LANGFUSE_PUBLIC_KEY": "public-key-" + marker, + "LANGFUSE_SECRET_KEY": "secret-key-" + marker, + }, + }, + ) + assert readded.status_code == 200, readded.text + expected_readded_callback: Final = {"langfuse": expected_callbacks["langfuse"]} + callbacks_peer_readded: Final = eventually( + lambda: rig.peer.request("GET", "/get/config/callbacks"), + lambda response: ( + response.status_code == 200 and _callback_entries_match(response, expected_readded_callback) + ), + seconds=20, + ) + assert _callback_entries_match(callbacks_peer_readded, expected_readded_callback), callbacks_peer_readded.text + barrier_markers: Final = (marker + "-barrier-first", marker + "-barrier-peer") + for worker, proxy, barrier_marker in zip( + ("first", "peer"), (rig.first, rig.peer), barrier_markers, strict=True + ): + barrier: Final = proxy.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": barrier_marker}], "stream": False}, + key=api_key, + ) + assert barrier.status_code == 200, barrier.text + + langfuse_deliveries: Final = eventually( + collect_after_langfuse, + lambda requests: all( + value.encode() in b"".join(request.body for request in requests) for value in barrier_markers + ), + seconds=25, + ) + langfuse_bodies: Final = b"".join(request.body for request in langfuse_deliveries) + assert all(value.encode() in langfuse_bodies for value in barrier_markers) + leaked_absence_markers: Final = tuple(value for value in absence_markers if value.encode() in langfuse_bodies) + assert leaked_absence_markers == (), leaked_absence_markers + + missing: Final = rig.first.request("POST", "/config/callback/delete", {"callback_name": "not-a-callback"}) + assert missing.status_code == 404, missing.text + assert missing.json() == {"detail": {"error": "Callback 'not-a-callback' not found in active configuration"}}, ( + missing.text + ) + + +def test_dashboard_field_updates_preserve_plugin_key_and_apply_at_runtime( + gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + assert os.environ.get("LITELLM_LICENSE"), ( + "LITELLM_LICENSE must be set: max_request_size_mb enforcement is an enterprise feature" + ) + plugin_key: Final = "plugin-key-" + uuid.uuid4().hex + plugin_body: Final = {"message": "plugin-" + uuid.uuid4().hex} + passthrough_body: Final = {"message": "passthrough-" + uuid.uuid4().hex} + passthrough_path: Final = "/integration-pt-" + uuid.uuid4().hex + + def plugin(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/rpc" + assert request.headers["authorization"] == f"Bearer {plugin_key}" + assert _JSON_OBJECT.validate_json(request.body) == plugin_body, request.body + return Reply(body=json.dumps({"received": plugin_body}).encode()) + + def passthrough(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/echo" + assert request.headers["x-integration"] == "yes" + assert _JSON_OBJECT.validate_json(request.body) == passthrough_body, request.body + return Reply(body=json.dumps({"received": passthrough_body}).encode()) + + with ( + scratch_database() as database_url, + wire_server(plugin) as plugin_wire, + wire_server(passthrough) as passthrough_wire, + ): + monkeypatch.setenv("DATABASE_URL", database_url) + environment: Final = { + "DATABASE_URL": database_url, + "PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": "600", + } + with ( + owned_proxy(gateway, tmp_path, environment) as candidate, + owned_proxy(gateway, tmp_path, environment) as peer, + ): + plugin_value: Final = [ + { + "name": "integration-plugin", + "display_name": "Integration", + "url": plugin_wire.url, + "plugin_key": plugin_key, + } + ] + plugin_update: Final = candidate.request( + "POST", + "/config/field/update", + {"field_name": "plugins", "field_value": plugin_value, "config_type": "general_settings"}, + ) + assert plugin_update.status_code == 200, plugin_update.text + assert object_value(plugin_update.json())["param_name"] == "general_settings", plugin_update.text + plugin_info: Final = object_value(candidate.get("/config/field/info?field_name=plugins")) + assert plugin_info["field_name"] == "plugins" + assert plugin_info["field_value"] == [ + { + "name": "integration-plugin", + "display_name": "Integration", + "url": plugin_wire.url, + "plugin_key": "***", + } + ] + reposted_plugins: Final = [ + { + "name": "integration-plugin", + "display_name": "Integration updated", + "url": plugin_wire.url, + "plugin_key": "***", + } + ] + repost: Final = candidate.request( + "POST", + "/config/field/update", + {"field_name": "plugins", "field_value": reposted_plugins, "config_type": "general_settings"}, + ) + assert repost.status_code == 200, repost.text + assert _config_section(database_url, "general_settings") == { + "plugins": [ + { + "name": "integration-plugin", + "display_name": "Integration updated", + "url": plugin_wire.url, + "plugin_key": plugin_key, + } + ] + } + blank_key_plugins: Final = [ + { + "name": "integration-plugin", + "display_name": "Integration updated", + "url": plugin_wire.url, + "plugin_key": "", + } + ] + blank_repost: Final = candidate.request( + "POST", + "/config/field/update", + {"field_name": "plugins", "field_value": blank_key_plugins, "config_type": "general_settings"}, + ) + assert blank_repost.status_code == 200, blank_repost.text + assert _config_section(database_url, "general_settings") == { + "plugins": [ + { + "name": "integration-plugin", + "display_name": "Integration updated", + "url": plugin_wire.url, + "plugin_key": plugin_key, + } + ] + } + plugin_response: Final = candidate.request("POST", "/plugin-proxy/integration-plugin/rpc", plugin_body) + assert plugin_response.status_code == 200, plugin_response.text + assert plugin_response.json() == {"received": plugin_body}, plugin_response.text + peer_plugin_response: Final = eventually( + lambda: peer.request("POST", "/plugin-proxy/integration-plugin/rpc", plugin_body), + lambda response: response.status_code == 200 and response.json() == {"received": plugin_body}, + seconds=20, + ) + assert peer_plugin_response.json() == {"received": plugin_body}, peer_plugin_response.text + + passthrough_value: Final = [ + { + "path": passthrough_path, + "target": passthrough_wire.url + "/echo", + "headers": {"x-integration": "yes"}, + "auth": True, + } + ] + passthrough_update: Final = candidate.request( + "POST", + "/config/field/update", + { + "field_name": "pass_through_endpoints", + "field_value": passthrough_value, + "config_type": "general_settings", + }, + ) + assert passthrough_update.status_code == 200, passthrough_update.text + passthrough_response: Final = candidate.request("POST", passthrough_path, passthrough_body) + assert passthrough_response.status_code == 200, passthrough_response.text + assert passthrough_response.json() == {"received": passthrough_body}, passthrough_response.text + peer_passthrough_response: Final = eventually( + lambda: peer.request("POST", passthrough_path, passthrough_body), + lambda response: response.status_code == 200 and response.json() == {"received": passthrough_body}, + seconds=20, + ) + assert peer_passthrough_response.json() == {"received": passthrough_body}, peer_passthrough_response.text + + alerting_args: Final = { + "daily_report_frequency": 43200, + "report_check_interval": 300, + "budget_alert_ttl": 86400, + "outage_alert_ttl": 60, + "region_outage_alert_ttl": 60, + "minor_outage_alert_threshold": 2, + "major_outage_alert_threshold": 5, + "max_outage_alert_list_size": 20, + "daily_spend_per_user_threshold": 25.0, + "monthly_spend_per_user_threshold": 500.0, + "spend_anomaly_multiplier": 3.5, + "spend_anomaly_baseline_days": 14, + "spend_anomaly_min_spend": 12.0, + "user_spend_check_interval": 3600, + } + alerting_update: Final = candidate.request( + "POST", + "/config/field/update", + {"field_name": "alerting_args", "field_value": alerting_args, "config_type": "general_settings"}, + ) + assert alerting_update.status_code == 200, alerting_update.text + assert _config_section(database_url, "general_settings") == { + "plugins": [ + { + "name": "integration-plugin", + "display_name": "Integration updated", + "url": plugin_wire.url, + "plugin_key": plugin_key, + } + ], + "pass_through_endpoints": passthrough_value, + "alerting_args": alerting_args, + } + alerting_info: Final = object_value(candidate.get("/config/field/info?field_name=alerting_args")) + assert alerting_info["field_value"] == alerting_args + + invalid: Final = candidate.request( + "POST", + "/config/field/update", + { + "field_name": "alerting_args", + "field_value": {"daily_spend_per_user_threshold": -1}, + "config_type": "general_settings", + }, + ) + assert invalid.status_code == 400, invalid.text + assert invalid.json() == { + "detail": { + "error": "Invalid alerting_args: daily_spend_per_user_threshold: Input should be greater than 0" + } + }, invalid.text + + max_size_update: Final = candidate.request( + "POST", + "/config/field/update", + {"field_name": "max_request_size_mb", "field_value": 1, "config_type": "general_settings"}, + ) + assert max_size_update.status_code == 200, max_size_update.text + expected_settings: Final = { + "plugins": [ + { + "name": "integration-plugin", + "display_name": "Integration updated", + "url": plugin_wire.url, + "plugin_key": plugin_key, + } + ], + "pass_through_endpoints": passthrough_value, + "alerting_args": alerting_args, + "max_request_size_mb": 1, + } + assert _config_section(database_url, "general_settings") == expected_settings + max_size_info: Final = object_value(candidate.get("/config/field/info?field_name=max_request_size_mb")) + assert max_size_info["field_value"] == 1 + list_after_size: Final = TypeAdapter(list[dict[str, JsonValue]]).validate_json( + candidate.request("GET", "/config/list", params={"config_type": "general_settings"}).content + ) + max_size_listed: Final = next( + entry for entry in list_after_size if entry["field_name"] == "max_request_size_mb" + ) + assert max_size_listed["field_value"] == 1 and max_size_listed["stored_in_db"] is True + oversized: Final = candidate.request( + "POST", + "/v1/chat/completions", + { + "model": "openai/gpt-4o-mini", + "messages": [{"role": "user", "content": "x" * (1024 * 1024 + 1)}], + }, + ) + assert oversized.status_code == 413, oversized.text + assert oversized.text == '{"error":"Request size is too large. Max size is 1 MB"}' + peer_oversized: Final = eventually( + lambda: peer.request( + "POST", + "/v1/chat/completions", + { + "model": "openai/gpt-4o-mini", + "messages": [{"role": "user", "content": "x" * (1024 * 1024 + 1)}], + }, + ), + lambda response: ( + response.status_code == 413 + and response.text == '{"error":"Request size is too large. Max size is 1 MB"}' + ), + seconds=20, + ) + assert peer_oversized.text == '{"error":"Request size is too large. Max size is 1 MB"}', peer_oversized.text + + plugin_requests: Final = plugin_wire.drain() + assert tuple((request.method, request.target) for request in plugin_requests) == ( + ("POST", "/rpc"), + ("POST", "/rpc"), + ), plugin_requests + passthrough_requests: Final = passthrough_wire.drain() + assert tuple((request.method, request.target) for request in passthrough_requests) == ( + ("POST", "/echo"), + ("POST", "/echo"), + ), passthrough_requests + + @pytest.mark.covers("mgmt.model.block.changes_serving_and_preserves_control") def test_model_block_changes_actual_route_and_leaves_other_route_working(gateway: Gateway) -> None: with gateway.scenario() as scenario: diff --git a/tests/integration/security/test_config_overrides.py b/tests/integration/security/test_config_overrides.py new file mode 100644 index 00000000000..72546c54222 --- /dev/null +++ b/tests/integration/security/test_config_overrides.py @@ -0,0 +1,1103 @@ +from __future__ import annotations + +import base64 +import json +import ssl +import uuid +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from pathlib import Path +from typing import Final +from urllib.parse import quote + +import httpx +import pytest +from integration._support.client import Gateway, object_value, string_value +from integration._support.database import read_rows, scratch_database +from integration._support.process import owned_proxy +from integration._support.tls import write_self_signed_cert +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_VAULT_ENVIRONMENT: Final = ( + "HCP_VAULT_ADDR", + "HCP_VAULT_TOKEN", + "HCP_VAULT_APPROLE_ROLE_ID", + "HCP_VAULT_APPROLE_SECRET_ID", + "HCP_VAULT_APPROLE_MOUNT_PATH", + "HCP_VAULT_CLIENT_CERT", + "HCP_VAULT_CLIENT_KEY", + "HCP_VAULT_CERT_ROLE", + "HCP_VAULT_NAMESPACE", + "HCP_VAULT_LOGIN_NAMESPACE", + "HCP_VAULT_SECRET_NAMESPACE", + "HCP_VAULT_MOUNT_NAME", + "HCP_VAULT_PATH_PREFIX", +) +_CYBERARK_ENVIRONMENT: Final = ( + "CYBERARK_API_BASE", + "CYBERARK_ACCOUNT", + "CYBERARK_USERNAME", + "CYBERARK_API_KEY", + "CYBERARK_CLIENT_CERT", + "CYBERARK_CLIENT_KEY", + "CYBERARK_SSL_VERIFY", + "CYBERARK_REFRESH_INTERVAL", +) +_SECRET_MANAGER_ENVIRONMENT: Final = (*_VAULT_ENVIRONMENT, *_CYBERARK_ENVIRONMENT) + + +@contextmanager +def _config_proxy( + gateway: Gateway, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + overrides: Mapping[str, str] | None = None, +) -> Iterator[tuple[Gateway, str]]: + with scratch_database() as database_url: + with monkeypatch.context() as scoped_environment: + scoped_environment.setenv("DATABASE_URL", database_url) + with owned_proxy( + gateway, + tmp_path, + {"DATABASE_URL": database_url, **(overrides or {})}, + remove_environment=_SECRET_MANAGER_ENVIRONMENT, + ) as candidate: + yield candidate, database_url + + +def _override_values(candidate: Gateway, path: str, config_type: str) -> dict[str, JsonValue]: + response: Final = candidate.request("GET", path) + assert response.status_code == 200, response.text + body: Final = object_value(_JSON_OBJECT.validate_json(response.content)) + assert body["config_type"] == config_type, response.text + return object_value(body["values"]) + + +def _stored_override(database_url: str, config_type: str) -> dict[str, JsonValue]: + rows: Final = read_rows( + 'SELECT config_value FROM "LiteLLM_ConfigOverrides" WHERE config_type = %s', + (config_type,), + database_url=database_url, + ) + assert len(rows) == 1, rows + config_value: Final = rows[0]["config_value"] + return _JSON_OBJECT.validate_json(config_value) if isinstance(config_value, str) else object_value(config_value) + + +def _masked(value: str) -> str: + return value[:4] + "*" * (len(value) - 8) + value[-4:] + + +def _assert_non_admin_refusal(response: httpx.Response, route: str, user_id: str) -> None: + assert response.status_code == 401, response.text + masked_user_id: Final = user_id[:6] + "*" * (len(user_id) - 8) + user_id[-2:] + assert response.json() == { + "error": { + "message": ( + "Authentication Error, Only proxy admin can be used to generate, delete, update info for " + f"new keys/users/teams. Route={route}. Your role=internal_user. Your user_id={masked_user_id}" + ), + "type": "auth_error", + "param": "None", + "code": "401", + } + }, response.text + + +def _completion(text: str) -> Reply: + return Reply( + body=json.dumps( + { + "id": "chatcmpl-" + uuid.uuid4().hex, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ).encode() + ) + + +def _new_model(candidate: Gateway, name: str, provider: str, secret_name: str) -> str: + response: Final = candidate.request( + "POST", + "/model/new", + { + "model_name": name, + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": f"os.environ/{secret_name}", + "api_base": provider + "/v1", + }, + "model_info": {}, + }, + ) + assert response.status_code == 200, response.text + return string_value(object_value(response.json()["model_info"])["id"]) + + +def _chat(candidate: Gateway, model: str, marker: str) -> None: + response: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": marker}]}, + ) + assert response.status_code == 200, response.text + assert response.json()["choices"][0]["message"]["content"] == marker, response.text + + +def test_saved_vault_config_reads_back_masked_and_refuses_non_admins( + gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def vault_response(request: Request) -> Reply: + return Reply() + + with wire_server(vault_response) as vault: + config: Final = { + "vault_addr": vault.url, + "vault_token": "vault-token-" + uuid.uuid4().hex, + "approle_role_id": "approle-role-" + uuid.uuid4().hex, + "approle_secret_id": "approle-secret-" + uuid.uuid4().hex, + "approle_mount_path": "auth/approle-integration", + "client_cert": "/integration/client.crt", + "client_key": "client-key-" + uuid.uuid4().hex, + "vault_cert_role": "integration-role", + "vault_namespace": "integration/root", + "vault_login_namespace": "integration/login", + "vault_secret_namespace": "integration/secrets", + "vault_mount_name": "integration-kv", + "vault_path_prefix": "integration/prefix", + } + with _config_proxy(gateway, tmp_path, monkeypatch) as (candidate, database_url): + saved: Final = candidate.request("POST", "/config_overrides/hashicorp_vault", config) + assert saved.status_code == 200, saved.text + assert saved.json() == { + "message": "Hashicorp Vault configuration updated successfully", + "status": "success", + }, saved.text + readback: Final = candidate.request("GET", "/config_overrides/hashicorp_vault") + assert readback.status_code == 200, readback.text + readback_body: Final = object_value(_JSON_OBJECT.validate_json(readback.content)) + assert readback_body["config_type"] == "hashicorp_vault", readback.text + values: Final = object_value(readback_body["values"]) + expected_values: Final = { + **config, + "vault_token": _masked(string_value(config["vault_token"])), + "approle_secret_id": _masked(string_value(config["approle_secret_id"])), + "client_key": _masked(string_value(config["client_key"])), + } + assert values == expected_values + assert all( + string_value(config[field]).encode() not in readback.content + for field in ("vault_token", "approle_secret_id", "client_key") + ) + + with candidate.scenario() as scenario: + non_admin_user_id: Final = "integration-vault-non-admin" + non_admin: Final = scenario.key( + user_id=scenario.user(user_id=non_admin_user_id, user_role="internal_user") + ) + denied_get: Final = candidate.request("GET", "/config_overrides/hashicorp_vault", key=non_admin) + _assert_non_admin_refusal(denied_get, "/config_overrides/hashicorp_vault", non_admin_user_id) + denied_post: Final = candidate.request( + "POST", "/config_overrides/hashicorp_vault", config, key=non_admin + ) + _assert_non_admin_refusal(denied_post, "/config_overrides/hashicorp_vault", non_admin_user_id) + + stored_before_scoped: Final = _stored_override(database_url, "hashicorp_vault") + scoped_route_key: Final = scenario.key( + user_id=scenario.user(user_role="internal_user"), + allowed_routes=["/config_overrides/hashicorp_vault"], + ) + scoped_get: Final = candidate.request("GET", "/config_overrides/hashicorp_vault", key=scoped_route_key) + assert scoped_get.status_code == 403, scoped_get.text + assert scoped_get.json() == {"detail": "Only admin users can view config overrides"}, scoped_get.text + scoped_post: Final = candidate.request( + "POST", "/config_overrides/hashicorp_vault", config, key=scoped_route_key + ) + assert scoped_post.status_code == 403, scoped_post.text + assert scoped_post.json() == {"detail": "Only admin users can update config overrides"}, ( + scoped_post.text + ) + assert _stored_override(database_url, "hashicorp_vault") == stored_before_scoped + + viewer_key: Final = scenario.key(user_id=scenario.user(user_role="proxy_admin_viewer")) + viewer_get: Final = candidate.request("GET", "/config_overrides/hashicorp_vault", key=viewer_key) + assert viewer_get.status_code == 200, viewer_get.text + viewer_values: Final = object_value( + object_value(_JSON_OBJECT.validate_json(viewer_get.content))["values"] + ) + assert viewer_values == expected_values, viewer_get.text + assert all( + string_value(config[field]).encode() not in viewer_get.content + for field in ("vault_token", "approle_secret_id", "client_key") + ) + viewer_post: Final = candidate.request( + "POST", "/config_overrides/hashicorp_vault", config, key=viewer_key + ) + assert viewer_post.status_code == 403, viewer_post.text + assert viewer_post.json() == { + "error": { + "message": "user not allowed to access this route, role= proxy_admin_viewer. " + "Trying to access: /config_overrides/hashicorp_vault", + "type": "auth_error", + "param": "None", + "code": "403", + } + }, viewer_post.text + assert _stored_override(database_url, "hashicorp_vault") == stored_before_scoped + + env_values: Final = { + "HCP_VAULT_ADDR": vault.url, + "HCP_VAULT_TOKEN": "env-vault-token-" + uuid.uuid4().hex, + "HCP_VAULT_APPROLE_ROLE_ID": "env-role-" + uuid.uuid4().hex, + "HCP_VAULT_APPROLE_SECRET_ID": "env-secret-" + uuid.uuid4().hex, + "HCP_VAULT_APPROLE_MOUNT_PATH": "auth/env-approle", + "HCP_VAULT_CLIENT_CERT": "/integration/env-client.crt", + "HCP_VAULT_CLIENT_KEY": "env-client-key-" + uuid.uuid4().hex, + "HCP_VAULT_CERT_ROLE": "env-integration-role", + "HCP_VAULT_NAMESPACE": "env/root", + "HCP_VAULT_LOGIN_NAMESPACE": "env/login", + "HCP_VAULT_SECRET_NAMESPACE": "env/secrets", + "HCP_VAULT_MOUNT_NAME": "env-kv", + "HCP_VAULT_PATH_PREFIX": "env/prefix", + } + env_config: Final = { + "vault_addr": env_values["HCP_VAULT_ADDR"], + "vault_token": env_values["HCP_VAULT_TOKEN"], + "approle_role_id": env_values["HCP_VAULT_APPROLE_ROLE_ID"], + "approle_secret_id": env_values["HCP_VAULT_APPROLE_SECRET_ID"], + "approle_mount_path": env_values["HCP_VAULT_APPROLE_MOUNT_PATH"], + "client_cert": env_values["HCP_VAULT_CLIENT_CERT"], + "client_key": env_values["HCP_VAULT_CLIENT_KEY"], + "vault_cert_role": env_values["HCP_VAULT_CERT_ROLE"], + "vault_namespace": env_values["HCP_VAULT_NAMESPACE"], + "vault_login_namespace": env_values["HCP_VAULT_LOGIN_NAMESPACE"], + "vault_secret_namespace": env_values["HCP_VAULT_SECRET_NAMESPACE"], + "vault_mount_name": env_values["HCP_VAULT_MOUNT_NAME"], + "vault_path_prefix": env_values["HCP_VAULT_PATH_PREFIX"], + } + with _config_proxy(gateway, tmp_path, monkeypatch, env_values) as (candidate, database_url): + assert ( + read_rows( + 'SELECT config_type FROM "LiteLLM_ConfigOverrides" WHERE config_type = %s', + ("hashicorp_vault",), + database_url=database_url, + ) + == [] + ) + env_response: Final = candidate.request("GET", "/config_overrides/hashicorp_vault") + assert env_response.status_code == 200, env_response.text + env_body: Final = object_value(_JSON_OBJECT.validate_json(env_response.content)) + assert env_body["config_type"] == "hashicorp_vault", env_response.text + env_readback: Final = object_value(env_body["values"]) + assert env_readback == { + **env_config, + "vault_token": _masked(string_value(env_config["vault_token"])), + "approle_secret_id": _masked(string_value(env_config["approle_secret_id"])), + "client_key": _masked(string_value(env_config["client_key"])), + } + assert all( + string_value(env_config[field]).encode() not in env_response.content + for field in ("vault_token", "approle_secret_id", "client_key") + ) + + +def test_saved_cyberark_config_reads_back_masked_and_refuses_non_admins( + gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with wire_server(lambda request: Reply()) as cyberark: + config: Final = { + "cyberark_api_base": cyberark.url, + "cyberark_account": "integration-account", + "cyberark_username": "integration-user", + "cyberark_api_key": "cyberark-api-key-" + uuid.uuid4().hex, + "client_cert": "/integration/client.crt", + "client_key": "cyberark-client-key-" + uuid.uuid4().hex, + "ssl_verify": "true", + "refresh_interval": "300", + } + with _config_proxy(gateway, tmp_path, monkeypatch) as (candidate, _database_url): + saved: Final = candidate.request("POST", "/config_overrides/cyberark", config) + assert saved.status_code == 200, saved.text + assert saved.json() == { + "message": "CyberArk configuration updated successfully", + "status": "success", + }, saved.text + readback: Final = candidate.request("GET", "/config_overrides/cyberark") + assert readback.status_code == 200, readback.text + readback_body: Final = object_value(_JSON_OBJECT.validate_json(readback.content)) + assert readback_body["config_type"] == "cyberark", readback.text + values: Final = object_value(readback_body["values"]) + assert values == { + **config, + "cyberark_api_key": _masked(string_value(config["cyberark_api_key"])), + "client_key": _masked(string_value(config["client_key"])), + } + assert all( + string_value(config[field]).encode() not in readback.content + for field in ("cyberark_api_key", "client_key") + ) + with candidate.scenario() as scenario: + non_admin_user_id: Final = "integration-cyberark-non-admin" + non_admin: Final = scenario.key( + user_id=scenario.user(user_id=non_admin_user_id, user_role="internal_user") + ) + denied_get: Final = candidate.request("GET", "/config_overrides/cyberark", key=non_admin) + _assert_non_admin_refusal(denied_get, "/config_overrides/cyberark", non_admin_user_id) + denied_post: Final = candidate.request("POST", "/config_overrides/cyberark", config, key=non_admin) + _assert_non_admin_refusal(denied_post, "/config_overrides/cyberark", non_admin_user_id) + + stored_before_scoped: Final = _stored_override(_database_url, "cyberark") + scoped_route_key: Final = scenario.key( + user_id=scenario.user(user_role="internal_user"), + allowed_routes=["/config_overrides/cyberark"], + ) + scoped_get: Final = candidate.request("GET", "/config_overrides/cyberark", key=scoped_route_key) + assert scoped_get.status_code == 403, scoped_get.text + assert scoped_get.json() == {"detail": "Only admin users can view config overrides"}, scoped_get.text + scoped_post: Final = candidate.request( + "POST", "/config_overrides/cyberark", config, key=scoped_route_key + ) + assert scoped_post.status_code == 403, scoped_post.text + assert scoped_post.json() == {"detail": "Only admin users can update config overrides"}, ( + scoped_post.text + ) + assert _stored_override(_database_url, "cyberark") == stored_before_scoped + + viewer_key: Final = scenario.key(user_id=scenario.user(user_role="proxy_admin_viewer")) + viewer_get: Final = candidate.request("GET", "/config_overrides/cyberark", key=viewer_key) + assert viewer_get.status_code == 200, viewer_get.text + viewer_values: Final = object_value( + object_value(_JSON_OBJECT.validate_json(viewer_get.content))["values"] + ) + assert viewer_values == values, viewer_get.text + assert all( + string_value(config[field]).encode() not in viewer_get.content + for field in ("cyberark_api_key", "client_key") + ) + viewer_post: Final = candidate.request("POST", "/config_overrides/cyberark", config, key=viewer_key) + assert viewer_post.status_code == 403, viewer_post.text + assert viewer_post.json() == { + "error": { + "message": "user not allowed to access this route, role= proxy_admin_viewer. " + "Trying to access: /config_overrides/cyberark", + "type": "auth_error", + "param": "None", + "code": "403", + } + }, viewer_post.text + assert _stored_override(_database_url, "cyberark") == stored_before_scoped + + env_values: Final = { + "CYBERARK_API_BASE": cyberark.url, + "CYBERARK_ACCOUNT": "env-account", + "CYBERARK_USERNAME": "env-user", + "CYBERARK_API_KEY": "env-cyberark-api-key-" + uuid.uuid4().hex, + "CYBERARK_CLIENT_CERT": "/integration/env-client.crt", + "CYBERARK_CLIENT_KEY": "env-cyberark-client-key-" + uuid.uuid4().hex, + "CYBERARK_SSL_VERIFY": "false", + "CYBERARK_REFRESH_INTERVAL": "600", + } + env_config: Final = { + "cyberark_api_base": env_values["CYBERARK_API_BASE"], + "cyberark_account": env_values["CYBERARK_ACCOUNT"], + "cyberark_username": env_values["CYBERARK_USERNAME"], + "cyberark_api_key": env_values["CYBERARK_API_KEY"], + "client_cert": env_values["CYBERARK_CLIENT_CERT"], + "client_key": env_values["CYBERARK_CLIENT_KEY"], + "ssl_verify": env_values["CYBERARK_SSL_VERIFY"], + "refresh_interval": env_values["CYBERARK_REFRESH_INTERVAL"], + } + with _config_proxy(gateway, tmp_path, monkeypatch, env_values) as (env_candidate, env_database_url): + assert ( + read_rows( + 'SELECT config_type FROM "LiteLLM_ConfigOverrides" WHERE config_type = %s', + ("cyberark",), + database_url=env_database_url, + ) + == [] + ) + env_response: Final = env_candidate.request("GET", "/config_overrides/cyberark") + assert env_response.status_code == 200, env_response.text + env_body: Final = object_value(_JSON_OBJECT.validate_json(env_response.content)) + assert env_body["config_type"] == "cyberark", env_response.text + assert object_value(env_body["values"]) == { + **env_config, + "cyberark_api_key": _masked(env_config["cyberark_api_key"]), + "client_key": _masked(env_config["client_key"]), + }, env_response.text + assert all( + env_config[field].encode() not in env_response.content for field in ("cyberark_api_key", "client_key") + ) + + +def test_vault_partial_update_keeps_omitted_fields_encrypts_and_drives_secret_reads( + gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + provider_secret: Final = "vault-provider-secret-" + uuid.uuid4().hex + vault_token: Final = "vault-token-" + uuid.uuid4().hex + marker_one: Final = "vault-runtime-one-" + uuid.uuid4().hex + marker_two: Final = "vault-runtime-two-" + uuid.uuid4().hex + path_one: Final = "/v1/integration/team/kv-original/data/initial-prefix/secret-one" + path_two: Final = "/v1/integration/team/kv-updated/data/secret-two" + + def vault(request: Request) -> Reply: + assert request.method == "GET" + assert request.headers["x-vault-token"] == vault_token + secrets: Final = { + path_one: provider_secret, + path_two: provider_secret, + } + if request.target not in secrets: + return Reply(status=404, body=b'{"errors":["secret not found"]}') + return Reply(body=json.dumps({"data": {"data": {"key": secrets[request.target]}}}).encode()) + + def provider(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/v1/chat/completions" + body: Final = _JSON_OBJECT.validate_json(request.body) + messages: Final = body["messages"] + assert isinstance(messages, list) + message: Final = object_value(messages[0]) + marker: Final = string_value(message["content"]) + if marker == "after-delete": + return _completion(marker) + assert request.headers["authorization"] == f"Bearer {provider_secret}", ( + "Provider request did not use the Vault-resolved secret" + ) + assert marker in {marker_one, marker_two}, request.body + return _completion(marker) + + with wire_server(vault) as vault_wire, wire_server(provider) as provider_wire: + with _config_proxy(gateway, tmp_path, monkeypatch) as (candidate, database_url): + full_config: Final = { + "vault_addr": vault_wire.url, + "vault_token": vault_token, + "vault_secret_namespace": "integration/team", + "vault_mount_name": "kv-original", + "vault_path_prefix": "initial-prefix", + } + full_update: Final = candidate.request("POST", "/config_overrides/hashicorp_vault", full_config) + assert full_update.status_code == 200, full_update.text + assert full_update.json() == { + "message": "Hashicorp Vault configuration updated successfully", + "status": "success", + }, full_update.text + stored_full: Final = _stored_override(database_url, "hashicorp_vault") + assert set(stored_full) == set(full_config), stored_full + serialized_full: Final = json.dumps(stored_full) + assert all(value not in serialized_full for value in full_config.values()), serialized_full + + with candidate.scenario() as scenario: + model_one: Final = scenario.model( + api_key="os.environ/secret-one", + api_base=provider_wire.url + "/v1", + ) + response_one: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": model_one, "messages": [{"role": "user", "content": marker_one}]}, + ) + assert response_one.status_code == 200, response_one.text + assert response_one.json()["choices"][0]["message"]["content"] == marker_one, response_one.text + first_secret_reads: Final = tuple( + (request.method, request.target) + for request in vault_wire.drain() + if request.target in {path_one, path_two} + ) + assert first_secret_reads == (("GET", path_one),) + + partial_config: Final = { + "vault_addr": vault_wire.url, + "vault_secret_namespace": "integration/team", + "vault_mount_name": "kv-updated", + "vault_path_prefix": "", + } + partial_update: Final = candidate.request("POST", "/config_overrides/hashicorp_vault", partial_config) + assert partial_update.status_code == 200, partial_update.text + assert partial_update.json() == { + "message": "Hashicorp Vault configuration updated successfully", + "status": "success", + }, partial_update.text + stored_partial: Final = _stored_override(database_url, "hashicorp_vault") + assert set(stored_partial) == { + "vault_addr", + "vault_token", + "vault_secret_namespace", + "vault_mount_name", + }, stored_partial + serialized_partial: Final = json.dumps(stored_partial) + assert all( + value not in serialized_partial + for value in ( + vault_wire.url, + vault_token, + "integration/team", + "kv-updated", + ) + ), serialized_partial + readback: Final = _override_values(candidate, "/config_overrides/hashicorp_vault", "hashicorp_vault") + assert readback == { + "vault_addr": vault_wire.url, + "vault_token": _masked(vault_token), + "vault_secret_namespace": "integration/team", + "vault_mount_name": "kv-updated", + } + readback_response: Final = candidate.request("GET", "/config_overrides/hashicorp_vault") + assert readback_response.status_code == 200, readback_response.text + assert vault_token not in readback_response.text, readback_response.text + + model_two: Final = scenario.model( + api_key="os.environ/secret-two", + api_base=provider_wire.url + "/v1", + ) + response_two: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": model_two, "messages": [{"role": "user", "content": marker_two}]}, + ) + assert response_two.status_code == 200, response_two.text + assert response_two.json()["choices"][0]["message"]["content"] == marker_two, response_two.text + second_secret_reads: Final = tuple( + (request.method, request.target) + for request in vault_wire.drain() + if request.target in {path_one, path_two} + ) + assert second_secret_reads == (("GET", path_two),) + provider_requests: Final = provider_wire.drain() + assert tuple((request.method, request.target) for request in provider_requests) == ( + ("POST", "/v1/chat/completions"), + ("POST", "/v1/chat/completions"), + ) + + deleted: Final = candidate.request("DELETE", "/config_overrides/hashicorp_vault") + assert deleted.status_code == 200, deleted.text + assert deleted.json() == { + "message": "Hashicorp Vault configuration deleted successfully", + "status": "success", + }, deleted.text + vault_wire.drain() + assert ( + read_rows( + 'SELECT config_type FROM "LiteLLM_ConfigOverrides" WHERE config_type = %s', + ("hashicorp_vault",), + database_url=database_url, + ) + == [] + ) + empty_readback: Final = candidate.request("GET", "/config_overrides/hashicorp_vault") + assert empty_readback.status_code == 200, empty_readback.text + empty_values: Final = object_value( + object_value(_JSON_OBJECT.validate_json(empty_readback.content))["values"] + ) + assert empty_values == { + field: None + for field in ( + "vault_addr", + "vault_token", + "approle_role_id", + "approle_secret_id", + "approle_mount_path", + "client_cert", + "client_key", + "vault_cert_role", + "vault_namespace", + "vault_login_namespace", + "vault_secret_namespace", + "vault_mount_name", + "vault_path_prefix", + ) + }, empty_readback.text + model_after_delete: Final = scenario.model( + api_key="os.environ/LITELLM_MASTER_KEY", + api_base=provider_wire.url + "/v1", + ) + response_after_delete: Final = candidate.request( + "POST", + "/v1/chat/completions", + { + "model": model_after_delete, + "messages": [{"role": "user", "content": "after-delete"}], + }, + ) + assert response_after_delete.status_code == 200, response_after_delete.text + assert response_after_delete.json()["choices"][0]["message"]["content"] == "after-delete", ( + response_after_delete.text + ) + after_delete_requests: Final = vault_wire.drain() + assert after_delete_requests == (), after_delete_requests + assert first_secret_reads + second_secret_reads == ( + ("GET", path_one), + ("GET", path_two), + ) + + +def test_cyberark_incremental_update_encrypts_reloads_on_restart_and_delete_clears( + gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + provider_secret: Final = "cyberark-provider-secret-" + uuid.uuid4().hex + api_key: Final = "cyberark-api-key-" + uuid.uuid4().hex + session_token: Final = "integration-session-" + uuid.uuid4().hex + account: Final = "integration-account" + username: Final = "integration-user" + auth_path: Final = f"/authn/{account}/{username}/authenticate" + default_user_auth_path: Final = f"/authn/{account}/admin/authenticate" + secret_names: Final = ( + "integration-cyberark-secret-one", + "integration-cyberark-secret-two", + "integration-cyberark-secret-three", + "integration-cyberark-secret-four", + ) + encoded_secret_names: Final = frozenset(quote(name, safe="") for name in secret_names) + + def cyberark(request: Request) -> Reply: + if request.method == "POST" and request.target in (auth_path, default_user_auth_path): + assert request.body == api_key.encode() + return Reply(body=session_token.encode()) + expected_target: Final = "/secrets/integration-account/variable/" + assert request.method == "GET" + assert request.target.startswith(expected_target), request.target + encoded_name: Final = request.target.removeprefix(expected_target) + expected_header: Final = f'Token token="{base64.b64encode(session_token.encode()).decode()}"' + assert request.headers["authorization"] == expected_header + if encoded_name not in encoded_secret_names: + return Reply(status=404, body=b"secret not found") + return Reply(body=provider_secret.encode()) + + def provider(request: Request) -> Reply: + if request.method == "GET": + assert request.target == "/v1/models", request.target + return Reply(status=404) + assert (request.method, request.target) == ("POST", "/v1/chat/completions"), ( + f"Unexpected provider request {request.method} {request.target}" + ) + body: Final = _JSON_OBJECT.validate_json(request.body) + messages: Final = body["messages"] + assert isinstance(messages, list) + marker: Final = string_value(object_value(messages[0])["content"]) + if marker == "after-delete": + return _completion(marker) + assert request.headers["authorization"] == f"Bearer {provider_secret}", ( + "Provider request did not use the CyberArk-resolved secret" + ) + return _completion(marker) + + with ( + scratch_database() as database_url, + wire_server(cyberark) as cyberark_wire, + wire_server(provider) as provider_wire, + ): + monkeypatch.setenv("DATABASE_URL", database_url) + environment: Final = {"DATABASE_URL": database_url} + with owned_proxy( + gateway, + tmp_path, + environment, + remove_environment=_SECRET_MANAGER_ENVIRONMENT, + ) as first: + full_config: Final = { + "cyberark_api_base": cyberark_wire.url, + "cyberark_account": account, + "cyberark_username": username, + "cyberark_api_key": api_key, + "ssl_verify": "true", + "refresh_interval": "300", + } + saved: Final = first.request("POST", "/config_overrides/cyberark", full_config) + assert saved.status_code == 200, saved.text + assert saved.json() == { + "message": "CyberArk configuration updated successfully", + "status": "success", + }, saved.text + stored_full: Final = _stored_override(database_url, "cyberark") + assert set(stored_full) == set(full_config), stored_full + serialized_full: Final = json.dumps(stored_full) + assert all(value not in serialized_full for value in full_config.values()), serialized_full + + model_one: Final = "integration-cyberark-" + uuid.uuid4().hex + _new_model(first, model_one, provider_wire.url, secret_names[0]) + _chat(first, model_one, "cyberark-runtime-one-" + uuid.uuid4().hex) + first_proxy_requests: Final = cyberark_wire.drain() + assert any(request.target == auth_path for request in first_proxy_requests) + assert any(request.target.endswith(quote(secret_names[0], safe="")) for request in first_proxy_requests) + + partial: Final = first.request( + "POST", + "/config_overrides/cyberark", + {"cyberark_api_base": cyberark_wire.url, "refresh_interval": "301"}, + ) + assert partial.status_code == 200, partial.text + assert partial.json() == { + "message": "CyberArk configuration updated successfully", + "status": "success", + }, partial.text + stored_partial: Final = _stored_override(database_url, "cyberark") + assert set(stored_partial) == set(full_config), stored_partial + serialized_partial: Final = json.dumps(stored_partial) + assert all(value not in serialized_partial for value in (cyberark_wire.url, account, username, api_key)), ( + serialized_partial + ) + assert string_value( + _override_values(first, "/config_overrides/cyberark", "cyberark")["cyberark_api_key"] + ) == _masked(api_key) + model_two: Final = "integration-cyberark-" + uuid.uuid4().hex + _new_model(first, model_two, provider_wire.url, secret_names[1]) + _chat(first, model_two, "cyberark-runtime-two-" + uuid.uuid4().hex) + second_proxy_requests: Final = cyberark_wire.drain() + + with owned_proxy( + gateway, + tmp_path, + environment, + remove_environment=_SECRET_MANAGER_ENVIRONMENT, + ) as restarted: + model_three: Final = "integration-cyberark-" + uuid.uuid4().hex + _new_model(restarted, model_three, provider_wire.url, secret_names[2]) + _chat(restarted, model_three, "cyberark-runtime-three-" + uuid.uuid4().hex) + restarted_proxy_requests: Final = cyberark_wire.drain() + known_secret_names: Final = frozenset(quote(name, safe="") for name in secret_names[:3]) + observed_secret_names: Final = frozenset( + request.target.rsplit("/", 1)[-1] + for request in ( + *first_proxy_requests, + *second_proxy_requests, + *restarted_proxy_requests, + ) + if request.method == "GET" + and request.target.startswith("/secrets/") + and request.target.rsplit("/", 1)[-1] in known_secret_names + ) + assert observed_secret_names == known_secret_names + + dashboard_edit: Final = restarted.request( + "POST", + "/config_overrides/cyberark", + { + "cyberark_api_base": cyberark_wire.url, + "cyberark_account": account, + "cyberark_username": "", + "client_cert": "", + "ssl_verify": "true", + "refresh_interval": "301", + }, + ) + assert dashboard_edit.status_code == 200, dashboard_edit.text + assert dashboard_edit.json() == { + "message": "CyberArk configuration updated successfully", + "status": "success", + }, dashboard_edit.text + stored_dashboard_edit: Final = _stored_override(database_url, "cyberark") + assert set(stored_dashboard_edit) == set(full_config) - {"cyberark_username"}, stored_dashboard_edit + serialized_dashboard_edit: Final = json.dumps(stored_dashboard_edit) + assert all(value not in serialized_dashboard_edit for value in (cyberark_wire.url, account, api_key)), ( + serialized_dashboard_edit + ) + dashboard_edit_readback: Final = restarted.request("GET", "/config_overrides/cyberark") + assert dashboard_edit_readback.status_code == 200, dashboard_edit_readback.text + dashboard_edit_values: Final = object_value( + object_value(_JSON_OBJECT.validate_json(dashboard_edit_readback.content))["values"] + ) + assert dashboard_edit_values == { + "cyberark_api_base": cyberark_wire.url, + "cyberark_account": account, + "cyberark_api_key": _masked(api_key), + "ssl_verify": "true", + "refresh_interval": "301", + }, dashboard_edit_readback.text + model_four: Final = "integration-cyberark-" + uuid.uuid4().hex + _new_model(restarted, model_four, provider_wire.url, secret_names[3]) + _chat(restarted, model_four, "cyberark-runtime-four-" + uuid.uuid4().hex) + blanked_requests: Final = cyberark_wire.drain() + assert any( + request.method == "POST" and request.target == default_user_auth_path for request in blanked_requests + ), blanked_requests + assert not any(request.target == auth_path for request in blanked_requests), blanked_requests + assert any( + request.method == "GET" and request.target.endswith(quote(secret_names[3], safe="")) + for request in blanked_requests + ), blanked_requests + + deleted: Final = restarted.request("DELETE", "/config_overrides/cyberark") + assert deleted.status_code == 200, deleted.text + assert deleted.json() == { + "message": "CyberArk configuration deleted successfully", + "status": "success", + }, deleted.text + assert ( + read_rows( + 'SELECT config_type FROM "LiteLLM_ConfigOverrides" WHERE config_type = %s', + ("cyberark",), + database_url=database_url, + ) + == [] + ) + deleted_secret_reads: Final = tuple( + request.target + for request in cyberark_wire.drain() + if request.method == "GET" + and request.target.startswith("/secrets/") + and request.target.rsplit("/", 1)[-1] in encoded_secret_names + ) + assert deleted_secret_reads == () + model_after_delete: Final = "integration-cyberark-" + uuid.uuid4().hex + _new_model(restarted, model_after_delete, provider_wire.url, "LITELLM_MASTER_KEY") + _chat(restarted, model_after_delete, "after-delete") + requests_after_delete: Final = cyberark_wire.drain() + assert requests_after_delete == (), requests_after_delete + empty_readback: Final = restarted.request("GET", "/config_overrides/cyberark") + assert empty_readback.status_code == 200, empty_readback.text + empty_values: Final = object_value( + object_value(_JSON_OBJECT.validate_json(empty_readback.content))["values"] + ) + assert empty_values == { + field: None + for field in ( + "cyberark_api_base", + "cyberark_account", + "cyberark_username", + "cyberark_api_key", + "client_cert", + "client_key", + "ssl_verify", + "refresh_interval", + ) + }, empty_readback.text + + +def test_vault_approle_and_cert_logins_use_login_namespace_and_drive_secret_reads( + gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + provider_secret: Final = "vault-auth-provider-secret-" + uuid.uuid4().hex + approle_token: Final = "approle-token-" + uuid.uuid4().hex + cert_token: Final = "cert-token-" + uuid.uuid4().hex + role_id: Final = "approle-role-" + uuid.uuid4().hex + secret_id: Final = "approle-secret-" + uuid.uuid4().hex + mount_path: Final = "approle-" + uuid.uuid4().hex + cert_role: Final = "cert-role-" + uuid.uuid4().hex + base_namespace: Final = "ns/base-" + uuid.uuid4().hex[:8] + login_namespace: Final = "ns/login-" + uuid.uuid4().hex[:8] + secret_namespace: Final = "ns/secrets-" + uuid.uuid4().hex[:8] + mount_name: Final = "kv-" + uuid.uuid4().hex[:8] + secret_a: Final = "secret-a-" + uuid.uuid4().hex + secret_b: Final = "secret-b-" + uuid.uuid4().hex + marker_a: Final = "vault-approle-runtime-" + uuid.uuid4().hex + marker_b: Final = "vault-cert-runtime-" + uuid.uuid4().hex + login_approle_path: Final = f"/v1/auth/{mount_path}/login" + login_cert_path: Final = "/v1/auth/cert/login" + secret_a_path: Final = f"/v1/{secret_namespace}/{mount_name}/data/{secret_a}" + secret_b_path: Final = f"/v1/{secret_namespace}/{mount_name}/data/{secret_b}" + + def vault(request: Request) -> Reply: + if request.method == "POST" and request.target == login_approle_path: + assert _JSON_OBJECT.validate_json(request.body) == { + "role_id": role_id, + "secret_id": secret_id, + }, request.body + return Reply(body=json.dumps({"auth": {"client_token": approle_token, "lease_duration": 3600}}).encode()) + if request.method == "POST" and request.target == login_cert_path: + assert _JSON_OBJECT.validate_json(request.body) == {"name": cert_role}, request.body + return Reply(body=json.dumps({"auth": {"client_token": cert_token, "lease_duration": 3600}}).encode()) + if request.method == "GET" and request.target in {secret_a_path, secret_b_path}: + return Reply(body=json.dumps({"data": {"data": {"key": provider_secret}}}).encode()) + return Reply(status=404, body=b'{"errors":["secret not found"]}') + + def provider(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/v1/chat/completions" + body: Final = _JSON_OBJECT.validate_json(request.body) + messages: Final = body["messages"] + assert isinstance(messages, list) + marker: Final = string_value(object_value(messages[0])["content"]) + if marker == "after-delete": + return _completion(marker) + assert request.headers["authorization"] == f"Bearer {provider_secret}", ( + "Provider request did not use the Vault-resolved secret" + ) + assert marker in {marker_a, marker_b}, request.body + return _completion(marker) + + cert_dir: Final = tmp_path / "vault-client-cert" + cert_dir.mkdir() + cert_file, key_file = write_self_signed_cert(cert_dir) + client_cert_der: Final = ssl.PEM_cert_to_DER_cert(cert_file.read_text()) + server_cert_dir: Final = tmp_path / "vault-server-cert" + server_cert_dir.mkdir() + server_cert, server_key = write_self_signed_cert(server_cert_dir) + presented_certs: Final[list[bytes | None]] = [] + + class RecordingSSLSocket(ssl.SSLSocket): + def do_handshake(self) -> None: + super().do_handshake() + presented_certs.append(self.getpeercert(binary_form=True)) + + vault_tls_context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + vault_tls_context.load_cert_chain(server_cert, server_key) + vault_tls_context.verify_mode = ssl.CERT_OPTIONAL + vault_tls_context.load_verify_locations(cafile=str(cert_file)) + vault_tls_context.sslsocket_class = RecordingSSLSocket + with ( + wire_server(vault) as vault_wire, + wire_server(vault, tls=vault_tls_context) as vault_tls_wire, + wire_server(provider) as provider_wire, + ): + with _config_proxy(gateway, tmp_path, monkeypatch, {"SSL_CERT_FILE": str(server_cert)}) as ( + candidate, + database_url, + ): + full_config: Final = { + "vault_addr": vault_wire.url, + "approle_role_id": role_id, + "approle_secret_id": secret_id, + "approle_mount_path": mount_path, + "vault_namespace": base_namespace, + "vault_login_namespace": login_namespace, + "vault_secret_namespace": secret_namespace, + "vault_mount_name": mount_name, + } + saved: Final = candidate.request("POST", "/config_overrides/hashicorp_vault", full_config) + assert saved.status_code == 200, saved.text + assert saved.json() == { + "message": "Hashicorp Vault configuration updated successfully", + "status": "success", + }, saved.text + stored_full: Final = _stored_override(database_url, "hashicorp_vault") + assert set(stored_full) == set(full_config), stored_full + serialized_full: Final = json.dumps(stored_full) + assert all(value not in serialized_full for value in full_config.values()), serialized_full + + with candidate.scenario() as scenario: + model_a: Final = scenario.model( + api_key=f"os.environ/{secret_a}", + api_base=provider_wire.url + "/v1", + ) + response_a: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": model_a, "messages": [{"role": "user", "content": marker_a}]}, + ) + assert response_a.status_code == 200, response_a.text + assert response_a.json()["choices"][0]["message"]["content"] == marker_a, response_a.text + approle_targets: Final = frozenset({login_approle_path, login_cert_path, secret_a_path, secret_b_path}) + approle_phase: Final = tuple( + request for request in vault_wire.drain() if request.target in approle_targets + ) + assert tuple((request.method, request.target) for request in approle_phase) == ( + ("POST", login_approle_path), + ("GET", secret_a_path), + ), approle_phase + assert approle_phase[0].headers["x-vault-namespace"] == login_namespace + assert _JSON_OBJECT.validate_json(approle_phase[0].body) == { + "role_id": role_id, + "secret_id": secret_id, + } + assert approle_phase[1].headers["x-vault-token"] == approle_token + + partial_config: Final = { + "vault_addr": vault_tls_wire.url, + "approle_role_id": "", + "approle_secret_id": "", + "client_cert": str(cert_file), + "client_key": str(key_file), + "vault_cert_role": cert_role, + } + partial: Final = candidate.request("POST", "/config_overrides/hashicorp_vault", partial_config) + assert partial.status_code == 200, partial.text + assert partial.json() == { + "message": "Hashicorp Vault configuration updated successfully", + "status": "success", + }, partial.text + vault_wire.drain() + readback: Final = _override_values(candidate, "/config_overrides/hashicorp_vault", "hashicorp_vault") + assert readback == { + "vault_addr": vault_tls_wire.url, + "approle_mount_path": mount_path, + "client_cert": str(cert_file), + "client_key": _masked(str(key_file)), + "vault_cert_role": cert_role, + "vault_namespace": base_namespace, + "vault_login_namespace": login_namespace, + "vault_secret_namespace": secret_namespace, + "vault_mount_name": mount_name, + }, readback + + model_b: Final = scenario.model( + api_key=f"os.environ/{secret_b}", + api_base=provider_wire.url + "/v1", + ) + response_b: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": model_b, "messages": [{"role": "user", "content": marker_b}]}, + ) + assert response_b.status_code == 200, response_b.text + assert response_b.json()["choices"][0]["message"]["content"] == marker_b, response_b.text + plain_phase_b: Final = vault_wire.drain() + assert plain_phase_b == (), plain_phase_b + cert_phase: Final = tuple( + request for request in vault_tls_wire.drain() if request.target in approle_targets + ) + assert all(request.target != login_approle_path for request in cert_phase), cert_phase + assert cert_phase[0].target == login_cert_path, cert_phase + cert_logins: Final = tuple(request for request in cert_phase if request.target == login_cert_path) + assert all( + request.method == "POST" + and request.headers["x-vault-namespace"] == login_namespace + and _JSON_OBJECT.validate_json(request.body) == {"name": cert_role} + for request in cert_logins + ), cert_logins + cert_secret_reads: Final = tuple( + request for request in cert_phase if request.target in {secret_a_path, secret_b_path} + ) + assert secret_b_path in tuple(request.target for request in cert_secret_reads), cert_secret_reads + assert all(request.headers["x-vault-token"] == cert_token for request in cert_secret_reads), ( + cert_secret_reads + ) + assert client_cert_der in presented_certs, presented_certs + + deleted: Final = candidate.request("DELETE", "/config_overrides/hashicorp_vault") + assert deleted.status_code == 200, deleted.text + assert deleted.json() == { + "message": "Hashicorp Vault configuration deleted successfully", + "status": "success", + }, deleted.text + vault_wire.drain() + vault_tls_wire.drain() + assert ( + read_rows( + 'SELECT config_type FROM "LiteLLM_ConfigOverrides" WHERE config_type = %s', + ("hashicorp_vault",), + database_url=database_url, + ) + == [] + ) + model_after_delete: Final = scenario.model( + api_key="os.environ/LITELLM_MASTER_KEY", + api_base=provider_wire.url + "/v1", + ) + response_after_delete: Final = candidate.request( + "POST", + "/v1/chat/completions", + { + "model": model_after_delete, + "messages": [{"role": "user", "content": "after-delete"}], + }, + ) + assert response_after_delete.status_code == 200, response_after_delete.text + assert response_after_delete.json()["choices"][0]["message"]["content"] == "after-delete", ( + response_after_delete.text + ) + after_delete_requests: Final = vault_wire.drain() + assert after_delete_requests == (), after_delete_requests + after_delete_tls_requests: Final = vault_tls_wire.drain() + assert after_delete_tls_requests == (), after_delete_tls_requests + + provider_requests: Final = provider_wire.drain() + assert tuple((request.method, request.target) for request in provider_requests) == ( + ("POST", "/v1/chat/completions"), + ("POST", "/v1/chat/completions"), + ("POST", "/v1/chat/completions"), + ), provider_requests From 27209b92b3236f6ff05ea5a19b45a73cc30187ca Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:00:38 +0000 Subject: [PATCH 10/51] test(integration): cover mcp_runtime endpoint contracts (#44671) * test(integration): cover mcp_runtime endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(integration): ruff format and import order in mcp runtime tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): assert out-of-path calls and untrusted forwarded headers are refused Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): move the untrusted forwarded-header discovery case into its own test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): hash the token row lookup, widen spend polling, and union open servers after listing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): tighten mcp runtime contracts per review (sdk client, toolset scope, forwarded trust, callback html, session tools/call) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): assert the sdk unknown-path error without pinning the sdk's exception wrappers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover trailing-path and group csv mcp paths, rfc 9207 iss-less and error-iss callbacks, and id_token exchange Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover lite logout spelling, team-mapped and unmapped IdP subjects Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover per-server mcp-session resource and root /token spelling Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): unpack the sealed client code in the step-up tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/mcp/test_mcp_oauth_flows.py | 1357 ++++++++++++++++- tests/integration/mcp/test_mcp_transports.py | 406 ++++- .../mcp/test_oauth_configuration.py | 248 +++ 3 files changed, 1987 insertions(+), 24 deletions(-) diff --git a/tests/integration/mcp/test_mcp_oauth_flows.py b/tests/integration/mcp/test_mcp_oauth_flows.py index 0bd299f3bbd..4fe2821d7aa 100644 --- a/tests/integration/mcp/test_mcp_oauth_flows.py +++ b/tests/integration/mcp/test_mcp_oauth_flows.py @@ -1,21 +1,24 @@ import base64 import hashlib import json +import os import re import secrets import textwrap import time import uuid -from collections.abc import Iterator +from collections.abc import Iterator, Mapping from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path from typing import Final -from urllib.parse import parse_qs, urlsplit +from urllib.parse import parse_qs, urljoin, urlsplit import httpx import jwt import pytest +import yaml +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, generate_private_key from integration._support.client import Gateway, eventually, gateway_from_environment from integration._support.database import read_rows from integration._support.mcp import ( @@ -38,6 +41,7 @@ from integration._support.mcp_grants import create_toolset from integration._support.oauth_server import AuthorizationServer, oauth_server from integration._support.process import owned_proxy from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import TypeAdapter ADD: Final = {"a": 2, "b": 3} CLIENT_REDIRECT: Final = "http://127.0.0.1:9/cb" @@ -48,6 +52,33 @@ def _base(gateway: Gateway) -> str: return str(gateway.client.base_url).rstrip("/") +def _response_object(response: httpx.Response) -> dict[str, object]: + return TypeAdapter(dict[str, object]).validate_python(response.json()) + + +def _open_aliases() -> frozenset[str]: + rows: Final = read_rows('SELECT alias FROM "LiteLLM_MCPServerTable" WHERE allow_all_keys', ()) + return frozenset(str(row["alias"]) for row in rows) + + +def _without_foreign_open_servers(tools: tuple[str, ...], open_aliases: frozenset[str]) -> set[str]: + prefixes: Final = tuple(f"{alias}-" for alias in open_aliases) + return {tool for tool in tools if not tool.startswith(prefixes)} + + +def _assert_upstream_call(call: Mapping[str, object], name: str, arguments: Mapping[str, object]) -> None: + body: Final = call["body"] + assert isinstance(body, Mapping), call + params: Final = body["params"] + assert isinstance(params, Mapping), call + assert set(params) == {"name", "arguments", "_meta"}, call + assert params["name"] == name, call + assert params["arguments"] == arguments, call + metadata: Final = params["_meta"] + assert isinstance(metadata, Mapping), call + assert set(metadata) == {"progressToken"}, call + + def _authorizations(peer: McpPeer) -> tuple[bytes | None, ...]: return tuple( value if isinstance(value := call["headers"].get(b"authorization"), bytes) else None @@ -246,9 +277,14 @@ class _Pkce: def _authorize_through_gateway( - gateway: Gateway, auth: AuthorizationServer, alias: str, key: str, client_id: str, pkce: _Pkce, + gateway: Gateway, + auth: AuthorizationServer, + alias: str, + key: str, + client_id: str, + pkce: _Pkce, scope: str = "tools.call", -) -> str: +) -> tuple[str, str]: started: Final = gateway.client.get( f"/{alias}/authorize", params={ @@ -279,7 +315,8 @@ def _authorize_through_gateway( final: Final = parse_qs(urlsplit(returned.headers["location"]).query) assert returned.headers["location"].startswith(CLIENT_REDIRECT) assert final["state"] == ["client-state"], final - return final["code"][0] + upstream_code: Final = parse_qs(urlsplit(callback).query)["code"][0] + return final["code"][0], upstream_code def _redeem(gateway: Gateway, alias: str, key: str, client_id: str, code: str, pkce: _Pkce) -> httpx.Response: @@ -296,6 +333,187 @@ def _redeem(gateway: Gateway, alias: str, key: str, client_id: str, code: str, p ) +def _expected_authorization_header(auth_method: str, secret: str) -> str | None: + if auth_method == "client_secret_post": + return None + return "Basic " + base64.b64encode(f"ac-client:{secret}".encode()).decode() + + +def _assert_upstream_token_request( + request: Request, auth_method: str, secret: str, expected_form: Mapping[str, str] +) -> None: + """One scripted-IdP token request must carry the gateway's exact form and, for + client_secret_basic, the stored credentials in a Basic header, and never the + caller's placeholder secret.""" + form: Final = {name: values[0] for name, values in parse_qs(request.body.decode()).items()} + assert form == dict(expected_form), form + assert request.headers.get("authorization") == _expected_authorization_header(auth_method, secret), request.headers + assert "dummy" not in request.body.decode() and "dummy" not in repr(request.headers), request + + +def _upstream_token_requests(auth: AuthorizationServer) -> tuple[Request, ...]: + return tuple(item for item in auth.drain() if item.target.startswith("/token")) + + +@pytest.mark.parametrize("auth_method", ("client_secret_post", "client_secret_basic")) +def test_upstream_token_requests_carry_the_gateway_callback_and_server_credentials( + gateway: Gateway, auth_method: str +) -> None: + with mcp_peer() as peer, oauth_server() as auth, gateway.scenario() as scenario: + alias: Final = "rt3" + uuid.uuid4().hex[:10] + secret: Final = "rt3-secret-" + uuid.uuid4().hex + identity: Final = _register_oauth( + scenario, + peer, + auth, + alias, + auth_type="oauth2", + oauth2_flow="authorization_code", + credentials={ + "client_id": "ac-client", + "client_secret": secret, + "scopes": ["tools.call"], + "token_endpoint_auth_method": auth_method, + }, + ) + # A second oauth2 server keeps the root /token path honest: with two registered + # oauth2 servers the single-server root fallback cannot resolve, so the root + # spelling works only when client_id itself names the server. + _register_oauth( + scenario, + peer, + auth, + "rt3o" + uuid.uuid4().hex[:9], + auth_type="oauth2", + oauth2_flow="authorization_code", + credentials={"client_id": "ac-other", "client_secret": "other-secret", "scopes": ["tools.call"]}, + ) + persisted: Final = gateway.request("GET", f"/v1/mcp/server/{identity}") + assert persisted.status_code == 200, persisted.text + persisted_body: Final = _response_object(persisted) + assert persisted_body["credentials"] == {"scopes": ["tools.call"]}, persisted.text + owner: Final = scenario.key(object_permission={"mcp_servers": [identity]}) + registered: Final = gateway.client.post( + f"/{alias}/register", json={"redirect_uris": [CLIENT_REDIRECT], "client_name": "integration"} + ) + assert registered.status_code in (200, 201), registered.text + client_id: Final = str(_response_object(registered)["client_id"]) + + def expected_code_form(code: str, verifier: str) -> dict[str, str]: + return { + "grant_type": "authorization_code", + "code": code, + "redirect_uri": f"{_base(gateway)}/callback", + "code_verifier": verifier, + **({"client_id": "ac-client", "client_secret": secret} if auth_method == "client_secret_post" else {}), + } + + def expected_refresh_form(refresh_token: str) -> dict[str, str]: + return { + "grant_type": "refresh_token", + "refresh_token": refresh_token, + "scope": "tools.call", + **({"client_id": "ac-client", "client_secret": secret} if auth_method == "client_secret_post" else {}), + } + + pkce: Final = _Pkce(secrets.token_urlsafe(32)) + code, upstream_code = _authorize_through_gateway(gateway, auth, alias, owner, client_id, pkce) + authorize_requests: Final = tuple(item for item in auth.drain() if item.target.startswith("/authorize")) + assert len(authorize_requests) == 1, authorize_requests + authorize_query: Final = parse_qs(urlsplit(authorize_requests[0].target).query) + assert authorize_query["redirect_uri"] == [f"{_base(gateway)}/callback"], authorize_query + + redeemed: Final = gateway.client.post( + f"/{alias}/token", + headers={"x-litellm-api-key": owner}, + data={ + "grant_type": "authorization_code", + "code": code, + "code_verifier": pkce.verifier, + "client_id": client_id, + "client_secret": "dummy", + "redirect_uri": CLIENT_REDIRECT, + }, + ) + assert redeemed.status_code == 200, redeemed.text + issued: Final = _response_object(redeemed) + initial_token_requests: Final = _upstream_token_requests(auth) + assert len(initial_token_requests) == 1, initial_token_requests + _assert_upstream_token_request( + initial_token_requests[0], auth_method, secret, expected_code_form(upstream_code, pkce.verifier) + ) + + upstream_refresh_token: Final = str(issued["refresh_token"]) + refreshed: Final = gateway.client.post( + f"/{alias}/token", + headers={"x-litellm-api-key": owner}, + data={ + "grant_type": "refresh_token", + "refresh_token": upstream_refresh_token, + "client_id": client_id, + "client_secret": "dummy", + "scope": "tools.call", + }, + ) + assert refreshed.status_code == 200, refreshed.text + assert _response_object(refreshed)["access_token"] != issued["access_token"], refreshed.text + refresh_requests: Final = _upstream_token_requests(auth) + assert len(refresh_requests) == 1, refresh_requests + _assert_upstream_token_request( + refresh_requests[0], auth_method, secret, expected_refresh_form(upstream_refresh_token) + ) + + pkce_root: Final = _Pkce(secrets.token_urlsafe(32)) + root_code, root_upstream_code = _authorize_through_gateway(gateway, auth, alias, owner, client_id, pkce_root) + root_authorize_requests: Final = tuple(item for item in auth.drain() if item.target.startswith("/authorize")) + assert len(root_authorize_requests) == 1, root_authorize_requests + root_authorize_query: Final = parse_qs(urlsplit(root_authorize_requests[0].target).query) + assert root_authorize_query["redirect_uri"] == [f"{_base(gateway)}/callback"], root_authorize_query + + root_redeemed: Final = gateway.client.post( + "/token", + headers={"x-litellm-api-key": owner}, + data={ + "grant_type": "authorization_code", + "code": root_code, + "code_verifier": pkce_root.verifier, + "client_id": alias, + "client_secret": "dummy", + "redirect_uri": CLIENT_REDIRECT, + }, + ) + assert root_redeemed.status_code == 200, root_redeemed.text + root_issued: Final = _response_object(root_redeemed) + root_token_requests: Final = _upstream_token_requests(auth) + assert len(root_token_requests) == 1, root_token_requests + _assert_upstream_token_request( + root_token_requests[0], + auth_method, + secret, + expected_code_form(root_upstream_code, pkce_root.verifier), + ) + + root_refresh_token: Final = str(root_issued["refresh_token"]) + root_refreshed: Final = gateway.client.post( + "/token", + headers={"x-litellm-api-key": owner}, + data={ + "grant_type": "refresh_token", + "refresh_token": root_refresh_token, + "client_id": alias, + "client_secret": "dummy", + "scope": "tools.call", + }, + ) + assert root_refreshed.status_code == 200, root_refreshed.text + assert _response_object(root_refreshed)["access_token"] != root_issued["access_token"], root_refreshed.text + root_refresh_requests: Final = _upstream_token_requests(auth) + assert len(root_refresh_requests) == 1, root_refresh_requests + _assert_upstream_token_request( + root_refresh_requests[0], auth_method, secret, expected_refresh_form(root_refresh_token) + ) + + def test_per_user_authorization_code_with_pkce_binds_the_token_to_the_authorizing_user(gateway: Gateway) -> None: with mcp_peer() as peer, oauth_server() as auth, gateway.scenario() as scenario: alias: Final = "ac" + uuid.uuid4().hex[:8] @@ -321,13 +539,13 @@ def test_per_user_authorization_code_with_pkce_binds_the_token_to_the_authorizin f"/{alias}/register", json={"redirect_uris": [CLIENT_REDIRECT], "client_name": "integration"} ) assert registered.status_code in (200, 201), registered.text - client_id: Final = registered.json()["client_id"] + client_id: Final = str(_response_object(registered)["client_id"]) pkce: Final = _Pkce(secrets.token_urlsafe(32)) - code: Final = _authorize_through_gateway(gateway, auth, alias, owner, client_id, pkce) + code, _ = _authorize_through_gateway(gateway, auth, alias, owner, client_id, pkce) wrong_verifier: Final = _redeem(gateway, alias, owner, client_id, code, _Pkce("wrong-" + pkce.verifier)) assert wrong_verifier.status_code == 400, wrong_verifier.text assert tool_calls(peer.drain()) == () - code2: Final = _authorize_through_gateway(gateway, auth, alias, owner, client_id, pkce) + code2, _ = _authorize_through_gateway(gateway, auth, alias, owner, client_id, pkce) redeemed: Final = _redeem(gateway, alias, owner, client_id, code2, pkce) assert redeemed.status_code == 200, redeemed.text issued: Final = redeemed.json() @@ -408,12 +626,14 @@ def _ui_session_cookie(gateway: Gateway, user_id: str) -> dict[str, str]: return {"token": jwt.encode(claims, gateway.key, algorithm="HS256")} -def _gateway_session_bearer(gateway: Gateway, user_id: str, resource: str | None = None) -> str: +def _gateway_session_sign_in( + gateway: Gateway, user_id: str, resource: str | None = None +) -> tuple[dict[str, object], str]: registered: Final = gateway.client.post( "/register", json={"redirect_uris": [CLIENT_REDIRECT], "client_name": "integration"} ) assert registered.status_code in (200, 201), registered.text - client_id: Final = registered.json()["client_id"] + client_id: Final = str(_response_object(registered)["client_id"]) pkce: Final = _Pkce(secrets.token_urlsafe(48)) cookies: Final = _ui_session_cookie(gateway, user_id) started: Final = gateway.client.get( @@ -448,7 +668,461 @@ def _gateway_session_bearer(gateway: Gateway, user_id: str, resource: str | None }, ) assert issued.status_code == 200, issued.text - return _issued_token(issued.json()) + return _response_object(issued), client_id + + +def _gateway_session_bearer(gateway: Gateway, user_id: str, resource: str | None = None) -> str: + issued, _ = _gateway_session_sign_in(gateway, user_id, resource) + return _issued_token(issued) + + +def test_spec_client_discovers_the_aggregate_authorization_server_and_signs_in_to_a_session_bearer( + gateway: Gateway, +) -> None: + base: Final = _base(gateway) + protected: Final = gateway.client.get("/.well-known/oauth-protected-resource/mcp") + assert protected.status_code == 200, protected.text + assert _response_object(protected) == { + "authorization_servers": [f"{base}/mcp"], + "resource": f"{base}/mcp", + "scopes_supported": [], + }, protected.text + authorization_server: Final = gateway.client.get("/.well-known/oauth-authorization-server/mcp") + assert authorization_server.status_code == 200, authorization_server.text + authorization_metadata: Final = _response_object(authorization_server) + assert authorization_metadata == { + "issuer": f"{base}/mcp", + "authorization_endpoint": f"{base}/authorize/mcp-session", + "token_endpoint": f"{base}/token", + "introspection_endpoint": f"{base}/introspect", + "registration_endpoint": f"{base}/register", + "response_types_supported": ["code"], + "scopes_supported": [], + "grant_types_supported": ["authorization_code", "refresh_token"], + "code_challenge_methods_supported": ["S256"], + "token_endpoint_auth_methods_supported": ["none", "client_secret_post"], + }, authorization_server.text + registration_endpoint: Final = urlsplit(str(authorization_metadata["registration_endpoint"])).path + authorization_endpoint: Final = urlsplit(str(authorization_metadata["authorization_endpoint"])).path + token_endpoint: Final = urlsplit(str(authorization_metadata["token_endpoint"])).path + with mcp_peer() as peer, gateway.scenario() as scenario: + alias: Final = "rt5" + uuid.uuid4().hex[:10] + identity: Final = register_mcp(scenario, peer, alias) + member: Final = scenario.member(scenario.team(object_permission={"mcp_servers": [identity]})) + registered: Final = gateway.client.post( + registration_endpoint, json={"redirect_uris": [CLIENT_REDIRECT], "client_name": "integration"} + ) + assert registered.status_code in (200, 201), registered.text + client_id: Final = str(_response_object(registered)["client_id"]) + pkce: Final = _Pkce(secrets.token_urlsafe(48)) + cookies: Final = _ui_session_cookie(gateway, member) + started: Final = gateway.client.get( + authorization_endpoint, + params={ + "client_id": client_id, + "redirect_uri": CLIENT_REDIRECT, + "response_type": "code", + "state": "rt5-state", + "code_challenge": pkce.challenge, + "code_challenge_method": "S256", + "resource": f"{base}/mcp", + }, + cookies=cookies, + ) + assert started.status_code == 303, started.text + connect_location: Final = urlsplit(started.headers["location"]) + assert ( + connect_location.scheme + "://" + connect_location.netloc + connect_location.path == f"{base}/ui/connect" + ), started.headers["location"] + connect_flow: Final = parse_qs(connect_location.query)["connect_flow"][0] + flow_cookie: Final = f"mcp_connect_flow_{connect_flow}" + assert flow_cookie in started.cookies, started.headers + flow_cookies: Final = {**cookies, flow_cookie: started.cookies[flow_cookie]} + described: Final = gateway.client.get("/authorize/flow", params={"flow": connect_flow}, cookies=flow_cookies) + assert described.status_code == 200, described.text + assert _response_object(described) == { + "state": "unscoped", + "client_origin": "http://127.0.0.1:9", + "server_id": None, + "server_name": None, + "connected": None, + }, described.text + completed: Final = gateway.client.post("/authorize/complete", data={"flow": connect_flow}, cookies=flow_cookies) + assert completed.status_code == 303, completed.text + callback: Final = parse_qs(urlsplit(completed.headers["location"]).query) + assert callback["state"] == ["rt5-state"], completed.headers["location"] + assert set(callback) == {"code", "state"}, callback + token: Final = gateway.client.post( + token_endpoint, + data={ + "grant_type": "authorization_code", + "code": callback["code"][0], + "redirect_uri": CLIENT_REDIRECT, + "client_id": client_id, + "code_verifier": pkce.verifier, + }, + ) + assert token.status_code == 200, token.text + token_body: Final = _response_object(token) + assert set(token_body) == {"access_token", "token_type", "expires_in", "refresh_token"}, token.text + assert token_body["token_type"] == "Bearer", token.text + assert isinstance(token_body["expires_in"], int) and token_body["expires_in"] > 0, token.text + assert str(token_body["access_token"]).startswith("llm_session_"), token.text + assert str(token_body["refresh_token"]).startswith("llm_srefresh_"), token.text + bearer: Final = str(token_body["access_token"]) + initialized: Final = gateway.client.post( + "/mcp", + headers={"Authorization": f"Bearer {bearer}", **ACCEPT}, + json={"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": dict(INITIALIZE)}, + ) + assert initialized.status_code == 200, initialized.text + open_before: Final = _open_aliases() + listed_response: Final = gateway.client.post( + "/mcp", + headers={"Authorization": f"Bearer {bearer}", **ACCEPT}, + json={"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}, + ) + assert listed_response.status_code == 200, listed_response.text + listed: Final = _outcome_from_rpc(listed_response) + assert listed.ok, listed.raw + open_after: Final = open_before | _open_aliases() + assert _without_foreign_open_servers(listed.tools, open_after) == { + f"{alias}-add", + f"{alias}-multiply", + f"{alias}-fail", + }, listed.tools + peer.drain() + called_response: Final = gateway.client.post( + "/mcp", + headers={"Authorization": f"Bearer {bearer}", **ACCEPT}, + json={ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": f"{alias}-add", "arguments": {"a": 20, "b": 22}}, + }, + ) + assert called_response.status_code == 200, called_response.text + called: Final = _outcome_from_rpc(called_response) + assert called.ok and called.text == "42", called.raw + calls: Final = tool_calls(peer.drain()) + assert len(calls) == 1, calls + _assert_upstream_call(calls[0], "add", {"a": 20, "b": 22}) + + +def test_per_server_resource_scopes_the_session_bearer_to_the_named_server(gateway: Gateway) -> None: + base: Final = _base(gateway) + with mcp_peer() as peer_a, mcp_peer() as peer_b, gateway.scenario() as scenario: + alias_a: Final = "sca" + uuid.uuid4().hex[:10] + alias_b: Final = "scb" + uuid.uuid4().hex[:10] + server_a: Final = register_mcp(scenario, peer_a, alias_a) + server_b: Final = register_mcp(scenario, peer_b, alias_b) + member: Final = scenario.member(scenario.team(object_permission={"mcp_servers": [server_a, server_b]})) + registered: Final = gateway.client.post( + "/register", json={"redirect_uris": [CLIENT_REDIRECT], "client_name": "integration"} + ) + assert registered.status_code in (200, 201), registered.text + client_id: Final = str(_response_object(registered)["client_id"]) + pkce: Final = _Pkce(secrets.token_urlsafe(48)) + cookies: Final = _ui_session_cookie(gateway, member) + started: Final = gateway.client.get( + "/authorize/mcp-session", + params={ + "client_id": client_id, + "redirect_uri": CLIENT_REDIRECT, + "response_type": "code", + "state": "sca-state", + "code_challenge": pkce.challenge, + "code_challenge_method": "S256", + "resource": f"{base}/mcp/{alias_a}", + }, + cookies=cookies, + ) + assert started.status_code == 303, started.text + connect_location: Final = urlsplit(started.headers["location"]) + assert ( + connect_location.scheme + "://" + connect_location.netloc + connect_location.path == f"{base}/ui/connect" + ), started.headers["location"] + connect_flow: Final = parse_qs(connect_location.query)["connect_flow"][0] + flow_cookie: Final = f"mcp_connect_flow_{connect_flow}" + assert flow_cookie in started.cookies, started.headers + flow_cookies: Final = {**cookies, flow_cookie: started.cookies[flow_cookie]} + described: Final = gateway.client.get("/authorize/flow", params={"flow": connect_flow}, cookies=flow_cookies) + assert described.status_code == 200, described.text + assert _response_object(described) == { + "state": "m2m", + "client_origin": "http://127.0.0.1:9", + "server_id": server_a, + "server_name": alias_a, + "connected": True, + }, described.text + completed: Final = gateway.client.post("/authorize/complete", data={"flow": connect_flow}, cookies=flow_cookies) + assert completed.status_code == 303, completed.text + callback: Final = parse_qs(urlsplit(completed.headers["location"]).query) + assert callback["state"] == ["sca-state"], completed.headers["location"] + assert set(callback) == {"code", "state"}, callback + token: Final = gateway.client.post( + "/token", + data={ + "grant_type": "authorization_code", + "code": callback["code"][0], + "redirect_uri": CLIENT_REDIRECT, + "client_id": client_id, + "code_verifier": pkce.verifier, + }, + ) + assert token.status_code == 200, token.text + token_body: Final = _response_object(token) + assert set(token_body) == {"access_token", "token_type", "expires_in", "refresh_token"}, token.text + bearer: Final = str(token_body["access_token"]) + assert bearer.startswith("llm_session_"), token.text + initialized: Final = gateway.client.post( + "/mcp", + headers={"Authorization": f"Bearer {bearer}", **ACCEPT}, + json={"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": dict(INITIALIZE)}, + ) + assert initialized.status_code == 200, initialized.text + open_before: Final = _open_aliases() + listed_response: Final = gateway.client.post( + "/mcp", + headers={"Authorization": f"Bearer {bearer}", **ACCEPT}, + json={"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}, + ) + assert listed_response.status_code == 200, listed_response.text + listed: Final = _outcome_from_rpc(listed_response) + assert listed.ok, listed.raw + open_after: Final = open_before | _open_aliases() + assert _without_foreign_open_servers(listed.tools, open_after) == { + f"{alias_a}-add", + f"{alias_a}-multiply", + f"{alias_a}-fail", + }, listed.tools + peer_a.drain() + peer_b.drain() + called_response: Final = gateway.client.post( + "/mcp", + headers={"Authorization": f"Bearer {bearer}", **ACCEPT}, + json={ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": f"{alias_a}-add", "arguments": {"a": 20, "b": 22}}, + }, + ) + assert called_response.status_code == 200, called_response.text + called: Final = _outcome_from_rpc(called_response) + assert called.ok and called.text == "42", called.raw + calls: Final = tool_calls(peer_a.drain()) + assert len(calls) == 1, calls + _assert_upstream_call(calls[0], "add", {"a": 20, "b": 22}) + assert tool_calls(peer_b.drain()) == () + refused_response: Final = gateway.client.post( + "/mcp", + headers={"Authorization": f"Bearer {bearer}", **ACCEPT}, + json={ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": f"{alias_b}-add", "arguments": ADD}, + }, + ) + refused: Final = _outcome_from_rpc(refused_response) + assert not refused.ok, refused.raw + assert tool_calls(peer_b.drain()) == () + + +def test_rotated_session_refresh_token_cannot_be_replayed_after_revocation(gateway: Gateway) -> None: + with mcp_peer() as peer, gateway.scenario() as scenario: + alias: Final = "rt6" + uuid.uuid4().hex[:10] + identity: Final = register_mcp(scenario, peer, alias) + member: Final = scenario.member(scenario.team(object_permission={"mcp_servers": [identity]})) + issued, client_id = _gateway_session_sign_in(gateway, member) + r1: Final = str(issued["refresh_token"]) + a1: Final = str(issued["access_token"]) + assert r1.startswith("llm_srefresh_"), r1 + assert a1.startswith("llm_session_"), a1 + rotated: Final = gateway.client.post( + "/token", + data={"grant_type": "refresh_token", "refresh_token": r1, "client_id": client_id}, + ) + assert rotated.status_code == 200, rotated.text + rotated_body: Final = _response_object(rotated) + assert set(rotated_body) == {"access_token", "token_type", "expires_in", "refresh_token"}, rotated.text + r2: Final = str(rotated_body["refresh_token"]) + a2: Final = str(rotated_body["access_token"]) + assert r2 != r1 and r2.startswith("llm_srefresh_"), rotated.text + assert a2 != a1 and a2.startswith("llm_session_"), rotated.text + replayed: Final = gateway.client.post( + "/token", + data={"grant_type": "refresh_token", "refresh_token": r1, "client_id": client_id}, + ) + assert replayed.status_code == 400, replayed.text + assert _response_object(replayed) == { + "error": "invalid_grant", + "error_description": "the refresh token was already used", + }, replayed.text + revoked: Final = gateway.client.post("/revoke", data={"token": r2, "client_id": client_id}) + assert revoked.status_code == 200, revoked.text + assert _response_object(revoked) == {}, revoked.text + revoked_again: Final = gateway.client.post("/revoke", data={"token": r2, "client_id": client_id}) + assert revoked_again.status_code == 200, revoked_again.text + assert _response_object(revoked_again) == {}, revoked_again.text + after_revoke: Final = gateway.client.post( + "/token", + data={"grant_type": "refresh_token", "refresh_token": r2, "client_id": client_id}, + ) + assert after_revoke.status_code == 400, after_revoke.text + assert _response_object(after_revoke) == { + "error": "invalid_grant", + "error_description": "the refresh token was revoked", + }, after_revoke.text + unknown_client: Final = gateway.client.post( + "/revoke", data={"token": r2, "client_id": "dcr-unknown-" + uuid.uuid4().hex} + ) + assert unknown_client.status_code == 401, unknown_client.text + assert _response_object(unknown_client) == { + "error": "invalid_client", + "error_description": "unknown or malformed client_id", + }, unknown_client.text + open_before: Final = _open_aliases() + listed_response: Final = gateway.client.post( + "/mcp", + headers={"Authorization": f"Bearer {a2}", **ACCEPT}, + json={"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}, + ) + assert listed_response.status_code == 200, listed_response.text + listed: Final = _outcome_from_rpc(listed_response) + assert listed.ok, listed.raw + open_after: Final = open_before | _open_aliases() + assert _without_foreign_open_servers(listed.tools, open_after) == { + f"{alias}-add", + f"{alias}-multiply", + f"{alias}-fail", + }, listed.tools + + +def _consent_form_fields(page: str) -> dict[str, str]: + return dict(re.findall(r'', page)) + + +def test_lite_logout_revokes_the_cli_refresh_token_sent_the_way_the_cli_sends_it(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + team_id: Final = scenario.team() + member: Final = scenario.member(team_id) + discovered: Final = gateway.client.get("/.well-known/litellm-cli-auth") + assert discovered.status_code == 200, discovered.text + contract: Final = _response_object(discovered) + base: Final = _base(gateway) + assert (contract["resource"], contract["token_endpoint"], contract["revocation_endpoint"]) == ( + base, + f"{base}/token", + f"{base}/revoke", + ), discovered.text + resource: Final = str(contract["resource"]) + registered: Final = gateway.client.post( + str(contract["registration_endpoint"]), + json={ + "client_name": "LiteLLM CLI", + "redirect_uris": [CLIENT_REDIRECT], + "grant_types": ["authorization_code", "refresh_token"], + "response_types": ["code"], + "token_endpoint_auth_method": "none", + }, + ) + assert registered.status_code in (200, 201), registered.text + client_id: Final = str(_response_object(registered)["client_id"]) + pkce: Final = _Pkce(secrets.token_urlsafe(64)) + cookies: Final = _ui_session_cookie(gateway, member) + consent: Final = gateway.client.get( + str(contract["authorization_endpoint"]), + params={ + "response_type": "code", + "client_id": client_id, + "redirect_uri": CLIENT_REDIRECT, + "state": "lite-login-state", + "code_challenge": pkce.challenge, + "code_challenge_method": "S256", + "resource": resource, + }, + cookies=cookies, + ) + assert consent.status_code == 200, consent.text + consent_fields: Final = _consent_form_fields(consent.text) + assert set(consent_fields) == {"flow", "team_id"} and consent_fields["team_id"] == team_id, consent.text + approved: Final = gateway.client.post( + "/authorize/complete", + data={**consent_fields, "decision": "approve"}, + cookies={**cookies, **dict(consent.cookies)}, + ) + assert approved.status_code == 303, approved.text + callback: Final = parse_qs(urlsplit(approved.headers["location"]).query) + assert callback["state"] == ["lite-login-state"], approved.headers["location"] + issued: Final = gateway.client.post( + str(contract["token_endpoint"]), + data={ + "grant_type": "authorization_code", + "code": callback["code"][0], + "redirect_uri": CLIENT_REDIRECT, + "client_id": client_id, + "code_verifier": pkce.verifier, + "resource": resource, + }, + ) + assert issued.status_code == 200, issued.text + issued_body: Final = _response_object(issued) + assert (issued_body["user_id"], issued_body["team_id"]) == (member, team_id), issued.text + r1: Final = str(issued_body["refresh_token"]) + assert r1.startswith("llm_srefresh_"), issued.text + + def refresh(refresh_token: str) -> httpx.Response: + return gateway.client.post( + str(contract["token_endpoint"]), + data={ + "grant_type": "refresh_token", + "refresh_token": refresh_token, + "client_id": client_id, + "resource": resource, + }, + ) + + def revoke(refresh_token: str) -> httpx.Response: + return gateway.client.post( + str(contract["revocation_endpoint"]), + data={"token": refresh_token, "token_type_hint": "refresh_token", "client_id": client_id}, + ) + + rotated: Final = refresh(r1) + assert rotated.status_code == 200, rotated.text + rotated_body: Final = _response_object(rotated) + r2: Final = str(rotated_body["refresh_token"]) + assert r2 != r1 and r2.startswith("llm_srefresh_"), rotated.text + assert (rotated_body["user_id"], rotated_body["team_id"]) == (member, team_id), rotated.text + replayed: Final = refresh(r1) + assert replayed.status_code == 400, replayed.text + assert _response_object(replayed) == { + "error": "invalid_grant", + "error_description": "the refresh token was already used", + }, replayed.text + logged_out: Final = revoke(r2) + assert logged_out.status_code == 200, logged_out.text + assert _response_object(logged_out) == {}, logged_out.text + logged_out_again: Final = revoke(r2) + assert logged_out_again.status_code == 200, logged_out_again.text + assert _response_object(logged_out_again) == {}, logged_out_again.text + after_logout: Final = refresh(r2) + assert after_logout.status_code == 400, after_logout.text + assert _response_object(after_logout) == { + "error": "invalid_grant", + "error_description": "the refresh token was revoked", + }, after_logout.text + completion: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "lite logout access token"}]}, + key=str(rotated_body["access_token"]), + ) + assert completion.status_code == 200, completion.text def _toolset_rpc(gateway: Gateway, bearer: str, name: str, method: str, params: dict[str, object]) -> Outcome: @@ -584,6 +1258,623 @@ def test_token_exchange_callers_with_different_subject_tokens_own_separate_listi assert tool_calls(peer.drain()) == (), "a blocked probe reached the peer" +@dataclass(frozen=True, slots=True) +class _IdpRig: + gateway: Gateway + private_key: RSAPrivateKey + other_private_key: RSAPrivateKey + kid: str + dead_jwks: Wire + + +def _proxy_config_with_general_settings(directory: Path, name: str, overrides: Mapping[str, object]) -> Path: + raw_config: Final[object] = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config: Final = TypeAdapter(dict[str, object]).validate_python(raw_config) + general_settings: Final = TypeAdapter(dict[str, object]).validate_python(config["general_settings"]) + merged: Final = {**config, "general_settings": {**general_settings, **overrides}} + config_path: Final = directory / name + config_path.write_text(yaml.safe_dump(merged)) + return config_path + + +def _idp_proxy_config(directory: Path, team_id_jwt_field: str | None = None) -> Path: + return _proxy_config_with_general_settings( + directory, + "mcp_idp.yaml", + { + "enable_jwt_auth": True, + "litellm_jwtauth": { + "user_id_jwt_field": "sub", + "user_id_upsert": True, + **({} if team_id_jwt_field is None else {"team_id_jwt_field": team_id_jwt_field}), + }, + "use_x_forwarded_for": True, + "mcp_trusted_proxy_ranges": ["127.0.0.1/32"], + }, + ) + + +def _signed_subject_token(private_key: RSAPrivateKey, claims: Mapping[str, str], kid: str) -> str: + now: Final = int(time.time()) + return jwt.encode({**claims, "iat": now, "exp": now + 300}, private_key, algorithm="RS256", headers={"kid": kid}) + + +def _token_exchange_form(client_id: str, subject_token: str) -> dict[str, str]: + return { + "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", + "client_id": client_id, + "subject_token": subject_token, + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + "requested_token_type": "urn:ietf:params:oauth:token-type:access_token", + } + + +def _registered_dcr_client(gateway: Gateway) -> str: + registered: Final = gateway.client.post( + "/register", json={"redirect_uris": [CLIENT_REDIRECT], "client_name": "integration"} + ) + assert registered.status_code in (200, 201), registered.text + return str(_response_object(registered)["client_id"]) + + +@contextmanager +def _idp_gateway(directory: Path, team_id_jwt_field: str | None, workers: int) -> Iterator[_IdpRig]: + assert os.environ.get("LITELLM_LICENSE"), ( + "enable_jwt_auth is enterprise-only: these tests need LITELLM_LICENSE (CI forwards it); " + "without it /token answers 400 'JWT auth is an enterprise only feature; no license is set' " + "and the failure reads as a regression instead of a missing license" + ) + private_key: Final = generate_private_key(public_exponent=65537, key_size=2048) + other_private_key: Final = generate_private_key(public_exponent=65537, key_size=2048) + kid: Final = "kid-a" + public_jwk: Final = TypeAdapter(dict[str, object]).validate_python( + json.loads(jwt.algorithms.RSAAlgorithm.to_jwk(private_key.public_key())) + ) + + def respond_good(request: Request) -> Reply: + assert request.method == "GET", request + return Reply(body=json.dumps({"keys": [{**public_jwk, "kid": kid}]}).encode()) + + def respond_dead(request: Request) -> Reply: + assert request.method == "GET", request + return Reply(drop_connection=True) + + config_path: Final = _idp_proxy_config(directory, team_id_jwt_field) + with ( + wire_server(respond_good) as good_jwks, + wire_server(respond_dead) as dead_jwks, + gateway_from_environment() as gateway, + owned_proxy( + gateway, + directory, + {"JWT_PUBLIC_KEY_URL": f"{good_jwks.url},{dead_jwks.url}"}, + config=config_path, + remove_environment=("PROXY_BASE_URL",), + workers=workers, + ) as candidate, + ): + yield _IdpRig(candidate, private_key, other_private_key, kid, dead_jwks) + + +@pytest.fixture(scope="module") +def idp_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_IdpRig]: + with _idp_gateway(tmp_path_factory.mktemp("mcp-idp"), None, workers=2) as rig: + yield rig + + +@pytest.fixture(scope="module") +def idp_team_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_IdpRig]: + with _idp_gateway(tmp_path_factory.mktemp("mcp-idp-team"), "team_id", workers=1) as rig: + yield rig + + +@pytest.fixture(scope="module") +def forwarded_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("mcp-forwarded") + config: Final = _proxy_config_with_general_settings( + directory, + "mcp_forwarded.yaml", + {"use_x_forwarded_for": True, "mcp_trusted_proxy_ranges": ["127.0.0.1/32"]}, + ) + with ( + gateway_from_environment() as gateway, + owned_proxy( + gateway, + directory, + {"FORWARDED_ALLOW_IPS": "192.0.2.1"}, + config=config, + remove_environment=("PROXY_BASE_URL",), + workers=1, + ) as candidate, + ): + yield candidate + + +@pytest.fixture( + scope="module", + params=( + {}, + {"use_x_forwarded_for": True, "mcp_trusted_proxy_ranges": ["192.0.2.0/24"]}, + ), + ids=("xff-off", "outside-trusted-range"), +) +def untrusted_gateway(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("mcp-untrusted-forwarded") + config: Final = _proxy_config_with_general_settings( + directory, + "mcp_untrusted.yaml", + TypeAdapter(dict[str, object]).validate_python(request.param), + ) + with ( + gateway_from_environment() as gateway, + owned_proxy( + gateway, + directory, + {"FORWARDED_ALLOW_IPS": "192.0.2.1"}, + config=config, + remove_environment=("PROXY_BASE_URL",), + workers=1, + ) as candidate, + ): + yield candidate + + +def test_idp_subject_token_exchange_mints_a_credential_for_the_mapped_user_and_refuses_bad_subjects( + idp_rig: _IdpRig, +) -> None: + registered: Final = idp_rig.gateway.client.post( + "/register", json={"redirect_uris": [CLIENT_REDIRECT], "client_name": "integration"} + ) + assert registered.status_code in (200, 201), registered.text + client_id: Final = str(_response_object(registered)["client_id"]) + subject: Final = "mcp-idp-user-" + uuid.uuid4().hex + + def token(private_key: RSAPrivateKey, user_id: str, kid: str) -> str: + now: Final = int(time.time()) + return jwt.encode( + {"sub": user_id, "iat": now, "exp": now + 300}, + private_key, + algorithm="RS256", + headers={"kid": kid}, + ) + + valid_subject: Final = token(idp_rig.private_key, subject, idp_rig.kid) + exchange_form: Final = { + "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", + "client_id": client_id, + "subject_token": valid_subject, + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + "requested_token_type": "urn:ietf:params:oauth:token-type:access_token", + } + id_token_exchange: Final = idp_rig.gateway.client.post( + "/token", + data={ + **exchange_form, + "subject_token_type": "urn:ietf:params:oauth:token-type:id_token", + }, + ) + assert id_token_exchange.status_code == 200, id_token_exchange.text + id_token_exchange_body: Final = _response_object(id_token_exchange) + assert set(id_token_exchange_body) == { + "access_token", + "token_type", + "expires_in", + "refresh_token", + "user_id", + "team_id", + "issued_token_type", + }, id_token_exchange.text + assert id_token_exchange_body["token_type"] == "Bearer", id_token_exchange.text + assert id_token_exchange_body["issued_token_type"] == "urn:ietf:params:oauth:token-type:access_token", ( + id_token_exchange.text + ) + assert id_token_exchange_body["user_id"] == subject, id_token_exchange.text + assert id_token_exchange_body["team_id"] is None, id_token_exchange.text + assert isinstance(id_token_exchange_body["expires_in"], int) and id_token_exchange_body["expires_in"] > 0, ( + id_token_exchange.text + ) + assert str(id_token_exchange_body["refresh_token"]).startswith("llm_srefresh_"), id_token_exchange.text + exchanged: Final = idp_rig.gateway.client.post("/token", data=exchange_form) + assert exchanged.status_code == 200, exchanged.text + outage_form: Final = { + **exchange_form, + "subject_token": token(idp_rig.other_private_key, subject, "kid-b"), + } + unavailable: Final = idp_rig.gateway.client.post("/token", data=outage_form) + dead_requests: Final = idp_rig.dead_jwks.drain() + assert unavailable.status_code == 503, unavailable.text + assert _response_object(unavailable) == { + "error": "temporarily_unavailable", + "error_description": ( + "the gateway could not verify subject_token because its identity provider or database is unavailable; retry" + ), + }, unavailable.text + assert tuple(request.method for request in dead_requests) == ("GET", "GET", "GET"), dead_requests + exchanged_body: Final = _response_object(exchanged) + assert set(exchanged_body) == { + "access_token", + "token_type", + "expires_in", + "refresh_token", + "user_id", + "team_id", + "issued_token_type", + }, exchanged.text + assert exchanged_body["token_type"] == "Bearer", exchanged.text + assert exchanged_body["issued_token_type"] == "urn:ietf:params:oauth:token-type:access_token", exchanged.text + assert exchanged_body["user_id"] == subject, exchanged.text + assert exchanged_body["team_id"] is None, exchanged.text + assert isinstance(exchanged_body["expires_in"], int) and exchanged_body["expires_in"] > 0, exchanged.text + assert str(exchanged_body["refresh_token"]).startswith("llm_srefresh_"), exchanged.text + access_token: Final = str(exchanged_body["access_token"]) + id_token_access_token: Final = str(id_token_exchange_body["access_token"]) + with idp_rig.gateway.scenario() as scenario: + model: Final = scenario.model() + completion: Final = idp_rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "mcp idp spend"}]}, + key=access_token, + ) + assert completion.status_code == 200, completion.text + request_id: Final = str(_response_object(completion)["id"]) + id_token_completion: Final = idp_rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "mcp idp id token spend"}]}, + key=id_token_access_token, + ) + assert id_token_completion.status_code == 200, id_token_completion.text + id_token_request_id: Final = str(_response_object(id_token_completion)["id"]) + request_ids: Final = tuple(sorted((request_id, id_token_request_id))) + spend: Final = eventually( + lambda: read_rows( + "SELECT request_id, metadata->>'user_api_key_user_id' AS user_api_key_user_id " + 'FROM "LiteLLM_SpendLogs" WHERE request_id IN (%s, %s) ORDER BY request_id', + request_ids, + ), + lambda rows: len(rows) == 2, + seconds=70, + ) + assert spend == [ + {"request_id": request_ids[0], "user_api_key_user_id": subject}, + {"request_id": request_ids[1], "user_api_key_user_id": subject}, + ], spend + user: Final = eventually( + lambda: read_rows('SELECT user_id, user_role, teams FROM "LiteLLM_UserTable" WHERE user_id = %s', (subject,)), + lambda rows: len(rows) == 1, + seconds=70, + ) + assert user == [{"user_id": subject, "user_role": None, "teams": []}], user + access_token_hash: Final = hashlib.sha256(access_token.encode()).hexdigest() + verification_tokens: Final = read_rows( + 'SELECT token, user_id FROM "LiteLLM_VerificationToken" WHERE token = %s', + (access_token_hash,), + ) + assert verification_tokens == [], verification_tokens + missing_subject: Final = idp_rig.gateway.client.post( + "/token", + data={key: value for key, value in exchange_form.items() if key != "subject_token"}, + ) + assert missing_subject.status_code == 400, missing_subject.text + assert _response_object(missing_subject) == { + "error": "invalid_request", + "error_description": "subject_token and subject_token_type are required", + }, missing_subject.text + rejected_subject: Final = idp_rig.gateway.client.post( + "/token", data={**exchange_form, "subject_token": token(idp_rig.other_private_key, subject, idp_rig.kid)} + ) + assert rejected_subject.status_code == 400, rejected_subject.text + assert _response_object(rejected_subject) == { + "error": "invalid_request", + "error_description": "subject_token was rejected by the gateway's JWT auth", + }, rejected_subject.text + non_jwt: Final = idp_rig.gateway.client.post("/token", data={**exchange_form, "subject_token": "not-a-jwt"}) + assert non_jwt.status_code == 400, non_jwt.text + assert _response_object(non_jwt) == { + "error": "invalid_request", + "error_description": "subject_token is not a JWT", + }, non_jwt.text + + +def test_idp_subject_token_exchange_binds_the_team_claim_and_refuses_a_signed_subject_with_no_user( + idp_team_rig: _IdpRig, +) -> None: + client_id: Final = _registered_dcr_client(idp_team_rig.gateway) + subject: Final = "mcp-idp-team-user-" + uuid.uuid4().hex + with idp_team_rig.gateway.scenario() as scenario: + model: Final = scenario.model() + team_id: Final = scenario.team() + exchanged: Final = idp_team_rig.gateway.client.post( + "/token", + data=_token_exchange_form( + client_id, + _signed_subject_token(idp_team_rig.private_key, {"sub": subject, "team_id": team_id}, idp_team_rig.kid), + ), + ) + assert exchanged.status_code == 200, exchanged.text + body: Final = _response_object(exchanged) + assert (body["user_id"], body["team_id"], body["issued_token_type"]) == ( + subject, + team_id, + "urn:ietf:params:oauth:token-type:access_token", + ), exchanged.text + completion: Final = idp_team_rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "mcp idp team spend"}]}, + key=str(body["access_token"]), + ) + assert completion.status_code == 200, completion.text + request_id: Final = str(_response_object(completion)["id"]) + spend: Final = eventually( + lambda: read_rows( + "SELECT team_id, metadata->>'user_api_key_user_id' AS user_api_key_user_id, " + "metadata->>'user_api_key_team_id' AS user_api_key_team_id " + 'FROM "LiteLLM_SpendLogs" WHERE request_id = %s', + (request_id,), + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + assert spend == [{"team_id": team_id, "user_api_key_user_id": subject, "user_api_key_team_id": team_id}], spend + no_user: Final = idp_team_rig.gateway.client.post( + "/token", + data=_token_exchange_form( + client_id, _signed_subject_token(idp_team_rig.private_key, {"team_id": team_id}, idp_team_rig.kid) + ), + ) + assert no_user.status_code == 400, no_user.text + assert _response_object(no_user) == { + "error": "invalid_request", + "error_description": "subject_token names no user the gateway knows", + }, no_user.text + + +def test_idp_jwks_server_error_answers_temporarily_unavailable(tmp_path: Path) -> None: + pytest.skip( + "BUG: an IdP JWKS answering HTTP 503 makes the gateway /token token-exchange answer 400 invalid_request " + "(subject_token rejected) instead of 503 temporarily_unavailable" + ) + assert os.environ.get("LITELLM_LICENSE"), ( + "enable_jwt_auth is enterprise-only: these tests need LITELLM_LICENSE (CI forwards it); " + "without it /token answers 400 'JWT auth is an enterprise only feature; no license is set' " + "and the failure reads as a regression instead of a missing license" + ) + private_key: Final = generate_private_key(public_exponent=65537, key_size=2048) + kid: Final = "mcp-runtime-503" + + def respond(request: Request) -> Reply: + assert request.method == "GET", request + return Reply(status=503, body=b'{"error":"jwks_down"}') + + config: Final = _idp_proxy_config(tmp_path) + with ( + wire_server(respond) as jwks, + gateway_from_environment() as gateway, + owned_proxy( + gateway, + tmp_path, + {"JWT_PUBLIC_KEY_URL": jwks.url}, + config=config, + remove_environment=("PROXY_BASE_URL",), + workers=2, + ) as idp_gateway, + ): + registered: Final = idp_gateway.client.post( + "/register", json={"redirect_uris": [CLIENT_REDIRECT], "client_name": "integration"} + ) + assert registered.status_code in (200, 201), registered.text + client_id: Final = str(_response_object(registered)["client_id"]) + now: Final = int(time.time()) + subject_token: Final = jwt.encode( + {"sub": "mcp-idp-http-503-" + uuid.uuid4().hex, "iat": now, "exp": now + 300}, + private_key, + algorithm="RS256", + headers={"kid": kid}, + ) + exchange_form: Final = { + "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", + "client_id": client_id, + "subject_token": subject_token, + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + "requested_token_type": "urn:ietf:params:oauth:token-type:access_token", + } + response: Final = idp_gateway.client.post("/token", data=exchange_form) + jwks_requests: Final = jwks.drain() + assert response.status_code == 503, ( + f"request={exchange_form!r}, jwks_requests={jwks_requests!r}, " + f"response_status={response.status_code}, response={response.text}" + ) + assert _response_object(response) == { + "error": "temporarily_unavailable", + "error_description": ( + "the gateway could not verify subject_token because its identity provider or database is unavailable; " + "retry" + ), + }, response.text + + +def test_standard_pattern_discovery_names_the_per_server_issuer_and_follows_forwarded_headers( + gateway: Gateway, forwarded_gateway: Gateway +) -> None: + with mcp_peer() as peer, gateway.scenario() as scenario: + alias: Final = "rt8" + uuid.uuid4().hex[:10] + register_mcp( + scenario, + peer, + alias, + auth_type="oauth2", + oauth2_flow="authorization_code", + credentials={ + "client_id": "rt8-client", + "client_secret": "rt8-secret", + "scopes": ["tools.call"], + }, + ) + base: Final = _base(gateway) + protected: Final = gateway.client.get(f"/.well-known/oauth-protected-resource/mcp/{alias}") + assert protected.status_code == 200, protected.text + aggregate_issuer: Final = f"{base}/mcp" + expected_protected: Final = { + "authorization_servers": [aggregate_issuer], + "resource": f"{base}/mcp/{alias}", + "scopes_supported": ["tools.call"], + } + assert _response_object(protected) == expected_protected, protected.text + aggregate_authorization: Final = gateway.client.get("/.well-known/oauth-authorization-server/mcp") + assert aggregate_authorization.status_code == 200, aggregate_authorization.text + assert _response_object(aggregate_authorization)["issuer"] == aggregate_issuer, aggregate_authorization.text + authorization: Final = gateway.client.get(f"/.well-known/oauth-authorization-server/mcp/{alias}") + assert authorization.status_code == 200, authorization.text + expected_authorization: Final = { + "issuer": f"{base}/mcp/{alias}", + "authorization_endpoint": f"{base}/{alias}/authorize", + "token_endpoint": f"{base}/{alias}/token", + "response_types_supported": ["code"], + "scopes_supported": ["tools.call"], + "grant_types_supported": ["authorization_code", "refresh_token"], + "code_challenge_methods_supported": ["S256"], + "token_endpoint_auth_methods_supported": ["client_secret_post"], + "registration_endpoint": f"{base}/{alias}/register", + } + assert _response_object(authorization) == expected_authorization, authorization.text + legacy_authorization: Final = gateway.client.get(f"/.well-known/oauth-authorization-server/{alias}") + assert legacy_authorization.status_code == 200, legacy_authorization.text + assert _response_object(legacy_authorization) == {**expected_authorization, "issuer": f"{base}/{alias}"}, ( + legacy_authorization.text + ) + legacy_protected: Final = gateway.client.get(f"/.well-known/oauth-protected-resource/{alias}/mcp") + assert legacy_protected.status_code == 200, legacy_protected.text + assert _response_object(legacy_protected) == { + "authorization_servers": [f"{base}/mcp"], + "resource": f"{base}/{alias}/mcp", + "scopes_supported": ["tools.call"], + }, legacy_protected.text + root_protected: Final = gateway.client.get("/.well-known/oauth-protected-resource") + assert root_protected.status_code == 200, root_protected.text + assert _response_object(root_protected) == { + "authorization_servers": [f"{base}/mcp"], + "resource": base, + "scopes_supported": [], + }, root_protected.text + anonymous: Final = gateway.client.post( + f"/mcp/{alias}", + headers=ACCEPT, + json={"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": dict(INITIALIZE)}, + ) + assert anonymous.status_code == 401, anonymous.text + challenge: Final = anonymous.headers["www-authenticate"] + metadata_ref: Final = re.search(r'resource_metadata="([^"]+)"', challenge) + assert metadata_ref is not None, challenge + metadata: Final = gateway.client.get(urljoin(base + "/", metadata_ref.group(1))) + assert metadata.status_code == 200, metadata.text + assert _response_object(metadata) == expected_protected, metadata.text + with mcp_peer() as peer, forwarded_gateway.scenario() as scenario: + alias: Final = "rt8f" + uuid.uuid4().hex[:9] + register_mcp( + scenario, + peer, + alias, + auth_type="oauth2", + oauth2_flow="authorization_code", + credentials={"client_id": "rt8f-client", "client_secret": "rt8f-secret"}, + ) + forwarded_base: Final = "https://gw.example.test" + forwarded_headers: Final = { + "X-Forwarded-Proto": "https", + "X-Forwarded-Host": "gw.example.test", + } + forwarded_prm: Final = forwarded_gateway.client.get( + f"/.well-known/oauth-protected-resource/mcp/{alias}", headers=forwarded_headers + ) + assert forwarded_prm.status_code == 200, forwarded_prm.text + assert _response_object(forwarded_prm) == { + "authorization_servers": [f"{forwarded_base}/mcp"], + "resource": f"{forwarded_base}/mcp/{alias}", + "scopes_supported": [], + }, forwarded_prm.text + forwarded_as: Final = forwarded_gateway.client.get( + f"/.well-known/oauth-authorization-server/mcp/{alias}", headers=forwarded_headers + ) + assert forwarded_as.status_code == 200, forwarded_as.text + assert _response_object(forwarded_as) == { + "issuer": f"{forwarded_base}/mcp/{alias}", + "authorization_endpoint": f"{forwarded_base}/{alias}/authorize", + "token_endpoint": f"{forwarded_base}/{alias}/token", + "response_types_supported": ["code"], + "scopes_supported": [], + "grant_types_supported": ["authorization_code", "refresh_token"], + "code_challenge_methods_supported": ["S256"], + "token_endpoint_auth_methods_supported": ["client_secret_post"], + "registration_endpoint": f"{forwarded_base}/{alias}/register", + }, forwarded_as.text + local_base: Final = _base(forwarded_gateway) + local_as: Final = forwarded_gateway.client.get(f"/.well-known/oauth-authorization-server/mcp/{alias}") + assert local_as.status_code == 200, local_as.text + assert _response_object(local_as) == { + "issuer": f"{local_base}/mcp/{alias}", + "authorization_endpoint": f"{local_base}/{alias}/authorize", + "token_endpoint": f"{local_base}/{alias}/token", + "response_types_supported": ["code"], + "scopes_supported": [], + "grant_types_supported": ["authorization_code", "refresh_token"], + "code_challenge_methods_supported": ["S256"], + "token_endpoint_auth_methods_supported": ["client_secret_post"], + "registration_endpoint": f"{local_base}/{alias}/register", + }, local_as.text + + +def test_standard_pattern_discovery_ignores_forwarded_headers_from_an_untrusted_peer( + untrusted_gateway: Gateway, +) -> None: + with mcp_peer() as peer, untrusted_gateway.scenario() as scenario: + alias: Final = "rt8u" + uuid.uuid4().hex[:9] + register_mcp( + scenario, + peer, + alias, + auth_type="oauth2", + oauth2_flow="authorization_code", + credentials={ + "client_id": "rt8u-client", + "client_secret": "rt8u-secret", + "scopes": ["tools.call"], + }, + ) + base: Final = _base(untrusted_gateway) + untrusted_headers: Final = { + "X-Forwarded-Proto": "https", + "X-Forwarded-Host": "evil.example", + } + authorization: Final = untrusted_gateway.client.get( + f"/.well-known/oauth-authorization-server/mcp/{alias}", headers=untrusted_headers + ) + assert authorization.status_code == 200, authorization.text + assert _response_object(authorization) == { + "issuer": f"{base}/mcp/{alias}", + "authorization_endpoint": f"{base}/{alias}/authorize", + "token_endpoint": f"{base}/{alias}/token", + "response_types_supported": ["code"], + "scopes_supported": ["tools.call"], + "grant_types_supported": ["authorization_code", "refresh_token"], + "code_challenge_methods_supported": ["S256"], + "token_endpoint_auth_methods_supported": ["client_secret_post"], + "registration_endpoint": f"{base}/{alias}/register", + }, authorization.text + assert "evil.example" not in authorization.text, authorization.text + protected: Final = untrusted_gateway.client.get( + f"/.well-known/oauth-protected-resource/mcp/{alias}", headers=untrusted_headers + ) + assert protected.status_code == 200, protected.text + assert _response_object(protected) == { + "authorization_servers": [f"{base}/mcp"], + "resource": f"{base}/mcp/{alias}", + "scopes_supported": ["tools.call"], + }, protected.text + assert "evil.example" not in protected.text, protected.text + + @contextmanager def _scope_enforcing_peer(auth: AuthorizationServer) -> Iterator[tuple[McpPeer, Wire]]: with scripted_peer( @@ -591,6 +1882,7 @@ def _scope_enforcing_peer(auth: AuthorizationServer) -> Iterator[tuple[McpPeer, ScriptedTool("write_op", lambda _: text_result("write completed")), ScriptedTool("forbid_op", lambda _: text_result("must not execute")), ) as inner: + def enforce(request: Request) -> Reply: body: Final = json.loads(request.body) if request.body else {} if body.get("method") == "tools/call": @@ -602,9 +1894,18 @@ def _scope_enforcing_peer(auth: AuthorizationServer) -> Iterator[tuple[McpPeer, with auth.lock: granted: Final = auth.access_tokens.get(token, {}).get("scope", "") if required not in granted.split(): - return Reply(status=403, headers={"WWW-Authenticate": f'Bearer error="insufficient_scope", scope="{required}"'}) - forwarded: Final = httpx.request(request.method, inner.url, content=request.body, headers=dict(request.headers)) - return Reply(status=forwarded.status_code, body=forwarded.content, content_type=forwarded.headers.get("content-type", "application/json")) + return Reply( + status=403, + headers={"WWW-Authenticate": f'Bearer error="insufficient_scope", scope="{required}"'}, + ) + forwarded: Final = httpx.request( + request.method, inner.url, content=request.body, headers=dict(request.headers) + ) + return Reply( + status=forwarded.status_code, + body=forwarded.content, + content_type=forwarded.headers.get("content-type", "application/json"), + ) with wire_server(enforce) as wire: yield McpPeer(wire.url + "/mcp", inner.calls), wire @@ -612,16 +1913,25 @@ def _scope_enforcing_peer(auth: AuthorizationServer) -> Iterator[tuple[McpPeer, @pytest.mark.parametrize("entry", ["rest", "server_mcp"]) def test_managed_scope_step_up_preserves_read_and_write_without_retry(gateway: Gateway, entry: EntryPoint) -> None: - with oauth_server(scopes=("tools.read", "tools.write")) as auth, _scope_enforcing_peer(auth) as (peer, wire), gateway.scenario() as scenario: + with ( + oauth_server(scopes=("tools.read", "tools.write")) as auth, + _scope_enforcing_peer(auth) as (peer, wire), + gateway.scenario() as scenario, + ): alias: Final = "scope" + uuid.uuid4().hex[:8] identity: Final = _register_oauth( - scenario, peer, auth, alias, auth_type="oauth2", oauth2_flow="authorization_code", + scenario, + peer, + auth, + alias, + auth_type="oauth2", + oauth2_flow="authorization_code", credentials={"client_id": "scope-client", "client_secret": "scope-secret", "scopes": ["tools.read"]}, ) owner: Final = scenario.key(user_id=scenario.user(), object_permission={"mcp_servers": [identity]}) stranger: Final = scenario.key(user_id=scenario.user(), object_permission={"mcp_servers": [identity]}) pkce: Final = _Pkce(secrets.token_urlsafe(32)) - code: Final = _authorize_through_gateway(gateway, auth, alias, owner, "scope-client", pkce, "tools.read") + code, _ = _authorize_through_gateway(gateway, auth, alias, owner, "scope-client", pkce, "tools.read") first: Final = _redeem(gateway, alias, owner, "scope-client", code, pkce) assert first.status_code == 200, first.text caller: Final = McpCaller(gateway, owner, entry, alias) @@ -630,16 +1940,21 @@ def test_managed_scope_step_up_preserves_read_and_write_without_retry(gateway: G peer.drain() wire.drain() denied: Final = ( - call_tool(gateway, owner, identity, f"{alias}-write_op", {}) if entry == "rest" + call_tool(gateway, owner, identity, f"{alias}-write_op", {}) + if entry == "rest" else caller.rpc("tools/call", {"name": f"{alias}-write_op", "arguments": {}}) ) assert denied.status_code == 403, denied.text challenge: Final = denied.headers["www-authenticate"] assert 'error="insufficient_scope"' in challenge and 'scope="tools.write"' in challenge assert f'resource_metadata="{_base(gateway)}/.well-known/oauth-protected-resource/' in challenge - attempts: Final = tuple(request for request in wire.drain() if request.body and json.loads(request.body).get("method") == "tools/call") + attempts: Final = tuple( + request + for request in wire.drain() + if request.body and json.loads(request.body).get("method") == "tools/call" + ) assert len(attempts) == 1 and tool_calls(peer.drain()) == (), "denied operation was retried or executed" - second_code: Final = _authorize_through_gateway(gateway, auth, alias, owner, "scope-client", pkce, "tools.write") + second_code, _ = _authorize_through_gateway(gateway, auth, alias, owner, "scope-client", pkce, "tools.write") second: Final = _redeem(gateway, alias, owner, "scope-client", second_code, pkce) assert second.status_code == 200, second.text with auth.lock: @@ -681,7 +1996,7 @@ def test_managed_step_up_advertises_sdk_compatible_metadata_and_bound_permission user_id=scenario.user(), object_permission={"mcp_servers": [identity], "mcp_toolsets": [toolset_id]} ) pkce: Final = _Pkce(secrets.token_urlsafe(32)) - code: Final = _authorize_through_gateway(gateway, auth, alias, owner, "scope-client", pkce, "tools.read") + code, _ = _authorize_through_gateway(gateway, auth, alias, owner, "scope-client", pkce, "tools.read") first: Final = _redeem(gateway, alias, owner, "scope-client", code, pkce) assert first.status_code == 200, first.text path: Final = route.format(toolset=toolset_name) diff --git a/tests/integration/mcp/test_mcp_transports.py b/tests/integration/mcp/test_mcp_transports.py index b448b8b4a64..2b6494914c8 100644 --- a/tests/integration/mcp/test_mcp_transports.py +++ b/tests/integration/mcp/test_mcp_transports.py @@ -1,27 +1,122 @@ +import builtins import json import uuid +from collections.abc import Mapping from typing import Final +import httpx import pytest +from mcp.shared.exceptions import MCPError from integration._support.client import Gateway +from integration._support.database import read_rows from integration._support.mcp import ( ENTRY_POINTS, + INITIALIZE, PEER_KINDS, EntryPoint, McpCaller, + Outcome, PeerKind, + _outcome_from_rpc, + _parse_rpc_body, mcp_peer, official_client_outcomes, peer_of, register_mcp, tool_calls, ) +from integration._support.mcp_grants import create_toolset ADD: Final = {"http": "add", "sse": "add", "stdio": "add", "openapi": "getpet"} ARGUMENTS: Final = {"add": {"a": 3, "b": 4}, "getpet": {"petId": "7"}} EXPECTED: Final = {"add": "7", "getpet": json.dumps({"id": "7", "name": "integration-pet"})} +def _streamable_rpc(gateway: Gateway, path: str, key: str, method: str, params: Mapping[str, object]) -> httpx.Response: + return gateway.client.post( + path, + json={"jsonrpc": "2.0", "id": 1, "method": method, "params": params}, + headers={"Authorization": f"Bearer {key}", "Accept": "application/json, text/event-stream"}, + ) + + +def _streamable_tools(gateway: Gateway, path: str, key: str) -> tuple[httpx.Response, Outcome]: + response: Final = _streamable_rpc(gateway, path, key, "tools/list", {}) + return response, _outcome_from_rpc(response) + + +def _list_and_call( + gateway: Gateway, client: str, path: str, key: str, name: str, arguments: Mapping[str, object] +) -> tuple[Outcome, Outcome]: + if client == "official-sdk": + return official_client_outcomes(gateway, key, path, name, dict(arguments)) + initialized: Final = _streamable_rpc(gateway, path, key, "initialize", dict(INITIALIZE)) + assert initialized.status_code == 200, initialized.text + listed_response, listed = _streamable_tools(gateway, path, key) + assert listed_response.status_code == 200, listed_response.text + called_response: Final = _streamable_rpc( + gateway, path, key, "tools/call", {"name": name, "arguments": dict(arguments)} + ) + assert called_response.status_code == 200, called_response.text + return listed, _outcome_from_rpc(called_response) + + +def _refused_call( + gateway: Gateway, + client: str, + path: str, + key: str, + name: str, + arguments: Mapping[str, object], + text: str = "Error: User not allowed to call this tool.", +) -> Outcome: + expected: Final = text + if client == "official-sdk": + _, refused = official_client_outcomes(gateway, key, path, name, dict(arguments)) + assert (refused.status, refused.error, refused.text) == (200, expected, expected), refused + return refused + response: Final = _streamable_rpc(gateway, path, key, "tools/call", {"name": name, "arguments": dict(arguments)}) + assert response.status_code == 200, response.text + assert _parse_rpc_body(response) == { + "jsonrpc": "2.0", + "id": 1, + "result": { + "content": [{"text": expected, "type": "text"}], + "isError": True, + }, + }, response.text + return _outcome_from_rpc(response) + + +def _leaf_exceptions(error: BaseException) -> tuple[BaseException, ...]: + if isinstance(error, builtins.BaseExceptionGroup): + return tuple(leaf for child in error.exceptions for leaf in _leaf_exceptions(child)) + return (error,) + + +def _assert_upstream_call(call: Mapping[str, object], name: str, arguments: Mapping[str, object]) -> None: + body: Final = call["body"] + assert isinstance(body, Mapping), call + params: Final = body["params"] + assert isinstance(params, Mapping), call + assert set(params) == {"name", "arguments", "_meta"}, call + assert params["name"] == name, call + assert params["arguments"] == arguments, call + metadata: Final = params["_meta"] + assert isinstance(metadata, Mapping), call + assert set(metadata) == {"progressToken"}, call + + +def _open_aliases() -> frozenset[str]: + rows: Final = read_rows('SELECT alias FROM "LiteLLM_MCPServerTable" WHERE allow_all_keys', ()) + return frozenset(str(row["alias"]) for row in rows) + + +def _without_foreign_open_servers(tools: tuple[str, ...], open_aliases: frozenset[str]) -> set[str]: + prefixes: Final = tuple(f"{alias}-" for alias in open_aliases) + return {tool for tool in tools if not tool.startswith(prefixes)} + + def _peer_saw_call(peer_kind: PeerKind, observed: tuple[dict[str, object], ...], tool: str) -> bool: if peer_kind == "openapi": return any(item.get("path") == "/pets/7" and item.get("method") == "GET" for item in observed) @@ -95,6 +190,306 @@ def test_official_client_session_lists_and_calls_through_gateway( assert _peer_saw_call(peer_kind, peer.drain(), "add") +@pytest.mark.parametrize("client", ("raw-jsonrpc", "official-sdk")) +def test_standard_per_server_path_scopes_the_session_to_the_named_servers(gateway: Gateway, client: str) -> None: + open_before: Final = _open_aliases() + with mcp_peer() as peer_a, mcp_peer() as peer_b, mcp_peer() as peer_c, gateway.scenario() as scenario: + alias_a: Final = "rt1a" + uuid.uuid4().hex[:10] + alias_b: Final = "rt1b" + uuid.uuid4().hex[:10] + alias_c: Final = "rt1c" + uuid.uuid4().hex[:10] + server_a: Final = register_mcp(scenario, peer_a, alias_a) + server_b: Final = register_mcp(scenario, peer_b, alias_b) + server_c: Final = register_mcp(scenario, peer_c, alias_c) + key: Final = scenario.key(object_permission={"mcp_servers": [server_a, server_b, server_c]}) + open_aliases: Final = open_before | _open_aliases() + + first_path: Final = f"/mcp/{alias_a}" + listed, called = _list_and_call(gateway, client, first_path, key, f"{alias_a}-add", {"a": 20, "b": 22}) + assert listed.ok, listed.raw + assert _without_foreign_open_servers(listed.tools, open_aliases) == { + f"{alias_a}-add", + f"{alias_a}-multiply", + f"{alias_a}-fail", + }, listed.tools + assert called.ok and called.text == "42", called.raw + _refused_call( + gateway, + client, + first_path, + key, + f"{alias_b}-add", + {"a": 20, "b": 22}, + f"Error: Tool {alias_b}-add not found", + ) + calls_a: Final = tool_calls(peer_a.drain()) + assert len(calls_a) == 1, calls_a + _assert_upstream_call(calls_a[0], "add", {"a": 20, "b": 22}) + assert tool_calls(peer_b.drain()) == () + assert tool_calls(peer_c.drain()) == () + + combined_path: Final = f"/mcp/{alias_a},{alias_b}" + combined_listed, combined_called = _list_and_call( + gateway, client, combined_path, key, f"{alias_b}-multiply", {"a": 6, "b": 7} + ) + assert combined_listed.ok, combined_listed.raw + assert _without_foreign_open_servers(combined_listed.tools, open_aliases) == { + f"{alias_a}-add", + f"{alias_a}-multiply", + f"{alias_a}-fail", + f"{alias_b}-add", + f"{alias_b}-multiply", + f"{alias_b}-fail", + }, combined_listed.tools + assert combined_called.ok and combined_called.text == "42", combined_called.raw + _refused_call(gateway, client, combined_path, key, f"{alias_c}-add", {"a": 20, "b": 22}) + assert tool_calls(peer_a.drain()) == () + calls_b: Final = tool_calls(peer_b.drain()) + assert len(calls_b) == 1, calls_b + _assert_upstream_call(calls_b[0], "multiply", {"a": 6, "b": 7}) + assert tool_calls(peer_c.drain()) == () + + +@pytest.mark.parametrize("client", ("raw-jsonrpc", "official-sdk")) +def test_standard_comma_path_with_a_trailing_segment_scopes_a_proxy_admin_session( + gateway: Gateway, client: str +) -> None: + open_before: Final = _open_aliases() + with mcp_peer() as peer_a, mcp_peer() as peer_b, mcp_peer() as peer_c, gateway.scenario() as scenario: + alias_a: Final = "rt1pa" + uuid.uuid4().hex[:10] + alias_b: Final = "rt1pb" + uuid.uuid4().hex[:10] + alias_c: Final = "rt1pc" + uuid.uuid4().hex[:10] + register_mcp(scenario, peer_a, alias_a) + register_mcp(scenario, peer_b, alias_b) + register_mcp(scenario, peer_c, alias_c) + open_aliases: Final = open_before | _open_aliases() + path: Final = f"/mcp/{alias_a},{alias_c}/tools" + + listed, called = _list_and_call(gateway, client, path, gateway.key, f"{alias_c}-add", {"a": 20, "b": 22}) + assert listed.ok, listed.raw + assert _without_foreign_open_servers(listed.tools, open_aliases) == { + f"{alias_a}-add", + f"{alias_a}-multiply", + f"{alias_a}-fail", + f"{alias_c}-add", + f"{alias_c}-multiply", + f"{alias_c}-fail", + }, listed.tools + assert called.ok and called.text == "42", called.raw + _refused_call(gateway, client, path, gateway.key, f"{alias_b}-add", {"a": 20, "b": 22}) + assert tool_calls(peer_a.drain()) == () + assert tool_calls(peer_b.drain()) == () + calls_c: Final = tool_calls(peer_c.drain()) + assert len(calls_c) == 1, calls_c + _assert_upstream_call(calls_c[0], "add", {"a": 20, "b": 22}) + + +@pytest.mark.parametrize("client", ("raw-jsonrpc", "official-sdk")) +def test_standard_comma_path_with_a_trailing_segment_admits_a_scoped_key(gateway: Gateway, client: str) -> None: + pytest.skip( + "BUG: a non-admin key gets 401 'Only proxy admin can be used...' on /mcp/{a},{b}/tools because " + "mcp_inference_routes only admits the single-segment /mcp/{subpath}, while " + "_get_mcp_servers_in_path parses the trailing path" + ) + open_before: Final = _open_aliases() + with mcp_peer() as peer_a, mcp_peer() as peer_b, mcp_peer() as peer_c, gateway.scenario() as scenario: + alias_a: Final = "rt1sa" + uuid.uuid4().hex[:10] + alias_b: Final = "rt1sb" + uuid.uuid4().hex[:10] + alias_c: Final = "rt1sc" + uuid.uuid4().hex[:10] + server_a: Final = register_mcp(scenario, peer_a, alias_a) + server_b: Final = register_mcp(scenario, peer_b, alias_b) + server_c: Final = register_mcp(scenario, peer_c, alias_c) + key: Final = scenario.key(object_permission={"mcp_servers": [server_a, server_b, server_c]}) + open_aliases: Final = open_before | _open_aliases() + path: Final = f"/mcp/{alias_a},{alias_c}/tools" + + listed, called = _list_and_call(gateway, client, path, key, f"{alias_c}-add", {"a": 20, "b": 22}) + assert listed.ok, listed.raw + assert _without_foreign_open_servers(listed.tools, open_aliases) == { + f"{alias_a}-add", + f"{alias_a}-multiply", + f"{alias_a}-fail", + f"{alias_c}-add", + f"{alias_c}-multiply", + f"{alias_c}-fail", + }, listed.tools + assert called.ok and called.text == "42", called.raw + _refused_call(gateway, client, path, key, f"{alias_b}-add", {"a": 20, "b": 22}) + assert tool_calls(peer_a.drain()) == () + assert tool_calls(peer_b.drain()) == () + calls_c: Final = tool_calls(peer_c.drain()) + assert len(calls_c) == 1, calls_c + _assert_upstream_call(calls_c[0], "add", {"a": 20, "b": 22}) + + +@pytest.mark.parametrize("client", ("raw-jsonrpc", "official-sdk")) +def test_legacy_server_path_resolves_lists_groups_and_fails_closed_on_unknown_names( + gateway: Gateway, client: str +) -> None: + open_before: Final = _open_aliases() + with mcp_peer() as peer_a, mcp_peer() as peer_b, mcp_peer() as peer_c, gateway.scenario() as scenario: + alias_a: Final = "rt2a" + uuid.uuid4().hex[:10] + alias_b: Final = "rt2b" + uuid.uuid4().hex[:10] + alias_c: Final = "rt2c" + uuid.uuid4().hex[:10] + group: Final = "rt2g" + uuid.uuid4().hex[:10] + server_a: Final = register_mcp(scenario, peer_a, alias_a, mcp_access_groups=[group]) + server_b: Final = register_mcp(scenario, peer_b, alias_b, mcp_access_groups=[group]) + server_c: Final = register_mcp(scenario, peer_c, alias_c) + key: Final = scenario.key(object_permission={"mcp_servers": [server_a, server_b, server_c]}) + open_aliases: Final = open_before | _open_aliases() + + direct_path: Final = f"/{alias_a},{alias_b}/mcp" + direct_listed, direct_called = _list_and_call( + gateway, client, direct_path, key, f"{alias_a}-add", {"a": 20, "b": 22} + ) + assert direct_listed.ok, direct_listed.raw + assert _without_foreign_open_servers(direct_listed.tools, open_aliases) == { + f"{alias_a}-add", + f"{alias_a}-multiply", + f"{alias_a}-fail", + f"{alias_b}-add", + f"{alias_b}-multiply", + f"{alias_b}-fail", + }, direct_listed.tools + assert direct_called.ok and direct_called.text == "42", direct_called.raw + direct_calls_a: Final = tool_calls(peer_a.drain()) + assert len(direct_calls_a) == 1, direct_calls_a + _assert_upstream_call(direct_calls_a[0], "add", {"a": 20, "b": 22}) + assert tool_calls(peer_b.drain()) == () + assert tool_calls(peer_c.drain()) == () + + mixed_path: Final = f"/{alias_a},%20{alias_a}%20,nope{uuid.uuid4().hex}/mcp" + mixed_listed, mixed_called = _list_and_call( + gateway, client, mixed_path, key, f"{alias_a}-multiply", {"a": 6, "b": 7} + ) + assert mixed_listed.ok, mixed_listed.raw + assert _without_foreign_open_servers(mixed_listed.tools, open_aliases) == { + f"{alias_a}-add", + f"{alias_a}-multiply", + f"{alias_a}-fail", + }, mixed_listed.tools + assert mixed_called.ok and mixed_called.text == "42", mixed_called.raw + mixed_calls_a: Final = tool_calls(peer_a.drain()) + assert len(mixed_calls_a) == 1, mixed_calls_a + _assert_upstream_call(mixed_calls_a[0], "multiply", {"a": 6, "b": 7}) + assert tool_calls(peer_b.drain()) == () + assert tool_calls(peer_c.drain()) == () + + unknown_path: Final = f"/nope1{uuid.uuid4().hex},nope2{uuid.uuid4().hex}/mcp" + assert peer_a.drain() == () + assert peer_b.drain() == () + assert peer_c.drain() == () + if client == "official-sdk": + with pytest.raises(builtins.ExceptionGroup) as caught: + official_client_outcomes(gateway, key, unknown_path, f"{alias_a}-add", {"a": 20, "b": 22}) + leaves: Final = _leaf_exceptions(caught.value) + assert len(leaves) == 1, repr(leaves) + leaf: Final = leaves[0] + assert isinstance(leaf, MCPError), repr(leaves) + assert leaf.error.code == -32601, repr(leaves) + assert leaf.error.message == "Not Found" and leaf.error.data is None, repr(leaves) + else: + unknown_initialized: Final = _streamable_rpc(gateway, unknown_path, key, "initialize", dict(INITIALIZE)) + assert unknown_initialized.status_code == 404, unknown_initialized.text + unknown_listed: Final = _streamable_rpc(gateway, unknown_path, key, "tools/list", {}) + assert unknown_listed.status_code == 404, unknown_listed.text + assert peer_a.drain() == () + assert peer_b.drain() == () + assert peer_c.drain() == () + + group_path: Final = f"/{group}/mcp" + group_listed, group_called = _list_and_call( + gateway, client, group_path, key, f"{alias_b}-multiply", {"a": 6, "b": 7} + ) + assert group_listed.ok, group_listed.raw + assert _without_foreign_open_servers(group_listed.tools, open_aliases) == { + f"{alias_a}-add", + f"{alias_a}-multiply", + f"{alias_a}-fail", + f"{alias_b}-add", + f"{alias_b}-multiply", + f"{alias_b}-fail", + }, group_listed.tools + assert group_called.ok and group_called.text == "42", group_called.raw + assert tool_calls(peer_a.drain()) == () + group_calls_b: Final = tool_calls(peer_b.drain()) + assert len(group_calls_b) == 1, group_calls_b + _assert_upstream_call(group_calls_b[0], "multiply", {"a": 6, "b": 7}) + assert tool_calls(peer_c.drain()) == () + + mixed_group_path: Final = f"/{group},{alias_c}/mcp" + mixed_group_listed, mixed_group_called = _list_and_call( + gateway, client, mixed_group_path, key, f"{alias_c}-add", {"a": 20, "b": 22} + ) + assert mixed_group_listed.ok, mixed_group_listed.raw + assert _without_foreign_open_servers(mixed_group_listed.tools, open_aliases) == { + f"{alias_a}-add", + f"{alias_a}-multiply", + f"{alias_a}-fail", + f"{alias_b}-add", + f"{alias_b}-multiply", + f"{alias_b}-fail", + f"{alias_c}-add", + f"{alias_c}-multiply", + f"{alias_c}-fail", + }, mixed_group_listed.tools + assert mixed_group_called.ok and mixed_group_called.text == "42", mixed_group_called.raw + assert tool_calls(peer_a.drain()) == () + assert tool_calls(peer_b.drain()) == () + mixed_group_calls_c: Final = tool_calls(peer_c.drain()) + assert len(mixed_group_calls_c) == 1, mixed_group_calls_c + _assert_upstream_call(mixed_group_calls_c[0], "add", {"a": 20, "b": 22}) + + +@pytest.mark.parametrize("client", ("raw-jsonrpc", "official-sdk")) +def test_legacy_toolset_path_scopes_the_session_to_the_toolsets_tools(gateway: Gateway, client: str) -> None: + pytest.skip( + "BUG: a /{toolset}/mcp session lists every granted server's tools instead of the toolset's " + "tools (the toolset's tool filter is not applied on the legacy path)" + ) + open_before: Final = _open_aliases() + with mcp_peer() as peer_a, mcp_peer() as peer_b, mcp_peer() as peer_c, gateway.scenario() as scenario: + alias_a: Final = "rt2ta" + uuid.uuid4().hex[:10] + alias_b: Final = "rt2tb" + uuid.uuid4().hex[:10] + alias_c: Final = "rt2tc" + uuid.uuid4().hex[:10] + server_a: Final = register_mcp(scenario, peer_a, alias_a) + server_b: Final = register_mcp(scenario, peer_b, alias_b) + server_c: Final = register_mcp(scenario, peer_c, alias_c) + toolset_name: Final = "rt2t" + uuid.uuid4().hex[:10] + toolset_id: Final = create_toolset(scenario, ((server_a, "add"),), toolset_name=toolset_name) + toolset_key: Final = scenario.key( + object_permission={ + "mcp_servers": [server_a, server_b, server_c], + "mcp_toolsets": [toolset_id], + } + ) + open_aliases: Final = open_before | _open_aliases() + + toolset_path: Final = f"/{toolset_name}/mcp" + toolset_listed, toolset_called = _list_and_call( + gateway, client, toolset_path, toolset_key, f"{alias_a}-add", {"a": 20, "b": 22} + ) + assert toolset_listed.ok, toolset_listed.raw + assert _without_foreign_open_servers(toolset_listed.tools, open_aliases) == {f"{alias_a}-add"}, ( + toolset_listed.tools + ) + assert toolset_called.ok and toolset_called.text == "42", toolset_called.raw + _refused_call( + gateway, + client, + toolset_path, + toolset_key, + f"{alias_a}-multiply", + {"a": 6, "b": 7}, + f"Error: Tool 'multiply' is not allowed for your key/team on server '{alias_a}'. Contact proxy admin for access.", + ) + _refused_call(gateway, client, toolset_path, toolset_key, f"{alias_b}-add", {"a": 20, "b": 22}) + toolset_calls_a: Final = tool_calls(peer_a.drain()) + assert len(toolset_calls_a) == 1, toolset_calls_a + _assert_upstream_call(toolset_calls_a[0], "add", {"a": 20, "b": 22}) + assert tool_calls(peer_b.drain()) == () + assert tool_calls(peer_c.drain()) == () + + @pytest.mark.parametrize("peer_kind", ("http", "sse", "stdio")) def test_prompts_resources_and_templates_are_proxied_from_rich_peer(gateway: Gateway, peer_kind: PeerKind) -> None: with peer_of(peer_kind, rich=True) as peer, gateway.scenario() as scenario: @@ -177,14 +572,19 @@ def test_pinned_revision_pairs_list_and_call_through_gateway( key: Final = scenario.key(object_permission={"mcp_servers": [identity]}) endpoint: Final = str(gateway.client.base_url).rstrip("/") + ("/mcp/sse" if ingress == "sse" else "/mcp") client: Final = MCPClient( - server_url=endpoint, transport_type=MCPTransport(ingress), protocol_version=downstream, - extra_headers={"Authorization": f"Bearer {key}", "x-mcp-servers": identity}, timeout=15, + server_url=endpoint, + transport_type=MCPTransport(ingress), + protocol_version=downstream, + extra_headers={"Authorization": f"Bearer {key}", "x-mcp-servers": identity}, + timeout=15, ) async def exercise() -> None: tools: Final = await client.list_tools(raise_on_error=True) assert f"{alias}-add" in tuple(tool.name for tool in tools) - result: Final = await client.call_tool(CallToolRequestParams(name=f"{alias}-add", arguments={"a": 3, "b": 4})) + result: Final = await client.call_tool( + CallToolRequestParams(name=f"{alias}-add", arguments={"a": 3, "b": 4}) + ) assert result.is_error is False assert result.content[0].text == "7" diff --git a/tests/integration/mcp/test_oauth_configuration.py b/tests/integration/mcp/test_oauth_configuration.py index fe2b1069f04..e3a42d630cb 100644 --- a/tests/integration/mcp/test_oauth_configuration.py +++ b/tests/integration/mcp/test_oauth_configuration.py @@ -9,15 +9,263 @@ from pathlib import Path from typing import Final, Literal from urllib.parse import parse_qs, urlsplit +import httpx import pytest from integration._support.client import Gateway, Scenario, eventually from integration._support.database import read_rows from integration._support.mcp import McpPeer, call_tool, mcp_peer, register_mcp, tool_names +from integration._support.oauth_server import oauth_server from integration._support.process import owned_proxy from integration._support.wire import Reply, Request, Wire, wire_server from pydantic import TypeAdapter _Upstream = Callable[[Request], Reply] +CLIENT_REDIRECT: Final = "http://127.0.0.1:9/cb" + + +def _response_object(response: httpx.Response) -> dict[str, object]: + return TypeAdapter(dict[str, object]).validate_python(response.json()) + + +def _start_callback_flow( + gateway: Gateway, + alias: str, + key: str, + client_id: str, + expected_authorization_url: str | None = None, +) -> tuple[str, str, str]: + started: Final = gateway.client.get( + f"/{alias}/authorize", + params={ + "client_id": client_id, + "redirect_uri": CLIENT_REDIRECT, + "response_type": "code", + "state": "client-state", + "code_challenge": "A" * 43, + "code_challenge_method": "S256", + }, + headers={"x-litellm-api-key": key}, + ) + assert started.status_code in (302, 307), started.text + location: Final = urlsplit(started.headers["location"]) + if expected_authorization_url is not None: + assert location.scheme + "://" + location.netloc + location.path == expected_authorization_url, started.text + relay_state: Final = parse_qs(location.query)["state"][0] + cookie_name: Final = f"mcp_oauth_state_{relay_state}" + assert cookie_name in started.cookies, started.headers + return relay_state, cookie_name, started.cookies[cookie_name] + + +def _assert_cleared_oauth_state_cookie(response: httpx.Response, cookie_name: str) -> None: + cleared: Final = response.headers["set-cookie"] + assert cleared.startswith(f'{cookie_name}="";'), cleared + assert "Max-Age=0" in cleared, cleared + assert "Path=/" in cleared, cleared + assert "HttpOnly" in cleared, cleared + assert "SameSite=lax" in cleared, cleared + + +def test_callback_forwards_the_code_only_for_the_sealed_issuer_and_relays_idp_errors( + gateway: Gateway, +) -> None: + with mcp_peer() as peer, oauth_server() as auth, gateway.scenario() as scenario: + alias: Final = "rt4" + uuid.uuid4().hex[:10] + identity: Final = register_mcp( + scenario, + peer, + alias, + issuer=auth.issuer, + authorization_url=auth.issuer + "/authorize", + token_url=auth.issuer + "/token", + registration_url=auth.issuer + "/register", + auth_type="oauth2", + oauth2_flow="authorization_code", + credentials={"client_id": "ac-client", "client_secret": "ac-secret", "scopes": ["tools.call"]}, + ) + key: Final = scenario.key(object_permission={"mcp_servers": [identity]}) + registered: Final = gateway.client.post( + f"/{alias}/register", json={"redirect_uris": [CLIENT_REDIRECT], "client_name": "integration"} + ) + assert registered.status_code in (200, 201), registered.text + client_id: Final = str(_response_object(registered)["client_id"]) + + wrong_relay, wrong_cookie, wrong_cookie_value = _start_callback_flow(gateway, alias, key, client_id) + wrong_code: Final = "callback-code-" + uuid.uuid4().hex + wrong: Final = gateway.client.get( + "/callback", + params={"code": wrong_code, "state": wrong_relay, "iss": "http://127.0.0.1:1/other"}, + cookies={wrong_cookie: wrong_cookie_value}, + ) + assert wrong.status_code == 400, wrong.text + assert wrong.headers["content-type"] == "text/html; charset=utf-8", wrong.headers + assert wrong.text == ( + "

Authentication failed

Error: invalid_issuer

" + "

This authorization response came from a different identity provider than the one this " + "MCP server is configured to use.

" + "

You can close this window and try again.

" + ), wrong.text + assert "location" not in wrong.headers, wrong.headers + assert wrong_code not in wrong.text, wrong.text + _assert_cleared_oauth_state_cookie(wrong, wrong_cookie) + + correct_relay, correct_cookie, correct_cookie_value = _start_callback_flow(gateway, alias, key, client_id) + correct: Final = gateway.client.get( + "/callback", + params={"code": "c", "state": correct_relay, "iss": auth.issuer}, + cookies={correct_cookie: correct_cookie_value}, + ) + assert correct.status_code == 302, correct.text + correct_location: Final = urlsplit(correct.headers["location"]) + assert correct_location.scheme + "://" + correct_location.netloc + correct_location.path == CLIENT_REDIRECT, ( + correct.headers["location"] + ) + correct_query: Final = parse_qs(correct_location.query) + assert set(correct_query) == {"code", "state"}, correct.headers["location"] + assert correct_query["state"] == ["client-state"], correct.headers["location"] + assert len(correct_query["code"]) == 1 and correct_query["code"][0].startswith("llm_scope_"), correct.headers[ + "location" + ] + _assert_cleared_oauth_state_cookie(correct, correct_cookie) + + error_relay, error_cookie, error_cookie_value = _start_callback_flow(gateway, alias, key, client_id) + error: Final = gateway.client.get( + "/callback", + params={"error": "access_denied", "error_description": "x", "state": error_relay}, + cookies={error_cookie: error_cookie_value}, + ) + assert error.status_code == 302, error.text + error_location: Final = urlsplit(error.headers["location"]) + assert parse_qs(error_location.query) == { + "error": ["access_denied"], + "error_description": ["x"], + "state": ["client-state"], + }, error.headers["location"] + _assert_cleared_oauth_state_cookie(error, error_cookie) + + error_issuer_relay, error_issuer_cookie, error_issuer_cookie_value = _start_callback_flow( + gateway, alias, key, client_id + ) + error_issuer: Final = gateway.client.get( + "/callback", + params={ + "error": "access_denied", + "error_description": "x", + "state": error_issuer_relay, + "iss": "http://127.0.0.1:1/other", + }, + cookies={error_issuer_cookie: error_issuer_cookie_value}, + ) + assert error_issuer.status_code == 400, error_issuer.text + assert error_issuer.headers["content-type"] == "text/html; charset=utf-8", error_issuer.headers + assert error_issuer.text == ( + "

Authentication failed

Error: invalid_issuer

" + "

Unexpected authorization issuer

" + "

You can close this window and try again.

" + ), error_issuer.text + assert "location" not in error_issuer.headers, error_issuer.headers + _assert_cleared_oauth_state_cookie(error_issuer, error_issuer_cookie) + + missing: Final = gateway.client.get("/callback") + assert missing.status_code == 400, missing.text + assert missing.headers["content-type"] == "text/html; charset=utf-8", missing.headers + assert missing.text == ( + "

Authentication failed

Error: invalid_request

" + "

Missing authorization 'code' and 'state' parameter(s).

" + "

You can close this window and try again.

" + ), missing.text + + +def test_callback_refuses_a_response_without_iss_when_the_authorization_server_advertises_iss_support( + gateway: Gateway, +) -> None: + wire_holder: Final[list[Wire]] = [] + + def respond(request: Request) -> Reply: + path: Final = urlsplit(request.target).path + if request.method == "GET" and path in ( + "/.well-known/oauth-authorization-server", + "/.well-known/openid-configuration", + ): + issuer: Final = wire_holder[0].url + return Reply( + body=json.dumps( + { + "issuer": issuer, + "authorization_endpoint": issuer + "/authorize", + "token_endpoint": issuer + "/token", + "registration_endpoint": issuer + "/register", + "response_types_supported": ["code"], + "grant_types_supported": ["authorization_code", "refresh_token"], + "code_challenge_methods_supported": ["S256"], + "token_endpoint_auth_methods_supported": ["client_secret_post"], + "authorization_response_iss_parameter_supported": True, + } + ).encode() + ) + return Reply(status=404, body=json.dumps({"error": "not_found"}).encode()) + + with wire_server(respond) as wire, mcp_peer() as peer, gateway.scenario() as scenario: + wire_holder.append(wire) + alias: Final = "rt5" + uuid.uuid4().hex[:10] + identity: Final = register_mcp( + scenario, + peer, + alias, + issuer=wire.url, + auth_type="oauth2", + oauth2_flow="authorization_code", + credentials={"client_id": "ac-client", "client_secret": "ac-secret", "scopes": ["tools.call"]}, + ) + key: Final = scenario.key(object_permission={"mcp_servers": [identity]}) + + relay, cookie, cookie_value = _start_callback_flow( + gateway, alias, key, "ac-client", expected_authorization_url=wire.url + "/authorize" + ) + code: Final = "callback-code-" + uuid.uuid4().hex + missing_issuer: Final = gateway.client.get( + "/callback", + params={"code": code, "state": relay}, + cookies={cookie: cookie_value}, + ) + assert missing_issuer.status_code == 400, missing_issuer.text + assert missing_issuer.headers["content-type"] == "text/html; charset=utf-8", missing_issuer.headers + assert missing_issuer.text == ( + "

Authentication failed

Error: invalid_issuer

" + "

This authorization response came from a different identity provider than the one this " + "MCP server is configured to use.

" + "

You can close this window and try again.

" + ), missing_issuer.text + assert "location" not in missing_issuer.headers, missing_issuer.headers + assert code not in missing_issuer.text, missing_issuer.text + _assert_cleared_oauth_state_cookie(missing_issuer, cookie) + + trusted_relay, trusted_cookie, trusted_cookie_value = _start_callback_flow( + gateway, alias, key, "ac-client", expected_authorization_url=wire.url + "/authorize" + ) + trusted_code: Final = "callback-code-" + uuid.uuid4().hex + trusted: Final = gateway.client.get( + "/callback", + params={"code": trusted_code, "state": trusted_relay, "iss": wire.url}, + cookies={trusted_cookie: trusted_cookie_value}, + ) + assert trusted.status_code == 302, trusted.text + trusted_location: Final = urlsplit(trusted.headers["location"]) + assert trusted_location.scheme + "://" + trusted_location.netloc + trusted_location.path == CLIENT_REDIRECT, ( + trusted.headers["location"] + ) + trusted_query: Final = parse_qs(trusted_location.query) + assert set(trusted_query) == {"code", "state"}, trusted.headers["location"] + assert trusted_query["state"] == ["client-state"], trusted.headers["location"] + assert len(trusted_query["code"]) == 1 and trusted_query["code"][0].startswith("llm_scope_"), trusted.headers[ + "location" + ] + assert trusted_query["code"][0] != trusted_code, trusted.headers["location"] + _assert_cleared_oauth_state_cookie(trusted, trusted_cookie) + + wire_requests: Final = wire.drain() + wire_method_paths: Final = tuple((request.method, urlsplit(request.target).path) for request in wire_requests) + assert wire_method_paths == (("GET", "/.well-known/oauth-authorization-server"),), wire_method_paths + assert not any(method == "POST" and path == "/token" for method, path in wire_method_paths), wire_method_paths @pytest.mark.covers("other.mcp.oauth.discovery_cannot_erase_configured_authorization_endpoint") From 1047a450bfe660b626040a1b19f60e1cabab91a1 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:08:35 +0000 Subject: [PATCH 11/51] test(integration): pin assistants and evals endpoint contracts (#44734) * test(integration): cover assistants_evals endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(tests): replace exception assertions flagged by ruff Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(openai): type assistant run options Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(unit): cover assistant and eval contract paths Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(openai): keep assistant run options unvalidated Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): pin assistants and evals wire contracts as BUG skips Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover every eval route, full responses, image parts and full SDK stream payloads Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): relay upstream eval run model, send reasoning_effort, cover SDK root alias, done frame and client disconnect Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): name reasoning_effort in the run BUG skip Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): use a non-null marker thread and name the eval run model restamp in its BUG skip Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): mark disconnect cases with a messages.create the proxy relays on main Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(assistants_evals): cover thread message shapes, raw run options, header leaks and /v1 eval bases Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): expect the OpenAI /cancel path for eval run cancel Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_missing_body_param_status.py | 150 ++++++- .../providers/test_assistants_evals_wire.py | 343 ++++++++++++++ .../providers/test_provider_lookup_status.py | 423 +++++++++++++++++- .../streaming/test_assistants_evals_stream.py | 403 +++++++++++++++++ 4 files changed, 1316 insertions(+), 3 deletions(-) create mode 100644 tests/integration/providers/test_assistants_evals_wire.py create mode 100644 tests/integration/streaming/test_assistants_evals_stream.py diff --git a/tests/integration/compatibility/test_missing_body_param_status.py b/tests/integration/compatibility/test_missing_body_param_status.py index f809a67c0ce..a452110f138 100644 --- a/tests/integration/compatibility/test_missing_body_param_status.py +++ b/tests/integration/compatibility/test_missing_body_param_status.py @@ -24,8 +24,10 @@ from integration._support.database import read_rows from integration._support.process import owned_proxy_process, paused, stop_child from integration._support.upstream import ScenarioHandle, delete_scenario, register_scenario from integration._support.wire import Reply, Request, Wire, wire_server -from openai import AsyncOpenAI, BadRequestError, OpenAI +from openai import APIStatusError, AsyncOpenAI, BadRequestError, OpenAI from openai.types import Video +from openai.types.eval_create_response import EvalCreateResponse as _Eval +from openai.types.evals.run_create_response import RunCreateResponse as _EvalRun from pydantic import JsonValue from litellm.responses.utils import ResponsesAPIRequestUtils @@ -42,6 +44,64 @@ from tests.integration.cost_calculation.cost_tracking_case import ( ) _Route = tuple[str, str, tuple[str, ...], dict[str, JsonValue], str] +_EVAL_DATA_SOURCE_CONFIG: Final[dict[str, JsonValue]] = { + "type": "custom", + "item_schema": {"type": "object", "properties": {"a": {"type": "string"}}}, +} +_EVAL_DATA_SOURCE_CONFIG_RESPONSE: Final[dict[str, JsonValue]] = { + "type": "custom", + "schema": {"type": "object", "properties": {"a": {"type": "string"}}}, +} +_EVAL_TESTING_CRITERIA: Final[list[JsonValue]] = [ + {"type": "string_check", "name": "exact", "input": "{{item.a}}", "reference": "{{item.a}}", "operation": "eq"} +] +_EVAL_RUN_DATA_SOURCE: Final[dict[str, JsonValue]] = { + "type": "jsonl", + "source": {"type": "file_id", "id": "file-abc"}, +} +_EVAL_CREATE_REQUEST: Final[dict[str, JsonValue]] = { + "data_source_config": _EVAL_DATA_SOURCE_CONFIG, + "testing_criteria": _EVAL_TESTING_CRITERIA, + "name": "nightly", + "metadata": {"suite": "nightly"}, +} +_EVAL_RUN_REQUEST: Final[dict[str, JsonValue]] = { + "data_source": _EVAL_RUN_DATA_SOURCE, + "name": "run-1", + "metadata": {"suite": "nightly"}, +} +_EVAL_CREATE_UPSTREAM_RESPONSE: Final[dict[str, JsonValue]] = { + "id": "eval_abc", + "object": "eval", + "created_at": 1700000000, + "name": "nightly", + "data_source_config": _EVAL_DATA_SOURCE_CONFIG_RESPONSE, + "testing_criteria": _EVAL_TESTING_CRITERIA, + "metadata": {"suite": "nightly"}, +} +_EVAL_CREATE_RESPONSE: Final[dict[str, JsonValue]] = { + **_EVAL_CREATE_UPSTREAM_RESPONSE, + "updated_at": None, +} +_EVAL_RUN_RESPONSE: Final[dict[str, JsonValue]] = { + "id": "run_abc", + "object": "eval.run", + "created_at": 1700000001, + "eval_id": "eval_abc", + "started_at": None, + "completed_at": None, + "data_source": _EVAL_RUN_DATA_SOURCE, + "error": None, + "per_model_usage": [], + "per_testing_criteria_results": [], + "report_url": "https://example.invalid/evals/run_abc", + "result_counts": {"errored": 0, "failed": 0, "passed": 0, "total": 0}, + "shared_with_openai": None, + "name": "run-1", + "model": "gpt-4o-mini", + "metadata": {"suite": "nightly"}, + "status": "queued", +} _ROUTES: Final[dict[str, _Route]] = { "acompletion": ( "/v1/chat/completions", @@ -1648,6 +1708,94 @@ def test_upstream_pause_and_worker_kill_preserve_required_body_status( assert recovered.status_code == 200, recovered.text +def test_eval_create_and_run_forward_client_metadata(gateway: Gateway) -> None: + pytest.skip( + "BUG: evals.create and evals.runs.create drop the client metadata upstream, " + "and the run response model is overwritten with the routing alias" + ) + + def respond(request: Request) -> Reply: + if request.method == "POST" and request.target == "/v1/evals": + return Reply(body=json.dumps(_EVAL_CREATE_UPSTREAM_RESPONSE).encode()) + if request.method == "POST" and request.target == "/v1/evals/eval_abc/runs": + return Reply(body=json.dumps(_EVAL_RUN_RESPONSE).encode()) + return Reply(status=404, body=b'{"error":"unexpected upstream request"}') + + with wire_server(respond) as wire, gateway.scenario() as scenario: + alias: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url, api_key="synthetic-openai-key") + eventually( + lambda: gateway.request( + "POST", + "/v1/chat/completions", + {"model": alias, "messages": [{"role": "user", "content": "model readiness"}]}, + ), + lambda response: not (response.status_code == 400 and "Invalid model name" in response.text), + seconds=30, + ) + wire.drain() + with OpenAI( + base_url=f"{str(gateway.client.base_url).rstrip('/')}/v1", api_key=gateway.key, max_retries=0 + ) as client: + try: + create_raw: Final = client.evals.with_raw_response.create( + name="nightly", + data_source_config=_EVAL_DATA_SOURCE_CONFIG, + testing_criteria=_EVAL_TESTING_CRITERIA, + metadata={"suite": "nightly"}, + extra_body={"model": alias}, + ) + except APIStatusError as error: + create_error_text: Final = error.response.text + create_failed_requests: Final = wire.drain() + assert [(request.method, request.target) for request in create_failed_requests] == [ + ("POST", "/v1/evals") + ], create_error_text + assert create_failed_requests[0].headers["authorization"] == "Bearer synthetic-openai-key", ( + create_error_text + ) + assert json.loads(create_failed_requests[0].body) == _EVAL_CREATE_REQUEST, create_error_text + pytest.fail(f"SDK Eval creation failed with HTTP {error.response.status_code}: {create_error_text}") + try: + run_raw: Final = client.evals.runs.with_raw_response.create( + "eval_abc", + data_source=_EVAL_RUN_DATA_SOURCE, + name="run-1", + metadata={"suite": "nightly"}, + extra_body={"model": alias}, + ) + except APIStatusError as error: + run_error_text: Final = error.response.text + run_failed_requests: Final = wire.drain() + assert [(request.method, request.target) for request in run_failed_requests] == [ + ("POST", "/v1/evals"), + ("POST", "/v1/evals/eval_abc/runs"), + ], run_error_text + assert tuple(request.headers["authorization"] for request in run_failed_requests) == ( + "Bearer synthetic-openai-key", + "Bearer synthetic-openai-key", + ), run_error_text + assert json.loads(run_failed_requests[0].body) == _EVAL_CREATE_REQUEST, run_error_text + assert json.loads(run_failed_requests[1].body) == _EVAL_RUN_REQUEST, run_error_text + pytest.fail(f"SDK Eval run creation failed with HTTP {error.response.status_code}: {run_error_text}") + response_text: Final = "\n".join((create_raw.http_response.text, run_raw.http_response.text)) + requests: Final = wire.drain() + assert [(request.method, request.target) for request in requests] == [ + ("POST", "/v1/evals"), + ("POST", "/v1/evals/eval_abc/runs"), + ], response_text + assert tuple(request.headers["authorization"] for request in requests) == ( + "Bearer synthetic-openai-key", + "Bearer synthetic-openai-key", + ), response_text + assert json.loads(requests[0].body) == _EVAL_CREATE_REQUEST, response_text + assert json.loads(requests[1].body) == _EVAL_RUN_REQUEST, response_text + assert (create_raw.status_code, run_raw.status_code) == (200, 200), response_text + assert isinstance(create_raw.parse(), _Eval), response_text + assert json.loads(create_raw.http_response.text) == _EVAL_CREATE_RESPONSE, response_text + assert isinstance(run_raw.parse(), _EvalRun), response_text + assert json.loads(run_raw.http_response.text) == _EVAL_RUN_RESPONSE, response_text + + _VIDEO_UPSTREAM_ID: Final = "video_upstream" _VIDEO_PROVIDER_KEY: Final = "video-provider-key" _VIDEO_BODY: Final = { diff --git a/tests/integration/providers/test_assistants_evals_wire.py b/tests/integration/providers/test_assistants_evals_wire.py new file mode 100644 index 00000000000..02f7307e725 --- /dev/null +++ b/tests/integration/providers/test_assistants_evals_wire.py @@ -0,0 +1,343 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Final +from urllib.parse import parse_qs, urlsplit + +import pytest +import yaml +from integration._support.client import Gateway +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from openai import APIStatusError, OpenAI +from openai.types.beta.thread import Thread as _Thread +from openai.types.beta.threads.message import Message as _Message +from openai.types.beta.threads.run import Run as _Run +from pydantic import JsonValue + +_THREAD: Final[dict[str, JsonValue]] = { + "id": "thread_abc", + "object": "thread", + "created_at": 1700000000, + "metadata": {"project": "alpha"}, + "tool_resources": {"code_interpreter": {"file_ids": ["file-abc"]}, "file_search": None}, +} +_MESSAGE: Final[dict[str, JsonValue]] = { + "id": "msg_abc", + "assistant_id": None, + "attachments": [{"file_id": "file-abc", "tools": [{"type": "file_search"}]}], + "completed_at": None, + "object": "thread.message", + "content": [{"type": "text", "text": {"value": "Summarize the attachment", "annotations": []}}], + "created_at": 1700000001, + "incomplete_at": None, + "incomplete_details": None, + "metadata": {"turn": "2"}, + "role": "user", + "run_id": None, + "status": "completed", + "thread_id": "thread_abc", +} +_IMAGE_PARTS: Final[list[JsonValue]] = [ + {"type": "text", "text": "Compare these"}, + {"type": "image_file", "image_file": {"file_id": "file-img", "detail": "low"}}, + {"type": "image_url", "image_url": {"url": "https://example.invalid/chart.png", "detail": "high"}}, +] +_ATTACHMENTS: Final[list[JsonValue]] = [{"file_id": "file-abc", "tools": [{"type": "file_search"}]}] +_THREAD_MESSAGES: Final[tuple[dict[str, JsonValue], ...]] = ( + {"role": "user", "content": "What is in the report?", "metadata": {"turn": "1"}}, + { + "role": "user", + "content": [{"type": "text", "text": "What is in the report?"}], + "attachments": _ATTACHMENTS, + "metadata": {"turn": "1"}, + }, + {"role": "user", "content": _IMAGE_PARTS, "attachments": _ATTACHMENTS, "metadata": {"turn": "1"}}, +) +_RUN_TOOL_FUNCTION: Final[dict[str, JsonValue]] = { + "name": "lookup", + "parameters": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}, +} +_RAW_RUN_REQUEST: Final[dict[str, JsonValue]] = { + "assistant_id": "asst_abc", + "tools": [{"type": "function", "function": _RUN_TOOL_FUNCTION}], + "tool_choice": {"type": "function", "function": {"name": "lookup"}}, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": {"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]}, + "strict": True, + }, + }, +} +_RUN: Final[dict[str, JsonValue]] = { + "id": "run_abc", + "assistant_id": "asst_abc", + "cancelled_at": None, + "completed_at": None, + "created_at": 1700000002, + "expires_at": None, + "failed_at": None, + "incomplete_details": None, + "instructions": "Be terse", + "last_error": None, + "max_completion_tokens": 50, + "max_prompt_tokens": 500, + "metadata": {"run": "r1"}, + "model": "gpt-4o", + "object": "thread.run", + "parallel_tool_calls": False, + "required_action": None, + "response_format": {"type": "json_object"}, + "started_at": None, + "status": "queued", + "thread_id": "thread_abc", + "tool_choice": "auto", + "tools": [{"type": "code_interpreter"}], + "truncation_strategy": {"type": "last_messages", "last_messages": 4}, + "usage": None, + "temperature": 0.3, + "top_p": 0.9, +} +_RUN_REQUEST: Final[dict[str, JsonValue]] = { + "assistant_id": "asst_abc", + "instructions": "Be terse", + "additional_instructions": "Cite sources", + "additional_messages": [{"role": "user", "content": "Also check page 2"}], + "metadata": {"run": "r1"}, + "model": "gpt-4o", + "tools": [{"type": "code_interpreter"}], + "temperature": 0.3, + "top_p": 0.9, + "max_prompt_tokens": 500, + "max_completion_tokens": 50, + "truncation_strategy": {"type": "last_messages", "last_messages": 4}, + "tool_choice": "auto", + "response_format": {"type": "json_object"}, + "parallel_tool_calls": False, + "reasoning_effort": "low", +} + + +def _assistant_config(tmp_path: Path, api_base: str) -> Path: + config: Final[dict[str, JsonValue]] = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["assistant_settings"] = { + "custom_llm_provider": "openai", + "litellm_params": {"api_base": api_base, "api_key": "synthetic-openai-key"}, + } + config_path: Final = tmp_path / "assistants_evals.yaml" + config_path.write_text(yaml.safe_dump(config), encoding="utf-8") + return config_path + + +def _respond(request: Request) -> Reply: + path: Final = request.target.partition("?")[0] + responses: Final = { + "/v1/threads": _THREAD, + "/v1/threads/thread_abc/messages": _MESSAGE, + "/v1/threads/thread_abc/runs": _RUN, + } + if request.method != "POST" or path not in responses: + return Reply(status=404, body=b'{"error":"unexpected upstream request"}') + return Reply(body=json.dumps(responses[path]).encode()) + + +@pytest.mark.parametrize("message", _THREAD_MESSAGES, ids=("string", "parts-attachments", "image-parts-attachments")) +@pytest.mark.parametrize("prefix", ("", "/v1"), ids=("root-alias", "v1-alias")) +def test_create_thread_forwards_sdk_body( + gateway: Gateway, tmp_path: Path, prefix: str, message: dict[str, JsonValue] +) -> None: + pytest.skip("BUG: POST /threads reaches the upstream with body {} and drops messages, metadata and tool_resources") + thread_request: Final[dict[str, JsonValue]] = { + "messages": [message], + "metadata": {"project": "alpha"}, + "tool_resources": {"code_interpreter": {"file_ids": ["file-abc"]}}, + } + with wire_server(_respond) as wire: + config: Final = _assistant_config(tmp_path, f"{wire.url}/v1") + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate: + base_url: Final = f"{str(candidate.client.base_url).rstrip('/')}{prefix}" + with OpenAI(base_url=base_url, api_key=candidate.key, max_retries=0) as client: + try: + raw: Final = client.beta.threads.with_raw_response.create( + messages=[message], + metadata={"project": "alpha"}, + tool_resources={"code_interpreter": {"file_ids": ["file-abc"]}}, + ) + except APIStatusError as error: + failure_text: Final = error.response.text + failed_requests: Final = wire.drain() + assert [(request.method, request.target) for request in failed_requests] == [ + ("POST", "/v1/threads") + ], failure_text + assert json.loads(failed_requests[0].body) == thread_request, failure_text + pytest.fail(f"SDK thread creation failed with HTTP {error.response.status_code}: {failure_text}") + assert raw.status_code == 200, raw.http_response.text + requests: Final = wire.drain() + assert [(request.method, request.target) for request in requests] == [("POST", "/v1/threads")], ( + raw.http_response.text + ) + assert json.loads(requests[0].body) == thread_request, raw.http_response.text + assert requests[0].headers["authorization"] == "Bearer synthetic-openai-key", raw.http_response.text + assert isinstance(raw.parse(), _Thread), raw.http_response.text + assert json.loads(raw.http_response.text) == _THREAD, raw.http_response.text + + +@pytest.mark.parametrize( + ("content", "prefix"), + ( + ("Summarize the attachment", ""), + ("Summarize the attachment", "/v1"), + ([{"type": "text", "text": "Summarize the attachment"}], ""), + ([{"type": "text", "text": "Summarize the attachment"}], "/v1"), + (_IMAGE_PARTS, ""), + (_IMAGE_PARTS, "/v1"), + ), + ids=( + "string-root-alias", + "string-v1-alias", + "parts-root-alias", + "parts-v1-alias", + "image-parts-root-alias", + "image-parts-v1-alias", + ), +) +def test_add_message_forwards_content_parts_attachments_metadata( + gateway: Gateway, tmp_path: Path, content: JsonValue, prefix: str +) -> None: + with wire_server(_respond) as wire: + config: Final = _assistant_config(tmp_path, f"{wire.url}/v1") + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate: + base_url: Final = f"{str(candidate.client.base_url).rstrip('/')}{prefix}" + with OpenAI(base_url=base_url, api_key=candidate.key, max_retries=0) as client: + raw: Final = client.beta.threads.messages.with_raw_response.create( + "thread_abc", + role="user", + content=content, + attachments=[{"file_id": "file-abc", "tools": [{"type": "file_search"}]}], + metadata={"turn": "2"}, + ) + assert raw.status_code == 200, raw.http_response.text + requests: Final = wire.drain() + assert [(request.method, request.target) for request in requests] == [ + ("POST", "/v1/threads/thread_abc/messages") + ], raw.http_response.text + assert requests[0].headers["authorization"] == "Bearer synthetic-openai-key", raw.http_response.text + assert json.loads(requests[0].body) == { + "role": "user", + "content": content, + "attachments": [{"file_id": "file-abc", "tools": [{"type": "file_search"}]}], + "metadata": {"turn": "2"}, + }, raw.http_response.text + assert isinstance(raw.parse(), _Message), raw.http_response.text + assert json.loads(raw.http_response.text) == _MESSAGE, raw.http_response.text + + +@pytest.mark.parametrize("prefix", ("", "/v1"), ids=("root-alias", "v1-alias")) +def test_run_forwards_every_sdk_field(gateway: Gateway, tmp_path: Path, prefix: str) -> None: + pytest.skip( + "BUG: runs.create drops additional_messages, temperature, top_p, max_prompt_tokens, max_completion_tokens, " + "truncation_strategy, tool_choice, response_format, parallel_tool_calls and reasoning_effort, sends nulls, " + "then polls " + "GET /runs/{id}" + ) + with wire_server(_respond) as wire: + config: Final = _assistant_config(tmp_path, f"{wire.url}/v1") + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate: + base_url: Final = f"{str(candidate.client.base_url).rstrip('/')}{prefix}" + with OpenAI(base_url=base_url, api_key=candidate.key, max_retries=0) as client: + try: + raw: Final = client.beta.threads.runs.with_raw_response.create( + "thread_abc", + assistant_id="asst_abc", + instructions="Be terse", + additional_instructions="Cite sources", + additional_messages=[{"role": "user", "content": "Also check page 2"}], + metadata={"run": "r1"}, + model="gpt-4o", + tools=[{"type": "code_interpreter"}], + temperature=0.3, + top_p=0.9, + max_prompt_tokens=500, + max_completion_tokens=50, + truncation_strategy={"type": "last_messages", "last_messages": 4}, + tool_choice="auto", + response_format={"type": "json_object"}, + parallel_tool_calls=False, + reasoning_effort="low", + ) + except APIStatusError as error: + failure_text: Final = error.response.text + failed_requests: Final = wire.drain() + assert [(request.method, request.target) for request in failed_requests] == [ + ("POST", "/v1/threads/thread_abc/runs") + ], failure_text + assert json.loads(failed_requests[0].body) == _RUN_REQUEST, failure_text + pytest.fail(f"SDK run creation failed with HTTP {error.response.status_code}: {failure_text}") + assert raw.status_code == 200, raw.http_response.text + requests: Final = wire.drain() + assert [(request.method, request.target) for request in requests] == [ + ("POST", "/v1/threads/thread_abc/runs") + ], raw.http_response.text + assert requests[0].headers["authorization"] == "Bearer synthetic-openai-key", raw.http_response.text + assert json.loads(requests[0].body) == _RUN_REQUEST, raw.http_response.text + run: Final[_Run] = raw.parse() + assert json.loads(raw.http_response.text) == _RUN, raw.http_response.text + assert run.status == "queued", raw.http_response.text + + +@pytest.mark.parametrize("prefix", ("", "/v1"), ids=("root-alias", "v1-alias")) +def test_run_raw_body_forwards_stream_false_and_typed_options(gateway: Gateway, tmp_path: Path, prefix: str) -> None: + pytest.skip( + "BUG: runs.create with a raw stream false body drops tool_choice and response_format, then polls GET /runs/{id}" + ) + with wire_server(_respond) as wire: + config: Final = _assistant_config(tmp_path, f"{wire.url}/v1") + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate: + response: Final = candidate.client.post( + f"{prefix}/threads/thread_abc/runs", + json={**_RAW_RUN_REQUEST, "stream": False}, + headers={"Authorization": f"Bearer {candidate.key}"}, + ) + requests: Final = wire.drain() + assert [(request.method, request.target) for request in requests] == [ + ("POST", "/v1/threads/thread_abc/runs") + ], response.text + assert requests[0].headers["authorization"] == "Bearer synthetic-openai-key", response.text + upstream_body: Final = json.loads(requests[0].body) + assert upstream_body.pop("stream", False) is False, response.text + assert upstream_body == _RAW_RUN_REQUEST, response.text + assert response.status_code == 200, response.text + assert response.headers["content-type"].startswith("application/json"), response.text + assert response.json() == _RUN, response.text + assert _Run.model_validate(response.json()).status == "queued", response.text + + +@pytest.mark.parametrize("prefix", ("", "/v1"), ids=("root-alias", "v1-alias")) +def test_run_include_query_reaches_upstream(gateway: Gateway, tmp_path: Path, prefix: str) -> None: + pytest.skip("BUG: runs.create include[] query param never reaches the upstream") + with wire_server(_respond) as wire: + config: Final = _assistant_config(tmp_path, f"{wire.url}/v1") + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate: + base_url: Final = f"{str(candidate.client.base_url).rstrip('/')}{prefix}" + with OpenAI(base_url=base_url, api_key=candidate.key, max_retries=0) as client: + raw: Final = client.beta.threads.runs.with_raw_response.create( + "thread_abc", + assistant_id="asst_abc", + include=["step_details.tool_calls[*].file_search.results[*].content"], + ) + assert raw.status_code == 200, raw.http_response.text + requests: Final = wire.drain() + assert [(request.method, urlsplit(request.target).path) for request in requests] == [ + ("POST", "/v1/threads/thread_abc/runs") + ], raw.http_response.text + assert parse_qs(urlsplit(requests[0].target).query) == { + "include[]": ["step_details.tool_calls[*].file_search.results[*].content"] + }, raw.http_response.text + assert json.loads(requests[0].body) == {"assistant_id": "asst_abc"}, raw.http_response.text + assert requests[0].headers["authorization"] == "Bearer synthetic-openai-key", raw.http_response.text + run: Final[_Run] = raw.parse() + assert isinstance(run, _Run), raw.http_response.text + assert json.loads(raw.http_response.text) == _RUN, raw.http_response.text diff --git a/tests/integration/providers/test_provider_lookup_status.py b/tests/integration/providers/test_provider_lookup_status.py index 2bc28d2b648..a231e7df9e4 100644 --- a/tests/integration/providers/test_provider_lookup_status.py +++ b/tests/integration/providers/test_provider_lookup_status.py @@ -3,21 +3,137 @@ from __future__ import annotations import asyncio import json import uuid +from collections.abc import Callable +from dataclasses import dataclass from pathlib import Path -from typing import Final, Literal +from typing import Final, Literal, Protocol +from urllib.parse import parse_qs, urlsplit import httpx import pytest from integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, object_value from integration._support.process import owned_proxy_process from integration._support.upstream import ScenarioHandle, delete_scenario, register_scenario +from integration._support.wire import Reply, Request, wire_server from openai import APIStatusError, AsyncOpenAI, OpenAI -from pydantic import JsonValue +from openai.pagination import SyncCursorPage +from openai.types.eval_create_response import EvalCreateResponse as _Eval +from openai.types.eval_delete_response import EvalDeleteResponse as _EvalDelete +from openai.types.eval_list_response import EvalListResponse as _EvalList +from openai.types.eval_retrieve_response import EvalRetrieveResponse as _EvalRetrieve +from openai.types.eval_update_response import EvalUpdateResponse as _EvalUpdate +from openai.types.evals.run_cancel_response import RunCancelResponse as _EvalRunCancel +from openai.types.evals.run_create_response import RunCreateResponse as _EvalRunCreate +from openai.types.evals.run_delete_response import RunDeleteResponse as _EvalRunDelete +from openai.types.evals.run_list_response import RunListResponse as _EvalRunList +from openai.types.evals.run_retrieve_response import RunRetrieveResponse as _EvalRunRetrieve +from pydantic import BaseModel, JsonValue from litellm.proxy.openai_files_endpoints.common_utils import encode_file_id_with_model from litellm.types.videos.utils import encode_video_id_with_provider from tests.integration.cost_calculation.cost_tracking_case import JsonResponse, RoutedResponse +_EVAL_DATA_SOURCE_CONFIG: Final[dict[str, JsonValue]] = { + "type": "custom", + "item_schema": {"type": "object", "properties": {"a": {"type": "string"}}}, +} +_EVAL_DATA_SOURCE_CONFIG_RESPONSE: Final[dict[str, JsonValue]] = { + "type": "custom", + "schema": {"type": "object", "properties": {"a": {"type": "string"}}}, +} +_EVAL_TESTING_CRITERIA: Final[list[JsonValue]] = [ + {"type": "string_check", "name": "exact", "input": "{{item.a}}", "reference": "{{item.a}}", "operation": "eq"} +] +_EVAL_CREATE_UPSTREAM_RESPONSE: Final[dict[str, JsonValue]] = { + "id": "eval_abc", + "object": "eval", + "created_at": 1700000000, + "name": "nightly", + "data_source_config": _EVAL_DATA_SOURCE_CONFIG_RESPONSE, + "testing_criteria": _EVAL_TESTING_CRITERIA, + "metadata": {"suite": "nightly"}, +} +_EVAL_CREATE_RESPONSE: Final[dict[str, JsonValue]] = { + **_EVAL_CREATE_UPSTREAM_RESPONSE, + "updated_at": None, +} +_EVAL_UPDATE_UPSTREAM_RESPONSE: Final[dict[str, JsonValue]] = { + **_EVAL_CREATE_UPSTREAM_RESPONSE, + "name": "renamed", + "metadata": {"team": "search"}, +} +_EVAL_UPDATE_RESPONSE: Final[dict[str, JsonValue]] = { + **_EVAL_UPDATE_UPSTREAM_RESPONSE, + "updated_at": None, +} +_EVAL_RUN_DATA_SOURCE: Final[dict[str, JsonValue]] = { + "type": "jsonl", + "source": {"type": "file_id", "id": "file-abc"}, +} +_EVAL_RUN_RESPONSE: Final[dict[str, JsonValue]] = { + "id": "run_abc", + "object": "eval.run", + "created_at": 1700000001, + "eval_id": "eval_abc", + "started_at": None, + "completed_at": None, + "data_source": _EVAL_RUN_DATA_SOURCE, + "error": None, + "per_model_usage": [], + "per_testing_criteria_results": [], + "report_url": "https://example.invalid/evals/run_abc", + "result_counts": {"errored": 0, "failed": 0, "passed": 0, "total": 0}, + "shared_with_openai": None, + "name": "run-1", + "model": "gpt-4o-mini", + "metadata": {"suite": "nightly"}, + "status": "queued", +} +_EVAL_RUN_CANCEL_RESPONSE: Final[dict[str, JsonValue]] = { + **_EVAL_RUN_RESPONSE, + "status": "cancelled", +} +_EVAL_DELETE_RESPONSE: Final[dict[str, JsonValue]] = {"object": "eval.deleted", "deleted": True, "eval_id": "eval_abc"} +_EVAL_RUN_DELETE_RESPONSE: Final[dict[str, JsonValue]] = { + "object": "eval.run.deleted", + "deleted": True, + "run_id": "run_abc", +} +_EVAL_LIST_UPSTREAM_RESPONSE: Final[dict[str, JsonValue]] = { + "object": "list", + "data": [_EVAL_CREATE_UPSTREAM_RESPONSE], + "first_id": "eval_abc", + "last_id": "eval_abc", + "has_more": False, +} +_EVAL_LIST_RESPONSE: Final[dict[str, JsonValue]] = {**_EVAL_LIST_UPSTREAM_RESPONSE, "data": [_EVAL_CREATE_RESPONSE]} +_EVAL_RUN_LIST_RESPONSE: Final[dict[str, JsonValue]] = { + "object": "list", + "data": [_EVAL_RUN_RESPONSE], + "first_id": "run_abc", + "last_id": "run_abc", + "has_more": False, +} +_EVAL_UPSTREAM_RESPONSES: Final[dict[tuple[str, str], dict[str, JsonValue]]] = { + ("POST", "/v1/evals"): _EVAL_CREATE_UPSTREAM_RESPONSE, + ("GET", "/v1/evals"): _EVAL_LIST_UPSTREAM_RESPONSE, + ("GET", "/v1/evals/eval_abc"): _EVAL_CREATE_UPSTREAM_RESPONSE, + ("POST", "/v1/evals/eval_abc"): _EVAL_UPDATE_UPSTREAM_RESPONSE, + ("DELETE", "/v1/evals/eval_abc"): _EVAL_DELETE_RESPONSE, + ("POST", "/v1/evals/eval_abc/runs"): _EVAL_RUN_RESPONSE, + ("GET", "/v1/evals/eval_abc/runs"): _EVAL_RUN_LIST_RESPONSE, + ("GET", "/v1/evals/eval_abc/runs/run_abc"): _EVAL_RUN_RESPONSE, + ("POST", "/v1/evals/eval_abc/runs/run_abc/cancel"): _EVAL_RUN_CANCEL_RESPONSE, + ("DELETE", "/v1/evals/eval_abc/runs/run_abc"): _EVAL_RUN_DELETE_RESPONSE, +} + + +def _evals_respond(request: Request) -> Reply: + body: Final = _EVAL_UPSTREAM_RESPONSES.get((request.method, urlsplit(request.target).path)) + if body is None: + return Reply(status=404, body=b'{"error":"unexpected upstream request"}') + return Reply(body=json.dumps(body).encode()) + def _provider_error(status: int) -> dict[str, JsonValue]: return { @@ -106,6 +222,309 @@ def _assert_provider_error(response: httpx.Response, status: int) -> None: assert "scripted provider status" in str(error.get("message")), response.text +def _assert_no_gateway_headers(gateway: Gateway, requests: tuple[Request, ...], context: str) -> None: + leaked: Final = [ + {name: value for name, value in request.headers.items() if name == "x-litellm-model" or gateway.key in value} + for request in requests + ] + assert leaked == [{} for _ in requests], context + + +def test_eval_update_forwards_only_client_fields(gateway: Gateway) -> None: + pytest.skip( + "BUG: evals.update returns 500 Object of type datetime is not JSON serializable and never reaches the upstream" + ) + with wire_server(_evals_respond) as wire, gateway.scenario() as scenario: + alias: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url, api_key="synthetic-openai-key") + _ready(gateway, alias) + wire.drain() + with OpenAI( + base_url=f"{str(gateway.client.base_url).rstrip('/')}/v1", api_key=gateway.key, max_retries=0 + ) as client: + try: + raw: Final = client.evals.with_raw_response.update( + "eval_abc", name="renamed", metadata={"team": "search"}, extra_body={"model": alias} + ) + except APIStatusError as error: + failure_text: Final = error.response.text + failed_requests: Final = wire.drain() + assert [(request.method, request.target) for request in failed_requests] == [ + ("POST", "/v1/evals/eval_abc") + ], failure_text + assert failed_requests[0].headers["authorization"] == "Bearer synthetic-openai-key", failure_text + assert json.loads(failed_requests[0].body) == {"name": "renamed", "metadata": {"team": "search"}}, ( + failure_text + ) + pytest.fail(f"SDK Eval update failed with HTTP {error.response.status_code}: {failure_text}") + assert raw.status_code == 200, raw.http_response.text + requests: Final = wire.drain() + assert [(request.method, request.target) for request in requests] == [("POST", "/v1/evals/eval_abc")], ( + raw.http_response.text + ) + assert requests[0].headers["authorization"] == "Bearer synthetic-openai-key", raw.http_response.text + _assert_no_gateway_headers(gateway, requests, raw.http_response.text) + assert json.loads(requests[0].body) == {"name": "renamed", "metadata": {"team": "search"}}, ( + raw.http_response.text + ) + evaluation: Final[_EvalUpdate] = raw.parse() + assert json.loads(raw.http_response.text) == _EVAL_UPDATE_RESPONSE, raw.http_response.text + assert evaluation.name == "renamed", raw.http_response.text + + +class _RawResponse(Protocol): + @property + def http_response(self) -> httpx.Response: ... + + def parse(self) -> object: ... + + +@dataclass(frozen=True, slots=True) +class _ModelRouting: + body: dict[str, object] | None + query: dict[str, object] | None + headers: dict[str, str] | None + + +@dataclass(frozen=True, slots=True) +class _EvalCall: + send: Callable[[OpenAI, _ModelRouting], _RawResponse] + parsed: type[BaseModel] + upstream: tuple[str, str, dict[str, list[str]], JsonValue] + response: dict[str, JsonValue] + + +_EVAL_CREATE_CALL: Final = _EvalCall( + send=lambda client, routing: client.evals.with_raw_response.create( + name="nightly", + data_source_config=_EVAL_DATA_SOURCE_CONFIG, + testing_criteria=_EVAL_TESTING_CRITERIA, + extra_body=routing.body, + ), + parsed=_Eval, + upstream=( + "POST", + "/v1/evals", + {}, + {"name": "nightly", "data_source_config": _EVAL_DATA_SOURCE_CONFIG, "testing_criteria": _EVAL_TESTING_CRITERIA}, + ), + response=_EVAL_CREATE_RESPONSE, +) +_EVAL_CREATE_WITH_METADATA_CALL: Final = _EvalCall( + send=lambda client, routing: client.evals.with_raw_response.create( + name="nightly", + data_source_config=_EVAL_DATA_SOURCE_CONFIG, + testing_criteria=_EVAL_TESTING_CRITERIA, + metadata={"suite": "nightly"}, + extra_body=routing.body, + ), + parsed=_Eval, + upstream=( + "POST", + "/v1/evals", + {}, + { + "name": "nightly", + "data_source_config": _EVAL_DATA_SOURCE_CONFIG, + "testing_criteria": _EVAL_TESTING_CRITERIA, + "metadata": {"suite": "nightly"}, + }, + ), + response=_EVAL_CREATE_RESPONSE, +) +_EVAL_LIST_CALL: Final = _EvalCall( + send=lambda client, routing: client.evals.with_raw_response.list( + limit=2, after="eval_000", order="asc", order_by="updated_at", extra_query=routing.query + ), + parsed=_EvalList, + upstream=( + "GET", + "/v1/evals", + {"limit": ["2"], "after": ["eval_000"], "order": ["asc"], "order_by": ["updated_at"]}, + None, + ), + response=_EVAL_LIST_RESPONSE, +) +_EVAL_RETRIEVE_CALL: Final = _EvalCall( + send=lambda client, routing: client.evals.with_raw_response.retrieve("eval_abc", extra_query=routing.query), + parsed=_EvalRetrieve, + upstream=("GET", "/v1/evals/eval_abc", {}, None), + response=_EVAL_CREATE_RESPONSE, +) +_EVAL_UPDATE_CALL: Final = _EvalCall( + send=lambda client, routing: client.evals.with_raw_response.update( + "eval_abc", name="renamed", metadata={"team": "search"}, extra_body=routing.body + ), + parsed=_EvalUpdate, + upstream=("POST", "/v1/evals/eval_abc", {}, {"name": "renamed", "metadata": {"team": "search"}}), + response=_EVAL_UPDATE_RESPONSE, +) +_EVAL_DELETE_CALL: Final = _EvalCall( + send=lambda client, routing: client.evals.with_raw_response.delete("eval_abc", extra_query=routing.query), + parsed=_EvalDelete, + upstream=("DELETE", "/v1/evals/eval_abc", {}, None), + response=_EVAL_DELETE_RESPONSE, +) +_EVAL_RUN_CREATE_WITH_METADATA_CALL: Final = _EvalCall( + send=lambda client, routing: client.evals.runs.with_raw_response.create( + "eval_abc", + data_source=_EVAL_RUN_DATA_SOURCE, + name="run-1", + metadata={"suite": "nightly"}, + extra_body=routing.body, + ), + parsed=_EvalRunCreate, + upstream=( + "POST", + "/v1/evals/eval_abc/runs", + {}, + {"data_source": _EVAL_RUN_DATA_SOURCE, "name": "run-1", "metadata": {"suite": "nightly"}}, + ), + response=_EVAL_RUN_RESPONSE, +) +_EVAL_RUN_LIST_CALL: Final = _EvalCall( + send=lambda client, routing: client.evals.runs.with_raw_response.list( + "eval_abc", limit=3, after="run_000", order="desc", extra_query=routing.query + ), + parsed=_EvalRunList, + upstream=("GET", "/v1/evals/eval_abc/runs", {"limit": ["3"], "after": ["run_000"], "order": ["desc"]}, None), + response=_EVAL_RUN_LIST_RESPONSE, +) +_EVAL_RUN_LIST_BY_STATUS_CALL: Final = _EvalCall( + send=lambda client, routing: client.evals.runs.with_raw_response.list( + "eval_abc", limit=3, after="run_000", order="desc", status="completed", extra_query=routing.query + ), + parsed=_EvalRunList, + upstream=( + "GET", + "/v1/evals/eval_abc/runs", + {"limit": ["3"], "after": ["run_000"], "order": ["desc"], "status": ["completed"]}, + None, + ), + response=_EVAL_RUN_LIST_RESPONSE, +) +_EVAL_RUN_RETRIEVE_CALL: Final = _EvalCall( + send=lambda client, routing: client.evals.runs.with_raw_response.retrieve( + "run_abc", eval_id="eval_abc", extra_query=routing.query + ), + parsed=_EvalRunRetrieve, + upstream=("GET", "/v1/evals/eval_abc/runs/run_abc", {}, None), + response=_EVAL_RUN_RESPONSE, +) +_EVAL_RUN_CANCEL_CALL: Final = _EvalCall( + send=lambda client, routing: client.evals.runs.with_raw_response.cancel( + "run_abc", eval_id="eval_abc", extra_query=routing.query + ), + parsed=_EvalRunCancel, + # OpenAI cancel path per openai-python 3.30.0 evals.runs.cancel, checked 2026-10-14 + upstream=("POST", "/v1/evals/eval_abc/runs/run_abc/cancel", {}, {}), + response=_EVAL_RUN_CANCEL_RESPONSE, +) +_EVAL_RUN_DELETE_CALL: Final = _EvalCall( + send=lambda client, routing: client.evals.runs.with_raw_response.delete( + "run_abc", eval_id="eval_abc", extra_query=routing.query + ), + parsed=_EvalRunDelete, + upstream=("DELETE", "/v1/evals/eval_abc/runs/run_abc", {}, None), + response=_EVAL_RUN_DELETE_RESPONSE, +) +_EVAL_ROUTES_WITHOUT_KNOWN_BUGS: Final = ( + _EVAL_CREATE_CALL, + _EVAL_LIST_CALL, + _EVAL_RETRIEVE_CALL, + _EVAL_DELETE_CALL, + _EVAL_RUN_LIST_CALL, + _EVAL_RUN_DELETE_CALL, +) +_EVERY_EVAL_ROUTE: Final = ( + _EVAL_CREATE_WITH_METADATA_CALL, + _EVAL_LIST_CALL, + _EVAL_RETRIEVE_CALL, + _EVAL_UPDATE_CALL, + _EVAL_DELETE_CALL, + _EVAL_RUN_CREATE_WITH_METADATA_CALL, + _EVAL_RUN_LIST_BY_STATUS_CALL, + _EVAL_RUN_RETRIEVE_CALL, + _EVAL_RUN_CANCEL_CALL, + _EVAL_RUN_DELETE_CALL, +) + + +def _model_routing(alias: str, spelling: str) -> _ModelRouting: + if spelling == "x-litellm-model": + return _ModelRouting(body=None, query=None, headers={"x-litellm-model": alias}) + return _ModelRouting(body={"model": alias}, query={"model": alias}, headers=None) + + +def _parsed_types(parsed: object) -> tuple[type, ...]: + if isinstance(parsed, SyncCursorPage): + return tuple(type(item) for item in parsed.data) + return (type(parsed),) + + +def _observed(request: Request) -> tuple[str, str, dict[str, list[str]], JsonValue]: + target: Final = urlsplit(request.target) + body: Final[JsonValue] = json.loads(request.body) if request.body else None + return request.method, target.path, parse_qs(target.query), body + + +def _assert_eval_sdk_paths(gateway: Gateway, suffix: str, spelling: str, calls: tuple[_EvalCall, ...]) -> None: + with wire_server(_evals_respond) as wire, gateway.scenario() as scenario: + alias: Final = scenario.model( + model="openai/gpt-4o-mini", api_base=f"{wire.url}{suffix}", api_key="synthetic-openai-key" + ) + _ready(gateway, alias) + wire.drain() + routing: Final = _model_routing(alias, spelling) + with OpenAI( + base_url=f"{str(gateway.client.base_url).rstrip('/')}/v1", + api_key=gateway.key, + max_retries=0, + default_headers=routing.headers, + ) as client: + try: + raws: Final = tuple(call.send(client, routing) for call in calls) + except APIStatusError as error: + failure_text: Final = error.response.text + observed: Final = [_observed(request) for request in wire.drain()] + pytest.fail(f"SDK eval call failed with HTTP {error.response.status_code}: {failure_text}\n{observed}") + response_text: Final = "\n".join(raw.http_response.text for raw in raws) + requests: Final = wire.drain() + assert [_observed(request) for request in requests] == [call.upstream for call in calls], response_text + assert {request.headers["authorization"] for request in requests} == {"Bearer synthetic-openai-key"}, ( + response_text + ) + _assert_no_gateway_headers(gateway, requests, response_text) + assert [json.loads(raw.http_response.text) for raw in raws] == [call.response for call in calls], response_text + assert [_parsed_types(raw.parse()) for raw in raws] == [(call.parsed,) for call in calls], response_text + + +@pytest.mark.parametrize("spelling", ("model-param", "x-litellm-model")) +def test_eval_routes_reach_sdk_paths_with_bare_api_base(gateway: Gateway, spelling: str) -> None: + _assert_eval_sdk_paths(gateway, "", spelling, _EVAL_ROUTES_WITHOUT_KNOWN_BUGS) + + +@pytest.mark.parametrize("spelling", ("model-param", "x-litellm-model")) +@pytest.mark.parametrize("suffix", ("/v1", "/v1/"), ids=("v1", "v1-trailing-slash")) +def test_eval_routes_reach_sdk_paths_with_v1_api_base(gateway: Gateway, suffix: str, spelling: str) -> None: + pytest.skip("BUG: an api_base ending in /v1 sends every eval request to /v1/v1/evals") + _assert_eval_sdk_paths(gateway, suffix, spelling, _EVAL_ROUTES_WITHOUT_KNOWN_BUGS) + + +@pytest.mark.parametrize("spelling", ("model-param", "x-litellm-model")) +@pytest.mark.parametrize("suffix", ("", "/v1", "/v1/"), ids=("bare", "v1", "v1-trailing-slash")) +def test_every_eval_route_reaches_sdk_path_and_relays_upstream_body( + gateway: Gateway, suffix: str, spelling: str +) -> None: + pytest.skip( + "BUG: evals.runs.cancel returns only {id, object, status} " + "instead of relaying the full upstream eval run (the route's response_model=CancelRunResponse strips it), " + "evals.update returns 500, create and run create drop metadata, runs.list drops status, " + "run create and retrieve overwrite the run's model with the routing alias, " + "and an api_base ending in /v1 sends every eval request to /v1/v1/evals" + ) + _assert_eval_sdk_paths(gateway, suffix, spelling, _EVERY_EVAL_ROUTE) + + @pytest.mark.parametrize( "status", (400, 401, 404, 429, 500), ids=("bad-request", "unauthorized", "not-found", "rate-limit", "server-error") ) diff --git a/tests/integration/streaming/test_assistants_evals_stream.py b/tests/integration/streaming/test_assistants_evals_stream.py new file mode 100644 index 00000000000..9f9e0275154 --- /dev/null +++ b/tests/integration/streaming/test_assistants_evals_stream.py @@ -0,0 +1,403 @@ +from __future__ import annotations + +import json +import threading +from collections.abc import Callable, Iterator +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from openai import APITimeoutError, OpenAI +from pydantic import JsonValue + +_QUEUED_RUN: Final[dict[str, JsonValue]] = { + "id": "run_abc", + "assistant_id": "asst_abc", + "cancelled_at": None, + "completed_at": None, + "created_at": 1700000002, + "expires_at": None, + "failed_at": None, + "incomplete_details": None, + "instructions": "Be terse", + "last_error": None, + "max_completion_tokens": 50, + "max_prompt_tokens": 500, + "metadata": {"run": "r1"}, + "model": "gpt-4o", + "object": "thread.run", + "parallel_tool_calls": False, + "required_action": None, + "response_format": {"type": "json_object"}, + "started_at": None, + "status": "queued", + "thread_id": "thread_abc", + "tool_choice": "auto", + "tools": [{"type": "code_interpreter"}], + "truncation_strategy": {"type": "last_messages", "last_messages": 4}, + "usage": None, + "temperature": 0.3, + "top_p": 0.9, +} +_MESSAGE_CREATED: Final[dict[str, JsonValue]] = { + "id": "msg_abc", + "assistant_id": "asst_abc", + "attachments": None, + "completed_at": None, + "content": [], + "created_at": 1700000003, + "incomplete_at": None, + "incomplete_details": None, + "metadata": {}, + "object": "thread.message", + "role": "assistant", + "run_id": "run_abc", + "status": "in_progress", + "thread_id": "thread_abc", +} +_MESSAGE_DELTA: Final[dict[str, JsonValue]] = { + "id": "msg_abc", + "object": "thread.message.delta", + "delta": {"content": [{"index": 0, "type": "text", "text": {"value": "Hi"}}]}, +} +_COMPLETED_RUN: Final[dict[str, JsonValue]] = { + **_QUEUED_RUN, + "completed_at": 1700000004, + "status": "completed", + "usage": {"completion_tokens": 1, "prompt_tokens": 1, "total_tokens": 2}, +} +_RUN_REQUEST: Final[dict[str, JsonValue]] = { + "assistant_id": "asst_abc", + "instructions": "Be terse", + "additional_instructions": "Cite sources", + "additional_messages": [{"role": "user", "content": "Also check page 2"}], + "metadata": {"run": "r1"}, + "model": "gpt-4o", + "tools": [{"type": "code_interpreter"}], + "temperature": 0.3, + "top_p": 0.9, + "max_prompt_tokens": 500, + "max_completion_tokens": 50, + "truncation_strategy": {"type": "last_messages", "last_messages": 4}, + "tool_choice": "auto", + "response_format": {"type": "json_object"}, + "parallel_tool_calls": False, + "reasoning_effort": "low", +} + + +def _assistant_config(tmp_path: Path, api_base: str) -> Path: + config: Final[dict[str, JsonValue]] = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["assistant_settings"] = { + "custom_llm_provider": "openai", + "litellm_params": {"api_base": api_base, "api_key": "synthetic-openai-key"}, + } + config_path: Final = tmp_path / "assistants_evals_stream.yaml" + config_path.write_text(yaml.safe_dump(config), encoding="utf-8") + return config_path + + +def _frame(event: str, data: dict[str, JsonValue] | str) -> bytes: + encoded_data: Final = data if isinstance(data, str) else json.dumps(data) + return f"event: {event}\ndata: {encoded_data}\n\n".encode() + + +def _respond(request: Request) -> Reply: + if request.method != "POST" or request.target != "/v1/threads/thread_abc/runs": + return Reply(status=404, body=b'{"error":"unexpected upstream request"}') + return Reply( + content_type="text/event-stream", + chunks=( + _frame("thread.run.created", _QUEUED_RUN), + _frame("thread.message.created", _MESSAGE_CREATED), + _frame("thread.message.delta", _MESSAGE_DELTA), + _frame("thread.run.completed", _COMPLETED_RUN), + _frame("done", "[DONE]"), + ), + ) + + +def _parse_frame(frame: str) -> tuple[str, JsonValue]: + lines: Final = frame.splitlines() + event: Final = next((line.removeprefix("event: ") for line in lines if line.startswith("event: ")), "") + data: Final = next((line.removeprefix("data: ") for line in lines if line.startswith("data: ")), "") + return event, data if data == "[DONE]" else json.loads(data) + + +def _parse_frames(body: str) -> tuple[tuple[str, JsonValue], ...]: + return tuple(_parse_frame(frame) for frame in body.strip().split("\n\n")) + + +_EXPECTED_FRAMES: Final = ( + ("thread.run.created", _QUEUED_RUN), + ("thread.message.created", _MESSAGE_CREATED), + ("thread.message.delta", _MESSAGE_DELTA), + ("thread.run.completed", _COMPLETED_RUN), + ("done", "[DONE]"), +) + + +class _RecordingStream(httpx.SyncByteStream): + def __init__(self, inner: httpx.SyncByteStream, sink: list[bytes]) -> None: + self._inner: Final = inner + self._sink: Final = sink + + def __iter__(self) -> Iterator[bytes]: + for chunk in self._inner: + self._sink.append(chunk) + yield chunk + + def close(self) -> None: + self._inner.close() + + +class _RecordingTransport(httpx.BaseTransport): + """Records the bytes the SDK client read off the proxy socket, so the done frame the SDK swallows is visible.""" + + def __init__(self) -> None: + self.received: Final[list[bytes]] = [] + self._inner: Final = httpx.HTTPTransport() + + def handle_request(self, request: httpx.Request) -> httpx.Response: + response: Final = self._inner.handle_request(request) + assert isinstance(response.stream, httpx.SyncByteStream) + return httpx.Response( + status_code=response.status_code, + headers=response.headers, + stream=_RecordingStream(response.stream, self.received), + extensions=response.extensions, + request=request, + ) + + def close(self) -> None: + self._inner.close() + + def text(self) -> str: + return b"".join(self.received).decode() + + +@pytest.mark.parametrize("prefix", ("", "/v1"), ids=("root-alias", "v1-alias")) +def test_run_stream_relays_upstream_events(gateway: Gateway, tmp_path: Path, prefix: str) -> None: + pytest.skip( + "BUG: streamed run frames carry the event name inside data with no event: line and drop run options upstream" + ) + with wire_server(_respond) as wire: + config: Final = _assistant_config(tmp_path, f"{wire.url}/v1") + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate: + body: Final[dict[str, JsonValue]] = {**_RUN_REQUEST, "stream": True} + response: Final = candidate.request("POST", f"{prefix}/threads/thread_abc/runs", body) + assert response.status_code == 200, response.text + assert response.headers["content-type"].startswith("text/event-stream"), response.text + assert _parse_frames(response.text) == _EXPECTED_FRAMES, response.text + requests: Final = wire.drain() + assert [(request.method, request.target) for request in requests] == [ + ("POST", "/v1/threads/thread_abc/runs") + ], response.text + assert requests[0].headers["authorization"] == "Bearer synthetic-openai-key", response.text + assert json.loads(requests[0].body) == body, response.text + + +@pytest.mark.parametrize("prefix", ("", "/v1"), ids=("root-alias", "v1-alias")) +def test_sdk_run_stream_event_handler_sees_events(gateway: Gateway, tmp_path: Path, prefix: str) -> None: + pytest.skip("BUG: runs.stream drops additional_messages, sampling, token caps and other run options upstream") + with wire_server(_respond) as wire: + config: Final = _assistant_config(tmp_path, f"{wire.url}/v1") + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate: + transport: Final = _RecordingTransport() + with OpenAI( + base_url=f"{str(candidate.client.base_url).rstrip('/')}{prefix}", + api_key=candidate.key, + max_retries=0, + http_client=httpx.Client(transport=transport), + ) as client: + with client.beta.threads.runs.stream( + thread_id="thread_abc", + assistant_id="asst_abc", + instructions="Be terse", + additional_instructions="Cite sources", + additional_messages=[{"role": "user", "content": "Also check page 2"}], + metadata={"run": "r1"}, + model="gpt-4o", + tools=[{"type": "code_interpreter"}], + temperature=0.3, + top_p=0.9, + max_prompt_tokens=500, + max_completion_tokens=50, + truncation_strategy={"type": "last_messages", "last_messages": 4}, + tool_choice="auto", + response_format={"type": "json_object"}, + parallel_tool_calls=False, + reasoning_effort="low", + ) as stream: + events: Final = tuple((event.event, event.data.to_dict(mode="json")) for event in stream) + response_text: Final = transport.text() + assert events == _EXPECTED_FRAMES[:-1], response_text + assert _parse_frames(response_text) == _EXPECTED_FRAMES, response_text + requests: Final = wire.drain() + assert [(request.method, request.target) for request in requests] == [ + ("POST", "/v1/threads/thread_abc/runs") + ], response_text + assert requests[0].headers["authorization"] == "Bearer synthetic-openai-key", response_text + assert json.loads(requests[0].body) == {**_RUN_REQUEST, "stream": True}, response_text + + +@pytest.mark.parametrize("prefix", ("", "/v1"), ids=("root-alias", "v1-alias")) +def test_sdk_run_create_stream_yields_events(gateway: Gateway, tmp_path: Path, prefix: str) -> None: + pytest.skip("BUG: runs.create(stream=True) yields null event names and drops run options upstream") + with wire_server(_respond) as wire: + config: Final = _assistant_config(tmp_path, f"{wire.url}/v1") + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate: + transport: Final = _RecordingTransport() + with OpenAI( + base_url=f"{str(candidate.client.base_url).rstrip('/')}{prefix}", + api_key=candidate.key, + max_retries=0, + http_client=httpx.Client(transport=transport), + ) as client: + with client.beta.threads.runs.create( + "thread_abc", + assistant_id="asst_abc", + instructions="Be terse", + additional_instructions="Cite sources", + additional_messages=[{"role": "user", "content": "Also check page 2"}], + metadata={"run": "r1"}, + model="gpt-4o", + tools=[{"type": "code_interpreter"}], + temperature=0.3, + top_p=0.9, + max_prompt_tokens=500, + max_completion_tokens=50, + truncation_strategy={"type": "last_messages", "last_messages": 4}, + tool_choice="auto", + response_format={"type": "json_object"}, + parallel_tool_calls=False, + reasoning_effort="low", + stream=True, + ) as stream: + events: Final = tuple((event.event, event.data.to_dict(mode="json")) for event in stream) + response_text: Final = transport.text() + assert events == _EXPECTED_FRAMES[:-1], response_text + assert _parse_frames(response_text) == _EXPECTED_FRAMES, response_text + requests: Final = wire.drain() + assert [(request.method, request.target) for request in requests] == [ + ("POST", "/v1/threads/thread_abc/runs") + ], response_text + assert requests[0].headers["authorization"] == "Bearer synthetic-openai-key", response_text + assert json.loads(requests[0].body) == {**_RUN_REQUEST, "stream": True}, response_text + + +_MARKER_MESSAGE: Final[dict[str, JsonValue]] = { + "id": "msg_marker", + "assistant_id": None, + "attachments": [], + "completed_at": None, + "object": "thread.message", + "content": [{"type": "text", "text": {"value": "marker", "annotations": []}}], + "created_at": 1700000005, + "incomplete_at": None, + "incomplete_details": None, + "metadata": {}, + "role": "user", + "run_id": None, + "status": "completed", + "thread_id": "thread_abc", +} +_HELD_FRAME: Final = b":" + b"x" * 4_000_000 + b"\n\n" + + +def _held_respond(gate: threading.Event, run_reply: Reply) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.method == "POST" and request.target == "/v1/threads/thread_abc/runs": + return run_reply + if request.method == "GET" and request.target == "/v1/threads/thread_abc/runs/run_abc": + return Reply(body=json.dumps(_QUEUED_RUN).encode()) + if request.method == "POST" and request.target == "/v1/threads/thread_abc/messages": + return Reply(body=json.dumps(_MARKER_MESSAGE).encode()) + return Reply(status=404, body=b'{"error":"unexpected upstream request"}') + + assert run_reply.gate_after_first is gate + return respond + + +def _marker_message(base_url: str, key: str) -> str: + with OpenAI(base_url=base_url, api_key=key, max_retries=0) as client: + raw: Final = client.beta.threads.messages.with_raw_response.create("thread_abc", role="user", content="marker") + assert json.loads(raw.http_response.text) == _MARKER_MESSAGE, raw.http_response.text + return raw.http_response.text + + +def test_run_stream_client_disconnect_releases_upstream_without_polling(gateway: Gateway, tmp_path: Path) -> None: + pytest.skip( + "BUG: runs.create(stream=True) yields null event names, so the first SDK event is not thread.run.created" + ) + gate: Final = threading.Event() + run_reply: Final = Reply( + content_type="text/event-stream", + chunks=( + _frame("thread.run.created", _QUEUED_RUN), + _HELD_FRAME, + _frame("thread.message.created", _MESSAGE_CREATED), + _frame("thread.run.completed", _COMPLETED_RUN), + _frame("done", "[DONE]"), + ), + gate_after_first=gate, + ) + with wire_server(_held_respond(gate, run_reply)) as wire: + config: Final = _assistant_config(tmp_path, f"{wire.url}/v1") + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate: + base_url: Final = f"{str(candidate.client.base_url).rstrip('/')}/v1" + try: + with OpenAI(base_url=base_url, api_key=candidate.key, max_retries=0) as client: + with client.beta.threads.runs.create("thread_abc", assistant_id="asst_abc", stream=True) as stream: + first: Final = next(iter(stream)) + finally: + gate.set() + response_text: Final = json.dumps({"event": first.event, "data": first.data.to_dict(mode="json")}) + assert (first.event, first.data.to_dict(mode="json")) == ("thread.run.created", _QUEUED_RUN), response_text + assert wire.disconnected.get(timeout=10) == "/v1/threads/thread_abc/runs", response_text + marker_text: Final = _marker_message(base_url, candidate.key) + requests: Final = wire.drain() + assert [(request.method, request.target) for request in requests] == [ + ("POST", "/v1/threads/thread_abc/runs"), + ("POST", "/v1/threads/thread_abc/messages"), + ], f"{response_text}\n{marker_text}" + assert json.loads(requests[0].body) == {"assistant_id": "asst_abc", "stream": True}, response_text + + +def test_run_client_disconnect_does_not_leave_the_proxy_polling(gateway: Gateway, tmp_path: Path) -> None: + pytest.skip("BUG: after the client of a non-stream runs.create disconnects, the proxy keeps polling GET /runs/{id}") + gate: Final = threading.Event() + queued_body: Final = json.dumps(_QUEUED_RUN).encode() + run_reply: Final = Reply(chunks=(queued_body[:20], queued_body[20:]), gate_after_first=gate) + with wire_server(_held_respond(gate, run_reply)) as wire: + config: Final = _assistant_config(tmp_path, f"{wire.url}/v1") + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate: + base_url: Final = f"{str(candidate.client.base_url).rstrip('/')}/v1" + try: + with OpenAI( + base_url=base_url, api_key=candidate.key, max_retries=0, timeout=httpx.Timeout(30, read=2) + ) as client: + with pytest.raises(APITimeoutError): + client.beta.threads.runs.create("thread_abc", assistant_id="asst_abc") + finally: + gate.set() + marker_text: Final = _marker_message(base_url, candidate.key) + requests: Final = wire.drain() + assert [(request.method, request.target) for request in requests] == [ + ("POST", "/v1/threads/thread_abc/runs"), + ("POST", "/v1/threads/thread_abc/messages"), + ], marker_text + assert json.loads(requests[0].body) == {"assistant_id": "asst_abc"}, marker_text + with OpenAI(base_url=base_url, api_key=candidate.key, max_retries=0) as client: + raw: Final = client.beta.threads.runs.with_raw_response.create("thread_abc", assistant_id="asst_abc") + assert raw.headers["content-type"].startswith("application/json"), raw.http_response.text + assert json.loads(raw.http_response.text) == _QUEUED_RUN, raw.http_response.text + assert raw.parse().status == "queued", raw.http_response.text + assert [(request.method, request.target) for request in wire.drain()] == [ + ("POST", "/v1/threads/thread_abc/runs") + ], raw.http_response.text From 5f7de8d6ac84b352fa96e905e21fa04e307b9659 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:12:28 +0000 Subject: [PATCH 12/51] test(integration): cover keys_auth endpoint contracts (#44674) * test(integration): cover keys_auth endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover hashed, path and token spellings for keys_auth contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): strengthen JWT mapping lifecycle coverage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover reset_spend over plaintext and hashed path spellings with reset_to 0 and 0.02 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): drop Final in reset loop and use drain-then-fresh-refusal in warmed-peer delete proof Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): disable model info refresh in JWT lifecycle rig Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): drop unused discovery constant Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover cache invalidation and key paths Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo --- .../test_jwt_key_mapping_lifecycle.py | 863 ++++++++++++++++++ .../authorization/test_warmed_policy.py | 179 +++- .../test_reader_writer_regeneration.py | 213 ++++- .../management/test_key_delete_revocation.py | 233 +++++ .../spend/test_key_budget_lockout.py | 283 +++++- 5 files changed, 1766 insertions(+), 5 deletions(-) create mode 100644 tests/integration/authorization/test_jwt_key_mapping_lifecycle.py create mode 100644 tests/integration/management/test_key_delete_revocation.py diff --git a/tests/integration/authorization/test_jwt_key_mapping_lifecycle.py b/tests/integration/authorization/test_jwt_key_mapping_lifecycle.py new file mode 100644 index 00000000000..03ca94dd561 --- /dev/null +++ b/tests/integration/authorization/test_jwt_key_mapping_lifecycle.py @@ -0,0 +1,863 @@ +import json +import os +import time +import uuid +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path +from typing import Final, Literal + +import httpx +import jwt +import pytest +from cryptography.hazmat.primitives.asymmetric import rsa +from jwt.algorithms import RSAAlgorithm +from pydantic import BaseModel, JsonValue, TypeAdapter +from redis import Redis + +from tests.integration._support.client import ( + Gateway, + delete_key_if_present, + eventually, + gateway_from_environment, +) +from tests.integration._support.database import read_rows +from tests.integration._support.process import owned_proxy +from tests.integration._support.wire import Reply, Request, Wire, wire_server + +_AUDIENCE: Final = "litellm-integration" +_CLIENT_CLAIM: Final = "client_id" +_KEY_ID: Final = "integration-signing-key" +_UPSTREAM_KEY: Final = "synthetic-openai-key" +_CHANNEL: Final = "litellm_proxy.auth_cache_invalidation" +_CHAT_BODY: Final = TypeAdapter(dict[str, JsonValue]) +pytestmark: Final = pytest.mark.timeout(180) + + +class _Usage(BaseModel): + prompt_tokens: int + completion_tokens: int + total_tokens: int + + +class _AssistantMessage(BaseModel): + content: str + + +class _Choice(BaseModel): + message: _AssistantMessage + + +class _ChatResponse(BaseModel): + id: str + choices: tuple[_Choice, ...] + usage: _Usage + + +class _KeyResponse(BaseModel): + key: str + + +class _DeleteResponse(BaseModel): + deleted_keys: tuple[str, ...] + + +class _JWTMappingResponse(BaseModel): + id: str + jwt_issuer: str | None + jwt_claim_name: str + jwt_claim_value: str + is_active: bool + + +class _JWTMappingRow(BaseModel): + id: str + token: str + jwt_issuer: str + is_active: bool + + +class _SpendRow(BaseModel): + api_key: str + + +class _CountRow(BaseModel): + count: int + + +class _MappingIdRow(BaseModel): + id: str + + +class _ModelEntry(BaseModel): + id: str + + +class _ModelList(BaseModel): + data: tuple[_ModelEntry, ...] + + +class _UserMessage(BaseModel): + content: str + + +class _ChatRequest(BaseModel): + model: str + messages: tuple[_UserMessage, ...] + + +class _ProxyError(BaseModel): + message: str + type: str + param: str + code: str + + +class _ErrorResponse(BaseModel): + error: _ProxyError + + +@dataclass(frozen=True, slots=True) +class _JWTLifecycleRuntime: + first: Gateway + second: Gateway + upstream: Wire + model: str + private_key_a: rsa.RSAPrivateKey + private_key_b: rsa.RSAPrivateKey + issuer_a: str + issuer_b: str + + +def _proxy_config( + directory: Path, + model: str, + upstream_url: str, + issuers: tuple[dict[str, JsonValue], ...] = (), +) -> Path: + config: Final = directory / "jwt_key_mapping_config.yaml" + config.write_text( + json.dumps( + { + "model_list": ( + { + "model_name": model, + "litellm_params": { + "model": f"openai/{model}", + "api_base": f"{upstream_url}/v1", + "api_key": _UPSTREAM_KEY, + }, + }, + ), + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "store_model_in_db": True, + "disable_model_info_refresh": True, + "proxy_batch_write_at": 1, + "proxy_batch_polling_interval": 1, + "enable_jwt_auth": True, + "litellm_jwtauth": { + "user_id_jwt_field": "sub", + "virtual_key_claim_field": _CLIENT_CLAIM, + "unregistered_jwt_client_behavior": "reject", + **({"issuers": issuers} if issuers else {}), + }, + }, + "litellm_settings": { + "enable_redis_auth_cache": True, + "cache": True, + "cache_params": { + "type": "redis", + "host": "os.environ/REDIS_HOST", + "port": "os.environ/REDIS_PORT", + }, + }, + } + ) + ) + return config + + +def _issuer_config(issuer: str, jwks_url: str) -> dict[str, JsonValue]: + return { + "issuer": issuer, + "jwks_url": jwks_url, + "audience": _AUDIENCE, + "virtual_key_claim_field": _CLIENT_CLAIM, + "user_id_jwt_field": "sub", + } + + +def _jwks_body(private_key: rsa.RSAPrivateKey) -> bytes: + public_jwk: Final = _CHAT_BODY.validate_json(RSAAlgorithm.to_jwk(private_key.public_key())) + return json.dumps({"keys": [{**public_jwk, "kid": _KEY_ID, "use": "sig", "alg": "RS256"}]}).encode() + + +def _jwks_respond(private_key: rsa.RSAPrivateKey) -> Callable[[Request], Reply]: + body: Final = _jwks_body(private_key) + + def respond(request: Request) -> Reply: + assert request.method == "GET" + assert request.target == "/jwks" + return Reply(body=body) + + return respond + + +def _signed_jwt(private_key: rsa.RSAPrivateKey, client_id: str, issuer: str | None = None) -> str: + now: Final = int(time.time()) + claims: Final[dict[str, JsonValue]] = { + "sub": "integration-user-" + uuid.uuid4().hex, + "client_id": client_id, + "aud": _AUDIENCE, + "iat": now, + "exp": now + 300, + **({"iss": issuer} if issuer is not None else {}), + } + return jwt.encode(claims, private_key, algorithm="RS256", headers={"kid": _KEY_ID}) + + +def _chat_body(model: str, prompt: str) -> dict[str, JsonValue]: + return { + "model": model, + "messages": [{"role": "user", "content": prompt}], + } + + +def _wire_reply(identity: str) -> Reply: + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "integration-jwt-key-mapping", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "wire response"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 20, "completion_tokens": 20, "total_tokens": 40}, + } + ).encode() + ) + + +def _chat_respond(backend: str) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.method == "GET": + assert request.target == "/v1/models", request + assert request.headers["authorization"] == f"Bearer {_UPSTREAM_KEY}" + assert request.body == b"" + return Reply( + body=json.dumps( + { + "object": "list", + "data": [ + { + "id": backend, + "object": "model", + "created": 0, + "owned_by": "integration", + } + ], + } + ).encode() + ) + assert request.method == "POST", request + assert request.target == "/v1/chat/completions" + assert request.headers["authorization"] == f"Bearer {_UPSTREAM_KEY}" + body: Final = _CHAT_BODY.validate_python(json.loads(request.body)) + parsed: Final = _ChatRequest.model_validate_json(request.body) + assert len(parsed.messages) == 1 + assert body == _chat_body(backend, parsed.messages[0].content), body + return _wire_reply("chatcmpl-" + uuid.uuid4().hex) + + return respond + + +def _chat(gateway: Gateway, model: str, token: str, prompt: str) -> httpx.Response: + return gateway.request("POST", "/v1/chat/completions", _chat_body(model, prompt), key=token) + + +def _expected_chat_request(backend: str, prompt: str) -> tuple[str, str, dict[str, JsonValue]]: + body: Final = _chat_body(backend, prompt) + return "POST", "/v1/chat/completions", body + + +def _upstream_request_key( + request: tuple[str, str, dict[str, JsonValue] | None], +) -> tuple[str, str, str]: + return request[0], request[1], "" if request[2] is None else json.dumps(request[2], sort_keys=True) + + +def _assert_upstream_requests( + upstream: Wire, + expected: tuple[tuple[str, str, dict[str, JsonValue] | None], ...], +) -> None: + observed: Final = tuple( + sorted( + ( + ( + request.method, + request.target, + None if not request.body else _CHAT_BODY.validate_python(json.loads(request.body)), + ) + for request in upstream.drain() + ), + key=_upstream_request_key, + ) + ) + assert observed == tuple(sorted(expected, key=_upstream_request_key)), observed + + +def _assert_chat(gateway: Gateway, model: str, token: str, prompt: str, key_hash: str) -> _ChatResponse: + response: Final = _chat(gateway, model, token, prompt) + assert response.status_code == 200, response.text + payload: Final = _ChatResponse.model_validate_json(response.content) + assert payload.choices[0].message.content == "wire response", response.text + assert payload.usage == _Usage(prompt_tokens=20, completion_tokens=20, total_tokens=40), response.text + spend: Final = TypeAdapter(tuple[_SpendRow, ...]).validate_python( + eventually( + lambda: read_rows( + 'SELECT api_key FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (payload.id,), + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + ) + assert spend == (_SpendRow(api_key=key_hash),) + return payload + + +def _model_is_available(gateway: Gateway, model: str) -> bool: + response: Final = gateway.request("GET", "/v1/models") + if response.status_code != 200: + return False + payload: Final = _ModelList.model_validate_json(response.content) + return any(entry.id == model for entry in payload.data) + + +def _create_key(gateway: Gateway, model: str) -> str: + response: Final = gateway.request("POST", "/key/generate", {"models": [model]}) + assert response.status_code == 200, response.text + return _KeyResponse.model_validate_json(response.content).key + + +def _create_mapping( + gateway: Gateway, + key: str, + client_id: str, + issuer: str | None = None, + spelling: Literal["key", "token"] = "key", +) -> _JWTMappingResponse: + body: Final = { + "jwt_claim_name": _CLIENT_CLAIM, + "jwt_claim_value": client_id, + spelling: key if spelling == "key" else sha256(key.encode()).hexdigest(), + **({"jwt_issuer": issuer} if issuer is not None else {}), + } + response: Final = gateway.request("POST", "/jwt/key/mapping/new", body) + assert response.status_code == 200, response.text + return _JWTMappingResponse.model_validate_json(response.content) + + +def _delete_mapping_if_present(gateway: Gateway, identity: str) -> None: + rows: Final = TypeAdapter(tuple[_MappingIdRow, ...]).validate_python( + read_rows('SELECT id FROM "LiteLLM_JWTKeyMapping" WHERE id=%s', (identity,)) + ) + if not rows: + return + response: Final = gateway.request("POST", "/jwt/key/mapping/delete", {"id": identity}) + assert response.status_code == 200, response.text + + +def _assert_unmapped(response: httpx.Response, client_id: str) -> None: + assert response.status_code == 403, response.text + error: Final = _ErrorResponse.model_validate_json(response.content) + expected: Final = f"JWT Key Mapping: No registered mapping for {_CLIENT_CLAIM}='{client_id}'. Access denied." + assert error.error == _ProxyError(message=expected, type="auth_error", param="None", code="403"), response.text + + +def _mapping_info(gateway: Gateway, identity: str) -> _JWTMappingResponse: + response: Final = gateway.request("GET", "/jwt/key/mapping/info", params={"id": identity}) + assert response.status_code == 200, response.text + return _JWTMappingResponse.model_validate_json(response.content) + + +def _mapping_row(gateway: Gateway, identity: str) -> _JWTMappingRow: + rows: Final = TypeAdapter(tuple[_JWTMappingRow, ...]).validate_python( + read_rows( + 'SELECT id, token, jwt_issuer, is_active FROM "LiteLLM_JWTKeyMapping" WHERE id=%s', + (identity,), + ) + ) + assert len(rows) == 1 + return rows[0] + + +def _spend_count(gateway: Gateway, key_hash: str) -> int: + rows: Final = TypeAdapter(tuple[_CountRow, ...]).validate_python( + read_rows('SELECT count(*) AS count FROM "LiteLLM_SpendLogs" WHERE api_key=%s', (key_hash,)) + ) + assert len(rows) == 1 + return rows[0].count + + +@pytest.fixture(scope="module") +def jwt_lifecycle(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_JWTLifecycleRuntime]: + directory: Final = tmp_path_factory.mktemp("jwt-key-mapping-lifecycle") + model: Final = "jwt-key-mapping-" + uuid.uuid4().hex + issuer_a: Final = "https://idp-a.integration.example" + issuer_b: Final = "https://idp-b.integration.example" + private_key_a: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + private_key_b: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + + with ( + gateway_from_environment() as base, + wire_server(_chat_respond(model)) as upstream, + wire_server(_jwks_respond(private_key_a)) as jwks_a, + wire_server(_jwks_respond(private_key_b)) as jwks_b, + Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache, + ): + issuers: Final = ( + _issuer_config(issuer_a, f"{jwks_a.url}/jwks"), + _issuer_config(issuer_b, f"{jwks_b.url}/jwks"), + ) + config: Final = _proxy_config(directory, model, upstream.url, issuers) + baseline: Final = cache.pubsub_numsub(_CHANNEL)[0][1] + with ( + owned_proxy(base, directory / "worker-one", {}, config=config) as first, + owned_proxy(base, directory / "worker-two", {}, config=config) as second, + ): + subscribers: Final = eventually( + lambda: cache.pubsub_numsub(_CHANNEL)[0][1], + lambda count: count >= baseline + 2, + seconds=20, + ) + assert subscribers >= baseline + 2 + eventually(lambda: _model_is_available(first, model), bool, seconds=30) + eventually(lambda: _model_is_available(second, model), bool, seconds=30) + _assert_upstream_requests(upstream, ()) + yield _JWTLifecycleRuntime( + first=first, + second=second, + upstream=upstream, + model=model, + private_key_a=private_key_a, + private_key_b=private_key_b, + issuer_a=issuer_a, + issuer_b=issuer_b, + ) + + +def test_updating_a_warmed_mapping_deactivates_and_repoints_it_on_both_workers( + jwt_lifecycle: _JWTLifecycleRuntime, +) -> None: + first: Final = jwt_lifecycle.first + second: Final = jwt_lifecycle.second + upstream: Final = jwt_lifecycle.upstream + model: Final = jwt_lifecycle.model + backend: Final = model + client_id: Final = "integration-client-" + uuid.uuid4().hex + issuer: Final = jwt_lifecycle.issuer_a + prompts: Final = ( + "mapping warm primary " + uuid.uuid4().hex, + "mapping warm peer " + uuid.uuid4().hex, + "mapping inactive primary " + uuid.uuid4().hex, + "mapping inactive peer " + uuid.uuid4().hex, + "mapping repointed primary " + uuid.uuid4().hex, + "mapping repointed peer " + uuid.uuid4().hex, + "mapping restored primary " + uuid.uuid4().hex, + "mapping restored peer " + uuid.uuid4().hex, + ) + token: Final = _signed_jwt(jwt_lifecycle.private_key_a, client_id, issuer) + upstream.drain() + with first.scenario() as scenario: + old_key: Final = _create_key(first, model) + new_key: Final = _create_key(first, model) + scenario.cleanups.callback(delete_key_if_present, first, old_key) + scenario.cleanups.callback(delete_key_if_present, first, new_key) + old_hash: Final = sha256(old_key.encode()).hexdigest() + new_hash: Final = sha256(new_key.encode()).hexdigest() + mapping: Final = _create_mapping(first, old_key, client_id, issuer) + scenario.cleanups.callback(_delete_mapping_if_present, first, mapping.id) + assert mapping.jwt_issuer == issuer + assert _mapping_row(first, mapping.id).jwt_issuer == issuer + + first_response: Final = _assert_chat(first, model, token, prompts[0], old_hash) + second_response: Final = _assert_chat(second, model, token, prompts[1], old_hash) + assert first_response.id != second_response.id + + deactivated_response: Final = first.request( + "POST", "/jwt/key/mapping/update", {"id": mapping.id, "is_active": False} + ) + assert deactivated_response.status_code == 200, deactivated_response.text + deactivated: Final = _JWTMappingResponse.model_validate_json(deactivated_response.content) + assert deactivated.is_active is False, deactivated_response.text + assert _mapping_info(first, mapping.id).is_active is False + inactive_row: Final = _mapping_row(first, mapping.id) + assert inactive_row.token == old_hash and inactive_row.is_active is False + old_spend_count: Final = _spend_count(first, old_hash) + + refused_first: Final = _chat(first, model, token, prompts[2]) + _assert_unmapped(refused_first, client_id) + refused_second: Final = eventually( + lambda: _chat(second, model, token, prompts[3]), + lambda response: response.status_code == 403, + seconds=3, + ) + _assert_unmapped(refused_second, client_id) + assert _spend_count(first, old_hash) == old_spend_count + + reactivated_response: Final = first.request( + "POST", + "/jwt/key/mapping/update", + {"id": mapping.id, "is_active": True, "token": new_hash}, + ) + assert reactivated_response.status_code == 200, reactivated_response.text + reactivated: Final = _JWTMappingResponse.model_validate_json(reactivated_response.content) + assert reactivated.is_active is True, reactivated_response.text + assert _mapping_info(first, mapping.id).is_active is True + active_row: Final = _mapping_row(first, mapping.id) + assert active_row.token == new_hash and active_row.is_active is True + _assert_chat(first, model, token, prompts[4], new_hash) + _assert_chat(second, model, token, prompts[5], new_hash) + + restored_response: Final = first.request( + "POST", + "/jwt/key/mapping/update", + {"id": mapping.id, "key": old_key}, + ) + assert restored_response.status_code == 200, restored_response.text + restored: Final = _JWTMappingResponse.model_validate_json(restored_response.content) + assert restored.is_active is True, restored_response.text + assert _mapping_info(first, mapping.id).is_active is True + restored_row: Final = _mapping_row(first, mapping.id) + assert restored_row.token == old_hash and restored_row.is_active is True + _assert_chat(first, model, token, prompts[6], old_hash) + _assert_chat(second, model, token, prompts[7], old_hash) + served_prompts: Final = prompts[0:2] + prompts[4:8] + _assert_upstream_requests(upstream, tuple(_expected_chat_request(backend, prompt) for prompt in served_prompts)) + + +def test_reactivating_a_global_mapping_restores_issuer_jwt_callers_on_both_workers( + jwt_lifecycle: _JWTLifecycleRuntime, +) -> None: + pytest.skip( + "BUG: reactivating a global JWT key mapping keeps refusing issuer-scoped JWTs with 403 No registered mapping " + "because the miss was cached under the issuer key and update evicts only the global key" + ) + first: Final = jwt_lifecycle.first + second: Final = jwt_lifecycle.second + upstream: Final = jwt_lifecycle.upstream + model: Final = jwt_lifecycle.model + client_id: Final = "integration-client-" + uuid.uuid4().hex + prompts: Final = tuple(f"global mapping reactivation {index} {uuid.uuid4().hex}" for index in range(6)) + token: Final = _signed_jwt(jwt_lifecycle.private_key_a, client_id, jwt_lifecycle.issuer_a) + upstream.drain() + + with first.scenario() as scenario: + key: Final = _create_key(first, model) + scenario.cleanups.callback(delete_key_if_present, first, key) + key_hash: Final = sha256(key.encode()).hexdigest() + mapping: Final = _create_mapping(first, key, client_id) + scenario.cleanups.callback(_delete_mapping_if_present, first, mapping.id) + assert mapping.jwt_issuer is None + assert _mapping_info(first, mapping.id).jwt_issuer is None + assert _mapping_row(first, mapping.id) == _JWTMappingRow( + id=mapping.id, + token=key_hash, + jwt_issuer="", + is_active=True, + ) + _assert_chat(first, model, token, prompts[0], key_hash) + _assert_chat(second, model, token, prompts[1], key_hash) + + deactivated_response: Final = first.request( + "POST", + "/jwt/key/mapping/update", + {"id": mapping.id, "is_active": False}, + ) + assert deactivated_response.status_code == 200, deactivated_response.text + assert _mapping_info(first, mapping.id).is_active is False + assert _mapping_row(first, mapping.id) == _JWTMappingRow( + id=mapping.id, + token=key_hash, + jwt_issuer="", + is_active=False, + ) + _assert_unmapped(_chat(first, model, token, prompts[2]), client_id) + refused_second: Final = eventually( + lambda: _chat(second, model, token, prompts[3]), + lambda response: response.status_code == 403, + seconds=3, + ) + _assert_unmapped(refused_second, client_id) + assert _spend_count(first, key_hash) == 2 + + reactivated_response: Final = first.request( + "POST", + "/jwt/key/mapping/update", + {"id": mapping.id, "is_active": True}, + ) + assert reactivated_response.status_code == 200, reactivated_response.text + reactivated: Final = _JWTMappingResponse.model_validate_json(reactivated_response.content) + assert reactivated.is_active is True, reactivated_response.text + assert reactivated.jwt_issuer is None, reactivated_response.text + assert _mapping_info(first, mapping.id).jwt_issuer is None + assert _mapping_row(first, mapping.id) == _JWTMappingRow( + id=mapping.id, + token=key_hash, + jwt_issuer="", + is_active=True, + ) + _assert_chat(first, model, token, prompts[4], key_hash) + _assert_chat(second, model, token, prompts[5], key_hash) + assert _spend_count(first, key_hash) == 4 + _assert_upstream_requests( + upstream, + tuple(_expected_chat_request(model, prompt) for prompt in prompts[0:2] + prompts[4:6]), + ) + + +@pytest.mark.parametrize("scope", ("global", "issuer")) +@pytest.mark.parametrize("spelling", ("body", "path", "plaintext_path")) +def test_regenerating_a_jwt_mapped_key_moves_jwt_callers_to_the_new_hash( + jwt_lifecycle: _JWTLifecycleRuntime, spelling: str, scope: str +) -> None: + first: Final = jwt_lifecycle.first + second: Final = jwt_lifecycle.second + upstream: Final = jwt_lifecycle.upstream + model: Final = jwt_lifecycle.model + backend: Final = model + client_id: Final = "integration-client-" + uuid.uuid4().hex + prompts: Final = ( + "regenerate warm primary " + uuid.uuid4().hex, + "regenerate warm peer " + uuid.uuid4().hex, + "regenerated primary " + uuid.uuid4().hex, + "regenerated peer " + uuid.uuid4().hex, + ) + issuer: Final = jwt_lifecycle.issuer_a if scope == "issuer" else None + token: Final = _signed_jwt(jwt_lifecycle.private_key_a, client_id, jwt_lifecycle.issuer_a) + upstream.drain() + with first.scenario() as scenario: + old_key: Final = _create_key(first, model) + scenario.cleanups.callback(delete_key_if_present, first, old_key) + old_hash: Final = sha256(old_key.encode()).hexdigest() + mapping: Final = _create_mapping(first, old_key, client_id, issuer) + scenario.cleanups.callback(_delete_mapping_if_present, first, mapping.id) + expected_issuer: Final = issuer if issuer is not None else "" + assert mapping.jwt_issuer == issuer + assert _mapping_info(first, mapping.id).jwt_issuer == issuer + assert _mapping_row(first, mapping.id).jwt_issuer == expected_issuer + _assert_chat(first, model, token, prompts[0], old_hash) + _assert_chat(second, model, token, prompts[1], old_hash) + + path_key: Final = old_hash if spelling == "path" else old_key + regenerated_response: Final = ( + first.request("POST", "/key/regenerate", {"key": old_key}) + if spelling == "body" + else first.request("POST", f"/key/{path_key}/regenerate", {}) + ) + assert regenerated_response.status_code == 200, regenerated_response.text + new_key: Final = _KeyResponse.model_validate_json(regenerated_response.content).key + scenario.cleanups.callback(delete_key_if_present, first, new_key) + new_hash: Final = sha256(new_key.encode()).hexdigest() + mapped_row: Final = _mapping_row(first, mapping.id) + assert mapped_row.token == new_hash and mapped_row.is_active is True + assert mapped_row.jwt_issuer == expected_issuer + assert _mapping_info(first, mapping.id).jwt_issuer == issuer + _assert_chat(first, model, token, prompts[2], new_hash) + _assert_chat(second, model, token, prompts[3], new_hash) + _assert_upstream_requests(upstream, tuple(_expected_chat_request(backend, prompt) for prompt in prompts)) + + +@pytest.mark.parametrize("scope", ("global", "issuer")) +def test_deleting_a_jwt_mapped_key_stops_its_mapping_on_both_workers( + jwt_lifecycle: _JWTLifecycleRuntime, scope: str +) -> None: + first: Final = jwt_lifecycle.first + second: Final = jwt_lifecycle.second + upstream: Final = jwt_lifecycle.upstream + model: Final = jwt_lifecycle.model + backend: Final = model + client_id: Final = "integration-client-" + uuid.uuid4().hex + prompts: Final = ( + "delete mapped warm primary " + uuid.uuid4().hex, + "delete mapped warm peer " + uuid.uuid4().hex, + "delete mapped primary " + uuid.uuid4().hex, + "delete mapped peer " + uuid.uuid4().hex, + ) + issuer: Final = jwt_lifecycle.issuer_a if scope == "issuer" else None + token: Final = _signed_jwt(jwt_lifecycle.private_key_a, client_id, jwt_lifecycle.issuer_a) + upstream.drain() + with first.scenario() as scenario: + key: Final = _create_key(first, model) + scenario.cleanups.callback(delete_key_if_present, first, key) + key_hash: Final = sha256(key.encode()).hexdigest() + mapping: Final = _create_mapping(first, key, client_id, issuer) + scenario.cleanups.callback(_delete_mapping_if_present, first, mapping.id) + expected_issuer: Final = issuer if issuer is not None else "" + assert mapping.jwt_issuer == issuer + assert _mapping_info(first, mapping.id).jwt_issuer == issuer + assert _mapping_row(first, mapping.id).jwt_issuer == expected_issuer + warm_first: Final = _assert_chat(first, model, token, prompts[0], key_hash) + warm_second: Final = _assert_chat(second, model, token, prompts[1], key_hash) + assert warm_first.id != warm_second.id + warm_spend_count: Final = _spend_count(first, key_hash) + assert warm_spend_count == 2 + + deletion: Final = first.request("POST", "/key/delete", {"keys": [key]}) + assert deletion.status_code == 200, deletion.text + assert _DeleteResponse.model_validate_json(deletion.content).deleted_keys == (key,), deletion.text + assert read_rows('SELECT id FROM "LiteLLM_JWTKeyMapping" WHERE id=%s', (mapping.id,)) == [] + second_refusal: Final = eventually( + lambda: _chat(second, model, token, prompts[2]), + lambda response: response.status_code == 403, + seconds=3, + ) + _assert_unmapped(second_refusal, client_id) + first_refusal: Final = _chat(first, model, token, prompts[3]) + _assert_unmapped(first_refusal, client_id) + assert _spend_count(first, key_hash) == warm_spend_count + _assert_upstream_requests(upstream, tuple(_expected_chat_request(backend, prompt) for prompt in prompts[:2])) + + +def test_issuer_scoped_mappings_with_the_same_claim_resolve_per_issuer( + jwt_lifecycle: _JWTLifecycleRuntime, +) -> None: + candidate: Final = jwt_lifecycle.first + upstream: Final = jwt_lifecycle.upstream + model: Final = jwt_lifecycle.model + backend: Final = model + client_id: Final = "integration-client-" + uuid.uuid4().hex + issuer_a: Final = jwt_lifecycle.issuer_a + issuer_b: Final = jwt_lifecycle.issuer_b + prompts: Final = tuple(f"issuer mapping {index} {uuid.uuid4().hex}" for index in range(4)) + upstream.drain() + + with candidate.scenario() as scenario: + key_a: Final = _create_key(candidate, model) + key_b: Final = _create_key(candidate, model) + scenario.cleanups.callback(delete_key_if_present, candidate, key_a) + scenario.cleanups.callback(delete_key_if_present, candidate, key_b) + hash_a: Final = sha256(key_a.encode()).hexdigest() + hash_b: Final = sha256(key_b.encode()).hexdigest() + mapping_a: Final = _create_mapping(candidate, key_a, client_id, issuer_a) + mapping_b: Final = _create_mapping(candidate, key_b, client_id, issuer_b, spelling="token") + scenario.cleanups.callback(_delete_mapping_if_present, candidate, mapping_a.id) + scenario.cleanups.callback(_delete_mapping_if_present, candidate, mapping_b.id) + + _assert_chat( + candidate, + model, + _signed_jwt(jwt_lifecycle.private_key_a, client_id, issuer_a), + prompts[0], + hash_a, + ) + _assert_chat( + candidate, + model, + _signed_jwt(jwt_lifecycle.private_key_b, client_id, issuer_b), + prompts[1], + hash_b, + ) + _assert_chat( + candidate, + model, + _signed_jwt(jwt_lifecycle.private_key_a, client_id, issuer_a), + prompts[2], + hash_a, + ) + _assert_chat( + candidate, + model, + _signed_jwt(jwt_lifecycle.private_key_b, client_id, issuer_b), + prompts[3], + hash_b, + ) + rows: Final = TypeAdapter(tuple[_JWTMappingRow, ...]).validate_python( + read_rows( + 'SELECT id, token, jwt_issuer, is_active FROM "LiteLLM_JWTKeyMapping" ' + "WHERE jwt_claim_name=%s AND jwt_claim_value=%s ORDER BY jwt_issuer", + (_CLIENT_CLAIM, client_id), + ) + ) + assert rows == ( + _JWTMappingRow(id=mapping_a.id, token=hash_a, jwt_issuer=issuer_a, is_active=True), + _JWTMappingRow(id=mapping_b.id, token=hash_b, jwt_issuer=issuer_b, is_active=True), + ) + _assert_upstream_requests( + upstream, + tuple(_expected_chat_request(backend, prompt) for prompt in prompts), + ) + + +def test_repointing_a_warmed_mapping_to_a_new_issuer_moves_it_on_both_workers( + jwt_lifecycle: _JWTLifecycleRuntime, +) -> None: + first: Final = jwt_lifecycle.first + second: Final = jwt_lifecycle.second + upstream: Final = jwt_lifecycle.upstream + model: Final = jwt_lifecycle.model + issuer_a: Final = jwt_lifecycle.issuer_a + issuer_b: Final = jwt_lifecycle.issuer_b + client_id: Final = "integration-client-" + uuid.uuid4().hex + prompts: Final = tuple(f"mapping repoint {index} {uuid.uuid4().hex}" for index in range(4)) + upstream.drain() + + with first.scenario() as scenario: + key: Final = _create_key(first, model) + scenario.cleanups.callback(delete_key_if_present, first, key) + key_hash: Final = sha256(key.encode()).hexdigest() + mapping: Final = _create_mapping(first, key, client_id, issuer_a) + scenario.cleanups.callback(_delete_mapping_if_present, first, mapping.id) + assert _mapping_row(first, mapping.id) == _JWTMappingRow( + id=mapping.id, + token=key_hash, + jwt_issuer=issuer_a, + is_active=True, + ) + + new_identity_token: Final = _signed_jwt(jwt_lifecycle.private_key_b, client_id, issuer_b) + _assert_unmapped(_chat(first, model, new_identity_token, prompts[0]), client_id) + _assert_unmapped(_chat(second, model, new_identity_token, prompts[1]), client_id) + + old_token: Final = _signed_jwt(jwt_lifecycle.private_key_a, client_id, issuer_a) + _assert_chat(first, model, old_token, prompts[2], key_hash) + _assert_chat(second, model, old_token, prompts[3], key_hash) + assert _spend_count(first, key_hash) == 2 + + update: Final = first.request( + "POST", + "/jwt/key/mapping/update", + {"id": mapping.id, "jwt_issuer": issuer_b}, + ) + assert update.status_code == 200, update.text + updated: Final = _JWTMappingResponse.model_validate_json(update.content) + assert updated.jwt_claim_value == client_id, update.text + assert updated.jwt_issuer == issuer_b, update.text + assert _mapping_info(first, mapping.id) == updated + assert _mapping_row(first, mapping.id) == _JWTMappingRow( + id=mapping.id, + token=key_hash, + jwt_issuer=issuer_b, + is_active=True, + ) + + _assert_chat(first, model, new_identity_token, prompts[0], key_hash) + _assert_chat(second, model, new_identity_token, prompts[1], key_hash) + assert _spend_count(first, key_hash) == 4 + _assert_unmapped(_chat(first, model, old_token, prompts[2]), client_id) + _assert_unmapped(_chat(second, model, old_token, prompts[3]), client_id) + assert _spend_count(first, key_hash) == 4 + _assert_upstream_requests( + upstream, + tuple(_expected_chat_request(model, prompt) for prompt in prompts), + ) diff --git a/tests/integration/authorization/test_warmed_policy.py b/tests/integration/authorization/test_warmed_policy.py index f114e78507a..b4ffc10c3c4 100644 --- a/tests/integration/authorization/test_warmed_policy.py +++ b/tests/integration/authorization/test_warmed_policy.py @@ -1,3 +1,4 @@ +import json import os import uuid from contextlib import ExitStack @@ -9,11 +10,187 @@ import psycopg import pytest from hypothesis import strategies as st from hypothesis.stateful import RuleBasedStateMachine, invariant, rule, run_state_machine_as_test -from pydantic import BaseModel, ConfigDict, JsonValue +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter from tests.integration._support.client import Gateway, eventually, object_value, string_value, team_admin_permissions from tests.integration._support.database import read_rows from tests.integration._support.generation import LIFECYCLE_SETTINGS, bounded_http_requests +from tests.integration._support.wire import Reply, Request, wire_server + +_CHAT_REQUEST: Final = TypeAdapter(dict[str, JsonValue]) +_EXPECTED_CHAT: Final = ("POST", "/v1/chat/completions") + + +class _Usage(BaseModel): + prompt_tokens: int + completion_tokens: int + total_tokens: int + + +class _AssistantMessage(BaseModel): + content: str + + +class _Choice(BaseModel): + message: _AssistantMessage + + +class _ChatResponse(BaseModel): + id: str + choices: tuple[_Choice, ...] + usage: _Usage + + +class _ModelEntry(BaseModel): + id: str + + +class _ModelList(BaseModel): + data: tuple[_ModelEntry, ...] + + +class _KeyInfo(BaseModel): + blocked: bool + + +class _KeyInfoResponse(BaseModel): + info: _KeyInfo + + +class _BlockedRow(BaseModel): + blocked: bool + + +class _AuthError(BaseModel): + type: str + + +class _ErrorResponse(BaseModel): + error: _AuthError + + +def _chat_body(model: str, prompt: str) -> dict[str, JsonValue]: + return { + "model": model, + "messages": [{"role": "user", "content": prompt}], + } + + +def _wire_completion(identity: str) -> Reply: + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "integration-keys-auth-block", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "wire response"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 20, "completion_tokens": 20, "total_tokens": 40}, + } + ).encode() + ) + + +def _wire_chat(gateway: Gateway, model: str, key: str, prompt: str) -> httpx.Response: + return gateway.request("POST", "/v1/chat/completions", _chat_body(model, prompt), key=key) + + +def _assert_wire_success(response: httpx.Response) -> _ChatResponse: + assert response.status_code == 200, response.text + payload: Final = _ChatResponse.model_validate_json(response.content) + assert payload.choices[0].message.content == "wire response", response.text + assert payload.usage == _Usage(prompt_tokens=20, completion_tokens=20, total_tokens=40), response.text + return payload + + +def _model_is_available(gateway: Gateway, model: str) -> bool: + response: Final = gateway.request("GET", "/v1/models") + if response.status_code != 200: + return False + models: Final = _ModelList.model_validate_json(response.content) + return model in tuple(entry.id for entry in models.data) + + +@pytest.mark.parametrize("hashed", (False, True), ids=("plaintext", "hashed")) +def test_key_block_and_unblock_reach_both_warmed_workers(gateway: Gateway, peer: Gateway, hashed: bool) -> None: + backend: Final = "integration-keys-auth-block" + prompts: Final = ( + "block warm primary " + uuid.uuid4().hex, + "block warm peer " + uuid.uuid4().hex, + "block rejected primary " + uuid.uuid4().hex, + "block rejected peer " + uuid.uuid4().hex, + "unblock primary " + uuid.uuid4().hex, + "unblock peer " + uuid.uuid4().hex, + ) + + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/v1/chat/completions" + assert request.headers["authorization"] == "Bearer synthetic-openai-key" + body: Final = _CHAT_REQUEST.validate_json(request.body) + assert body in tuple(_chat_body(backend, prompt) for prompt in prompts), body + return _wire_completion("chatcmpl-" + uuid.uuid4().hex) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"openai/{backend}", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key" + ) + eventually(lambda: _model_is_available(peer, model), bool, seconds=30) + key: Final = scenario.key(models=[model]) + key_hash: Final = sha256(key.encode()).hexdigest() + endpoint_key: Final = key_hash if hashed else key + warmed: Final = tuple( + _assert_wire_success(_wire_chat(worker, model, key, prompt)) + for worker, prompt in zip((gateway, peer), prompts[:2], strict=True) + ) + assert len(warmed) == 2 + assert tuple((request.method, request.target) for request in wire.drain()) == (_EXPECTED_CHAT,) * 2 + + blocked: Final = gateway.request("POST", "/key/block", {"key": endpoint_key}) + assert blocked.status_code == 200, blocked.text + assert _KeyInfo.model_validate_json(blocked.content).blocked is True, blocked.text + primary_refusal: Final = _wire_chat(gateway, model, key, prompts[2]) + assert primary_refusal.status_code == 401, primary_refusal.text + assert _ErrorResponse.model_validate_json(primary_refusal.content).error.type == "auth_error", ( + primary_refusal.text + ) + peer_refusal: Final = eventually( + lambda: _wire_chat(peer, model, key, prompts[3]), + lambda response: response.status_code == 401, + seconds=3, + ) + assert _ErrorResponse.model_validate_json(peer_refusal.content).error.type == "auth_error", peer_refusal.text + assert wire.drain() == (), "blocked requests reached the upstream" + blocked_info: Final = gateway.request("GET", "/key/info", params={"key": key}) + assert blocked_info.status_code == 200, blocked_info.text + assert _KeyInfoResponse.model_validate_json(blocked_info.content).info.blocked is True, blocked_info.text + blocked_rows: Final = TypeAdapter(tuple[_BlockedRow, ...]).validate_python( + read_rows('SELECT blocked FROM "LiteLLM_VerificationToken" WHERE token=%s', (key_hash,)) + ) + assert blocked_rows == (_BlockedRow(blocked=True),) + + unblocked: Final = gateway.request("POST", "/key/unblock", {"key": endpoint_key}) + assert unblocked.status_code == 200, unblocked.text + assert _KeyInfo.model_validate_json(unblocked.content).blocked is False, unblocked.text + served: Final = tuple( + _assert_wire_success(_wire_chat(worker, model, key, prompt)) + for worker, prompt in zip((gateway, peer), prompts[4:], strict=True) + ) + assert len(served) == 2 + assert tuple((request.method, request.target) for request in wire.drain()) == (_EXPECTED_CHAT,) * 2 + unblocked_info: Final = gateway.request("GET", "/key/info", params={"key": key}) + assert unblocked_info.status_code == 200, unblocked_info.text + assert _KeyInfoResponse.model_validate_json(unblocked_info.content).info.blocked is False, unblocked_info.text + unblocked_rows: Final = TypeAdapter(tuple[_BlockedRow, ...]).validate_python( + read_rows('SELECT blocked FROM "LiteLLM_VerificationToken" WHERE token=%s', (key_hash,)) + ) + assert unblocked_rows == (_BlockedRow(blocked=False),) SCIM_CORE_USER_SCHEMA: Final = "urn:ietf:params:scim:schemas:core:2.0:User" diff --git a/tests/integration/database/test_reader_writer_regeneration.py b/tests/integration/database/test_reader_writer_regeneration.py index 4161d0b04a5..62c152167ab 100644 --- a/tests/integration/database/test_reader_writer_regeneration.py +++ b/tests/integration/database/test_reader_writer_regeneration.py @@ -1,18 +1,21 @@ +import json import os import uuid +from concurrent.futures import ThreadPoolExecutor from hashlib import sha256 from pathlib import Path -from concurrent.futures import ThreadPoolExecutor from typing import Final from urllib.parse import urlsplit, urlunsplit +import httpx import psycopg import pytest -from psycopg import sql - from integration._support.client import Gateway, delete_key_if_present, eventually, string_value from integration._support.database import read_rows from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from psycopg import sql +from pydantic import BaseModel, JsonValue, TypeAdapter @pytest.mark.covers("other.database.regeneration.writer_updates_dependent_grants") @@ -129,3 +132,207 @@ def test_key_regeneration_uses_writer_with_a_real_readonly_reader(gateway: Gatew admin.execute(sql.SQL("DROP OWNED BY {}").format(sql.Identifier(role))) admin.execute(sql.SQL("DROP ROLE {}").format(sql.Identifier(role))) assert read_rows("SELECT rolname FROM pg_roles WHERE rolname=%s", (role,)) == [] + + +class _Usage(BaseModel): + prompt_tokens: int + completion_tokens: int + total_tokens: int + + +class _AssistantMessage(BaseModel): + content: str + + +class _Choice(BaseModel): + message: _AssistantMessage + + +class _ChatResponse(BaseModel): + id: str + choices: tuple[_Choice, ...] + usage: _Usage + + +class _GenerateKeyResponse(BaseModel): + key: str + + +class _Error(BaseModel): + type: str + + +class _ErrorResponse(BaseModel): + error: _Error + + +class _SpendLog(BaseModel): + api_key: str + spend: float + + +class _KeySpend(BaseModel): + spend: float + + +class _DeprecatedKey(BaseModel): + active_token_id: str + grace_seconds: float + + +_CHAT_BODY: Final = TypeAdapter(dict[str, JsonValue]) +_EXPECTED_CHAT: Final = ("POST", "/v1/chat/completions") + + +def _chat_body(model: str, prompt: str) -> dict[str, JsonValue]: + return { + "model": model, + "messages": [{"role": "user", "content": prompt}], + } + + +def _wire_reply(identity: str) -> Reply: + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "integration-keys-auth-grace", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "wire response"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 20, "completion_tokens": 20, "total_tokens": 40}, + } + ).encode() + ) + + +def _chat(gateway: Gateway, model: str, key: str, prompt: str) -> httpx.Response: + return gateway.request("POST", "/v1/chat/completions", _chat_body(model, prompt), key=key) + + +def _assert_success(response: httpx.Response) -> _ChatResponse: + assert response.status_code == 200, response.text + payload: Final = _ChatResponse.model_validate_json(response.content) + assert payload.choices[0].message.content == "wire response", response.text + assert payload.usage == _Usage(prompt_tokens=20, completion_tokens=20, total_tokens=40), response.text + return payload + + +@pytest.mark.parametrize("spelling", ("body", "path", "plaintext_path")) +def test_regenerate_with_a_grace_period_keeps_the_old_key_serving_billed_to_the_new_row( + gateway: Gateway, spelling: str +) -> None: + assert os.environ.get("LITELLM_KEY_ROTATION_GRACE_PERIOD") is None + backend: Final = "integration-keys-auth-grace" + prompts: Final = ( + "grace old key " + uuid.uuid4().hex, + "grace new key " + uuid.uuid4().hex, + "revoked second key " + uuid.uuid4().hex, + "final generated key " + uuid.uuid4().hex, + ) + + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/v1/chat/completions" + assert request.headers["authorization"] == "Bearer synthetic-openai-key" + body: Final = _CHAT_BODY.validate_json(request.body) + assert body in tuple(_chat_body(backend, prompt) for prompt in prompts), body + return _wire_reply("chatcmpl-" + uuid.uuid4().hex) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"openai/{backend}", + api_base=f"{wire.url}/v1", + api_key="synthetic-openai-key", + input_cost_per_token=0.001, + output_cost_per_token=0.002, + ) + created: Final = gateway.request("POST", "/key/generate", {"models": [model]}) + assert created.status_code == 200, created.text + key_one: Final = _GenerateKeyResponse.model_validate_json(created.content).key + scenario.cleanups.callback(delete_key_if_present, gateway, key_one) + hash_one: Final = sha256(key_one.encode()).hexdigest() + + path_key_one: Final = hash_one if spelling == "path" else key_one + generated_two: Final = ( + gateway.request("POST", "/key/regenerate", {"key": key_one, "grace_period": "1h"}) + if spelling == "body" + else gateway.request("POST", f"/key/{path_key_one}/regenerate", {"grace_period": "1h"}) + ) + assert generated_two.status_code == 200, generated_two.text + key_two: Final = _GenerateKeyResponse.model_validate_json(generated_two.content).key + scenario.cleanups.callback(delete_key_if_present, gateway, key_two) + hash_two: Final = sha256(key_two.encode()).hexdigest() + old_response: Final = _assert_success(_chat(gateway, model, key_one, prompts[0])) + new_response: Final = _assert_success(_chat(gateway, model, key_two, prompts[1])) + assert tuple((request.method, request.target) for request in wire.drain()) == (_EXPECTED_CHAT,) * 2 + + old_spend: Final = TypeAdapter(tuple[_SpendLog, ...]).validate_python( + eventually( + lambda: read_rows( + 'SELECT api_key, spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (old_response.id,), + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + ) + new_spend: Final = TypeAdapter(tuple[_SpendLog, ...]).validate_python( + eventually( + lambda: read_rows( + 'SELECT api_key, spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (new_response.id,), + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + ) + assert old_spend[0].api_key == hash_two + assert old_spend[0].spend > 0 + assert new_spend[0].api_key == hash_two + assert new_spend[0].spend > 0 + key_spend: Final = TypeAdapter(tuple[_KeySpend, ...]).validate_python( + eventually( + lambda: read_rows( + 'SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', + (hash_two,), + ), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) > 0, + seconds=70, + ) + ) + assert key_spend[0].spend > 0 + deprecated: Final = TypeAdapter(tuple[_DeprecatedKey, ...]).validate_python( + read_rows( + "SELECT active_token_id, EXTRACT(EPOCH FROM revoke_at - now())::float8 AS grace_seconds " + 'FROM "LiteLLM_DeprecatedVerificationToken" WHERE token=%s', + (hash_one,), + ) + ) + assert len(deprecated) == 1 + assert deprecated[0].active_token_id == hash_two + assert deprecated[0].grace_seconds == pytest.approx(3600, abs=120) + + path_key_two: Final = hash_two if spelling == "path" else key_two + generated_three: Final = ( + gateway.request("POST", "/key/regenerate", {"key": key_two}) + if spelling == "body" + else gateway.request("POST", f"/key/{path_key_two}/regenerate", {}) + ) + assert generated_three.status_code == 200, generated_three.text + key_three: Final = _GenerateKeyResponse.model_validate_json(generated_three.content).key + scenario.cleanups.callback(delete_key_if_present, gateway, key_three) + refused: Final = _chat(gateway, model, key_two, prompts[2]) + assert refused.status_code == 401, refused.text + assert _ErrorResponse.model_validate_json(refused.content).error.type == "token_not_found_in_db", refused.text + assert wire.drain() == (), "the revoked second key reached the upstream" + assert ( + _assert_success(_chat(gateway, model, key_three, prompts[3])).choices[0].message.content == "wire response" + ) + assert tuple((request.method, request.target) for request in wire.drain()) == (_EXPECTED_CHAT,) diff --git a/tests/integration/management/test_key_delete_revocation.py b/tests/integration/management/test_key_delete_revocation.py new file mode 100644 index 00000000000..5b3527a52ae --- /dev/null +++ b/tests/integration/management/test_key_delete_revocation.py @@ -0,0 +1,233 @@ +import json +import uuid +from hashlib import sha256 +from typing import Final + +import httpx +import pytest +from pydantic import BaseModel, JsonValue, TypeAdapter + +from tests.integration._support.client import Gateway, delete_key_if_present, eventually +from tests.integration._support.database import read_rows +from tests.integration._support.wire import Reply, Request, wire_server + +_CHAT_BODY: Final = TypeAdapter(dict[str, JsonValue]) +_EXPECTED_CHAT: Final = ("POST", "/v1/chat/completions") + + +class _Usage(BaseModel): + prompt_tokens: int + completion_tokens: int + total_tokens: int + + +class _AssistantMessage(BaseModel): + content: str + + +class _Choice(BaseModel): + message: _AssistantMessage + + +class _ChatResponse(BaseModel): + id: str + choices: tuple[_Choice, ...] + usage: _Usage + + +class _ModelEntry(BaseModel): + id: str + + +class _ModelList(BaseModel): + data: tuple[_ModelEntry, ...] + + +class _GenerateKeyResponse(BaseModel): + key: str + + +class _DeleteResponse(BaseModel): + deleted_keys: tuple[str, ...] + + +class _KeyInfo(BaseModel): + status: str + + +class _KeyInfoResponse(BaseModel): + info: _KeyInfo + + +class _Error(BaseModel): + type: str + + +class _ErrorResponse(BaseModel): + error: _Error + + +class _DeletedKeyRow(BaseModel): + token: str + + +def _chat_body(model: str, prompt: str) -> dict[str, JsonValue]: + return { + "model": model, + "messages": [{"role": "user", "content": prompt}], + } + + +def _wire_reply(identity: str) -> Reply: + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "integration-keys-auth-delete", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "wire response"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 20, "completion_tokens": 20, "total_tokens": 40}, + } + ).encode() + ) + + +def _chat(gateway: Gateway, model: str, key: str, prompt: str) -> httpx.Response: + return gateway.request("POST", "/v1/chat/completions", _chat_body(model, prompt), key=key) + + +def _assert_success(response: httpx.Response) -> _ChatResponse: + assert response.status_code == 200, response.text + payload: Final = _ChatResponse.model_validate_json(response.content) + assert payload.choices[0].message.content == "wire response", response.text + assert payload.usage == _Usage(prompt_tokens=20, completion_tokens=20, total_tokens=40), response.text + return payload + + +def _model_is_available(gateway: Gateway, model: str) -> bool: + response: Final = gateway.request("GET", "/v1/models") + if response.status_code != 200: + return False + models: Final = _ModelList.model_validate_json(response.content) + return model in tuple(entry.id for entry in models.data) + + +@pytest.mark.parametrize("spelling", ("keys", "hashed_keys", "key_aliases")) +def test_deleting_a_key_revokes_it(gateway: Gateway, spelling: str) -> None: + backend: Final = "integration-keys-auth-delete" + prompts: Final = ( + "delete warm " + uuid.uuid4().hex, + "delete reject " + uuid.uuid4().hex, + ) + + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/v1/chat/completions" + assert request.headers["authorization"] == "Bearer synthetic-openai-key" + body: Final = _CHAT_BODY.validate_json(request.body) + assert body in tuple(_chat_body(backend, prompt) for prompt in prompts), body + return _wire_reply("chatcmpl-" + uuid.uuid4().hex) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"openai/{backend}", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key" + ) + eventually(lambda: _model_is_available(gateway, model), bool, seconds=30) + alias: Final = "integration-delete-" + uuid.uuid4().hex + created_response: Final = gateway.request("POST", "/key/generate", {"key_alias": alias, "models": [model]}) + assert created_response.status_code == 200, created_response.text + key: Final = _GenerateKeyResponse.model_validate_json(created_response.content).key + scenario.cleanups.callback(delete_key_if_present, gateway, key) + key_hash: Final = sha256(key.encode()).hexdigest() + _assert_success(_chat(gateway, model, key, prompts[0])) + assert tuple((request.method, request.target) for request in wire.drain()) == (_EXPECTED_CHAT,) + + identifier: Final = {"keys": key, "hashed_keys": key_hash, "key_aliases": alias}[spelling] + delete_body: Final = {"key_aliases" if spelling == "key_aliases" else "keys": [identifier]} + deleted: Final = gateway.request("POST", "/key/delete", delete_body) + assert deleted.status_code == 200, deleted.text + assert _DeleteResponse.model_validate_json(deleted.content).deleted_keys == (identifier,), deleted.text + primary_refusal: Final = _chat(gateway, model, key, prompts[1]) + assert primary_refusal.status_code == 401, primary_refusal.text + assert _ErrorResponse.model_validate_json(primary_refusal.content).error.type == "token_not_found_in_db", ( + primary_refusal.text + ) + assert wire.drain() == (), "deleted-key requests reached the upstream" + info: Final = gateway.request("GET", "/key/info", params={"key": key_hash}) + assert info.status_code == 200, info.text + assert _KeyInfoResponse.model_validate_json(info.content).info.status == "deleted", info.text + live_rows: Final = read_rows('SELECT token FROM "LiteLLM_VerificationToken" WHERE token=%s', (key_hash,)) + deleted_rows: Final = TypeAdapter(tuple[_DeletedKeyRow, ...]).validate_python( + read_rows('SELECT token FROM "LiteLLM_DeletedVerificationToken" WHERE token=%s', (key_hash,)) + ) + assert live_rows == [] + assert deleted_rows == (_DeletedKeyRow(token=key_hash),) + + +def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gateway) -> None: + pytest.skip( + "BUG: POST /key/delete does not broadcast the key eviction, so a warmed peer keeps answering 200 for about " + "60 seconds until its in-memory cache TTL expires" + ) + backend: Final = "integration-keys-auth-delete" + prompts: Final = ( + "delete warm primary " + uuid.uuid4().hex, + "delete warm peer " + uuid.uuid4().hex, + "delete reject peer " + uuid.uuid4().hex, + "delete reject peer fresh " + uuid.uuid4().hex, + ) + + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/v1/chat/completions" + assert request.headers["authorization"] == "Bearer synthetic-openai-key" + body: Final = _CHAT_BODY.validate_json(request.body) + assert body in tuple(_chat_body(backend, prompt) for prompt in prompts), body + return _wire_reply("chatcmpl-" + uuid.uuid4().hex) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"openai/{backend}", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key" + ) + eventually(lambda: _model_is_available(peer, model), bool, seconds=30) + alias: Final = "integration-delete-" + uuid.uuid4().hex + created_response: Final = gateway.request("POST", "/key/generate", {"key_alias": alias, "models": [model]}) + assert created_response.status_code == 200, created_response.text + key: Final = _GenerateKeyResponse.model_validate_json(created_response.content).key + scenario.cleanups.callback(delete_key_if_present, gateway, key) + warm_responses: Final = tuple( + _assert_success(_chat(worker, model, key, prompt)) + for worker, prompt in zip((gateway, peer), prompts[:2], strict=True) + ) + assert len(warm_responses) == 2 + assert tuple((request.method, request.target) for request in wire.drain()) == (_EXPECTED_CHAT,) * 2 + deleted: Final = gateway.request("POST", "/key/delete", {"keys": [key]}) + assert deleted.status_code == 200, deleted.text + assert _DeleteResponse.model_validate_json(deleted.content).deleted_keys == (key,), deleted.text + peer_refusal: Final = eventually( + lambda: _chat(peer, model, key, prompts[2]), + lambda response: response.status_code == 401, + seconds=3, + ) + assert _ErrorResponse.model_validate_json(peer_refusal.content).error.type == "token_not_found_in_db", ( + peer_refusal.text + ) + wire.drain() + fresh_peer_refusal: Final = _chat(peer, model, key, prompts[3]) + assert fresh_peer_refusal.status_code == 401, fresh_peer_refusal.text + assert _ErrorResponse.model_validate_json(fresh_peer_refusal.content).error.type == "token_not_found_in_db", ( + fresh_peer_refusal.text + ) + fresh_gateway_refusal: Final = _chat(gateway, model, key, prompts[3]) + assert fresh_gateway_refusal.status_code == 401, fresh_gateway_refusal.text + assert _ErrorResponse.model_validate_json(fresh_gateway_refusal.content).error.type == ( + "token_not_found_in_db" + ), fresh_gateway_refusal.text + assert wire.drain() == (), "deleted-key requests reached the upstream" diff --git a/tests/integration/spend/test_key_budget_lockout.py b/tests/integration/spend/test_key_budget_lockout.py index 7108978f6e5..56b4db89bcb 100644 --- a/tests/integration/spend/test_key_budget_lockout.py +++ b/tests/integration/spend/test_key_budget_lockout.py @@ -1,13 +1,167 @@ +import json +import os import uuid from hashlib import sha256 +from pathlib import Path from typing import Final import httpx import pytest from integration._support.client import Gateway, eventually, object_value from integration._support.database import read_rows +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server from openai.types.chat import ChatCompletionChunk -from pydantic import JsonValue +from pydantic import BaseModel, JsonValue, TypeAdapter +from redis import Redis + +_CHAT_BODY: Final = TypeAdapter(dict[str, JsonValue]) +_CHANNEL: Final = "litellm_proxy.auth_cache_invalidation" +_EXPECTED_MODEL_DISCOVERY: Final = ("GET", "/v1/models") +_EXPECTED_CHAT: Final = ("POST", "/v1/chat/completions") + + +class _Usage(BaseModel): + prompt_tokens: int + completion_tokens: int + total_tokens: int + + +_FULL_USAGE: Final = _Usage(prompt_tokens=20, completion_tokens=20, total_tokens=40) +_POST_RESET_USAGE: Final = _Usage(prompt_tokens=10, completion_tokens=10, total_tokens=20) + + +class _AssistantMessage(BaseModel): + content: str + + +class _Choice(BaseModel): + message: _AssistantMessage + + +class _ChatResponse(BaseModel): + id: str + choices: tuple[_Choice, ...] + usage: _Usage + + +class _ModelInfoEntry(BaseModel): + model_name: str + + +class _ModelInfoResponse(BaseModel): + data: tuple[_ModelInfoEntry, ...] + + +class _BudgetError(BaseModel): + type: str + + +class _ErrorResponse(BaseModel): + error: _BudgetError + + +class _ResetResponse(BaseModel): + key_hash: str + spend: float + previous_spend: float + + +class _KeyInfo(BaseModel): + spend: float + max_budget: float + + +class _KeyInfoResponse(BaseModel): + info: _KeyInfo + + +class _SpendRow(BaseModel): + spend: float + + +class _UserMessage(BaseModel): + content: str + + +class _ChatRequest(BaseModel): + model: str + max_tokens: int + messages: tuple[_UserMessage, ...] + + +def _chat_body(model: str, prompt: str, max_tokens: int = 20) -> dict[str, JsonValue]: + return { + "model": model, + "max_tokens": max_tokens, + "messages": [{"role": "user", "content": prompt}], + } + + +def _wire_reply(identity: str, usage: _Usage) -> Reply: + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "integration-keys-auth-reset", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "wire response"}, + "finish_reason": "stop", + } + ], + "usage": usage.model_dump(), + } + ).encode() + ) + + +def _assert_wire_requests(wire: Wire, expected: tuple[tuple[str, str], ...]) -> None: + observed: Final = tuple( + sorted( + (request.method, request.target) + for request in wire.drain() + if (request.method, request.target) != _EXPECTED_MODEL_DISCOVERY + ) + ) + assert observed == tuple(sorted(expected)), observed + + +def _wire_chat(gateway: Gateway, model: str, key: str, prompt: str, max_tokens: int = 20) -> httpx.Response: + return gateway.request("POST", "/v1/chat/completions", _chat_body(model, prompt, max_tokens), key=key) + + +def _assert_wire_success(response: httpx.Response, expected_usage: _Usage) -> _ChatResponse: + assert response.status_code == 200, response.text + payload: Final = _ChatResponse.model_validate_json(response.content) + assert payload.choices[0].message.content == "wire response", response.text + assert payload.usage == expected_usage, response.text + return payload + + +def _model_is_available(gateway: Gateway, model: str) -> bool: + response: Final = gateway.request("GET", "/model/info") + if response.status_code != 200: + return False + models: Final = _ModelInfoResponse.model_validate_json(response.content) + return any(entry.model_name == model for entry in models.data) + + +def _spend(gateway: Gateway, key_hash: str) -> float: + rows: Final = TypeAdapter(tuple[_SpendRow, ...]).validate_python( + read_rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', (key_hash,)) + ) + assert len(rows) == 1 + return rows[0].spend + + +def _assert_budget_refusal(gateway: Gateway, model: str, key: str, prompt: str) -> None: + response: Final = _wire_chat(gateway, model, key, prompt) + assert response.status_code == 422, response.text + assert _ErrorResponse.model_validate_json(response.content).error.type == "budget_exceeded", response.text def test_an_exhausted_key_is_refused_inference_but_can_still_read_its_own_info(gateway: Gateway) -> None: @@ -113,6 +267,133 @@ def test_raising_a_spent_keys_budget_restores_serving(gateway: Gateway) -> None: assert len(upstream.get("/__observations").json()["requests"]) == 3 +@pytest.mark.parametrize("spelling", ("plaintext", "hashed")) +def test_reset_spend_lifts_an_exhausted_key_on_both_workers_for_both_path_spellings( + gateway: Gateway, tmp_path: Path, spelling: str +) -> None: + backend: Final = "integration-keys-auth-reset" + prompts: Final = tuple(f"reset spend case {index} {uuid.uuid4().hex}" for index in range(11)) + accepted_after_reset: Final = frozenset((prompts[3], prompts[4], prompts[7], prompts[8])) + + def respond(request: Request) -> Reply: + if request.method == "GET": + assert request.target == "/v1/models", request + assert request.headers["authorization"] == "Bearer synthetic-openai-key" + assert request.body == b"" + return Reply( + body=json.dumps( + { + "object": "list", + "data": [ + { + "id": backend, + "object": "model", + "created": 0, + "owned_by": "integration", + } + ], + } + ).encode() + ) + assert request.method == "POST" + assert request.target == "/v1/chat/completions" + assert request.headers["authorization"] == "Bearer synthetic-openai-key" + body: Final = _CHAT_BODY.validate_json(request.body) + parsed: Final = _ChatRequest.model_validate_json(request.body) + prompt: Final = parsed.messages[0].content + max_tokens: Final = 10 if prompt in accepted_after_reset else 20 + assert parsed.model == backend + assert parsed.max_tokens == max_tokens + assert len(parsed.messages) == 1 and prompt in prompts, request.body + assert body == _chat_body(backend, prompt, max_tokens), body + usage: Final = _POST_RESET_USAGE if prompt in accepted_after_reset else _FULL_USAGE + return _wire_reply("chatcmpl-" + uuid.uuid4().hex, usage) + + with ( + wire_server(respond) as wire, + gateway.scenario() as scenario, + Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache, + ): + model: Final = scenario.model( + model=f"openai/{backend}", + api_base=f"{wire.url}/v1", + api_key="synthetic-openai-key", + input_cost_per_token=0.001, + output_cost_per_token=0.002, + ) + subscriber_baseline: Final = cache.pubsub_numsub(_CHANNEL)[0][1] + with owned_proxy(gateway, tmp_path / "second-worker", {}) as second: + eventually( + lambda: cache.pubsub_numsub(_CHANNEL)[0][1], + lambda count: count >= subscriber_baseline + 1, + seconds=30, + ) + eventually(lambda: _model_is_available(second, model), bool, seconds=30) + key: Final = scenario.key(models=[model], max_budget=0.06) + key_hash: Final = sha256(key.encode()).hexdigest() + first_spend: Final = _assert_wire_success(_wire_chat(gateway, model, key, prompts[0]), _FULL_USAGE) + assert first_spend.usage.total_tokens == 40 + exhausted_spend: Final = eventually( + lambda: _spend(gateway, key_hash), lambda spend: spend >= 0.06, seconds=70 + ) + assert exhausted_spend == pytest.approx(0.06) + _assert_wire_requests(wire, (_EXPECTED_CHAT,)) + _assert_budget_refusal(gateway, model, key, prompts[1]) + _assert_budget_refusal(second, model, key, prompts[2]) + assert wire.drain() == (), "exhausted-key requests reached the upstream" + + identifier: Final = key if spelling == "plaintext" else key_hash + for cycle, reset_to in enumerate((0, 0.02)): + prior_spend = _spend(gateway, key_hash) + reset = gateway.request("POST", f"/key/{identifier}/reset_spend", {"reset_to": reset_to}) + assert reset.status_code == 200, reset.text + reset_result = _ResetResponse.model_validate_json(reset.content) + assert reset_result.key_hash == key_hash, reset.text + assert reset_result.spend == pytest.approx(reset_to), reset.text + assert reset_result.previous_spend == pytest.approx(prior_spend), reset.text + info_after_reset = gateway.request("GET", "/key/info", params={"key": key}) + assert info_after_reset.status_code == 200, info_after_reset.text + assert _KeyInfoResponse.model_validate_json(info_after_reset.content).info.spend == pytest.approx( + reset_to + ), info_after_reset.text + assert _spend(gateway, key_hash) == pytest.approx(reset_to) + peer_info_after_reset = eventually( + lambda: second.request("GET", "/key/info", params={"key": key}), + lambda response, reset_to=reset_to: ( + response.status_code == 200 + and _KeyInfoResponse.model_validate_json(response.content).info.spend == pytest.approx(reset_to) + ), + seconds=10, + ) + assert _KeyInfoResponse.model_validate_json(peer_info_after_reset.content).info.spend == pytest.approx( + reset_to + ), peer_info_after_reset.text + workers = (gateway, second) if spelling == "plaintext" else (second, gateway) + first_chat = _wire_chat(workers[0], model, key, prompts[3 + cycle * 4], max_tokens=10) + _assert_wire_success(first_chat, _POST_RESET_USAGE) + first_reported_spend = float(first_chat.headers["x-litellm-key-spend"]) + assert first_reported_spend == pytest.approx(reset_to + 0.03), first_chat.headers + eventually( + lambda: float(cache.get(f"spend:key:{key_hash}") or 0), + lambda counter, reset_to=reset_to: counter == pytest.approx(reset_to + 0.03), + seconds=30, + ) + second_chat = _wire_chat(workers[1], model, key, prompts[4 + cycle * 4], max_tokens=10) + _assert_wire_success(second_chat, _POST_RESET_USAGE) + second_reported_spend = float(second_chat.headers["x-litellm-key-spend"]) + assert reset_to + 0.03 <= second_reported_spend <= reset_to + 0.06, second_chat.headers + re_exhausted_spend = eventually( + lambda: _spend(gateway, key_hash), + lambda spend, reset_to=reset_to: spend >= reset_to + 0.06, + seconds=70, + ) + assert re_exhausted_spend == pytest.approx(reset_to + 0.06) + _assert_wire_requests(wire, (_EXPECTED_CHAT, _EXPECTED_CHAT)) + _assert_budget_refusal(gateway, model, key, prompts[5 + cycle * 4]) + _assert_budget_refusal(second, model, key, prompts[6 + cycle * 4]) + assert wire.drain() == (), "re-exhausted-key requests reached the upstream" + + def _denied_with_budget_exceeded(gateway: Gateway, model: str, key: str) -> None: denied: Final = _bounded_chat(gateway, model, key) assert denied.status_code == 422, denied.text From feb92d3255ab44f645d17eb3afe988eb0b50ad0a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:38:06 +0000 Subject: [PATCH 13/51] test(integration): cover orgs_users_customers endpoint contracts (#44725) --- .../test_access_group_model_listing.py | 354 +++++++++++ .../authorization/test_end_user_block.py | 367 ++++++++++++ .../test_user_self_update_guard.py | 287 +++++++++ .../management/test_organization_lifecycle.py | 150 ++++- .../test_user_delete_revokes_keys.py | 394 +++++++++++++ ..._user_updates_wedged_coordination_redis.py | 324 ++++++++++ .../spend/test_end_user_budget_enforcement.py | 200 +++++++ .../test_end_user_spend_without_proxy_user.py | 553 +++++++++++++++++- 8 files changed, 2626 insertions(+), 3 deletions(-) create mode 100644 tests/integration/authorization/test_end_user_block.py create mode 100644 tests/integration/authorization/test_user_self_update_guard.py create mode 100644 tests/integration/management/test_user_delete_revokes_keys.py diff --git a/tests/integration/authorization/test_access_group_model_listing.py b/tests/integration/authorization/test_access_group_model_listing.py index 9b37cc8f232..13439d19769 100644 --- a/tests/integration/authorization/test_access_group_model_listing.py +++ b/tests/integration/authorization/test_access_group_model_listing.py @@ -1,12 +1,15 @@ import uuid from collections.abc import Iterator from contextlib import contextmanager +from hashlib import sha256 from typing import Final import httpx import pytest +from pydantic import BaseModel, JsonValue from tests.integration._support.client import Gateway, eventually, object_value, string_value +from tests.integration._support.database import read_rows @contextmanager @@ -49,3 +52,354 @@ def test_team_key_lists_models_granted_through_team_access_group(gateway: Gatewa ) assert response.status_code == 200, response.text assert _listed_model_ids(response) == (model,), response.text + + +class _AccessGroupInfo(BaseModel): + access_group_id: str + assigned_team_ids: list[str] | None + assigned_key_ids: list[str] | None + + +def _denied_error(model: str, denied_type: str) -> dict[str, JsonValue]: + return { + "error": { + "message": f"The requested model '{model}' is not available for this API key, or the model name is invalid. Check the models available to you and try again.", + "type": denied_type, + "param": "model", + "code": "403", + } + } + + +def _observed(upstream: httpx.Client) -> list[dict[str, JsonValue]]: + response: Final = upstream.get("/__observations") + response.raise_for_status() + requests: Final = response.json()["requests"] + assert isinstance(requests, list) + return requests + + +def _chat(proxy: Gateway, model: str, key: str, text: str) -> httpx.Response: + return proxy.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": text}]}, + key=key, + ) + + +def _observation(text: str) -> dict[str, JsonValue]: + return { + "path": "/v1/chat/completions", + "authorization": "Bearer integration-provider-key", + "body": {"messages": [{"role": "user", "content": text}], "model": "gpt-4o-mini"}, + "method": "POST", + "api_key": "", + } + + +def _team_groups(team_id: str) -> list[dict[str, JsonValue]]: + return read_rows('SELECT access_group_ids FROM "LiteLLM_TeamTable" WHERE team_id=%s', (team_id,)) + + +def _key_groups(hashed: str) -> list[dict[str, JsonValue]]: + return read_rows('SELECT access_group_ids FROM "LiteLLM_VerificationToken" WHERE token=%s', (hashed,)) + + +def _group_row(access_group_id: str) -> list[dict[str, JsonValue]]: + return read_rows( + 'SELECT assigned_team_ids, assigned_key_ids FROM "LiteLLM_AccessGroupTable" WHERE access_group_id=%s', + (access_group_id,), + ) + + +def _assert_denied(response: httpx.Response, model: str, denied_type: str) -> None: + assert response.status_code == 403, response.text + assert response.json() == _denied_error(model, denied_type), response.text + + +def _assert_served(response: httpx.Response) -> None: + assert response.status_code == 200, response.text + + +@pytest.mark.parametrize("route_prefix", ("/v1/access_group", "/v1/unified_access_group")) +def test_access_group_assignment_grants_and_revokes_on_gateway_and_peer( + gateway: Gateway, peer: Gateway, route_prefix: str +) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model() + team_id: Final = scenario.team(models=["no-default-models"]) + team_key: Final = scenario.key(team_id=team_id) + assigned_key: Final = scenario.key(models=["no-default-models"]) + hashed_key: Final = sha256(assigned_key.encode()).hexdigest() + _observed(upstream) + _assert_denied(_chat(gateway, model, team_key, "control team"), model, "team_model_access_denied") + _assert_denied(_chat(gateway, model, assigned_key, "control key"), model, "key_model_access_denied") + assert _observed(upstream) == [] + + created: Final = gateway.request( + "POST", + route_prefix, + { + "access_group_name": f"integration-{uuid.uuid4().hex}", + "access_model_names": [model], + "assigned_team_ids": [team_id], + "assigned_key_ids": [hashed_key], + }, + ) + assert created.status_code == 201, created.text + access_group_id: Final = string_value(created.json()["access_group_id"]) + scenario.cleanups.callback(gateway.request, "DELETE", f"{route_prefix}/{access_group_id}") + info: Final = _AccessGroupInfo.model_validate(gateway.get(f"{route_prefix}/{access_group_id}")) + assert info.assigned_team_ids == [team_id] + assert info.assigned_key_ids == [hashed_key] + assert _group_row(access_group_id) == [{"assigned_team_ids": [team_id], "assigned_key_ids": [hashed_key]}] + assert _team_groups(team_id) == [{"access_group_ids": [access_group_id]}] + assert _key_groups(hashed_key) == [{"access_group_ids": [access_group_id]}] + + expected: Final = [] + for proxy in (gateway, peer): + for key in (team_key, assigned_key): + text: Final = f"granted {uuid.uuid4().hex}" + _assert_served( + eventually( + lambda p=proxy, k=key, t=text: _chat(p, model, k, t), + lambda response: response.status_code == 200, + seconds=30, + ) + ) + expected.append(_observation(text)) + assert _observed(upstream) == expected + + removed_team: Final = gateway.request("PUT", f"{route_prefix}/{access_group_id}", {"assigned_team_ids": []}) + assert removed_team.status_code == 200, removed_team.text + assert _team_groups(team_id) == [{"access_group_ids": []}] + _assert_denied(_chat(gateway, model, team_key, "denied"), model, "team_model_access_denied") + denied_observations: Final = _observed(upstream) + assert denied_observations == [], denied_observations + still_granted: Final = f"still granted {uuid.uuid4().hex}" + _assert_served(_chat(gateway, model, assigned_key, still_granted)) + assert _observed(upstream) == [_observation(still_granted)] + + removed_key: Final = gateway.request("PUT", f"{route_prefix}/{access_group_id}", {"assigned_key_ids": []}) + assert removed_key.status_code == 200, removed_key.text + assert _key_groups(hashed_key) == [{"access_group_ids": []}] + _assert_denied(_chat(gateway, model, assigned_key, "denied"), model, "key_model_access_denied") + assert _observed(upstream) == [] + + restored: Final = gateway.request( + "PUT", + f"{route_prefix}/{access_group_id}", + {"assigned_team_ids": [team_id], "assigned_key_ids": [hashed_key]}, + ) + assert restored.status_code == 200, restored.text + assert _observed(upstream) == [] + regranted: Final = [] + for key in (team_key, assigned_key): + text = f"re-granted {uuid.uuid4().hex}" + _assert_served(_chat(gateway, model, key, text)) + regranted.append(_observation(text)) + assert _observed(upstream) == regranted + + deleted: Final = gateway.request("DELETE", f"{route_prefix}/{access_group_id}") + assert deleted.status_code == 204, deleted.text + assert _group_row(access_group_id) == [] + assert _team_groups(team_id) == [{"access_group_ids": []}] + assert _key_groups(hashed_key) == [{"access_group_ids": []}] + denied_pairs: Final = ( + (team_key, "team_model_access_denied"), + (assigned_key, "key_model_access_denied"), + ) + for key, denied_type in denied_pairs: + _assert_denied(_chat(gateway, model, key, "deleted"), model, denied_type) + deleted_observations: Final = _observed(upstream) + assert deleted_observations == [], deleted_observations + for key, denied_type in denied_pairs: + _assert_denied(_chat(gateway, model, key, "deleted"), model, denied_type) + assert _observed(upstream) == [] + + +@pytest.mark.parametrize("route_prefix", ("/v1/access_group", "/v1/unified_access_group")) +def test_deleting_an_access_group_revokes_a_still_assigned_team_and_key(gateway: Gateway, route_prefix: str) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model() + team_id: Final = scenario.team(models=["no-default-models"]) + team_key: Final = scenario.key(team_id=team_id) + assigned_key: Final = scenario.key(models=["no-default-models"]) + hashed_key: Final = sha256(assigned_key.encode()).hexdigest() + created: Final = gateway.request( + "POST", + route_prefix, + { + "access_group_name": f"integration-{uuid.uuid4().hex}", + "access_model_names": [model], + "assigned_team_ids": [team_id], + "assigned_key_ids": [hashed_key], + }, + ) + assert created.status_code == 201, created.text + access_group_id: Final = string_value(created.json()["access_group_id"]) + scenario.cleanups.callback(gateway.request, "DELETE", f"{route_prefix}/{access_group_id}") + assert _group_row(access_group_id) == [{"assigned_team_ids": [team_id], "assigned_key_ids": [hashed_key]}] + assert _team_groups(team_id) == [{"access_group_ids": [access_group_id]}] + assert _key_groups(hashed_key) == [{"access_group_ids": [access_group_id]}] + assert _observed(upstream) == [] + + team_text: Final = f"delete warm team {uuid.uuid4().hex}" + team_response: Final = eventually( + lambda: _chat(gateway, model, team_key, team_text), + lambda response: response.status_code == 200, + seconds=30, + ) + _assert_served(team_response) + key_text: Final = f"delete warm key {uuid.uuid4().hex}" + key_response: Final = eventually( + lambda: _chat(gateway, model, assigned_key, key_text), + lambda response: response.status_code == 200, + seconds=30, + ) + _assert_served(key_response) + assert _observed(upstream) == [_observation(team_text), _observation(key_text)] + + deleted: Final = gateway.request("DELETE", f"{route_prefix}/{access_group_id}") + assert deleted.status_code == 204, deleted.text + assert _group_row(access_group_id) == [] + assert _team_groups(team_id) == [{"access_group_ids": []}] + assert _key_groups(hashed_key) == [{"access_group_ids": []}] + assert _observed(upstream) == [] + denied_pairs: Final = ( + (team_key, "team_model_access_denied"), + (assigned_key, "key_model_access_denied"), + ) + for key, denied_type in denied_pairs: + _assert_denied(_chat(gateway, model, key, "deleted"), model, denied_type) + assert _observed(upstream) == [] + + +@pytest.mark.parametrize("route_prefix", ("/v1/access_group", "/v1/unified_access_group")) +@pytest.mark.parametrize("revocation", ("team", "key", "delete")) +def test_access_group_revocation_reaches_a_warmed_peer_before_the_cache_ttl( + gateway: Gateway, peer: Gateway, route_prefix: str, revocation: str +) -> None: + pytest.skip("BUG: warmed peer returns 200 instead of 403 after access-group revocation") + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model() + team_id: Final = scenario.team(models=["no-default-models"]) + team_key: Final = scenario.key(team_id=team_id) + assigned_key: Final = scenario.key(models=["no-default-models"]) + hashed_key: Final = sha256(assigned_key.encode()).hexdigest() + created: Final = gateway.request( + "POST", + route_prefix, + { + "access_group_name": f"integration-{uuid.uuid4().hex}", + "access_model_names": [model], + "assigned_team_ids": [team_id], + "assigned_key_ids": [hashed_key], + }, + ) + assert created.status_code == 201, created.text + access_group_id: Final = string_value(created.json()["access_group_id"]) + scenario.cleanups.callback(gateway.request, "DELETE", f"{route_prefix}/{access_group_id}") + _observed(upstream) + + if revocation == "team": + warmed: Final = _chat(peer, model, team_key, "warm team") + assert warmed.status_code == 200, warmed.text + assert _observed(upstream) == [_observation("warm team")] + team_update: Final = gateway.request("PUT", f"{route_prefix}/{access_group_id}", {"assigned_team_ids": []}) + assert team_update.status_code == 200, team_update.text + _assert_denied( + eventually( + lambda: _chat(peer, model, team_key, "revoked team"), + lambda response: response.status_code == 403, + seconds=10, + ), + model, + "team_model_access_denied", + ) + elif revocation == "key": + warmed: Final = _chat(peer, model, assigned_key, "warm key") + assert warmed.status_code == 200, warmed.text + assert _observed(upstream) == [_observation("warm key")] + key_update: Final = gateway.request("PUT", f"{route_prefix}/{access_group_id}", {"assigned_key_ids": []}) + assert key_update.status_code == 200, key_update.text + _assert_denied( + eventually( + lambda: _chat(peer, model, assigned_key, "revoked key"), + lambda response: response.status_code == 403, + seconds=10, + ), + model, + "key_model_access_denied", + ) + else: + warmed_team: Final = _chat(peer, model, team_key, "warm delete team") + warmed_key: Final = _chat(peer, model, assigned_key, "warm delete key") + assert warmed_team.status_code == 200, warmed_team.text + assert warmed_key.status_code == 200, warmed_key.text + assert _observed(upstream) == [ + _observation("warm delete team"), + _observation("warm delete key"), + ] + deleted: Final = gateway.request("DELETE", f"{route_prefix}/{access_group_id}") + assert deleted.status_code == 204, deleted.text + _assert_denied( + eventually( + lambda: _chat(peer, model, team_key, "revoked delete team"), + lambda response: response.status_code == 403, + seconds=10, + ), + model, + "team_model_access_denied", + ) + _assert_denied( + eventually( + lambda: _chat(peer, model, assigned_key, "revoked delete key"), + lambda response: response.status_code == 403, + seconds=10, + ), + model, + "key_model_access_denied", + ) + assert _observed(upstream) == [] + + +def test_raw_key_in_assigned_key_ids_grants_the_same_access_as_the_hashed_key(gateway: Gateway) -> None: + pytest.skip("BUG: raw assigned key does not populate access_group_ids for its hashed database key") + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model() + key: Final = scenario.key(models=["no-default-models"]) + hashed_key: Final = sha256(key.encode()).hexdigest() + _observed(upstream) + _assert_denied(_chat(gateway, model, key, "control"), model, "key_model_access_denied") + assert _observed(upstream) == [] + + created: Final = gateway.request( + "POST", + "/v1/access_group", + { + "access_group_name": f"integration-{uuid.uuid4().hex}", + "access_model_names": [model], + "assigned_key_ids": [key], + }, + ) + assert created.status_code == 201, created.text + access_group_id: Final = string_value(created.json()["access_group_id"]) + scenario.cleanups.callback(gateway.request, "DELETE", f"/v1/access_group/{access_group_id}") + assert _key_groups(hashed_key) == [{"access_group_ids": [access_group_id]}] + text: Final = f"granted {uuid.uuid4().hex}" + _assert_served(_chat(gateway, model, key, text)) + assert _observed(upstream) == [_observation(text)] diff --git a/tests/integration/authorization/test_end_user_block.py b/tests/integration/authorization/test_end_user_block.py new file mode 100644 index 00000000000..9f4fdb7f8f0 --- /dev/null +++ b/tests/integration/authorization/test_end_user_block.py @@ -0,0 +1,367 @@ +import uuid +from contextlib import ExitStack +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, eventually, object_value +from integration._support.database import read_rows +from integration._support.process import owned_proxy +from pydantic import BaseModel, JsonValue + + +class _BlockedCustomerInfo(BaseModel): + user_id: str + blocked: bool + + +class _BlockUsersResponse(BaseModel): + blocked_users: list[_BlockedCustomerInfo] + + +def _blocked_row(user_id: str) -> list[dict[str, JsonValue]]: + return read_rows('SELECT blocked FROM "LiteLLM_EndUserTable" WHERE user_id=%s', (user_id,)) + + +def _observed(upstream: httpx.Client) -> list[dict[str, JsonValue]]: + response: Final = upstream.get("/__observations") + response.raise_for_status() + requests: Final = response.json()["requests"] + assert isinstance(requests, list) + return requests + + +def _chat(gateway: Gateway, model: str, key: str, end_user: str) -> httpx.Response: + return gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": f"end user block {uuid.uuid4().hex}"}], + "user": end_user, + }, + key=key, + ) + + +def _observation(text: str, end_user: str) -> dict[str, JsonValue]: + return { + "path": "/v1/chat/completions", + "authorization": "Bearer integration-provider-key", + "body": { + "messages": [{"role": "user", "content": text}], + "model": "gpt-4o-mini", + "user": end_user, + }, + "method": "POST", + "api_key": "", + } + + +def _assert_blocked_refusal(response: httpx.Response, end_user: str) -> None: + assert response.status_code == 400, response.text + assert response.json() == { + "error": { + "message": f"User blocked from making LLM API Calls. User={end_user}", + "type": "invalid_request_error", + "param": None, + "code": "400", + "provider_specific_fields": {"error": f"User blocked from making LLM API Calls. User={end_user}"}, + } + }, response.text + + +@dataclass(frozen=True, slots=True) +class _CallbackRig: + first: Gateway + second: Gateway + + +def _callback_config(directory: Path, blocked_user_list: list[str] | None = None) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + settings: Final = { + **object_value(config["litellm_settings"]), + "callbacks": ["blocked_user_check"], + } + if blocked_user_list is not None: + settings["blocked_user_list"] = blocked_user_list + config["litellm_settings"] = settings + path: Final = directory / "config.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +def _two_callback_proxies(gateway: Gateway, tmp_path: Path, stack: ExitStack) -> _CallbackRig: + config: Final = _callback_config(tmp_path) + first: Final = stack.enter_context(owned_proxy(gateway, tmp_path / "a", {}, config=config)) + second: Final = stack.enter_context(owned_proxy(gateway, tmp_path / "b", {}, config=config)) + return _CallbackRig(first, second) + + +def _one_callback_proxy(gateway: Gateway, tmp_path: Path, stack: ExitStack) -> Gateway: + config: Final = _callback_config(tmp_path) + return stack.enter_context(owned_proxy(gateway, tmp_path / "a", {}, config=config)) + + +@pytest.mark.parametrize("block_route", ("/customer/block", "/end_user/block")) +def test_block_route_marks_the_customer_blocked_and_update_unblocks(gateway: Gateway, block_route: str) -> None: + with gateway.scenario() as scenario: + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + created: Final = gateway.request("POST", "/customer/new", {"user_id": end_user, "max_budget": 100.0}) + assert created.status_code == 200, created.text + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + + gateway.request("POST", block_route, {"user_ids": [end_user]}) + assert eventually(lambda: _blocked_row(end_user), lambda value: value == [{"blocked": True}], seconds=15) == [ + {"blocked": True} + ] + info: Final = gateway.get("/customer/info", {"end_user_id": end_user}) + assert _BlockedCustomerInfo.model_validate(info).blocked is True + + updated: Final = gateway.request("POST", "/customer/update", {"user_id": end_user, "blocked": False}) + assert updated.status_code == 200, updated.text + assert eventually(lambda: _blocked_row(end_user), lambda value: value == [{"blocked": False}], seconds=15) == [ + {"blocked": False} + ] + assert ( + _BlockedCustomerInfo.model_validate(gateway.get("/customer/info", {"end_user_id": end_user})).blocked + is False + ) + + +def _assert_unseen_customer_is_blocked(gateway: Gateway, block_route: str) -> None: + with gateway.scenario() as scenario: + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + + blocked: Final = gateway.request("POST", block_route, {"user_ids": [end_user]}) + assert blocked.status_code == 200, blocked.text + blocked_response: Final = _BlockUsersResponse.model_validate_json(blocked.content) + assert blocked_response == _BlockUsersResponse( + blocked_users=[_BlockedCustomerInfo(user_id=end_user, blocked=True)] + ) + assert eventually(lambda: _blocked_row(end_user), lambda value: value == [{"blocked": True}], seconds=15) == [ + {"blocked": True} + ] + info: Final = gateway.request("GET", "/customer/info", params={"end_user_id": end_user}) + assert info.status_code == 200, info.text + assert _BlockedCustomerInfo.model_validate(info.json()).blocked is True + + updated: Final = gateway.request("POST", "/customer/update", {"user_id": end_user, "blocked": False}) + assert updated.status_code == 200, updated.text + assert eventually(lambda: _blocked_row(end_user), lambda value: value == [{"blocked": False}], seconds=15) == [ + {"blocked": False} + ] + info_after_update: Final = gateway.request("GET", "/customer/info", params={"end_user_id": end_user}) + assert info_after_update.status_code == 200, info_after_update.text + assert _BlockedCustomerInfo.model_validate(info_after_update.json()).blocked is False + + +def test_end_user_block_route_creates_and_blocks_an_unseen_customer(gateway: Gateway) -> None: + _assert_unseen_customer_is_blocked(gateway, "/end_user/block") + + +def test_customer_block_route_creates_and_blocks_an_unseen_customer(gateway: Gateway) -> None: + pytest.skip("BUG: POST /customer/block for an unseen customer returns 500 instead of 200 BlockUsersResponse") + _assert_unseen_customer_is_blocked(gateway, "/customer/block") + + +def test_listed_end_user_is_refused_and_unblock_restores_serving(gateway: Gateway, tmp_path: Path) -> None: + with ( + ExitStack() as stack, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + config: Final = _callback_config(tmp_path, blocked_user_list=[end_user]) + proxy: Final = stack.enter_context(owned_proxy(gateway, tmp_path / "listed", {}, config=config)) + with proxy.scenario() as scenario: + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model]) + assert _observed(upstream) == [] + + refused: Final = _chat(proxy, model, key, end_user) + _assert_blocked_refusal(refused, end_user) + assert _observed(upstream) == [] + + unblocked: Final = proxy.request("POST", "/customer/unblock", {"user_ids": [end_user]}) + assert unblocked.status_code == 200, unblocked.text + assert unblocked.json() == {"blocked_users": []}, unblocked.text + assert _blocked_row(end_user) == [] + + text: Final = f"end user unblocked {uuid.uuid4().hex}" + served: Final = proxy.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": text}], "user": end_user}, + key=key, + ) + assert served.status_code == 200, served.text + assert _observed(upstream) == [_observation(text, end_user)] + + +@pytest.mark.parametrize("block_route", ("/customer/block", "/end_user/block")) +def test_blocked_end_user_is_refused_on_every_proxy_and_unblock_restores( + gateway: Gateway, tmp_path: Path, block_route: str +) -> None: + pytest.skip("BUG: blocked end-user chat requests return HTTP 200 instead of HTTP 400") + with ( + ExitStack() as stack, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + rig: Final = _two_callback_proxies(gateway, tmp_path, stack) + with rig.first.scenario() as scenario: + _observed(upstream) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + created: Final = rig.first.request("POST", "/customer/new", {"user_id": end_user, "max_budget": 100.0}) + assert created.status_code == 200, created.text + scenario.cleanups.callback(rig.first.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model]) + served: Final = _chat(rig.first, model, key, end_user) + assert served.status_code == 200, served.text + _observed(upstream) + + rig.first.request("POST", block_route, {"user_ids": [end_user]}) + assert eventually( + lambda: _blocked_row(end_user), lambda value: value == [{"blocked": True}], seconds=15 + ) == [{"blocked": True}] + info: Final = rig.first.get("/customer/info", {"end_user_id": end_user}) + assert _BlockedCustomerInfo.model_validate(info).blocked is True + + denied_first: Final = _chat(rig.first, model, key, end_user) + denied_second: Final = _chat(rig.second, model, key, end_user) + for denied in (denied_first, denied_second): + _assert_blocked_refusal(denied, end_user) + upstream_calls: Final = _observed(upstream) + assert upstream_calls == [], upstream_calls + + updated: Final = rig.first.request("POST", "/customer/update", {"user_id": end_user, "blocked": False}) + assert updated.status_code == 200, updated.text + assert eventually( + lambda: _blocked_row(end_user), lambda value: value == [{"blocked": False}], seconds=15 + ) == [{"blocked": False}] + assert ( + _BlockedCustomerInfo.model_validate(rig.first.get("/customer/info", {"end_user_id": end_user})).blocked + is False + ) + + text: Final = f"end user unblocked {uuid.uuid4().hex}" + + def call(candidate: Gateway) -> httpx.Response: + return candidate.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": text}], "user": end_user}, + key=key, + ) + + assert ( + eventually( + lambda: call(rig.first), lambda response: response.status_code == 200, seconds=10 + ).status_code + == 200 + ) + assert ( + eventually( + lambda: call(rig.second), lambda response: response.status_code == 200, seconds=10 + ).status_code + == 200 + ) + expected_observations: Final = [ + { + "path": "/v1/chat/completions", + "authorization": "Bearer integration-provider-key", + "body": { + "messages": [{"role": "user", "content": text}], + "model": "gpt-4o-mini", + "user": end_user, + }, + "method": "POST", + "api_key": "", + } + ] * 2 + assert _observed(upstream) == expected_observations + + +def test_unblock_route_requires_the_blocked_user_check_callback(gateway: Gateway, tmp_path: Path) -> None: + with ExitStack() as stack: + callback_proxy: Final = _one_callback_proxy(gateway, tmp_path, stack) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + created: Final = callback_proxy.request( + "POST", "/customer/new", {"user_id": end_user, "max_budget": 100.0, "blocked": True} + ) + assert created.status_code == 200, created.text + stack.callback(callback_proxy.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + assert _blocked_row(end_user) == [{"blocked": True}] + + shared: Final = gateway.request("POST", "/customer/unblock", {"user_ids": [end_user]}) + assert shared.status_code == 400, shared.text + assert shared.json() == {"detail": {"error": "Blocked user check was never set. This call has no effect."}}, ( + shared.text + ) + assert _blocked_row(end_user) == [{"blocked": True}] + + callback_response: Final = callback_proxy.request("POST", "/customer/unblock", {"user_ids": [end_user]}) + assert callback_response.status_code == 400, callback_response.text + assert callback_response.json() == { + "detail": {"error": "Blocked user check was never set. This call has no effect."} + }, callback_response.text + assert _blocked_row(end_user) == [{"blocked": True}] + + +def test_block_route_returns_the_blocked_records(gateway: Gateway) -> None: + pytest.skip("BUG: POST /customer/block returns 500 instead of 200 BlockUsersResponse") + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + with gateway.scenario() as scenario: + created: Final = gateway.request("POST", "/customer/new", {"user_id": end_user}) + assert created.status_code == 200, created.text + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + blocked: Final = gateway.request("POST", "/customer/block", {"user_ids": [end_user]}) + assert blocked.status_code == 200, blocked.text + payload: Final = object_value(blocked.json()) + blocked_users: Final = payload["blocked_users"] + assert isinstance(blocked_users, list) and len(blocked_users) == 1 + assert object_value(blocked_users[0])["user_id"] == end_user + assert object_value(blocked_users[0])["blocked"] is True + + +def test_blocked_end_user_is_refused_without_the_callback(gateway: Gateway) -> None: + pytest.skip("BUG: blocked end-user request without callback returns 200 instead of 400") + with gateway.scenario() as scenario: + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + created: Final = gateway.request("POST", "/customer/new", {"user_id": end_user, "blocked": True}) + assert created.status_code == 200, created.text + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model]) + refused: Final = _chat(gateway, model, key, end_user) + _assert_blocked_refusal(refused, end_user) + + +def test_blocked_end_user_header_spelling_is_refused_with_the_callback(gateway: Gateway, tmp_path: Path) -> None: + pytest.skip("BUG: x-litellm-end-user-id request for a blocked customer returns 200 instead of 400") + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + config: Final = _callback_config(tmp_path, blocked_user_list=[end_user]) + with ( + ExitStack() as stack, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + proxy: Final = stack.enter_context(owned_proxy(gateway, tmp_path / "listed", {}, config=config)) + with proxy.scenario() as scenario: + created: Final = proxy.request("POST", "/customer/new", {"user_id": end_user, "max_budget": 100.0}) + assert created.status_code == 200, created.text + scenario.cleanups.callback(proxy.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model]) + refused: Final = proxy.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": f"blocked {uuid.uuid4().hex}"}]}, + key=key, + headers={"x-litellm-end-user-id": end_user}, + ) + _assert_blocked_refusal(refused, end_user) + assert _observed(upstream) == [] diff --git a/tests/integration/authorization/test_user_self_update_guard.py b/tests/integration/authorization/test_user_self_update_guard.py new file mode 100644 index 00000000000..d83960e1e85 --- /dev/null +++ b/tests/integration/authorization/test_user_self_update_guard.py @@ -0,0 +1,287 @@ +import os +import uuid +from typing import Final + +import pytest +from integration._support.client import Gateway, object_value +from integration._support.database import read_rows +from pydantic import BaseModel, JsonValue + +from litellm.models.user import LiteLLM_UserTable +from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken + +_SELF_SERVICE_ROUTES: Final = ("/user/update", "/user/bulk_update", "/user/new", "/user/info") + + +class _UserInfo(BaseModel): + user_id: str + user_alias: str | None + user_role: str + max_budget: float | None + + +def _user_row(user_id: str) -> list[dict[str, JsonValue]]: + return read_rows( + "SELECT user_id, user_alias, user_role, max_budget, model_max_budget, spend, object_permission_id, user_email" + ' FROM "LiteLLM_UserTable" WHERE user_id=%s', + (user_id,), + ) + + +def _user_info(gateway: Gateway, user_id: str) -> _UserInfo: + payload: Final = object_value(gateway.get("/user/info", {"user_id": user_id})) + return _UserInfo.model_validate(payload["user_info"]) + + +def _escalation_error(field: str) -> dict[str, JsonValue]: + return { + "error": { + "message": f"{{'error': \"Non-admin users cannot modify '{field}' on their own record. Contact your proxy admin.\"}}", + "type": "auth_error", + "param": "None", + "code": "403", + } + } + + +def _role_change_error() -> dict[str, JsonValue]: + return { + "error": { + "message": "Only proxy admins can modify user roles.", + "type": "auth_error", + "param": "None", + "code": "403", + } + } + + +def _ui_session_route_error(user_id: str) -> dict[str, JsonValue]: + masked_user_id: Final = f"{user_id[:6]}{'*' * (len(user_id) - 8)}{user_id[-2:]}" + return { + "error": { + "message": ( + "Authentication Error, Only proxy admin can be used to generate, delete, update info for new " + "keys/users/teams. Route=/user/update. Your role=internal_user. " + f"Your user_id={masked_user_id}" + ), + "type": "auth_error", + "param": "None", + "code": "401", + } + } + + +def test_internal_user_can_update_allowed_fields_on_own_record(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + user_id: Final = scenario.user(user_role="internal_user", max_budget=10.0) + key: Final = scenario.key(user_id=user_id, allowed_routes=list(_SELF_SERVICE_ROUTES)) + alias: Final = f"self-alias-{uuid.uuid4().hex}" + updated: Final = gateway.request("POST", "/user/update", {"user_id": user_id, "user_alias": alias}, key=key) + assert updated.status_code == 200, updated.text + assert object_value(object_value(updated.json())["data"])["user_alias"] == alias + assert _user_row(user_id)[0]["user_alias"] == alias + assert _user_info(gateway, user_id).user_alias == alias + + +@pytest.mark.parametrize("user_role", ("proxy_admin", "proxy_admin_viewer")) +def test_internal_user_cannot_promote_own_role(gateway: Gateway, user_role: str) -> None: + with gateway.scenario() as scenario: + user_id: Final = scenario.user(user_role="internal_user", max_budget=10.0) + key: Final = scenario.key(user_id=user_id, allowed_routes=list(_SELF_SERVICE_ROUTES)) + before_row: Final = _user_row(user_id) + before_info: Final = _user_info(gateway, user_id) + + refused: Final = gateway.request( + "POST", + "/user/update", + {"user_id": user_id, "user_role": user_role}, + key=key, + ) + assert refused.status_code == 403, refused.text + assert refused.json() == _role_change_error(), refused.text + assert _user_row(user_id) == before_row + assert _user_info(gateway, user_id) == before_info + + +def test_internal_user_can_update_allowed_fields_by_own_email(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + user_email: Final = f"integration-{uuid.uuid4().hex}@example.com" + user_id: Final = scenario.user( + user_email=user_email, + user_role="internal_user", + max_budget=10.0, + ) + key: Final = scenario.key(user_id=user_id, allowed_routes=list(_SELF_SERVICE_ROUTES)) + alias: Final = f"self-email-alias-{uuid.uuid4().hex}" + + updated: Final = gateway.request( + "POST", + "/user/update", + {"user_email": user_email, "user_alias": alias}, + key=key, + ) + assert updated.status_code == 200, updated.text + updated_data: Final = object_value(object_value(updated.json())["data"]) + assert updated_data["user_alias"] == alias + assert updated_data["user_id"] == user_id + assert _user_row(user_id)[0]["user_alias"] == alias + assert _user_info(gateway, user_id).user_alias == alias + + +def test_internal_user_cannot_escalate_protected_fields_by_own_email(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + user_email: Final = f"integration-{uuid.uuid4().hex}@example.com" + user_id: Final = scenario.user( + user_email=user_email, + user_role="internal_user", + max_budget=10.0, + ) + key: Final = scenario.key(user_id=user_id, allowed_routes=list(_SELF_SERVICE_ROUTES)) + before_row: Final = _user_row(user_id) + before_info: Final = _user_info(gateway, user_id) + + refused: Final = gateway.request( + "POST", + "/user/update", + {"user_email": user_email, "max_budget": 1000.0}, + key=key, + ) + assert refused.status_code == 403, refused.text + assert refused.json() == _escalation_error("max_budget"), refused.text + assert _user_row(user_id) == before_row + assert _user_info(gateway, user_id) == before_info + + +@pytest.mark.parametrize( + ("field", "value"), + (("max_budget", 1000), ("object_permission", {})), + ids=("max_budget", "object_permission"), +) +def test_ui_session_token_cannot_update_own_protected_fields( + gateway: Gateway, + monkeypatch: pytest.MonkeyPatch, + field: str, + value: JsonValue, +) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", os.environ.get("LITELLM_SALT_KEY", "sk-integration-salt")) + with gateway.scenario() as scenario: + user_id: Final = scenario.user(user_role="internal_user", max_budget=10.0) + token: Final = ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token( + LiteLLM_UserTable(user_id=user_id, user_role="internal_user", models=[]) + ) + self_info: Final = gateway.request("GET", "/user/info", params={"user_id": user_id}, key=token) + assert self_info.status_code == 200, self_info.text + assert object_value(object_value(self_info.json())["user_info"])["user_id"] == user_id, self_info.text + before_row: Final = _user_row(user_id) + before_info: Final = _user_info(gateway, user_id) + + refused: Final = gateway.request("POST", "/user/update", {"user_id": user_id, field: value}, key=token) + assert refused.status_code == 401, refused.text + assert refused.json() == _ui_session_route_error(user_id), refused.text + assert _user_row(user_id) == before_row + assert _user_info(gateway, user_id) == before_info + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("max_budget", 1000), + ("max_budget", "1000"), + ("model_max_budget", {"gpt-4o": 5.0}), + ("spend", 0), + ("object_permission", {"vector_stores": ["vs-1"]}), + ("object_permission", {}), + ), + ids=( + "max_budget_number", + "max_budget_string", + "model_max_budget", + "spend", + "object_permission", + "object_permission_empty", + ), +) +def test_internal_user_cannot_escalate_protected_fields(gateway: Gateway, field: str, value: JsonValue) -> None: + with gateway.scenario() as scenario: + user_id: Final = scenario.user(user_role="internal_user", max_budget=10.0) + key: Final = scenario.key(user_id=user_id, allowed_routes=list(_SELF_SERVICE_ROUTES)) + before_row: Final = _user_row(user_id) + before_info: Final = _user_info(gateway, user_id) + + refused: Final = gateway.request("POST", "/user/update", {"user_id": user_id, field: value}, key=key) + assert refused.status_code == 403, refused.text + assert refused.json() == _escalation_error(field), refused.text + assert _user_row(user_id) == before_row + assert _user_info(gateway, user_id) == before_info + + +def test_bulk_update_self_escalation_fails_per_user(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + user_id: Final = scenario.user(user_role="internal_user", max_budget=10.0) + key: Final = scenario.key(user_id=user_id, allowed_routes=list(_SELF_SERVICE_ROUTES)) + before_row: Final = _user_row(user_id) + + refused: Final = gateway.request( + "POST", + "/user/bulk_update", + {"users": [{"user_id": user_id, "max_budget": 1000}]}, + key=key, + ) + assert refused.status_code == 200, refused.text + assert refused.json() == { + "results": [ + { + "user_id": user_id, + "user_email": None, + "success": False, + "error": "403: {'error': \"Non-admin users cannot modify 'max_budget' on their own record. Contact your proxy admin.\"}", + "updated_user": None, + } + ], + "total_requested": 1, + "successful_updates": 0, + "failed_updates": 1, + }, refused.text + assert _user_row(user_id) == before_row + + +def test_bulk_update_user_role_requires_proxy_admin(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + user_id: Final = scenario.user(user_role="internal_user", max_budget=10.0) + key: Final = scenario.key(user_id=user_id, allowed_routes=list(_SELF_SERVICE_ROUTES)) + before_row: Final = _user_row(user_id) + + refused: Final = gateway.request( + "POST", + "/user/bulk_update", + {"users": [{"user_id": user_id, "user_role": "proxy_admin"}]}, + key=key, + ) + assert refused.status_code == 403, refused.text + assert refused.json() == {"detail": "Only proxy admins can modify user roles."}, refused.text + assert _user_row(user_id) == before_row + + +@pytest.mark.parametrize("role", ("proxy_admin", "proxy_admin_viewer")) +def test_new_user_with_admin_role_requires_proxy_admin(gateway: Gateway, role: str) -> None: + with gateway.scenario() as scenario: + user_id: Final = scenario.user(user_role="internal_user", max_budget=10.0) + key: Final = scenario.key(user_id=user_id, allowed_routes=list(_SELF_SERVICE_ROUTES)) + attempted_id: Final = f"integration-{uuid.uuid4().hex}" + + refused: Final = gateway.request( + "POST", + "/user/new", + {"user_id": attempted_id, "user_role": role, "auto_create_key": False}, + key=key, + ) + assert refused.status_code == 403, refused.text + assert refused.json() == { + "error": { + "message": f"Only proxy admins can create administrative users (proxy_admin, proxy_admin_viewer). Attempted to create user with role: {role}. Your role: internal_user", + "type": "internal_server_error", + "param": None, + "code": "403", + } + }, refused.text + assert _user_row(attempted_id) == [] diff --git a/tests/integration/management/test_organization_lifecycle.py b/tests/integration/management/test_organization_lifecycle.py index 7b8d1d7f7e7..0193bf517b2 100644 --- a/tests/integration/management/test_organization_lifecycle.py +++ b/tests/integration/management/test_organization_lifecycle.py @@ -1,9 +1,10 @@ import uuid from concurrent.futures import ThreadPoolExecutor +from hashlib import sha256 from typing import Final import httpx -from integration._support.client import Gateway, object_value, string_value +from integration._support.client import Gateway, eventually, object_value, string_value from integration._support.database import read_rows from pydantic import JsonValue @@ -87,3 +88,150 @@ def test_member_role_update_and_removal_persist_and_read_back(gateway: Gateway) assert removed.status_code == 200, removed.text assert _membership_rows(organization_id) == [] assert _listed(gateway, organization_id)["members"] == [] + + +def _observed(upstream: httpx.Client) -> list[dict[str, JsonValue]]: + response: Final = upstream.get("/__observations") + response.raise_for_status() + requests: Final = response.json()["requests"] + assert isinstance(requests, list) + return requests + + +def _chat(proxy: Gateway, model: str, key: str, text: str) -> httpx.Response: + return proxy.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": text}]}, + key=key, + ) + + +def _observation(text: str) -> dict[str, JsonValue]: + return { + "path": "/v1/chat/completions", + "authorization": "Bearer integration-provider-key", + "body": {"messages": [{"role": "user", "content": text}], "model": "gpt-4o-mini"}, + "method": "POST", + "api_key": "", + } + + +def _token_not_found_error(key: str) -> dict[str, JsonValue]: + hashed: Final = sha256(key.encode()).hexdigest() + return { + "error": { + "message": ( + "Authentication Error, Invalid proxy server token passed. " + f"Received API Key = sk-...{key[-4:]}, Key Hash (Token) ={hashed}. " + "Unable to find token in cache or `LiteLLM_VerificationTokenTable`" + ), + "type": "token_not_found_in_db", + "param": "key", + "code": "401", + } + } + + +def _new_key(gateway: Gateway, **fields: JsonValue) -> str: + created: Final = gateway.post("/key/generate", fields) + return string_value(created["key"]) + + +def _key_rows(*hashes: str) -> list[dict[str, JsonValue]]: + return read_rows( + 'SELECT token FROM "LiteLLM_VerificationToken" WHERE token = ANY(%s)', + (list(hashes),), + ) + + +def test_organization_delete_revokes_org_and_team_keys_on_gateway_and_peer(gateway: Gateway, peer: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model() + created_org: Final = gateway.post( + "/organization/new", {"organization_alias": f"integration-{uuid.uuid4().hex}"} + ) + scenario.cleanups.callback(scenario.delete_budget, string_value(created_org["budget_id"])) + organization_id: Final = string_value(created_org["organization_id"]) + other_organization_id: Final = scenario.organization() + member: Final = scenario.user() + membership: Final = gateway.request( + "POST", + "/organization/member_add", + {"organization_id": organization_id, "member": {"role": "internal_user", "user_id": member}}, + ) + assert membership.status_code == 200, membership.text + assert _membership_rows(organization_id) == [{"user_id": member, "user_role": "internal_user"}] + team_id: Final = string_value( + gateway.post( + "/team/new", + {"team_alias": f"integration-{uuid.uuid4().hex}", "organization_id": organization_id}, + )["team_id"] + ) + organization_key: Final = _new_key(gateway, organization_id=organization_id) + team_key: Final = _new_key(gateway, team_id=team_id) + other_key: Final = _new_key(gateway, organization_id=other_organization_id) + warmed_info: Final = peer.request("GET", "/organization/info", params={"organization_id": organization_id}) + assert warmed_info.status_code == 200, warmed_info.text + + assert _observed(upstream) == [] + for proxy in (gateway, peer): + for key in (organization_key, team_key, other_key): + text: Final = f"warm {uuid.uuid4().hex}" + response: Final = _chat(proxy, model, key, text) + assert response.status_code == 200, response.text + assert _observed(upstream) == [_observation(text)] + + deleted: Final = gateway.request("DELETE", "/organization/delete", {"organization_ids": [organization_id]}) + assert deleted.status_code == 200, deleted.text + + for key in (organization_key, team_key): + refused: Final = _chat(gateway, model, key, f"refused {uuid.uuid4().hex}") + assert refused.status_code == 401, refused.text + assert refused.json() == _token_not_found_error(key), refused.text + for key in (organization_key, team_key): + refused_peer: Final = eventually( + lambda k=key: _chat(peer, model, k, f"refused {uuid.uuid4().hex}"), + lambda response: response.status_code == 401, + seconds=10, + ) + assert refused_peer.json() == _token_not_found_error(key), refused_peer.text + refused_observations: Final = _observed(upstream) + assert refused_observations == [], refused_observations + for proxy in (gateway, peer): + for key in (organization_key, team_key): + refused_again: Final = _chat(proxy, model, key, f"refused {uuid.uuid4().hex}") + assert refused_again.status_code == 401, refused_again.text + assert _observed(upstream) == [] + + for proxy in (gateway, peer): + text: Final = f"served {uuid.uuid4().hex}" + response: Final = _chat(proxy, model, other_key, text) + assert response.status_code == 200, response.text + assert _observed(upstream) == [_observation(text)] + + hashed: Final = tuple(sha256(key.encode()).hexdigest() for key in (organization_key, team_key)) + assert _key_rows(*hashed) == [] + assert read_rows('SELECT team_id FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team_id,)) == [] + assert _membership_rows(organization_id) == [] + assert ( + read_rows( + 'SELECT organization_id FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', + (organization_id,), + ) + == [] + ) + info: Final = gateway.request("GET", "/organization/info", params={"organization_id": organization_id}) + assert info.status_code == 404, info.text + assert info.json() == {"detail": {"error": "Organization not found"}}, info.text + peer_info: Final = eventually( + lambda: peer.request("GET", "/organization/info", params={"organization_id": organization_id}), + lambda response: response.status_code == 404, + seconds=10, + ) + assert peer_info.json() == {"detail": {"error": "Organization not found"}}, peer_info.text + alive: Final = gateway.request("GET", "/organization/info", params={"organization_id": other_organization_id}) + assert alive.status_code == 200, alive.text diff --git a/tests/integration/management/test_user_delete_revokes_keys.py b/tests/integration/management/test_user_delete_revokes_keys.py new file mode 100644 index 00000000000..0cf2db9c3e6 --- /dev/null +++ b/tests/integration/management/test_user_delete_revokes_keys.py @@ -0,0 +1,394 @@ +import os +import uuid +from hashlib import sha256 +from typing import Final + +import httpx +import pytest +from integration._support.client import Gateway, eventually, object_value, string_value +from integration._support.database import read_rows +from pydantic import JsonValue + +from litellm.models.user import LiteLLM_UserTable +from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken + + +def _observed(upstream: httpx.Client) -> list[dict[str, JsonValue]]: + response: Final = upstream.get("/__observations") + response.raise_for_status() + requests: Final = response.json()["requests"] + assert isinstance(requests, list) + return requests + + +def _chat(proxy: Gateway, model: str, key: str, text: str) -> httpx.Response: + return proxy.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": text}]}, + key=key, + ) + + +def _observation(text: str) -> dict[str, JsonValue]: + return { + "path": "/v1/chat/completions", + "authorization": "Bearer integration-provider-key", + "body": {"messages": [{"role": "user", "content": text}], "model": "gpt-4o-mini"}, + "method": "POST", + "api_key": "", + } + + +def _token_not_found_error(key: str) -> dict[str, JsonValue]: + hashed: Final = sha256(key.encode()).hexdigest() + return { + "error": { + "message": ( + "Authentication Error, Invalid proxy server token passed. " + f"Received API Key = sk-...{key[-4:]}, Key Hash (Token) ={hashed}. " + "Unable to find token in cache or `LiteLLM_VerificationTokenTable`" + ), + "type": "token_not_found_in_db", + "param": "key", + "code": "401", + } + } + + +def _blocked_key_error() -> dict[str, JsonValue]: + return { + "error": { + "message": "Authentication Error, Key is blocked. Update via `/key/unblock` if you're an admin.", + "type": "auth_error", + "param": "None", + "code": "401", + } + } + + +def _user_rows(user_id: str) -> dict[str, list[dict[str, JsonValue]]]: + return { + "user": read_rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id=%s', (user_id,)), + "keys": read_rows('SELECT token FROM "LiteLLM_VerificationToken" WHERE user_id=%s', (user_id,)), + "team_memberships": read_rows('SELECT user_id FROM "LiteLLM_TeamMembership" WHERE user_id=%s', (user_id,)), + "org_memberships": read_rows( + 'SELECT user_id FROM "LiteLLM_OrganizationMembership" WHERE user_id=%s', (user_id,) + ), + "invitations": read_rows('SELECT id FROM "LiteLLM_InvitationLink" WHERE user_id=%s', (user_id,)), + } + + +def _new_user(gateway: Gateway, **fields: JsonValue) -> str: + created: Final = gateway.post( + "/user/new", {"user_id": f"integration-{uuid.uuid4().hex}", "auto_create_key": False, **fields} + ) + return string_value(created["user_id"]) + + +def _new_key(gateway: Gateway, **fields: JsonValue) -> str: + created: Final = gateway.post("/key/generate", fields) + return string_value(created["key"]) + + +def test_user_delete_revokes_keys_and_memberships(gateway: Gateway, peer: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model() + team_id: Final = scenario.team() + organization_id: Final = scenario.organization() + user_id: Final = _new_user(gateway) + gateway.post("/team/member_add", {"team_id": team_id, "member": {"role": "user", "user_id": user_id}}) + gateway.post( + "/organization/member_add", + {"organization_id": organization_id, "member": {"role": "internal_user", "user_id": user_id}}, + ) + keys: Final = ( + _new_key(gateway, user_id=user_id, models=[model]), + _new_key(gateway, user_id=user_id, models=[model]), + ) + invitation: Final = gateway.post("/invitation/new", {"user_id": user_id}) + assert string_value(object_value(invitation)["user_id"]) == user_id + + assert _observed(upstream) == [] + expected: Final = [] + for proxy in (gateway, peer): + for key in keys: + text: Final = f"warm {uuid.uuid4().hex}" + assert _chat(proxy, model, key, text).status_code == 200 + expected.append(_observation(text)) + assert _observed(upstream) == expected + + deleted: Final = gateway.request("POST", "/user/delete", {"user_ids": [user_id]}) + assert deleted.status_code == 200, deleted.text + assert deleted.json() == 1 + + for key in keys: + denied: Final = _chat(gateway, model, key, f"denied {uuid.uuid4().hex}") + assert denied.status_code == 401, denied.text + assert denied.json() == _token_not_found_error(key), denied.text + for key in keys: + denied_peer: Final = eventually( + lambda k=key: _chat(peer, model, k, f"denied {uuid.uuid4().hex}"), + lambda response: response.status_code == 401, + seconds=10, + ) + assert denied_peer.json() == _token_not_found_error(key), denied_peer.text + + denied_observations: Final = _observed(upstream) + assert denied_observations == [], denied_observations + for proxy in (gateway, peer): + for key in keys: + denied_again: Final = _chat(proxy, model, key, f"denied {uuid.uuid4().hex}") + assert denied_again.status_code == 401, denied_again.text + assert _observed(upstream) == [] + + assert _user_rows(user_id) == { + "user": [], + "keys": [], + "team_memberships": [], + "org_memberships": [], + "invitations": [], + } + team_row: Final = read_rows('SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id=%s', (team_id,)) + assert team_row == [ + {"members_with_roles": [{"role": "admin", "user_email": None, "user_id": "default_user_id"}]} + ] + team_info: Final = object_value(object_value(gateway.get("/team/info", {"team_id": team_id}))["team_info"]) + assert team_info["members_with_roles"] == [ + {"role": "admin", "user_alias": None, "user_id": "default_user_id", "user_email": None} + ] + + info: Final = gateway.request("GET", "/user/info", params={"user_id": user_id}) + assert info.status_code == 404, info.text + assert info.json() == { + "error": { + "message": f"User {user_id} not found", + "type": "internal_server_error", + "param": None, + "code": "404", + } + }, info.text + + +def test_org_admin_can_only_delete_users_inside_their_orgs( + gateway: Gateway, +) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model() + org_a: Final = scenario.organization() + org_b: Final = scenario.organization() + admin: Final = scenario.org_member(org_a, "org_admin") + admin_key: Final = scenario.key(user_id=admin) + + victim_b: Final = scenario.org_member(org_b, "internal_user") + victim_b_key: Final = scenario.key(user_id=victim_b, models=[model]) + victim_overlap: Final = scenario.org_member(org_a, "internal_user") + gateway.post( + "/organization/member_add", + {"organization_id": org_b, "member": {"role": "internal_user", "user_id": victim_overlap}}, + ) + victim_no_org: Final = scenario.user(user_role="internal_user") + victim_a: Final = _new_user(gateway) + gateway.post( + "/organization/member_add", + {"organization_id": org_a, "member": {"role": "internal_user", "user_id": victim_a}}, + ) + + assert _observed(upstream) == [] + refused: Final = gateway.request( + "POST", + "/user/delete", + {"user_ids": [victim_b], "organization_id": org_a}, + key=admin_key, + ) + assert refused.status_code == 403, refused.text + assert refused.json() == { + "detail": { + "error": ( + f"User {victim_b} is not within your admin scope. " + "Only PROXY_ADMIN may delete users outside your administered organizations." + ) + } + }, refused.text + assert _user_rows(victim_b)["user"] != [] + + refused_overlap: Final = gateway.request( + "POST", + "/user/delete", + {"user_ids": [victim_overlap], "organization_id": org_a}, + key=admin_key, + ) + assert refused_overlap.status_code == 403, refused_overlap.text + assert refused_overlap.json() == { + "detail": { + "error": ( + f"User {victim_overlap} is not within your admin scope. " + "Only PROXY_ADMIN may delete users outside your administered organizations." + ) + } + }, refused_overlap.text + assert _user_rows(victim_overlap)["user"] != [] + + refused_no_org: Final = gateway.request( + "POST", + "/user/delete", + {"user_ids": [victim_no_org], "organization_id": org_a}, + key=admin_key, + ) + assert refused_no_org.status_code == 403, refused_no_org.text + assert refused_no_org.json() == { + "detail": { + "error": ( + f"User {victim_no_org} is not within your admin scope. " + "Only PROXY_ADMIN may delete users outside your administered organizations." + ) + } + }, refused_no_org.text + assert _user_rows(victim_no_org)["user"] != [] + + alive: Final = f"still alive {uuid.uuid4().hex}" + served: Final = _chat(gateway, model, victim_b_key, alive) + assert served.status_code == 200, served.text + assert _observed(upstream) == [_observation(alive)] + + deleted: Final = gateway.request( + "POST", + "/user/delete", + {"user_ids": [victim_a], "organization_id": org_a}, + key=admin_key, + ) + assert deleted.status_code == 200, deleted.text + assert deleted.json() == 1 + assert _user_rows(victim_a)["user"] == [] + + +def test_scim_user_delete_revokes_keys(gateway: Gateway, peer: Gateway, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", os.environ.get("LITELLM_SALT_KEY", "sk-integration-salt")) + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model() + user_id: Final = _new_user(gateway) + remaining_user_id: Final = _new_user(gateway) + scenario.cleanups.callback(gateway.request, "POST", "/user/delete", {"user_ids": [remaining_user_id]}) + ui_token: Final = ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token( + LiteLLM_UserTable(user_id=user_id, user_role="internal_user", models=[]) + ) + team_id: Final = scenario.team( + members_with_roles=[ + {"role": "user", "user_id": user_id}, + {"role": "user", "user_id": remaining_user_id}, + ] + ) + initial_team_rows: Final = read_rows( + 'SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id=%s', + (team_id,), + ) + assert len(initial_team_rows) == 1, initial_team_rows + initial_members: Final = initial_team_rows[0]["members_with_roles"] + assert isinstance(initial_members, list), initial_team_rows + initial_member_id_roles: Final = tuple( + ( + string_value(object_value(member)["user_id"]), + string_value(object_value(member)["role"]), + ) + for member in initial_members + ) + assert (user_id, "user") in initial_member_id_roles, initial_team_rows + assert (remaining_user_id, "user") in initial_member_id_roles, initial_team_rows + expected_remaining_members: Final = tuple(member for member in initial_member_id_roles if member[0] != user_id) + organization_id: Final = scenario.organization() + membership: Final = gateway.request( + "POST", + "/organization/member_add", + {"organization_id": organization_id, "member": {"role": "internal_user", "user_id": user_id}}, + ) + assert membership.status_code == 200, membership.text + invitation: Final = gateway.post("/invitation/new", {"user_id": user_id}) + assert string_value(object_value(invitation)["user_id"]) == user_id + key: Final = _new_key(gateway, user_id=user_id, models=[model]) + scenario.cleanups.callback(gateway.request, "POST", "/key/delete", {"keys": [key]}) + hashed: Final = sha256(key.encode()).hexdigest() + + gateway_info: Final = gateway.request("GET", "/user/info", params={"user_id": user_id}) + assert gateway_info.status_code == 200, gateway_info.text + peer_info: Final = peer.request("GET", "/user/info", params={"user_id": user_id}) + assert peer_info.status_code == 200, peer_info.text + for proxy in (gateway, peer): + session_info: Final = proxy.request("GET", "/user/info", params={"user_id": user_id}, key=ui_token) + assert session_info.status_code == 200, session_info.text + assert object_value(object_value(session_info.json())["user_info"])["user_id"] == user_id, session_info.text + assert _observed(upstream) == [] + + deleted: Final = gateway.request("DELETE", f"/scim/v2/Users/{user_id}") + assert deleted.status_code == 204, deleted.text + + expected_info: Final = { + "error": { + "message": f"User {user_id} not found", + "type": "internal_server_error", + "param": None, + "code": "404", + } + } + denied_info: Final = gateway.request("GET", "/user/info", params={"user_id": user_id}) + assert denied_info.status_code == 404, denied_info.text + assert denied_info.json() == expected_info, denied_info.text + denied_peer_info: Final = eventually( + lambda: peer.request("GET", "/user/info", params={"user_id": user_id}), + lambda response: response.status_code == 404, + seconds=10, + ) + assert denied_peer_info.json() == expected_info, denied_peer_info.text + for proxy in (gateway, peer): + expired_session: Final = proxy.request("GET", "/user/info", params={"user_id": user_id}, key=ui_token) + assert expired_session.status_code == 404, expired_session.text + assert expired_session.json() == expected_info, expired_session.text + + rows: Final = _user_rows(user_id) + assert rows["user"] == [] + assert rows["team_memberships"] == [] + assert rows["org_memberships"] == [] + assert rows["invitations"] == [] + assert rows["keys"] == [{"token": hashed}] + team_rows: Final = read_rows( + 'SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id=%s', + (team_id,), + ) + assert len(team_rows) == 1, team_rows + members: Final = team_rows[0]["members_with_roles"] + assert isinstance(members, list), team_rows + member_id_roles: Final = tuple( + ( + string_value(object_value(member)["user_id"]), + string_value(object_value(member)["role"]), + ) + for member in members + ) + assert member_id_roles == expected_remaining_members, team_rows + assert ( + read_rows( + 'SELECT user_id FROM "LiteLLM_OrganizationMembership" WHERE organization_id=%s', + (organization_id,), + ) + == [] + ) + assert read_rows('SELECT blocked FROM "LiteLLM_VerificationToken" WHERE token=%s', (hashed,)) == [ + {"blocked": True} + ] + + denied: Final = _chat(gateway, model, key, f"denied {uuid.uuid4().hex}") + assert denied.status_code == 401, denied.text + assert denied.json() == _blocked_key_error(), denied.text + denied_peer: Final = _chat(peer, model, key, f"denied {uuid.uuid4().hex}") + assert denied_peer.status_code == 401, denied_peer.text + assert denied_peer.json() == _blocked_key_error(), denied_peer.text + assert _observed(upstream) == [] + assert _observed(upstream) == [] diff --git a/tests/integration/management/test_user_updates_wedged_coordination_redis.py b/tests/integration/management/test_user_updates_wedged_coordination_redis.py index 79b1f3166e3..41fefdcd3b2 100644 --- a/tests/integration/management/test_user_updates_wedged_coordination_redis.py +++ b/tests/integration/management/test_user_updates_wedged_coordination_redis.py @@ -1,4 +1,5 @@ import os +import re import time import uuid from collections.abc import Callable, Mapping @@ -7,6 +8,7 @@ from pathlib import Path from typing import Final from urllib.parse import urlsplit, urlunsplit +import httpx import psycopg import pytest from psycopg import sql @@ -16,6 +18,7 @@ from redis.client import PubSub from tests.integration._support.budget import case_timeout_seconds from tests.integration._support.client import JSON_OBJECT, Gateway, eventually, object_value, string_value +from tests.integration._support.database import read_rows from tests.integration._support.process import owned_proxy_process from tests.integration._support.redis_process import owned_redis @@ -345,3 +348,324 @@ def test_user_budget_updates_return_promptly_while_coordination_redis_is_wedged( finally: admin.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(identity))) record_property("cell_elapsed_seconds", timings) + + +_USER_BUDGET_REFUSAL: Final = re.compile( + r"^ExceededBudget: User=(?P\S+) over budget\. Spend=(?P[0-9.eE+-]+), Budget=(?P[0-9.eE+-]+)$" +) + + +def _observed(upstream: httpx.Client) -> list[dict[str, JsonValue]]: + response: Final = upstream.get("/__observations") + response.raise_for_status() + requests: Final = response.json()["requests"] + assert isinstance(requests, list) + return requests + + +def _user_chat(proxy: Gateway, model: str, key: str, text: str) -> httpx.Response: + return proxy.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": text}]}, + key=key, + ) + + +def _chat_observation(text: str) -> dict[str, JsonValue]: + return { + "path": "/v1/chat/completions", + "authorization": "Bearer integration-provider-key", + "body": {"messages": [{"role": "user", "content": text}], "model": "gpt-4o-mini"}, + "method": "POST", + "api_key": "", + } + + +def _user_row(user_id: str) -> list[dict[str, JsonValue]]: + return read_rows('SELECT models, max_budget, metadata FROM "LiteLLM_UserTable" WHERE user_id=%s', (user_id,)) + + +def _partial_user_row(user_id: str) -> list[dict[str, JsonValue]]: + return read_rows( + "SELECT models, max_budget, metadata, user_alias, rpm_limit, budget_duration, spend " + 'FROM "LiteLLM_UserTable" WHERE user_id=%s', + (user_id,), + ) + + +def _assert_user_budget_refusal(response: httpx.Response, user_id: str) -> None: + assert response.status_code == 422, response.text + assert response.json()["error"]["type"] == "budget_exceeded", response.text + match: Final = _USER_BUDGET_REFUSAL.match(response.json()["error"]["message"]) + assert match is not None, response.text + assert match.group("user") == user_id, response.text + assert float(match.group("spend")) > 0, response.text + + +def _user_model_denied_error(model: str) -> dict[str, JsonValue]: + return { + "error": { + "message": f"The requested model '{model}' is not available for this API key, or the model name is invalid. Check the models available to you and try again.", + "type": "key_model_access_denied", + "param": "model", + "code": "403", + } + } + + +def _model_user(gateway: Gateway, model: str, max_budget: float = 10.0) -> str: + created: Final = gateway.post( + "/user/new", + { + "user_id": f"integration-{uuid.uuid4().hex}", + "user_role": "internal_user", + "auto_create_key": False, + "max_budget": max_budget, + "models": [model], + "metadata": {"dept": "qa"}, + }, + ) + return string_value(created["user_id"]) + + +def _wait_for_spend(user_id: str, spend: float) -> None: + eventually( + lambda: read_rows('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=%s', (user_id,)), + lambda rows: bool(rows) and float(rows[0]["spend"]) >= spend, + seconds=70, + ) + + +def test_user_budget_update_propagates_to_gateway_and_peer(gateway: Gateway, peer: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + user_id: Final = _model_user(gateway, model) + scenario.cleanups.callback(lambda: gateway.request("POST", "/user/delete", {"user_ids": [user_id]})) + key: Final = string_value(gateway.post("/key/generate", {"user_id": user_id})["key"]) + + assert _observed(upstream) == [] + for proxy in (gateway, peer): + warm: Final = f"warm {uuid.uuid4().hex}" + assert ( + eventually( + lambda p=proxy, t=warm: _user_chat(p, model, key, t), + lambda response: response.status_code == 200, + seconds=75, + ).status_code + == 200 + ) + assert _observed(upstream) == [_chat_observation(warm)] + _wait_for_spend(user_id, 0.11) + + shrink: Final = gateway.request("POST", "/user/update", {"user_id": user_id, "max_budget": 0.01}) + assert shrink.status_code == 200, shrink.text + _assert_user_budget_refusal(_user_chat(gateway, model, key, f"denied {uuid.uuid4().hex}"), user_id) + _assert_user_budget_refusal( + eventually( + lambda: _user_chat(peer, model, key, f"denied {uuid.uuid4().hex}"), + lambda response: response.status_code == 422, + seconds=10, + ), + user_id, + ) + denied_observations: Final = _observed(upstream) + assert denied_observations == [], denied_observations + for proxy in (gateway, peer): + _assert_user_budget_refusal(_user_chat(proxy, model, key, f"denied {uuid.uuid4().hex}"), user_id) + assert _observed(upstream) == [] + + restored: Final = gateway.request("POST", "/user/update", {"user_id": user_id, "max_budget": 100.0}) + assert restored.status_code == 200, restored.text + for proxy in (gateway, peer): + text: Final = f"restored {uuid.uuid4().hex}" + assert ( + eventually( + lambda p=proxy, t=text: _user_chat(p, model, key, t), + lambda response: response.status_code == 200, + seconds=30, + ).status_code + == 200 + ) + assert _observed(upstream) == [_chat_observation(text)] + + _wait_for_spend(user_id, 0.23) + email: Final = f"{user_id}@integration.local" + gateway.post("/user/update", {"user_id": user_id, "user_email": email}) + spelled: Final = gateway.request("POST", "/user/update", {"user_email": email, "max_budget": 0.01}) + assert spelled.status_code == 200, spelled.text + _assert_user_budget_refusal(_user_chat(gateway, model, key, f"denied {uuid.uuid4().hex}"), user_id) + assert _observed(upstream) == [] + + +def test_user_models_update_with_metadata_propagates_to_gateway_and_peer(gateway: Gateway, peer: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + user_id: Final = _model_user(gateway, model) + scenario.cleanups.callback(lambda: gateway.request("POST", "/user/delete", {"user_ids": [user_id]})) + key: Final = string_value(gateway.post("/key/generate", {"user_id": user_id})["key"]) + + assert _observed(upstream) == [] + for proxy in (gateway, peer): + warm: Final = f"warm {uuid.uuid4().hex}" + assert ( + eventually( + lambda p=proxy, t=warm: _user_chat(p, model, key, t), + lambda response: response.status_code == 200, + seconds=75, + ).status_code + == 200 + ) + assert _observed(upstream) == [_chat_observation(warm)] + + narrowed: Final = gateway.request( + "POST", + "/user/update", + {"user_id": user_id, "models": ["no-default-models"], "metadata": {"dept": "restricted"}}, + ) + assert narrowed.status_code == 200, narrowed.text + denied_gateway: Final = _user_chat(gateway, model, key, f"denied {uuid.uuid4().hex}") + assert denied_gateway.status_code == 403, denied_gateway.text + assert denied_gateway.json() == _user_model_denied_error(model), denied_gateway.text + denied_peer: Final = eventually( + lambda: _user_chat(peer, model, key, f"denied {uuid.uuid4().hex}"), + lambda response: response.status_code == 403, + seconds=30, + ) + assert denied_peer.json() == _user_model_denied_error(model), denied_peer.text + model_denied_observations: Final = _observed(upstream) + assert model_denied_observations == [], model_denied_observations + for proxy in (gateway, peer): + denied_again: Final = _user_chat(proxy, model, key, f"denied {uuid.uuid4().hex}") + assert denied_again.status_code == 403, denied_again.text + assert denied_again.json() == _user_model_denied_error(model), denied_again.text + assert _observed(upstream) == [] + + restored: Final = gateway.request( + "POST", + "/user/update", + {"user_id": user_id, "models": [model], "metadata": {"dept": "qa"}}, + ) + assert restored.status_code == 200, restored.text + for proxy in (gateway, peer): + text: Final = f"restored {uuid.uuid4().hex}" + assert ( + eventually( + lambda p=proxy, t=text: _user_chat(p, model, key, t), + lambda response: response.status_code == 200, + seconds=75, + ).status_code + == 200 + ) + assert _observed(upstream) == [_chat_observation(text)] + + +def test_partial_user_update_preserves_sibling_fields(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + user_id: Final = _model_user(gateway, model) + scenario.cleanups.callback(lambda: gateway.request("POST", "/user/delete", {"user_ids": [user_id]})) + + initial: Final = gateway.request( + "POST", + "/user/update", + { + "user_id": user_id, + "user_alias": "sibling-alias", + "rpm_limit": 123, + "budget_duration": "1d", + "spend": 0.25, + }, + ) + assert initial.status_code == 200, initial.text + before: Final = _partial_user_row(user_id) + assert float(str(before[0]["spend"])) == pytest.approx(0.25), before + updated: Final = gateway.request("POST", "/user/update", {"user_id": user_id, "tpm_limit": 1234}) + assert updated.status_code == 200, updated.text + after: Final = _partial_user_row(user_id) + assert after == before + info: Final = object_value(gateway.get("/user/info", {"user_id": user_id})["user_info"]) + assert info["models"] == [model], info + assert info["max_budget"] == 10.0, info + assert info["metadata"] == {"dept": "qa"}, info + assert info["user_alias"] == "sibling-alias", info + assert info["rpm_limit"] == 123, info + assert info["budget_duration"] == "1d", info + assert info["tpm_limit"] == 1234, info + assert float(str(info["spend"])) == pytest.approx(0.25), info + assert float(str(after[0]["spend"])) == pytest.approx(0.25), after + + +def test_user_models_only_update_is_enforced_on_the_gateway(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + user_id: Final = _model_user(gateway, model, max_budget=100.0) + scenario.cleanups.callback(lambda: gateway.request("POST", "/user/delete", {"user_ids": [user_id]})) + key: Final = string_value(gateway.post("/key/generate", {"user_id": user_id})["key"]) + + assert _observed(upstream) == [] + warm: Final = f"warm {uuid.uuid4().hex}" + warmed: Final = _user_chat(gateway, model, key, warm) + assert warmed.status_code == 200, warmed.text + assert _observed(upstream) == [_chat_observation(warm)] + + narrowed: Final = gateway.request("POST", "/user/update", {"user_id": user_id, "models": ["no-default-models"]}) + assert narrowed.status_code == 200, narrowed.text + assert _observed(upstream) == [] + denied: Final = _user_chat(gateway, model, key, f"denied {uuid.uuid4().hex}") + assert denied.status_code == 403, denied.text + assert denied.json() == _user_model_denied_error(model), denied.text + assert _observed(upstream) == [] + + restored: Final = gateway.request("POST", "/user/update", {"user_id": user_id, "models": [model]}) + assert restored.status_code == 200, restored.text + text: Final = f"restored {uuid.uuid4().hex}" + served: Final = _user_chat(gateway, model, key, text) + assert served.status_code == 200, served.text + assert _observed(upstream) == [_chat_observation(text)] + + +def test_user_models_update_without_metadata_propagates_to_gateway_and_peer(gateway: Gateway, peer: Gateway) -> None: + pytest.skip("BUG: models-only /user/update leaves warmed peer serving the revoked model with 200") + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + user_id: Final = _model_user(gateway, model, max_budget=100.0) + scenario.cleanups.callback(lambda: gateway.request("POST", "/user/delete", {"user_ids": [user_id]})) + key: Final = string_value(gateway.post("/key/generate", {"user_id": user_id})["key"]) + + _observed(upstream) + for proxy in (gateway, peer): + warm: Final = f"warm {uuid.uuid4().hex}" + warmed: Final = _user_chat(proxy, model, key, warm) + assert warmed.status_code == 200, warmed.text + assert _observed(upstream) == [_chat_observation(warm)] + + narrowed: Final = gateway.request("POST", "/user/update", {"user_id": user_id, "models": ["no-default-models"]}) + assert narrowed.status_code == 200, narrowed.text + denied_gateway: Final = _user_chat(gateway, model, key, f"denied {uuid.uuid4().hex}") + assert denied_gateway.status_code == 403, denied_gateway.text + assert denied_gateway.json() == _user_model_denied_error(model), denied_gateway.text + denied_peer: Final = eventually( + lambda: _user_chat(peer, model, key, f"denied {uuid.uuid4().hex}"), + lambda response: response.status_code == 403, + seconds=10, + ) + assert denied_peer.json() == _user_model_denied_error(model), denied_peer.text + denied_observations: Final = _observed(upstream) + assert denied_observations == [], denied_observations + for proxy in (gateway, peer): + refused = _user_chat(proxy, model, key, f"denied {uuid.uuid4().hex}") + assert refused.json() == _user_model_denied_error(model), refused.text + assert _observed(upstream) == [] diff --git a/tests/integration/spend/test_end_user_budget_enforcement.py b/tests/integration/spend/test_end_user_budget_enforcement.py index de0bfcd5194..29b18f1f261 100644 --- a/tests/integration/spend/test_end_user_budget_enforcement.py +++ b/tests/integration/spend/test_end_user_budget_enforcement.py @@ -436,3 +436,203 @@ def test_key_end_user_budget_id_refuses_second_request(gateway: Gateway) -> None other_end_user_id, {"budget_id": None, "max_budget": None}, ) + + +class _Budget(BaseModel): + budget_id: str + max_budget: float | None + + +class _CustomerInfo(BaseModel): + user_id: str + blocked: bool + alias: str | None + spend: float + litellm_budget_table: _Budget | None + + +def _new_customer(gateway: Gateway, user_id: str, *, path: str = "/customer/new", **fields: JsonValue) -> None: + response: Final = gateway.request("POST", path, {"user_id": user_id, **fields}) + assert response.status_code == 200, response.text + + +def _customer_info(gateway: Gateway, user_id: str) -> _CustomerInfo: + response: Final = gateway.request("GET", "/customer/info", params={"end_user_id": user_id}) + assert response.status_code == 200, response.text + return _CustomerInfo.model_validate_json(response.content) + + +def _chat(gateway: Gateway, model: str, key: str, end_user: str, *, stream: bool = False) -> httpx.Response: + body: Final[dict[str, JsonValue]] = { + "model": model, + "messages": [{"role": "user", "content": f"end user budget {uuid.uuid4().hex}"}], + "user": end_user, + } + if stream: + body["stream"] = True + return gateway.request("POST", "/v1/chat/completions", body, key=key) + + +def _observed_requests(upstream: httpx.Client) -> list[dict[str, JsonValue]]: + response: Final = upstream.get("/__observations") + response.raise_for_status() + requests: Final = response.json()["requests"] + assert isinstance(requests, list) + return requests + + +def _expected_observation(body: dict[str, JsonValue]) -> dict[str, JsonValue]: + return { + "path": "/v1/chat/completions", + "authorization": "Bearer integration-provider-key", + "body": body, + "method": "POST", + "api_key": "", + } + + +def _sent_body(response: httpx.Response, end_user: str) -> dict[str, JsonValue]: + sent: Final = object_value(json.loads(response.request.content)) + expected: Final[dict[str, JsonValue]] = {"messages": sent["messages"], "model": "gpt-4o-mini", "user": end_user} + if sent.get("stream") is True: + expected["stream"] = True + expected["stream_options"] = {"include_usage": True} + return expected + + +def _end_user_row(user_id: str) -> list[dict[str, JsonValue]]: + return read_rows('SELECT spend, blocked FROM "LiteLLM_EndUserTable" WHERE user_id=%s', (user_id,)) + + +def _daily_rows(user_id: str) -> list[dict[str, JsonValue]]: + return read_rows( + 'SELECT model, model_group, spend, api_requests, successful_requests FROM "LiteLLM_DailyEndUserSpend" WHERE end_user_id=%s', + (user_id,), + ) + + +_REFUSAL_MESSAGE: Final = re.compile( + r"^ExceededBudget: End User=(?P\S+) over budget\. Spend=(?P[0-9.eE+-]+), Budget=(?P[0-9.eE+-]+)$" +) + + +def _assert_budget_refusal(response: httpx.Response, end_user: str, budget: float) -> None: + error: Final = object_value(response.json()["error"]) + assert error["type"] == "budget_exceeded" + assert error["param"] is None + assert error["code"] == "422" + matched: Final = _REFUSAL_MESSAGE.match(str(error["message"])) + assert matched is not None, error["message"] + assert matched.group("user") == end_user + assert float(matched.group("spend")) == pytest.approx(0.06) + assert float(matched.group("budget")) == budget + + +def test_customer_budget_refuses_calls_and_writes_daily_spend(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + _observed_requests(upstream) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + _new_customer(gateway, end_user, max_budget=0.05) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + info: Final = _customer_info(gateway, end_user) + assert info.blocked is False + assert info.litellm_budget_table is not None and info.litellm_budget_table.max_budget == 0.05, info + joined: Final = read_rows( + 'SELECT b.max_budget FROM "LiteLLM_EndUserTable" e JOIN "LiteLLM_BudgetTable" b' + " ON e.budget_id = b.budget_id WHERE e.user_id = %s", + (end_user,), + ) + assert len(joined) == 1 and float(str(joined[0]["max_budget"])) == 0.05, joined + + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model]) + served: Final = _chat(gateway, model, key, end_user) + assert served.status_code == 200, served.text + assert _observed_requests(upstream) == [_expected_observation(_sent_body(served, end_user))] + + charged: Final = eventually( + lambda: _end_user_row(end_user), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) >= 0.05, + seconds=70, + ) + assert float(str(charged[0]["spend"])) == pytest.approx(20 * 0.001 + 20 * 0.002) + daily: Final = eventually(lambda: _daily_rows(end_user), lambda rows: len(rows) == 1, seconds=70) + assert daily[0]["model"] == "openai/gpt-4o-mini" + assert daily[0]["model_group"] == model + assert float(str(daily[0]["spend"])) == pytest.approx(0.06) + assert int(str(daily[0]["api_requests"])) == 1 + assert int(str(daily[0]["successful_requests"])) == 1 + + denied: Final = _chat(gateway, model, key, end_user) + assert denied.status_code == 422, denied.text + _assert_budget_refusal(denied, end_user, 0.05) + assert _observed_requests(upstream) == [] + + +def test_customer_budget_update_takes_effect_without_restart(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + _observed_requests(upstream) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + _new_customer(gateway, end_user, path="/end_user/new", max_budget=1.0) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model]) + served: Final = _chat(gateway, model, key, end_user) + assert served.status_code == 200, served.text + assert _observed_requests(upstream) == [_expected_observation(_sent_body(served, end_user))] + + updated: Final = gateway.request("POST", "/customer/update", {"user_id": end_user, "max_budget": 0.01}) + assert updated.status_code == 200, updated.text + info: Final = _customer_info(gateway, end_user) + assert info.litellm_budget_table is not None and info.litellm_budget_table.max_budget == 0.01, info + budget_row: Final = read_rows( + 'SELECT b.max_budget FROM "LiteLLM_EndUserTable" e JOIN "LiteLLM_BudgetTable" b' + " ON e.budget_id = b.budget_id WHERE e.user_id = %s", + (end_user,), + ) + assert len(budget_row) == 1 and float(str(budget_row[0]["max_budget"])) == 0.01, budget_row + eventually( + lambda: _end_user_row(end_user), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) >= 0.05, + seconds=70, + ) + denied: Final = _chat(gateway, model, key, end_user) + assert denied.status_code == 422, denied.text + _assert_budget_refusal(denied, end_user, 0.01) + assert _observed_requests(upstream) == [] + + +def test_streamed_call_is_refused_as_json_once_over_budget(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + _observed_requests(upstream) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + _new_customer(gateway, end_user, max_budget=0.05) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model]) + served: Final = _chat(gateway, model, key, end_user, stream=True) + assert served.status_code == 200, served.text + assert served.headers["content-type"].startswith("text/event-stream") + assert 'data: {"id":"chatcmpl-' in served.text and "data: [DONE]" in served.text, served.text + assert _observed_requests(upstream) == [_expected_observation(_sent_body(served, end_user))] + eventually( + lambda: _end_user_row(end_user), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) >= 0.05, + seconds=70, + ) + + denied: Final = _chat(gateway, model, key, end_user, stream=True) + assert denied.status_code == 422, denied.text + assert denied.headers["content-type"].startswith("application/json") + assert "data:" not in denied.text, denied.text + _assert_budget_refusal(denied, end_user, 0.05) + assert _observed_requests(upstream) == [] diff --git a/tests/integration/spend/test_end_user_spend_without_proxy_user.py b/tests/integration/spend/test_end_user_spend_without_proxy_user.py index 15d77f16db4..16210438e82 100644 --- a/tests/integration/spend/test_end_user_spend_without_proxy_user.py +++ b/tests/integration/spend/test_end_user_spend_without_proxy_user.py @@ -1,9 +1,40 @@ +import json import uuid -from typing import Final +from collections.abc import Mapping +from pathlib import Path +from typing import Final, Literal +import httpx import pytest -from integration._support.client import Gateway, eventually +import yaml +from integration._support.client import Gateway, eventually, object_value from integration._support.database import read_rows +from integration._support.process import owned_proxy +from integration._support.upstream import JsonResponse, SseResponse, delete_scenario, register_scenario +from pydantic import BaseModel, JsonValue, TypeAdapter + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +class _AnthropicTextBlock(BaseModel): + type: Literal["text"] + text: str + + +class _AnthropicMessageUsage(BaseModel): + input_tokens: int + output_tokens: int + + +class _AnthropicMessageResponse(BaseModel): + id: str + type: Literal["message"] + role: Literal["assistant"] + model: str + content: list[_AnthropicTextBlock] + stop_reason: str | None + stop_sequence: str | None + usage: _AnthropicMessageUsage @pytest.mark.covers("spend.end_user.charged_when_key_has_no_user_id_and_auth_cache_is_redis") @@ -27,3 +58,521 @@ def test_end_user_spend_lands_for_key_without_user_id_when_auth_cache_is_redis(g seconds=70, ) assert float(charged[0]["spend"]) == pytest.approx(20 * 0.001 + 20 * 0.002) + + +def _observed(upstream: httpx.Client) -> list[dict[str, JsonValue]]: + response: Final = upstream.get("/__observations") + response.raise_for_status() + requests: Final = response.json()["requests"] + assert isinstance(requests, list) + return requests + + +def _observation(path: str, body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return { + "path": path, + "authorization": "Bearer integration-provider-key", + "body": dict(body), + "method": "POST", + "api_key": "", + } + + +def _new_customer(gateway: Gateway, user_id: str, **fields: JsonValue) -> None: + response: Final = gateway.request("POST", "/customer/new", {"user_id": user_id, **fields}) + assert response.status_code == 200, response.text + + +def _charged_spend(user_id: str) -> float: + charged: Final = eventually( + lambda: read_rows('SELECT spend FROM "LiteLLM_EndUserTable" WHERE user_id=%s', (user_id,)), + lambda values: len(values) == 1 and float(str(values[0]["spend"])) >= 0.05, + seconds=70, + ) + return float(str(charged[0]["spend"])) + + +def _assert_daily_row(user_id: str, model: str, upstream_model: str = "openai/gpt-4o-mini") -> None: + daily: Final = eventually( + lambda: read_rows( + 'SELECT model, model_group, spend, api_requests FROM "LiteLLM_DailyEndUserSpend" WHERE end_user_id=%s', + (user_id,), + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + assert daily[0]["model"] == upstream_model + assert daily[0]["model_group"] == model + assert float(str(daily[0]["spend"])) == pytest.approx(0.06) + assert int(str(daily[0]["api_requests"])) == 1 + + +def _assert_budget_refusal(response: httpx.Response, end_user: str) -> None: + assert response.status_code == 422, response.text + error: Final = object_value(response.json()["error"]) + assert error["type"] == "budget_exceeded" + assert error["param"] is None + assert error["code"] == "422" + assert str(error["message"]).startswith(f"ExceededBudget: End User={end_user} over budget. Spend="), error[ + "message" + ] + + +def _json_object(raw: bytes) -> dict[str, JsonValue]: + return _JSON_OBJECT.validate_json(raw) + + +_CHAT_SPELLINGS: Final = ( + pytest.param("x-litellm-end-user-id", id="header-x-litellm-end-user-id"), + pytest.param("x-litellm-customer-id", id="header-x-litellm-customer-id"), + pytest.param("litellm_metadata.user", id="body-litellm_metadata-user"), + pytest.param("metadata.user_id", id="body-metadata-user_id"), + pytest.param("litellm_metadata.user:json-string", id="body-litellm_metadata-json-string-user"), + pytest.param("metadata.user_id:json-string", id="body-metadata-json-string-user_id"), +) + + +def _chat_request( + gateway: Gateway, + model: str, + key: str, + spelling: str, + end_user: str, + text: str, + stream: bool = False, +) -> httpx.Response: + headers: Final[dict[str, str]] = {} + body: Final[dict[str, JsonValue]] = { + "model": model, + "messages": [{"role": "user", "content": text}], + } + if spelling.startswith("x-"): + headers[spelling] = end_user + elif spelling == "litellm_metadata.user": + body["litellm_metadata"] = {"user": end_user} + elif spelling == "litellm_metadata.user:json-string": + body["litellm_metadata"] = json.dumps({"user": end_user}) + elif spelling == "metadata.user_id:json-string": + body["metadata"] = json.dumps({"user_id": end_user}) + else: + body["metadata"] = {"user_id": end_user} + if stream: + body["stream"] = True + return gateway.request("POST", "/v1/chat/completions", body, key=key, headers=headers) + + +@pytest.mark.parametrize( + "spelling", + ( + pytest.param("x-litellm-end-user-id", id="header"), + pytest.param("metadata.user_id", id="metadata"), + ), +) +def test_streamed_end_user_spelling_is_charged_and_budget_enforced(gateway: Gateway, spelling: str) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + _observed(upstream) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + _new_customer(gateway, end_user, max_budget=0.05) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model]) + + text: Final = f"streamed end user spelling {uuid.uuid4().hex}" + served: Final = _chat_request(gateway, model, key, spelling, end_user, text, stream=True) + assert served.status_code == 200, served.text + assert served.headers["content-type"].startswith("text/event-stream"), served.headers + chunks: Final = tuple( + _json_object(line.removeprefix("data: ").strip().encode()) + for line in served.text.splitlines() + if line.startswith("data: ") and line.removeprefix("data: ").strip() != "[DONE]" + ) + assert chunks + assert object_value(chunks[-1]["choices"][0])["finish_reason"] == "stop" + assert "data: [DONE]" in served.text + observed: Final = _observed(upstream) + assert observed == [ + _observation( + "/v1/chat/completions", + { + "messages": [{"role": "user", "content": text}], + "model": "gpt-4o-mini", + "stream": True, + "stream_options": {"include_usage": True}, + }, + ) + ] + spend: Final = _charged_spend(end_user) + assert spend == pytest.approx(20 * 0.001 + 20 * 0.002) + _assert_daily_row(end_user, model) + + denied: Final = _chat_request( + gateway, + model, + key, + spelling, + end_user, + f"again {uuid.uuid4().hex}", + stream=True, + ) + assert denied.headers["content-type"].startswith("application/json"), denied.headers + assert "data:" not in denied.text + _assert_budget_refusal(denied, end_user) + assert _observed(upstream) == [] + + +@pytest.mark.parametrize("stream", (False, True), ids=("non-streaming", "streaming")) +def test_anthropic_messages_metadata_user_id_is_charged_and_budget_enforced(gateway: Gateway, stream: bool) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + _observed(upstream) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + _new_customer(gateway, end_user, max_budget=0.05) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + scenario_id: Final = f"sc-{uuid.uuid4().hex}" + response: Final = ( + SseResponse( + content_type="text/event-stream", + frames=( + "event: message_start\ndata: " + '{"type":"message_start","message":{"id":"msg_$REQUEST_ID","type":"message",' + '"role":"assistant","model":"claude-sonnet-5-5","content":[],"stop_reason":null,' + '"stop_sequence":null,"usage":{"input_tokens":20,"output_tokens":0}}}', + "event: content_block_start\ndata: " + '{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}', + "event: content_block_delta\ndata: " + '{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"scripted answer"}}', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}', + "event: message_delta\ndata: " + '{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},' + '"usage":{"output_tokens":20}}', + 'event: message_stop\ndata: {"type":"message_stop"}', + ), + ) + if stream + else JsonResponse( + content_type="application/json", + body={ + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "scripted answer"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 20, "output_tokens": 20}, + }, + ) + ) + handle: Final = register_scenario(scenario_id, response) + scenario.cleanups.callback(delete_scenario, handle) + model: Final = scenario.model( + model="anthropic/claude-sonnet-5-5", + api_base=handle.api_base(), + input_cost_per_token=0.001, + output_cost_per_token=0.002, + ) + key: Final = scenario.key(models=[model]) + text: Final = f"anthropic end user {uuid.uuid4().hex}" + body: Final[dict[str, JsonValue]] = { + "model": model, + "max_tokens": 64, + "messages": [{"role": "user", "content": text}], + "metadata": {"user_id": end_user}, + "stream": stream, + } + served: Final = gateway.client.request( + "POST", + "/v1/messages", + json=body, + headers={"x-api-key": key, "anthropic-version": "2023-06-01"}, + ) + assert served.status_code == 200, served.text + if stream: + assert served.headers["content-type"].startswith("text/event-stream"), served.headers + message_events: Final = tuple( + _json_object(line.removeprefix("data: ").strip().encode()) + for line in served.text.splitlines() + if line.startswith("data: ") + ) + assert tuple(str(event["type"]) for event in message_events) == ( + "message_start", + "content_block_start", + "content_block_delta", + "content_block_stop", + "message_delta", + "message_stop", + ) + else: + message: Final = _AnthropicMessageResponse.model_validate_json(served.content) + assert message.type == "message" + assert message.role == "assistant" + assert message.content == [_AnthropicTextBlock(type="text", text="scripted answer")] + assert message.stop_reason == "end_turn" + assert message.stop_sequence is None + assert message.usage == _AnthropicMessageUsage(input_tokens=20, output_tokens=20) + observed: Final = _observed(upstream) + assert observed == [ + { + "path": f"/{scenario_id}/v1/messages", + "authorization": "", + "body": { + "model": "claude-sonnet-5-5", + "max_tokens": 64, + "messages": [{"role": "user", "content": text}], + "metadata": {"user_id": end_user}, + "stream": stream, + }, + "method": "POST", + "api_key": "", + } + ] + spend: Final = _charged_spend(end_user) + assert spend == pytest.approx(20 * 0.001 + 20 * 0.002) + _assert_daily_row(end_user, model, "anthropic/claude-sonnet-5-5") + + denied: Final = gateway.client.request( + "POST", + "/v1/messages", + json={**body, "messages": [{"role": "user", "content": f"again {uuid.uuid4().hex}"}]}, + headers={"x-api-key": key, "anthropic-version": "2023-06-01"}, + ) + _assert_budget_refusal(denied, end_user) + assert _observed(upstream) == [] + + +@pytest.mark.parametrize("spelling", _CHAT_SPELLINGS) +def test_end_user_id_spelling_is_charged_and_budget_enforced(gateway: Gateway, spelling: str) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + _observed(upstream) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + _new_customer(gateway, end_user, max_budget=0.05) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model]) + + text: Final = f"end user spelling {uuid.uuid4().hex}" + served: Final = _chat_request(gateway, model, key, spelling, end_user, text) + assert served.status_code == 200, served.text + assert _observed(upstream) == [ + _observation( + "/v1/chat/completions", + {"messages": [{"role": "user", "content": text}], "model": "gpt-4o-mini"}, + ) + ] + spend: Final = _charged_spend(end_user) + assert spend == pytest.approx(20 * 0.001 + 20 * 0.002) + _assert_daily_row(end_user, model) + + denied: Final = _chat_request(gateway, model, key, spelling, end_user, f"again {uuid.uuid4().hex}") + _assert_budget_refusal(denied, end_user) + assert _observed(upstream) == [] + + +def test_safety_identifier_on_responses_is_charged_and_budget_enforced(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + _observed(upstream) + scenario_id: Final = f"sc-{uuid.uuid4().hex}" + handle: Final = register_scenario( + scenario_id, + JsonResponse( + content_type="application/json", + body={ + "id": f"resp_{scenario_id}", + "object": "response", + "created_at": 1759700000, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "scripted answer", "annotations": []}], + } + ], + "usage": { + "input_tokens": 20, + "output_tokens": 20, + "total_tokens": 40, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + ), + ) + scenario.cleanups.callback( + lambda: httpx.delete(f"{handle.control_url}/__scenarios/{handle.scenario_id}", trust_env=False) + ) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + _new_customer(gateway, end_user, max_budget=0.05) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + model: Final = scenario.model( + api_base=f"{gateway.upstream_url}/{handle.scenario_id}/v1", + input_cost_per_token=0.001, + output_cost_per_token=0.002, + ) + key: Final = scenario.key(models=[model]) + + input_message: Final[dict[str, JsonValue]] = { + "role": "user", + "content": [{"type": "input_text", "text": f"responses spelling {uuid.uuid4().hex}"}], + } + + def call() -> httpx.Response: + return gateway.request( + "POST", + "/v1/responses", + {"model": model, "input": [input_message], "safety_identifier": end_user}, + key=key, + ) + + served: Final = call() + assert served.status_code == 200, served.text + assert _observed(upstream) == [ + _observation( + f"/{handle.scenario_id}/v1/responses", + {"model": "gpt-4o-mini", "input": [input_message], "safety_identifier": end_user}, + ) + ] + spend: Final = _charged_spend(end_user) + assert spend == pytest.approx(0.06) + _assert_daily_row(end_user, model) + + denied: Final = call() + _assert_budget_refusal(denied, end_user) + assert _observed(upstream) == [] + + +def test_end_user_header_wins_over_body_user(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + _observed(upstream) + header_user: Final = f"integration-end-user-a-{uuid.uuid4().hex}" + body_user: Final = f"integration-end-user-b-{uuid.uuid4().hex}" + _new_customer(gateway, header_user, max_budget=0.05) + _new_customer(gateway, body_user, max_budget=1.0) + for end_user in (header_user, body_user): + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model]) + + text: Final = f"header precedence {uuid.uuid4().hex}" + served: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": text}], "user": body_user}, + key=key, + headers={"x-litellm-end-user-id": header_user}, + ) + assert served.status_code == 200, served.text + assert _observed(upstream) == [ + _observation( + "/v1/chat/completions", + { + "messages": [{"role": "user", "content": text}], + "model": "gpt-4o-mini", + "user": body_user, + }, + ) + ] + assert _charged_spend(header_user) == pytest.approx(0.06) + _assert_daily_row(header_user, model) + body_rows: Final = read_rows('SELECT spend FROM "LiteLLM_EndUserTable" WHERE user_id=%s', (body_user,)) + assert all(float(str(row["spend"])) == 0.0 for row in body_rows), body_rows + assert ( + read_rows( + 'SELECT id FROM "LiteLLM_DailyEndUserSpend" WHERE end_user_id=%s', + (body_user,), + ) + == [] + ) + + denied: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": f"again {uuid.uuid4().hex}"}], "user": body_user}, + key=key, + headers={"x-litellm-end-user-id": header_user}, + ) + _assert_budget_refusal(denied, header_user) + assert _observed(upstream) == [] + + +def _config_with_general_settings(tmp_path: Path, general_settings: Mapping[str, JsonValue]) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["general_settings"] = {**object_value(config["general_settings"]), **dict(general_settings)} + path: Final = tmp_path / "config.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@pytest.mark.parametrize( + ("general_settings", "injects_user_body_field"), + ( + pytest.param( + {"user_header_mappings": [{"header_name": "x-integration-end-user", "litellm_user_role": "customer"}]}, + False, + id="config-user_header_mappings", + ), + pytest.param({"user_header_name": "x-integration-end-user"}, True, id="config-user_header_name"), + ), +) +def test_configured_end_user_header_is_charged_and_budget_enforced( + gateway: Gateway, tmp_path: Path, general_settings: Mapping[str, JsonValue], injects_user_body_field: bool +) -> None: + config: Final = _config_with_general_settings(tmp_path, general_settings) + with ( + owned_proxy(gateway, tmp_path, {}, config=config) as candidate, + candidate.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + _observed(upstream) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + _new_customer(candidate, end_user, max_budget=0.05) + scenario.cleanups.callback(candidate.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model]) + headers: Final = {"x-integration-end-user": end_user} + + text: Final = f"configured header {uuid.uuid4().hex}" + served: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": text}]}, + key=key, + headers=headers, + ) + assert served.status_code == 200, served.text + expected_body: Final[dict[str, JsonValue]] = { + "messages": [{"role": "user", "content": text}], + "model": "gpt-4o-mini", + } + if injects_user_body_field: + expected_body["user"] = end_user + assert _observed(upstream) == [_observation("/v1/chat/completions", expected_body)] + assert _charged_spend(end_user) == pytest.approx(0.06) + _assert_daily_row(end_user, model) + + denied: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": f"again {uuid.uuid4().hex}"}]}, + key=key, + headers=headers, + ) + _assert_budget_refusal(denied, end_user) + assert _observed(upstream) == [] From cfa98e2f40403c3976253d25b5f5072e3da3603e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:38:17 +0000 Subject: [PATCH 14/51] test(integration): cover pass_through endpoint contracts (#44690) * test(integration): cover pass_through endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): run configured endpoint contracts on one proxy worker Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): prove pass-through streams, SDK spellings, endpoint renames and bedrock stream bodies Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover pass-through wire behavior Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover Vertex and Gemini passthrough authentication Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * Prove pass-through runtime method, credential-less Anthropic, stream barriers and Vertex strip layers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(pass_through): gate unquoted Bedrock stream targets and assert configured endpoint part types and methods after rename Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): assert pass-through query pairs so duplicate params fail Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...t_passthrough_upstream_error_visibility.py | 366 ++++++++++++- .../test_anthropic_passthrough_wire.py | 332 +++++++++++ .../test_bedrock_passthrough_stream_wire.py | 308 +++++++++++ .../test_openai_passthrough_multipart_wire.py | 276 ++++++++++ .../providers/test_vertex_passthrough_wire.py | 516 ++++++++++++++++++ .../spend/test_passthrough_request_tags.py | 362 ++++++++++++ 6 files changed, 2156 insertions(+), 4 deletions(-) create mode 100644 tests/integration/providers/test_anthropic_passthrough_wire.py create mode 100644 tests/integration/providers/test_openai_passthrough_multipart_wire.py create mode 100644 tests/integration/providers/test_vertex_passthrough_wire.py diff --git a/tests/integration/observability/test_passthrough_upstream_error_visibility.py b/tests/integration/observability/test_passthrough_upstream_error_visibility.py index 0d86440045b..83560f863ed 100644 --- a/tests/integration/observability/test_passthrough_upstream_error_visibility.py +++ b/tests/integration/observability/test_passthrough_upstream_error_visibility.py @@ -1,16 +1,20 @@ import gzip import json +import threading from hashlib import sha256 from pathlib import Path from typing import Final +from urllib.parse import parse_qsl, urlsplit import httpx import pytest import yaml +from google import genai +from google.genai import types from integration._support.client import Gateway, eventually, object_value from integration._support.database import read_rows from integration._support.process import owned_proxy_process -from integration._support.wire import Reply, Request, wire_server +from integration._support.wire import Reply, Request, Wire, wire_server from openai import AsyncOpenAI, NotFoundError, OpenAI from pydantic import JsonValue @@ -265,9 +269,7 @@ def test_gemini_streaming_passthrough_request_tags_reach_spend_log(gateway: Gate raw_tags: Final = row["request_tags"] request_tags: Final = json.loads(raw_tags) if isinstance(raw_tags, str) else raw_tags policy_tags: Final = [ - tag - for tag in request_tags - if not (isinstance(tag, str) and tag.startswith("User-Agent: ")) + tag for tag in request_tags if not (isinstance(tag, str) and tag.startswith("User-Agent: ")) ] assert policy_tags == ["gemini-js-sdk", "pass-through-endpoint"], row assert row["call_type"] == "pass_through_endpoint", row @@ -650,3 +652,359 @@ def test_budget_rejected_call_keeps_budget_normalized_error(gateway: Gateway, tm ) ) assert len(budget_rows) == 1, budget_rows + + +_GEMINI_REPLY: Final[dict[str, JsonValue]] = { + "candidates": [{"content": {"parts": [{"text": "scripted gemini"}], "role": "model"}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 2, "totalTokenCount": 5}, +} +_GEMINI_SSE: Final = (f"data: {json.dumps(_GEMINI_REPLY)}\r\n\r\n".encode(),) +_GEMINI_BODY: Final[dict[str, JsonValue]] = { + "contents": [{"role": "user", "parts": [{"text": "hi"}]}], + "systemInstruction": {"parts": [{"text": "be terse"}]}, + "generationConfig": {"temperature": 0.2, "maxOutputTokens": 32, "thinkingConfig": {"thinkingBudget": 0}}, +} +_GEMINI_SDK_BODY: Final[dict[str, JsonValue]] = { + **_GEMINI_BODY, + "systemInstruction": {"parts": [{"text": "be terse"}], "role": "user"}, +} + + +def _upstream_query(request: Request) -> tuple[str, list[tuple[str, str]]]: + split: Final = urlsplit(request.target) + return split.path, parse_qsl(split.query, keep_blank_values=True) + + +def test_gemini_passthrough_both_auth_spellings_swap_the_virtual_key_for_the_proxy_key( + gateway: Gateway, tmp_path: Path +) -> None: + def respond(request: Request) -> Reply: + if ":streamGenerateContent" in request.target: + return Reply(chunks=_GEMINI_SSE, content_type="text/event-stream") + return Reply(body=json.dumps(_GEMINI_REPLY).encode()) + + path: Final = tmp_path / "gemini-auth-spellings.yaml" + model_path: Final = "/v1beta/models/gemini-2.5-flash" + with wire_server(respond) as wire: + _gemini_config(path, wire.url) + with owned_proxy_process(gateway, tmp_path, {}, config=path, workers=2) as owned: + candidate: Final = owned.gateway + with candidate.scenario() as scenario: + key: Final = scenario.key() + sent: Final[list[httpx.Request]] = [] + client: Final = genai.Client( + api_key=key, + http_options=types.HttpOptions( + base_url=f"{str(candidate.client.base_url).rstrip('/')}/gemini", + client_args={"event_hooks": {"request": [sent.append]}, "timeout": 30, "trust_env": False}, + ), + ) + generated: Final = client.models.generate_content( + model="gemini-2.5-flash", + contents="hi", + config=types.GenerateContentConfig( + system_instruction="be terse", + temperature=0.2, + max_output_tokens=32, + thinking_config=types.ThinkingConfig(thinking_budget=0), + ), + ) + streamed_text: Final = "".join( + chunk.text or "" + for chunk in client.models.generate_content_stream( + model="gemini-2.5-flash", + contents="hi", + config=types.GenerateContentConfig( + system_instruction="be terse", + temperature=0.2, + max_output_tokens=32, + thinking_config=types.ThinkingConfig(thinking_budget=0), + ), + ) + ) + assert generated.text == "scripted gemini", generated + assert streamed_text == "scripted gemini", streamed_text + assert len(sent) == 2, sent + assert all(request.headers.get("x-goog-api-key") == key for request in sent), sent + assert parse_qsl(sent[1].url.query.decode(), keep_blank_values=True) == [("alt", "sse")], sent[1].url + query_spelling: Final = candidate.client.post( + f"/gemini{model_path}:generateContent", params={"key": key}, json=_GEMINI_BODY + ) + assert query_spelling.status_code == 200, query_spelling.text + assert query_spelling.json() == _GEMINI_REPLY, query_spelling.text + with candidate.client.stream( + "POST", + f"/gemini{model_path}:streamGenerateContent", + params={"key": key, "alt": "sse"}, + json=_GEMINI_BODY, + ) as streamed_response: + streamed: Final = streamed_response.read() + assert streamed_response.status_code == 200, streamed + assert streamed == b"".join(_GEMINI_SSE), streamed + received: Final = wire.drain() + assert [(request.method, *_upstream_query(request)) for request in received] == [ + ("POST", f"{model_path}:generateContent", [("key", "scripted")]), + ("POST", f"{model_path}:streamGenerateContent", [("alt", "sse"), ("key", "scripted")]), + ("POST", f"{model_path}:generateContent", [("key", "scripted")]), + ("POST", f"{model_path}:streamGenerateContent", [("key", "scripted"), ("alt", "sse")]), + ], received + for upstream, sdk_request in zip(received[:2], sent, strict=True): + sdk_body: Final = json.loads(sdk_request.content) + assert json.loads(upstream.body) == sdk_body, upstream.body + assert sdk_body == _GEMINI_SDK_BODY, sdk_request.content + for request in received[2:]: + assert json.loads(request.body) == _GEMINI_BODY, request.body + for request in received: + assert {name: value for name, value in request.headers.items() if key in value} == {}, ( + request.headers + ) + assert "x-goog-api-key" not in request.headers, request.headers + assert key not in request.target, request.target + + +_CHAT_REPLY: Final[dict[str, JsonValue]] = { + "id": "chatcmpl-scripted", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "scripted chat"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}, +} +_RESPONSE_OBJECT: Final[dict[str, JsonValue]] = { + "id": "resp_scripted", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "id": "msg_scripted", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "scripted response", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": { + "input_tokens": 5, + "output_tokens": 3, + "total_tokens": 8, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, +} +_RESPONSE_EVENTS: Final[tuple[dict[str, JsonValue], ...]] = ( + { + "type": "response.created", + "sequence_number": 0, + "response": {**_RESPONSE_OBJECT, "status": "in_progress", "output": []}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_scripted", + "output_index": 0, + "content_index": 0, + "delta": "scripted response", + "logprobs": [], + }, + {"type": "response.completed", "sequence_number": 2, "response": _RESPONSE_OBJECT}, +) +_RESPONSE_SSE: Final = tuple( + f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in _RESPONSE_EVENTS +) +_ASSISTANTS_PAGE: Final[dict[str, JsonValue]] = { + "object": "list", + "data": [ + { + "id": "asst_scripted", + "object": "assistant", + "created_at": 1, + "name": "scripted", + "description": None, + "model": "gpt-4o-mini", + "instructions": None, + "tools": [], + "metadata": {}, + } + ], + "first_id": "asst_scripted", + "last_id": "asst_scripted", + "has_more": False, +} + + +_RESPONSE_STREAM_GATE: Final = threading.Event() +_RESPONSE_STREAM_GATE.set() + + +def _openai_peer(request: Request) -> Reply: + if request.target.startswith("/v1/assistants"): + return Reply(body=json.dumps(_ASSISTANTS_PAGE).encode()) + if request.target == "/v1/chat/completions": + return Reply(body=json.dumps(_CHAT_REPLY).encode()) + if json.loads(request.body).get("stream") is True: + return Reply(chunks=_RESPONSE_SSE, content_type="text/event-stream", gate_after_first=_RESPONSE_STREAM_GATE) + return Reply(body=json.dumps(_RESPONSE_OBJECT).encode()) + + +def _openai_sdk(candidate: Gateway, key: str, prefix: str, sent: list[httpx.Request]) -> OpenAI: + return OpenAI( + api_key=key, + base_url=f"{str(candidate.client.base_url).rstrip('/')}{prefix}/v1", + max_retries=0, + http_client=httpx.Client(timeout=30, trust_env=False, event_hooks={"request": [sent.append]}), + ) + + +def _assert_openai_upstream_auth(request: Request, key: str) -> None: + assert request.headers["authorization"] == "Bearer scripted", request.headers + assert {name: value for name, value in request.headers.items() if key in value} == {}, request.headers + assert key.encode() not in request.body, request.body + + +def _assert_openai_sdk_request( + wire: Wire, + sent: list[httpx.Request], + key: str, + method: str, + target: str, + expected_body: dict[str, JsonValue] | None, +) -> Request: + received: Final = wire.drain() + assert [(request.method, request.target) for request in received] == [(method, target)], received + assert len(sent) == 1, sent + client_request: Final = sent[0] + upstream: Final = received[0] + _assert_openai_upstream_auth(upstream, key) + if expected_body is None: + assert client_request.content == b"", client_request.content + assert upstream.body == b"", upstream.body + else: + assert json.loads(client_request.content) == expected_body, client_request.content + assert json.loads(upstream.body) == expected_body, upstream.body + return upstream + + +_OPENAI_CHAT_BODY: Final[dict[str, JsonValue]] = { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hi"}], + "user": "end-user-1", + "store": True, + "temperature": 0.2, +} +_OPENAI_RESPONSES_BODY: Final[dict[str, JsonValue]] = { + "model": "gpt-4o-mini", + "input": "hi", + "store": False, + "user": "end-user-1", + "temperature": 0.2, +} +_OPENAI_RESPONSES_STREAM_BODY: Final[dict[str, JsonValue]] = { + **_OPENAI_RESPONSES_BODY, + "stream": True, +} + + +@pytest.mark.parametrize( + ("base_path", "surface"), + [ + pytest.param("/openai", "chat", id="openai-chat"), + pytest.param("/openai_passthrough", "chat", id="openai-passthrough-chat"), + pytest.param("/openai_passthrough", "responses", id="openai-passthrough-responses"), + pytest.param("/openai_passthrough", "responses-stream", id="openai-passthrough-responses-stream"), + pytest.param("/openai", "assistants", id="openai-assistants"), + pytest.param("/openai_passthrough", "assistants", id="openai-passthrough-assistants"), + ], +) +def test_openai_passthrough_sdk_success_on_both_aliases_forwards_the_native_request( + gateway: Gateway, tmp_path: Path, base_path: str, surface: str +) -> None: + path: Final = tmp_path / "openai-success.yaml" + with wire_server(_openai_peer) as wire: + _openai_config(path, wire.url) + with owned_proxy_process(gateway, tmp_path, {}, config=path, workers=2) as owned: + candidate: Final = owned.gateway + with candidate.scenario() as scenario: + key: Final = scenario.key() + sent: Final[list[httpx.Request]] = [] + with _openai_sdk(candidate, key, base_path, sent) as sdk: + if surface == "chat": + chat: Final = sdk.chat.completions.create( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + user="end-user-1", + store=True, + temperature=0.2, + ) + assert chat.model_dump(exclude_unset=True) == _CHAT_REPLY, chat + _assert_openai_sdk_request(wire, sent, key, "POST", "/v1/chat/completions", _OPENAI_CHAT_BODY) + elif surface == "responses": + response: Final = sdk.responses.create( + model="gpt-4o-mini", + input="hi", + store=False, + user="end-user-1", + temperature=0.2, + ) + assert response.output_text == "scripted response", response + assert response.id == "resp_scripted", response + _assert_openai_sdk_request(wire, sent, key, "POST", "/v1/responses", _OPENAI_RESPONSES_BODY) + elif surface == "responses-stream": + _RESPONSE_STREAM_GATE.clear() + try: + stream: Final = iter( + sdk.responses.create( + model="gpt-4o-mini", + input="hi", + store=False, + user="end-user-1", + temperature=0.2, + stream=True, + ) + ) + first_event: Final = next(stream) + finally: + _RESPONSE_STREAM_GATE.set() + assert first_event.type == _RESPONSE_EVENTS[0]["type"], first_event + events: Final = (first_event, *stream) + streamed_text: Final = "".join( + event.delta for event in events if event.type == "response.output_text.delta" + ) + final: Final = next(event.response for event in events if event.type == "response.completed") + assert [event.type for event in events] == [event["type"] for event in _RESPONSE_EVENTS], events + assert streamed_text == "scripted response", events + assert final.output_text == "scripted response", final + _assert_openai_sdk_request( + wire, sent, key, "POST", "/v1/responses", _OPENAI_RESPONSES_STREAM_BODY + ) + else: + assistants: Final = sdk.beta.assistants.list(limit=2, order="desc") + assert [assistant.id for assistant in assistants.data] == ["asst_scripted"], assistants + upstream: Final = _assert_openai_sdk_request( + wire, + sent, + key, + "GET", + "/v1/assistants?limit=2&order=desc", + None, + ) + assert upstream.headers["openai-beta"] == "assistants=v2", upstream.headers + + +def test_openai_passthrough_responses_metadata_reaches_upstream(gateway: Gateway, tmp_path: Path) -> None: + pytest.skip("BUG: /openai_passthrough/v1/responses drops the native metadata field before forwarding upstream") + path: Final = tmp_path / "openai-metadata.yaml" + with wire_server(_openai_peer) as wire: + _openai_config(path, wire.url) + with owned_proxy_process(gateway, tmp_path, {}, config=path, workers=2) as owned: + candidate: Final = owned.gateway + with candidate.scenario() as scenario: + sent: Final[list[httpx.Request]] = [] + with _openai_sdk(candidate, scenario.key(), "/openai_passthrough", sent) as sdk: + sdk.responses.create(model="gpt-4o-mini", input="hi", metadata={"session": "scripted"}) + received: Final = wire.drain() + assert [json.loads(request.body) for request in received] == [json.loads(sent[0].content)], received diff --git a/tests/integration/providers/test_anthropic_passthrough_wire.py b/tests/integration/providers/test_anthropic_passthrough_wire.py new file mode 100644 index 00000000000..a791eb9c2a2 --- /dev/null +++ b/tests/integration/providers/test_anthropic_passthrough_wire.py @@ -0,0 +1,332 @@ +import json +import threading +from collections.abc import Iterator +from itertools import accumulate, dropwhile +from pathlib import Path +from typing import Final + +import anthropic +import httpx +import pytest +from integration._support.client import Gateway, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +pytestmark = pytest.mark.timeout(240) + +_API_KEY: Final = "scripted-anthropic-key" +_OAUTH_TOKEN: Final = "sk-ant-oat01-scripted-proxy-token" +_MODEL: Final = "claude-sonnet-4-5" +_TEXT: Final = "scripted passthrough answer" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_SYSTEM: Final[list[JsonValue]] = [{"type": "text", "text": "You are a terse assistant"}] +_TOOLS: Final[list[JsonValue]] = [ + { + "name": "get_weather", + "description": "Look up the weather", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } +] +_THINKING: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 1024} +_MESSAGE: Final[dict[str, JsonValue]] = { + "id": "msg_scripted", + "type": "message", + "role": "assistant", + "model": _MODEL, + "content": [{"type": "text", "text": _TEXT}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 7, "output_tokens": 3}, +} +_SSE_EVENTS: Final[tuple[dict[str, JsonValue], ...]] = ( + {"type": "message_start", "message": {**_MESSAGE, "content": [], "stop_reason": None}}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": _TEXT}}, + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 3}, + }, + {"type": "message_stop"}, +) +_SSE_FRAMES: Final = tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in _SSE_EVENTS) +_CLAUDE_CODE_STREAM_GATE: Final = threading.Event() + + +def _respond(request: Request) -> Reply: + if _JSON_OBJECT.validate_json(request.body).get("stream") is True: + gate: Final = _CLAUDE_CODE_STREAM_GATE if request.headers.get("x-app") == "cli" else None + return Reply(chunks=_SSE_FRAMES, content_type="text/event-stream", gate_after_first=gate) + return Reply(body=json.dumps(_MESSAGE).encode()) + + +@pytest.fixture(scope="module") +def rig() -> Iterator[Gateway]: + with gateway_from_environment() as gateway: + yield gateway + + +@pytest.fixture(scope="module") +def wire() -> Iterator[Wire]: + with wire_server(_respond) as served: + yield served + + +@pytest.fixture(autouse=True) +def _drained(wire: Wire) -> None: + wire.drain() + + +def _config(directory: Path) -> Path: + path: Final = directory / "anthropic-passthrough.yaml" + path.write_text( + "model_list: []\n" + "general_settings:\n" + " master_key: os.environ/LITELLM_MASTER_KEY\n" + " database_url: os.environ/DATABASE_URL\n" + " store_model_in_db: true\n" + "router_settings:\n" + " disable_cooldowns: true\n" + ) + return path + + +@pytest.fixture(scope="module") +def api_key_proxy(rig: Gateway, wire: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("anthropic-passthrough-api-key") + overrides: Final = {"ANTHROPIC_API_BASE": wire.url, "ANTHROPIC_API_KEY": _API_KEY} + with owned_proxy(rig, directory, overrides, config=_config(directory), workers=2) as owned: + yield owned + + +@pytest.fixture(scope="module") +def oauth_proxy(rig: Gateway, wire: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("anthropic-passthrough-oauth") + overrides: Final = {"ANTHROPIC_API_BASE": wire.url, "ANTHROPIC_API_KEY": _OAUTH_TOKEN} + with owned_proxy(rig, directory, overrides, config=_config(directory), workers=2) as owned: + yield owned + + +@pytest.fixture(scope="module") +def credentialless_proxy(rig: Gateway, wire: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("anthropic-passthrough-credentialless") + overrides: Final = {"ANTHROPIC_API_BASE": wire.url, "ANTHROPIC_API_KEY": "", "ANTHROPIC_AUTH_TOKEN": ""} + with owned_proxy(rig, directory, overrides, config=_config(directory), workers=2) as owned: + yield owned + + +def _sdk(proxy: Gateway, key: str, sent: list[httpx.Request]) -> anthropic.Anthropic: + return anthropic.Anthropic( + api_key=key, + base_url=f"{str(proxy.client.base_url).rstrip('/')}/anthropic", + max_retries=0, + http_client=httpx.Client(timeout=30, trust_env=False, event_hooks={"request": [sent.append]}), + ) + + +def _only_upstream_request(wire: Wire) -> Request: + received: Final = wire.drain() + assert [(request.method, request.target) for request in received] == [("POST", "/v1/messages")], received + return received[0] + + +def _assert_virtual_key_absent(upstream: Request, key: str) -> None: + leaked: Final = {name: value for name, value in upstream.headers.items() if key in value} + assert leaked == {}, upstream.headers + assert key.encode() not in upstream.body, upstream.body + + +def _assert_sdk_call_reached_upstream_intact(upstream: Request, sent: httpx.Request, key: str) -> None: + assert _JSON_OBJECT.validate_json(upstream.body) == _JSON_OBJECT.validate_json(sent.content), upstream.body + assert upstream.headers["x-api-key"] == _API_KEY, upstream.headers + assert "authorization" not in upstream.headers, upstream.headers + assert upstream.headers["anthropic-version"] == sent.headers["anthropic-version"], upstream.headers + assert upstream.headers["anthropic-beta"] == "interleaved-thinking-2025-05-14", upstream.headers + _assert_virtual_key_absent(upstream, key) + + +@pytest.mark.parametrize("stream", [False, True], ids=["json", "sse"]) +def test_anthropic_sdk_call_reaches_upstream_with_full_body_and_proxy_key( + api_key_proxy: Gateway, wire: Wire, stream: bool +) -> None: + with api_key_proxy.scenario() as scenario: + key: Final = scenario.key() + sent: Final[list[httpx.Request]] = [] + with _sdk(api_key_proxy, key, sent) as sdk: + arguments: Final = { + "model": _MODEL, + "max_tokens": 2048, + "system": _SYSTEM, + "tools": _TOOLS, + "thinking": _THINKING, + "messages": [{"role": "user", "content": "weather in Paris?"}], + "extra_headers": {"anthropic-beta": "interleaved-thinking-2025-05-14"}, + } + if stream: + with sdk.messages.stream(**arguments) as events: + message = events.get_final_message() + else: + message = sdk.messages.create(**arguments) + assert message.model_dump(exclude_none=True) == anthropic.types.Message.model_validate(_MESSAGE).model_dump( + exclude_none=True + ), message + upstream: Final = _only_upstream_request(wire) + assert len(sent) == 1, sent + _assert_sdk_call_reached_upstream_intact(upstream, sent[0], key) + + +@pytest.mark.parametrize("auth_style", ["api-key", "oauth"]) +def test_anthropic_proxy_credentials_override_caller_credentials( + api_key_proxy: Gateway, oauth_proxy: Gateway, wire: Wire, auth_style: str +) -> None: + proxy: Final = api_key_proxy if auth_style == "api-key" else oauth_proxy + caller_api_key: Final = f"caller-anthropic-key-{auth_style}" + caller_authorization: Final = f"Bearer caller-anthropic-token-{auth_style}" + body: Final[dict[str, JsonValue]] = { + "model": _MODEL, + "max_tokens": 64, + "messages": [{"role": "user", "content": "hi"}], + } + with ( + proxy.scenario() as scenario, + httpx.Client(base_url=str(proxy.client.base_url), timeout=30, trust_env=False) as client, + ): + key: Final = scenario.key() + response: Final = client.post( + "/anthropic/v1/messages", + json=body, + headers={ + "x-litellm-api-key": key, + "x-api-key": caller_api_key, + "Authorization": caller_authorization, + }, + ) + assert response.status_code == 200, response.text + message: Final = anthropic.types.Message.model_validate_json(response.content) + assert message.model_dump(exclude_none=True) == anthropic.types.Message.model_validate(_MESSAGE).model_dump( + exclude_none=True + ), response.text + upstream: Final = _only_upstream_request(wire) + assert _JSON_OBJECT.validate_json(upstream.body) == body, upstream.body + assert { + name: value + for name, value in upstream.headers.items() + if caller_api_key in value or caller_authorization in value + } == {}, upstream.headers + if auth_style == "api-key": + assert upstream.headers["x-api-key"] == _API_KEY, upstream.headers + assert "authorization" not in upstream.headers, upstream.headers + else: + assert upstream.headers["authorization"] == f"Bearer {_OAUTH_TOKEN}", upstream.headers + assert "x-api-key" not in upstream.headers, upstream.headers + _assert_virtual_key_absent(upstream, key) + + +def test_claude_code_bearer_stream_merges_caller_beta_with_the_credential_beta( + oauth_proxy: Gateway, wire: Wire +) -> None: + _CLAUDE_CODE_STREAM_GATE.clear() + with oauth_proxy.scenario() as scenario: + key: Final = scenario.key() + body: Final[dict[str, JsonValue]] = { + "model": _MODEL, + "max_tokens": 2048, + "stream": True, + "system": _SYSTEM, + "tools": _TOOLS, + "thinking": _THINKING, + "messages": [{"role": "user", "content": "weather in Paris?"}], + } + with oauth_proxy.client.stream( + "POST", + "/anthropic/v1/messages", + json=body, + headers={ + "Authorization": f"Bearer {key}", + "anthropic-version": "2023-06-01", + "anthropic-beta": "claude-code-20250219,interleaved-thinking-2025-05-14", + "x-app": "cli", + }, + ) as response: + raw: Final = iter(response.iter_raw()) + try: + assert response.status_code == 200, response.status_code + assert response.headers.get("content-type", "").startswith("text/event-stream"), response.headers + first_frame: Final = next( + dropwhile( + lambda buffered: b"\n\n" not in buffered, + accumulate(raw, lambda buffered, chunk: buffered + chunk, initial=b""), + ) + ) + assert first_frame == _SSE_FRAMES[0], first_frame + finally: + _CLAUDE_CODE_STREAM_GATE.set() + streamed: Final = first_frame + b"".join(raw) + assert streamed == b"".join(_SSE_FRAMES), streamed + _CLAUDE_CODE_STREAM_GATE.clear() + upstream: Final = _only_upstream_request(wire) + assert _JSON_OBJECT.validate_json(upstream.body) == body, upstream.body + assert upstream.headers["authorization"] == f"Bearer {_OAUTH_TOKEN}", upstream.headers + assert "x-api-key" not in upstream.headers, upstream.headers + assert upstream.headers["anthropic-version"] == "2023-06-01", upstream.headers + assert upstream.headers["anthropic-beta"] == ( + "claude-code-20250219,interleaved-thinking-2025-05-14,oauth-2025-04-20" + ), upstream.headers + assert upstream.headers["x-app"] == "cli", upstream.headers + _assert_virtual_key_absent(upstream, key) + + +@pytest.mark.parametrize( + ("credential_header", "credential"), + [("authorization", "Bearer caller-anthropic-oauth-token"), ("x-api-key", "caller-anthropic-api-key")], + ids=["caller-oauth-token", "caller-api-key"], +) +def test_credentialless_proxy_forwards_the_caller_anthropic_credential_and_never_the_virtual_key( + credentialless_proxy: Gateway, wire: Wire, credential_header: str, credential: str +) -> None: + body: Final[dict[str, JsonValue]] = { + "model": _MODEL, + "max_tokens": 64, + "messages": [{"role": "user", "content": "hi"}], + } + with ( + credentialless_proxy.scenario() as scenario, + httpx.Client(base_url=str(credentialless_proxy.client.base_url), timeout=30, trust_env=False) as client, + ): + key: Final = scenario.key() + response: Final = client.post( + "/anthropic/v1/messages", json=body, headers={"x-litellm-api-key": key, credential_header: credential} + ) + assert response.status_code == 200, response.text + message: Final = anthropic.types.Message.model_validate_json(response.content) + assert message.model_dump(exclude_none=True) == anthropic.types.Message.model_validate(_MESSAGE).model_dump( + exclude_none=True + ), response.text + upstream: Final = _only_upstream_request(wire) + assert _JSON_OBJECT.validate_json(upstream.body) == body, upstream.body + assert upstream.headers[credential_header] == credential, upstream.headers + assert {"authorization", "x-api-key"} & set(upstream.headers) == {credential_header}, upstream.headers + _assert_virtual_key_absent(upstream, key) + + refused: Final = client.post("/anthropic/v1/messages", json=body, headers={"x-litellm-api-key": key}) + assert refused.status_code == 401, refused.text + assert "The LiteLLM virtual key is not forwarded to Anthropic" in refused.text, refused.text + assert wire.drain() == () + + +def test_anthropic_sdk_metadata_user_id_reaches_upstream(api_key_proxy: Gateway, wire: Wire) -> None: + pytest.skip("BUG: /anthropic/v1/messages drops the native metadata.user_id field before forwarding upstream") + with api_key_proxy.scenario() as scenario: + key: Final = scenario.key() + sent: Final[list[httpx.Request]] = [] + with _sdk(api_key_proxy, key, sent) as sdk: + sdk.messages.create( + model=_MODEL, + max_tokens=64, + metadata={"user_id": "user_scripted_session_1"}, + messages=[{"role": "user", "content": "hi"}], + ) + upstream: Final = _only_upstream_request(wire) + assert _JSON_OBJECT.validate_json(upstream.body) == _JSON_OBJECT.validate_json(sent[0].content), upstream.body diff --git a/tests/integration/providers/test_bedrock_passthrough_stream_wire.py b/tests/integration/providers/test_bedrock_passthrough_stream_wire.py index bb9bbc65f30..74d3ac7d2ef 100644 --- a/tests/integration/providers/test_bedrock_passthrough_stream_wire.py +++ b/tests/integration/providers/test_bedrock_passthrough_stream_wire.py @@ -1,10 +1,22 @@ +import base64 import json +import threading +import uuid +from collections.abc import Callable, Iterable +from pathlib import Path from typing import Final +from urllib.parse import quote, unquote +import boto3 import pytest +import yaml +from botocore.awsrequest import AWSPreparedRequest +from botocore.config import Config from integration._support.client import Gateway +from integration._support.process import owned_proxy from integration._support.upstream import _aws_event_frame from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue _MODEL_ID: Final = "anthropic.claude-sonnet-5-v1:0" _EVENT_STREAM: Final = "application/vnd.amazon.eventstream" @@ -40,3 +52,299 @@ def test_bedrock_passthrough_converse_stream_response_carries_event_stream_conte assert len(wire.drain()) == 1, response.text assert response.headers.get("content-type") == _EVENT_STREAM, dict(response.headers) assert response.content == _STREAM_BYTES, response.text + + +_ACCOUNT: Final = "123456789012" +_CONVERSE_BODY: Final[dict[str, JsonValue]] = { + "messages": [{"role": "user", "content": [{"text": "full body"}]}], + "system": [{"text": "be terse"}], + "inferenceConfig": {"maxTokens": 64, "temperature": 0.2, "stopSequences": ["END"]}, + "toolConfig": { + "tools": [ + { + "toolSpec": { + "name": "lookup", + "description": "look a value up", + "inputSchema": {"json": {"type": "object", "properties": {"q": {"type": "string"}}}}, + } + } + ], + "toolChoice": {"auto": {}}, + }, +} +_INVOKE_BODY: Final[dict[str, JsonValue]] = { + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 64, + "system": "be terse", + "messages": [{"role": "user", "content": "full body"}], +} +_CONVERSE_REPLY: Final[dict[str, JsonValue]] = { + "output": {"message": {"role": "assistant", "content": [{"text": "bedrock converse"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 5, "outputTokens": 2, "totalTokens": 7}, + "metrics": {"latencyMs": 12}, +} +_INVOKE_REPLY: Final[dict[str, JsonValue]] = { + "id": "msg_bedrock", + "type": "message", + "role": "assistant", + "model": "claude", + "content": [{"type": "text", "text": "bedrock invoke"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 5, "output_tokens": 2}, +} + + +_CONVERSE_STREAM_FRAMES: Final = tuple(_aws_event_frame(kind, payload, "sc", "u") for kind, payload in _EVENTS) +_INVOKE_STREAM_FRAMES: Final = ( + _aws_event_frame( + "chunk", + {"bytes": base64.b64encode(json.dumps(_INVOKE_REPLY).encode()).decode()}, + "sc", + "u", + ), +) + + +_GATED_STREAM_TARGETS: Final = (f"/model/{_MODEL_ID}/converse-stream", "-direct-") +_BEDROCK_STREAM_GATE: Final = threading.Event() +_BEDROCK_STREAM_GATE.set() + + +def _full_body_peer(request: Request) -> Reply: + if request.target.endswith("/converse-stream"): + gated: Final = any(marker in unquote(request.target) for marker in _GATED_STREAM_TARGETS) + return Reply( + chunks=_CONVERSE_STREAM_FRAMES, + content_type=_EVENT_STREAM, + gate_after_first=_BEDROCK_STREAM_GATE if gated else None, + ) + if request.target.endswith("/invoke-with-response-stream"): + return Reply(chunks=_INVOKE_STREAM_FRAMES, content_type=_EVENT_STREAM) + if request.target.endswith("/converse"): + return Reply(body=json.dumps(_CONVERSE_REPLY).encode()) + return Reply(body=json.dumps(_INVOKE_REPLY).encode()) + + +def _boto3_runtime(gateway: Gateway, key: str): # noqa: ANN202 - boto3 clients are untyped + client: Final = boto3.client( + "bedrock-runtime", + endpoint_url=f"{str(gateway.client.base_url).rstrip('/')}/bedrock", + region_name="us-east-1", + aws_access_key_id="AKIACALLERSIDE", + aws_secret_access_key="caller-side-secret", + config=Config(retries={"max_attempts": 1}), + ) + + def bearer(request: AWSPreparedRequest, **_: object) -> None: + request.headers["Authorization"] = f"Bearer {key}" + + client.meta.events.register("before-send.bedrock-runtime.*", bearer) + return client + + +def _events_read_before_the_upstream_finishes(open_stream: Callable[[], Iterable[object]]) -> tuple[object, ...]: + _BEDROCK_STREAM_GATE.clear() + events: Final = iter(open_stream()) + try: + first: Final = next(events) + finally: + _BEDROCK_STREAM_GATE.set() + return (first, *events) + + +def _assert_signed_with_the_deployment_key(request: Request, key: str) -> None: + authorization: Final = request.headers["authorization"] + assert authorization.startswith("AWS4-HMAC-SHA256 Credential=AKIASCRIPTEDPROVIDER/"), request.headers + assert "/us-east-1/bedrock/aws4_request" in authorization, request.headers + assert {name: value for name, value in request.headers.items() if key in value} == {}, request.headers + assert key not in request.target, request.target + + +def test_boto3_converse_invoke_and_stream_methods_forward_full_bodies_for_deployments_ids_and_profile_arns( + gateway: Gateway, tmp_path: Path +) -> None: + suffix: Final = uuid.uuid4().hex[:12] + raw_id: Final = f"anthropic.claude-scripted-{suffix}-v1:0" + profile_arn: Final = f"arn:aws:bedrock:us-east-1:{_ACCOUNT}:application-inference-profile/{suffix}" + router_name: Final = f"bedrock-router-{suffix}" + base_config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + with wire_server(_full_body_peer) as wire: + deployment_params: Final = { + "api_base": wire.url, + "aws_access_key_id": "AKIASCRIPTEDPROVIDER", + "aws_secret_access_key": "scripted-secret", + "aws_region_name": "us-east-1", + "use_in_pass_through": True, + } + config: Final = { + **base_config, + "model_list": [ + *base_config.get("model_list", []), + { + "model_name": router_name, + "litellm_params": {"model": f"bedrock/{_MODEL_ID}", **deployment_params}, + }, + { + "model_name": raw_id, + "litellm_params": {"model": f"bedrock/{raw_id}", **deployment_params}, + }, + { + "model_name": profile_arn, + "litellm_params": {"model": f"bedrock/{profile_arn}", **deployment_params}, + }, + ], + } + config_path: Final = tmp_path / "bedrock-passthrough.yaml" + config_path.write_text(yaml.safe_dump(config)) + with ( + owned_proxy(gateway, tmp_path, {}, config=config_path, workers=1) as candidate, + candidate.scenario() as scenario, + ): + key: Final = scenario.key() + runtime: Final = _boto3_runtime(candidate, key) + router_converse: Final = runtime.converse(modelId=router_name, **_CONVERSE_BODY) + profile_converse: Final = runtime.converse(modelId=profile_arn, **_CONVERSE_BODY) + raw_invoke: Final = runtime.invoke_model(modelId=raw_id, body=json.dumps(_INVOKE_BODY)) + profile_invoke: Final = runtime.invoke_model(modelId=profile_arn, body=json.dumps(_INVOKE_BODY)) + raw_converse_stream: Final = runtime.converse_stream(modelId=raw_id, **_CONVERSE_BODY) + profile_converse_stream: Final = runtime.converse_stream(modelId=profile_arn, **_CONVERSE_BODY) + raw_invoke_stream: Final = runtime.invoke_model_with_response_stream( + modelId=raw_id, body=json.dumps(_INVOKE_BODY) + ) + profile_invoke_stream: Final = runtime.invoke_model_with_response_stream( + modelId=profile_arn, body=json.dumps(_INVOKE_BODY) + ) + router_converse_stream: Final = _events_read_before_the_upstream_finishes( + lambda: runtime.converse_stream(modelId=router_name, **_CONVERSE_BODY)["stream"] + ) + router_invoke_stream: Final = runtime.invoke_model_with_response_stream( + modelId=router_name, body=json.dumps(_INVOKE_BODY) + ) + assert tuple( + {name: reply[name] for name in _CONVERSE_REPLY} for reply in (router_converse, profile_converse) + ) == (_CONVERSE_REPLY, _CONVERSE_REPLY), (router_converse, profile_converse) + assert tuple(json.loads(response["body"].read()) for response in (raw_invoke, profile_invoke)) == ( + _INVOKE_REPLY, + _INVOKE_REPLY, + ), (raw_invoke, profile_invoke) + expected_converse_events: Final = [{kind: payload} for kind, payload in _EVENTS] + assert [list(response["stream"]) for response in (raw_converse_stream, profile_converse_stream)] == [ + expected_converse_events, + expected_converse_events, + ], (raw_converse_stream, profile_converse_stream) + assert [ + [json.loads(event["chunk"]["bytes"]) for event in response["body"]] + for response in (raw_invoke_stream, profile_invoke_stream) + ] == [[_INVOKE_REPLY], [_INVOKE_REPLY]], (raw_invoke_stream, profile_invoke_stream) + assert list(router_converse_stream) == expected_converse_events, router_converse_stream + assert [json.loads(event["chunk"]["bytes"]) for event in router_invoke_stream["body"]] == [_INVOKE_REPLY] + + received: Final = wire.drain() + encoded_arn: Final = quote(profile_arn, safe=":") + assert [(request.method, request.target) for request in received] == [ + ("POST", f"/model/{_MODEL_ID}/converse"), + ("POST", f"/model/{encoded_arn}/converse"), + ("POST", f"/model/{raw_id}/invoke"), + ("POST", f"/model/{encoded_arn}/invoke"), + ("POST", f"/model/{raw_id}/converse-stream"), + ("POST", f"/model/{encoded_arn}/converse-stream"), + ("POST", f"/model/{raw_id}/invoke-with-response-stream"), + ("POST", f"/model/{encoded_arn}/invoke-with-response-stream"), + ("POST", f"/model/{_MODEL_ID}/converse-stream"), + ("POST", f"/model/{_MODEL_ID}/invoke-with-response-stream"), + ], received + assert tuple(json.loads(request.body) for request in received) == ( + _CONVERSE_BODY, + _CONVERSE_BODY, + _INVOKE_BODY, + _INVOKE_BODY, + _CONVERSE_BODY, + _CONVERSE_BODY, + _INVOKE_BODY, + _INVOKE_BODY, + _CONVERSE_BODY, + _INVOKE_BODY, + ), tuple(request.body for request in received) + for request in received: + _assert_signed_with_the_deployment_key(request, key) + + +def test_boto3_direct_raw_model_ids_forward_full_bodies(gateway: Gateway, tmp_path: Path) -> None: + direct_raw_id: Final = f"anthropic.claude-direct-{uuid.uuid4().hex[:12]}-v1:0" + with wire_server(_full_body_peer) as wire: + config_path: Final = tmp_path / "bedrock-direct.yaml" + config_path.write_text( + yaml.safe_dump( + { + "model_list": [{"model_name": "bedrock/*", "litellm_params": {"model": "bedrock/*"}}], + "general_settings": {"supported_db_objects": []}, + } + ) + ) + aws_environment: Final = { + "AWS_ACCESS_KEY_ID": "AKIASCRIPTEDPROVIDER", + "AWS_SECRET_ACCESS_KEY": "scripted-secret", + "AWS_REGION_NAME": "us-east-1", + "AWS_BEDROCK_RUNTIME_ENDPOINT": wire.url, + } + with ( + owned_proxy( + gateway, + tmp_path, + aws_environment, + config=config_path, + remove_environment=( + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_SECURITY_TOKEN", + "AWS_BEARER_TOKEN_BEDROCK", + "AWS_PROFILE", + "AWS_DEFAULT_PROFILE", + "AWS_SHARED_CREDENTIALS_FILE", + "AWS_CONFIG_FILE", + "AWS_WEB_IDENTITY_TOKEN_FILE", + "AWS_ROLE_ARN", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", + "AWS_REGION", + "AWS_DEFAULT_REGION", + ), + workers=1, + ) as candidate, + candidate.scenario() as scenario, + ): + key: Final = scenario.key() + runtime: Final = _boto3_runtime(candidate, key) + direct_converse: Final = runtime.converse(modelId=direct_raw_id, **_CONVERSE_BODY) + direct_invoke: Final = runtime.invoke_model(modelId=direct_raw_id, body=json.dumps(_INVOKE_BODY)) + direct_converse_stream: Final = _events_read_before_the_upstream_finishes( + lambda: runtime.converse_stream(modelId=direct_raw_id, **_CONVERSE_BODY)["stream"] + ) + direct_invoke_stream: Final = runtime.invoke_model_with_response_stream( + modelId=direct_raw_id, body=json.dumps(_INVOKE_BODY) + ) + assert {name: direct_converse[name] for name in _CONVERSE_REPLY} == _CONVERSE_REPLY, direct_converse + assert json.loads(direct_invoke["body"].read()) == _INVOKE_REPLY, direct_invoke + assert list(direct_converse_stream) == [{kind: payload} for kind, payload in _EVENTS], ( + direct_converse_stream + ) + assert [json.loads(event["chunk"]["bytes"]) for event in direct_invoke_stream["body"]] == [_INVOKE_REPLY] + + received: Final = wire.drain() + assert [(request.method, request.target) for request in received] == [ + ("POST", f"/model/{direct_raw_id}/converse"), + ("POST", f"/model/{direct_raw_id}/invoke"), + ("POST", f"/model/{direct_raw_id}/converse-stream"), + ("POST", f"/model/{direct_raw_id}/invoke-with-response-stream"), + ], received + assert tuple(json.loads(request.body) for request in received) == ( + _CONVERSE_BODY, + _INVOKE_BODY, + _CONVERSE_BODY, + _INVOKE_BODY, + ), tuple(request.body for request in received) + for request in received: + _assert_signed_with_the_deployment_key(request, key) diff --git a/tests/integration/providers/test_openai_passthrough_multipart_wire.py b/tests/integration/providers/test_openai_passthrough_multipart_wire.py new file mode 100644 index 00000000000..50c6fe3ac0e --- /dev/null +++ b/tests/integration/providers/test_openai_passthrough_multipart_wire.py @@ -0,0 +1,276 @@ +import io +import json +import threading +from collections.abc import Iterator +from email.parser import BytesParser +from email.policy import HTTP +from itertools import accumulate, dropwhile +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from openai import OpenAI +from pydantic import BaseModel + +pytestmark = pytest.mark.timeout(240) + +_PNG_A: Final = b"\x89PNG\r\n\x1a\nscripted-image-a" +_PNG_B: Final = b"\x89PNG\r\n\x1a\nscripted-image-b" +_MASK: Final = b"\x89PNG\r\n\x1a\nscripted-mask" +_AUDIO: Final = b"RIFF\x24\x00\x00\x00WAVEscripted-audio" +_JSONL: Final = b'{"custom_id":"1","method":"POST","url":"/v1/chat/completions","body":{}}\n' +_EDITED: Final = "c2NyaXB0ZWQtZWRpdA==" +_TRANSCRIPT: Final = "scripted transcript" +_IMAGE_STREAM_FRAMES: Final = ( + b'event: image_generation.partial_image\ndata: {"type":"image_generation.partial_image"}\n\n', + b'event: image_generation.completed\ndata: {"type":"image_generation.completed"}\n\n', +) +_IMAGE_STREAM_GATE: Final = threading.Event() +_FILE_OBJECT: Final = { + "id": "file-scripted", + "object": "file", + "bytes": len(_JSONL), + "created_at": 1, + "filename": "batch.jsonl", + "purpose": "batch", + "status": "processed", +} + +Part = tuple[str, str | None, str | None, bytes] + + +class _Image(BaseModel): + b64_json: str + + +class _ImageResponse(BaseModel): + created: int + data: tuple[_Image, ...] + + +def _respond(request: Request) -> Reply: + if request.target == "/v1/audio/transcriptions": + return Reply(body=_TRANSCRIPT.encode(), content_type="text/plain") + if request.target == "/v1/files": + return Reply(body=json.dumps(_FILE_OBJECT).encode()) + if request.target == "/v1/images/edits": + parts: Final = _parts(request.headers["content-type"], request.body) + if ("stream", None, None, b"true") in parts: + return Reply( + chunks=_IMAGE_STREAM_FRAMES, + content_type="text/event-stream", + gate_after_first=_IMAGE_STREAM_GATE, + ) + return Reply(body=json.dumps({"created": 1, "data": [{"b64_json": _EDITED}]}).encode()) + + +@pytest.fixture(scope="module") +def wire() -> Iterator[Wire]: + with wire_server(_respond) as served: + yield served + + +@pytest.fixture(autouse=True) +def _drained(wire: Wire) -> None: + wire.drain() + + +@pytest.fixture(scope="module") +def proxy(wire: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("openai-passthrough-multipart") + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["environment_variables"] = {"OPENAI_API_BASE": wire.url, "OPENAI_API_KEY": "scripted"} + path: Final = directory / "openai-multipart.yaml" + path.write_text(yaml.safe_dump(config)) + with gateway_from_environment() as rig, owned_proxy(rig, directory, {}, config=path, workers=2) as owned: + yield owned + + +def _parts(content_type: str, body: bytes) -> tuple[Part, ...]: + envelope: Final = f"content-type: {content_type}\r\n\r\n".encode() + body + parsed: Final = BytesParser(policy=HTTP).parsebytes(envelope) + assert parsed.is_multipart(), content_type + return tuple( + ( + str(part.get_param("name", header="content-disposition")), + part.get_filename(), + part.get_content_type() if part.get_filename() is not None else None, + part.get_payload(decode=True), + ) + for part in parsed.iter_parts() + ) + + +def _only_upstream(wire: Wire, target: str) -> Request: + received: Final = wire.drain() + assert [(request.method, request.target) for request in received] == [("POST", target)], received + upstream: Final = received[0] + assert upstream.headers["authorization"] == "Bearer scripted", upstream.headers + return upstream + + +def _sdk(proxy: Gateway, key: str, prefix: str) -> OpenAI: + return OpenAI( + api_key=key, + base_url=f"{str(proxy.client.base_url).rstrip('/')}{prefix}/v1", + max_retries=0, + http_client=httpx.Client(timeout=30, trust_env=False), + ) + + +def _image_edit_files( + stream: bool, image_field: str, mask_field: str +) -> list[tuple[str, tuple[str | None, str | bytes] | tuple[str, bytes, str]]]: + return [ + ("model", (None, "gpt-image-1")), + ("prompt", (None, "make them match")), + ("n", (None, "2")), + ("stream", (None, "true" if stream else "false")), + (image_field, ("a.png", _PNG_A, "image/png")), + (image_field, ("b.png", _PNG_B, "image/png")), + (mask_field, ("m.png", _MASK, "image/png")), + ] + + +def _expected_image_edit_parts(stream: bool, image_field: str, mask_field: str) -> tuple[Part, ...]: + return ( + ("model", None, None, b"gpt-image-1"), + ("prompt", None, None, b"make them match"), + ("n", None, None, b"2"), + ("stream", None, None, b"true" if stream else b"false"), + (image_field, "a.png", "image/png", _PNG_A), + (image_field, "b.png", "image/png", _PNG_B), + (mask_field, "m.png", "image/png", _MASK), + ) + + +@pytest.mark.parametrize( + ("image_field", "mask_field"), + [("image[]", "mask"), ("image", "mask"), ("image[]", "mask[]")], + ids=["bracketed-images", "plain-repeated-images", "bracketed-mask"], +) +def test_curl_image_edit_forwards_repeated_image_parts_mask_and_string_typed_fields( + proxy: Gateway, wire: Wire, image_field: str, mask_field: str +) -> None: + with proxy.scenario() as scenario: + response: Final = proxy.client.post( + "/openai/v1/images/edits", + headers={"Authorization": f"Bearer {scenario.key()}"}, + files=_image_edit_files(False, image_field, mask_field), + ) + assert response.status_code == 200, response.text + assert _ImageResponse.model_validate_json(response.content).data == (_Image(b64_json=_EDITED),), response.text + upstream: Final = _only_upstream(wire, "/v1/images/edits") + assert _parts(upstream.headers["content-type"], upstream.body) == _expected_image_edit_parts( + False, image_field, mask_field + ), upstream.body + + +def test_curl_streaming_image_edit_relays_each_sse_frame_as_the_upstream_sends_it(proxy: Gateway, wire: Wire) -> None: + pytest.skip("BUG: multipart image edits with stream=true do not relay the SSE response incrementally") + _IMAGE_STREAM_GATE.clear() + with ( + proxy.scenario() as scenario, + proxy.client.stream( + "POST", + "/openai/v1/images/edits", + headers={"Authorization": f"Bearer {scenario.key()}"}, + files=_image_edit_files(True, "image[]", "mask"), + ) as response, + ): + raw: Final = iter(response.iter_raw()) + try: + assert response.status_code == 200, response.status_code + assert response.headers.get("content-type") == "text/event-stream", response.headers + first_frame: Final = next( + dropwhile( + lambda buffered: b"\n\n" not in buffered, + accumulate(raw, lambda buffered, chunk: buffered + chunk, initial=b""), + ) + ) + assert first_frame == _IMAGE_STREAM_FRAMES[0], first_frame + finally: + _IMAGE_STREAM_GATE.set() + streamed: Final = first_frame + b"".join(raw) + assert streamed == b"".join(_IMAGE_STREAM_FRAMES), streamed + upstream: Final = _only_upstream(wire, "/v1/images/edits") + assert _parts(upstream.headers["content-type"], upstream.body) == _expected_image_edit_parts( + True, "image[]", "mask" + ) + + +def test_sdk_image_edit_with_two_images_reaches_upstream_as_two_file_parts(proxy: Gateway, wire: Wire) -> None: + with proxy.scenario() as scenario, _sdk(proxy, scenario.key(), "/openai") as sdk: + edited: Final = sdk.images.edit( + model="gpt-image-1", + image=[("a.png", io.BytesIO(_PNG_A), "image/png"), ("b.png", io.BytesIO(_PNG_B), "image/png")], + mask=("m.png", io.BytesIO(_MASK), "image/png"), + prompt="make them match", + n=2, + ) + assert [image.b64_json for image in edited.data or []] == [_EDITED], edited + upstream: Final = _only_upstream(wire, "/v1/images/edits") + assert _parts(upstream.headers["content-type"], upstream.body) == ( + ("prompt", None, None, b"make them match"), + ("model", None, None, b"gpt-image-1"), + ("n", None, None, b"2"), + ("image[]", "a.png", "image/png", _PNG_A), + ("image[]", "b.png", "image/png", _PNG_B), + ("mask", "m.png", "image/png", _MASK), + ), upstream.body + + +def test_sdk_transcription_and_file_upload_forward_the_file_and_its_text_fields(proxy: Gateway, wire: Wire) -> None: + with proxy.scenario() as scenario: + key: Final = scenario.key() + with _sdk(proxy, key, "/openai") as sdk: + transcript: Final = sdk.audio.transcriptions.create( + model="whisper-1", file=("clip.wav", io.BytesIO(_AUDIO), "audio/wav"), response_format="text" + ) + assert transcript == _TRANSCRIPT, transcript + transcription: Final = _only_upstream(wire, "/v1/audio/transcriptions") + assert _parts(transcription.headers["content-type"], transcription.body) == ( + ("model", None, None, b"whisper-1"), + ("response_format", None, None, b"text"), + ("file", "clip.wav", "audio/wav", _AUDIO), + ), transcription.body + with _sdk(proxy, key, "/openai_passthrough") as sdk: + uploaded: Final = sdk.files.create( + file=("batch.jsonl", io.BytesIO(_JSONL), "application/jsonl"), purpose="batch" + ) + assert uploaded.model_dump(exclude_unset=True) == _FILE_OBJECT, uploaded + upload: Final = _only_upstream(wire, "/v1/files") + assert _parts(upload.headers["content-type"], upload.body) == ( + ("purpose", None, None, b"batch"), + ("file", "batch.jsonl", "application/jsonl", _JSONL), + ), upload.body + + +def test_file_less_multipart_form_stays_multipart_with_every_repeated_field(proxy: Gateway, wire: Wire) -> None: + with proxy.scenario() as scenario: + response: Final = proxy.client.post( + "/openai/v1/images/edits", + headers={"Authorization": f"Bearer {scenario.key()}"}, + files=[ + ("model", (None, "gpt-image-1")), + ("prompt", (None, "no file")), + ("image_url[]", (None, "https://example.com/a.png")), + ("image_url[]", (None, "https://example.com/b.png")), + ], + ) + assert response.status_code == 200, response.text + parsed: Final = _ImageResponse.model_validate_json(response.content) + assert parsed == _ImageResponse(created=1, data=(_Image(b64_json=_EDITED),)), response.text + upstream: Final = _only_upstream(wire, "/v1/images/edits") + assert upstream.headers["content-type"].startswith("multipart/form-data; boundary="), upstream.headers + assert _parts(upstream.headers["content-type"], upstream.body) == ( + ("model", None, None, b"gpt-image-1"), + ("prompt", None, None, b"no file"), + ("image_url[]", None, None, b"https://example.com/a.png"), + ("image_url[]", None, None, b"https://example.com/b.png"), + ), upstream.body diff --git a/tests/integration/providers/test_vertex_passthrough_wire.py b/tests/integration/providers/test_vertex_passthrough_wire.py new file mode 100644 index 00000000000..eb9471bc948 --- /dev/null +++ b/tests/integration/providers/test_vertex_passthrough_wire.py @@ -0,0 +1,516 @@ +import contextlib +import datetime +import json +import socket +import socketserver +import ssl +import threading +from collections.abc import Generator, Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from itertools import accumulate, dropwhile +from pathlib import Path +from queue import Empty, SimpleQueue +from typing import Final + +import anthropic +import httpx +import pytest +import yaml +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import NameOID +from google import genai +from google.genai import types +from google.oauth2.credentials import Credentials +from integration._support.client import Gateway, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.vertex import service_account_json +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +pytestmark = pytest.mark.timeout(240) + +_PROJECT: Final = "scripted-project" +_TOKEN: Final = "scripted-vertex-token" +_CALLER_GOOGLE_TOKEN: Final = "caller-google-token" +_GEMINI_ROUTER_NAME: Final = "vertex-gemini-router" +_REGIONAL: Final = "us-central1-aiplatform.googleapis.com:443" +_GLOBAL: Final = "aiplatform.googleapis.com:443" +_BYO: Final = "us-east5-aiplatform.googleapis.com:443" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_GEMINI_REPLY: Final[dict[str, JsonValue]] = { + "candidates": [{"content": {"parts": [{"text": "scripted vertex"}], "role": "model"}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 2, "totalTokenCount": 5}, +} +_GEMINI_SSE: Final = (f"data: {json.dumps(_GEMINI_REPLY)}\r\n\r\n".encode(),) +_GEMINI_BODY: Final[dict[str, JsonValue]] = { + "contents": [{"role": "user", "parts": [{"text": "hi"}]}], + "systemInstruction": {"parts": [{"text": "be terse"}]}, + "generationConfig": {"temperature": 0.2, "maxOutputTokens": 32}, +} +_GEMINI_SDK_BODY: Final[dict[str, JsonValue]] = { + **_GEMINI_BODY, + "systemInstruction": {"parts": [{"text": "be terse"}], "role": "user"}, +} +_CLAUDE_REPLY: Final[dict[str, JsonValue]] = { + "id": "msg_scripted", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "scripted claude"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 3, "output_tokens": 2}, +} +_CLAUDE_EVENTS: Final[tuple[dict[str, JsonValue], ...]] = ( + {"type": "message_start", "message": {**_CLAUDE_REPLY, "content": [], "stop_reason": None}}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "scripted claude"}}, + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 2}, + }, + {"type": "message_stop"}, +) +_CLAUDE_SSE: Final = tuple( + f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in _CLAUDE_EVENTS +) + + +_VERTEX_STREAM_GATE: Final = threading.Event() +_VERTEX_STREAM_GATE.set() + + +@dataclass(frozen=True, slots=True) +class _Tunnel: + url: str + authorities: SimpleQueue[str] + + def drain(self) -> list[str]: + drained: Final[list[str]] = [] + with contextlib.suppress(Empty): + while True: + drained.append(self.authorities.get_nowait()) + return drained + + +def _tls_context(directory: Path) -> ssl.SSLContext: + key: Final = ec.generate_private_key(ec.SECP256R1()) + now: Final = datetime.datetime.now(datetime.UTC) + name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "aiplatform.googleapis.com")]) + certificate: Final = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(days=1)) + .not_valid_after(now + datetime.timedelta(days=1)) + .sign(key, hashes.SHA256()) + ) + certificate_file: Final = directory / "vertex.pem" + key_file: Final = directory / "vertex.key" + certificate_file.write_bytes(certificate.public_bytes(serialization.Encoding.PEM)) + key_file.write_bytes( + key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()) + ) + context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(certificate_file, key_file) + return context + + +def _pipe(source: socket.socket, sink: socket.socket) -> None: + with contextlib.suppress(OSError): + for chunk in iter(lambda: source.recv(65536), b""): + sink.sendall(chunk) + with contextlib.suppress(OSError): + sink.shutdown(socket.SHUT_WR) + + +@contextmanager +def _vertex_tunnel(destination: Wire) -> Generator[_Tunnel, None, None]: + authorities: Final[SimpleQueue[str]] = SimpleQueue() + destination_port: Final = int(destination.url.rsplit(":", 1)[1]) + + class Tunnel(socketserver.StreamRequestHandler): + rbufsize = 0 + request: socket.socket + + def handle(self) -> None: + authority: Final = self.rfile.readline().decode().split()[1] + while self.rfile.readline() not in (b"\r\n", b""): + pass + authorities.put(authority) + if authority not in (_REGIONAL, _GLOBAL, _BYO): + self.wfile.write(b"HTTP/1.1 403 Forbidden\r\ncontent-length: 0\r\n\r\n") + return + self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n") + self.request.settimeout(10) + with socket.create_connection(("127.0.0.1", destination_port), timeout=10) as upstream: + outbound: Final = threading.Thread(target=_pipe, args=(self.request, upstream)) + outbound.start() + _pipe(upstream, self.request) + outbound.join(timeout=12) + + with socketserver.ThreadingTCPServer(("127.0.0.1", 0), Tunnel) as server: + thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}) + thread.start() + try: + yield _Tunnel(f"http://127.0.0.1:{server.server_address[1]}", authorities) + finally: + server.shutdown() + thread.join(timeout=6) + + +def _vertex_peer(request: Request) -> Reply: + if "enerateContent" in request.target: + if ":streamGenerateContent" in request.target: + return Reply(chunks=_GEMINI_SSE, content_type="text/event-stream") + return Reply(body=json.dumps(_GEMINI_REPLY).encode()) + if ":streamRawPredict" in request.target or request.target.endswith(":rawPredict?alt=sse"): + return Reply(chunks=_CLAUDE_SSE, content_type="text/event-stream", gate_after_first=_VERTEX_STREAM_GATE) + return Reply(body=json.dumps(_CLAUDE_REPLY).encode()) + + +def _token_peer(request: Request) -> Reply: + assert request.target == "/_oauth/token", request.target + return Reply(body=json.dumps({"access_token": _TOKEN, "expires_in": 3600, "token_type": "Bearer"}).encode()) + + +@dataclass(frozen=True, slots=True) +class _Rig: + proxy: Gateway + vertex: Wire + tokens: Wire + tunnel: _Tunnel + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("vertex-passthrough") + with ( + wire_server(_vertex_peer, tls=_tls_context(directory)) as vertex, + wire_server(_token_peer) as tokens, + _vertex_tunnel(vertex) as tunnel, + ): + credentials: Final = service_account_json(_PROJECT, tokens.url) + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["model_list"] = [ + *config.get("model_list", []), + { + "model_name": _GEMINI_ROUTER_NAME, + "litellm_params": { + "model": "vertex_ai/gemini-2.5-flash", + "vertex_project": _PROJECT, + "vertex_location": "us-central1", + "vertex_credentials": credentials, + "use_in_pass_through": True, + }, + }, + { + "model_name": "claude-sonnet-4-5", + "litellm_params": { + "model": "vertex_ai/claude-sonnet-4-5", + "vertex_project": _PROJECT, + "vertex_location": "global", + "vertex_credentials": credentials, + "use_in_pass_through": True, + }, + }, + ] + path: Final = directory / "vertex-passthrough.yaml" + path.write_text(yaml.safe_dump(config)) + environment: Final = { + "HTTPS_PROXY": tunnel.url, + "NO_PROXY": "127.0.0.1,localhost", + "AIOHTTP_TRUST_ENV": "True", + "SSL_VERIFY": "False", + } + with ( + gateway_from_environment() as base, + owned_proxy(base, directory, environment, config=path, workers=2) as proxy, + ): + yield _Rig(proxy, vertex, tokens, tunnel) + + +@pytest.fixture(autouse=True) +def _drained(rig: _Rig) -> None: + rig.vertex.drain() + rig.tunnel.drain() + + +def _assert_minted_token_and_no_virtual_key(upstream: Request, key: str) -> None: + assert upstream.headers["authorization"] == f"Bearer {_TOKEN}", upstream.headers + assert {name: value for name, value in upstream.headers.items() if key in value} == {}, upstream.headers + assert key not in upstream.target, upstream.target + + +@pytest.mark.parametrize("alias", ["/vertex_ai", "/vertex-ai"]) +def test_google_genai_vertex_calls_reach_the_router_deployment_with_a_minted_token(rig: _Rig, alias: str) -> None: + with rig.proxy.scenario() as scenario: + key: Final = scenario.key() + sent: Final[list[httpx.Request]] = [] + client: Final = genai.Client( + vertexai=True, + project=_PROJECT, + location="us-central1", + credentials=Credentials(token=key), + http_options=types.HttpOptions( + base_url=f"{str(rig.proxy.client.base_url).rstrip('/')}{alias}", + client_args={"event_hooks": {"request": [sent.append]}, "timeout": 30, "trust_env": False}, + ), + ) + generated: Final = client.models.generate_content( + model=_GEMINI_ROUTER_NAME, + contents="hi", + config=types.GenerateContentConfig( + system_instruction="be terse", + temperature=0.2, + max_output_tokens=32, + ), + ) + streamed: Final = "".join( + chunk.text or "" + for chunk in client.models.generate_content_stream( + model=_GEMINI_ROUTER_NAME, + contents="hi", + config=types.GenerateContentConfig( + system_instruction="be terse", + temperature=0.2, + max_output_tokens=32, + ), + ) + ) + assert generated.text == "scripted vertex", generated + assert streamed == "scripted vertex", streamed + received: Final = rig.vertex.drain() + model_path: Final = f"/v1beta1/projects/{_PROJECT}/locations/us-central1/publishers/google/models" + assert [(request.method, request.target) for request in received] == [ + ("POST", f"{model_path}/gemini-2.5-flash:generateContent"), + ("POST", f"{model_path}/gemini-2.5-flash:streamGenerateContent?alt=sse"), + ], received + assert rig.tunnel.drain() == [_REGIONAL, _REGIONAL] + assert len(sent) == len(received), sent + for upstream, sdk_request in zip(received, sent, strict=True): + sdk_body: Final = _JSON_OBJECT.validate_json(sdk_request.content) + assert _JSON_OBJECT.validate_json(upstream.body) == sdk_body, upstream.body + assert sdk_body == _GEMINI_SDK_BODY, upstream.body + _assert_minted_token_and_no_virtual_key(upstream, key) + + +def test_vertex_abbreviated_beta_route_uses_router_project_and_location(rig: _Rig) -> None: + with rig.proxy.scenario() as scenario: + key: Final = scenario.key() + response: Final = rig.proxy.client.post( + f"/vertex-ai/v1beta1/publishers/google/models/{_GEMINI_ROUTER_NAME}:generateContent", + json=_GEMINI_BODY, + headers={"Authorization": f"Bearer {key}"}, + ) + assert response.status_code == 200, response.text + assert response.json() == _GEMINI_REPLY, response.text + received: Final = rig.vertex.drain() + assert [(request.method, request.target) for request in received] == [ + ( + "POST", + f"/v1beta1/projects/{_PROJECT}/locations/us-central1/publishers/google/models/" + "gemini-2.5-flash:generateContent", + ) + ], received + assert rig.tunnel.drain() == [_REGIONAL] + assert _JSON_OBJECT.validate_json(received[0].body) == _GEMINI_BODY, received[0].body + _assert_minted_token_and_no_virtual_key(received[0], key) + + +@pytest.mark.parametrize("alias", ["/vertex_ai", "/vertex-ai"]) +def test_vertex_rest_spellings_mint_token_and_add_sse_query_upstream(rig: _Rig, alias: str) -> None: + model_path: Final = f"/v1/projects/{_PROJECT}/locations/us-central1/publishers/google/models" + with rig.proxy.scenario() as scenario: + key: Final = scenario.key() + headers: Final = {"Authorization": f"Bearer {key}"} + generated: Final = rig.proxy.client.post( + f"{alias}{model_path}/{_GEMINI_ROUTER_NAME}:generateContent", json=_GEMINI_BODY, headers=headers + ) + assert generated.status_code == 200, generated.text + assert generated.json() == _GEMINI_REPLY, generated.text + with rig.proxy.client.stream( + "POST", + f"{alias}{model_path}/{_GEMINI_ROUTER_NAME}:streamGenerateContent", + json=_GEMINI_BODY, + headers=headers, + ) as streamed_response: + streamed: Final = streamed_response.read() + assert streamed_response.status_code == 200, streamed + assert streamed == b"".join(_GEMINI_SSE), streamed + received: Final = rig.vertex.drain() + assert [(request.method, request.target) for request in received] == [ + ("POST", f"{model_path}/gemini-2.5-flash:generateContent"), + ("POST", f"{model_path}/gemini-2.5-flash:streamGenerateContent?alt=sse"), + ], received + assert rig.tunnel.drain() == [_REGIONAL, _REGIONAL] + for upstream in received: + assert _JSON_OBJECT.validate_json(upstream.body) == _GEMINI_BODY, upstream.body + _assert_minted_token_and_no_virtual_key(upstream, key) + + +@pytest.mark.parametrize("alias", ["/vertex_ai", "/vertex-ai"]) +@pytest.mark.parametrize("stream", [False, True], ids=["rawPredict", "streamRawPredict"]) +def test_anthropic_vertex_sdk_partner_call_on_the_global_location_reaches_upstream_intact( + rig: _Rig, alias: str, stream: bool +) -> None: + sent: Final[list[httpx.Request]] = [] + with rig.proxy.scenario() as scenario: + key: Final = scenario.key() + client: Final = anthropic.AnthropicVertex( + base_url=f"{str(rig.proxy.client.base_url).rstrip('/')}{alias}/v1", + region="global", + project_id=_PROJECT, + access_token=key, + max_retries=0, + http_client=httpx.Client(timeout=30, trust_env=False, event_hooks={"request": [sent.append]}), + ) + arguments: Final[dict[str, JsonValue]] = { + "model": "claude-sonnet-4-5", + "max_tokens": 64, + "system": "be terse", + "messages": [{"role": "user", "content": "hi"}], + } + if stream: + with client.messages.stream(**arguments) as events: # type: ignore[arg-type] + text = events.get_final_text() + else: + text = "".join( + block.text + for block in client.messages.create(**arguments).content # type: ignore[arg-type] + if block.type == "text" + ) + client.close() + assert text == "scripted claude" + action: Final = "streamRawPredict" if stream else "rawPredict" + model_path: Final = f"/v1/projects/{_PROJECT}/locations/global/publishers/anthropic/models/claude-sonnet-4-5" + received: Final = rig.vertex.drain() + assert [(request.method, request.target) for request in received] == [ + ("POST", f"{model_path}:{action}" + ("?alt=sse" if stream else "")) + ], received + assert rig.tunnel.drain() == [_GLOBAL] + upstream: Final = received[0] + assert _JSON_OBJECT.validate_json(upstream.body) == _JSON_OBJECT.validate_json(sent[0].content), upstream.body + _assert_minted_token_and_no_virtual_key(upstream, key) + + +@pytest.mark.parametrize("alias", ["/vertex_ai", "/vertex-ai"]) +@pytest.mark.parametrize("action", ["rawPredict", "streamRawPredict"]) +def test_vertex_partner_stream_body_adds_the_sse_query_and_relays_each_frame_before_the_upstream_finishes( + rig: _Rig, alias: str, action: str +) -> None: + model_path: Final = f"/v1/projects/{_PROJECT}/locations/global/publishers/anthropic/models/claude-sonnet-4-5" + body: Final[dict[str, JsonValue]] = { + "anthropic_version": "vertex-2023-10-16", + "max_tokens": 64, + "stream": True, + "messages": [{"role": "user", "content": "hi"}], + } + with rig.proxy.scenario() as scenario: + key: Final = scenario.key() + _VERTEX_STREAM_GATE.clear() + with rig.proxy.client.stream( + "POST", f"{alias}{model_path}:{action}", json=body, headers={"Authorization": f"Bearer {key}"} + ) as response: + raw: Final = iter(response.iter_raw()) + try: + assert response.status_code == 200, response.status_code + first_frame: Final = next( + dropwhile( + lambda buffered: b"\n\n" not in buffered, + accumulate(raw, lambda buffered, chunk: buffered + chunk, initial=b""), + ), + b"", + ) + assert first_frame == _CLAUDE_SSE[0], first_frame + finally: + _VERTEX_STREAM_GATE.set() + streamed: Final = first_frame + b"".join(raw) + assert streamed == b"".join(_CLAUDE_SSE), streamed + received: Final = rig.vertex.drain() + assert [(request.method, request.target) for request in received] == [ + ("POST", f"{model_path}:{action}?alt=sse") + ], received + assert rig.tunnel.drain() == [_GLOBAL] + assert _JSON_OBJECT.validate_json(received[0].body) == body, received[0].body + _assert_minted_token_and_no_virtual_key(received[0], key) + + +def test_credential_less_vertex_project_forwards_the_caller_google_token_and_strips_the_virtual_key( + rig: _Rig, +) -> None: + model_path: Final = "/v1/projects/byo-project/locations/us-east5/publishers/anthropic/models/claude-byo" + with rig.proxy.scenario() as scenario: + key: Final = scenario.key() + body: Final[dict[str, JsonValue]] = { + "anthropic_version": "vertex-2023-10-16", + "max_tokens": 64, + "messages": [{"role": "user", "content": "hi"}], + } + response: Final = rig.proxy.client.post( + f"/vertex_ai{model_path}:rawPredict", + json=body, + headers={"Authorization": f"Bearer {key}", "x-goog-api-key": _CALLER_GOOGLE_TOKEN}, + ) + assert response.status_code == 200, response.text + assert response.json() == _CLAUDE_REPLY, response.text + received: Final = rig.vertex.drain() + assert [(request.method, request.target) for request in received] == [("POST", f"{model_path}:rawPredict")], ( + received + ) + assert rig.tunnel.drain() == [_BYO] + upstream: Final = received[0] + assert upstream.headers["x-goog-api-key"] == _CALLER_GOOGLE_TOKEN, upstream.headers + assert {name: value for name, value in upstream.headers.items() if key in value} == {}, upstream.headers + assert "authorization" not in upstream.headers, upstream.headers + assert "x-litellm-api-key" not in upstream.headers, upstream.headers + assert _JSON_OBJECT.validate_json(upstream.body) == body, upstream.body + + +@pytest.mark.parametrize("alias", ["/vertex_ai", "/vertex-ai"]) +def test_credential_less_vertex_project_forwards_the_caller_bearer_token_when_the_virtual_key_rides_in_x_litellm_api_key( + rig: _Rig, alias: str +) -> None: + model_path: Final = "/v1/projects/byo-project/locations/us-east5/publishers/anthropic/models/claude-byo" + with rig.proxy.scenario() as scenario: + key: Final = scenario.key() + second_key: Final = scenario.key() + body: Final[dict[str, JsonValue]] = { + "anthropic_version": "vertex-2023-10-16", + "max_tokens": 64, + "messages": [{"role": "user", "content": "hi"}], + } + response: Final = rig.proxy.client.post( + f"{alias}{model_path}:rawPredict", + json=body, + headers={ + "x-litellm-api-key": f"Bearer {key}", + "Authorization": f"Bearer {_CALLER_GOOGLE_TOKEN}", + "api-key": second_key, + "x-caller-echo": key, + "x-caller-trace": "trace-1", + }, + ) + assert response.status_code == 200, response.text + assert response.json() == _CLAUDE_REPLY, response.text + received: Final = rig.vertex.drain() + assert [(request.method, request.target) for request in received] == [("POST", f"{model_path}:rawPredict")], ( + received + ) + assert rig.tunnel.drain() == [_BYO] + upstream: Final = received[0] + assert upstream.headers["authorization"] == f"Bearer {_CALLER_GOOGLE_TOKEN}", upstream.headers + assert {name: value for name, value in upstream.headers.items() if key in value} == {}, upstream.headers + assert {name: value for name, value in upstream.headers.items() if second_key in value} == {}, upstream.headers + assert "x-litellm-api-key" not in upstream.headers, upstream.headers + assert "x-goog-api-key" not in upstream.headers, upstream.headers + assert upstream.headers["x-caller-trace"] == "trace-1", upstream.headers + assert key not in upstream.target, upstream.target + assert _JSON_OBJECT.validate_json(upstream.body) == body, upstream.body diff --git a/tests/integration/spend/test_passthrough_request_tags.py b/tests/integration/spend/test_passthrough_request_tags.py index e6f5e15e161..f3c0b66b20d 100644 --- a/tests/integration/spend/test_passthrough_request_tags.py +++ b/tests/integration/spend/test_passthrough_request_tags.py @@ -1,9 +1,12 @@ import json import uuid from collections.abc import Callable, Mapping +from email.parser import BytesParser +from email.policy import HTTP from hashlib import sha256 from pathlib import Path from typing import Final +from urllib.parse import parse_qsl import pytest from integration._support.client import Gateway, JsonValue, Scenario, eventually, object_value @@ -419,3 +422,362 @@ def test_anthropic_passthrough_spend_row_carries_key_team_project_tags_and_spend row ) assert _spend_logs_metadata(row) == {"cost_center": marker, "team_field": marker}, row + + +_CONTRACT_TOKEN_ENV: Final = "INTEGRATION_PASSTHROUGH_CONTRACT_TOKEN" +_CONTRACT_TOKEN: Final = "scripted-configured-endpoint-token" +_ENDPOINTS: Final = TypeAdapter(list[dict[str, JsonValue]]) + + +def _received_reply(request: Request) -> Reply: + return Reply(body=json.dumps({"received": request.target}).encode()) + + +def _create_endpoint(gateway: Gateway, scenario: Scenario, body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + response: Final = gateway.request("POST", "/config/pass_through_endpoint", dict(body)) + assert response.status_code == 200, response.text + created: Final = _ENDPOINTS.validate_python(response.json()["endpoints"]) + assert len(created) == 1, response.text + endpoint_id: Final = str(created[0]["id"]) + scenario.cleanups.callback( + lambda: gateway.request("DELETE", "/config/pass_through_endpoint", params={"endpoint_id": endpoint_id}) + ) + return created[0] + + +def _readback(gateway: Gateway, endpoint_id: str) -> list[dict[str, JsonValue]]: + response: Final = gateway.request("GET", "/config/pass_through_endpoint", params={"endpoint_id": endpoint_id}) + assert response.status_code == 200, response.text + return _ENDPOINTS.validate_python(response.json()["endpoints"]) + + +def _stored_endpoint_ids() -> list[JsonValue]: + rows: Final = read_rows('SELECT param_value FROM "LiteLLM_Config" WHERE param_name = %s', ("general_settings",)) + assert len(rows) == 1, rows + settings: Final = object_value(rows[0]["param_value"]) + stored: Final = _ENDPOINTS.validate_python(settings.get("pass_through_endpoints") or []) + return [endpoint["id"] for endpoint in stored] + + +def _stored_endpoint(endpoint_id: str) -> dict[str, JsonValue]: + rows: Final = read_rows('SELECT param_value FROM "LiteLLM_Config" WHERE param_name = %s', ("general_settings",)) + assert len(rows) == 1, rows + settings: Final = object_value(rows[0]["param_value"]) + stored: Final = _ENDPOINTS.validate_python(settings.get("pass_through_endpoints") or []) + matches: Final = [endpoint for endpoint in stored if endpoint.get("id") == endpoint_id] + assert len(matches) == 1, matches + return matches[0] + + +def _query(request: Request) -> tuple[str, tuple[tuple[str, str], ...]]: + path, _, query = request.target.partition("?") + return path, tuple(parse_qsl(query, keep_blank_values=True)) + + +def _multipart_parts(content_type: str, body: bytes) -> list[tuple[str, str | None, str | None, bytes]]: + parsed: Final = BytesParser(policy=HTTP).parsebytes(f"content-type: {content_type}\r\n\r\n".encode() + body) + assert parsed.is_multipart(), content_type + return [ + ( + str(part.get_param("name", header="content-disposition")), + part.get_filename(), + part.get("content-type"), + part.get_payload(decode=True), + ) + for part in parsed.iter_parts() + ] + + +def test_configured_endpoint_forwards_json_multipart_and_query_and_an_update_keeps_omitted_fields( + gateway: Gateway, tmp_path: Path +) -> None: + marker: Final = uuid.uuid4().hex + path: Final = f"/integration-contract-{marker}" + subpath_root: Final = f"/integration-contract-subpath-{marker}" + with ( + wire_server(_received_reply) as wire, + owned_proxy(gateway, tmp_path, {_CONTRACT_TOKEN_ENV: _CONTRACT_TOKEN}) as candidate, + candidate.scenario() as scenario, + ): + created: Final = _create_endpoint( + candidate, + scenario, + { + "path": path, + "target": f"{wire.url}/base", + "headers": {"Authorization": f"Bearer os.environ/{_CONTRACT_TOKEN_ENV}", "x-static": "static-v1"}, + "methods": ["POST"], + "default_query_params": {"api-version": "2024-01-01"}, + "auth": False, + }, + ) + _create_endpoint( + candidate, + scenario, + {"path": subpath_root, "target": f"{wire.url}/nested", "include_subpath": True, "auth": True}, + ) + endpoint_id: Final = str(created["id"]) + body: Final[dict[str, JsonValue]] = {"input": marker, "options": {"trace": marker, "stream": False}, "n": 2} + + sent_json: Final = candidate.client.post(path, params={"trace": "1"}, json=body) + assert sent_json.status_code == 200, sent_json.text + assert sent_json.json() == {"received": "/base?api-version=2024-01-01&trace=1"}, sent_json.text + overridden: Final = candidate.client.post(path, params={"api-version": "2025-02-02"}, json=body) + assert overridden.status_code == 200, overridden.text + form: Final = candidate.client.post( + path, + files=[ + ("tag", (None, "first")), + ("tag", (None, "second")), + ("file", ("one.bin", b"\x00one", "application/octet-stream")), + ("file", ("two.bin", b"\x00two", "application/octet-stream")), + ], + ) + assert form.status_code == 200, form.text + refused_verb: Final = candidate.client.get(path) + assert refused_verb.status_code == 405, refused_verb.text + nested: Final = candidate.request("POST", f"{subpath_root}/sub/route", body, params={"q": "1"}) + assert nested.status_code == 200, nested.text + before: Final = wire.drain() + assert [(request.method, *_query(request)) for request in before] == [ + ("POST", "/base", (("api-version", "2024-01-01"), ("trace", "1"))), + ("POST", "/base", (("api-version", "2025-02-02"),)), + ("POST", "/base", (("api-version", "2024-01-01"),)), + ("POST", "/nested/sub/route", (("q", "1"),)), + ], before + for request in before[:3]: + assert request.headers["authorization"] == f"Bearer {_CONTRACT_TOKEN}", request.headers + assert request.headers["x-static"] == "static-v1", request.headers + assert [json.loads(request.body) for request in (before[0], before[1], before[3])] == [body] * 3, before + assert _multipart_parts(before[2].headers["content-type"], before[2].body) == [ + ("tag", None, None, b"first"), + ("tag", None, None, b"second"), + ("file", "one.bin", "application/octet-stream", b"\x00one"), + ("file", "two.bin", "application/octet-stream", b"\x00two"), + ], before[2].body + + retargeted: Final = candidate.request( + "POST", f"/config/pass_through_endpoint/{endpoint_id}", {"path": path, "target": f"{wire.url}/v2"} + ) + assert retargeted.status_code == 200, retargeted.text + expected: Final = {**created, "target": f"{wire.url}/v2"} + assert _ENDPOINTS.validate_python(retargeted.json()["endpoints"]) == [expected], retargeted.text + retargeted_readback: Final = _readback(candidate, endpoint_id) + assert retargeted_readback == [expected] + assert _stored_endpoint(endpoint_id) == { + key: value for key, value in retargeted_readback[0].items() if key != "is_from_config" + } + after_retarget: Final = candidate.client.post(path, json=body) + assert after_retarget.status_code == 200, after_retarget.text + assert after_retarget.json() == {"received": "/v2?api-version=2024-01-01"}, after_retarget.text + retargeted_request: Final = wire.drain() + assert [(request.method, request.target) for request in retargeted_request] == [ + ("POST", "/v2?api-version=2024-01-01") + ], retargeted_request + assert retargeted_request[0].headers["authorization"] == f"Bearer {_CONTRACT_TOKEN}", retargeted_request + + locked: Final = candidate.request( + "POST", + f"/config/pass_through_endpoint/{endpoint_id}", + {"path": path, "target": f"{wire.url}/v2", "auth": True, "headers": {"x-static": "static-v2"}}, + ) + assert locked.status_code == 200, locked.text + locked_expected: Final = {**expected, "auth": True, "headers": {"x-static": "static-v2"}} + locked_readback: Final = _readback(candidate, endpoint_id) + assert locked_readback == [locked_expected] + assert _stored_endpoint(endpoint_id) == { + key: value for key, value in locked_readback[0].items() if key != "is_from_config" + } + anonymous: Final = candidate.client.post(path, json=body) + assert anonymous.status_code == 401, anonymous.text + assert wire.drain() == () + keyed: Final = candidate.request("POST", path, body) + assert keyed.status_code == 200, keyed.text + locked_request: Final = wire.drain() + assert [(request.method, request.target) for request in locked_request] == [ + ("POST", "/v2?api-version=2024-01-01") + ], locked_request + assert locked_request[0].headers["x-static"] == "static-v2", locked_request[0].headers + assert "authorization" not in locked_request[0].headers, locked_request[0].headers + refused_after_update: Final = candidate.request("GET", path) + assert refused_after_update.status_code == 405, refused_after_update.text + keyed_form: Final = candidate.client.post( + path, + headers={"Authorization": f"Bearer {candidate.key}"}, + files=[ + ("tag", (None, "first")), + ("tag", (None, "second")), + ("file", ("one.bin", b"\x00one", "application/octet-stream")), + ("file", ("two.bin", b"\x00two", "application/octet-stream")), + ], + ) + assert keyed_form.status_code == 200, keyed_form.text + locked_form: Final = wire.drain() + assert [(request.method, request.target) for request in locked_form] == [ + ("POST", "/v2?api-version=2024-01-01") + ], locked_form + assert _multipart_parts(locked_form[0].headers["content-type"], locked_form[0].body) == [ + ("tag", None, None, b"first"), + ("tag", None, None, b"second"), + ("file", "one.bin", "application/octet-stream", b"\x00one"), + ("file", "two.bin", "application/octet-stream", b"\x00two"), + ], locked_form[0].body + + +def test_renaming_a_subpath_endpoint_moves_both_routes_keeps_stored_fields_and_never_forwards_caller_headers( + gateway: Gateway, tmp_path: Path +) -> None: + marker: Final = uuid.uuid4().hex + path: Final = f"/integration-rename-{marker}" + renamed_path: Final = f"/integration-renamed-{marker}" + body: Final[dict[str, JsonValue]] = {"input": marker} + with ( + wire_server(_received_reply) as wire, + owned_proxy(gateway, tmp_path, {}, workers=1) as candidate, + candidate.scenario() as scenario, + ): + created: Final = _create_endpoint( + candidate, + scenario, + { + "path": path, + "target": f"{wire.url}/base", + "include_subpath": True, + "auth": True, + "headers": {"x-static": "v1"}, + "methods": ["POST"], + }, + ) + endpoint_id: Final = str(created["id"]) + expected: Final = { + "id": endpoint_id, + "path": path, + "target": f"{wire.url}/base", + "headers": {"x-static": "v1"}, + "default_query_params": {}, + "include_subpath": True, + "cost_per_request": 0.0, + "timeout": None, + "auth": True, + "guardrails": None, + "is_from_config": False, + "methods": ["POST"], + } + assert created == expected, created + assert _readback(candidate, endpoint_id) == [expected] + key: Final = scenario.key(allowed_passthrough_routes=[path, renamed_path]) + caller_headers: Final = {"x-caller": f"caller-{marker}"} + first: Final = candidate.request("POST", path, body, key=key, headers=caller_headers) + assert first.status_code == 200, first.text + assert first.json() == {"received": "/base"}, first.text + second: Final = candidate.request("POST", f"{path}/sub", body, key=key, headers=caller_headers) + assert second.status_code == 200, second.text + assert second.json() == {"received": "/base/sub"}, second.text + initial_requests: Final = wire.drain() + assert [(request.method, request.target) for request in initial_requests] == [ + ("POST", "/base"), + ("POST", "/base/sub"), + ], initial_requests + + updated: Final = candidate.request( + "POST", + f"/config/pass_through_endpoint/{endpoint_id}", + {"path": renamed_path, "target": f"{wire.url}/base"}, + ) + assert updated.status_code == 200, updated.text + updated_expected: Final = {**expected, "path": renamed_path} + assert _ENDPOINTS.validate_python(updated.json()["endpoints"]) == [updated_expected], updated.text + updated_readback: Final = _readback(candidate, endpoint_id) + assert updated_readback == [updated_expected] + assert _stored_endpoint(endpoint_id) == { + key: value for key, value in updated_readback[0].items() if key != "is_from_config" + } + + old_path: Final = candidate.request("POST", path, body, key=key, headers=caller_headers) + old_subpath: Final = candidate.request("POST", f"{path}/sub", body, key=key, headers=caller_headers) + assert old_path.status_code == 404, old_path.text + assert old_subpath.status_code == 404, old_subpath.text + assert wire.drain() == () + + renamed: Final = candidate.request("POST", renamed_path, body, key=key, headers=caller_headers) + assert renamed.status_code == 200, renamed.text + assert renamed.json() == {"received": "/base"}, renamed.text + renamed_subpath: Final = candidate.request("POST", f"{renamed_path}/sub", body, key=key, headers=caller_headers) + assert renamed_subpath.status_code == 200, renamed_subpath.text + assert renamed_subpath.json() == {"received": "/base/sub"}, renamed_subpath.text + refused_verbs: Final = tuple( + candidate.request("GET", renamed_path, key=key, headers=caller_headers).status_code for _ in range(4) + ) + assert refused_verbs == (405, 405, 405, 405), refused_verbs + anonymous: Final = candidate.client.post(f"{renamed_path}/sub", json=body, headers=caller_headers) + assert anonymous.status_code == 401, anonymous.text + renamed_requests: Final = wire.drain() + assert [(request.method, request.target) for request in (*initial_requests, *renamed_requests)] == [ + ("POST", "/base"), + ("POST", "/base/sub"), + ("POST", "/base"), + ("POST", "/base/sub"), + ], renamed_requests + for upstream in (*initial_requests, *renamed_requests): + assert upstream.headers["x-static"] == "v1", upstream.headers + assert "x-caller" not in upstream.headers, upstream.headers + assert {name: value for name, value in upstream.headers.items() if key in value} == {}, upstream.headers + assert key not in upstream.target, upstream.target + assert wire.drain() == () + + +def test_configured_endpoint_without_auth_serves_its_subpaths_without_a_key(gateway: Gateway) -> None: + pytest.skip("BUG: an auth=false include_subpath pass-through endpoint answers 401 on every subpath request") + marker: Final = uuid.uuid4().hex + path: Final = f"/integration-open-subpath-{marker}" + with wire_server(_received_reply) as wire, gateway.scenario() as scenario: + _create_endpoint( + gateway, scenario, {"path": path, "target": f"{wire.url}/open", "include_subpath": True, "auth": False} + ) + response: Final = gateway.client.post(f"{path}/sub", json={"input": marker}) + assert response.status_code == 200, response.text + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/open/sub")] + + +def test_deleted_configured_endpoint_leaves_readback_storage_and_runtime_and_team_get_filters( + gateway: Gateway, tmp_path: Path +) -> None: + marker: Final = uuid.uuid4().hex + with ( + wire_server(_received_reply) as wire, + owned_proxy(gateway, tmp_path, {}) as candidate, + candidate.scenario() as scenario, + ): + allowed: Final = _create_endpoint( + candidate, scenario, {"path": f"/integration-allowed-{marker}", "target": f"{wire.url}/allowed"} + ) + hidden: Final = _create_endpoint( + candidate, scenario, {"path": f"/integration-hidden-{marker}", "target": f"{wire.url}/hidden"} + ) + team: Final = scenario.team(metadata={"allowed_passthrough_routes": [allowed["path"]]}) + team_view: Final = candidate.request("GET", f"/config/pass_through_endpoint/team/{team}") + assert team_view.status_code == 200, team_view.text + assert [endpoint["path"] for endpoint in _ENDPOINTS.validate_python(team_view.json()["endpoints"])] == [ + allowed["path"] + ], team_view.text + + served: Final = candidate.request("POST", str(hidden["path"]), {"input": marker}) + assert served.status_code == 200, served.text + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/hidden")] + + deleted: Final = candidate.request( + "DELETE", "/config/pass_through_endpoint", params={"endpoint_id": str(hidden["id"])} + ) + assert deleted.status_code == 200, deleted.text + assert _ENDPOINTS.validate_python(deleted.json()["endpoints"]) == [hidden], deleted.text + assert _readback(candidate, str(hidden["id"])) == [] + listed: Final = candidate.request("GET", "/config/pass_through_endpoint") + assert listed.status_code == 200, listed.text + listed_ids: Final = [endpoint["id"] for endpoint in _ENDPOINTS.validate_python(listed.json()["endpoints"])] + assert hidden["id"] not in listed_ids and allowed["id"] in listed_ids, listed.text + stored_ids: Final = _stored_endpoint_ids() + assert hidden["id"] not in stored_ids and allowed["id"] in stored_ids, stored_ids + + refused: Final = candidate.request("POST", str(hidden["path"]), {"input": marker}) + assert refused.status_code == 404, refused.text + assert wire.drain() == () From 46a389c60bd46c3d91074d1513f718cee4da2370 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:59:25 +0000 Subject: [PATCH 15/51] feat(claude-code): emit installationPreference in marketplace.json entries (#41430) * feat(claude-code): emit installationPreference in marketplace.json entries Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(claude-code): satisfy marketplace CI checks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(claude-code): qualify marketplace entry as read only Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(claude-code): suppress typed marketplace assembly warning Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(claude-code): restrict installation_preference to the documented values Accept only available, auto_install, or required so a typo returns 422 instead of being silently dropped, and cover the PUT full-replace clearing behavior with a test * fix(claude-code): echo installation_preference on the public skill hub * refactor(claude-code): read installation_preference with one membership check and cover hand-edited rows * test(integration): audit installation_preference on the Claude Code marketplace * test(integration): kill a serving worker in the marketplace chaos cell * test(integration): give the marketplace worker-kill cell the two-boot timeout * fix(proxy): keep installationPreference on the MarketplacePluginEntry model * fix(proxy): refuse self-installing preferences off sha256 archives, accept the camelCase key, and echo the preference on POST and PUT * fix(claude-code): serve installation_preference only on archive plugins and refuse both key spellings Claude Code 2.1.75 and older reject a marketplace.json once any plugin carries installationPreference, and every Claude Code release before archive support rejects a catalog with an archive plugin. Requiring an archive source for every value, available included, keeps the field off catalogs those clients can read. A request that sends installation_preference and installationPreference together now gets a 422. * fix(claude-code): read the preference spelling that carries a value when the other is null --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/_lazy_openapi_snapshot.json | 71 ++- .../claude_code_marketplace.py | 66 ++- .../public_endpoints/public_endpoints.py | 3 + litellm/types/proxy/claude_code_endpoints.py | 35 +- ...ude_code_plugin_installation_preference.py | 540 ++++++++++++++++++ .../test_claude_code_marketplace.py | 272 ++++++++- .../public_endpoints/test_public_endpoints.py | 42 ++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 19 + 8 files changed, 1033 insertions(+), 15 deletions(-) create mode 100644 tests/integration/management/test_claude_code_plugin_installation_preference.py diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index ee49644f442..91fae2d1058 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -7134,6 +7134,22 @@ "title": "Id", "type": "string" }, + "installation_preference": { + "anyOf": [ + { + "enum": [ + "available", + "auto_install", + "required" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Installation Preference" + }, "keywords": { "anyOf": [ { @@ -7231,6 +7247,23 @@ "title": "Id", "type": "string" }, + "installation_preference": { + "anyOf": [ + { + "enum": [ + "available", + "auto_install", + "required" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Plugin installation preference", + "title": "Installation Preference" + }, "name": { "description": "Plugin name", "title": "Name", @@ -7328,6 +7361,23 @@ "description": "Plugin homepage URL", "title": "Homepage" }, + "installation_preference": { + "anyOf": [ + { + "enum": [ + "available", + "auto_install", + "required" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Emitted as installationPreference on this plugin's marketplace.json entry. Needs an archive source, the only plugins Claude Desktop lists from a marketplace.json URL, and 'auto_install' or 'required' also need its sha256. Claude Desktop honors the mark only when the archive and marketplace.json share its inference gateway origin", + "title": "Installation Preference" + }, "keywords": { "anyOf": [ { @@ -7478,6 +7528,23 @@ "description": "Plugin homepage URL", "title": "Homepage" }, + "installation_preference": { + "anyOf": [ + { + "enum": [ + "available", + "auto_install", + "required" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Emitted as installationPreference on this plugin's marketplace.json entry. Needs an archive source, the only plugins Claude Desktop lists from a marketplace.json URL, and 'auto_install' or 'required' also need its sha256. Claude Desktop honors the mark only when the archive and marketplace.json share its inference gateway origin", + "title": "Installation Preference" + }, "keywords": { "anyOf": [ { @@ -7672,7 +7739,7 @@ ] }, "post": { - "description": "Register a new plugin in the LiteLLM marketplace.\n\nLiteLLM acts as a registry/discovery layer. Plugins are hosted on\nGitHub/GitLab/Bitbucket or as a zip archive on any https host (e.g. S3).\nClaude Code clones the git source or downloads the archive when users install.\n\nThis endpoint is create-only and never overwrites. If a plugin with\nthe same name already exists it returns 409 Conflict; use\nPUT /claude-code/plugins/{plugin_name} to update an existing plugin.\n\nRequires a proxy admin API key.\n\nParameters:\n - name: Plugin name (kebab-case)\n - source: Plugin source reference (github, url, git-subdir, or archive format)\n - version: Semantic version (optional)\n - description: Plugin description (optional)\n - author: Author information (optional)\n - homepage: Plugin homepage URL (optional)\n - keywords: Search keywords (optional)\n - category: Plugin category (optional)\n\nReturns:\n Registration status (action is always \"created\") and plugin information.\n\nExample:\n ```bash\n curl -X POST http://localhost:4000/claude-code/plugins \\\n -H \"Authorization: Bearer sk-...\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"name\": \"my-plugin\",\n \"source\": {\"source\": \"github\", \"repo\": \"org/my-plugin\"},\n \"version\": \"1.0.0\",\n \"description\": \"My awesome plugin\"\n }'\n ```", + "description": "Register a new plugin in the LiteLLM marketplace.\n\nLiteLLM acts as a registry/discovery layer. Plugins are hosted on\nGitHub/GitLab/Bitbucket or as a zip archive on any https host (e.g. S3).\nClaude Code clones the git source or downloads the archive when users install.\n\nThis endpoint is create-only and never overwrites. If a plugin with\nthe same name already exists it returns 409 Conflict; use\nPUT /claude-code/plugins/{plugin_name} to update an existing plugin.\n\nRequires a proxy admin API key.\n\nParameters:\n - name: Plugin name (kebab-case)\n - source: Plugin source reference (github, url, git-subdir, or archive format)\n - version: Semantic version (optional)\n - description: Plugin description (optional)\n - author: Author information (optional)\n - homepage: Plugin homepage URL (optional)\n - keywords: Search keywords (optional)\n - category: Plugin category (optional)\n - installation_preference: 'available', 'auto_install', or 'required' (optional)\n\nReturns:\n Registration status (action is always \"created\") and plugin information.\n\nExample:\n ```bash\n curl -X POST http://localhost:4000/claude-code/plugins \\\n -H \"Authorization: Bearer sk-...\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"name\": \"my-plugin\",\n \"source\": {\"source\": \"github\", \"repo\": \"org/my-plugin\"},\n \"version\": \"1.0.0\",\n \"description\": \"My awesome plugin\"\n }'\n ```", "operationId": "register_plugin_claude_code_plugins_post", "requestBody": { "content": { @@ -7807,7 +7874,7 @@ ] }, "put": { - "description": "Update an existing plugin in the LiteLLM marketplace.\n\nThe plugin is identified by its name in the path, which is the resource\nidentity and cannot be changed here. This is a full replace, not a merge:\nthe manifest is rebuilt from the request body, so any optional field left\nout is reset to its default (e.g. an omitted version is cleared, not kept).\nSend the full desired state.\n\nReturns 404 if no plugin with the given name exists; use\nPOST /claude-code/plugins to create a new plugin.\n\nRequires a proxy admin API key.\n\nParameters:\n - plugin_name: Name of the plugin to update (path parameter)\n - source: Plugin source reference (github, url, git-subdir, or archive format)\n - version: Semantic version (optional)\n - description: Plugin description (optional)\n - author: Author information (optional)\n - homepage: Plugin homepage URL (optional)\n - keywords: Search keywords (optional)\n - category: Plugin category (optional)\n\nReturns:\n Update status (action is always \"updated\") and plugin information.\n\nExample:\n ```bash\n curl -X PUT http://localhost:4000/claude-code/plugins/my-plugin \\\n -H \"Authorization: Bearer sk-...\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"source\": {\"source\": \"github\", \"repo\": \"org/my-plugin\"},\n \"version\": \"2.0.0\",\n \"description\": \"My awesome plugin\"\n }'\n ```", + "description": "Update an existing plugin in the LiteLLM marketplace.\n\nThe plugin is identified by its name in the path, which is the resource\nidentity and cannot be changed here. This is a full replace, not a merge:\nthe manifest is rebuilt from the request body, so any optional field left\nout is reset to its default (e.g. an omitted version is cleared, not kept).\nSend the full desired state.\n\nReturns 404 if no plugin with the given name exists; use\nPOST /claude-code/plugins to create a new plugin.\n\nRequires a proxy admin API key.\n\nParameters:\n - plugin_name: Name of the plugin to update (path parameter)\n - source: Plugin source reference (github, url, git-subdir, or archive format)\n - version: Semantic version (optional)\n - description: Plugin description (optional)\n - author: Author information (optional)\n - homepage: Plugin homepage URL (optional)\n - keywords: Search keywords (optional)\n - category: Plugin category (optional)\n - installation_preference: 'available', 'auto_install', or 'required' (optional)\n\nReturns:\n Update status (action is always \"updated\") and plugin information.\n\nExample:\n ```bash\n curl -X PUT http://localhost:4000/claude-code/plugins/my-plugin \\\n -H \"Authorization: Bearer sk-...\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"source\": {\"source\": \"github\", \"repo\": \"org/my-plugin\"},\n \"version\": \"2.0.0\",\n \"description\": \"My awesome plugin\"\n }'\n ```", "operationId": "update_plugin_claude_code_plugins__plugin_name__put", "parameters": [ { diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index 2f0bdd20a8c..806f3ff7535 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -21,11 +21,13 @@ import json import re from collections.abc import Mapping, Sequence from datetime import datetime, timezone -from typing import Annotated, Final, Protocol, TypedDict +from typing import Annotated, Final, Protocol, TypedDict, get_args from urllib.parse import urlsplit from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import JSONResponse +from pydantic import TypeAdapter +from typing_extensions import ReadOnly from litellm._logging import verbose_proxy_logger from litellm.proxy._types import CommonProxyErrors, ProxyException, UserAPIKeyAuth @@ -37,6 +39,7 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.resource_ownership import is_proxy_admin from litellm.repositories.table_repositories import ClaudeCodePluginRepository from litellm.types.proxy.claude_code_endpoints import ( + InstallationPreference, ListPluginsResponse, PluginListItem, PluginResponse, @@ -70,6 +73,24 @@ class _MarketplaceEntry(TypedDict, total=False): homepage: object keywords: object category: object + installationPreference: ReadOnly[InstallationPreference] + + +_INSTALLATION_PREFERENCES: Final[tuple[InstallationPreference, ...]] = get_args(InstallationPreference) +MANIFEST_FIELDS: Final = TypeAdapter(dict[str, object]) + + +_SELF_INSTALLING_PREFERENCES: Final[frozenset[InstallationPreference]] = frozenset({"auto_install", "required"}) + + +def get_installation_preference(manifest: Mapping[str, object]) -> InstallationPreference | None: + value: Final = manifest.get("installation_preference") + if value is None or value in _INSTALLATION_PREFERENCES: + return value + verbose_proxy_logger.warning( + "Plugin %s has unknown installation_preference %r, serving it as unset", manifest.get("name"), value + ) + return None async def get_prisma_client() -> object: @@ -139,10 +160,16 @@ async def get_marketplace(request: Request, key: str | None = None): verbose_proxy_logger.warning("Plugin %s has no source field, skipping", plugin.name) continue - entry: _MarketplaceEntry = { - "name": plugin.name, - "source": manifest["source"], - } + installation_preference = get_installation_preference(manifest) + entry: _MarketplaceEntry = ( + {"name": plugin.name, "source": manifest["source"]} + if installation_preference is None + else { + "name": plugin.name, + "source": manifest["source"], + "installationPreference": installation_preference, + } + ) if plugin.version: entry["version"] = plugin.version @@ -249,6 +276,27 @@ def _validate_plugin_source(source: Mapping[str, str]) -> None: ) +def _validate_installation_preference(spec: PluginSpec) -> None: + if spec.installation_preference is None: + return + if spec.source.get("source") != "archive": + raise HTTPException( + status_code=400, + detail={ + "error": "installation_preference needs an archive source, the only plugins Claude Desktop lists " + "from a marketplace.json URL" + }, + ) + if spec.installation_preference in _SELF_INSTALLING_PREFERENCES and not spec.source.get("sha256"): + raise HTTPException( + status_code=400, + detail={ + "error": f"installation_preference '{spec.installation_preference}' needs the archive's sha256, " + "since Claude Desktop installs only pinned archives on its own" + }, + ) + + def _build_plugin_manifest(name: str, spec: PluginSpec) -> Mapping[str, object]: """Build the stored manifest dict shared by plugin create and update.""" dumped: Final[Mapping[str, object]] = spec.model_dump(exclude_none=True) @@ -309,6 +357,7 @@ async def register_plugin( - homepage: Plugin homepage URL (optional) - keywords: Search keywords (optional) - category: Plugin category (optional) + - installation_preference: 'available', 'auto_install', or 'required' (optional) Returns: Registration status (action is always "created") and plugin information. @@ -340,6 +389,7 @@ async def register_plugin( ) _validate_plugin_source(request.source) + _validate_installation_preference(request) existing: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.find_unique( where={"name": request.name} @@ -377,6 +427,7 @@ async def register_plugin( version=plugin.version, description=plugin.description, source=request.source, + installation_preference=request.installation_preference, enabled=plugin.enabled, ), ) @@ -438,6 +489,7 @@ async def list_plugins( category=manifest.get("category"), domain=manifest.get("domain"), namespace=manifest.get("namespace"), + installation_preference=get_installation_preference(MANIFEST_FIELDS.validate_python(manifest)), enabled=p.enabled, created_at=p.created_at.isoformat() if p.created_at else None, updated_at=p.updated_at.isoformat() if p.updated_at else None, @@ -511,6 +563,7 @@ async def get_plugin( "homepage": manifest.get("homepage"), "keywords": manifest.get("keywords"), "category": manifest.get("category"), + "installation_preference": get_installation_preference(manifest), "enabled": plugin.enabled, "created_at": plugin.created_at.isoformat() if plugin.created_at else None, "updated_at": plugin.updated_at.isoformat() if plugin.updated_at else None, @@ -561,6 +614,7 @@ async def update_plugin( - homepage: Plugin homepage URL (optional) - keywords: Search keywords (optional) - category: Plugin category (optional) + - installation_preference: 'available', 'auto_install', or 'required' (optional) Returns: Update status (action is always "updated") and plugin information. @@ -585,6 +639,7 @@ async def update_plugin( prisma_client: Final = await get_prisma_client() _validate_plugin_source(request.source) + _validate_installation_preference(request) existing: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.find_unique( where={"name": plugin_name} @@ -618,6 +673,7 @@ async def update_plugin( version=plugin.version, description=plugin.description, source=request.source, + installation_preference=request.installation_preference, enabled=plugin.enabled, ), ) diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 1d3516b9ebd..137f91f9b6e 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -322,6 +322,8 @@ async def get_mcp_servers(): async def public_skill_hub(): """Return enabled (public) Claude Code skills — no auth required.""" from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import ( + MANIFEST_FIELDS, + get_installation_preference, get_prisma_client, ) from litellm.types.proxy.claude_code_endpoints import ( @@ -352,6 +354,7 @@ async def public_skill_hub(): homepage=manifest.get("homepage"), domain=manifest.get("domain"), namespace=manifest.get("namespace"), + installation_preference=get_installation_preference(MANIFEST_FIELDS.validate_python(manifest)), ) ) return ListPluginsResponse(plugins=items, count=len(items)) diff --git a/litellm/types/proxy/claude_code_endpoints.py b/litellm/types/proxy/claude_code_endpoints.py index 28285838637..19e40aa5c5d 100644 --- a/litellm/types/proxy/claude_code_endpoints.py +++ b/litellm/types/proxy/claude_code_endpoints.py @@ -2,10 +2,16 @@ Claude Code Marketplace endpoint types for LiteLLM Proxy """ -from pydantic import Field +from typing import Final, Literal, TypeAlias + +from pydantic import AliasChoices, Field, TypeAdapter, model_validator from litellm.types.llms.base import LiteLLMBaseModel +InstallationPreference: TypeAlias = Literal["available", "auto_install", "required"] +_BOTH_PREFERENCE_SPELLINGS: Final = frozenset({"installation_preference", "installationPreference"}) +_REQUEST_KEYS: Final = TypeAdapter(dict[str, object]) + class PluginAuthor(LiteLLMBaseModel): """Plugin author information.""" @@ -43,6 +49,30 @@ class PluginSpec(LiteLLMBaseModel): category: str | None = Field(None, description="Plugin category") domain: str | None = Field(None, description="Skill domain (e.g., 'Productivity')") namespace: str | None = Field(None, description="Skill namespace within domain (e.g., 'workflows')") + installation_preference: InstallationPreference | None = Field( + None, + validation_alias=AliasChoices("installation_preference", "installationPreference"), + description=( + "Emitted as installationPreference on this plugin's marketplace.json entry. Needs an archive source, " + "the only plugins Claude Desktop lists from a marketplace.json URL, and 'auto_install' or 'required' " + "also need its sha256. Claude Desktop honors the mark only when the archive and marketplace.json share " + "its inference gateway origin" + ), + ) + + @model_validator(mode="before") + @classmethod + def reject_both_preference_spellings(cls, data: object) -> object: + if not isinstance(data, dict): + return data + fields: Final = _REQUEST_KEYS.validate_python(data) + if not _BOTH_PREFERENCE_SPELLINGS.issubset(fields): + return fields + if all(fields[key] is not None for key in _BOTH_PREFERENCE_SPELLINGS): + raise ValueError("send installation_preference or installationPreference, not both") + return { + key: value for key, value in fields.items() if key not in _BOTH_PREFERENCE_SPELLINGS or value is not None + } class RegisterPluginRequest(PluginSpec): @@ -81,6 +111,7 @@ class PluginResponse(LiteLLMBaseModel): version: str | None = Field(None, description="Plugin version") description: str | None = Field(None, description="Plugin description") source: dict[str, str] = Field(..., description="Plugin source reference") + installation_preference: InstallationPreference | None = Field(None, description="Plugin installation preference") enabled: bool = Field(..., description="Whether plugin is enabled") @@ -106,6 +137,7 @@ class PluginListItem(LiteLLMBaseModel): category: str | None = None domain: str | None = None namespace: str | None = None + installation_preference: InstallationPreference | None = None enabled: bool created_at: str | None updated_at: str | None @@ -129,6 +161,7 @@ class MarketplacePluginEntry(LiteLLMBaseModel): homepage: str | None = None keywords: list[str] | None = None category: str | None = None + installationPreference: InstallationPreference | None = None class MarketplaceResponse(LiteLLMBaseModel): diff --git a/tests/integration/management/test_claude_code_plugin_installation_preference.py b/tests/integration/management/test_claude_code_plugin_installation_preference.py new file mode 100644 index 00000000000..b8afd3ed65d --- /dev/null +++ b/tests/integration/management/test_claude_code_plugin_installation_preference.py @@ -0,0 +1,540 @@ +import json +import uuid +from collections.abc import Mapping +from concurrent.futures import ThreadPoolExecutor +from functools import partial +from pathlib import Path +from typing import Final + +import httpx +import pytest +from pydantic import JsonValue + +from tests.integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, object_value, string_value +from tests.integration._support.database import read_rows, write_rows +from tests.integration._support.process import OwnedProxy, owned_proxy_process + +PREFERENCES: Final = ("available", "auto_install", "required") +INVALID_PREFERENCES: Final[tuple[JsonValue, ...]] = ( + "", + 0, + ["auto_install"], + "x" * 5000, + "auto-install", + "AUTO_INSTALL", +) +INVALID_IDS: Final = ("empty-string", "int", "list", "5kb-string", "hyphenated", "upper-case") +SCHEMAS_WITH_PREFERENCE: Final = ("RegisterPluginRequest", "UpdatePluginRequest", "PluginListItem") +MARKETPLACE: Final = "/claude-code/marketplace.json" +PLUGINS: Final = "/claude-code/plugins" +SKILL_HUB: Final = "/public/skill_hub" +PLUGIN_ROWS: Final = 'SELECT manifest_json, enabled, files_json FROM "LiteLLM_ClaudeCodePluginTable" WHERE name = %s' + + +def plugin_name() -> str: + return f"audit-{uuid.uuid4().hex[:12]}" + + +def archive_source(name: str, digest: str = "a" * 64) -> dict[str, JsonValue]: + return {"source": "archive", "url": f"https://files.example.com/{name}.zip", "sha256": digest} + + +def registration(name: str, **fields: JsonValue) -> dict[str, JsonValue]: + return {"name": name, "source": archive_source(name), **fields} + + +def replacement(name: str, **fields: JsonValue) -> dict[str, JsonValue]: + return {"source": archive_source(name), **fields} + + +def delete_plugin(gateway: Gateway, name: str) -> None: + response: Final = gateway.request("DELETE", f"{PLUGINS}/{name}") + assert response.status_code in {200, 404}, response.text + + +def register(scenario: Scenario, body: Mapping[str, JsonValue]) -> httpx.Response: + scenario.cleanups.callback(delete_plugin, scenario.gateway, string_value(body["name"])) + return scenario.gateway.request("POST", PLUGINS, body) + + +def registered(scenario: Scenario, name: str, **fields: JsonValue) -> None: + response: Final = register(scenario, registration(name, **fields)) + assert response.status_code == 200, response.text + + +def replace(gateway: Gateway, name: str, body: Mapping[str, JsonValue], *, key: str | None = None) -> httpx.Response: + return gateway.request("PUT", f"{PLUGINS}/{name}", body, key=key) + + +def replaced(gateway: Gateway, name: str, **fields: JsonValue) -> None: + response: Final = replace(gateway, name, replacement(name, **fields)) + assert response.status_code == 200, response.text + + +def entries(response: httpx.Response, name: str) -> tuple[dict[str, JsonValue], ...]: + assert response.status_code == 200, response.text + plugins: Final = JSON_OBJECT.validate_json(response.content)["plugins"] + assert isinstance(plugins, list), response.text + return tuple(object_value(plugin) for plugin in plugins if object_value(plugin)["name"] == name) + + +def marketplace_entries(gateway: Gateway, name: str, *, key: str | None = None) -> tuple[dict[str, JsonValue], ...]: + return entries(gateway.client.get(MARKETPLACE, params=None if key is None else {"key": key}), name) + + +def marketplace_entry(gateway: Gateway, name: str, *, key: str | None = None) -> dict[str, JsonValue]: + found: Final = marketplace_entries(gateway, name, key=key) + assert len(found) == 1, found + return found[0] + + +def listed_item(gateway: Gateway, name: str) -> dict[str, JsonValue]: + found: Final = entries(gateway.request("GET", PLUGINS), name) + assert len(found) == 1, found + return found[0] + + +def skill_hub_entries(gateway: Gateway, name: str) -> tuple[dict[str, JsonValue], ...]: + return entries(gateway.client.get(SKILL_HUB), name) + + +def detail(gateway: Gateway, name: str, *, key: str | None = None) -> httpx.Response: + return gateway.request("GET", f"{PLUGINS}/{name}", key=key) + + +def detail_body(gateway: Gateway, name: str, *, key: str | None = None) -> dict[str, JsonValue]: + response: Final = detail(gateway, name, key=key) + assert response.status_code == 200, response.text + return JSON_OBJECT.validate_json(response.content) + + +def stored_rows(name: str) -> list[dict[str, JsonValue]]: + return read_rows(PLUGIN_ROWS, (name,)) + + +def stored_manifest(name: str) -> dict[str, JsonValue]: + rows: Final = stored_rows(name) + assert len(rows) == 1, rows + return JSON_OBJECT.validate_json(string_value(rows[0]["manifest_json"])) + + +def assert_preference_served(gateway: Gateway, name: str, preference: str) -> None: + assert marketplace_entry(gateway, name)["installationPreference"] == preference + assert listed_item(gateway, name)["installation_preference"] == preference + assert detail_body(gateway, name)["installation_preference"] == preference + hub: Final = skill_hub_entries(gateway, name) + assert len(hub) == 1, hub + assert hub[0]["installation_preference"] == preference + + +def assert_preference_unset(gateway: Gateway, name: str) -> None: + assert "installationPreference" not in marketplace_entry(gateway, name) + assert listed_item(gateway, name)["installation_preference"] is None + assert detail_body(gateway, name)["installation_preference"] is None + hub: Final = skill_hub_entries(gateway, name) + assert len(hub) == 1, hub + assert hub[0]["installation_preference"] is None + + +@pytest.mark.parametrize("preference", PREFERENCES) +def test_registered_preference_is_served_on_the_marketplace(gateway: Gateway, preference: str) -> None: + name: Final = plugin_name() + with gateway.scenario() as scenario: + registered(scenario, name, installation_preference=preference) + assert marketplace_entry(gateway, name) == { + "name": name, + "source": archive_source(name), + "installationPreference": preference, + "version": "1.0.0", + } + assert stored_manifest(name)["installation_preference"] == preference + + +def test_preference_echoes_on_list_detail_and_skill_hub(gateway: Gateway) -> None: + name: Final = plugin_name() + with gateway.scenario() as scenario: + registered(scenario, name, installation_preference="required") + assert_preference_served(gateway, name, "required") + + +def test_peer_process_serves_the_stored_preference(gateway: Gateway, peer: Gateway) -> None: + name: Final = plugin_name() + with gateway.scenario() as scenario: + registered(scenario, name, installation_preference="auto_install") + assert_preference_served(peer, name, "auto_install") + + +def test_omitted_preference_is_unset(gateway: Gateway) -> None: + name: Final = plugin_name() + with gateway.scenario() as scenario: + registered(scenario, name) + assert_preference_unset(gateway, name) + assert stored_manifest(name).get("installation_preference") is None + + +def test_explicit_null_preference_is_unset(gateway: Gateway) -> None: + name: Final = plugin_name() + with gateway.scenario() as scenario: + registered(scenario, name, installation_preference=None) + assert_preference_unset(gateway, name) + assert stored_manifest(name).get("installation_preference") is None + + +@pytest.mark.parametrize("preference", INVALID_PREFERENCES, ids=INVALID_IDS) +def test_invalid_preference_is_refused_on_register(gateway: Gateway, preference: JsonValue) -> None: + name: Final = plugin_name() + with gateway.scenario() as scenario: + response: Final = register(scenario, registration(name, installation_preference=preference)) + assert response.status_code == 422, response.text + assert "installation_preference" in response.text + assert stored_rows(name) == [] + + +@pytest.mark.parametrize("preference", INVALID_PREFERENCES, ids=INVALID_IDS) +def test_invalid_preference_is_refused_on_update(gateway: Gateway, preference: JsonValue) -> None: + name: Final = plugin_name() + with gateway.scenario() as scenario: + registered(scenario, name, installation_preference="available") + response: Final = replace(gateway, name, replacement(name, installation_preference=preference)) + assert response.status_code == 422, response.text + assert "installation_preference" in response.text + assert stored_manifest(name)["installation_preference"] == "available" + assert marketplace_entry(gateway, name)["installationPreference"] == "available" + + +@pytest.mark.parametrize("preference", PREFERENCES) +def test_preference_on_a_git_source_is_refused(gateway: Gateway, preference: str) -> None: + name: Final = plugin_name() + with gateway.scenario() as scenario: + body: Final = { + "name": name, + "source": {"source": "github", "repo": "example-org/example-plugin"}, + "installation_preference": preference, + } + response: Final = register(scenario, body) + assert response.status_code == 400, response.text + assert "archive source" in response.text + assert stored_rows(name) == [] + + +def test_both_preference_spellings_are_refused(gateway: Gateway) -> None: + name: Final = plugin_name() + with gateway.scenario() as scenario: + response: Final = register( + scenario, registration(name, installation_preference="required", installationPreference="available") + ) + assert response.status_code == 422, response.text + assert "not both" in response.text + assert stored_rows(name) == [] + + +def test_duplicate_json_key_keeps_the_last_value(gateway: Gateway) -> None: + name: Final = plugin_name() + raw: Final = ( + f'{{"name": "{name}", "source": {json.dumps(archive_source(name))}, ' + '"installation_preference": "required", "installation_preference": "available"}' + ) + with gateway.scenario() as scenario: + scenario.cleanups.callback(delete_plugin, gateway, name) + response: Final = gateway.client.post( + PLUGINS, + content=raw.encode(), + headers={"Authorization": f"Bearer {gateway.key}", "Content-Type": "application/json"}, + ) + assert response.status_code == 200, response.text + assert marketplace_entry(gateway, name)["installationPreference"] == "available" + assert stored_manifest(name)["installation_preference"] == "available" + + +def test_repeated_identical_update_is_idempotent(gateway: Gateway) -> None: + name: Final = plugin_name() + with gateway.scenario() as scenario: + registered(scenario, name, installation_preference="required") + replaced(gateway, name, installation_preference="required") + first: Final = stored_rows(name) + replaced(gateway, name, installation_preference="required") + second: Final = stored_rows(name) + assert len(first) == 1, first + assert second == first + assert_preference_served(gateway, name, "required") + + +def test_duplicate_name_is_refused_and_keeps_the_first_preference(gateway: Gateway) -> None: + name: Final = plugin_name() + with gateway.scenario() as scenario: + registered(scenario, name, installation_preference="required") + response: Final = register(scenario, registration(name, installation_preference="available")) + assert response.status_code == 409, response.text + assert len(stored_rows(name)) == 1 + assert_preference_served(gateway, name, "required") + + +def test_update_changes_and_clears_the_preference(gateway: Gateway) -> None: + name: Final = plugin_name() + with gateway.scenario() as scenario: + registered(scenario, name, installation_preference="available") + replaced(gateway, name, installation_preference="required") + assert_preference_served(gateway, name, "required") + replaced(gateway, name) + assert_preference_unset(gateway, name) + replaced(gateway, name, installation_preference="auto_install") + assert_preference_served(gateway, name, "auto_install") + replaced(gateway, name, installation_preference=None) + assert_preference_unset(gateway, name) + assert stored_manifest(name).get("installation_preference") is None + + +def test_update_replaces_the_whole_manifest(gateway: Gateway) -> None: + name: Final = plugin_name() + with gateway.scenario() as scenario: + registered( + scenario, + name, + description="first", + keywords=["one"], + author={"name": "audit"}, + homepage="https://example.com/first", + category="tools", + installation_preference="required", + ) + before: Final = detail_body(gateway, name) + assert before["description"] == "first" + assert before["installation_preference"] == "required" + replaced(gateway, name, version="2.0.0", installation_preference="auto_install") + after: Final = detail_body(gateway, name) + assert after["updated_at"] != before["updated_at"] + assert {field: value for field, value in after.items() if field != "updated_at"} == { + **{field: value for field, value in before.items() if field != "updated_at"}, + "version": "2.0.0", + "description": None, + "author": None, + "homepage": None, + "keywords": None, + "category": None, + "installation_preference": "auto_install", + } + assert marketplace_entry(gateway, name) == { + "name": name, + "source": archive_source(name), + "installationPreference": "auto_install", + "version": "2.0.0", + } + assert stored_rows(name)[0]["files_json"] == "{}" + + +def test_unauthenticated_requests_are_refused(gateway: Gateway) -> None: + name: Final = plugin_name() + unauthenticated: Final = gateway.client + with gateway.scenario() as scenario: + anonymous_registration: Final = unauthenticated.post( + PLUGINS, json=registration(name, installation_preference="required") + ) + assert anonymous_registration.status_code == 401, anonymous_registration.text + assert stored_rows(name) == [] + registered(scenario, name, installation_preference="required") + refused: Final = ( + unauthenticated.put(f"{PLUGINS}/{name}", json=replacement(name, installation_preference="available")), + unauthenticated.post(f"{PLUGINS}/{name}/disable"), + unauthenticated.post(f"{PLUGINS}/{name}/enable"), + unauthenticated.delete(f"{PLUGINS}/{name}"), + unauthenticated.get(PLUGINS), + unauthenticated.get(f"{PLUGINS}/{name}"), + ) + assert [response.status_code for response in refused] == [401] * len(refused), [r.text for r in refused] + assert stored_manifest(name)["installation_preference"] == "required" + assert stored_rows(name)[0]["enabled"] is True + assert marketplace_entry(gateway, name)["installationPreference"] == "required" + + +def test_non_admin_key_cannot_modify_the_catalog(gateway: Gateway) -> None: + name: Final = plugin_name() + other: Final = plugin_name() + with gateway.scenario() as scenario: + key: Final = scenario.key() + registered(scenario, name, installation_preference="required") + scenario.cleanups.callback(delete_plugin, gateway, other) + refused: Final = ( + gateway.request("POST", PLUGINS, registration(other, installation_preference="available"), key=key), + replace(gateway, name, replacement(name, installation_preference="available"), key=key), + gateway.request("DELETE", f"{PLUGINS}/{name}", key=key), + ) + assert [response.status_code for response in refused] == [403] * len(refused), [r.text for r in refused] + flips: Final = ( + gateway.request("POST", f"{PLUGINS}/{name}/disable", key=key), + gateway.request("POST", f"{PLUGINS}/{name}/enable", key=key), + ) + assert all(response.status_code in {401, 403} for response in flips), [r.text for r in flips] + assert stored_rows(other) == [] + assert stored_manifest(name)["installation_preference"] == "required" + assert stored_rows(name)[0]["enabled"] is True + assert detail_body(gateway, name, key=key)["installation_preference"] == "required" + + +def test_disabled_plugin_keeps_its_preference_for_granted_keys(gateway: Gateway) -> None: + name: Final = plugin_name() + with gateway.scenario() as scenario: + registered(scenario, name, installation_preference="auto_install") + granted: Final = scenario.key(object_permission={"skills": [name]}) + disabled: Final = gateway.request("POST", f"{PLUGINS}/{name}/disable") + assert disabled.status_code == 200, disabled.text + assert marketplace_entries(gateway, name) == () + assert skill_hub_entries(gateway, name) == () + assert marketplace_entry(gateway, name, key=granted)["installationPreference"] == "auto_install" + assert detail_body(gateway, name, key=granted)["installation_preference"] == "auto_install" + assert stored_rows(name)[0]["enabled"] is False + enabled: Final = gateway.request("POST", f"{PLUGINS}/{name}/enable") + assert enabled.status_code == 200, enabled.text + assert_preference_served(gateway, name, "auto_install") + assert stored_manifest(name)["installation_preference"] == "auto_install" + + +def test_non_granted_key_cannot_read_a_disabled_plugin(gateway: Gateway) -> None: + name: Final = plugin_name() + with gateway.scenario() as scenario: + registered(scenario, name, installation_preference="required") + plain: Final = scenario.key() + assert detail_body(gateway, name, key=plain)["installation_preference"] == "required" + disabled: Final = gateway.request("POST", f"{PLUGINS}/{name}/disable") + assert disabled.status_code == 200, disabled.text + refused: Final = detail(gateway, name, key=plain) + assert refused.status_code == 403, refused.text + assert marketplace_entries(gateway, name, key=plain) == () + assert skill_hub_entries(gateway, name) == () + + +def test_hand_edited_unknown_preference_is_not_served(gateway: Gateway, peer: Gateway) -> None: + name: Final = plugin_name() + with gateway.scenario() as scenario: + registered(scenario, name, installation_preference="required") + edited: Final = {**stored_manifest(name), "installation_preference": "someday"} + write_rows( + 'UPDATE "LiteLLM_ClaudeCodePluginTable" SET manifest_json = %s WHERE name = %s', + (json.dumps(edited), name), + ) + assert stored_manifest(name)["installation_preference"] == "someday" + assert_preference_unset(gateway, name) + assert_preference_unset(peer, name) + liveliness: Final = gateway.client.get("/health/liveliness") + assert liveliness.status_code == 200, liveliness.text + + +def preference_shapes(schemas: Mapping[str, JsonValue], schema: str) -> list[JsonValue]: + properties: Final = object_value(object_value(schemas[schema])["properties"]) + assert "installation_preference" in properties, (schema, sorted(properties)) + shapes: Final = object_value(properties["installation_preference"])["anyOf"] + assert isinstance(shapes, list), (schema, shapes) + return shapes + + +@pytest.mark.parametrize("schema", SCHEMAS_WITH_PREFERENCE) +def test_openapi_declares_the_preference_enum(gateway: Gateway, schema: str) -> None: + response: Final = gateway.client.get("/openapi.json") + assert response.status_code == 200, response.text + schemas: Final = object_value(object_value(JSON_OBJECT.validate_json(response.content)["components"])["schemas"]) + shapes: Final = preference_shapes(schemas, schema) + assert {"type": "string", "enum": list(PREFERENCES)} in shapes, (schema, shapes) + assert {"type": "null"} in shapes, (schema, shapes) + + +def test_burst_reads_across_processes_while_the_preference_flips(gateway: Gateway, peer: Gateway) -> None: + name: Final = plugin_name() + targets: Final = (gateway, peer) + flips: Final = tuple(PREFERENCES[(index + 1) % len(PREFERENCES)] for index in range(13)) + with gateway.scenario() as scenario: + registered(scenario, name, installation_preference="available") + + def job(index: int) -> tuple[str, JsonValue]: + if index % 4 == 3: + return "flip", replace( + gateway, name, replacement(name, installation_preference=flips[index // 4]) + ).status_code + return "read", marketplace_entry(targets[index % 2], name).get("installationPreference") + + with ThreadPoolExecutor(max_workers=8) as pool: + outcomes: Final = tuple(pool.map(job, range(53))) + assert [result for kind, result in outcomes if kind == "flip"] == [200] * len(flips), outcomes + observed: Final = tuple(result for kind, result in outcomes if kind == "read") + assert len(observed) == 40, outcomes + assert all(value in PREFERENCES for value in observed), observed + settled: Final = stored_manifest(name)["installation_preference"] + assert settled in PREFERENCES, settled + assert marketplace_entry(gateway, name)["installationPreference"] == settled + assert marketplace_entry(peer, name)["installationPreference"] == settled + assert len(stored_rows(name)) == 1 + replaced(gateway, name, installation_preference="required") + assert_preference_served(gateway, name, "required") + assert_preference_served(peer, name, "required") + assert stored_manifest(name)["installation_preference"] == "required" + + +def test_repeated_identical_reads_are_stable(gateway: Gateway, peer: Gateway) -> None: + name: Final = plugin_name() + targets: Final = (gateway, peer) + with gateway.scenario() as scenario: + registered(scenario, name, installation_preference="required") + before: Final = listed_item(gateway, name) + bodies: Final = tuple(marketplace_entry(targets[index % 2], name) for index in range(10)) + assert all(body == bodies[0] for body in bodies), bodies + assert bodies[0]["installationPreference"] == "required" + assert listed_item(peer, name) == before + assert len(stored_rows(name)) == 1 + + +@pytest.mark.timeout(240) +def test_registration_burst_survives_a_worker_kill(gateway: Gateway, tmp_path: Path) -> None: + names: Final = tuple(plugin_name() for _ in range(20)) + preferences: Final = tuple(PREFERENCES[index % len(PREFERENCES)] for index in range(len(names))) + with gateway.scenario() as scenario, owned_proxy_process(gateway, tmp_path, {}, workers=2) as owned: + for name in names: + scenario.cleanups.callback(delete_plugin, gateway, name) + workers: Final = eventually(owned.worker_pids, lambda found: len(found) == 2, seconds=30) + + def submit(index: int) -> int: + try: + return owned.gateway.request( + "POST", PLUGINS, registration(names[index], installation_preference=preferences[index]) + ).status_code + except httpx.TransportError: + return -1 + + with ThreadPoolExecutor(max_workers=8) as pool: + futures: Final = tuple(pool.submit(submit, index) for index in range(len(names))) + eventually(lambda: sum(1 for future in futures if future.done()), lambda done: done >= 3, seconds=30) + owned.kill_worker(min(workers)) + statuses: Final = tuple(future.result() for future in futures) + assert set(statuses) <= {200, -1}, statuses + assert statuses.count(200) >= 3, statuses + served: Final = tuple(marketplace_entries(owned.gateway, name) for name in names) + for index, found in enumerate(served): + assert len(found) == (1 if statuses[index] == 200 else len(found)), (names[index], statuses[index], found) + assert len(found) <= 1, (names[index], found) + assert all(entry["installationPreference"] == preferences[index] for entry in found), (names[index], found) + assert [marketplace_entries(gateway, name) for name in names] == list(served) + assert sum(len(stored_rows(name)) for name in names) == sum(len(found) for found in served) + + +@pytest.mark.timeout(240) +def test_registrations_survive_a_proxy_restart(gateway: Gateway, tmp_path: Path) -> None: + names: Final = tuple(plugin_name() for _ in range(10)) + preferences: Final = tuple(PREFERENCES[index % len(PREFERENCES)] for index in range(len(names))) + with gateway.scenario() as scenario: + for name in names: + scenario.cleanups.callback(delete_plugin, gateway, name) + + def submit(target: Gateway, index: int) -> int: + return target.request( + "POST", PLUGINS, registration(names[index], installation_preference=preferences[index]) + ).status_code + + with owned_proxy_process(gateway, tmp_path, {}) as first, ThreadPoolExecutor(max_workers=8) as pool: + statuses: Final = tuple(pool.map(partial(submit, first.gateway), range(len(names)))) + assert statuses == (200,) * len(names), statuses + with owned_proxy_process(gateway, tmp_path, {}) as second: + served: Final = tuple(marketplace_entry(second.gateway, name)["installationPreference"] for name in names) + assert served == preferences + listed: Final = second.gateway.request("GET", PLUGINS) + assert listed.status_code == 200, listed.text + plugins: Final = JSON_OBJECT.validate_json(listed.content)["plugins"] + assert isinstance(plugins, list), listed.text + assert sum(1 for plugin in plugins if object_value(plugin)["name"] in names) == len(names), listed.text + assert sum(len(stored_rows(name)) for name in names) == len(names) diff --git a/tests/unit/proxy/anthropic_endpoints/test_claude_code_marketplace.py b/tests/unit/proxy/anthropic_endpoints/test_claude_code_marketplace.py index ccc63f12157..6f426a4a3a7 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_claude_code_marketplace.py +++ b/tests/unit/proxy/anthropic_endpoints/test_claude_code_marketplace.py @@ -4,20 +4,16 @@ Unit tests for claude_code_marketplace.py source validation. Covers the git-subdir and archive source types added alongside the existing github and url types. """ -from typing import Final import json +from typing import Final +from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import HTTPException -from unittest.mock import AsyncMock, MagicMock +from pydantic import ValidationError import litellm from litellm.proxy._types import LiteLLM_ObjectPermissionTable, ProxyException, UserAPIKeyAuth -from litellm.proxy.proxy_server import LitellmUserRoles -from litellm.types.proxy.claude_code_endpoints import ( - RegisterPluginRequest, - UpdatePluginRequest, -) from litellm.proxy.anthropic_endpoints.claude_code_endpoints import claude_code_marketplace from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import ( delete_plugin, @@ -29,6 +25,12 @@ from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketp register_plugin, update_plugin, ) +from litellm.proxy.proxy_server import LitellmUserRoles +from litellm.types.proxy.claude_code_endpoints import ( + MarketplaceResponse, + RegisterPluginRequest, + UpdatePluginRequest, +) from tests._master_key import MASTER_KEY @@ -365,6 +367,225 @@ async def test_get_marketplace_key_query_param_adds_granted_disabled_plugins(mon assert exc_info.value.code == "401" +@pytest.mark.asyncio +@pytest.mark.parametrize("preference", ["available", "auto_install", "required"]) +async def test_get_marketplace_emits_installation_preference(preference): + await register_plugin( + request=RegisterPluginRequest( + name="s3-skill", + source=_ARCHIVE_SOURCE, + installation_preference=preference, + ), + user_api_key_dict=_USER, + ) + + marketplace = json.loads((await get_marketplace(request=MagicMock())).body) + + assert marketplace["plugins"] == [ + {"name": "s3-skill", "source": _ARCHIVE_SOURCE, "version": "1.0.0", "installationPreference": preference} + ] + + +@pytest.mark.parametrize( + "build_request", + [ + lambda preference: RegisterPluginRequest( + name="s3-skill", source=_ARCHIVE_SOURCE, installation_preference=preference + ), + lambda preference: UpdatePluginRequest(source=_ARCHIVE_SOURCE, installation_preference=preference), + ], + ids=["register", "update"], +) +def test_plugin_request_rejects_unknown_installation_preference(build_request): + with pytest.raises(ValidationError, match="installation_preference"): + build_request("auto-install") + + +@pytest.mark.asyncio +async def test_get_marketplace_omits_installation_preference_when_unset(): + await register_plugin( + request=RegisterPluginRequest(name="manual-install-plugin", source=_GIT_SUBDIR_SOURCE), + user_api_key_dict=_USER, + ) + + marketplace = json.loads((await get_marketplace(request=MagicMock())).body) + + assert "installationPreference" not in marketplace["plugins"][0] + + +@pytest.mark.asyncio +async def test_update_plugin_propagates_installation_preference_to_get_and_list(): + name = "updated-install-plugin" + await register_plugin( + request=RegisterPluginRequest(name=name, source=_ARCHIVE_SOURCE), + user_api_key_dict=_USER, + ) + + response = await update_plugin( + plugin_name=name, + request=UpdatePluginRequest( + source=_ARCHIVE_SOURCE, + installation_preference="auto_install", + ), + user_api_key_dict=_USER, + ) + + plugin = await get_plugin(plugin_name=name, user_api_key_dict=_USER) + listed_plugin = (await list_plugins(user_api_key_dict=_USER)).plugins[0] + + assert response.plugin.installation_preference == "auto_install" + assert plugin["installation_preference"] == "auto_install" + assert listed_plugin.installation_preference == "auto_install" + + +@pytest.mark.asyncio +async def test_update_plugin_without_installation_preference_clears_it(): + name = "cleared-install-plugin" + await register_plugin( + request=RegisterPluginRequest(name=name, source=_ARCHIVE_SOURCE, installation_preference="auto_install"), + user_api_key_dict=_USER, + ) + + await update_plugin( + plugin_name=name, + request=UpdatePluginRequest(source=_ARCHIVE_SOURCE), + user_api_key_dict=_USER, + ) + + marketplace = json.loads((await get_marketplace(request=MagicMock())).body) + assert "installationPreference" not in marketplace["plugins"][0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stored_value", ["auto-install", 1, True, None]) +async def test_stored_unknown_installation_preference_is_not_served(stored_value, caplog): + name = "hand-edited-plugin" + table = litellm.proxy.proxy_server.prisma_client.db.litellm_claudecodeplugintable + await table.create( + data={ + "name": name, + "manifest_json": json.dumps( + {"name": name, "source": _ARCHIVE_SOURCE, "installation_preference": stored_value} + ), + } + ) + + marketplace = json.loads((await get_marketplace(request=MagicMock())).body) + listed_plugin = (await list_plugins(user_api_key_dict=_USER)).plugins[0] + plugin = await get_plugin(plugin_name=name, user_api_key_dict=_USER) + + assert marketplace["plugins"] == [{"name": name, "source": _ARCHIVE_SOURCE}] + assert listed_plugin.installation_preference is None + assert plugin["installation_preference"] is None + unknown_value_warnings = [r for r in caplog.records if "unknown installation_preference" in r.getMessage()] + assert len(unknown_value_warnings) == (0 if stored_value is None else 3) + + +@pytest.mark.asyncio +async def test_register_accepts_the_camel_case_installation_preference_key(): + request = RegisterPluginRequest.model_validate( + {"name": "s3-skill", "source": _ARCHIVE_SOURCE, "installationPreference": "auto_install"} + ) + + response = await register_plugin(request=request, user_api_key_dict=_USER) + marketplace = json.loads((await get_marketplace(request=MagicMock())).body) + + assert response.plugin.installation_preference == "auto_install" + assert (await _read_stored_manifest("s3-skill"))["installation_preference"] == "auto_install" + assert marketplace["plugins"][0]["installationPreference"] == "auto_install" + + +_NON_ARCHIVE_SOURCES = [ + {"source": "github", "repo": "org/repo"}, + {"source": "url", "url": "https://github.com/org/repo.git"}, + _GIT_SUBDIR_SOURCE, +] +_UNPINNED_ARCHIVE_SOURCE = {"source": "archive", "url": "https://artifacts.example.com/plugin.zip"} + + +async def _assert_preference_refused(source, preference): + with pytest.raises(HTTPException) as register_error: + await register_plugin( + request=RegisterPluginRequest(name="no-op-plugin", source=source, installation_preference=preference), + user_api_key_dict=_USER, + ) + await register_plugin(request=RegisterPluginRequest(name="no-op-plugin", source=source), user_api_key_dict=_USER) + with pytest.raises(HTTPException) as update_error: + await update_plugin( + plugin_name="no-op-plugin", + request=UpdatePluginRequest(source=source, installation_preference=preference), + user_api_key_dict=_USER, + ) + + assert register_error.value.status_code == 400 + assert update_error.value.status_code == 400 + assert "installationPreference" not in json.loads((await get_marketplace(request=MagicMock())).body)["plugins"][0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("preference", ["available", "auto_install", "required"]) +@pytest.mark.parametrize("source", _NON_ARCHIVE_SOURCES) +async def test_every_preference_needs_an_archive_source(preference, source): + await _assert_preference_refused(source, preference) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("preference", ["auto_install", "required"]) +async def test_self_installing_preference_needs_the_archive_sha256(preference): + await _assert_preference_refused(_UNPINNED_ARCHIVE_SOURCE, preference) + + +@pytest.mark.asyncio +async def test_available_preference_is_accepted_on_an_unpinned_archive(): + response = await register_plugin( + request=RegisterPluginRequest( + name="listed-plugin", source=_UNPINNED_ARCHIVE_SOURCE, installation_preference="available" + ), + user_api_key_dict=_USER, + ) + marketplace = json.loads((await get_marketplace(request=MagicMock())).body) + + assert response.plugin.installation_preference == "available" + assert marketplace["plugins"][0]["installationPreference"] == "available" + + +@pytest.mark.parametrize( + "build_request", + [ + lambda body: RegisterPluginRequest.model_validate({"name": "s3-skill", **body}), + UpdatePluginRequest.model_validate, + ], + ids=["register", "update"], +) +def test_plugin_request_rejects_both_preference_spellings(build_request): + with pytest.raises(ValidationError, match="not both"): + build_request( + { + "source": _ARCHIVE_SOURCE, + "installation_preference": "auto_install", + "installationPreference": "available", + } + ) + + +@pytest.mark.parametrize( + "build_request", + [ + lambda body: RegisterPluginRequest.model_validate({"name": "s3-skill", **body}), + UpdatePluginRequest.model_validate, + ], + ids=["register", "update"], +) +@pytest.mark.parametrize( + "null_key, value_key", + [("installation_preference", "installationPreference"), ("installationPreference", "installation_preference")], +) +def test_plugin_request_reads_the_spelling_that_carries_a_value(build_request, null_key, value_key): + request = build_request({"source": _ARCHIVE_SOURCE, null_key: None, value_key: "auto_install"}) + + assert request.installation_preference == "auto_install" + + @pytest.mark.asyncio async def test_register_plugin_git_subdir_missing_url(): """git-subdir without url field raises HTTP 400.""" @@ -706,3 +927,40 @@ async def test_register_plugin_git_subdir(): assert response.plugin.source["url"] == "https://github.com/test-org/monorepo.git" assert response.plugin.source["path"] == "plugins/my-plugin" assert response.plugin.enabled is True + + +def test_marketplace_response_round_trips_installation_preference(): + catalog: Final = { + "name": "litellm", + "owner": {"name": "LiteLLM", "email": "support@litellm.ai"}, + "plugins": [ + { + "name": "auto", + "source": {"source": "url", "url": "https://example.com/auto.tgz"}, + "installationPreference": "auto_install", + }, + {"name": "manual", "source": {"source": "url", "url": "https://example.com/manual.tgz"}}, + ], + } + + round_tripped: Final = MarketplaceResponse.model_validate(catalog).model_dump(exclude_none=True) + + assert round_tripped["plugins"][0]["installationPreference"] == "auto_install" + assert "installationPreference" not in round_tripped["plugins"][1] + + +def test_marketplace_response_rejects_unknown_installation_preference(): + catalog: Final = { + "name": "litellm", + "owner": {"name": "LiteLLM", "email": "support@litellm.ai"}, + "plugins": [ + { + "name": "auto", + "source": {"source": "url", "url": "https://example.com/auto.tgz"}, + "installationPreference": "sometimes", + } + ], + } + + with pytest.raises(ValidationError): + MarketplaceResponse.model_validate(catalog) diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 129dbaf9d5e..d16661b5221 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -1,6 +1,8 @@ +import asyncio import json import re from datetime import datetime, timezone +from types import SimpleNamespace from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -1534,3 +1536,43 @@ async def test_fetch_remote_autorouter_presets_parses_and_rejects_empty(monkeypa response.content = b"{}" with pytest.raises(ValueError, match="empty"): await _fetch_remote_autorouter_presets("https://example.test/presets.json") + + +@pytest.mark.parametrize("preference", ["auto_install", None]) +def test_public_skill_hub_echoes_registered_installation_preference(preference): + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import register_plugin + from litellm.types.proxy.claude_code_endpoints import RegisterPluginRequest + + table: Final = MagicMock() + table.find_unique = AsyncMock(return_value=None) + table.create = AsyncMock(side_effect=lambda data: SimpleNamespace(id="plugin-id", **data)) + table.find_many = AsyncMock( + side_effect=lambda where: [SimpleNamespace(id="plugin-id", **table.create.call_args.kwargs["data"])] + ) + prisma: Final = MagicMock() + prisma.db.litellm_claudecodeplugintable = table + app: Final = FastAPI() + app.include_router(router) + + with patch("litellm.proxy.proxy_server.prisma_client", prisma): + asyncio.run( + register_plugin( + request=RegisterPluginRequest( + name="security-review", + source={ + "source": "archive", + "url": "https://plugins.example.com/security-review-1.0.0.zip", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }, + installation_preference=preference, + ), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin"), + ) + ) + response: Final = TestClient(app).get("/public/skill_hub") + + assert response.status_code == 200 + assert [(p["name"], p["installation_preference"]) for p in response.json()["plugins"]] == [ + ("security-review", preference) + ] diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 98071b81e6f..356684ee937 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -2449,6 +2449,7 @@ export interface paths { * - homepage: Plugin homepage URL (optional) * - keywords: Search keywords (optional) * - category: Plugin category (optional) + * - installation_preference: 'available', 'auto_install', or 'required' (optional) * * Returns: * Registration status (action is always "created") and plugin information. @@ -2515,6 +2516,7 @@ export interface paths { * - homepage: Plugin homepage URL (optional) * - keywords: Search keywords (optional) * - category: Plugin category (optional) + * - installation_preference: 'available', 'auto_install', or 'required' (optional) * * Returns: * Update status (action is always "updated") and plugin information. @@ -40238,6 +40240,8 @@ export interface components { homepage?: string | null; /** Id */ id: string; + /** Installation Preference */ + installation_preference?: ("available" | "auto_install" | "required") | null; /** Keywords */ keywords?: string[] | null; /** Name */ @@ -40273,6 +40277,11 @@ export interface components { * @description Plugin unique ID */ id: string; + /** + * Installation Preference + * @description Plugin installation preference + */ + installation_preference?: ("available" | "auto_install" | "required") | null; /** * Name * @description Plugin name @@ -41691,6 +41700,11 @@ export interface components { * @description Plugin homepage URL */ homepage?: string | null; + /** + * Installation Preference + * @description Emitted as installationPreference on this plugin's marketplace.json entry. Needs an archive source, the only plugins Claude Desktop lists from a marketplace.json URL, and 'auto_install' or 'required' also need its sha256. Claude Desktop honors the mark only when the archive and marketplace.json share its inference gateway origin + */ + installation_preference?: ("available" | "auto_install" | "required") | null; /** * Keywords * @description Search keywords @@ -47848,6 +47862,11 @@ export interface components { * @description Plugin homepage URL */ homepage?: string | null; + /** + * Installation Preference + * @description Emitted as installationPreference on this plugin's marketplace.json entry. Needs an archive source, the only plugins Claude Desktop lists from a marketplace.json URL, and 'auto_install' or 'required' also need its sha256. Claude Desktop honors the mark only when the archive and marketplace.json share its inference gateway origin + */ + installation_preference?: ("available" | "auto_install" | "required") | null; /** * Keywords * @description Search keywords From 14e820ef7c35c0fda5bdd0c7dd2f6b81d2f8ae3d Mon Sep 17 00:00:00 2001 From: Oliver Jensen Date: Wed, 14 Oct 2026 10:01:21 +0200 Subject: [PATCH 16/51] fix(proxy): scope /global/activity/cache_hits analytics to the caller's own and permitted-team logs (#46210) Non-admin callers previously received deployment-wide cache activity, error breakdowns, every key alias on the proxy, and could filter the aggregates by any other tenant's key alias. All four queries now take the caller's read scope as bind parameters, which also shrinks the key-alias and model filter options to the caller's own visibility. The three-way scope resolution (admin / internal user / fallback) now lives in resolve_spend_log_read_scope, shared with /spend/tags and /spend/logs/session/ui instead of being duplicated per endpoint. --- .../analytics_endpoints.py | 8 ++ .../analytics_endpoints/cache_activity.py | 25 ++++- litellm/proxy/auth/authorization.py | 29 +++++ .../integration/spend/test_cache_activity.py | 103 +++++++++++++++--- .../test_analytics_endpoints.py | 64 +++++++++-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 3 + 6 files changed, 202 insertions(+), 30 deletions(-) diff --git a/litellm/proxy/analytics_endpoints/analytics_endpoints.py b/litellm/proxy/analytics_endpoints/analytics_endpoints.py index 3af88182d7d..3eb93a3e970 100644 --- a/litellm/proxy/analytics_endpoints/analytics_endpoints.py +++ b/litellm/proxy/analytics_endpoints/analytics_endpoints.py @@ -7,6 +7,8 @@ from fastapi import APIRouter, Depends, HTTPException, status from litellm.proxy._types import * from litellm.proxy.analytics_endpoints.cache_activity import CacheActivityResponse, get_cache_activity +from litellm.proxy.auth.authorization import resolve_spend_log_read_scope +from litellm.proxy.auth.authorization_dependencies import LogTeamLookupDependency from litellm.proxy.auth.user_api_key_auth import user_api_key_auth router: Final = APIRouter() @@ -30,6 +32,8 @@ def _parse_date(value: str, param_name: str) -> datetime: include_in_schema=False, ) async def get_global_activity( + log_team_lookup: LogTeamLookupDependency, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], start_date: Annotated[str, fastapi.Query(description="Time from which to start viewing spend")], end_date: Annotated[str, fastapi.Query(description="Time till which to view spend")], key_aliases: Annotated[ @@ -41,6 +45,9 @@ async def get_global_activity( Cache activity for the Admin UI cache dashboard, aggregated per call_type: cache hits vs successful LLM API requests vs failed requests, plus totals for the stat cards and the available key-alias/model filter options. + + Admin callers see proxy-wide activity; all other callers see only their own + spend logs and those of teams they may view, exactly like ``/spend/logs/ui``. """ from litellm.proxy.proxy_server import prisma_client @@ -54,6 +61,7 @@ async def get_global_activity( return await get_cache_activity( prisma_client=prisma_client, + read_scope=await resolve_spend_log_read_scope(user_api_key_dict, log_team_lookup), start_date=_parse_date(start_date, "start_date"), end_date=_parse_date(end_date, "end_date"), key_aliases=key_aliases or [], diff --git a/litellm/proxy/analytics_endpoints/cache_activity.py b/litellm/proxy/analytics_endpoints/cache_activity.py index c860f3c8a88..8a3f71e656e 100644 --- a/litellm/proxy/analytics_endpoints/cache_activity.py +++ b/litellm/proxy/analytics_endpoints/cache_activity.py @@ -7,6 +7,7 @@ from typing import Final, Protocol from pydantic import TypeAdapter from litellm.proxy._types import LiteLLMRoutes +from litellm.proxy.auth.authorization import ReadScope, read_scope_bind_params from litellm.types.llms.base import LiteLLMBaseModel UNKNOWN_CALL_TYPE: Final = "Unknown" @@ -84,6 +85,9 @@ GROUPS_SQL: Final = """ AND ($4::jsonb = '[]'::jsonb OR sl."model" IN (SELECT jsonb_array_elements_text($4::jsonb))) AND sl."call_type" NOT IN (SELECT jsonb_array_elements_text($5::jsonb)) + AND ($6::boolean + OR sl."user" = $7 + OR sl."team_id" IN (SELECT jsonb_array_elements_text($8::jsonb))) GROUP BY 1 ORDER BY (COUNT(*)) DESC """ @@ -105,6 +109,9 @@ ERROR_BREAKDOWN_SQL: Final = """ AND ($4::jsonb = '[]'::jsonb OR sl."model" IN (SELECT jsonb_array_elements_text($4::jsonb))) AND sl."call_type" NOT IN (SELECT jsonb_array_elements_text($5::jsonb)) + AND ($6::boolean + OR sl."user" = $7 + OR sl."team_id" IN (SELECT jsonb_array_elements_text($8::jsonb))) GROUP BY 1, 2, 3 ORDER BY (COUNT(*)) DESC """ @@ -117,6 +124,9 @@ KEY_ALIAS_OPTIONS_SQL: Final = """ sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC') AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') AND sl."call_type" NOT IN (SELECT jsonb_array_elements_text($3::jsonb)) + AND ($4::boolean + OR sl."user" = $5 + OR sl."team_id" IN (SELECT jsonb_array_elements_text($6::jsonb))) ORDER BY 1 """ @@ -128,6 +138,9 @@ MODEL_OPTIONS_SQL: Final = """ AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') AND sl."model" != '' AND sl."call_type" NOT IN (SELECT jsonb_array_elements_text($3::jsonb)) + AND ($4::boolean + OR sl."user" = $5 + OR sl."team_id" IN (SELECT jsonb_array_elements_text($6::jsonb))) ORDER BY 1 """ @@ -162,6 +175,7 @@ def compute_totals(groups: Sequence[CacheActivityGroup]) -> CacheActivityTotals: async def get_cache_activity( prisma_client: _SupportsRawQueryDb, + read_scope: ReadScope, start_date: datetime, end_date: datetime, key_aliases: Sequence[str], @@ -169,13 +183,16 @@ async def get_cache_activity( ) -> CacheActivityResponse: key_aliases_json: Final = json.dumps(list(key_aliases)) models_json: Final = json.dumps(list(models)) + scope_params: Final = read_scope_bind_params(read_scope) group_rows, error_rows, key_alias_rows, model_rows = await asyncio.gather( - prisma_client.db.query_raw(GROUPS_SQL, start_date, end_date, key_aliases_json, models_json, INFO_ROUTES_JSON), prisma_client.db.query_raw( - ERROR_BREAKDOWN_SQL, start_date, end_date, key_aliases_json, models_json, INFO_ROUTES_JSON + GROUPS_SQL, start_date, end_date, key_aliases_json, models_json, INFO_ROUTES_JSON, *scope_params ), - prisma_client.db.query_raw(KEY_ALIAS_OPTIONS_SQL, start_date, end_date, INFO_ROUTES_JSON), - prisma_client.db.query_raw(MODEL_OPTIONS_SQL, start_date, end_date, INFO_ROUTES_JSON), + prisma_client.db.query_raw( + ERROR_BREAKDOWN_SQL, start_date, end_date, key_aliases_json, models_json, INFO_ROUTES_JSON, *scope_params + ), + prisma_client.db.query_raw(KEY_ALIAS_OPTIONS_SQL, start_date, end_date, INFO_ROUTES_JSON, *scope_params), + prisma_client.db.query_raw(MODEL_OPTIONS_SQL, start_date, end_date, INFO_ROUTES_JSON, *scope_params), ) groups: Final = _groups_adapter.validate_python(group_rows or []) return CacheActivityResponse( diff --git a/litellm/proxy/auth/authorization.py b/litellm/proxy/auth/authorization.py index bb498c33763..2ca007a5739 100644 --- a/litellm/proxy/auth/authorization.py +++ b/litellm/proxy/auth/authorization.py @@ -1,5 +1,7 @@ +import json from collections.abc import Awaitable, Callable, Iterable, Sequence from dataclasses import dataclass +from functools import partial from typing import Final, TypeAlias from litellm.proxy._types import KeyManagementRoutes, LiteLLM_TeamTable, LitellmUserRoles, UserAPIKeyAuth @@ -66,6 +68,33 @@ async def can_read_log_owner( return False +async def resolve_spend_log_read_scope( + auth: UserAPIKeyAuth, + permitted_team_lookup: Callable[[UserAPIKeyAuth], Awaitable[Sequence[str]]], +) -> ReadScope: + """Admin roles read every spend log; internal users read their own rows and + those of teams they may view; any other caller reads only their own rows, + which is no rows at all when the key carries no user_id.""" + if auth.user_role in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): + return AllRows() + is_internal_user: Final = auth.user_role in ( + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + ) + if is_internal_user and auth.user_id is not None: + return await resolve_owned_read_scope(auth.user_id, partial(permitted_team_lookup, auth)) + return OwnedRows(auth.user_id) + + +def read_scope_bind_params(scope: ReadScope) -> tuple[bool, str | None, str]: + """Encode a scope as (grant_all, user_id, team_ids_json) bind parameters, so a + static SQL clause like ($N::boolean OR "user" = $N+1 OR team_id IN (...$N+2)) + can enforce it without interpolating SQL text.""" + if isinstance(scope, AllRows): + return (True, None, "[]") + return (False, scope.user_id, json.dumps(list(scope.team_ids))) + + async def resolve_trace_read_scope( auth: UserAPIKeyAuth, permitted_team_lookup: Callable[[], Awaitable[Sequence[str]]], diff --git a/tests/integration/spend/test_cache_activity.py b/tests/integration/spend/test_cache_activity.py index c381d51c402..aebba663694 100644 --- a/tests/integration/spend/test_cache_activity.py +++ b/tests/integration/spend/test_cache_activity.py @@ -31,11 +31,18 @@ AT_NOON: Final = DAY.replace(hour=12) RUN: Final = uuid.uuid4() INFERENCE_KEY: Final = f"ca-inference-{RUN}" INFO_ONLY_KEY: Final = f"ca-info-only-{RUN}" +OUTSIDER_KEY: Final = f"ca-outsider-{RUN}" INFERENCE_ALIAS: Final = f"alias-inference-{RUN}" INFO_ONLY_ALIAS: Final = f"alias-info-only-{RUN}" +OUTSIDER_ALIAS: Final = f"alias-outsider-{RUN}" INFERENCE_MODEL: Final = f"gpt-5.4-mini-{RUN}" INFO_ONLY_MODEL: Final = f"ghost-model-{RUN}" NO_FILTER: Final = "[]" +OWNER: Final = f"owner-{RUN}" +OWNER_TEAM: Final = f"team-{RUN}" +OUTSIDER: Final = f"outsider-{RUN}" +GRANT_ALL: Final = (True, None, "[]") +NO_GRANTS: Final = (False, None, "[]") @pytest_asyncio.fixture(scope="module", loop_scope="session") @@ -44,11 +51,21 @@ async def db() -> AsyncIterator[Prisma]: yield client -async def _spend_log(db, api_key: str, call_type: str, status: str, model: str = "", error_code: str = "") -> None: +async def _spend_log( + db, + api_key: str, + call_type: str, + status: str, + model: str = "", + error_code: str = "", + user: str = "", + team_id: str = "", +) -> None: metadata: Final = {"error_information": {"error_code": error_code, "error_class": "ProxyException"}} await db.execute_raw( 'INSERT INTO "LiteLLM_SpendLogs" ("request_id", "call_type", "api_key", "startTime", "endTime", "model", ' - '"status", "metadata") VALUES ($1, $2, $3, $4::timestamp, $4::timestamp, $5, $6, $7::jsonb)', + '"status", "metadata", "user", "team_id") ' + "VALUES ($1, $2, $3, $4::timestamp, $4::timestamp, $5, $6, $7::jsonb, $8, $9)", str(uuid.uuid4()), call_type, api_key, @@ -56,29 +73,38 @@ async def _spend_log(db, api_key: str, call_type: str, status: str, model: str = model, status, json.dumps(metadata if status == "failure" else {}), + user, + team_id, ) @pytest.fixture(scope="module", autouse=True) async def seeded(db): - for token, alias in ((INFERENCE_KEY, INFERENCE_ALIAS), (INFO_ONLY_KEY, INFO_ONLY_ALIAS)): + for token, alias in ( + (INFERENCE_KEY, INFERENCE_ALIAS), + (INFO_ONLY_KEY, INFO_ONLY_ALIAS), + (OUTSIDER_KEY, OUTSIDER_ALIAS), + ): await db.execute_raw( 'INSERT INTO "LiteLLM_VerificationToken" ("token", "key_alias") VALUES ($1, $2)', token, alias ) - await _spend_log(db, INFERENCE_KEY, "acompletion", "success", model=INFERENCE_MODEL) - await _spend_log(db, INFERENCE_KEY, "acompletion", "failure", model=INFERENCE_MODEL, error_code="429") - await _spend_log(db, INFERENCE_KEY, "", "failure", error_code="401") - await _spend_log(db, INFERENCE_KEY, "/model/info", "failure", error_code="401") - await _spend_log(db, INFO_ONLY_KEY, "/v1/models", "failure", model=INFO_ONLY_MODEL, error_code="401") - await _spend_log(db, INFO_ONLY_KEY, "/key/info", "success") + await _spend_log(db, INFERENCE_KEY, "acompletion", "success", model=INFERENCE_MODEL, user=OWNER) + await _spend_log( + db, INFERENCE_KEY, "acompletion", "failure", model=INFERENCE_MODEL, error_code="429", team_id=OWNER_TEAM + ) + await _spend_log(db, INFERENCE_KEY, "", "failure", error_code="401", user=OUTSIDER) + await _spend_log(db, INFERENCE_KEY, "/model/info", "failure", error_code="401", user=OWNER) + await _spend_log(db, INFO_ONLY_KEY, "/v1/models", "failure", model=INFO_ONLY_MODEL, error_code="401", user=OUTSIDER) + await _spend_log(db, INFO_ONLY_KEY, "/key/info", "success", user=OUTSIDER) + await _spend_log(db, OUTSIDER_KEY, "acompletion", "success", model=INFERENCE_MODEL, user=OUTSIDER) yield - keys: Final = [INFERENCE_KEY, INFO_ONLY_KEY] + keys: Final = [INFERENCE_KEY, INFO_ONLY_KEY, OUTSIDER_KEY] await db.execute_raw('DELETE FROM "LiteLLM_SpendLogs" WHERE "api_key" = ANY($1::text[])', keys) await db.execute_raw('DELETE FROM "LiteLLM_VerificationToken" WHERE "token" = ANY($1::text[])', keys) -async def _groups(db, key_aliases: list[str]) -> dict[str, dict]: - rows: Final = await db.query_raw(GROUPS_SQL, DAY, DAY, json.dumps(key_aliases), NO_FILTER, INFO_ROUTES_JSON) +async def _groups(db, key_aliases: list[str], scope: tuple = GRANT_ALL) -> dict[str, dict]: + rows: Final = await db.query_raw(GROUPS_SQL, DAY, DAY, json.dumps(key_aliases), NO_FILTER, INFO_ROUTES_JSON, *scope) return {row["call_type"]: row for row in rows} @@ -95,7 +121,13 @@ async def test_key_with_only_info_route_traffic_has_no_groups(db): async def test_error_breakdown_drops_info_routes(db): rows: Final = await db.query_raw( - ERROR_BREAKDOWN_SQL, DAY, DAY, json.dumps([INFERENCE_ALIAS, INFO_ONLY_ALIAS]), NO_FILTER, INFO_ROUTES_JSON + ERROR_BREAKDOWN_SQL, + DAY, + DAY, + json.dumps([INFERENCE_ALIAS, INFO_ONLY_ALIAS]), + NO_FILTER, + INFO_ROUTES_JSON, + *GRANT_ALL, ) assert {(row["call_type"], row["error_code"], row["count"]) for row in rows} == { ("acompletion", "429", 1), @@ -104,9 +136,50 @@ async def test_error_breakdown_drops_info_routes(db): async def test_filter_options_only_offer_values_that_return_analytics(db): - key_alias_rows: Final = await db.query_raw(KEY_ALIAS_OPTIONS_SQL, DAY, DAY, INFO_ROUTES_JSON) - model_rows: Final = await db.query_raw(MODEL_OPTIONS_SQL, DAY, DAY, INFO_ROUTES_JSON) + key_alias_rows: Final = await db.query_raw(KEY_ALIAS_OPTIONS_SQL, DAY, DAY, INFO_ROUTES_JSON, *GRANT_ALL) + model_rows: Final = await db.query_raw(MODEL_OPTIONS_SQL, DAY, DAY, INFO_ROUTES_JSON, *GRANT_ALL) key_aliases: Final = {row["key_alias"] for row in key_alias_rows} models: Final = {row["model"] for row in model_rows} assert INFERENCE_ALIAS in key_aliases and INFO_ONLY_ALIAS not in key_aliases assert INFERENCE_MODEL in models and INFO_ONLY_MODEL not in models + + +def _owner_scope(team_ids: list[str]) -> tuple: + return (False, OWNER, json.dumps(team_ids)) + + +async def test_scoped_groups_cover_own_rows_and_permitted_team_rows_only(db): + own_only: Final = await _groups(db, [INFERENCE_ALIAS], scope=_owner_scope([])) + assert {(g["api_requests"], g["failed_requests"]) for g in own_only.values()} == {(1, 0)} + + with_team: Final = await _groups(db, [INFERENCE_ALIAS], scope=_owner_scope([OWNER_TEAM])) + assert (with_team["acompletion"]["api_requests"], with_team["acompletion"]["failed_requests"]) == (1, 1) + assert "Unknown" not in with_team, "the outsider's row must stay invisible" + + +async def test_scope_without_grants_returns_nothing(db): + assert await _groups(db, [INFERENCE_ALIAS], scope=NO_GRANTS) == {} + + +async def test_scoped_error_breakdown_excludes_other_users_failures(db): + rows: Final = await db.query_raw( + ERROR_BREAKDOWN_SQL, + DAY, + DAY, + json.dumps([INFERENCE_ALIAS, INFO_ONLY_ALIAS]), + NO_FILTER, + INFO_ROUTES_JSON, + *_owner_scope([OWNER_TEAM]), + ) + assert {(row["call_type"], row["error_code"], row["count"]) for row in rows} == {("acompletion", "429", 1)} + + +async def test_scoped_filter_options_hide_other_users_key_aliases(db): + unscoped_rows: Final = await db.query_raw(KEY_ALIAS_OPTIONS_SQL, DAY, DAY, INFO_ROUTES_JSON, *GRANT_ALL) + assert OUTSIDER_ALIAS in {row["key_alias"] for row in unscoped_rows}, ( + "seed must make the outsider visible to admins" + ) + + scoped_rows: Final = await db.query_raw(KEY_ALIAS_OPTIONS_SQL, DAY, DAY, INFO_ROUTES_JSON, *_owner_scope([])) + scoped_aliases: Final = {row["key_alias"] for row in scoped_rows} + assert INFERENCE_ALIAS in scoped_aliases and OUTSIDER_ALIAS not in scoped_aliases diff --git a/tests/unit/proxy/analytics_endpoints/test_analytics_endpoints.py b/tests/unit/proxy/analytics_endpoints/test_analytics_endpoints.py index 69188ebd5a2..1c706978fed 100644 --- a/tests/unit/proxy/analytics_endpoints/test_analytics_endpoints.py +++ b/tests/unit/proxy/analytics_endpoints/test_analytics_endpoints.py @@ -7,12 +7,13 @@ totals for the stat cards, and the filter options for the UI dropdowns. """ import json +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import HTTPException -from litellm.proxy._types import LiteLLMRoutes +from litellm.proxy._types import LiteLLMRoutes, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.analytics_endpoints.analytics_endpoints import get_global_activity from litellm.proxy.analytics_endpoints.cache_activity import ( ERROR_BREAKDOWN_SQL, @@ -23,6 +24,23 @@ from litellm.proxy.analytics_endpoints.cache_activity import ( compute_totals, ) +ADMIN: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin-user") +INTERNAL_USER: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="user-1") + + +async def _no_teams(auth: UserAPIKeyAuth) -> tuple[str, ...]: + return () + + +async def _team_nine(auth: UserAPIKeyAuth) -> tuple[str, ...]: + return ("team-9",) + + +async def _activity(auth: UserAPIKeyAuth, team_lookup=_no_teams, **overrides): + params = {"start_date": "2026-07-01", "end_date": "2026-07-27", "key_aliases": [], "models": [], **overrides} + return await get_global_activity(log_team_lookup=team_lookup, user_api_key_dict=auth, **params) + + GROUP_ROWS = [ { "call_type": "acompletion", @@ -78,7 +96,7 @@ def mock_prisma(monkeypatch: pytest.MonkeyPatch) -> MagicMock: @pytest.mark.asyncio async def test_returns_groups_totals_and_filter_options(mock_prisma: MagicMock): - response = await get_global_activity(start_date="2026-07-01", end_date="2026-07-27", key_aliases=[], models=[]) + response = await _activity(ADMIN) assert [group.call_type for group in response.groups] == ["acompletion", "Unknown"] assert response.groups[0].api_requests == 1000 @@ -99,12 +117,7 @@ async def test_returns_groups_totals_and_filter_options(mock_prisma: MagicMock): @pytest.mark.asyncio async def test_filters_are_passed_to_sql_as_json_arrays(mock_prisma: MagicMock): - await get_global_activity( - start_date="2026-07-01", - end_date="2026-07-27", - key_aliases=["my-key"], - models=["gpt-5.1", "claude-opus-4-8"], - ) + await _activity(ADMIN, key_aliases=["my-key"], models=["gpt-5.1", "claude-opus-4-8"]) filtered_calls = [ call for call in mock_prisma.db.query_raw.call_args_list if call.args[0] in (GROUPS_SQL, ERROR_BREAKDOWN_SQL) @@ -120,9 +133,9 @@ async def test_every_query_excludes_the_same_info_routes(mock_prisma: MagicMock) """Regression for LIT-5884: failed info-route calls are spend-logged but are not inference traffic, so the groups, error breakdown and both filter-option queries all receive the same exclusion list. What the SQL does with it is covered against Postgres in tests/integration/spend/test_cache_activity.py.""" - await get_global_activity(start_date="2026-07-01", end_date="2026-07-27", key_aliases=[], models=[]) + await _activity(ADMIN) - exclusions_by_query = {call.args[0]: json.loads(call.args[-1]) for call in mock_prisma.db.query_raw.call_args_list} + exclusions_by_query = {call.args[0]: json.loads(call.args[-4]) for call in mock_prisma.db.query_raw.call_args_list} assert set(exclusions_by_query) == {GROUPS_SQL, ERROR_BREAKDOWN_SQL, KEY_ALIAS_OPTIONS_SQL, MODEL_OPTIONS_SQL} for excluded_call_types in exclusions_by_query.values(): assert excluded_call_types == LiteLLMRoutes.info_routes.value @@ -130,10 +143,39 @@ async def test_every_query_excludes_the_same_info_routes(mock_prisma: MagicMock) assert "" not in excluded_call_types +@pytest.mark.asyncio +async def test_admin_queries_grant_all_rows(mock_prisma: MagicMock): + await _activity(ADMIN) + + scope_by_query = {call.args[0]: call.args[-3:] for call in mock_prisma.db.query_raw.call_args_list} + assert set(scope_by_query) == {GROUPS_SQL, ERROR_BREAKDOWN_SQL, KEY_ALIAS_OPTIONS_SQL, MODEL_OPTIONS_SQL} + for scope_args in scope_by_query.values(): + assert scope_args == (True, None, "[]") + + +@pytest.mark.asyncio +async def test_internal_user_queries_are_scoped_to_own_and_team_rows(mock_prisma: MagicMock): + await _activity(INTERNAL_USER, team_lookup=_team_nine) + + scope_by_query = {call.args[0]: call.args[-3:] for call in mock_prisma.db.query_raw.call_args_list} + assert set(scope_by_query) == {GROUPS_SQL, ERROR_BREAKDOWN_SQL, KEY_ALIAS_OPTIONS_SQL, MODEL_OPTIONS_SQL} + for scope_args in scope_by_query.values(): + assert scope_args == (False, "user-1", json.dumps(["team-9"])) + + +@pytest.mark.asyncio +async def test_key_without_user_id_grants_no_rows(mock_prisma: MagicMock): + no_user = UserAPIKeyAuth(user_role=LitellmUserRoles.TEAM, user_id=None) + await _activity(no_user) + + for call in mock_prisma.db.query_raw.call_args_list: + assert call.args[-3:] == (False, None, "[]") + + @pytest.mark.asyncio async def test_rejects_malformed_dates_with_400(mock_prisma: MagicMock): with pytest.raises(HTTPException) as exc_info: - await get_global_activity(start_date="07/01/2026", end_date="2026-07-27", key_aliases=[], models=[]) + await _activity(ADMIN, start_date="07/01/2026") assert exc_info.value.status_code == 400 mock_prisma.db.query_raw.assert_not_called() diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 356684ee937..2cc91879f7e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -6147,6 +6147,9 @@ export interface paths { * @description Cache activity for the Admin UI cache dashboard, aggregated per call_type: * cache hits vs successful LLM API requests vs failed requests, plus totals * for the stat cards and the available key-alias/model filter options. + * + * Admin callers see proxy-wide activity; all other callers see only their own + * spend logs and those of teams they may view, exactly like ``/spend/logs/ui``. */ get: operations["get_global_activity_global_activity_cache_hits_get"]; put?: never; From b37eac29f0a9be3c222c32d84a553b19cb006f3e Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 14 Oct 2026 01:12:11 -0700 Subject: [PATCH 17/51] fix(anthropic): cache tool result screenshots on the /v1/messages to Responses bridge (#46324) * fix(anthropic): put a tool_result cache breakpoint on its hoisted images on the Responses bridge A marked tool_result whose images get moved into a trailing user message now carries the breakpoint on the last moved image instead of the function_call_output, so the prefix OpenAI caches includes the images * test(anthropic): cover a marked tool_result with two screenshots on the Responses bridge * fix(anthropic): keep every tool result screenshot inside the breakpoint that covers it on the Responses bridge A marked tool_result's screenshots now go right after its output, so earlier unmarked screenshots land inside the breakpoint and later tool outputs stay after it. A cache_control mark on a block inside a tool_result is read too. * fix(anthropic): cover only the screenshots up to a mark inside a tool result, and drop raw breakpoints under drop_params A cache mark on a text block inside a tool result now covers that result only up to the marked block, the way Anthropic caches it, so screenshots after the marked text stay outside the breakpoint. A mark on the tool result itself still covers every screenshot in it A deployment that reads no prompt cache breakpoints now drops raw prompt_cache_breakpoint marks, top-level and inside tool results, when drop_params is on, and still forwards them as sent without it * fix(anthropic): keep raw breakpoints under drop_params for models that take them * docs(anthropic): say a mark inside a tool result covers its whole text and documents --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: yuneng-jiang --- .../responses_adapters/handler.py | 62 ++++-- .../responses_adapters/transformation.py | 125 +++++++++--- ...ses_bridge_prompt_cache_breakpoint_wire.py | 36 ++++ .../test_responses_adapters_handler.py | 108 ++++++++++ .../test_responses_adapters_transformation.py | 184 +++++++++++++++++- 5 files changed, 474 insertions(+), 41 deletions(-) diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py index 5c5fb71c295..42188e536a1 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py @@ -4,13 +4,17 @@ Handler for the Anthropic v1/messages -> OpenAI Responses API path. Used when the target model is an OpenAI or Azure model. """ -from collections.abc import AsyncIterator, Coroutine, Mapping, Sequence +from collections.abc import AsyncIterator, Callable, Coroutine, Mapping, Sequence from typing import Final, TypeAlias from pydantic import TypeAdapter, ValidationError import litellm -from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook +from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + supports_openai_prompt_cache_breakpoint, +) +from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( @@ -31,6 +35,7 @@ from .streaming_iterator import AnthropicResponsesStreamWrapper from .transformation import LiteLLMAnthropicToResponsesAPIAdapter AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]] +BlockRewrite: TypeAlias = Callable[[Mapping[str, object]], Mapping[str, object]] _BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]]) _ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter() @@ -60,10 +65,46 @@ def _reads_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, ob ) -def _blocks_with_prompt_cache_breakpoints(content: object) -> Sequence[Mapping[str, object]]: - """The content's blocks with each ``cache_control`` mark rewritten, empty when it is not a list of blocks.""" +def _model_takes_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, object]) -> bool: + provider: Final = forwarded_kwargs.get("custom_llm_provider") + provider_name: Final = provider if isinstance(provider, str) else None + base_model: Final = forwarded_kwargs.get("base_model") + return supports_openai_prompt_cache_breakpoint(model, provider_name) or ( + isinstance(base_model, str) + and bool(base_model) + and supports_openai_prompt_cache_breakpoint(base_model, provider_name) + ) + + +def _without_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str, object]: + return {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"} + + +def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, object]) -> BlockRewrite | None: + """Client cache marks become breakpoints for a deployment that reads them, and a model that takes no breakpoints + drops raw ones under drop_params.""" + if _reads_prompt_cache_breakpoint(model, forwarded_kwargs): + return cache_control_as_prompt_cache_breakpoint + if _model_takes_prompt_cache_breakpoint(model, forwarded_kwargs): + return None + if normalize_drop_params(forwarded_kwargs.get("drop_params")) is True or litellm.drop_params is True: + return _without_prompt_cache_breakpoint + return None + + +def _rewritten_block(block: Mapping[str, object], rewrite: BlockRewrite) -> Mapping[str, object]: + rewritten: Final = rewrite(block) + if rewritten.get("type") != "tool_result": + return rewritten + inner_blocks: Final = _rewritten_blocks(rewritten.get("content"), rewrite) + return {**rewritten, "content": list(inner_blocks)} if inner_blocks else rewritten + + +def _rewritten_blocks(content: object, rewrite: BlockRewrite) -> Sequence[Mapping[str, object]]: + """The content's blocks each rewritten, a tool_result's inner blocks included, empty when it is not a list of + blocks.""" try: - return [cache_control_as_prompt_cache_breakpoint(block) for block in _BLOCK_LIST.validate_python(content)] + return [_rewritten_block(block, rewrite) for block in _BLOCK_LIST.validate_python(content)] except ValidationError: return () @@ -92,22 +133,19 @@ def _build_responses_kwargs( Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses(). """ forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs) - reads_prompt_cache_breakpoint: Final = _reads_prompt_cache_breakpoint(model, forwarded_kwargs) + block_rewrite: Final = _prompt_cache_block_rewrite(model, forwarded_kwargs) request_messages: Final[AnthropicRequestMessages] = ( [ { **message, - "content": list(_blocks_with_prompt_cache_breakpoints(message.get("content"))) - or message.get("content"), + "content": list(_rewritten_blocks(message.get("content"), block_rewrite)) or message.get("content"), } for message in messages ] - if reads_prompt_cache_breakpoint + if block_rewrite else messages ) - request_system: Final = ( - (list(_blocks_with_prompt_cache_breakpoints(system)) or system) if reads_prompt_cache_breakpoint else system - ) + request_system: Final = (list(_rewritten_blocks(system, block_rewrite)) or system) if block_rewrite else system # Build a typed AnthropicMessagesRequest for the adapter request_data: Final[AnthropicMessagesRequest] = { diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py index e033f1562cd..a9f7a554041 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py @@ -6,7 +6,7 @@ path used for OpenAI and Azure models. """ import json -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Mapping, Sequence from itertools import groupby from typing import Any, Final, cast @@ -127,17 +127,93 @@ class LiteLLMAnthropicToResponsesAPIAdapter: return {"type": "input_file", "file_url": url} return None + @staticmethod + def _content_blocks(content: object) -> tuple[Mapping[str, object], ...]: + if not isinstance(content, list): + return () + return tuple( + cast(Mapping[str, object], block) # cast-ok: untrusted client payload + for block in cast(Sequence[object], content) # cast-ok: untyped client content list + if isinstance(block, dict) + ) + + @classmethod + def _tool_result_image_parts( + cls, blocks: Sequence[Mapping[str, object]] + ) -> tuple[dict[str, object], ...]: # mutable-ok: json parts + """Each image block as an input_image part, keeping a breakpoint the image carries.""" + sources: Final = tuple( + (cls._translate_anthropic_image_source_to_url(block.get("source")), block.get("prompt_cache_breakpoint")) + for block in blocks + if block.get("type") == "image" + ) + return tuple( + with_prompt_cache_breakpoint({"type": "input_image", "image_url": url}, breakpoint) + for url, breakpoint in sources + if url + ) + + @classmethod + def _tool_result_images_by_breakpoint( + cls, block: Mapping[str, object] + ) -> tuple[object, tuple[dict[str, object], ...], tuple[dict[str, object], ...]]: # mutable-ok: json parts + """The breakpoint covering a tool_result, its images at or before that breakpoint, and its images after it. + + A mark on the tool_result covers all its images, and a mark on a block inside it covers the images up to that + block, the images Anthropic caches for the mark. The result's text and documents join into one output, which + the mark covers whole. + """ + inner: Final = cls._content_blocks(block.get("content")) + own_breakpoint: Final = block.get("prompt_cache_breakpoint") + if own_breakpoint is not None: + return own_breakpoint, cls._tool_result_image_parts(inner), () + last_marked: Final = next( + ( + position + for position in reversed(range(len(inner))) + if inner[position].get("prompt_cache_breakpoint") is not None + ), + None, + ) + if last_marked is None: + return None, (), cls._tool_result_image_parts(inner) + return ( + inner[last_marked].get("prompt_cache_breakpoint"), + cls._tool_result_image_parts(inner[: last_marked + 1]), + cls._tool_result_image_parts(inner[last_marked + 1 :]), + ) + + @staticmethod + def _tool_result_images_message( + image_parts: Sequence[Mapping[str, object]], + prompt_cache_breakpoint: object, + ) -> dict[str, object]: # mutable-ok: API message payload + """Tool result images as one user message, the tool_result's breakpoint on the last image. + + OpenAI caches only up to a breakpoint, so a marked result's images go right after its output and ahead of any + later output, the images Anthropic caches for that mark. + """ + *earlier_images, last_image = image_parts + return { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": TOOL_RESULT_IMAGE_BOUNDARY}, + *earlier_images, + with_prompt_cache_breakpoint(last_image, prompt_cache_breakpoint), + ], + } + @staticmethod def _tool_result_output_value( output_text: str, file_parts: tuple[dict[str, str], ...], # mutable-ok: json content parts - tool_result: Mapping[str, object], + prompt_cache_breakpoint: object, ) -> str | list[dict[str, str]]: # mutable-ok: API message payload """Plain string output, or a part list when document file parts or a cache breakpoint are present. OpenAI reads a breakpoint only on a part, so a marked result ends on a part that carries it. """ - prompt_cache_breakpoint: Final = tool_result.get("prompt_cache_breakpoint") text_parts: Final = [{"type": "input_text", "text": output_text}] if output_text else [] parts: Final = [*text_parts, *file_parts] if not parts or (not file_parts and prompt_cache_breakpoint is None): @@ -292,6 +368,11 @@ class LiteLLMAnthropicToResponsesAPIAdapter: elif btype == "tool_result": tool_use_id = block.get("tool_use_id", "") inner = block.get("content") + tool_breakpoint, covered_image_parts, later_image_parts = ( + self._tool_result_images_by_breakpoint( + cast(Mapping[str, object], block) # cast-ok: TypedDict upcast + ) + ) document_candidates = ( tuple( self._translate_anthropic_document_block_to_file_part(c) @@ -311,43 +392,37 @@ class LiteLLMAnthropicToResponsesAPIAdapter: c.get("text", "") for c in inner if isinstance(c, dict) and c.get("type") == "text" ] output_text = "\n".join(parts) - image_candidates = tuple( - self._translate_anthropic_image_source_to_url(c.get("source")) - for c in inner - if isinstance(c, dict) and c.get("type") == "image" - ) - image_urls = tuple(url for url in image_candidates if url) - if image_urls: + if covered_image_parts or later_image_parts: output_text = ( f"{output_text}\n{TOOL_RESULT_IMAGE_PLACEHOLDER}" if output_text else TOOL_RESULT_IMAGE_PLACEHOLDER ) - tool_image_parts.extend( - {"type": "input_image", "image_url": url} for url in image_urls - ) else: output_text = str(inner) + images_inside_breakpoint = ( + (*tool_image_parts, *covered_image_parts) if tool_breakpoint is not None else () + ) # tool_result is a top-level item, not inside the message input_items.append( { "type": "function_call_output", "call_id": tool_use_id, - "output": self._tool_result_output_value(output_text, tool_file_parts, block), + "output": self._tool_result_output_value( + output_text, + tool_file_parts, + None if images_inside_breakpoint else tool_breakpoint, + ), } ) + if images_inside_breakpoint: + input_items.append( + self._tool_result_images_message(images_inside_breakpoint, tool_breakpoint) + ) + tool_image_parts.clear() + tool_image_parts.extend(later_image_parts) if tool_image_parts: - boundary_part = { - "type": "input_text", - "text": TOOL_RESULT_IMAGE_BOUNDARY, - } - input_items.append( - { - "type": "message", - "role": "user", - "content": [boundary_part, *tool_image_parts], - } - ) + input_items.append(self._tool_result_images_message(tool_image_parts, None)) if user_parts: input_items.append( { diff --git a/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py b/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py index 055eec3f6f6..40a64af05ed 100644 --- a/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py +++ b/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py @@ -485,6 +485,42 @@ def test_anthropic_sdk_cache_control_on_a_tool_result_marks_its_function_call_ou ], "OpenAI caches up to a breakpoint on a function_call_output part (prompt caching guide, read 2026-10-12)" +def test_anthropic_sdk_cache_control_on_a_tool_result_with_a_screenshot_marks_the_screenshot(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + screenshot_result: Final = { + "type": "tool_result", + "tool_use_id": "call_1", + "content": [ + {"type": "text", "text": "captured"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "Zm9v"}}, + ], + "cache_control": _EPHEMERAL, + } + with bridge.gateway.scenario() as scenario, _anthropic(bridge) as client: + model: Final = scenario.model(model=pcb.MODEL, api_base=bridge.api_base, **_OPTED_IN) + message: Final = _anthropic_message( + client, + model, + [ + {"role": "user", "content": pcb.prompt(marker)}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "call_1", "name": "shot", "input": {}}]}, + {"role": "user", "content": [screenshot_result]}, + ], + ) + _anthropic_text(message, marker) + request: Final = pcb.posted(bridge.wire, marker) + _wire_body(request) + items: Final = pcb.input_items(request) + output_index: Final = next(index for index, item in enumerate(items) if item.get("call_id") == "call_1") + *_, screenshot = pcb.content_of(items[output_index + 1 :], "user") + assert screenshot == { + "type": "input_image", + "image_url": "data:image/png;base64,Zm9v", + "prompt_cache_breakpoint": pcb.EXPLICIT, + }, "OpenAI caches only the prefix up to the breakpoint, so the screenshot must sit inside it" + assert request.body.count(b"prompt_cache_breakpoint") == 1, request.body + + def test_anthropic_sdk_cache_control_on_an_assistant_turn_is_dropped(bridge: _Bridge) -> None: marker: Final = uuid.uuid4().hex messages: Final[list[anthropic.types.MessageParam]] = [ diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py index df565ef25b5..d458f408fbf 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py @@ -11,6 +11,10 @@ import respx sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) import litellm +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + TOOL_RESULT_IMAGE_BOUNDARY, + TOOL_RESULT_IMAGE_PLACEHOLDER, +) from litellm.llms.anthropic.pass_through.responses_adapters.handler import ( LiteLLMMessagesToResponsesAPIHandler, _build_responses_kwargs, @@ -255,6 +259,110 @@ def test_client_cache_control_reaches_gpt_5_6_as_a_prompt_cache_breakpoint_on_th ] +def test_client_cache_control_on_a_screenshot_inside_a_tool_result_reaches_gpt_5_6_on_that_screenshot() -> None: + responses_kwargs: Final = _build_responses_kwargs( + model="openai/gpt-5.6", + extra_kwargs=OPENAI_API, + max_tokens=1024, + messages=[ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_shot", + "content": [ + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": "c2hvdA=="}, + "cache_control": EPHEMERAL, + } + ], + } + ], + } + ], + ) + + assert responses_kwargs["input"] == [ + {"type": "function_call_output", "call_id": "toolu_shot", "output": TOOL_RESULT_IMAGE_PLACEHOLDER}, + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": TOOL_RESULT_IMAGE_BOUNDARY}, + { + "type": "input_image", + "image_url": "data:image/png;base64,c2hvdA==", + "prompt_cache_breakpoint": {"mode": "explicit"}, + }, + ], + }, + ] + + +RAW_BREAKPOINT: Final = {"mode": "explicit"} + + +RAW_LOG_OUTPUT: Final = [{"type": "input_text", "text": "build ok", "prompt_cache_breakpoint": RAW_BREAKPOINT}] +RAW_QUESTION: Final = {"type": "input_text", "text": "q", "prompt_cache_breakpoint": RAW_BREAKPOINT} + + +@pytest.mark.parametrize( + ("model", "extra_kwargs", "log_output", "question"), + [ + pytest.param("openai/gpt-5.5", OPENAI_API, RAW_LOG_OUTPUT, RAW_QUESTION, id="sent-as-is"), + pytest.param( + "openai/gpt-5.5", + {**OPENAI_API, "drop_params": True}, + "build ok", + {"type": "input_text", "text": "q"}, + id="dropped-under-drop-params", + ), + pytest.param( + "openai/gpt-5.6", + {**OTHER_HOST, "drop_params": True}, + RAW_LOG_OUTPUT, + RAW_QUESTION, + id="kept-under-drop-params-for-a-model-that-takes-them", + ), + pytest.param( + "openai/screenshots-prod", + {**OTHER_HOST, "drop_params": True, "base_model": "gpt-5.6"}, + RAW_LOG_OUTPUT, + RAW_QUESTION, + id="kept-under-drop-params-for-a-base-model-that-takes-them", + ), + ], +) +def test_raw_prompt_cache_breakpoints_reach_a_deployment_unless_drop_params_drops_them_for_a_model_that_takes_none( + model: str, extra_kwargs: dict[str, object], log_output: object, question: dict[str, object] +) -> None: + responses_kwargs: Final = _build_responses_kwargs( + model=model, + extra_kwargs=extra_kwargs, + max_tokens=1024, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "q", "prompt_cache_breakpoint": RAW_BREAKPOINT}, + { + "type": "tool_result", + "tool_use_id": "toolu_log", + "content": [{"type": "text", "text": "build ok", "prompt_cache_breakpoint": RAW_BREAKPOINT}], + }, + ], + } + ], + ) + + assert responses_kwargs["input"] == [ + {"type": "function_call_output", "call_id": "toolu_log", "output": log_output}, + {"type": "message", "role": "user", "content": [question]}, + ] + + @pytest.mark.parametrize( ("model", "extra_kwargs"), [ diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py index 1532b38bfae..69c2f9210c3 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -5,7 +5,7 @@ Tests for LiteLLMAnthropicToResponsesAPIAdapter import json import os -from typing import Any, Dict, List +from typing import Any, Dict, Final, List from unittest.mock import MagicMock import pytest @@ -623,9 +623,7 @@ class TestTranslateMessagesToResponsesInput: } ] result = _translate_messages(messages) - assert result == [ - {"type": "reasoning", "summary": [{"type": "summary_text", "text": "Private reasoning."}]} - ] + assert result == [{"type": "reasoning", "summary": [{"type": "summary_text", "text": "Private reasoning."}]}] def test_consecutive_thinking_blocks_become_one_reasoning_item(self): """Summary parts of one upstream reasoning item are regrouped into that item.""" @@ -2189,6 +2187,184 @@ class TestPromptCacheBreakpointToResponses: "output": [{"type": "input_text", "text": "r", "prompt_cache_breakpoint": self.EXPLICIT}], } + def test_a_marked_tool_result_with_screenshots_moves_its_breakpoint_to_the_last_screenshot(self): + items: Final = _ADAPTER.translate_messages_to_responses_input( + [ + {"role": "user", "content": [{"type": "text", "text": "q"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_01", "name": "t", "input": {}}]}, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [ + {"type": "text", "text": "r"}, + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": "Zmlyc3Q="}, + }, + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": "bGFzdA=="}, + }, + ], + "prompt_cache_breakpoint": self.EXPLICIT, + } + ], + }, + ] + ) + *_, tool_output, screenshots = items + assert tool_output == { + "type": "function_call_output", + "call_id": "toolu_01", + "output": f"r\n{TOOL_RESULT_IMAGE_PLACEHOLDER}", + } + *_, first_screenshot, last_screenshot = screenshots["content"] + assert first_screenshot == {"type": "input_image", "image_url": "data:image/png;base64,Zmlyc3Q="} + assert last_screenshot == { + "type": "input_image", + "image_url": "data:image/png;base64,bGFzdA==", + "prompt_cache_breakpoint": self.EXPLICIT, + } + + SCREENSHOT: Final = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "c2hvdA=="}} + MARKED_SCREENSHOT_PART: Final = { + "type": "input_image", + "image_url": "data:image/png;base64,c2hvdA==", + "prompt_cache_breakpoint": EXPLICIT, + } + LOG_OUTPUT: Final = {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok"} + + def test_an_unmarked_screenshot_before_a_marked_tool_result_lands_inside_that_breakpoint(self): + items: Final = _ADAPTER.translate_messages_to_responses_input( + [ + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_shot", "content": [self.SCREENSHOT]}, + { + "type": "tool_result", + "tool_use_id": "toolu_log", + "content": "build ok", + "prompt_cache_breakpoint": self.EXPLICIT, + }, + ], + } + ] + ) + _, log_output, screenshots = items + assert log_output == self.LOG_OUTPUT + assert screenshots["content"][-1] == self.MARKED_SCREENSHOT_PART + + @pytest.mark.parametrize("mark_on_the_screenshot", [False, True], ids=["mark-on-tool-result", "mark-on-screenshot"]) + def test_a_tool_result_after_a_marked_screenshot_stays_after_its_breakpoint(self, mark_on_the_screenshot: bool): + mark: Final = {"prompt_cache_breakpoint": self.EXPLICIT} + marked_result: Final = ( + {"type": "tool_result", "tool_use_id": "toolu_shot", "content": [{**self.SCREENSHOT, **mark}]} + if mark_on_the_screenshot + else {"type": "tool_result", "tool_use_id": "toolu_shot", "content": [self.SCREENSHOT], **mark} + ) + items: Final = _ADAPTER.translate_messages_to_responses_input( + [ + { + "role": "user", + "content": [ + marked_result, + {"type": "tool_result", "tool_use_id": "toolu_log", "content": "build ok"}, + ], + } + ] + ) + _, screenshots, log_output = items + assert screenshots["content"][-1] == self.MARKED_SCREENSHOT_PART + assert log_output == self.LOG_OUTPUT + + def test_a_breakpoint_on_text_inside_a_tool_result_marks_its_output(self): + items: Final = _ADAPTER.translate_messages_to_responses_input( + [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_log", + "content": [{"type": "text", "text": "build ok", "prompt_cache_breakpoint": self.EXPLICIT}], + } + ], + } + ] + ) + assert items == [ + { + "type": "function_call_output", + "call_id": "toolu_log", + "output": [{"type": "input_text", "text": "build ok", "prompt_cache_breakpoint": self.EXPLICIT}], + } + ] + + SCREENSHOT_PART: Final = {"type": "input_image", "image_url": "data:image/png;base64,c2hvdA=="} + MARKED_LOG_TEXT: Final = {"type": "text", "text": "build ok", "prompt_cache_breakpoint": EXPLICIT} + LOG_TEXT_WITH_SCREENSHOT: Final = f"build ok\n{TOOL_RESULT_IMAGE_PLACEHOLDER}" + + @pytest.mark.parametrize( + ("content", "output", "screenshot"), + [ + pytest.param( + [MARKED_LOG_TEXT, SCREENSHOT], + [{"type": "input_text", "text": LOG_TEXT_WITH_SCREENSHOT, "prompt_cache_breakpoint": EXPLICIT}], + SCREENSHOT_PART, + id="screenshot-after-the-marked-text", + ), + pytest.param( + [SCREENSHOT, MARKED_LOG_TEXT], + LOG_TEXT_WITH_SCREENSHOT, + MARKED_SCREENSHOT_PART, + id="screenshot-before-the-marked-text", + ), + ], + ) + def test_a_breakpoint_on_text_inside_a_tool_result_covers_only_the_screenshots_before_that_text( + self, content: list[dict[str, object]], output: object, screenshot: dict[str, object] + ): + items: Final = _ADAPTER.translate_messages_to_responses_input( + [{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_shot", "content": content}]}] + ) + assert items == [ + {"type": "function_call_output", "call_id": "toolu_shot", "output": output}, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": TOOL_RESULT_IMAGE_BOUNDARY}, screenshot], + }, + ] + + def test_a_later_tool_output_stays_after_the_last_marked_screenshot_of_an_earlier_result(self): + marked_screenshot: Final = {**self.SCREENSHOT, "prompt_cache_breakpoint": self.EXPLICIT} + items: Final = _ADAPTER.translate_messages_to_responses_input( + [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_shot", + "content": [marked_screenshot, marked_screenshot], + }, + {"type": "tool_result", "tool_use_id": "toolu_log", "content": "build ok"}, + ], + } + ] + ) + _, screenshots, log_output = items + assert screenshots["content"] == [ + {"type": "input_text", "text": TOOL_RESULT_IMAGE_BOUNDARY}, + self.MARKED_SCREENSHOT_PART, + self.MARKED_SCREENSHOT_PART, + ] + assert log_output == self.LOG_OUTPUT + def test_prompt_cache_options_forwarded_to_responses_kwargs(self): from litellm.llms.anthropic.pass_through.responses_adapters.handler import ( _build_responses_kwargs, From b22a0bd9e5531d443da2945951f99573c0a02413 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:31:51 +0000 Subject: [PATCH 18/51] test(integration): cover fine_tuning endpoint contracts (#44697) * test(integration): cover fine_tuning endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): prove fine_tuning ownership hook, header spelling and drain order Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(fine_tuning): assert unified job id and retrieve ownership through route-granted keys Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(fine_tuning): assert managed file and job ownership rows in Postgres Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(fine_tuning): cover validation_file ownership, default retrieve ownership and full job objects Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../compatibility/test_fine_tuning_jobs.py | 442 ++++++++++++++++ .../security/test_fine_tuning_managed_ids.py | 492 ++++++++++++++++++ 2 files changed, 934 insertions(+) create mode 100644 tests/integration/compatibility/test_fine_tuning_jobs.py create mode 100644 tests/integration/security/test_fine_tuning_managed_ids.py diff --git a/tests/integration/compatibility/test_fine_tuning_jobs.py b/tests/integration/compatibility/test_fine_tuning_jobs.py new file mode 100644 index 00000000000..9678198352f --- /dev/null +++ b/tests/integration/compatibility/test_fine_tuning_jobs.py @@ -0,0 +1,442 @@ +import base64 +import json +import re +import uuid +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from email.message import Message +from email.parser import BytesParser +from email.policy import HTTP +from typing import Final +from urllib.parse import urlsplit + +import pytest +import yaml +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from openai import OpenAI +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +OPENAI_KEY: Final = "sk-fine-tuning-openai" +AZURE_KEY: Final = "azure-fine-tuning-key" +AMBIENT_OPENAI_KEY: Final = "sk-fine-tuning-ambient-environment" +AMBIENT_AZURE_KEY: Final = "azure-fine-tuning-ambient-environment" +AZURE_API_VERSION: Final = "2024-10-21" +MANAGED_MODEL: Final = "gpt-4.1-openai" +DEPLOYMENT_KEYS: Final = ("sk-fine-tuning-deployment-a", "sk-fine-tuning-deployment-b") +PROVIDER_JOB: Final = "ftjob-provider-123" +RUN: Final = uuid.uuid4().hex[:12] +MANAGED_JOB: Final = f"ftjob-managed-{RUN}" +TRAINING_LINE: Final = b'{"messages": [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]}\n' +JOB_PATH: Final = re.compile(r"^/(?:openai/)?(?:v1/)?fine_tuning/jobs/([^/]+)(/cancel)?$") +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +PREFIXES: Final = (pytest.param("/v1", id="v1"), pytest.param("", id="unversioned")) + + +class Job(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + object: str + created_at: int + error: JsonValue + fine_tuned_model: str | None + finished_at: int | None + hyperparameters: dict[str, JsonValue] + model: str + organization_id: str + result_files: list[str] + seed: int + status: str + trained_tokens: int | None + training_file: str + validation_file: str | None + estimated_finish: int | None = None + integrations: JsonValue = None + metadata: JsonValue = None + method: JsonValue = None + + +def _client_job(job_id: str, status: str, training_file: str = "file-abc123") -> Job: + return Job.model_validate( + { + **_job(job_id, status, training_file), + "hyperparameters": {"batch_size": None, "learning_rate_multiplier": None, "n_epochs": 3}, + } + ) + + +def _job(job_id: str, status: str, training_file: str = "file-abc123", **fields: JsonValue) -> dict[str, JsonValue]: + return { + "id": job_id, + "object": "fine_tuning.job", + "created_at": 1700000000, + "error": None, + "fine_tuned_model": None, + "finished_at": None, + "hyperparameters": {"n_epochs": 3}, + "model": "gpt-4o-mini-2024-07-18", + "organization_id": "org-fine-tuning", + "result_files": [], + "seed": 42, + "status": status, + "trained_tokens": None, + "training_file": training_file, + "validation_file": None, + **fields, + } + + +def _provider_file_id(bearer: str) -> str: + return f"file-{RUN}-" + bearer.removeprefix("sk-fine-tuning-") + + +def _bearer(request: Request) -> str: + return request.headers.get("authorization", "").removeprefix("Bearer ") + + +def _environment(wire: Wire) -> dict[str, str]: + return { + "OPENAI_BASE_URL": f"{wire.url}/v1", + "OPENAI_API_BASE": f"{wire.url}/v1", + "OPENAI_API_KEY": AMBIENT_OPENAI_KEY, + "AZURE_API_BASE": wire.url, + "AZURE_API_KEY": AMBIENT_AZURE_KEY, + "AZURE_API_VERSION": AZURE_API_VERSION, + } + + +def _reply(body: Mapping[str, JsonValue]) -> Reply: + return Reply(body=json.dumps(body).encode()) + + +def _respond(request: Request) -> Reply: + path: Final = urlsplit(request.target).path + if (request.method, path) == ("GET", "/v1/models"): + return _reply({"object": "list", "data": []}) + if (request.method, path) == ("POST", "/v1/files"): + return _reply( + { + "id": _provider_file_id(_bearer(request)), + "object": "file", + "bytes": len(TRAINING_LINE), + "created_at": 1700000000, + "filename": "train.jsonl", + "purpose": "fine-tune", + "status": "processed", + } + ) + if (request.method, path) in {("POST", "/v1/fine_tuning/jobs"), ("POST", "/openai/fine_tuning/jobs")}: + sent: Final = JSON_OBJECT.validate_json(request.body) + job_id: Final = MANAGED_JOB if _bearer(request) in DEPLOYMENT_KEYS else PROVIDER_JOB + return _reply(_job(job_id, "validating_files", str(sent["training_file"]))) + matched: Final = JOB_PATH.match(path) + if matched and matched.group(1) and request.method == "GET": + return _reply(_job(matched.group(1), "running")) + if matched and matched.group(2) and request.method == "POST": + return _reply(_job(matched.group(1), "cancelled")) + return Reply( + status=404, body=json.dumps({"error": {"message": f"unscripted {request.method} {request.target}"}}).encode() + ) + + +def _config(wire: Wire) -> dict[str, JsonValue]: + return { + "model_list": [ + { + "model_name": MANAGED_MODEL, + "litellm_params": {"model": "openai/gpt-4.1", "api_key": key, "api_base": f"{wire.url}/v1"}, + "model_info": {"supported_endpoints": ["/chat/completions", "/fine_tuning"]}, + } + for key in DEPLOYMENT_KEYS + ], + "finetune_settings": [ + {"custom_llm_provider": "openai", "api_key": OPENAI_KEY, "api_base": f"{wire.url}/v1"}, + { + "custom_llm_provider": "azure", + "api_key": AZURE_KEY, + "api_base": wire.url, + "api_version": AZURE_API_VERSION, + }, + ], + "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY", "database_url": "os.environ/DATABASE_URL"}, + } + + +@dataclass(frozen=True, slots=True) +class Rig: + proxy: Gateway + wire: Wire + + def sdk(self, prefix: str) -> OpenAI: + return OpenAI( + base_url=f"{str(self.proxy.client.base_url).rstrip('/')}{prefix}", api_key=self.proxy.key, max_retries=0 + ) + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("fine-tuning") + with gateway_from_environment() as shared, wire_server(_respond) as wire: + config: Final = directory / "config.yaml" + config.write_text(yaml.safe_dump(_config(wire))) + with owned_proxy(shared, directory, _environment(wire), config=config) as proxy: + boot: Final[list[Request]] = [] + eventually(lambda: boot.extend(wire.drain()) or len(boot), lambda count: count >= len(DEPLOYMENT_KEYS)) + assert sorted((request.method, request.target, _bearer(request)) for request in boot) == [ + ("GET", "/v1/models", key) for key in DEPLOYMENT_KEYS + ] + yield Rig(proxy, wire) + assert wire.drain() == () + + +def _drained(rig: Rig) -> tuple[Request, ...]: + return rig.wire.drain() + + +def _only(requests: tuple[Request, ...], method: str, target: str) -> Request: + assert [(request.method, request.target) for request in requests] == [(method, target)] + return requests[0] + + +def _decoded(unified_id: str) -> str: + return base64.urlsafe_b64decode(unified_id + "=" * (-len(unified_id) % 4)).decode() + + +def _multipart_parts(request: Request) -> tuple[Message, ...]: + envelope: Final = f"content-type: {request.headers['content-type']}\r\n\r\n".encode() + request.body + parsed: Final = BytesParser(policy=HTTP).parsebytes(envelope) + assert parsed.is_multipart(), request.headers["content-type"] + return tuple(parsed.iter_parts()) + + +@pytest.mark.parametrize("prefix", PREFIXES) +def test_create_forwards_every_documented_optional_field(rig: Rig, prefix: str) -> None: + with rig.sdk(prefix) as client: + raw: Final = client.fine_tuning.jobs.with_raw_response.create( + model="gpt-4o-mini", + training_file="file-abc123", + validation_file="file-xyz789", + hyperparameters={"n_epochs": 3, "batch_size": "auto", "learning_rate_multiplier": 0.1}, + suffix="custom-model", + seed=42, + extra_body={"custom_llm_provider": "openai"}, + ) + sent: Final = _only(_drained(rig), "POST", "/v1/fine_tuning/jobs") + assert sent.headers["authorization"] == f"Bearer {OPENAI_KEY}" + assert JSON_OBJECT.validate_json(sent.body) == { + "model": "gpt-4o-mini", + "training_file": "file-abc123", + "validation_file": "file-xyz789", + "hyperparameters": {"n_epochs": 3, "batch_size": "auto", "learning_rate_multiplier": 0.1}, + "suffix": "custom-model", + "seed": 42, + }, sent.body + assert raw.status_code == 200, raw.text + assert Job.model_validate_json(raw.text) == _client_job(PROVIDER_JOB, "validating_files"), raw.text + + +def test_create_forwards_the_method_object(rig: Rig) -> None: + pytest.skip( + "BUG: create drops the documented method object, the provider receives only model, training_file and hyperparameters {}" + ) + with rig.sdk("/v1") as client: + raw: Final = client.fine_tuning.jobs.with_raw_response.create( + model="gpt-4o-mini", + training_file="file-abc123", + method={"type": "supervised", "supervised": {"hyperparameters": {"n_epochs": 2}}}, + extra_body={"custom_llm_provider": "openai"}, + ) + sent: Final = _only(_drained(rig), "POST", "/v1/fine_tuning/jobs") + assert raw.status_code == 200, raw.text + assert JSON_OBJECT.validate_json(sent.body) == { + "model": "gpt-4o-mini", + "training_file": "file-abc123", + "hyperparameters": {}, + "method": {"type": "supervised", "supervised": {"hyperparameters": {"n_epochs": 2}}}, + }, sent.body + + +def test_create_forwards_integrations_as_the_sdk_sends_them(rig: Rig) -> None: + pytest.skip( + "BUG: create rejects the SDK integrations objects with 422 because the proxy types integrations as list[str]" + ) + with rig.sdk("/v1") as client: + raw: Final = client.fine_tuning.jobs.with_raw_response.create( + model="gpt-4o-mini", + training_file="file-abc123", + integrations=[{"type": "wandb", "wandb": {"project": "fine-tuning"}}], + extra_body={"custom_llm_provider": "openai"}, + ) + sent: Final = _only(_drained(rig), "POST", "/v1/fine_tuning/jobs") + assert raw.status_code == 200, raw.text + assert JSON_OBJECT.validate_json(sent.body) == { + "model": "gpt-4o-mini", + "training_file": "file-abc123", + "hyperparameters": {}, + "integrations": [{"type": "wandb", "wandb": {"project": "fine-tuning"}}], + }, sent.body + + +@pytest.mark.parametrize("prefix", PREFIXES) +def test_retrieve_forwards_the_raw_provider_job_id(rig: Rig, prefix: str) -> None: + with rig.sdk(prefix) as client: + raw: Final = client.fine_tuning.jobs.with_raw_response.retrieve( + PROVIDER_JOB, extra_query={"custom_llm_provider": "openai"} + ) + sent: Final = _only(_drained(rig), "GET", f"/v1/fine_tuning/jobs/{PROVIDER_JOB}") + assert sent.headers["authorization"] == f"Bearer {OPENAI_KEY}" + assert sent.body == b"" + assert raw.status_code == 200, raw.text + assert Job.model_validate_json(raw.text) == _client_job(PROVIDER_JOB, "running"), raw.text + + +@pytest.mark.parametrize("prefix", PREFIXES) +def test_cancel_sends_the_cancel_to_the_provider(rig: Rig, prefix: str) -> None: + with rig.sdk(prefix) as client: + raw: Final = client.fine_tuning.jobs.with_raw_response.cancel( + PROVIDER_JOB, extra_body={"custom_llm_provider": "openai"} + ) + sent: Final = _only(_drained(rig), "POST", f"/v1/fine_tuning/jobs/{PROVIDER_JOB}/cancel") + assert sent.headers["authorization"] == f"Bearer {OPENAI_KEY}" + assert sent.body == b"" + assert raw.status_code == 200, raw.text + assert Job.model_validate_json(raw.text) == _client_job(PROVIDER_JOB, "cancelled"), raw.text + + +def test_azure_create_merges_finetune_settings_and_keeps_azure_fields(rig: Rig) -> None: + with rig.sdk("/v1") as client: + raw: Final = client.fine_tuning.jobs.with_raw_response.create( + model="gpt-35-turbo-1106", + training_file="file-abc", + extra_body={ + "custom_llm_provider": "azure", + "hyperparameters": {"n_epochs": 3, "prompt_loss_weight": 0.01}, + "trainingType": 2, + }, + ) + sent: Final = _only(_drained(rig), "POST", f"/openai/fine_tuning/jobs?api-version={AZURE_API_VERSION}") + assert sent.headers["api-key"] == AZURE_KEY + assert JSON_OBJECT.validate_json(sent.body) == { + "model": "gpt-35-turbo-1106", + "training_file": "file-abc", + "hyperparameters": {"n_epochs": 3}, + "trainingType": 2, + "prompt_loss_weight": 0.01, + }, sent.body + assert raw.status_code == 200, raw.text + assert Job.model_validate_json(raw.text) == _client_job(PROVIDER_JOB, "validating_files", "file-abc"), raw.text + + +def test_azure_cancel_merges_finetune_settings(rig: Rig) -> None: + with rig.sdk("/v1") as client: + raw: Final = client.fine_tuning.jobs.with_raw_response.cancel( + PROVIDER_JOB, extra_body={"custom_llm_provider": "azure"} + ) + sent: Final = _only( + _drained(rig), "POST", f"/openai/fine_tuning/jobs/{PROVIDER_JOB}/cancel?api-version={AZURE_API_VERSION}" + ) + assert sent.headers["api-key"] == AZURE_KEY + assert sent.body == b"" + assert raw.status_code == 200, raw.text + assert Job.model_validate_json(raw.text) == _client_job(PROVIDER_JOB, "cancelled"), raw.text + + +def test_create_reads_the_documented_custom_llm_provider_header(rig: Rig) -> None: + pytest.skip( + "BUG: create ignores the documented custom-llm-provider header and returns 500" + " Invalid request, No litellm managed file id or custom_llm_provider provided." + ) + with rig.sdk("/v1") as client: + raw: Final = client.fine_tuning.jobs.with_raw_response.create( + model="gpt-35-turbo-1106", training_file="file-abc", extra_headers={"custom-llm-provider": "azure"} + ) + sent: Final = _only(_drained(rig), "POST", f"/openai/fine_tuning/jobs?api-version={AZURE_API_VERSION}") + assert sent.headers["api-key"] == AZURE_KEY + assert JSON_OBJECT.validate_json(sent.body) == { + "model": "gpt-35-turbo-1106", + "training_file": "file-abc", + "hyperparameters": {}, + }, sent.body + assert raw.status_code == 200, raw.text + assert Job.model_validate_json(raw.text) == _client_job(PROVIDER_JOB, "validating_files", "file-abc"), raw.text + + +def test_cancel_reads_the_documented_custom_llm_provider_header(rig: Rig) -> None: + pytest.skip( + "BUG: cancel ignores the documented custom-llm-provider header and the azure finetune_settings entry" + " and sends the cancel to the default OpenAI base URL and key" + ) + with rig.sdk("/v1") as client: + raw: Final = client.fine_tuning.jobs.with_raw_response.cancel( + PROVIDER_JOB, extra_headers={"custom-llm-provider": "azure"} + ) + sent: Final = _only( + _drained(rig), "POST", f"/openai/fine_tuning/jobs/{PROVIDER_JOB}/cancel?api-version={AZURE_API_VERSION}" + ) + assert sent.headers["api-key"] == AZURE_KEY + assert sent.body == b"" + assert raw.status_code == 200, raw.text + assert Job.model_validate_json(raw.text) == _client_job(PROVIDER_JOB, "cancelled"), raw.text + + +def test_retrieve_reads_the_documented_custom_llm_provider_header(rig: Rig) -> None: + pytest.skip( + "BUG: retrieve ignores the documented custom-llm-provider header and returns 400" + " Invalid request, No litellm managed file id or custom_llm_provider provided." + ) + with rig.sdk("/v1") as client: + raw: Final = client.fine_tuning.jobs.with_raw_response.retrieve( + PROVIDER_JOB, extra_headers={"custom-llm-provider": "azure"} + ) + sent: Final = _only( + _drained(rig), "GET", f"/openai/fine_tuning/jobs/{PROVIDER_JOB}?api-version={AZURE_API_VERSION}" + ) + assert sent.headers["api-key"] == AZURE_KEY + assert sent.body == b"" + assert raw.status_code == 200, raw.text + assert Job.model_validate_json(raw.text) == _client_job(PROVIDER_JOB, "running"), raw.text + + +def test_managed_create_sends_the_provider_file_id_of_the_pinned_deployment(rig: Rig) -> None: + pytest.skip( + "BUG: managed create sends the base64 LiteLLM file id and the prefixed model openai/gpt-4.1 to the provider" + " and returns the decoded unified id as training_file" + ) + with rig.sdk("/v1") as client: + uploaded: Final = client.files.with_raw_response.create( + file=("train.jsonl", TRAINING_LINE, "application/jsonl"), + purpose="fine-tune", + extra_body={"target_model_names": MANAGED_MODEL}, + ) + assert uploaded.status_code == 200, uploaded.text + managed_file: Final = JSON_OBJECT.validate_json(uploaded.text)["id"] + assert isinstance(managed_file, str) and _decoded(managed_file).startswith("litellm_proxy:"), uploaded.text + uploads: Final = _drained(rig) + assert sorted((request.method, request.target, _bearer(request)) for request in uploads) == [ + ("POST", "/v1/files", key) for key in DEPLOYMENT_KEYS + ] + for upload in uploads: + parts = {part.get_param("name", header="content-disposition"): part for part in _multipart_parts(upload)} + assert sorted(parts) == ["file", "purpose"], upload.body[:300] + assert parts["file"].get_filename() == "train.jsonl" + assert parts["file"].get_payload(decode=True) == TRAINING_LINE + assert parts["purpose"].get_filename() is None + assert parts["purpose"].get_content() == "fine-tune" + + raw: Final = client.fine_tuning.jobs.with_raw_response.create(model=MANAGED_MODEL, training_file=managed_file) + sent: Final = _only(_drained(rig), "POST", "/v1/fine_tuning/jobs") + deployment_key: Final = _bearer(sent) + assert deployment_key in DEPLOYMENT_KEYS, sent.headers + assert JSON_OBJECT.validate_json(sent.body) == { + "model": "gpt-4.1", + "training_file": _provider_file_id(deployment_key), + "hyperparameters": {}, + }, sent.body + assert raw.status_code == 200, raw.text + job: Final = Job.model_validate_json(raw.text) + assert re.fullmatch( + rf"litellm_proxy;model_id:[0-9a-f]{{64}};generic_response_id:{MANAGED_JOB}", _decoded(job.id) + ), raw.text + assert job.training_file == managed_file, raw.text diff --git a/tests/integration/security/test_fine_tuning_managed_ids.py b/tests/integration/security/test_fine_tuning_managed_ids.py new file mode 100644 index 00000000000..0c1a0f71b3f --- /dev/null +++ b/tests/integration/security/test_fine_tuning_managed_ids.py @@ -0,0 +1,492 @@ +import base64 +import json +import re +import uuid +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final +from urllib.parse import urlsplit + +import httpx +import pytest +import yaml +from integration._support.database import read_rows +from integration._support.client import Gateway, eventually, gateway_from_environment, object_value, string_value +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from openai import OpenAI +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +MANAGED_MODEL: Final = "gpt-4.1-managed-ids" +DEPLOYMENT_KEY: Final = "sk-fine-tuning-managed-deployment" +RUN: Final = uuid.uuid4().hex[:12] +PROVIDER_FILE: Final = f"file-managed-{RUN}" +PROVIDER_JOB: Final = f"ftjob-managed-{RUN}" +DEFAULT_PROVIDER_FILE: Final = f"file-managed-default-{RUN}" +DEFAULT_PROVIDER_JOB: Final = f"ftjob-managed-default-{RUN}" +TRAINING_LINE: Final = b'{"messages": [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]}\n' +JOB_PATH: Final = re.compile(r"^/v1/fine_tuning/jobs/([^/]+?)(/cancel)?$") +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +class Job(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + object: str + created_at: int + error: JsonValue + fine_tuned_model: str | None + finished_at: int | None + hyperparameters: dict[str, JsonValue] + status: str + model: str + organization_id: str + result_files: list[str] + seed: int + trained_tokens: int | None + training_file: str + validation_file: str | None + estimated_finish: int | None = None + integrations: JsonValue = None + metadata: JsonValue = None + method: JsonValue = None + + +class ProxyError(BaseModel): + model_config = ConfigDict(extra="ignore") + + error: dict[str, JsonValue] + + +def _job(job_id: str, status: str, file_id: str) -> dict[str, JsonValue]: + return { + "id": job_id, + "object": "fine_tuning.job", + "created_at": 1700000000, + "error": None, + "fine_tuned_model": None, + "finished_at": None, + "hyperparameters": {"n_epochs": 3}, + "model": "gpt-4.1", + "organization_id": "org-fine-tuning", + "result_files": [], + "seed": 42, + "status": status, + "trained_tokens": None, + "training_file": file_id, + "validation_file": None, + } + + +def _client_job(job_id: str, status: str, file_id: str) -> Job: + return Job.model_validate( + { + **_job(job_id, status, file_id), + "hyperparameters": {"batch_size": None, "learning_rate_multiplier": None, "n_epochs": 3}, + } + ) + + +def _reply(body: Mapping[str, JsonValue]) -> Reply: + return Reply(body=json.dumps(body).encode()) + + +def _responder(file_id: str, job_id: str) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + return _respond(request, file_id, job_id) + + return respond + + +def _respond(request: Request, file_id: str, job_id: str) -> Reply: + path: Final = urlsplit(request.target).path + if (request.method, path) == ("GET", "/v1/models"): + return _reply({"object": "list", "data": []}) + if (request.method, path) == ("POST", "/v1/files"): + return _reply( + { + "id": file_id, + "object": "file", + "bytes": len(TRAINING_LINE), + "created_at": 1700000000, + "filename": "train.jsonl", + "purpose": "fine-tune", + "status": "processed", + } + ) + if (request.method, path) == ("POST", "/v1/fine_tuning/jobs"): + return _reply(_job(job_id, "validating_files", file_id)) + matched: Final = JOB_PATH.match(path) + if matched and request.method == "GET" and not matched.group(2): + return _reply(_job(matched.group(1), "running", file_id)) + if matched and request.method == "POST" and matched.group(2): + return _reply(_job(matched.group(1), "cancelled", file_id)) + return Reply( + status=404, body=json.dumps({"error": {"message": f"unscripted {request.method} {request.target}"}}).encode() + ) + + +def _config(wire: Wire, litellm_settings: dict[str, JsonValue]) -> dict[str, JsonValue]: + return { + "model_list": [ + { + "model_name": MANAGED_MODEL, + "litellm_params": {"model": "openai/gpt-4.1", "api_key": DEPLOYMENT_KEY, "api_base": f"{wire.url}/v1"}, + "model_info": {"supported_endpoints": ["/chat/completions", "/fine_tuning"]}, + } + ], + "finetune_settings": [ + {"custom_llm_provider": "openai", "api_key": "sk-fine-tuning-raw-provider", "api_base": f"{wire.url}/v1"} + ], + "litellm_settings": litellm_settings, + "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY", "database_url": "os.environ/DATABASE_URL"}, + } + + +def _unified_job_id(model_id: str, job_id: str) -> str: + return ( + base64.urlsafe_b64encode(f"litellm_proxy;model_id:{model_id};generic_response_id:{job_id}".encode()) + .decode() + .rstrip("=") + ) + + +def _deployment_id(proxy: Gateway) -> str: + listed: Final = proxy.get("/v1/model/info")["data"] + assert isinstance(listed, list), listed + ids: Final = [ + string_value(object_value(object_value(entry)["model_info"])["id"]) + for entry in listed + if object_value(entry)["model_name"] == MANAGED_MODEL + ] + assert len(ids) == 1, listed + return ids[0] + + +def _targets(requests: tuple[Request, ...]) -> list[tuple[str, str, str]]: + return [(request.method, request.target, request.headers.get("authorization", "")) for request in requests] + + +@dataclass(frozen=True, slots=True) +class Rig: + proxy: Gateway + wire: Wire + owner: str + owner_user: str + intruder: str + intruder_user: str + managed_file: str + provider_file: str + managed_job: str + created_text: str + deployment_id: str + provider_job: str + + def sdk(self, key: str) -> OpenAI: + return OpenAI(base_url=f"{str(self.proxy.client.base_url).rstrip('/')}/v1", api_key=key, max_retries=0) + + +@contextmanager +def _rig(directory: Path, litellm_settings: dict[str, JsonValue], file_id: str, job_id: str) -> Iterator[Rig]: + with gateway_from_environment() as shared, wire_server(_responder(file_id, job_id)) as wire: + config: Final = directory / "config.yaml" + config.write_text(yaml.safe_dump(_config(wire, litellm_settings))) + with owned_proxy(shared, directory, {}, config=config) as proxy, proxy.scenario() as scenario: + eventually( + lambda: _targets(wire.drain()), lambda seen: seen == [("GET", "/v1/models", f"Bearer {DEPLOYMENT_KEY}")] + ) + deployment_id: Final = _deployment_id(proxy) + owner_user: Final = scenario.user() + owner: Final = scenario.key(user_id=owner_user) + intruder_user: Final = scenario.user() + intruder: Final = scenario.key(user_id=intruder_user) + with OpenAI( + base_url=f"{str(proxy.client.base_url).rstrip('/')}/v1", api_key=owner, max_retries=0 + ) as client: + uploaded: Final = client.files.with_raw_response.create( + file=("train.jsonl", TRAINING_LINE, "application/jsonl"), + purpose="fine-tune", + extra_body={"target_model_names": MANAGED_MODEL}, + ) + assert uploaded.status_code == 200, uploaded.text + managed_file: Final = uploaded.parse().id + created: Final = client.fine_tuning.jobs.with_raw_response.create( + model=MANAGED_MODEL, training_file=managed_file + ) + assert _targets(wire.drain()) == [ + ("POST", "/v1/files", f"Bearer {DEPLOYMENT_KEY}"), + ("POST", "/v1/fine_tuning/jobs", f"Bearer {DEPLOYMENT_KEY}"), + ] + yield Rig( + proxy, + wire, + owner, + owner_user, + intruder, + intruder_user, + managed_file, + file_id, + Job.model_validate_json(created.text).id, + created.text, + deployment_id, + job_id, + ) + assert wire.drain() == () + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + with _rig( + tmp_path_factory.mktemp("fine-tuning-managed-ids"), {"require_managed_files": True}, PROVIDER_FILE, PROVIDER_JOB + ) as built: + yield built + + +@pytest.fixture(scope="module") +def default_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + with _rig( + tmp_path_factory.mktemp("fine-tuning-managed-ids-default"), {}, DEFAULT_PROVIDER_FILE, DEFAULT_PROVIDER_JOB + ) as built: + yield built + + +@pytest.mark.parametrize("fixture", ["rig", "default_rig"]) +def test_managed_file_and_job_rows_record_the_owner(fixture: str, request: pytest.FixtureRequest) -> None: + built: Final[Rig] = request.getfixturevalue(fixture) + file_rows: Final = read_rows( + 'SELECT model_mappings, flat_model_file_ids, created_by, team_id FROM "LiteLLM_ManagedFileTable" WHERE unified_file_id = %s', + (built.managed_file,), + ) + assert file_rows == [ + { + "model_mappings": {built.deployment_id: built.provider_file}, + "flat_model_file_ids": [built.provider_file], + "created_by": built.owner_user, + "team_id": None, + } + ], file_rows + job_rows: Final = eventually( + lambda: read_rows( + 'SELECT unified_object_id, model_object_id, file_purpose, created_by, team_id FROM "LiteLLM_ManagedObjectTable" WHERE unified_object_id = %s', + (built.managed_job,), + ), + lambda rows: len(rows) == 1, + ) + assert job_rows == [ + { + "unified_object_id": built.managed_job, + "model_object_id": built.provider_job, + "file_purpose": "fine-tune", + "created_by": built.owner_user, + "team_id": None, + } + ], job_rows + + +def _refused(response: httpx.Response, wire: Wire, detail: str) -> None: + reached: Final = _targets(wire.drain()) + assert response.status_code == 403, response.text + assert ProxyError.model_validate_json(response.text).error["message"] == detail, response.text + assert reached == [] + + +@pytest.mark.parametrize("fixture", ["rig", "default_rig"]) +def test_owner_create_returns_the_unified_job_id(fixture: str, request: pytest.FixtureRequest) -> None: + built: Final[Rig] = request.getfixturevalue(fixture) + created: Final = Job.model_validate_json(built.created_text) + expected: Final = _client_job( + _unified_job_id(built.deployment_id, built.provider_job), "validating_files", built.provider_file + ) + assert created.model_dump(exclude={"training_file"}) == expected.model_dump(exclude={"training_file"}), ( + "training_file is excluded because test_managed_create_sends_the_provider_file_id_of_the_pinned_deployment " + "documents the known managed-create bug", + built.created_text, + ) + + +def test_another_key_cannot_create_with_or_cancel_the_owners_managed_ids(rig: Rig) -> None: + created: Final = rig.proxy.request( + "POST", + "/v1/fine_tuning/jobs", + {"model": MANAGED_MODEL, "training_file": rig.managed_file}, + key=rig.intruder, + ) + _refused( + created, + rig.wire, + f"User {rig.intruder_user} does not have access to the file {rig.managed_file}", + ) + cancelled: Final = rig.proxy.request("POST", f"/v1/fine_tuning/jobs/{rig.managed_job}/cancel", key=rig.intruder) + _refused(cancelled, rig.wire, "The caller does not have access to this managed fine-tuning job id.") + + +def test_another_key_cannot_cancel_the_owners_managed_job_by_default(default_rig: Rig) -> None: + cancelled: Final = default_rig.proxy.request( + "POST", f"/v1/fine_tuning/jobs/{default_rig.managed_job}/cancel", key=default_rig.intruder + ) + _refused( + cancelled, + default_rig.wire, + f"User {default_rig.intruder_user} does not have access to the object {default_rig.managed_job}", + ) + + +def test_another_key_cannot_create_with_the_owners_managed_file_by_default(default_rig: Rig) -> None: + pytest.skip( + "BUG: another key can create a fine-tuning job with the owner's managed training_file" + " when require_managed_files is off" + ) + created: Final = default_rig.proxy.request( + "POST", + "/v1/fine_tuning/jobs", + {"model": MANAGED_MODEL, "training_file": default_rig.managed_file}, + key=default_rig.intruder, + ) + _refused( + created, + default_rig.wire, + f"User {default_rig.intruder_user} does not have access to the file {default_rig.managed_file}", + ) + + +@pytest.mark.parametrize( + ("method", "path", "body", "kind"), + [ + pytest.param( + "POST", + "/v1/fine_tuning/jobs", + {"model": "gpt-4o-mini", "training_file": "file-raw123", "custom_llm_provider": "openai"}, + "file", + id="create", + ), + pytest.param("GET", "/v1/fine_tuning/jobs/ftjob-raw123", None, "fine-tuning job", id="retrieve"), + pytest.param("POST", "/v1/fine_tuning/jobs/ftjob-raw123/cancel", None, "fine-tuning job", id="cancel"), + ], +) +def test_raw_provider_ids_are_refused_when_managed_files_are_required( + rig: Rig, method: str, path: str, body: dict[str, JsonValue] | None, kind: str +) -> None: + response: Final = rig.proxy.request( + method, path, body, params={"custom_llm_provider": "openai"} if body is None else None + ) + reached: Final = _targets(rig.wire.drain()) + assert response.status_code == 400, response.text + assert ProxyError.model_validate_json(response.text).error["message"] == ( + f"Raw provider {kind} ids cannot be used when require_managed_files is enabled in litellm_settings. " + f"Use the LiteLLM managed {kind} id returned when the {kind} was created." + ), response.text + assert reached == [] + + +def test_raw_validation_file_is_refused_when_managed_files_are_required(rig: Rig) -> None: + rig.wire.drain() + response: Final = rig.proxy.request( + "POST", + "/v1/fine_tuning/jobs", + {"model": MANAGED_MODEL, "training_file": rig.managed_file, "validation_file": "file-raw123"}, + key=rig.owner, + ) + reached: Final = _targets(rig.wire.drain()) + assert response.status_code == 400, response.text + assert ProxyError.model_validate_json(response.text).error["message"] == ( + "Raw provider file ids cannot be used when require_managed_files is enabled in litellm_settings. " + "Use the LiteLLM managed file id returned when the file was created." + ), response.text + assert reached == [] + + +def test_intruder_cannot_use_the_owners_managed_validation_file(rig: Rig) -> None: + with rig.sdk(rig.intruder) as client: + uploaded: Final = client.files.with_raw_response.create( + file=("train.jsonl", TRAINING_LINE, "application/jsonl"), + purpose="fine-tune", + extra_body={"target_model_names": MANAGED_MODEL}, + ) + assert uploaded.status_code == 200, uploaded.text + intruder_file: Final = uploaded.parse().id + assert _targets(rig.wire.drain()) == [("POST", "/v1/files", f"Bearer {DEPLOYMENT_KEY}")] + response: Final = rig.proxy.request( + "POST", + "/v1/fine_tuning/jobs", + {"model": MANAGED_MODEL, "training_file": intruder_file, "validation_file": rig.managed_file}, + key=rig.intruder, + ) + _refused( + response, + rig.wire, + f"User {rig.intruder_user} does not have access to the file {rig.managed_file}", + ) + + +def test_keys_granted_the_retrieve_route_get_the_owner_check_and_the_decoded_job_id(rig: Rig) -> None: + with rig.proxy.scenario() as scenario: + owner: Final = scenario.key(user_id=rig.owner_user, allowed_routes=["/v1/fine_tuning/jobs"]) + intruder: Final = scenario.key(user_id=rig.intruder_user, allowed_routes=["/v1/fine_tuning/jobs"]) + _refused( + rig.proxy.request("GET", f"/v1/fine_tuning/jobs/{rig.managed_job}", key=intruder), + rig.wire, + "The caller does not have access to this managed fine-tuning job id.", + ) + with rig.sdk(owner) as client: + retrieved: Final = client.fine_tuning.jobs.with_raw_response.retrieve(rig.managed_job) + assert [(r.method, r.target, r.headers["authorization"], r.body) for r in rig.wire.drain()] == [ + ("GET", f"/v1/fine_tuning/jobs/{PROVIDER_JOB}", f"Bearer {DEPLOYMENT_KEY}", b"") + ] + assert retrieved.status_code == 200, retrieved.text + assert Job.model_validate_json(retrieved.text) == _client_job(rig.managed_job, "running", rig.provider_file), ( + retrieved.text + ) + + +def test_keys_granted_the_retrieve_route_get_default_job_ownership_and_decoded_id(default_rig: Rig) -> None: + with default_rig.proxy.scenario() as scenario: + owner: Final = scenario.key(user_id=default_rig.owner_user, allowed_routes=["/v1/fine_tuning/jobs"]) + intruder: Final = scenario.key(user_id=default_rig.intruder_user, allowed_routes=["/v1/fine_tuning/jobs"]) + _refused( + default_rig.proxy.request("GET", f"/v1/fine_tuning/jobs/{default_rig.managed_job}", key=intruder), + default_rig.wire, + f"User {default_rig.intruder_user} does not have access to the object {default_rig.managed_job}", + ) + with default_rig.sdk(owner) as client: + retrieved: Final = client.fine_tuning.jobs.with_raw_response.retrieve(default_rig.managed_job) + assert [ + (request.method, request.target, request.headers["authorization"], request.body) + for request in default_rig.wire.drain() + ] == [("GET", f"/v1/fine_tuning/jobs/{DEFAULT_PROVIDER_JOB}", f"Bearer {DEPLOYMENT_KEY}", b"")] + assert retrieved.status_code == 200, retrieved.text + assert Job.model_validate_json(retrieved.text) == _client_job( + default_rig.managed_job, "running", default_rig.provider_file + ), retrieved.text + + +def test_owner_retrieve_reaches_the_provider_with_the_decoded_job_id(rig: Rig) -> None: + pytest.skip("BUG: a virtual key gets 401 on GET /v1/fine_tuning/jobs/{id}, the route is missing from openai_routes") + _refused( + rig.proxy.request("GET", f"/v1/fine_tuning/jobs/{rig.managed_job}", key=rig.intruder), + rig.wire, + "The caller does not have access to this managed fine-tuning job id.", + ) + with rig.sdk(rig.owner) as client: + retrieved: Final = client.fine_tuning.jobs.with_raw_response.retrieve(rig.managed_job) + assert [(r.method, r.target, r.headers["authorization"], r.body) for r in rig.wire.drain()] == [ + ("GET", f"/v1/fine_tuning/jobs/{PROVIDER_JOB}", f"Bearer {DEPLOYMENT_KEY}", b"") + ] + assert retrieved.status_code == 200, retrieved.text + assert Job.model_validate_json(retrieved.text) == _client_job(rig.managed_job, "running", rig.provider_file), ( + retrieved.text + ) + + +def test_owner_cancel_reaches_the_provider_with_the_decoded_job_id(rig: Rig) -> None: + with rig.sdk(rig.owner) as client: + cancelled: Final = client.fine_tuning.jobs.with_raw_response.cancel(rig.managed_job) + assert [(r.method, r.target, r.headers["authorization"], r.body) for r in rig.wire.drain()] == [ + ("POST", f"/v1/fine_tuning/jobs/{PROVIDER_JOB}/cancel", f"Bearer {DEPLOYMENT_KEY}", b"") + ] + assert cancelled.status_code == 200, cancelled.text + assert Job.model_validate_json(cancelled.text) == _client_job(rig.managed_job, "cancelled", rig.provider_file), ( + cancelled.text + ) From 1aee1a21edec8606f6128ae97afe4f3c71bfe01f Mon Sep 17 00:00:00 2001 From: mrinal-berri Date: Wed, 14 Oct 2026 05:22:02 -0400 Subject: [PATCH 19/51] fix(databricks): send a single plain text block as a string for non-Claude models (#45686) * fix(databricks): send a single plain text block as a string for non-Claude models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(databricks): simplify single-text-block helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * docs(databricks): clarify json_object validation wording Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(databricks): integration cells for single text block content shape Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(databricks): rename list-content cells to stays_list Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): expect plain string content for bridged Databricks messages Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(databricks): cover the customer's Unity Catalog model from #45617 and #45618 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(databricks): drop casts from _collapse_single_text_block Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(databricks): assert the whole outgoing messages list in the collapse unit test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(databricks): cover system joins, fallback, SDK clients, and outage burst for one-block content --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../llms/databricks/chat/transformation.py | 12 +- .../providers/test_databricks_chat_wire.py | 577 ++++++++++++++++++ .../test_databricks_chat_transformation.py | 116 ++++ .../test_responses_api_bridge_flag.py | 7 +- 4 files changed, 706 insertions(+), 6 deletions(-) diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 4b6f178fa1a..a98cfdfe52d 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -75,6 +75,15 @@ def _is_bare_assistant_message(message_dict: Mapping[str, object]) -> bool: ) +def _collapse_single_text_block(message: Mapping[str, object]) -> Mapping[str, object]: + content: Final = message.get("content") + match content: + case [{"type": "text", "text": str() as text}] if content == [{"type": "text", "text": text}]: + return {**message, "content": text} + case _: + return message + + def _sanitize_empty_content(message_dict: dict[str, object]) -> None: """ Remove or filter content so empty text blocks are not sent. @@ -463,6 +472,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): Databricks does not support: - 'name' in user message. - litellm's internal `thinking_blocks` / `reasoning_content` on assistant messages. + - non-Claude json_object "json" checks read only string content, so a single text block is sent as a string. """ new_messages = [] for idx, message in enumerate(messages): @@ -478,7 +488,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): _sanitize_empty_content(cast(dict[str, object], _message)) if _is_bare_assistant_message(_message): continue - new_messages.append(_message) + new_messages.append(_collapse_single_text_block(_message) if "claude" not in model else _message) if "claude" not in model: new_messages = _split_parallel_tool_calls( diff --git a/tests/integration/providers/test_databricks_chat_wire.py b/tests/integration/providers/test_databricks_chat_wire.py index f6192d32c31..3d7f273279f 100644 --- a/tests/integration/providers/test_databricks_chat_wire.py +++ b/tests/integration/providers/test_databricks_chat_wire.py @@ -3,11 +3,15 @@ import json import uuid from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor +from pathlib import Path from typing import Final, Literal, TypeAlias +import anthropic +import openai import pytest from integration._support.client import Gateway, eventually from integration._support.database import read_rows +from integration._support.process import owned_proxy_process from integration._support.wire import Reply, Request, wire_server from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter @@ -201,6 +205,226 @@ def test_databricks_stream_final_usage_chunk_reaches_client_and_spend_log(gatewa assert (rows[0]["prompt_tokens"], rows[0]["completion_tokens"], rows[0]["total_tokens"]) == (12011, 8, 12019) +def _shape_reply(identity: str, stream: bool) -> Reply: + if stream: + chunks: Final = ( + _frame( + identity, + [{"index": 0, "delta": {"role": "assistant", "content": "ok"}, "finish_reason": None}], + ), + _frame(identity, [{"index": 0, "delta": {}, "finish_reason": "stop"}]), + b"data: [DONE]\n\n", + ) + return Reply(content_type="text/event-stream", chunks=chunks) + + body: Final = { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": _BACKEND, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + return Reply(body=json.dumps(body).encode()) + + +@pytest.mark.parametrize( + "case,endpoint,provider_model,request_body,expected_messages,responses_mode,upstream_path", + [ + pytest.param( + "chat_nonstream", + "/v1/chat/completions", + "databricks-meta-llama-3-3-70b-instruct", + { + "messages": [{"role": "user", "content": [{"type": "text", "text": "Reply in JSON"}]}], + "response_format": {"type": "json_object"}, + }, + [{"role": "user", "content": "Reply in JSON"}], + False, + "/chat/completions", + id="A1-chat-nonstream", + ), + pytest.param( + "chat_stream", + "/v1/chat/completions", + "databricks-meta-llama-3-3-70b-instruct", + { + "messages": [{"role": "user", "content": [{"type": "text", "text": "Reply in JSON"}]}], + "response_format": {"type": "json_object"}, + "stream": True, + }, + [{"role": "user", "content": "Reply in JSON"}], + False, + "/chat/completions", + id="A2-chat-stream", + ), + pytest.param( + "responses-mode-bridge", + "/v1/chat/completions", + "databricks-meta-llama-3-1-8b-instruct", + { + "messages": [{"role": "user", "content": "Reply in JSON"}], + "response_format": {"type": "json_object"}, + }, + [{"role": "user", "content": "Reply in JSON"}], + True, + "/chat/completions", + id="A3-responses-mode", + ), + pytest.param( + "responses-mode-bridge-unity-catalog", + "/v1/chat/completions", + "system.ai.deepseek-v4-1-flash", + { + "messages": [{"role": "user", "content": "Reply in JSON with key a. hi"}], + "response_format": {"type": "json_object"}, + }, + [{"role": "user", "content": "Reply in JSON with key a. hi"}], + True, + "/ai-gateway/mlflow/v1/chat/completions", + id="A3b-responses-mode-unity-catalog-model", + ), + pytest.param( + "messages-endpoint", + "/v1/messages", + "databricks-meta-llama-3-3-70b-instruct", + { + "max_tokens": 32, + "messages": [{"role": "user", "content": [{"type": "text", "text": "Reply in JSON"}]}], + }, + [{"role": "user", "content": "Reply in JSON"}], + False, + "/chat/completions", + id="A4-messages", + ), + pytest.param( + "responses-endpoint", + "/v1/responses", + "databricks-meta-llama-3-3-70b-instruct", + {"input": [{"role": "user", "content": [{"type": "input_text", "text": "Reply in JSON"}]}]}, + [{"role": "user", "content": "Reply in JSON"}], + False, + "/chat/completions", + id="A5-responses-list-input", + ), + ], +) +def test_databricks_non_claude_single_text_block_reaches_upstream_as_string( + gateway: Gateway, + case: str, + endpoint: str, + provider_model: str, + request_body: dict[str, JsonValue], + expected_messages: list[JsonValue], + responses_mode: bool, + upstream_path: str, +) -> None: + identity: Final = f"databricks-single-text-{case}-{uuid.uuid4().hex}" + streaming: Final = request_body.get("stream") is True + + def respond(request: Request) -> Reply: + assert request.method == "POST" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body["messages"] == expected_messages, body["messages"] + return _shape_reply(identity, streaming) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"databricks/{provider_model}", + api_base=wire.url, + api_key=_API_KEY, + model_info={"mode": "responses"} if responses_mode else None, + ) + headers: Final = {"anthropic-version": "2023-06-01"} if endpoint == "/v1/messages" else {} + body: Final = {"model": model, **request_body} + if streaming: + with gateway.client.stream( + "POST", endpoint, json=body, headers={"Authorization": f"Bearer {gateway.key}", **headers} + ) as response: + assert response.status_code == 200, response.read() + lines: Final = tuple(line for line in response.iter_lines() if line.startswith("data: ")) + assert lines[-1] == "data: [DONE]", lines + assert any('"content":"ok"' in line for line in lines), lines + else: + response: Final = gateway.request("POST", endpoint, body, headers=headers) + assert response.status_code == 200, response.text + assert _JSON_OBJECT.validate_json(response.content)["id"] + assert [(request.method, request.target) for request in wire.drain()] == [("POST", upstream_path)] + + +@pytest.mark.parametrize( + "case,provider_model,content,expected_content", + [ + pytest.param( + "cache-control", + "databricks-meta-llama-3-3-70b-instruct", + [{"type": "text", "text": "Reply in JSON", "cache_control": {"type": "ephemeral"}}], + [{"type": "text", "text": "Reply in JSON", "cache_control": {"type": "ephemeral"}}], + id="B1-cache-control-stays-list", + ), + pytest.param( + "two-text-blocks", + "databricks-meta-llama-3-3-70b-instruct", + [{"type": "text", "text": "Reply"}, {"type": "text", "text": " in JSON"}], + [{"type": "text", "text": "Reply"}, {"type": "text", "text": " in JSON"}], + id="B2-two-text-blocks-stays-list", + ), + pytest.param( + "text-and-image", + "databricks-meta-llama-3-3-70b-instruct", + [ + {"type": "text", "text": "Reply in JSON"}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.png"}}, + ], + [ + {"type": "text", "text": "Reply in JSON"}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.png"}}, + ], + id="B3-text-and-image-stays-list", + ), + pytest.param( + "claude", + "databricks-claude-opus-5-5", + [{"type": "text", "text": "Reply in JSON"}], + [{"type": "text", "text": "Reply in JSON"}], + id="B4-claude-stays-list", + ), + ], +) +def test_databricks_list_content_stays_list( + gateway: Gateway, + case: str, + provider_model: str, + content: list[JsonValue], + expected_content: list[JsonValue], +) -> None: + identity: Final = f"databricks-list-content-{case}-{uuid.uuid4().hex}" + expected_messages: Final = [{"role": "user", "content": expected_content}] + + def respond(request: Request) -> Reply: + assert request.method == "POST" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body["messages"] == expected_messages, body["messages"] + return _shape_reply(identity, False) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"databricks/{provider_model}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": content}]}, + ) + assert response.status_code == 200, response.text + assert _JSON_OBJECT.validate_json(response.content)["id"] + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")] + + def test_databricks_json_schema_refs_are_preserved_in_chat_completions(gateway: Gateway) -> None: def respond(request: Request) -> Reply: assert request.method == "POST" @@ -868,3 +1092,356 @@ def test_databricks_json_schema_burst_through_an_upstream_outage_keeps_every_ref ) assert tuple(_outcome(*result) for result in results) == expected assert tuple(_outcome(*result) for result in recovered) == ((200, _EXPECTED_JSON),) * len(recovery) + + +_LLAMA_BACKEND: Final = "databricks-meta-llama-3-3-70b-instruct" +_NO_CACHE: Final[dict[str, JsonValue]] = {"no-cache": True} + + +def _one_block(text: str) -> list[JsonValue]: + return [{"type": "text", "text": text}] + + +def _plain_reply(_request: Request) -> Reply: + return _shape_reply(f"databricks-plain-{uuid.uuid4().hex}", False) + + +def _sent_messages(request: Request) -> JsonValue: + return _JSON_OBJECT.validate_json(request.body)["messages"] + + +def _tool_call(identity: str, city: str) -> dict[str, JsonValue]: + return { + "id": identity, + "type": "function", + "function": {"name": "get_weather", "arguments": json.dumps({"city": city})}, + } + + +@pytest.mark.parametrize( + "messages,expected_messages", + [ + pytest.param( + [ + {"role": "system", "content": _one_block("Alpha rule.")}, + {"role": "system", "content": _one_block("Bravo rule.")}, + {"role": "user", "content": _one_block("Reply in JSON")}, + ], + [ + {"role": "system", "content": "Alpha rule.\n\nBravo rule."}, + {"role": "user", "content": "Reply in JSON"}, + ], + id="C1-two-one-block-system-messages", + ), + pytest.param( + [ + {"role": "system", "content": _one_block("Alpha rule.")}, + {"role": "developer", "content": _one_block("Bravo rule.")}, + {"role": "user", "content": "Reply in JSON"}, + ], + [ + {"role": "system", "content": "Alpha rule.\n\nBravo rule."}, + {"role": "user", "content": "Reply in JSON"}, + ], + id="C2-system-then-developer-one-block", + ), + pytest.param( + [ + {"role": "system", "content": _one_block("Alpha rule.")}, + {"role": "system", "content": "Bravo rule."}, + {"role": "user", "content": "Reply in JSON"}, + ], + [ + {"role": "system", "content": "Alpha rule.\n\nBravo rule."}, + {"role": "user", "content": "Reply in JSON"}, + ], + id="C3-one-block-then-string-system", + ), + pytest.param( + [ + {"role": "system", "content": _one_block("Alpha rule.")}, + { + "role": "system", + "content": [{"type": "text", "text": "Bravo", "cache_control": {"type": "ephemeral"}}], + }, + {"role": "user", "content": "Reply in JSON"}, + ], + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Alpha rule."}, + {"type": "text", "text": "Bravo", "cache_control": {"type": "ephemeral"}}, + ], + }, + {"role": "user", "content": "Reply in JSON"}, + ], + id="C4-one-block-then-cache-marked-system-stays-parts", + ), + pytest.param( + [ + {"role": "user", "content": _one_block("Weather in Paris and Rome?")}, + { + "role": "assistant", + "content": _one_block("Checking both."), + "tool_calls": [_tool_call("call-paris", "Paris"), _tool_call("call-rome", "Rome")], + }, + {"role": "tool", "tool_call_id": "call-paris", "content": _one_block("18C")}, + {"role": "tool", "tool_call_id": "call-rome", "content": _one_block("22C")}, + ], + [ + {"role": "user", "content": "Weather in Paris and Rome?"}, + {"role": "assistant", "content": "Checking both.", "tool_calls": [_tool_call("call-paris", "Paris")]}, + {"role": "tool", "tool_call_id": "call-paris", "content": "18C"}, + {"role": "assistant", "tool_calls": [_tool_call("call-rome", "Rome")]}, + {"role": "tool", "tool_call_id": "call-rome", "content": "22C"}, + ], + id="C5-parallel-tool-results-split-and-collapsed", + ), + pytest.param( + [ + {"role": "user", "content": [{"type": "text", "text": "Reply in JSON", "cache_control": None}]}, + {"role": "assistant", "content": [{"type": "text", "text": "ok", "annotations": []}]}, + {"role": "user", "content": _one_block(" ")}, + {"role": "user", "content": _one_block("x" * 5120)}, + ], + [ + {"role": "user", "content": [{"type": "text", "text": "Reply in JSON", "cache_control": None}]}, + {"role": "assistant", "content": [{"type": "text", "text": "ok", "annotations": []}]}, + {"role": "user"}, + {"role": "user", "content": "x" * 5120}, + ], + id="F1-extra-keys-stay-list-blank-dropped-5kb-collapsed", + ), + ], +) +def test_databricks_non_claude_message_shapes_reach_upstream( + gateway: Gateway, messages: list[JsonValue], expected_messages: list[JsonValue] +) -> None: + with wire_server(_plain_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"databricks/{_LLAMA_BACKEND}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": messages, "cache": _NO_CACHE}, + ) + received: Final = wire.drain() + assert response.status_code == 200, response.text + assert [(request.method, request.target) for request in received] == [("POST", "/chat/completions")] + assert _sent_messages(received[0]) == expected_messages + + +def test_databricks_responses_instructions_and_one_block_developer_reach_upstream_joined(gateway: Gateway) -> None: + with wire_server(_plain_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"databricks/{_LLAMA_BACKEND}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/responses", + { + "model": model, + "instructions": "Alpha rule.", + "input": [ + {"role": "developer", "content": [{"type": "input_text", "text": "Bravo rule."}]}, + {"role": "user", "content": [{"type": "input_text", "text": "Reply in JSON"}]}, + ], + }, + ) + received: Final = wire.drain() + assert response.status_code == 200, response.text + assert [(request.method, request.target) for request in received] == [("POST", "/chat/completions")] + assert _sent_messages(received[0]) == [ + {"role": "system", "content": "Alpha rule.\n\nBravo rule."}, + {"role": "user", "content": "Reply in JSON"}, + ] + + +def test_databricks_fallback_sends_the_collapsed_string_to_both_deployments(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + fallback_backend: Final = "databricks-meta-llama-3-1-8b-instruct" + + def respond(request: Request) -> Reply: + if _JSON_OBJECT.validate_json(request.body)["model"] == _LLAMA_BACKEND: + return Reply(status=500, body=b'{"error_code": "INTERNAL_ERROR", "message": "primary deployment is down"}') + return _shape_reply(f"databricks-fallback-{marker}", False) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + primary: Final = scenario.model(model=f"databricks/{_LLAMA_BACKEND}", api_base=wire.url, api_key=_API_KEY) + fallback: Final = scenario.model(model=f"databricks/{fallback_backend}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": primary, + "messages": [{"role": "user", "content": _one_block(f"Reply in JSON {marker}")}], + "response_format": {"type": "json_object"}, + "fallbacks": [fallback], + "num_retries": 0, + "cache": _NO_CACHE, + }, + ) + received: Final = wire.drain() + assert response.status_code == 200, response.text + assert _JSON_OBJECT.validate_json(response.content)["id"] == f"databricks-fallback-{marker}" + assert [_JSON_OBJECT.validate_json(request.body)["model"] for request in received] == [ + _LLAMA_BACKEND, + fallback_backend, + ] + assert [_sent_messages(request) for request in received] == [ + [{"role": "user", "content": f"Reply in JSON {marker}"}], + ] * 2 + + +def _proxy_root(gateway: Gateway) -> str: + return str(gateway.client.base_url).rstrip("/") + + +def test_databricks_openai_sdk_sync_one_block_reaches_upstream_as_string_and_logs_spend(gateway: Gateway) -> None: + identity: Final = f"databricks-openai-sdk-{uuid.uuid4().hex}" + with ( + wire_server(lambda _request: _shape_reply(identity, False)) as wire, + gateway.scenario() as scenario, + openai.OpenAI(base_url=f"{_proxy_root(gateway)}/v1", api_key=gateway.key, max_retries=0) as client, + ): + model: Final = scenario.model(model=f"databricks/{_LLAMA_BACKEND}", api_base=wire.url, api_key=_API_KEY) + completion: Final = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": [{"type": "text", "text": "Reply in JSON"}]}], + response_format={"type": "json_object"}, + extra_body={"cache": _NO_CACHE}, + ) + received: Final = wire.drain() + assert (completion.id, completion.choices[0].message.content) == (identity, "ok") + assert [_sent_messages(request) for request in received] == [[{"role": "user", "content": "Reply in JSON"}]] + rows: Final = eventually( + lambda: read_rows('SELECT status, call_type FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (identity,)), + lambda values: len(values) == 1, + seconds=70, + ) + assert (rows[0]["status"], rows[0]["call_type"]) == ("success", "acompletion") + + +async def test_databricks_openai_sdk_async_stream_one_block_reaches_upstream_as_string(gateway: Gateway) -> None: + identity: Final = f"databricks-openai-async-{uuid.uuid4().hex}" + with wire_server(lambda _request: _shape_reply(identity, True)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"databricks/{_LLAMA_BACKEND}", api_base=wire.url, api_key=_API_KEY) + async with openai.AsyncOpenAI( + base_url=f"{_proxy_root(gateway)}/v1", api_key=gateway.key, max_retries=0 + ) as client: + stream: Final = await client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": [{"type": "text", "text": "Reply in JSON"}]}], + response_format={"type": "json_object"}, + stream=True, + extra_body={"cache": _NO_CACHE}, + ) + chunks: Final = [chunk async for chunk in stream] + received: Final = wire.drain() + assert {chunk.id for chunk in chunks} == {identity} + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) == "ok" + assert [_sent_messages(request) for request in received] == [[{"role": "user", "content": "Reply in JSON"}]] + + +async def test_databricks_anthropic_sdk_async_stream_one_block_system_and_user_reach_upstream_as_strings( + gateway: Gateway, +) -> None: + identity: Final = f"databricks-anthropic-async-{uuid.uuid4().hex}" + with wire_server(lambda _request: _shape_reply(identity, True)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"databricks/{_LLAMA_BACKEND}", api_base=wire.url, api_key=_API_KEY) + async with anthropic.AsyncAnthropic( + base_url=_proxy_root(gateway), api_key=gateway.key, max_retries=0 + ) as client: + async with client.messages.stream( + model=model, + max_tokens=32, + system=[{"type": "text", "text": "Reply in JSON."}], + messages=[{"role": "user", "content": [{"type": "text", "text": "What is 2+2?"}]}], + ) as stream: + final: Final = await stream.get_final_message() + received: Final = wire.drain() + assert [block.text for block in final.content if block.type == "text"] == ["ok"] + assert [_sent_messages(request) for request in received] == [ + [{"role": "system", "content": "Reply in JSON."}, {"role": "user", "content": "What is 2+2?"}] + ] + + +def test_databricks_one_block_burst_through_an_upstream_outage_reaches_upstream_as_strings(gateway: Gateway) -> None: + plan: Final = tuple(_burst_request(index) for index in range(24)) + + def respond(request: Request) -> Reply: + if _OUTAGE.encode() in request.body: + error: Final = {"error_code": "TEMPORARILY_UNAVAILABLE", "message": _OUTAGE_MESSAGE} + return Reply(status=503, body=json.dumps(error).encode()) + body: Final = _JSON_OBJECT.validate_json(request.body) + return _upstream_reply(_JSON_SCHEMA_BACKEND, body.get("stream") is True) + + def one_block_body(endpoint: _Endpoint, model: str, stream: bool, prompt: str) -> dict[str, JsonValue]: + body: Final = _client_body(endpoint, model, stream, prompt) + match endpoint: + case "responses": + return {**body, "input": [{"role": "user", "content": [{"type": "input_text", "text": prompt}]}]} + case "chat" | "messages": + return {**body, "messages": [{"role": "user", "content": _one_block(prompt)}]} + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"databricks/{_JSON_SCHEMA_BACKEND}", api_base=wire.url, api_key=_API_KEY) + prompts: Final = tuple( + f"{_JSON_SCHEMA_PROMPT} {index} {_OUTAGE if down else ''}".strip() + for index, (_e, _s, down) in enumerate(plan) + ) + bodies: Final = tuple( + one_block_body(endpoint, model, stream, prompt) for (endpoint, stream, _down), prompt in zip(plan, prompts) + ) + with ThreadPoolExecutor(max_workers=12) as pool: + results: Final = tuple(pool.map(_call, itertools.repeat(gateway), (e for e, _s, _d in plan), bodies)) + received: Final = wire.drain() + assert sorted(json.dumps(_sent_messages(request)) for request in received) == sorted( + json.dumps([{"role": "user", "content": prompt}]) for prompt in prompts + ) + expected: Final = tuple( + (503, _OUTAGE_MESSAGE) if down else (200, _EXPECTED_JSON) for _endpoint, _stream, down in plan + ) + assert tuple(_outcome(*result) for result in results) == expected + + +def test_databricks_transform_request_shows_the_collapsed_string(gateway: Gateway, tmp_path: Path) -> None: + environment: Final = { + "DATABRICKS_API_BASE": "https://databricks.invalid/serving-endpoints", + "DATABRICKS_API_KEY": _API_KEY, + } + with owned_proxy_process(gateway, tmp_path, environment) as owned: + response: Final = owned.gateway.request( + "POST", + "/utils/transform_request", + { + "call_type": "completion", + "request_body": { + "model": f"databricks/{_LLAMA_BACKEND}", + "messages": [{"role": "user", "content": _one_block("Reply in JSON")}], + }, + }, + ) + assert response.status_code == 200, response.text + body: Final = _JSON_OBJECT.validate_json(response.content) + assert body["error"] is None, body + assert body["raw_request_api_base"] == "https://databricks.invalid/serving-endpoints/chat/completions" + sent: Final = body["raw_request_body"] + assert isinstance(sent, dict), body + assert sent["messages"] == [{"role": "user", "content": "Reply in JSON"}] + + +def test_databricks_non_string_text_block_is_rejected_before_reaching_upstream(gateway: Gateway) -> None: + with wire_server(_plain_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"databricks/{_LLAMA_BACKEND}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": [{"type": "text", "text": 42}]}], + "cache": _NO_CACHE, + }, + ) + received: Final = wire.drain() + assert response.status_code == 400, response.text + assert "'int' object has no attribute 'strip'" in response.text + assert received == () diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index fe76a667f3e..d25787d7028 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch import httpx import pytest import respx +from pydantic import TypeAdapter from fastapi.testclient import TestClient from pydantic import BaseModel, ConfigDict @@ -23,6 +24,7 @@ from litellm.llms.databricks.chat.transformation import ( DatabricksConfig, _sanitize_empty_content, ) +from litellm.types.llms.openai import AllMessageValues import asyncio from unittest.mock import Mock from litellm._version import version @@ -432,6 +434,119 @@ def test_transform_request_preserves_unity_model_service_name(): assert result["model"] == "system.ai.kimi-k3" +@pytest.mark.parametrize( + "model,messages,optional_params,expected_content", + [ + pytest.param( + "databricks-meta-llama-3-3-70b-instruct", + [{"role": "user", "content": [{"type": "text", "text": "Reply in JSON"}]}], + {"response_format": {"type": "json_object"}}, + "Reply in JSON", + id="user", + ), + pytest.param( + "databricks-meta-llama-3-3-70b-instruct", + [{"role": "system", "content": [{"type": "text", "text": "Reply in JSON"}]}], + {"response_format": {"type": "json_object"}}, + "Reply in JSON", + id="system", + ), + pytest.param( + "databricks-meta-llama-3-3-70b-instruct", + [{"role": "assistant", "content": [{"type": "text", "text": "Reply in JSON"}]}], + {"response_format": {"type": "json_object"}}, + "Reply in JSON", + id="assistant", + ), + pytest.param( + "databricks-meta-llama-3-3-70b-instruct", + [ + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{"type": "text", "text": "Reply in JSON"}], + } + ], + {"response_format": {"type": "json_object"}}, + "Reply in JSON", + id="tool", + ), + pytest.param( + "databricks-meta-llama-3-3-70b-instruct", + [ + { + "role": "user", + "content": [{"type": "text", "text": "Reply in JSON", "cache_control": {"type": "ephemeral"}}], + } + ], + {"response_format": {"type": "json_object"}}, + [{"type": "text", "text": "Reply in JSON", "cache_control": {"type": "ephemeral"}}], + id="cache-control", + ), + pytest.param( + "databricks-meta-llama-3-3-70b-instruct", + [{"role": "user", "content": [{"type": "text", "text": "Reply"}, {"type": "text", "text": " in JSON"}]}], + {"response_format": {"type": "json_object"}}, + [{"type": "text", "text": "Reply"}, {"type": "text", "text": " in JSON"}], + id="multiple-text-blocks", + ), + pytest.param( + "databricks-meta-llama-3-3-70b-instruct", + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Reply in JSON"}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.png"}}, + ], + } + ], + {"response_format": {"type": "json_object"}}, + [ + {"type": "text", "text": "Reply in JSON"}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.png"}}, + ], + id="text-and-image", + ), + pytest.param( + "databricks-meta-llama-3-3-70b-instruct", + [{"role": "user", "content": "Reply in JSON"}], + {"response_format": {"type": "json_object"}}, + "Reply in JSON", + id="plain-string", + ), + pytest.param( + "databricks-claude-sonnet-5", + [{"role": "user", "content": [{"type": "text", "text": "Reply in JSON"}]}], + {"response_format": {"type": "json_object"}}, + [{"type": "text", "text": "Reply in JSON"}], + id="claude", + ), + ], +) +def test_transform_request_collapses_single_plain_text_block_for_non_claude_models( + model: str, + messages: list[AllMessageValues], + optional_params: dict[str, object], + expected_content: object, +) -> None: + config: Final[DatabricksConfig] = DatabricksConfig() + result: Final[dict[str, object]] = TypeAdapter(dict[str, object]).validate_python( + config.transform_request( # pyright: ignore[reportUnknownMemberType] # inherited hook is untyped + model=model, + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + ) + outgoing_messages: Final[list[dict[str, object]]] = TypeAdapter(list[dict[str, object]]).validate_python( + result["messages"] + ) + + assert outgoing_messages == [{**messages[0], "content": expected_content}] + + def test_transform_request_strips_thinking_blocks_and_reasoning_content(): """Regression for LIT-6762: replaying an assistant turn that litellm decorated with `thinking_blocks` / `reasoning_content` made Databricks 400 with @@ -2117,6 +2232,7 @@ def mock_chat_streaming_response_chunks() -> List[str]: "databricks-meta-llama-3-3-70b-instruct", "databricks-qwen35-122b-a10b", "databricks-gpt-oss-120b", + "system.ai.deepseek-v4-1-flash", ), ) def test_databricks_non_claude_json_schema_preserves_refs(model: str) -> None: diff --git a/tests/unit/responses/test_responses_api_bridge_flag.py b/tests/unit/responses/test_responses_api_bridge_flag.py index fb1361c1f49..699144a31bb 100644 --- a/tests/unit/responses/test_responses_api_bridge_flag.py +++ b/tests/unit/responses/test_responses_api_bridge_flag.py @@ -356,11 +356,8 @@ class TestUseResponsesApiBridgeFlag: assert upstream.call_count == 1 request_body: Final = json.loads(upstream.calls[0].request.read()) assert request_body["messages"] == [ - { - "role": "system", - "content": [{"type": "text", "text": "You are terse."}, {"type": "text", "text": "Skills: none."}], - }, - {"role": "user", "content": [{"type": "text", "text": "Hello"}]}, + {"role": "system", "content": "You are terse.\n\nSkills: none."}, + {"role": "user", "content": "Hello"}, ] assert "client_metadata" not in request_body assert request_body["chat_template_kwargs"] == {"thinking": True} From 30b8db325e85740eb86901d08597da845b206504 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:38:25 +0000 Subject: [PATCH 20/51] feat(ui): show decision models as decisions and list the supported ones in the AI Hub (#45965) * feat(ui): show decision models as decisions and list the supported ones in the AI Hub * fix(ui): gate the decision garden's Add model links and sort the Mode column by its label The garden shows Add model only for providers the Add Model form can create and only when the session can add models, since a view-only admin's session role reads as Admin. The AI Hub's Mode column sorts by the label it shows, and the Models page parses its tab slug as a string literal * fix(ui): label the make-public dialog's decision models, clear the prefill after the add, and gate the garden's Add model by the Models page's rule * fix(ui): drop the dead decisions billing-row filter and the unread team fetch on the AI Hub The /decisions/ cost map keys never shipped, so the garden filter matched nothing. The AI Hub page fetched /team/list for every visitor and read it for nobody, since only proxy admins reach the add-model check. The org-admin page test now asserts the role string a session carries * fix(ui): keep self-hosted decision models off the /v1/decisions claim, order the garden one per provider, and show the decisions example on the public hub * fix(ui): mark decision models this proxy cannot serve yet in the AI Hub garden * fix(ui): use singular wording in the decision garden header when one model is left * fix(ui): list gpt-6-luna in the decision garden, send Foundry to its deployment-name docs, and record Models page tab clicks * fix(ui): say the decisions filter matched no model in decisions mode, and mock usePathname in the Models page integration test * fix(ui): stop saying deployed decision models still need adding --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../providers/useProxyDecisionProviders.ts | 17 ++ .../(dashboard)/model-hub-table/page.test.tsx | 57 ++++ .../app/(dashboard)/model-hub-table/page.tsx | 8 +- .../page.integration.test.tsx | 5 +- .../models-and-endpoints/page.test.tsx | 91 +++++- .../(dashboard)/models-and-endpoints/page.tsx | 46 ++- .../panels/AddModelPanel.integration.test.tsx | 80 ++++- .../panels/AddModelPanel.tsx | 38 ++- .../src/app/model_hub_table/page.tsx | 1 + .../components/AIHub/DecisionModelGarden.tsx | 159 ++++++++++ .../components/AIHub/ModelHubTable.test.tsx | 275 +++++++++++++++++- .../src/components/AIHub/ModelHubTable.tsx | 47 ++- .../components/AIHub/ModelHubTableColumns.tsx | 2 +- .../AIHub/forms/MakeModelPublicForm.test.tsx | 13 + .../AIHub/forms/MakeModelPublicForm.tsx | 3 +- .../components/PublicModelHubTableColumns.tsx | 3 +- .../src/components/model_filters.tsx | 15 +- .../publicModelHubFilters.test.ts | 6 + .../publicModelHub/publicModelHubFilters.ts | 3 + .../src/components/public_model_hub.test.tsx | 48 ++- .../src/components/public_model_hub.tsx | 77 +++-- .../src/lib/decisionModels.test.ts | 121 +++++++- .../src/lib/decisionModels.ts | 68 ++++- 23 files changed, 1086 insertions(+), 97 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/providers/useProxyDecisionProviders.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/model-hub-table/page.test.tsx create mode 100644 ui/litellm-dashboard/src/components/AIHub/DecisionModelGarden.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/providers/useProxyDecisionProviders.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/providers/useProxyDecisionProviders.ts new file mode 100644 index 00000000000..fd79bb2fbb4 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/providers/useProxyDecisionProviders.ts @@ -0,0 +1,17 @@ +import { getGuardrailUISettings } from "@/components/networking"; +import { parseProxyDecisionProviders } from "@/lib/decisionModels"; +import { useQuery } from "@tanstack/react-query"; +import { createQueryKeys } from "../common/queryKeysFactory"; + +const proxyDecisionProvidersKeys = createQueryKeys("proxyDecisionProviders"); + +export const useProxyDecisionProviders = (accessToken: string | null) => { + const query = { + queryKey: proxyDecisionProvidersKeys.list({}), + queryFn: async () => parseProxyDecisionProviders(await getGuardrailUISettings(accessToken ?? "")), + enabled: accessToken !== null, + retry: false, + staleTime: 24 * 60 * 60 * 1000, + }; + return useQuery(query); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/model-hub-table/page.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/model-hub-table/page.test.tsx new file mode 100644 index 00000000000..c6348cdb052 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/model-hub-table/page.test.tsx @@ -0,0 +1,57 @@ +import { render, screen } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import ModelHubTablePage from "./page"; + +const mockUseAuthorized = vi.fn(); +const hubProps = vi.fn(); + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => mockUseAuthorized() })); +vi.mock("@/components/AIHub/ModelHubTable", () => ({ + default: (props: { canAddModels: boolean }) => { + hubProps(props); + return
; + }, +})); +vi.mock("@/components/public_model_hub", () => ({ + default: () =>
, +})); + +const session = (userRole: string, isViewOnly = false) => ({ + accessToken: "token", + userRole, + userId: "user-1", + premiumUser: false, + isViewOnly, +}); + +describe("ModelHubTablePage", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("lets a proxy admin add models from the garden", () => { + mockUseAuthorized.mockReturnValue(session("Admin")); + render(); + expect(hubProps).toHaveBeenCalledWith(expect.objectContaining({ canAddModels: true })); + }); + + it("keeps the garden read-only for a view-only admin", () => { + mockUseAuthorized.mockReturnValue(session("Admin", true)); + render(); + expect(hubProps).toHaveBeenCalledWith(expect.objectContaining({ canAddModels: false })); + }); + + it("sends an org admin to the public hub under the role string a session carries", () => { + mockUseAuthorized.mockReturnValue(session("Org Admin")); + render(); + expect(screen.getByTestId("public-model-hub")).toBeInTheDocument(); + expect(hubProps).not.toHaveBeenCalled(); + }); + + it("sends a non-admin to the public hub", () => { + mockUseAuthorized.mockReturnValue(session("internal_user")); + render(); + expect(screen.getByTestId("public-model-hub")).toBeInTheDocument(); + expect(hubProps).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/model-hub-table/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/model-hub-table/page.tsx index c9eb8762ed2..d402af41112 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/model-hub-table/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/model-hub-table/page.tsx @@ -3,13 +3,18 @@ import ModelHubTable from "@/components/AIHub/ModelHubTable"; import PublicModelHub from "@/components/public_model_hub"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { canCreateModels } from "@/utils/modelPermissions"; import { isAdminRole } from "@/utils/roles"; export default function ModelHubTablePage() { - const { accessToken, userRole, premiumUser, isViewOnly } = useAuthorized(); + const { accessToken, userRole, userId: userID, premiumUser, isViewOnly } = useAuthorized(); if (!isAdminRole(userRole)) { return ; } + const canAddModels = canCreateModels( + { userRole, userID, isViewOnly }, + { teams: null, disabledForInternalUsers: false }, + ); return ( ); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.integration.test.tsx index fa62f43c2d1..4d18966625e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.integration.test.tsx @@ -39,7 +39,10 @@ vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ }), })); -vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn() }) })); +vi.mock("next/navigation", () => ({ + useRouter: () => ({ push: vi.fn() }), + usePathname: () => "/ui/models-and-endpoints", +})); vi.mock("@/components/networking", () => ({ serverRootPath: "", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx index a9cb68ae8dd..3cfed7297c8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx @@ -1,10 +1,15 @@ /* @vitest-environment jsdom */ -import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { render, screen } from "@testing-library/react"; +import { screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { OnUrlUpdateFunction } from "nuqs/adapters/testing"; +import { beforeEach, describe, expect, it, vi, type Mock } from "vitest"; +import { renderWithProviders } from "../../../../tests/test-utils"; +import { recordUiEvent } from "@/lib/telemetry/uiEvents"; import ModelsAndEndpointsPage from "./page"; +vi.mock("@/lib/telemetry/uiEvents", () => ({ recordUiEvent: vi.fn(() => Promise.resolve()) })); +vi.mock("next/navigation", () => ({ usePathname: () => "/ui/models-and-endpoints" })); + vi.mock("./panels/AllModelsPanel", () => ({ default: () =>
})); vi.mock("./panels/AddModelPanel", () => ({ default: () =>
})); vi.mock("./panels/AutoRoutersTabPanel", () => ({ default: () =>
})); @@ -61,14 +66,13 @@ const NON_ADMIN = { // A proxy_admin_viewer session: effectiveSessionRole masquerades the role as "Admin". const VIEW_ONLY_ADMIN = { ...ADMIN, isViewOnly: true }; -const renderPage = () => { - const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }); - return render( - - - , - ); -}; +const renderPage = (options?: { searchParams?: string; onUrlUpdate?: OnUrlUpdateFunction }) => + renderWithProviders(, options); + +const lastUrlParams = (onUrlUpdate: Mock): URLSearchParams | undefined => + onUrlUpdate.mock.calls.at(-1)?.[0].searchParams; + +const ADD_MODEL_PREFILL_LINK = "?tab=add&provider=typesafe&litellm_model=typesafe%2Fjev-latest"; describe("ModelsAndEndpointsPage", () => { beforeEach(() => { @@ -224,4 +228,69 @@ describe("ModelsAndEndpointsPage", () => { expect(screen.queryByRole("tab", { name: /Auto-Routers/ })).not.toBeInTheDocument(); }); }); + + describe("tab from the URL", () => { + it("opens the Add Model tab from ?tab=add for an admin", () => { + renderPage({ searchParams: "?tab=add" }); + + expect(screen.getByRole("tab", { name: "Add Model" })).toHaveAttribute("aria-selected", "true"); + expect(screen.getByTestId("panel-add")).toBeInTheDocument(); + expect(screen.queryByTestId("panel-all-models")).not.toBeInTheDocument(); + }); + + it("falls back to Deployed Models on ?tab=add for a view-only admin, who has no Add Model tab", () => { + mockUseAuthorized.mockReturnValue(VIEW_ONLY_ADMIN); + renderPage({ searchParams: "?tab=add" }); + + expect(screen.getByRole("tab", { name: "Deployed Models" })).toHaveAttribute("aria-selected", "true"); + expect(screen.getByTestId("panel-all-models")).toBeInTheDocument(); + expect(screen.queryByTestId("panel-add")).not.toBeInTheDocument(); + }); + + it("falls back to Deployed Models on an unknown ?tab", () => { + renderPage({ searchParams: "?tab=nope" }); + + expect(screen.getByRole("tab", { name: "Deployed Models" })).toHaveAttribute("aria-selected", "true"); + expect(screen.getByTestId("panel-all-models")).toBeInTheDocument(); + }); + + it("records a UI event for each tab the user clicks", async () => { + const user = userEvent.setup(); + renderPage(); + + await user.click(screen.getByRole("tab", { name: "Health Status" })); + + expect(recordUiEvent).toHaveBeenCalledWith({ + page: "models-and-endpoints", + action: "click", + target: "tab=health", + }); + }); + + it("writes the clicked tab to the URL and clears it for Deployed Models", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderPage({ onUrlUpdate }); + + await user.click(screen.getByRole("tab", { name: "Health Status" })); + await waitFor(() => expect(lastUrlParams(onUrlUpdate)?.get("tab")).toBe("health")); + expect(screen.getByTestId("panel-health")).toBeInTheDocument(); + + await user.click(screen.getByRole("tab", { name: "Deployed Models" })); + await waitFor(() => expect(lastUrlParams(onUrlUpdate)?.get("tab")).toBeNull()); + expect(screen.getByTestId("panel-all-models")).toBeInTheDocument(); + }); + + it("drops the Add Model prefill params when the user leaves the tab", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderPage({ searchParams: ADD_MODEL_PREFILL_LINK, onUrlUpdate }); + + await user.click(screen.getByRole("tab", { name: "Deployed Models" })); + + await waitFor(() => expect(lastUrlParams(onUrlUpdate)?.get("tab")).toBeNull()); + expect(lastUrlParams(onUrlUpdate)?.get("provider")).toBeNull(); + expect(lastUrlParams(onUrlUpdate)?.get("litellm_model")).toBeNull(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx index a54c4ee7292..9f1ee04c5a2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx @@ -2,10 +2,12 @@ import { useMemo, useState } from "react"; import { RefreshCw } from "lucide-react"; +import { parseAsString, useQueryStates } from "nuqs"; import { useQueryClient } from "@tanstack/react-query"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings"; +import { useUrlTab } from "@/hooks/useUrlTab"; import { all_admin_roles, internalUserRoles } from "@/utils/roles"; import { autoRouterCreationScope, canCreateModels } from "@/utils/modelPermissions"; import BetaBadge from "@/components/BetaBadge"; @@ -27,19 +29,26 @@ import PriceDataPanel from "@/app/(dashboard)/models-and-endpoints/panels/PriceD import { Button } from "@/components/ui/button"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -type ModelTabSlug = - | "add" - | "auto-routers" - | "llm-credentials" - | "pass-through" - | "health" - | "retry-settings" - | "model-group-alias" - | "access-group-budgets" - | "price-data"; +const MODEL_TAB_SLUGS = [ + "add", + "auto-routers", + "llm-credentials", + "pass-through", + "health", + "retry-settings", + "model-group-alias", + "access-group-budgets", + "price-data", +] as const; + +type ModelTabSlug = (typeof MODEL_TAB_SLUGS)[number]; const BASE_TAB_KEY = "all-models"; +const TAB_KEYS = [BASE_TAB_KEY, ...MODEL_TAB_SLUGS] as const; + +const PREFILL_URL_PARAMS = { provider: parseAsString, litellm_model: parseAsString }; + const TAB_LABELS: Record = { add: "Add Model", "auto-routers": "Auto-Routers", @@ -87,7 +96,8 @@ export default function ModelsAndEndpointsPage() { const { modelId, teamId, close } = useModelDetailRouting(); const { availableModelAccessGroups, allModelsOnProxy } = useModelDashboardData(); - const [activeKey, setActiveKey] = useState(BASE_TAB_KEY); + const [requestedTab, setRequestedTab] = useUrlTab(TAB_KEYS, BASE_TAB_KEY); + const [, setPrefillParams] = useQueryStates(PREFILL_URL_PARAMS); const [lastRefreshed, setLastRefreshed] = useState(""); const isInternalUser = userRole && internalUserRoles.includes(userRole); @@ -124,6 +134,18 @@ export default function ModelsAndEndpointsPage() { [canCreate, canViewAutoRouters, isAdmin, isInternalUser, isViewOnly], ); + const visibleKeys: readonly string[] = visibleSlugs.map((slug) => slug || BASE_TAB_KEY); + const activeKey = visibleKeys.includes(requestedTab) ? requestedTab : BASE_TAB_KEY; + + const handleTabChange = (key: string) => { + const next = TAB_KEYS.find((tabKey) => tabKey === key); + if (next === undefined) { + return; + } + setRequestedTab(next); + void setPrefillParams({ provider: null, litellm_model: null }); + }; + const allModelsLabel = isAdmin ? "Deployed Models" : "Your Models"; const tabLabel = (slug: "" | ModelTabSlug): React.ReactNode => { if (!slug) return allModelsLabel; @@ -192,7 +214,7 @@ export default function ModelsAndEndpointsPage() { modelAccessGroups={availableModelAccessGroups} /> ) : ( - +
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx index 05560c9e7cd..99656df3b0e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx @@ -1,5 +1,6 @@ import { useSyncExternalStore } from "react"; import { act, renderWithProviders, screen, waitFor, within } from "../../../../../tests/test-utils"; +import type { OnUrlUpdateFunction } from "nuqs/adapters/testing"; import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import AddModelPanel from "./AddModelPanel"; @@ -140,9 +141,9 @@ const baseModelInfo = { access_groups: undefined, mode: undefined }; const { api_base: _omitted, ...ALWAYS_MOUNTED_WITHOUT_API_BASE } = alwaysMounted; -const setup = async () => { +const setup = async (searchParams?: string, onUrlUpdate?: OnUrlUpdateFunction) => { const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never }); - renderWithProviders(); + renderWithProviders(, { searchParams, onUrlUpdate }); await screen.findByText("Provider"); const openAdvanced = async () => { @@ -595,3 +596,78 @@ describe("AddModelPanel decision models", () => { expect(note).not.toHaveTextContent("/chat/completions"); }); }); + +describe("AddModelPanel prefill from the URL", () => { + const COST_MAP = { + "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" }, + "gpt-4o-2024-08-06": { litellm_provider: "openai", mode: "chat" }, + }; + + beforeEach(() => { + vi.clearAllMocks(); + mockUseModelCostMap.mockReturnValue({ data: COST_MAP }); + mockPtuEnabled.mockReturnValue(false); + mockAuthorized.mockReturnValue(PROXY_ADMIN); + }); + + it("preselects the provider and model named in the URL and submits that model", async () => { + const { submit } = await setup("?provider=typesafe&litellm_model=typesafe%2Fjev-latest"); + + await waitFor(() => expect(screen.getByRole("combobox", { name: /provider/i })).toHaveValue("TypeSafe")); + expect(screen.getByRole("combobox", { name: "Select models" })).toBeInTheDocument(); + expect(screen.getByDisplayValue("typesafe/jev-latest")).toBeInTheDocument(); + expect(await screen.findByRole("note", { name: "Decision model notice" })).toBeInTheDocument(); + + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "typesafe/jev-latest", + litellm_params: { api_key: undefined, custom_llm_provider: "typesafe", model: "typesafe/jev-latest" }, + model_info: { ...baseModelInfo }, + }); + }); + + it("drops the prefill params from the URL once the prefilled model is added", async () => { + const onUrlUpdate = vi.fn(); + const { submit } = await setup("?provider=typesafe&litellm_model=typesafe%2Fjev-latest", onUrlUpdate); + await waitFor(() => expect(screen.getByRole("combobox", { name: /provider/i })).toHaveValue("TypeSafe")); + + await submit(); + + await waitFor(() => { + const params = onUrlUpdate.mock.calls.at(-1)?.[0].searchParams; + expect(params?.has("provider")).toBe(false); + expect(params?.has("litellm_model")).toBe(false); + }); + }); + + it("leaves the form on its defaults when the provider slug matches no provider, model included", async () => { + const { submitExpectingRejection } = await setup("?provider=unknownslug&litellm_model=typesafe%2Fjev-latest"); + + expect(screen.getByRole("combobox", { name: /provider/i })).toHaveValue(""); + expect(screen.queryByDisplayValue("typesafe/jev-latest")).not.toBeInTheDocument(); + + await submitExpectingRejection("Required"); + + expect(modelCreateCall).not.toHaveBeenCalled(); + }); + + it("keeps a provider the user picks after the prefill instead of re-applying the URL", async () => { + const { user, submit } = await setup("?provider=typesafe&litellm_model=typesafe%2Fjev-latest"); + await waitFor(() => expect(screen.getByRole("combobox", { name: /provider/i })).toHaveValue("TypeSafe")); + + const provider = screen.getByRole("combobox", { name: /provider/i }); + await user.clear(provider); + await user.type(provider, "OpenAI"); + await user.click(await screen.findByRole("option", { name: /OpenAI/ })); + await user.click(await screen.findByPlaceholderText("Select models")); + await user.click(await screen.findByText("gpt-4o-2024-08-06")); + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "gpt-4o-2024-08-06", + litellm_params: { ...alwaysMounted, model: "gpt-4o-2024-08-06" }, + model_info: { ...baseModelInfo }, + }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx index 4703bda8d20..392a3c3940a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx @@ -1,7 +1,8 @@ "use client"; -import { useMemo, useState } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; import { useForm, useWatch } from "react-hook-form"; +import { parseAsString, useQueryStates } from "nuqs"; import { useQueryClient } from "@tanstack/react-query"; import AddModelForm from "@/components/add_model/AddModelForm"; import { handleAddModelSubmit } from "@/components/add_model/handle_add_model_submit"; @@ -13,12 +14,15 @@ import { import { Providers, getPlaceholder, getProviderModels } from "@/components/provider_info_helpers"; import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; import { useCredentials } from "@/app/(dashboard)/hooks/credentials/useCredentials"; +import { useProviderFields } from "@/app/(dashboard)/hooks/providers/useProviderFields"; import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { buildDecisionCatalog } from "@/lib/decisionModels"; const INITIAL_VALUES: MountedFormValues = { litellm_credential_name: null }; +const PREFILL_URL_PARAMS = { provider: parseAsString, litellm_model: parseAsString }; + export default function AddModelPanel() { const { accessToken } = useAuthorized(); const form = useForm({ mode: "onChange", defaultValues: INITIAL_VALUES }); @@ -28,6 +32,31 @@ export default function AddModelPanel() { const { data: credentialsResponse } = useCredentials(); const { data: teams } = useTeams(); const [selectedProvider, setSelectedProvider] = useState(Providers.Anthropic); + const [{ provider: prefillProviderSlug, litellm_model: prefillModel }, setPrefillParams] = + useQueryStates(PREFILL_URL_PARAMS); + const { data: providerMetadata } = useProviderFields(); + const prefillProvider = useMemo( + () => + prefillProviderSlug === null + ? null + : providerMetadata?.find((info) => info.litellm_provider === prefillProviderSlug)?.provider ?? null, + [prefillProviderSlug, providerMetadata], + ); + const prefillApplied = useRef(false); + + useEffect(() => { + if (prefillApplied.current || prefillProvider === null) { + return; + } + prefillApplied.current = true; + setSelectedProvider(prefillProvider); + form.setValue("custom_llm_provider", prefillProvider); + if (prefillModel === null) { + return; + } + form.setValue("model", [prefillModel]); + }, [form, prefillModel, prefillProvider]); + const pickedProvider = useWatch({ control: form.control, name: "custom_llm_provider" }); const providerModels = useMemo( () => (typeof pickedProvider === "string" ? getProviderModels(pickedProvider, modelCostMapData) : []), @@ -48,7 +77,12 @@ export default function AddModelPanel() { await handleAddModelSubmit( mountedValues(), accessToken, - { resetFields: () => form.reset(INITIAL_VALUES) }, + { + resetFields: () => { + form.reset(INITIAL_VALUES); + void setPrefillParams({ provider: null, litellm_model: null }); + }, + }, refresh, ); return true; diff --git a/ui/litellm-dashboard/src/app/model_hub_table/page.tsx b/ui/litellm-dashboard/src/app/model_hub_table/page.tsx index 46c7394cff7..af6764dcd4c 100644 --- a/ui/litellm-dashboard/src/app/model_hub_table/page.tsx +++ b/ui/litellm-dashboard/src/app/model_hub_table/page.tsx @@ -22,6 +22,7 @@ function PublicModelHubTableContent() { premiumUser={false} userRole={null} canOpenPlayground={false} + canAddModels={false} /> ); } diff --git a/ui/litellm-dashboard/src/components/AIHub/DecisionModelGarden.tsx b/ui/litellm-dashboard/src/components/AIHub/DecisionModelGarden.tsx new file mode 100644 index 00000000000..dcd2528c761 --- /dev/null +++ b/ui/litellm-dashboard/src/components/AIHub/DecisionModelGarden.tsx @@ -0,0 +1,159 @@ +import React, { useMemo, useState } from "react"; +import { BookOpen, Plus, Sparkles } from "lucide-react"; + +import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; +import { useProviderFields } from "@/app/(dashboard)/hooks/providers/useProviderFields"; +import { useProxyDecisionProviders } from "@/app/(dashboard)/hooks/providers/useProxyDecisionProviders"; +import { Logo } from "@/components/molecules/logo/Logo"; +import { getProviderLogoAndName } from "@/components/provider_info_helpers"; +import { Button } from "@/components/ui/button"; +import { Card } from "@/components/ui/card"; +import { + DECISIONS_DOCS_URL, + DECISIONS_PROVIDERS_DOCS_URL, + DECISIONS_SELF_HOSTED_DOCS_URL, + DecisionGardenModel, + DecisionRoute, + buildDecisionGarden, +} from "@/lib/decisionModels"; +import { uiHref } from "@/utils/uiHref"; + +const PREVIEW_COUNT = 8; + +const plural = (count: number, one: string, many: string): string => (count === 1 ? one : many); + +const ROUTE_LINKS: Readonly> = { + decisions: { href: DECISIONS_DOCS_URL, label: "/v1/decisions docs" }, + "self-hosted": { href: DECISIONS_SELF_HOSTED_DOCS_URL, label: "Self-hosted, own route" }, + "needs-upgrade": { href: DECISIONS_PROVIDERS_DOCS_URL, label: "Needs a newer LiteLLM" }, +}; + +const DEPLOYMENT_NAMED_PROVIDERS: ReadonlySet = new Set(["azure", "azure_ai"]); + +const OWN_DEPLOYMENT_LINK = { href: DECISIONS_PROVIDERS_DOCS_URL, label: "Deploy under your own name" } as const; + +const needsOwnDeploymentName = (row: DecisionGardenModel): boolean => + row.route === "decisions" && DEPLOYMENT_NAMED_PROVIDERS.has(row.provider); + +const cardLink = (row: DecisionGardenModel) => + needsOwnDeploymentName(row) ? OWN_DEPLOYMENT_LINK : ROUTE_LINKS[row.route]; + +export const addModelHref = (row: DecisionGardenModel): string => { + const params = new URLSearchParams({ tab: "add", provider: row.provider, litellm_model: row.model }); + return `${uiHref("models-and-endpoints")}?${params.toString()}`; +}; + +interface DecisionModelCardProps { + readonly row: DecisionGardenModel; + readonly canAddModels: boolean; +} + +function DecisionModelCard({ row, canAddModels }: DecisionModelCardProps) { + const { displayName } = getProviderLogoAndName(row.provider); + const link = cardLink(row); + return ( +
+
+ + {displayName} +
+ + {row.model} + +
+ + {link.label} + + {canAddModels && ( + + )} +
+
+ ); +} + +interface DecisionModelGardenProps { + readonly accessToken: string | null; + readonly canAddModels: boolean; +} + +const DecisionModelGarden: React.FC = ({ accessToken, canAddModels }) => { + const { data: costMap } = useModelCostMap(true, true); + const { data: providerFields } = useProviderFields(); + const { data: proxyProviders, isLoading: proxyProvidersLoading } = useProxyDecisionProviders(accessToken); + const rows = useMemo(() => buildDecisionGarden(costMap, proxyProviders ?? null), [costMap, proxyProviders]); + const addableProviders = useMemo( + () => new Set((providerFields ?? []).map((info) => info.litellm_provider)), + [providerFields], + ); + const [showAll, setShowAll] = useState(false); + const isAddable = (row: DecisionGardenModel): boolean => + row.route === "decisions" && !needsOwnDeploymentName(row) && addableProviders.has(row.provider); + + if (proxyProvidersLoading || rows.length === 0) { + return null; + } + + const routedRows = rows.filter((row) => row.route === "decisions"); + const routedProviderCount = new Set(routedRows.map((row) => row.provider)).size; + const selfHostedCount = rows.filter((row) => row.route === "self-hosted").length; + const needsUpgradeCount = rows.filter((row) => row.route === "needs-upgrade").length; + const visibleRows = showAll ? rows : rows.slice(0, PREVIEW_COUNT); + + return ( + +
+
+ +
+
+

Decision models LiteLLM supports

+

+ {routedRows.length} decision {plural(routedRows.length, "model", "models")} across {routedProviderCount}{" "} + {plural(routedProviderCount, "provider", "providers")} {plural(routedRows.length, "answers", "answer")} on{" "} + /v1/decisions. + {selfHostedCount > 0 && + ` ${selfHostedCount} more ${plural(selfHostedCount, "is self-hosted and answers on its own route", "are self-hosted and answer on their own routes")}.`} + {needsUpgradeCount > 0 && + ` ${needsUpgradeCount} more ${plural(needsUpgradeCount, "needs", "need")} a newer LiteLLM version.`} +

+
+ +
+
+ {visibleRows.map((row) => ( + + ))} +
+ {rows.length > PREVIEW_COUNT && ( +
+ +
+ )} + + ); +}; + +export default DecisionModelGarden; diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx index cc3ac1d2591..d1a118f238d 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx @@ -3,7 +3,7 @@ import userEvent from "@testing-library/user-event"; import { act } from "@testing-library/react"; import type { MCPServerData } from "./MCPHubTableColumns"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { renderWithProviders, screen, waitFor } from "../../../tests/test-utils"; +import { renderWithProviders, screen, testQueryClient, waitFor, within } from "../../../tests/test-utils"; import ModelHubTable from "./ModelHubTable"; const mockUseUISettings = vi.hoisted(() => vi.fn()); @@ -24,8 +24,31 @@ vi.mock("@/components/networking", () => ({ makeMCPPublicCall: vi.fn(), getUiSettings: vi.fn(), getClaudeCodePluginsList: vi.fn(() => Promise.resolve({ plugins: [] })), + modelCostMap: vi.fn(() => Promise.resolve({})), + getProviderCreateMetadata: vi.fn(() => Promise.resolve([])), + getGuardrailUISettings: vi.fn(() => Promise.resolve({})), })); +const GARDEN_COST_MAP = { + "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" }, + "cloudflare/clef": { litellm_provider: "cloudflare", mode: "evaluation" }, + "newvendor/decider": { litellm_provider: "newvendor", mode: "evaluation" }, + "laya/english": { litellm_provider: "laya", mode: "evaluation", supported_endpoints: ["/v1/systemone"] }, + "claude-opus-4-8": { litellm_provider: "anthropic", mode: "chat" }, +}; + +const ADDABLE_PROVIDER_FIELDS = [ + { provider: "TypeSafe", provider_display_name: "TypeSafe", litellm_provider: "typesafe", credential_fields: [] }, + { provider: "Laya", provider_display_name: "Laya", litellm_provider: "laya", credential_fields: [] }, + { provider: "OpenAI", provider_display_name: "OpenAI", litellm_provider: "openai", credential_fields: [] }, + { + provider: "Azure_AI_Studio", + provider_display_name: "Azure AI Foundry (Studio)", + litellm_provider: "azure_ai", + credential_fields: [], + }, +]; + vi.mock("next/navigation", () => ({ useRouter: () => ({ replace: mockRouterReplace, @@ -72,6 +95,7 @@ describe("ModelHubTable", () => { writable: true, }); vi.clearAllMocks(); + testQueryClient.clear(); }); // Reusable helper function to setup mocks for auth redirect tests @@ -123,6 +147,7 @@ describe("ModelHubTable", () => { premiumUser={false} userRole={null} canOpenPlayground={false} + canAddModels={false} />, ); @@ -163,6 +188,7 @@ describe("ModelHubTable", () => { premiumUser={false} userRole={null} canOpenPlayground + canAddModels />, ); @@ -181,7 +207,14 @@ describe("ModelHubTable", () => { }); renderWithProviders( - , + , ); expect(await screen.findByText("No models yet")).toBeInTheDocument(); @@ -215,6 +248,7 @@ describe("ModelHubTable", () => { premiumUser={false} userRole={null} canOpenPlayground={false} + canAddModels={false} />, ); @@ -248,6 +282,7 @@ describe("ModelHubTable", () => { premiumUser={false} userRole="Admin" canOpenPlayground={canOpenPlayground} + canAddModels />, ); await user.click(await screen.findByRole("button", { name: model.model_group })); @@ -289,6 +324,241 @@ describe("ModelHubTable", () => { }); }); + describe("decision model garden", () => { + const renderHub = async ( + models: { model_group: string; providers: string[]; mode: string }[], + { + userRole = "Admin", + canAddModels = true, + loadProxySettings = async () => ({ decision_model_providers: ["typesafe", "cloudflare"] }), + costMap = GARDEN_COST_MAP, + }: { + userRole?: string; + canAddModels?: boolean; + loadProxySettings?: () => Promise; + costMap?: Record; + } = {}, + ) => { + vi.mocked(networking.modelHubCall).mockResolvedValue({ data: models }); + vi.mocked(networking.modelCostMap).mockResolvedValue(costMap); + vi.mocked(networking.getProviderCreateMetadata).mockResolvedValue([ + ...ADDABLE_PROVIDER_FIELDS, + { + provider: "NewVendor", + provider_display_name: "NewVendor", + litellm_provider: "newvendor", + credential_fields: [], + }, + ]); + vi.mocked(networking.getGuardrailUISettings).mockImplementation(loadProxySettings); + vi.mocked(networking.getConfigFieldSetting).mockResolvedValue({ field_value: false }); + vi.mocked(networking.getAgentsList).mockResolvedValue({ agents: [] }); + vi.mocked(networking.fetchMCPServers).mockResolvedValue([]); + vi.mocked(networking.getUiSettings).mockResolvedValue({ values: {} }); + mockUseUISettings.mockReturnValue({ data: { values: {} }, isLoading: false }); + const user = userEvent.setup(); + renderWithProviders( + , + ); + await screen.findByText(/Showing \d+ of \d+ models/); + return { user, modeSelect: () => screen.getByDisplayValue("All Modes") }; + }; + + const chatOnly = [{ model_group: "claude-opus-4-8", providers: ["anthropic"], mode: "chat" }]; + + it("lists the decision models from the cost map when the proxy has none configured", async () => { + await renderHub(chatOnly); + + const garden = await screen.findByTestId("decision-model-garden"); + expect(garden).toHaveTextContent("2 decision models across 2 providers answer on /v1/decisions"); + expect(within(garden).getByTestId("decision-model-card-typesafe/jev-latest")).toHaveTextContent("TypeSafe"); + expect(within(garden).getByTestId("decision-model-card-cloudflare/clef")).toHaveTextContent("Cloudflare"); + expect(within(garden).queryByText("claude-opus-4-8")).not.toBeInTheDocument(); + }); + + it("sends a self-hosted model that does not answer on /v1/decisions to its own docs, with no Add Model link", async () => { + await renderHub(chatOnly); + + const garden = await screen.findByTestId("decision-model-garden"); + expect(garden).toHaveTextContent("1 more is self-hosted and answers on its own route."); + const layaCard = within(garden).getByTestId("decision-model-card-laya/english"); + expect(within(layaCard).queryByRole("link", { name: "/v1/decisions docs" })).not.toBeInTheDocument(); + expect(within(layaCard).getByRole("link", { name: "Self-hosted, own route" })).toHaveAttribute( + "href", + "https://docs.litellm.ai/docs/decisions#self-hosted-laya-and-nimble", + ); + expect(await within(garden).findAllByRole("link", { name: "Add model" })).toHaveLength(1); + expect(within(layaCard).queryByRole("link", { name: "Add model" })).not.toBeInTheDocument(); + }); + + it("links each card to the decisions docs and to Add Model prefilled with that provider and model", async () => { + await renderHub(chatOnly); + + const card = within(await screen.findByTestId("decision-model-garden")).getByTestId( + "decision-model-card-typesafe/jev-latest", + ); + expect(within(card).getByRole("link", { name: "/v1/decisions docs" })).toHaveAttribute( + "href", + "https://docs.litellm.ai/docs/decisions", + ); + expect(await within(card).findByRole("link", { name: "Add model" })).toHaveAttribute( + "href", + expect.stringMatching(/\/models-and-endpoints\?tab=add&provider=typesafe&litellm_model=typesafe%2Fjev-latest$/), + ); + }); + + it("labels a model whose provider this proxy cannot serve on /v1/decisions, with no Add Model and no count", async () => { + await renderHub(chatOnly); + + const garden = await screen.findByTestId("decision-model-garden"); + expect(garden).toHaveTextContent("2 decision models across 2 providers answer on /v1/decisions"); + expect(garden).toHaveTextContent("1 more needs a newer LiteLLM version."); + const newCard = within(garden).getByTestId("decision-model-card-newvendor/decider"); + expect(within(newCard).getByRole("link", { name: "Needs a newer LiteLLM" })).toHaveAttribute( + "href", + "https://docs.litellm.ai/docs/decisions#supported-providers", + ); + expect(await within(garden).findAllByRole("link", { name: "Add model" })).toHaveLength(1); + expect(within(newCard).queryByRole("link", { name: "Add model" })).not.toBeInTheDocument(); + }); + + it("lists OpenAI's chat model that answers on /v1/decisions, with Add Model prefilled", async () => { + await renderHub(chatOnly, { + loadProxySettings: async () => ({ decision_model_providers: ["openai"] }), + costMap: { + "gpt-6-luna": { + litellm_provider: "openai", + mode: "chat", + supported_endpoints: ["/v1/chat/completions", "/v1/decisions"], + }, + "gpt-5.5": { litellm_provider: "openai", mode: "chat", supported_endpoints: ["/v1/chat/completions"] }, + }, + }); + + const garden = await screen.findByTestId("decision-model-garden"); + expect(garden).toHaveTextContent("1 decision model across 1 provider answers on /v1/decisions."); + const card = within(garden).getByTestId("decision-model-card-gpt-6-luna"); + expect(await within(card).findByRole("link", { name: "Add model" })).toHaveAttribute( + "href", + expect.stringMatching(/\?tab=add&provider=openai&litellm_model=gpt-6-luna$/), + ); + expect(within(garden).queryByTestId("decision-model-card-gpt-5.5")).not.toBeInTheDocument(); + }); + + it("sends a Foundry decision model to the docs for its deployment name instead of prefilling one Azure refuses", async () => { + await renderHub(chatOnly, { + loadProxySettings: async () => ({ decision_model_providers: ["typesafe", "azure_ai"] }), + costMap: { + "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" }, + "azure_ai/Microsoft-Decision-1": { litellm_provider: "azure_ai", mode: "evaluation" }, + }, + }); + + const garden = await screen.findByTestId("decision-model-garden"); + expect(garden).toHaveTextContent("2 decision models across 2 providers answer on /v1/decisions"); + const foundryCard = within(garden).getByTestId("decision-model-card-azure_ai/Microsoft-Decision-1"); + expect(within(foundryCard).getByRole("link", { name: "Deploy under your own name" })).toHaveAttribute( + "href", + "https://docs.litellm.ai/docs/decisions#supported-providers", + ); + expect(await within(garden).findAllByRole("link", { name: "Add model" })).toHaveLength(1); + expect(within(foundryCard).queryByRole("link", { name: "Add model" })).not.toBeInTheDocument(); + }); + + it("falls back to the cost map alone when this proxy's decision provider list cannot be read", async () => { + await renderHub(chatOnly, { + loadProxySettings: async () => { + throw new Error("Failed to get guardrail UI settings"); + }, + }); + + const garden = await screen.findByTestId("decision-model-garden"); + expect(garden).toHaveTextContent("3 decision models across 3 providers answer on /v1/decisions"); + expect(garden).not.toHaveTextContent("a newer LiteLLM version"); + }); + + it("offers a decisions mode option that keeps the garden, hides the chat rows, and points at the garden", async () => { + const { user, modeSelect } = await renderHub(chatOnly); + await screen.findByTestId("decision-model-garden"); + + expect(within(modeSelect()).queryByRole("option", { name: "evaluation" })).not.toBeInTheDocument(); + await user.selectOptions(modeSelect(), "decisions"); + + expect(screen.getByTestId("decision-model-garden")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "claude-opus-4-8" })).not.toBeInTheDocument(); + expect(screen.getByText("Showing 0 of 1 models")).toBeInTheDocument(); + expect(screen.getByText("No model on this proxy uses the decisions mode")).toBeInTheDocument(); + expect(screen.queryByText("Adjust the filters to see more models.")).not.toBeInTheDocument(); + }); + + it("hides the garden while a non-decision mode is selected", async () => { + const { user, modeSelect } = await renderHub(chatOnly); + await screen.findByTestId("decision-model-garden"); + + await user.selectOptions(modeSelect(), "chat"); + expect(screen.queryByTestId("decision-model-garden")).not.toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Clear Filters" })); + expect(await screen.findByTestId("decision-model-garden")).toBeInTheDocument(); + }); + + it("labels a configured evaluation model as decisions in the table and under the decisions filter", async () => { + const { user, modeSelect } = await renderHub([ + ...chatOnly, + { model_group: "jev-latest", providers: ["typesafe"], mode: "evaluation" }, + ]); + + const row = screen.getByRole("button", { name: "jev-latest" }).closest("tr"); + expect(row).toHaveTextContent("decisions"); + expect(row).not.toHaveTextContent("evaluation"); + + await user.selectOptions(modeSelect(), "decisions"); + expect(screen.getByText("Showing 1 of 2 models")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "jev-latest" })).toBeInTheDocument(); + }); + + it("drops the Add Model link for an admin session that cannot add models", async () => { + await renderHub(chatOnly, { canAddModels: false }); + + const garden = await screen.findByTestId("decision-model-garden"); + expect(within(garden).queryByRole("link", { name: "Add model" })).not.toBeInTheDocument(); + expect(within(garden).getAllByRole("link", { name: "/v1/decisions docs" })).toHaveLength(2); + }); + + it("drops the Add Model link on a card whose provider the Add Model form cannot create", async () => { + await renderHub(chatOnly); + + const garden = await screen.findByTestId("decision-model-garden"); + const typesafeCard = within(garden).getByTestId("decision-model-card-typesafe/jev-latest"); + const cloudflareCard = within(garden).getByTestId("decision-model-card-cloudflare/clef"); + expect(await within(typesafeCard).findByRole("link", { name: "Add model" })).toBeInTheDocument(); + expect(within(cloudflareCard).queryByRole("link", { name: "Add model" })).not.toBeInTheDocument(); + expect(within(cloudflareCard).getByRole("link", { name: "/v1/decisions docs" })).toBeInTheDocument(); + }); + + it("sorts the Mode column by the label it shows, placing decisions between chat and embedding", async () => { + const { user } = await renderHub([ + { model_group: "text-embedding-4", providers: ["openai"], mode: "embedding" }, + { model_group: "jev-latest", providers: ["typesafe"], mode: "evaluation" }, + ...chatOnly, + ]); + + await user.click(screen.getByTestId("sort-header-mode")); + + const order = screen + .getAllByRole("button", { name: /^(claude-opus-4-8|jev-latest|text-embedding-4)$/ }) + .map((button) => button.textContent); + expect(order).toEqual(["claude-opus-4-8", "jev-latest", "text-embedding-4"]); + }); + }); + describe("hub tabs", () => { const renderHub = async (agents: object[] = [], mcpServers: Promise = Promise.resolve([])) => { vi.mocked(networking.modelHubCall).mockResolvedValue({ @@ -308,6 +578,7 @@ describe("ModelHubTable", () => { premiumUser={false} userRole="Admin" canOpenPlayground + canAddModels />, ); return { user, search: await screen.findByPlaceholderText("Search model names...") }; diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx index 2007020e117..3e23d64d28a 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx @@ -44,7 +44,14 @@ import { checkTokenValidity } from "@/utils/jwtUtils"; import { getCookie } from "@/utils/cookieUtils"; import { getLoginUrl } from "@/utils/returnUrlUtils"; import { uiHref } from "@/utils/uiHref"; -import { DECISIONS_DOCS_URL, SYSTEM_ONE_PLAYGROUND_ROUTE, displayMode, isDecisionMode } from "@/lib/decisionModels"; +import { + DECISIONS_DOCS_URL, + DECISIONS_MODE, + SYSTEM_ONE_PLAYGROUND_ROUTE, + displayMode, + isDecisionMode, +} from "@/lib/decisionModels"; +import DecisionModelGarden from "@/components/AIHub/DecisionModelGarden"; interface ModelHubTableProps { accessToken: string | null; @@ -52,12 +59,29 @@ interface ModelHubTableProps { premiumUser: boolean; userRole: string | null; canOpenPlayground: boolean; + canAddModels: boolean; } function isMCPHubVisibilityDisabled(isLoading: boolean, servers: readonly MCPServerData[] | null): boolean { return isLoading || servers === null; } +function modelTableEmptyState( + models: readonly ModelHubData[], + showsDecisionGarden: boolean, + selectedMode: string, +): { title: string; body: string } { + if (showsDecisionGarden && selectedMode === DECISIONS_MODE && !models.some((model) => isDecisionMode(model.mode))) { + return { + title: "No model on this proxy uses the decisions mode", + body: "The decision models LiteLLM supports are listed below.", + }; + } + return models.length > 0 + ? { title: "No matching models", body: "Adjust the filters to see more models." } + : { title: "No models yet", body: "Models added to this proxy will appear here." }; +} + function HubEmptyState({ title, body }: { title: string; body: string }) { return (
@@ -76,6 +100,7 @@ const ModelHubTable: React.FC = ({ premiumUser, userRole, canOpenPlayground, + canAddModels, }) => { const syntaxTheme = useSyntaxTheme(prism); // Admin Viewer follows the read-parity rule: see the AI Hub catalog, but @@ -88,6 +113,8 @@ const ModelHubTable: React.FC = ({ const [isModalVisible, setIsModalVisible] = useState(false); const [selectedModel, setSelectedModel] = useState(null); const [filteredData, setFilteredData] = useState([]); + const [selectedMode, setSelectedMode] = useState(""); + const showDecisionGarden = !publicPage && (selectedMode === "" || selectedMode === DECISIONS_MODE); const [isMakePublicModalVisible, setIsMakePublicModalVisible] = useState(false); // Agent Hub state const [agentHubData, setAgentHubData] = useState(null); @@ -479,7 +506,12 @@ const ModelHubTable: React.FC = ({ )} {/* Filters */} - + {/* Model Table */} = ({ isLoading={loading} loadingMessage="Loading models…" noDataMessage={ - + } size="compact" /> @@ -511,6 +536,8 @@ const ModelHubTable: React.FC = ({ Showing {filteredData.length} of {modelHubData?.length || 0} models

+ + {showDecisionGarden && } {/* Agent Hub Tab */} diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx index 559c18930c7..1a5f8889c93 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx @@ -129,7 +129,7 @@ export const getModelHubTableColumns = ({ onModelClick }: ModelHubTableColumnsDe }, { id: "mode", - accessorKey: "mode", + accessorFn: (row) => (row.mode ? displayMode(row.mode) : ""), meta: { title: "Mode", className: "hidden lg:table-cell" }, header: ({ column }) => , size: 110, diff --git a/ui/litellm-dashboard/src/components/AIHub/forms/MakeModelPublicForm.test.tsx b/ui/litellm-dashboard/src/components/AIHub/forms/MakeModelPublicForm.test.tsx index bbeee883b1a..f99facad6fb 100644 --- a/ui/litellm-dashboard/src/components/AIHub/forms/MakeModelPublicForm.test.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/forms/MakeModelPublicForm.test.tsx @@ -329,6 +329,19 @@ describe("MakeModelPublicForm", () => { expect(screen.getAllByText("openai")).toHaveLength(2); }); + it("labels an evaluation-mode model as decisions, matching the mode filter", () => { + const decisionModel = { + ...mockProps.modelHubData[0], + model_group: "jev-latest", + providers: ["typesafe"], + mode: "evaluation", + }; + render(); + + expect(screen.getByText("decisions")).toBeInTheDocument(); + expect(screen.queryByText("evaluation")).not.toBeInTheDocument(); + }); + it("should handle submit error properly", async () => { const errorMessage = "Network error"; mockMakeModelGroupPublic.mockRejectedValueOnce(new Error(errorMessage)); diff --git a/ui/litellm-dashboard/src/components/AIHub/forms/MakeModelPublicForm.tsx b/ui/litellm-dashboard/src/components/AIHub/forms/MakeModelPublicForm.tsx index 073eedca191..20fa0295fed 100644 --- a/ui/litellm-dashboard/src/components/AIHub/forms/MakeModelPublicForm.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/forms/MakeModelPublicForm.tsx @@ -8,6 +8,7 @@ import { cn } from "@/lib/cva.config"; import { makeModelGroupPublic } from "../../networking"; import ModelFilters from "../../model_filters"; import { toast } from "@/lib/toast"; +import { displayMode } from "@/lib/decisionModels"; const STEP_TITLES = ["Select Models", "Confirm"]; @@ -186,7 +187,7 @@ const MakeModelPublicForm: React.FC = ({

{model.model_group}

- {model.mode && {model.mode}} + {model.mode && {displayMode(model.mode)}}
{model.providers.map((provider) => ( diff --git a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx index 2fae8044d6b..0288c58e31e 100644 --- a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx @@ -7,6 +7,7 @@ import { CellTooltip, IdentityCell, StatusBadge, type StatusTone } from "@/compo import { Badge } from "@/components/ui/badge"; import { getProviderLogoAndName } from "@/components/provider_info_helpers"; import { PUBLIC_MODEL_HUB_SORTABLE_FIELDS } from "@/components/publicModelHub/publicModelHubFilters"; +import { displayMode } from "@/lib/decisionModels"; export interface ModelGroupInfo { model_group: string; @@ -203,7 +204,7 @@ export const getPublicModelHubColumns = ({ onModelClick }: PublicModelHubColumns cell: ({ row }) => ( {getModeIcon(row.original.mode || "")} - {row.original.mode || "Chat"} + {row.original.mode ? displayMode(row.original.mode) : "Chat"} ), }, diff --git a/ui/litellm-dashboard/src/components/model_filters.tsx b/ui/litellm-dashboard/src/components/model_filters.tsx index 1eef5083f16..165cbf68fe0 100644 --- a/ui/litellm-dashboard/src/components/model_filters.tsx +++ b/ui/litellm-dashboard/src/components/model_filters.tsx @@ -26,6 +26,8 @@ interface ModelFiltersProps { onFilteredDataChange: (filteredData: ModelGroupInfo[]) => void; showFiltersCard?: boolean; className?: string; + extraModes?: readonly string[]; + onModeChange?: (mode: string) => void; } const ModelFilters: React.FC = ({ @@ -33,6 +35,8 @@ const ModelFilters: React.FC = ({ onFilteredDataChange, showFiltersCard = true, className = "", + extraModes = [], + onModeChange, }) => { const [searchTerm, setSearchTerm] = useState(""); const [selectedProvider, setSelectedProvider] = useState(""); @@ -50,7 +54,7 @@ const ModelFilters: React.FC = ({ }; const getUniqueModes = (data: ModelGroupInfo[]) => { - const modes = new Set(); + const modes = new Set(extraModes); data.forEach((model) => { const mode = displayMode(model.mode); if (mode) modes.add(mode); @@ -58,6 +62,11 @@ const ModelFilters: React.FC = ({ return Array.from(modes); }; + const changeMode = (mode: string) => { + setSelectedMode(mode); + onModeChange?.(mode); + }; + const getUniqueFeatures = (data: ModelGroupInfo[]) => { const features = new Set(); data.forEach((model) => { @@ -119,7 +128,7 @@ const ModelFilters: React.FC = ({ const resetFilters = () => { setSearchTerm(""); setSelectedProvider(""); - setSelectedMode(""); + changeMode(""); setSelectedFeature(""); }; @@ -159,7 +168,7 @@ const ModelFilters: React.FC = ({

Mode: