From 9968499aabf6d6b4e36579c05979fc1deda44098 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 30 Jun 2026 11:28:51 -0700 Subject: [PATCH 01/81] fix(ui): fix Router Settings Loadbalancing tab save (LIT-4057) The Loadbalancing tab rendered routing_groups as a generic text input and sent its array value back as the JSON string "[]", which fails Pydantic list validation on POST /config/update and returns 422. routing_groups has its own dedicated Routing Groups tab, so this tab must neither render nor write it; exclude it the same way retry_policy and model_group_retry_policy are excluded for the Model Retry Settings tab. The save was also fire-and-forget: setCallbacksCall was not awaited, so the rejected promise escaped the try/catch and the success toast fired unconditionally, showing success even when the backend rejected the change. Await the call, gate the success toast on resolution, and surface the error. --- .../ReliabilityRetriesSection.tsx | 5 ++- .../components/router_settings/index.test.tsx | 40 +++++++++++++++++++ .../src/components/router_settings/index.tsx | 13 +++--- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/ui/litellm-dashboard/src/components/router_settings/ReliabilityRetriesSection.tsx b/ui/litellm-dashboard/src/components/router_settings/ReliabilityRetriesSection.tsx index fa48c1c97b9..da089552b11 100644 --- a/ui/litellm-dashboard/src/components/router_settings/ReliabilityRetriesSection.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/ReliabilityRetriesSection.tsx @@ -20,14 +20,15 @@ const ReliabilityRetriesSection: React.FC = ({
{Object.entries(routerSettings) .filter( - ([param, value]) => + ([param]) => param != "fallbacks" && param != "context_window_fallbacks" && param != "routing_strategy_args" && param != "routing_strategy" && param != "enable_tag_filtering" && param != "retry_policy" && - param != "model_group_retry_policy", + param != "model_group_retry_policy" && + param != "routing_groups", ) .map(([param, value]) => (
diff --git a/ui/litellm-dashboard/src/components/router_settings/index.test.tsx b/ui/litellm-dashboard/src/components/router_settings/index.test.tsx index 78a0b4b0dff..0df1cd81d6b 100644 --- a/ui/litellm-dashboard/src/components/router_settings/index.test.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/index.test.tsx @@ -146,4 +146,44 @@ describe("RouterSettings", () => { expect(NotificationsManager.success).toHaveBeenCalledWith("router settings updated successfully"); }); + + it("should not render or save routing_groups (owned by the Routing Groups tab) (LIT-4057)", async () => { + const user = userEvent.setup(); + vi.mocked(getCallbacksCall).mockResolvedValue({ + router_settings: { + routing_strategy: "simple-shuffle", + num_retries: 3, + routing_groups: [{ group_name: "g1", models: ["gpt-4"], routing_strategy: "simple-shuffle" }], + }, + }); + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByTestId("strategy-select")).toBeInTheDocument(); + }); + expect(document.querySelector('input[name="routing_groups"]')).toBeNull(); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + const payload = vi.mocked(setCallbacksCall).mock.calls[0][1] as { + router_settings: Record; + }; + expect(payload.router_settings).not.toHaveProperty("routing_groups"); + }); + + it("should surface an error and not claim success when saving fails (LIT-4057)", async () => { + const user = userEvent.setup(); + vi.mocked(setCallbacksCall).mockRejectedValue(new Error("422 Unprocessable Entity")); + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByTestId("strategy-select")).toBeInTheDocument(); + }); + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => { + expect(NotificationsManager.fromBackend).toHaveBeenCalled(); + }); + expect(NotificationsManager.success).not.toHaveBeenCalled(); + }); }); diff --git a/ui/litellm-dashboard/src/components/router_settings/index.tsx b/ui/litellm-dashboard/src/components/router_settings/index.tsx index d3753529058..360c7f41138 100644 --- a/ui/litellm-dashboard/src/components/router_settings/index.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/index.tsx @@ -81,7 +81,7 @@ const RouterSettings: React.FC = ({ accessToken, userRole, }); }, [accessToken, userRole, userID]); - const handleSaveChanges = () => { + const handleSaveChanges = async () => { if (!accessToken) { return; } @@ -91,9 +91,9 @@ const RouterSettings: React.FC = ({ accessToken, userRole, const numberKeys = new Set(["allowed_fails", "cooldown_time", "num_retries", "timeout", "retry_after"]); const jsonKeys = new Set(["model_group_alias"]); - // retry_policy and model_group_retry_policy are owned exclusively by the - // Model Retry Settings tab; this page must not read or write them. - const tabOwnedKeys = new Set(["retry_policy", "model_group_retry_policy"]); + // retry_policy and model_group_retry_policy are owned by the Model Retry Settings tab; + // routing_groups is owned by the Routing Groups tab. This page must not read or write them. + const tabOwnedKeys = new Set(["retry_policy", "model_group_retry_policy", "routing_groups"]); const parseInputValue = (key: string, raw: string | undefined, fallback: unknown) => { if (raw === undefined) return fallback; @@ -172,12 +172,11 @@ const RouterSettings: React.FC = ({ accessToken, userRole, }; try { - setCallbacksCall(accessToken, payload); + await setCallbacksCall(accessToken, payload); + NotificationsManager.success("router settings updated successfully"); } catch (error) { NotificationsManager.fromBackend("Failed to update router settings: " + error); } - - NotificationsManager.success("router settings updated successfully"); }; if (!accessToken) { From 30141f86f824cdc53e425a4b27fe07e34cc61c64 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 30 Jun 2026 11:40:09 -0700 Subject: [PATCH 02/81] test(ui): make router settings save tests resilient to async timing Address Greptile P2: the routing_groups test read setCallbacksCall.mock.calls[0][1] immediately after the now-async save handler, so any latency in the mock would throw an opaque TypeError instead of a clean assertion failure. Assert through toHaveBeenCalledWith inside waitFor with expect.not.objectContaining, dropping the index access and the cast. Also drop the ticket id from the test names. --- .../src/components/router_settings/index.test.tsx | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ui/litellm-dashboard/src/components/router_settings/index.test.tsx b/ui/litellm-dashboard/src/components/router_settings/index.test.tsx index 0df1cd81d6b..94cbb94d164 100644 --- a/ui/litellm-dashboard/src/components/router_settings/index.test.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/index.test.tsx @@ -147,7 +147,7 @@ describe("RouterSettings", () => { expect(NotificationsManager.success).toHaveBeenCalledWith("router settings updated successfully"); }); - it("should not render or save routing_groups (owned by the Routing Groups tab) (LIT-4057)", async () => { + it("should not render or save routing_groups (owned by the Routing Groups tab)", async () => { const user = userEvent.setup(); vi.mocked(getCallbacksCall).mockResolvedValue({ router_settings: { @@ -165,13 +165,14 @@ describe("RouterSettings", () => { await user.click(screen.getByRole("button", { name: /save changes/i })); - const payload = vi.mocked(setCallbacksCall).mock.calls[0][1] as { - router_settings: Record; - }; - expect(payload.router_settings).not.toHaveProperty("routing_groups"); + await waitFor(() => + expect(setCallbacksCall).toHaveBeenCalledWith("test-token", { + router_settings: expect.not.objectContaining({ routing_groups: expect.anything() }), + }), + ); }); - it("should surface an error and not claim success when saving fails (LIT-4057)", async () => { + it("should surface an error and not claim success when saving fails", async () => { const user = userEvent.setup(); vi.mocked(setCallbacksCall).mockRejectedValue(new Error("422 Unprocessable Entity")); renderWithProviders(); From 540c860a9737838e5fa2146aa8d4cd2fab445548 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 30 Jun 2026 11:47:35 -0700 Subject: [PATCH 03/81] test(ui): add typed e2e for Router Settings Loadbalancing save (LIT-4057) Drives the real save flow against a live proxy: seeds a present routing_groups array (the LIT-4057 trigger) via the typed /config/update contract, changes num_retries on the Loadbalancing tab, and asserts the POST returns 200 instead of 422, the success toast appears, and the value still shows after a reload (the ticket's "refresh shows old values" symptom). The round-trip is typed against the OpenAPI-generated backend schema (ConfigYAML write, RouterSettingsResponse read) through a type-only import, so a backend contract drift fails the type check. --- .../tests/settings/routerSettings.spec.ts | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts index 98b86ec9b11..e560500fca6 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts @@ -3,6 +3,10 @@ import { ADMIN_STORAGE_PATH } from "../../constants"; import { navigateToPage } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; import { Role, users } from "../../fixtures/users"; +// Type-only import of the OpenAPI-generated backend schema; esbuild erases it at +// runtime, so the round-trip below is checked against the real /config/update and +// /router/settings contracts without bundling the 2 MB definition file. +import type { components } from "../../../src/lib/http/schema"; const PRIMARY = "fake-openai-gpt-4"; const FALLBACK = "fake-anthropic-claude"; @@ -99,3 +103,83 @@ test.describe("Router Settings - Fallbacks", () => { await expect(newRow).toHaveCount(1, { timeout: 10_000 }); }); }); + +type ConfigYAML = components["schemas"]["ConfigYAML"]; +type RouterSettingsResponse = components["schemas"]["RouterSettingsResponse"]; + +const BASE_URL = "http://localhost:4000"; +const ADMIN_AUTH = { Authorization: `Bearer ${users[Role.ProxyAdmin].password}` }; + +/** + * Merge a router_settings patch into the live config through the typed + * /config/update contract, preserving any other settings already present. + */ +async function patchRouterSettings( + request: import("@playwright/test").APIRequestContext, + patch: Partial>, +) { + const current = await request.get(`${BASE_URL}/get/config/callbacks`, { headers: ADMIN_AUTH }); + const existing = current.ok() ? (await current.json())?.router_settings ?? {} : {}; + const payload = { router_settings: { ...(existing as Record), ...patch } }; + await request.post(`${BASE_URL}/config/update`, { headers: ADMIN_AUTH, data: payload }); +} + +test.describe("Router Settings - Loadbalancing", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + // Seed a present routing_groups array (the LIT-4057 trigger) plus a known + // num_retries so the UI assertions are deterministic across reruns. + const ROUTING_GROUP = { group_name: "e2e-lit-4057", models: [PRIMARY], routing_strategy: "simple-shuffle" }; + + test.beforeEach(async ({ request }) => { + await patchRouterSettings(request, { num_retries: 3, routing_groups: [ROUTING_GROUP] }); + }); + + test.afterEach(async ({ request }) => { + await patchRouterSettings(request, { num_retries: 3, routing_groups: [] }); + }); + + test("saves the Loadbalancing tab without a 422 when routing_groups is present, and persists", async ({ + page, + request, + }) => { + await navigateToPage(page, Page.RouterSettings); + await page.getByRole("tab", { name: "Loadbalancing" }).click(); + + const numRetries = page.locator('input[name="num_retries"]'); + await expect(numRetries).toHaveValue("3", { timeout: 15_000 }); + // routing_groups belongs to its own tab and must not leak into this form. + await expect(page.locator('input[name="routing_groups"]')).toHaveCount(0); + + await numRetries.fill("5"); + + // LIT-4057: the tab used to serialize routing_groups as the string "[]", + // which the backend rejects with 422 while the UI still claimed success. + // Assert the save actually succeeds at the network level. + const saveResponse = page.waitForResponse( + (res) => res.url().includes("/config/update") && res.request().method() === "POST", + { timeout: 15_000 }, + ); + await page.getByRole("button", { name: /save changes/i }).click(); + expect((await saveResponse).status()).toBe(200); + + await expect(page.getByText(/router settings updated successfully/i).first()).toBeVisible({ timeout: 10_000 }); + + // The ticket's core symptom was that a refresh showed the old value. + await navigateToPage(page, Page.RouterSettings); + await page.getByRole("tab", { name: "Loadbalancing" }).click(); + await expect(page.locator('input[name="num_retries"]')).toHaveValue("5", { timeout: 15_000 }); + + // The typed backend read agrees the change persisted. + await expect + .poll( + async () => { + const res = await request.get(`${BASE_URL}/router/settings`, { headers: ADMIN_AUTH }); + const data = (await res.json()) as RouterSettingsResponse; + return data.current_values?.num_retries; + }, + { timeout: 10_000 }, + ) + .toBe(5); + }); +}); From 3971469b71e4454bc02dc5ee1d5b9bd1618566ad Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 30 Jun 2026 12:08:15 -0700 Subject: [PATCH 04/81] test(ui): harden Router Settings e2e and make its typing a real CI gate Address an adversarial review of the Loadbalancing e2e: - The "typed against the backend schema" claim was hollow: nothing type-checked e2e_tests (the root tsconfig excludes it and no CI step runs tsc), so a contract drift would compile and run unchanged. Add e2e_tests/tsconfig.json, a typecheck:e2e script, and a CircleCI step so the schema typing actually gates. - The two describe blocks both mutate the proxy's shared router_settings, and the Loadbalancing save echoes the whole settings object, so they could clobber each other under local fullyParallel. Run the file serially. - patchRouterSettings swallowed a failed seed, which surfaced later as a misleading UI timeout. Assert the write succeeded, and rely on the server-side merge instead of echoing the whole settings object back (drops a cast and a GET). - Empty routing_groups already reproduces the bug, so drop the non-empty seed and its model coupling. --- .circleci/config.yml | 8 +++++ .../tests/settings/routerSettings.spec.ts | 36 +++++++++++-------- ui/litellm-dashboard/e2e_tests/tsconfig.json | 9 +++++ ui/litellm-dashboard/package.json | 1 + 4 files changed, 39 insertions(+), 15 deletions(-) create mode 100644 ui/litellm-dashboard/e2e_tests/tsconfig.json diff --git a/.circleci/config.yml b/.circleci/config.yml index f13e9bf66f1..009884cbfe4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2687,6 +2687,14 @@ jobs: paths: - ui/litellm-dashboard/node_modules - ~/.cache/ms-playwright + - run: + name: Type-check E2E specs + # The specs type their request/response round-trips against the generated + # OpenAPI schema; this step turns that typing into a real gate, so a backend + # contract drift fails here instead of silently passing at runtime. + command: | + cd ui/litellm-dashboard + npm run typecheck:e2e - run: name: Build UI from source # Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`. diff --git a/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts index e560500fca6..64dab6d7cf6 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts @@ -3,11 +3,16 @@ import { ADMIN_STORAGE_PATH } from "../../constants"; import { navigateToPage } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; import { Role, users } from "../../fixtures/users"; -// Type-only import of the OpenAPI-generated backend schema; esbuild erases it at -// runtime, so the round-trip below is checked against the real /config/update and -// /router/settings contracts without bundling the 2 MB definition file. +// Type-only import of the OpenAPI-generated backend schema. esbuild erases it at +// runtime; the round-trip below is enforced by the `typecheck:e2e` CI step (tsc over +// e2e_tests), so a drift in the /config/update or /router/settings contract fails the +// build rather than silently passing here. import type { components } from "../../../src/lib/http/schema"; +// These tests mutate the proxy's shared router_settings, and the Loadbalancing save +// echoes the whole settings object, so they must not run concurrently. +test.describe.configure({ mode: "serial" }); + const PRIMARY = "fake-openai-gpt-4"; const FALLBACK = "fake-anthropic-claude"; @@ -111,32 +116,33 @@ const BASE_URL = "http://localhost:4000"; const ADMIN_AUTH = { Authorization: `Bearer ${users[Role.ProxyAdmin].password}` }; /** - * Merge a router_settings patch into the live config through the typed - * /config/update contract, preserving any other settings already present. + * Apply a router_settings patch through the typed /config/update contract. The + * server merges it over existing settings (request wins), so only the passed keys + * change. Fails loudly if the write is rejected instead of leaving a silent bad seed. */ async function patchRouterSettings( request: import("@playwright/test").APIRequestContext, patch: Partial>, ) { - const current = await request.get(`${BASE_URL}/get/config/callbacks`, { headers: ADMIN_AUTH }); - const existing = current.ok() ? (await current.json())?.router_settings ?? {} : {}; - const payload = { router_settings: { ...(existing as Record), ...patch } }; - await request.post(`${BASE_URL}/config/update`, { headers: ADMIN_AUTH, data: payload }); + const res = await request.post(`${BASE_URL}/config/update`, { + headers: ADMIN_AUTH, + data: { router_settings: patch }, + }); + expect(res.ok(), `seed /config/update failed: ${res.status()} ${await res.text()}`).toBeTruthy(); } test.describe("Router Settings - Loadbalancing", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); - // Seed a present routing_groups array (the LIT-4057 trigger) plus a known - // num_retries so the UI assertions are deterministic across reruns. - const ROUTING_GROUP = { group_name: "e2e-lit-4057", models: [PRIMARY], routing_strategy: "simple-shuffle" }; - + // Pin num_retries and an empty routing_groups so the assertions are deterministic. + // Empty already reproduces LIT-4057: the old tab serialized [] to the string "[]" + // and the save 422'd. test.beforeEach(async ({ request }) => { - await patchRouterSettings(request, { num_retries: 3, routing_groups: [ROUTING_GROUP] }); + await patchRouterSettings(request, { num_retries: 3, routing_groups: [] }); }); test.afterEach(async ({ request }) => { - await patchRouterSettings(request, { num_retries: 3, routing_groups: [] }); + await patchRouterSettings(request, { num_retries: 3 }); }); test("saves the Loadbalancing tab without a 422 when routing_groups is present, and persists", async ({ diff --git a/ui/litellm-dashboard/e2e_tests/tsconfig.json b/ui/litellm-dashboard/e2e_tests/tsconfig.json new file mode 100644 index 00000000000..abda3fdb8b6 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "noEmit": true, + "types": ["node"] + }, + "include": ["**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index a8948f4be34..1d2b505a172 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -19,6 +19,7 @@ "e2e:ui": "playwright test --ui --config e2e_tests/playwright.config.ts", "e2e:migration": "playwright test e2e_tests/tests/migration/migratedPages.spec.ts --config e2e_tests/playwright.config.ts", "e2e:migration:root": "playwright test --config e2e_tests/migration.serverRootPath.config.ts", + "typecheck:e2e": "tsc -p e2e_tests/tsconfig.json --noEmit", "knip": "knip", "knip:fix": "knip --fix", "gen:api": "node scripts/gen-api-types.mjs" From 4f41a9e140ebcf12ec56bfc8d24b4dd5ba78ed4f Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 30 Jun 2026 12:49:46 -0700 Subject: [PATCH 05/81] test(ui): drop the e2e typecheck CI gate, keep the typed import for the editor The e2e runs against the real proxy, so a contract drift already fails the test at runtime; tsc only checks the spec against schema.d.ts, a generated snapshot, so a backend change with a stale snapshot would pass tsc while the live test still catches it. The dedicated tsconfig + script + CI step were circular ceremony for that. Keep the zero-runtime-cost type-only import, which still catches mistakes in the editor, and make its comment honest about what enforces the contract. --- .circleci/config.yml | 8 -------- .../e2e_tests/tests/settings/routerSettings.spec.ts | 7 +++---- ui/litellm-dashboard/e2e_tests/tsconfig.json | 9 --------- ui/litellm-dashboard/package.json | 1 - 4 files changed, 3 insertions(+), 22 deletions(-) delete mode 100644 ui/litellm-dashboard/e2e_tests/tsconfig.json diff --git a/.circleci/config.yml b/.circleci/config.yml index 009884cbfe4..f13e9bf66f1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2687,14 +2687,6 @@ jobs: paths: - ui/litellm-dashboard/node_modules - ~/.cache/ms-playwright - - run: - name: Type-check E2E specs - # The specs type their request/response round-trips against the generated - # OpenAPI schema; this step turns that typing into a real gate, so a backend - # contract drift fails here instead of silently passing at runtime. - command: | - cd ui/litellm-dashboard - npm run typecheck:e2e - run: name: Build UI from source # Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`. diff --git a/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts index 64dab6d7cf6..3e140b9ab56 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts @@ -3,10 +3,9 @@ import { ADMIN_STORAGE_PATH } from "../../constants"; import { navigateToPage } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; import { Role, users } from "../../fixtures/users"; -// Type-only import of the OpenAPI-generated backend schema. esbuild erases it at -// runtime; the round-trip below is enforced by the `typecheck:e2e` CI step (tsc over -// e2e_tests), so a drift in the /config/update or /router/settings contract fails the -// build rather than silently passing here. +// Type-only import of the OpenAPI-generated backend schema, erased at runtime by +// esbuild. It types the round-trips below so mistakes surface in the editor; the live +// test against the real proxy is what actually enforces the contract. import type { components } from "../../../src/lib/http/schema"; // These tests mutate the proxy's shared router_settings, and the Loadbalancing save diff --git a/ui/litellm-dashboard/e2e_tests/tsconfig.json b/ui/litellm-dashboard/e2e_tests/tsconfig.json deleted file mode 100644 index abda3fdb8b6..00000000000 --- a/ui/litellm-dashboard/e2e_tests/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "../tsconfig.json", - "compilerOptions": { - "noEmit": true, - "types": ["node"] - }, - "include": ["**/*.ts"], - "exclude": ["node_modules"] -} diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 1d2b505a172..a8948f4be34 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -19,7 +19,6 @@ "e2e:ui": "playwright test --ui --config e2e_tests/playwright.config.ts", "e2e:migration": "playwright test e2e_tests/tests/migration/migratedPages.spec.ts --config e2e_tests/playwright.config.ts", "e2e:migration:root": "playwright test --config e2e_tests/migration.serverRootPath.config.ts", - "typecheck:e2e": "tsc -p e2e_tests/tsconfig.json --noEmit", "knip": "knip", "knip:fix": "knip --fix", "gen:api": "node scripts/gen-api-types.mjs" From ae8084de747c2e152d2594ddcc97b851da2d98f0 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 30 Jun 2026 15:25:55 -0700 Subject: [PATCH 06/81] ci(codspeed): pin benchmark runner to ubuntu-24.04 (#31746) * ci(codspeed): pin benchmark runner to ubuntu-24.04 ubuntu-latest resolves to different runner images between the BASE (main/staging) and HEAD (PR) runs, so CodSpeed reports 'Different runtime environments detected' and emits false-positive regressions (e.g. a -25.2% swing on test_completion_multi_turn in #31684, an MCP auth fix with no LLM code changes). Pinning the runner to a fixed image keeps BASE and HEAD on the same hardware so 1 ms swings on a ~3 ms benchmark stop blocking unrelated PRs. Fixes #31738 * ci(codspeed): stop running benchmarks on litellm_internal_staging The CodSpeed check flip-flops on internal staging and on PRs targeting it (e.g. "+11.75% improvement" on one run, "-25.36% regression" on the next) because the comparison flags "different runtime environments" and the benchmarks are only 3-4 ms, so sub-millisecond runner noise swings the result by 25-30%. Pinning the runner to ubuntu-24.04 in this PR helps the head side, but the internal_staging base is still recorded on the old unpinned runner, so comparisons keep flapping until the pin merges and the base is re-baselined. Until that settles, the red X's on internal staging make the OSS project look unhealthy and confuse contributors, so drop the litellm_internal_staging push and pull_request triggers and keep CodSpeed running on main only. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .github/workflows/codspeed.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 1fad82827ff..49f1d906069 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -4,11 +4,9 @@ on: push: branches: - main - - litellm_internal_staging pull_request: branches: - main - - litellm_internal_staging # Allow CodSpeed to trigger backtest performance analysis # in order to generate initial data workflow_dispatch: @@ -23,7 +21,7 @@ concurrency: jobs: benchmarks: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 15 steps: From a7d8c6f46760eae1051d92129507b17e50642234 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 30 Jun 2026 15:27:48 -0700 Subject: [PATCH 07/81] test(pass-through): de-flake vertex spend-log test by routing through the proxy (#31689) * test(pass-through): de-flake vertex spend-log assertion by re-billing The vertex pass-through spend-log test asserted that a single billed generateContent call moved the global spend aggregate within a fixed wait. CI failures show the call returning a valid response with real usage, yet spend never increasing over a 240s poll. Pass-through spend logging is best-effort: the success handler is enqueued on a background worker that can drop or time out an individual event under load and never retries it, so one billed call occasionally never reaches LiteLLM_SpendLogs. Waiting longer cannot recover a dropped event; only re-issuing the call can. Re-bill the call up to a few times and require at least one to be tracked, mirroring the sibling jest test that already retries. The test still fails hard if cost tracking is actually broken, since then every call records nothing. Also sum spend across all returned days instead of matching the runner's local 'today', removing a separate UTC-rollover flake. * test(pass-through): route vertex spend-log test through proxy via direct HTTP The vertexai SDK, configured with location="global" and an http api_endpoint override, intermittently sends generateContent to the public Vertex endpoint instead of the proxy. Proxy logs from a failing run show all 46 of the test's own spend-log polls reaching the proxy while zero generateContent calls did, so LiteLLM never saw the billed call and no spend was ever recorded; re-billing through the SDK could not help because every retry bypassed the proxy too. Issue the pass-through request directly over HTTP so it always hits the proxy, minting a Google token from the same service-account credentials, then assert that the specific call's own spend log lands with spend > 0, a gemini model, and custom_llm_provider vertex_ai. A small best-effort retry covers the rare case where the background logging worker drops a single event; failing every attempt still fails hard so the test keeps its teeth if cost tracking breaks. * test(pass-through): reuse LITE_LLM_ENDPOINT and drop needless async in get_tracked_spend --- tests/pass_through_tests/test_vertex_ai.py | 175 +++++++++++---------- 1 file changed, 96 insertions(+), 79 deletions(-) diff --git a/tests/pass_through_tests/test_vertex_ai.py b/tests/pass_through_tests/test_vertex_ai.py index e8223f2219c..35cb5f49c56 100644 --- a/tests/pass_through_tests/test_vertex_ai.py +++ b/tests/pass_through_tests/test_vertex_ai.py @@ -11,6 +11,7 @@ import json import os import pytest import asyncio +import requests # Path to your service account JSON file SERVICE_ACCOUNT_FILE = "path/to/your/service-account.json" @@ -57,98 +58,114 @@ def load_vertex_ai_credentials(): os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(temp_file.name) -async def call_spend_logs_endpoint(): - """ - Call this - curl -X GET "http://0.0.0.0:4000/spend/logs" -H "Authorization: Bearer sk-1234" - """ - import datetime - import requests - - todays_date = datetime.datetime.now().strftime("%Y-%m-%d") - url = f"http://0.0.0.0:4000/global/spend/logs?api_key=best-api-key-ever" - headers = {"Authorization": f"Bearer sk-1234"} - response = requests.get(url, headers=headers) - print("response from call_spend_logs_endpoint", response) - - if response.status_code != 200: - print(f"spend logs endpoint returned {response.status_code}: {response.text}") - return None - - json_response = response.json() - - # get spend for today - """ - json response looks like this - - [{'date': '2024-08-30', 'spend': 0.00016600000000000002, 'api_key': 'best-api-key-ever'}] - """ - print("json_response", json_response) - - todays_date = datetime.datetime.now().strftime("%Y-%m-%d") - for spend_log in json_response: - if spend_log["date"] == todays_date: - return spend_log["spend"] - - LITE_LLM_ENDPOINT = "http://localhost:4000" +SPEND_LOG_API_KEY = "best-api-key-ever" -def _is_vertex_quota_error(exc: Exception) -> bool: - message = str(exc) - return ( - "429" in message - or "Too Many Requests" in message - or "RESOURCE_EXHAUSTED" in message + +def get_tracked_spend() -> float: + """ + Total spend recorded under the pass-through key in the global spend view. + + Sums every day the endpoint returns instead of matching the runner's local + "today" so a UTC date rollover mid-test can't hide a freshly billed call, and + treats an unreachable endpoint as "nothing recorded yet" (0.0). + """ + url = f"{LITE_LLM_ENDPOINT}/global/spend/logs?api_key={SPEND_LOG_API_KEY}" + response = requests.get(url, headers={"Authorization": "Bearer sk-1234"}) + if response.status_code != 200: + print(f"global spend logs endpoint returned {response.status_code}: {response.text}") + return 0.0 + + rows = response.json() + print("global spend logs rows", rows) + return sum(float(row.get("spend") or 0.0) for row in rows) + + +VERTEX_PROJECT = "litellm-ci-cd" +VERTEX_MODEL = "gemini-3.1-flash-lite" +VERTEX_GENERATE_CONTENT_URL = ( + f"{LITE_LLM_ENDPOINT}/vertex_ai/v1/projects/{VERTEX_PROJECT}" + f"/locations/global/publishers/google/models/{VERTEX_MODEL}:generateContent" +) + + +def _vertex_access_token() -> str: + import google.auth + import google.auth.transport.requests + + credentials, _ = google.auth.default( + scopes=["https://www.googleapis.com/auth/cloud-platform"] ) + credentials.refresh(google.auth.transport.requests.Request()) + return credentials.token + + +def _spend_log_for_request(call_id: str) -> dict | None: + response = requests.get( + f"{LITE_LLM_ENDPOINT}/spend/logs?request_id={call_id}", + headers={"Authorization": "Bearer sk-1234"}, + timeout=30, + ) + if response.status_code != 200: + return None + rows = response.json() + return rows[0] if rows else None + + +def _is_vertex_quota_error(response: requests.Response) -> bool: + return response.status_code == 429 or "RESOURCE_EXHAUSTED" in response.text @pytest.mark.asyncio() async def test_basic_vertex_ai_pass_through_with_spendlog(): - - spend_before = await call_spend_logs_endpoint() or 0.0 load_vertex_ai_credentials() + access_token = _vertex_access_token() - vertexai.init( - project="litellm-ci-cd", - location="global", - api_endpoint=f"{LITE_LLM_ENDPOINT}/vertex_ai", - api_transport="rest", - ) + # Drive the pass-through over HTTP instead of the vertexai SDK: the SDK intermittently + # routes generateContent to the public Vertex endpoint rather than the proxy override, + # so the call never reaches LiteLLM and no spend is logged. A direct request always + # hits the proxy. Spend logging then runs on a best-effort background worker that can + # drop a single event, so retry a few billed calls and assert that one specific call's + # spend log lands. Failing every attempt still fails hard, which is the signal we want + # if cost tracking is broken. + max_attempts = 3 + poll_seconds = 60 + poll_interval = 5 - model = GenerativeModel(model_name="gemini-3.1-flash-lite") - try: - response = model.generate_content("hi") - except Exception as exc: - if _is_vertex_quota_error(exc): + for attempt in range(1, max_attempts + 1): + response = requests.post( + VERTEX_GENERATE_CONTENT_URL, + headers={ + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json", + }, + json={"contents": [{"role": "user", "parts": [{"text": "hi"}]}]}, + timeout=60, + ) + if _is_vertex_quota_error(response): pytest.skip("Vertex AI quota exhausted") - raise + assert ( + response.status_code == 200 + ), f"vertex pass-through call failed: {response.status_code} {response.text}" - print("response", response) + call_id = response.headers.get("x-litellm-call-id") + assert call_id, "proxy response missing x-litellm-call-id header" - # Spend logging is async/batched and can lag under CI load, so poll instead of - # sleeping a fixed amount. A transient empty read is skipped, not counted as 0.0 - # spend, which would spuriously fail the assertion on an otherwise-billed call. - max_wait = 240 # total seconds to wait - poll_interval = 10 # seconds between checks - elapsed = 0 - spend_after = spend_before - while elapsed < max_wait: - await asyncio.sleep(poll_interval) - elapsed += poll_interval - latest_spend = await call_spend_logs_endpoint() - if latest_spend is None: - print(f"spend logs unavailable (elapsed={elapsed}s), retrying") - continue - spend_after = latest_spend - print(f"spend_after (elapsed={elapsed}s)", spend_after) - if spend_after > spend_before: - break + for _ in range(poll_seconds // poll_interval): + await asyncio.sleep(poll_interval) + row = _spend_log_for_request(call_id) + if row is not None and float(row.get("spend") or 0) > 0: + assert "gemini" in row["model"], f"unexpected model in spend log: {row}" + assert ( + row["custom_llm_provider"] == "vertex_ai" + ), f"unexpected provider in spend log: {row}" + return - assert ( - spend_after > spend_before - ), "Spend should be greater than before after {}s. spend_before: {}, spend_after: {}".format( - elapsed, spend_before, spend_after + print(f"attempt {attempt}: spend log for call {call_id} not found yet, re-billing") + + pytest.fail( + f"Vertex pass-through spend never recorded after {max_attempts} billed calls" ) @@ -156,7 +173,7 @@ async def test_basic_vertex_ai_pass_through_with_spendlog(): @pytest.mark.skip(reason="skip flaky test - vertex pass through streaming is flaky") async def test_basic_vertex_ai_pass_through_streaming_with_spendlog(): - spend_before = await call_spend_logs_endpoint() or 0.0 + spend_before = get_tracked_spend() print("spend_before", spend_before) load_vertex_ai_credentials() @@ -176,7 +193,7 @@ async def test_basic_vertex_ai_pass_through_streaming_with_spendlog(): print("response", response) await asyncio.sleep(20) - spend_after = await call_spend_logs_endpoint() + spend_after = get_tracked_spend() print("spend_after", spend_after) assert ( spend_after > spend_before From 41f9d8de7b16516808bbad3b5be5da9dd736e698 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Tue, 30 Jun 2026 15:30:08 -0700 Subject: [PATCH 08/81] fix(proxy): extend banned-params + admin-clear lists for NVIDIA Riva (VERIA-493) (#31742) Two NVIDIA-Riva-specific fields consumed by the audio-transcription handler via the provider's `optional_params` passthrough were not covered by the proxy's existing banned-request-body list or the admin-config clearing list applied on `api_base` BYOK override: * `nvcf_function_id` * `use_ssl` Add both to `_BANNED_REQUEST_BODY_PARAMS` in `litellm/proxy/auth/auth_utils.py` and to the kwargs-only list in `_admin_config_fields_to_clear_on_base_override()` in `litellm/router_utils/clientside_credential_handler.py`, next to the analogous provider-specific entries already there (`aws_bedrock_*`, OCI provider fields, etc.). Same admin opt-ins as every other entry on those lists (`general_settings.allow_client_side_credentials` proxy-wide, or `configurable_clientside_auth_params` per deployment). Regression tests in `tests/test_litellm/proxy/auth/test_auth_utils.py` cover root-level rejection, the historical `api_key` bypass, both admin opt-in paths (proxy-wide and per-deployment), nested-container smuggling via the existing recursive walk, and clearing on `api_base` override. Mutation check verified. Resolves VERIA-493 --- litellm/proxy/auth/auth_utils.py | 6 + .../clientside_credential_handler.py | 7 + .../proxy/auth/test_auth_utils.py | 173 ++++++++++++++++++ 3 files changed, 186 insertions(+) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index b1bce352784..2bf0acc7232 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -278,6 +278,12 @@ _BANNED_REQUEST_BODY_PARAMS: Tuple[str, ...] = ( "s3_endpoint_url", "sagemaker_base_url", "deployment_url", + # NVIDIA Riva fields consumed by the audio-transcription handler + # via ``optional_params``. Banned for the same reason as the + # provider-specific entries above: a caller-supplied value retargets + # the request away from the admin's pinned configuration. + "nvcf_function_id", + "use_ssl", # SDK-only field; also rejected outright in is_request_body_safe. "model_list", # Observability credentials, hosts, and project identifiers: derived diff --git a/litellm/router_utils/clientside_credential_handler.py b/litellm/router_utils/clientside_credential_handler.py index e992ef63658..8234d89e248 100644 --- a/litellm/router_utils/clientside_credential_handler.py +++ b/litellm/router_utils/clientside_credential_handler.py @@ -52,6 +52,13 @@ def _admin_config_fields_to_clear_on_base_override() -> List[str]: "oci_tenancy", "oci_key", "oci_key_file", + # NVIDIA Riva fields — consumed by + # ``litellm/llms/nvidia_riva/audio_transcription/handler.py`` via + # optional_params and not declared on CredentialLiteLLMParams. + # Admin-pinned values must not flow through on a caller-redirected + # ``api_base`` for the same reason as the OCI entries above. + "nvcf_function_id", + "use_ssl", ] return typed_fields + kwargs_only_fields diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index cd8cf10d037..d5d2d27cb7e 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -1520,6 +1520,42 @@ class TestGetDynamicLitellmParamsClearsAdminConfigOnBaseOverride: assert "vertex_credentials" not in out assert "vertex_project" not in out + def test_clears_nvcf_function_id_on_base_override(self): + from litellm.router_utils.clientside_credential_handler import ( + get_dynamic_litellm_params, + ) + + admin_params = { + "model": "nvidia_riva/parakeet", + "api_base": "grpc.nvcf.nvidia.com:443", + "api_key": "nvapi-admin", + "nvcf_function_id": "admin-pinned-function", + } + out = get_dynamic_litellm_params( + litellm_params=dict(admin_params), + request_kwargs={"api_base": "self-hosted.example.com:50051"}, + ) + assert out["api_base"] == "self-hosted.example.com:50051" + assert "nvcf_function_id" not in out + + def test_clears_use_ssl_on_base_override(self): + from litellm.router_utils.clientside_credential_handler import ( + get_dynamic_litellm_params, + ) + + admin_params = { + "model": "nvidia_riva/parakeet", + "api_base": "grpc.nvcf.nvidia.com:443", + "api_key": "nvapi-admin", + "use_ssl": True, + } + out = get_dynamic_litellm_params( + litellm_params=dict(admin_params), + request_kwargs={"api_base": "self-hosted.example.com:50051"}, + ) + assert out["api_base"] == "self-hosted.example.com:50051" + assert "use_ssl" not in out + def test_caller_resupplied_value_overrides_admin_value_on_base_override(self): # When the caller redirects ``api_base`` and *also* supplies their # own value for one of the admin fields (e.g. ``organization``), @@ -1712,6 +1748,127 @@ class TestIsRequestBodySafeBlocksBedrockProjectOverride: ) +class TestIsRequestBodySafeBlocksNVCFFunctionOverride: + """``nvcf_function_id`` is rejected as a request-body param unless the + admin opted in proxy-wide or per-deployment.""" + + def test_nvcf_function_id_in_request_body_is_rejected(self): + with pytest.raises(ValueError, match="nvcf_function_id"): + is_request_body_safe( + request_body={ + "model": "nvidia_riva/parakeet", + "nvcf_function_id": "caller-supplied", + }, + general_settings={}, + llm_router=None, + model="nvidia_riva/parakeet", + ) + + def test_nvcf_function_id_with_api_key_still_rejected(self): + with pytest.raises(ValueError, match="nvcf_function_id"): + is_request_body_safe( + request_body={ + "model": "nvidia_riva/parakeet", + "api_key": "sk-anything", + "nvcf_function_id": "caller-supplied", + }, + general_settings={}, + llm_router=None, + model="nvidia_riva/parakeet", + ) + + def test_admin_opt_in_proxy_wide_allows_nvcf_function_id(self): + assert ( + is_request_body_safe( + request_body={ + "model": "nvidia_riva/parakeet", + "nvcf_function_id": "byok-function-id", + }, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="nvidia_riva/parakeet", + ) + is True + ) + + def test_admin_opt_in_per_deployment_allows_nvcf_function_id(self, monkeypatch): + """The error message lists per-deployment ``configurable_clientside_auth_params`` + as a second opt-in. Cover that path too so it can't silently regress.""" + from litellm.proxy.auth import auth_utils + + monkeypatch.setattr( + auth_utils, + "_allow_model_level_clientside_configurable_parameters", + lambda model, param, request_body_value, llm_router: param == "nvcf_function_id", + ) + + assert ( + is_request_body_safe( + request_body={ + "model": "nvidia_riva/parakeet", + "nvcf_function_id": "byok-function-id", + }, + general_settings={}, + llm_router=None, + model="nvidia_riva/parakeet", + ) + is True + ) + + +class TestIsRequestBodySafeBlocksRivaUseSsl: + """``use_ssl`` is rejected as a request-body param unless the admin + opted in proxy-wide or per-deployment.""" + + def test_use_ssl_in_request_body_is_rejected(self): + with pytest.raises(ValueError, match="use_ssl"): + is_request_body_safe( + request_body={ + "model": "nvidia_riva/parakeet", + "use_ssl": False, + }, + general_settings={}, + llm_router=None, + model="nvidia_riva/parakeet", + ) + + def test_admin_opt_in_proxy_wide_allows_use_ssl(self): + assert ( + is_request_body_safe( + request_body={ + "model": "nvidia_riva/parakeet", + "use_ssl": True, + }, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="nvidia_riva/parakeet", + ) + is True + ) + + def test_admin_opt_in_per_deployment_allows_use_ssl(self, monkeypatch): + from litellm.proxy.auth import auth_utils + + monkeypatch.setattr( + auth_utils, + "_allow_model_level_clientside_configurable_parameters", + lambda model, param, request_body_value, llm_router: param == "use_ssl", + ) + + assert ( + is_request_body_safe( + request_body={ + "model": "nvidia_riva/parakeet", + "use_ssl": True, + }, + general_settings={}, + llm_router=None, + model="nvidia_riva/parakeet", + ) + is True + ) + + # ── is_request_body_safe nested-config recursion (VERIA-6) ──────────────────── @@ -1748,6 +1905,22 @@ class TestIsRequestBodySafeNestedConfig: model="milvus-store", ) + def test_nested_nvcf_function_id_in_metadata_blocked(self): + """Smuggling ``nvcf_function_id`` via ``metadata`` / ``extra_body`` + is the same shape as the VERIA-6 ``api_base`` bypass — must be + rejected by the recursive walk so the NVCF override gate cannot + be sidestepped with nesting.""" + with pytest.raises(ValueError, match="nvcf_function_id"): + is_request_body_safe( + request_body={ + "model": "nvidia_riva/parakeet", + "litellm_metadata": {"nvcf_function_id": "attacker-via-metadata"}, + }, + general_settings={}, + llm_router=None, + model="nvidia_riva/parakeet", + ) + def test_nested_langfuse_host_in_embedding_config_blocked(self): """The recursion uses the *full* banned-param list, not a special subset — so any flag that's banned at the root is also banned From 833406a711111e8e1347eead3aac5a831de39ea8 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 30 Jun 2026 17:20:24 -0700 Subject: [PATCH 09/81] fix(ui): rotate model credentials in a dedicated modal so a normal save can't overwrite secrets (#28089) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(ui): add provider auth editing to the model edit view Provider API keys / auth could previously only be changed by hand-editing the raw litellm_params JSON, so there was no first-class way to rotate a model's key. Adds an Authentication section that renders the correct provider-specific fields (reusing ProviderSpecificFields) keyed off the model's custom_llm_provider; fields are blank ("leave blank to keep current") so untouched secrets are preserved and only entered values are PATCHed and encrypted at rest. Resolves LIT-3169 * refactor(ui): simplify model auth editing; fix stale credential branch Drop the onFieldsResolved/authFieldKeys round trip: the parent now resolves provider auth field keys itself via the new useProviderAuthFieldKeys hook (same metadata ProviderSpecificFields renders), removing the report-up effect and its stable-reference footgun. ProviderSpecificFields keeps only excludeKeys (real need: suppress duplicate visible inputs). Fix the stale Authentication branch: derive it from the live litellm_credential_name form value (Form.useWatch) instead of the server snapshot, so clearing/adding a credential mid-edit shows the right UI. Also skip inline auth updates entirely when a named credential is selected, so we never submit a credential name and raw inline auth together. * fix(ui): don't leak freshly-entered model auth secrets to display/console The auth values a user types are still sent in the PATCH request, but: - strip them from the locally-stored litellm_params after save so the read-only LiteLLM Params JSON doesn't render the plaintext key - remove the debug console.log in modelPatchUpdateCall that dumped the full update payload (incl. api_key / vertex_credentials) to the browser console on every model update Backend stores these encrypted and returns them masked on refetch. * fix(ui): don't require blank auth fields in model edit context Auth fields render blank ('leave blank to keep'), but required metadata (e.g. OpenAI api_key) added a required validation rule that blocked onFinish entirely — making it impossible to save any unrelated edit without re-entering the secret. Add a disableRequired prop to ProviderSpecificFields and set it in the model edit Authentication section. * fix(ui): rotate model credentials in a dedicated modal so a normal save can't overwrite secrets The model edit form seeded the read-only LiteLLM Params textarea with the whole litellm_params blob and re-sent all of it on every save. Because /model/info redacts secrets by masking them ("azur****BBCC") rather than removing them, any save re-encrypted the asterisk mask over the real value and silently destroyed credentials such as azure_ad_token, aws_session_token, watsonx token/zen_api_key and the OCI key fields. api_key, client_secret, vertex_credentials and the AWS access/secret keys were safe only because the backend strips those entirely Credential rotation now lives in a dedicated UpdateModelCredentialsModal that PATCHes only the fields the user types, decoupled from the params blob; the backend already merges partial litellm_params, so the rest of the deployment is left untouched. The general edit form drops masked values from both the textarea seed and the outbound payload, so a normal save can never carry a redacted secret Also removes the now-unused inline auth section and its excludeKeys and useProviderAuthFieldKeys plumbing, strips secret-leaking console.logs from the provider upload handler and the model-update response, and fixes a react-hooks/use-memo error that was failing the frontend-lint CI job * chore(ui): ratchet no-explicit-any lint metric to 2013 Removing the credential-echoing console.log (and its info: any param) from the provider upload handler dropped the tracked count by one; update the committed baseline so the Check lint budgets CI step is not stale * refactor(ui): scope the model credential modal to api-key rotation only Narrows UpdateModelCredentialsModal to a single API Key field. On submit it PATCHes only { api_key }, so the backend merge leaves every other deployment param untouched; a model authed via azure_ad_token, AWS keys, or a Vertex JSON won't have anything to rotate here yet, which is the intended scope for now. Drops the multi-field provider rendering this added earlier, which also removes the now-unused disableRequired prop from ProviderSpecificFields and reverts that shared component to its prior shape. The "Update API Key" trigger button is now an antd Button rather than a TremorButton, so the feature introduces no tremor. * refactor(ui): convert the model detail toolbar buttons from tremor to antd Switches Test Connection, Re-use Credentials and Delete Model to antd Button so the toolbar matches the Update API Key button and no longer mixes libraries; Delete Model uses antd's danger styling instead of hand-rolled red classes * style(ui): make the api-key modal submit button primary and drop the Need Help link --- ui/litellm-dashboard/eslint-metrics.json | 2 +- .../add_model/provider_specific_fields.tsx | 17 ---- .../src/components/model_info_view.test.tsx | 45 ++++++++++ .../src/components/model_info_view.tsx | 74 +++++++++++++---- .../src/components/networking.tsx | 5 +- .../update_model_credentials_modal.test.tsx | 83 +++++++++++++++++++ .../update_model_credentials_modal.tsx | 76 +++++++++++++++++ 7 files changed, 265 insertions(+), 37 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/update_model_credentials_modal.test.tsx create mode 100644 ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx diff --git a/ui/litellm-dashboard/eslint-metrics.json b/ui/litellm-dashboard/eslint-metrics.json index 09ad247391b..92c5a991eb6 100644 --- a/ui/litellm-dashboard/eslint-metrics.json +++ b/ui/litellm-dashboard/eslint-metrics.json @@ -1,5 +1,5 @@ { - "@typescript-eslint/no-explicit-any": 2014, + "@typescript-eslint/no-explicit-any": 2013, "complexity": 126, "max-depth": 61 } diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx index 045a9b0c1b6..205292edfb4 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx +++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx @@ -212,9 +212,7 @@ const ProviderSpecificFields: React.FC = ({ selecte reader.onload = (e) => { if (e.target) { const jsonStr = e.target.result as string; - console.log(`Setting field value from JSON, length: ${jsonStr.length}`); form.setFieldsValue({ vertex_credentials: jsonStr }); - console.log("Form values after setting:", form.getFieldsValue()); } }; reader.readAsText(file); @@ -222,14 +220,6 @@ const ProviderSpecificFields: React.FC = ({ selecte // Prevent upload return false; }, - onChange(info: any) { - console.log("Upload onChange triggered in ProviderSpecificFields"); - console.log("Current form values:", form.getFieldsValue()); - - if (info.file.status !== "uploading") { - console.log(info.file, info.fileList); - } - }, }; return ( @@ -271,16 +261,9 @@ const ProviderSpecificFields: React.FC = ({ selecte { - // First call the original onChange if (uploadProps?.onChange) { uploadProps.onChange(info); } - - // Check the field value after a short delay - setTimeout(() => { - const value = form.getFieldValue(field.key); - console.log(`${field.key} value after upload:`, JSON.stringify(value)); - }, 500); }} > }>Click to Upload diff --git a/ui/litellm-dashboard/src/components/model_info_view.test.tsx b/ui/litellm-dashboard/src/components/model_info_view.test.tsx index d91d5ec307e..2546601b4db 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.test.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.test.tsx @@ -633,6 +633,51 @@ describe("ModelInfoView", () => { expect(updatePayload.litellm_params).not.toHaveProperty("output_cost_per_token"); }); + it("never re-sends a masked secret on save (regression: masked auth value must not overwrite the real secret)", async () => { + // /model/info redacts secrets by masking (e.g. "azur****BBCC"), not removing them. + // A plain save re-PATCHes the whole litellm_params blob; if the masked value were + // sent, the backend would encrypt the asterisks over the real azure_ad_token and + // silently destroy the credential. The edit form must strip masked values entirely. + const maskedSecret = "azur********************************************BBCC"; + const maskedModelData = { + ...defaultModelData, + litellm_params: { + model: "azure/gpt-4o", + api_base: "https://example-az.openai.azure.com", + custom_llm_provider: "azure", + azure_ad_token: maskedSecret, + }, + }; + mockUseModelsInfo.mockReturnValue({ + data: { data: [maskedModelData] }, + isLoading: false, + error: null, + }); + mockModelInfoV1Call.mockResolvedValue({ data: [maskedModelData] }); + + const user = userEvent.setup(); + render(, { wrapper }); + + await waitFor(() => { + expect(screen.getByRole("button", { name: /edit settings/i })).toBeInTheDocument(); + }); + await user.click(screen.getByRole("button", { name: /edit settings/i })); + + await waitFor(() => { + expect(screen.getByRole("button", { name: /save changes/i })).toBeInTheDocument(); + }); + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => { + expect(mockModelPatchUpdateCall).toHaveBeenCalled(); + }); + + const updatePayload = mockModelPatchUpdateCall.mock.calls[0][1]; + expect(updatePayload.litellm_params.azure_ad_token).not.toBe(maskedSecret); + // No masked value may appear anywhere in the outbound params. + expect(JSON.stringify(updatePayload.litellm_params)).not.toContain("**"); + }); + it("should display health check model field for wildcard models", async () => { const wildcardModelData = { ...defaultModelData, diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index 66a00b9bbe3..45c5b0fd9b6 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -1,5 +1,6 @@ import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; import { useModelHub, useModelsInfo } from "@/app/(dashboard)/hooks/models/useModels"; +import { useQueryClient } from "@tanstack/react-query"; import { transformModelData } from "@/app/(dashboard)/models-and-endpoints/utils/modelDataTransformer"; import { InfoCircleOutlined } from "@ant-design/icons"; import { ArrowLeftIcon, KeyIcon, RefreshIcon, TrashIcon } from "@heroicons/react/outline"; @@ -40,6 +41,7 @@ import { testConnectionRequest, } from "./networking"; import { getProviderLogoAndName } from "./provider_info_helpers"; +import UpdateModelCredentialsModal from "./update_model_credentials_modal"; import NumericalInput from "./shared/numerical_input"; import { Tag } from "./tag_management/types"; import { getDisplayModelName } from "./view_model/model_name_display"; @@ -54,6 +56,18 @@ interface ModelInfoViewProps { modelAccessGroups: string[] | null; } +// The /model/info response redacts secrets by masking them (e.g. "sk-1****2345"), +// not by removing them. The edit form must never echo a masked value back on save: +// the backend would encrypt the asterisks and overwrite the real secret. A run of +// 2+ mask chars only appears in masker output (real config — incl. wildcard model +// names like "openai/*" — carries at most a single "*"), so this reliably detects a +// redacted value without a provider-metadata lookup. API-key rotation goes through +// UpdateModelCredentialsModal instead, which sends only the new key. +const isMaskedSecret = (value: unknown): boolean => typeof value === "string" && /\*{2,}/.test(value); + +const stripMaskedSecrets = (params: Record): Record => + Object.fromEntries(Object.entries(params).filter(([, value]) => !isMaskedSecret(value))); + export default function ModelInfoView({ modelId, onClose, @@ -64,10 +78,12 @@ export default function ModelInfoView({ modelAccessGroups, }: ModelInfoViewProps) { const [form] = Form.useForm(); + const queryClient = useQueryClient(); const [localModelData, setLocalModelData] = useState(null); const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); const [deleteLoading, setDeleteLoading] = useState(false); const [isCredentialModalOpen, setIsCredentialModalOpen] = useState(false); + const [isUpdateCredentialsModalOpen, setIsUpdateCredentialsModalOpen] = useState(false); const [isDirty, setIsDirty] = useState(false); const [isSaving, setIsSaving] = useState(false); const [isEditing, setIsEditing] = useState(false); @@ -351,9 +367,15 @@ export default function ModelInfoView({ return; } + // Final guard: never PATCH a redacted secret. The /model/info snapshot that + // seeds this form masks secrets, and any save re-sends the whole params blob; + // without this strip a masked value would be re-encrypted over the real secret. + // Credential rotation has its own dedicated path (UpdateModelCredentialsModal). + const safeLitellmParams = stripMaskedSecrets(updatedLitellmParams); + const updateData = { model_name: values.model_name, - litellm_params: updatedLitellmParams, + litellm_params: safeLitellmParams, model_info: updatedModelInfo, }; @@ -363,7 +385,7 @@ export default function ModelInfoView({ ...localModelData, model_name: values.model_name, litellm_model_name: values.litellm_model_name, - litellm_params: updatedLitellmParams, + litellm_params: safeLitellmParams, model_info: updatedModelInfo, }; @@ -511,36 +533,44 @@ export default function ModelInfoView({
- } onClick={handleTestConnection} className="flex items-center gap-2" data-testid="test-connection-button" > Test Connection - + - } + onClick={() => setIsUpdateCredentialsModalOpen(true)} + className="flex items-center" + disabled={!canEditModel} + data-testid="update-api-key-button" + > + Update API Key + + +
@@ -715,7 +745,7 @@ export default function ModelInfoView({ litellm_extra_params: JSON.stringify( Object.fromEntries( Object.entries(localModelData.litellm_params || {}).filter( - ([key]) => key !== "litellm_credential_name", + ([key, value]) => key !== "litellm_credential_name" && !isMaskedSecret(value), ), ), null, @@ -1375,6 +1405,18 @@ export default function ModelInfoView({ )} + {isUpdateCredentialsModalOpen && accessToken && ( + setIsUpdateCredentialsModalOpen(false)} + accessToken={accessToken} + modelId={modelId} + onUpdated={() => { + queryClient.invalidateQueries({ queryKey: ["models", "list"] }); + }} + /> + )} + {/* Edit Auto Router Modal */} { try { - console.log("Form Values in modelUpateCall:", formValues); // Log the form values before making the API call - + // Intentionally not logging the payload: it can contain freshly-entered + // provider secrets (api_key, vertex_credentials, AWS creds). const url = proxyBaseUrl ? `${proxyBaseUrl}/model/${modelId}/update` : `/model/${modelId}/update`; const response = await fetch(url, { method: "PATCH", @@ -2802,7 +2802,6 @@ export const modelPatchUpdateCall = async ( throw new Error("Network response was not ok"); } const data = await response.json(); - console.log("Update model Response:", data); return data; // Handle success - you might want to update some state or UI based on the created key } catch (error) { diff --git a/ui/litellm-dashboard/src/components/update_model_credentials_modal.test.tsx b/ui/litellm-dashboard/src/components/update_model_credentials_modal.test.tsx new file mode 100644 index 00000000000..ab18ae71203 --- /dev/null +++ b/ui/litellm-dashboard/src/components/update_model_credentials_modal.test.tsx @@ -0,0 +1,83 @@ +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import UpdateModelCredentialsModal from "./update_model_credentials_modal"; +import * as networking from "./networking"; + +vi.mock("./networking", async () => { + const actual = await vi.importActual("./networking"); + return { + ...actual, + modelPatchUpdateCall: vi.fn().mockResolvedValue({}), + }; +}); + +vi.mock("./molecules/notifications_manager", () => ({ + default: { success: vi.fn(), error: vi.fn(), info: vi.fn(), fromBackend: vi.fn() }, +})); + +const mockModelPatchUpdateCall = vi.mocked(networking.modelPatchUpdateCall); + +beforeAll(() => { + Object.defineProperty(window, "matchMedia", { + writable: true, + value: (query: string) => ({ + matches: false, + media: query, + onchange: null, + addListener: () => {}, + removeListener: () => {}, + addEventListener: () => {}, + removeEventListener: () => {}, + dispatchEvent: () => false, + }), + }); +}); + +const renderModal = (overrides: Partial[0]> = {}) => + render( + , + ); + +describe("UpdateModelCredentialsModal", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("sends a minimal PATCH with only the new api_key", async () => { + const user = userEvent.setup(); + const onUpdated = vi.fn(); + const onCancel = vi.fn(); + renderModal({ onUpdated, onCancel }); + + await user.type(screen.getByLabelText(/new api key/i), "sk-rotated-9988"); + await user.click(screen.getByRole("button", { name: /update api key/i })); + + await waitFor(() => expect(mockModelPatchUpdateCall).toHaveBeenCalledTimes(1)); + const [token, payload, modelId] = mockModelPatchUpdateCall.mock.calls[0]; + expect(token).toBe("test-token"); + expect(modelId).toBe("model-123"); + // Exactly the new key plus the id — nothing else from the deployment. + expect(payload).toEqual({ litellm_params: { api_key: "sk-rotated-9988" }, model_info: { id: "model-123" } }); + expect(onUpdated).toHaveBeenCalledTimes(1); + expect(onCancel).toHaveBeenCalledTimes(1); + }); + + it("does not call the update API when the field is left blank", async () => { + const user = userEvent.setup(); + renderModal(); + + await user.click(screen.getByRole("button", { name: /update api key/i })); + + // Required-field validation blocks submit; give it a tick then assert no call. + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(mockModelPatchUpdateCall).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx b/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx new file mode 100644 index 00000000000..238207a4aa8 --- /dev/null +++ b/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx @@ -0,0 +1,76 @@ +import { Button, Form, Input, Modal, Typography } from "antd"; +import { useState } from "react"; +import { modelPatchUpdateCall } from "./networking"; +import NotificationsManager from "./molecules/notifications_manager"; + +const { Text } = Typography; + +interface UpdateModelCredentialsModalProps { + open: boolean; + onCancel: () => void; + accessToken: string; + modelId: string; + onUpdated: () => void; +} + +export default function UpdateModelCredentialsModal({ + open, + onCancel, + accessToken, + modelId, + onUpdated, +}: UpdateModelCredentialsModalProps) { + const [form] = Form.useForm(); + const [isSaving, setIsSaving] = useState(false); + + const close = () => { + form.resetFields(); + onCancel(); + }; + + const handleSubmit = async (values: { api_key?: string }) => { + const apiKey = values.api_key?.trim(); + if (!apiKey) { + NotificationsManager.fromBackend("Enter a new API key"); + return; + } + setIsSaving(true); + try { + await modelPatchUpdateCall( + accessToken, + { litellm_params: { api_key: apiKey }, model_info: { id: modelId } }, + modelId, + ); + NotificationsManager.success("API key updated"); + form.resetFields(); + onUpdated(); + onCancel(); + } catch (error) { + console.error("Error updating API key:", error); + NotificationsManager.fromBackend("Failed to update API key"); + } finally { + setIsSaving(false); + } + }; + + return ( + + + Rotate this model's API key. Only the new key is sent; the rest of the deployment is left untouched. + +
+ + + +
+ + +
+
+
+ ); +} From 2860dad5145772070c6607883989454dbb2943d4 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Tue, 30 Jun 2026 18:17:56 -0700 Subject: [PATCH 10/81] feat(proxy): audit default user settings updates (#31753) * feat(proxy): audit default user settings updates Adds audit logging for the customer-impacting path: PATCH /update/internal_user_settings, which is what the admin dashboard hits when an admin changes Default User Settings and which today leaves no record of who changed what. Introduces the small framework that future system-wide settings audits will share: a CONFIG_TABLE_NAME enum value, a create_config_audit_log helper that reuses the existing create_object_audit_log path (so the enterprise gate and store_audit_logs flag still apply), and a _dump_redacted_config helper that strips secret leaves before the row is written using the same matcher /config/field/info applies for non-admins. The helper handles environment_variables as a special case where every value is redacted, since that section carries credentials under non-secret-looking uppercase keys (e.g. DATABASE_URL). Only update_internal_user_settings is wired up in this change. Coverage for the other LiteLLM_Config writers (/config/update sections, /config/field/update, /config/field/delete, /config/callback/delete, default_team_settings, mcp_semantic_filter, allowed_ip, sso_settings, ui_theme, ui_settings) is intentionally a follow-up so each can be verified live against the credential-bearing fields it actually carries. The audit-actor parameter on _update_litellm_setting is optional today so non-audited callers keep working unchanged; the follow-up will make it required once every caller is wired up. * fix(proxy): make audit-log call non-blocking and serializer defensive Greptile review of #31753 surfaced three robustness issues with the audit-log call path. The settings change always commits; these fixes prevent post-commit audit failures from surfacing as 500 responses. Switch the audit-log call in _update_litellm_setting from a blocking await to asyncio.create_task, matching the create_object_audit_log pattern every other call site uses (model_management_endpoints etc.). A transient prisma blip or a JSON serialization error in the audit row no longer turns a successful save_config into a 500 the caller sees. Add default=str to both json.dumps calls in _dump_redacted_config so a YAML-loaded value with a non-JSON-native leaf (datetime, custom object) serializes cleanly. The sibling audit-log serializers in team_endpoints.py already pass default=str for the same reason. Tighten the redact_all_values branch to redact wholesale for non-dict inputs rather than silently falling through to the key-name matcher; defensive against a future change that stores a section as a list or scalar. Each fix has a regression test mutation-checked against reverting the fix. * refactor(proxy): drop unreachable non-dict redact_all_values branch The defensive non-dict fallback in _dump_redacted_config emitted json.dumps("REDACTED") which, if ever hit, would crash LiteLLM_AuditLogs construction (mask_api_keys validator calls json.loads on the already- parsed bare string). Reachability is zero: redact_all_values is True only for param_name=="environment_variables", which is always a dict. Delete the dead branch and its test rather than ship provably-wrong defensive code with a test that green-lights it. --- litellm/proxy/_types.py | 1 + litellm/proxy/proxy_server.py | 46 +++++++- .../proxy_setting_endpoints.py | 28 ++++- tests/test_litellm/proxy/test_proxy_server.py | 106 ++++++++++++++++++ .../test_proxy_setting_endpoints.py | 91 +++++++++++++++ 5 files changed, 270 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5fe17d79ab5..a6ef7de07ae 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -189,6 +189,7 @@ class LitellmTableNames(str, enum.Enum): TOOL_TABLE_NAME = "LiteLLM_ToolTable" CACHE_CONFIG_TABLE_NAME = "LiteLLM_CacheConfig" CONFIG_OVERRIDES_TABLE_NAME = "LiteLLM_ConfigOverrides" + CONFIG_TABLE_NAME = "LiteLLM_Config" class Litellm_EntityType(enum.Enum): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 57814de6e3f..0158f601d32 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -425,7 +425,10 @@ from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import ( from litellm.proxy.management_endpoints.workflow_management_endpoints import ( router as workflow_management_router, ) -from litellm.proxy.management_helpers.audit_logs import create_audit_log_for_update +from litellm.proxy.management_helpers.audit_logs import ( + create_audit_log_for_update, + create_object_audit_log, +) from litellm.proxy.memory.memory_endpoints import router as memory_router from litellm.proxy.plugin_routes import ( router as plugin_router, @@ -14228,6 +14231,47 @@ def _redact_general_setting_value(field_name: str, value: JsonValue, is_full_adm return value +def _dump_redacted_config(value: Optional[JsonValue], *, redact_all_values: bool = False) -> Optional[str]: + # `default=str` matches the sibling audit-log serializers in + # team_endpoints.py and the LiteLLM_AuditLogs validator, so a YAML-loaded + # value with a non-JSON-native leaf (datetime, custom object) cannot turn + # an audit write into a 500. + if value is None: + return None + if redact_all_values and isinstance(value, dict): + return json.dumps({key: "REDACTED" for key in value}, default=str) + return json.dumps(_redact_secret_values_in_obj(value), default=str) + + +async def create_config_audit_log( + param_name: str, + action: AUDIT_ACTIONS, + before_value: Optional[JsonValue], + after_value: Optional[JsonValue], + user_api_key_dict: UserAPIKeyAuth, + table_name: LitellmTableNames = LitellmTableNames.CONFIG_TABLE_NAME, +) -> None: + """Record a system-wide settings change in LiteLLM_AuditLog. + + Secret leaves are redacted before the row is written. environment_variables + hold arbitrary credentials under non-secret-looking uppercase keys (e.g. + DATABASE_URL), so every value in that section is redacted rather than + relying on key-name matching; other sections reuse the same matcher + /config/field/info applies for non-admins. + """ + redact_all_values = param_name == "environment_variables" + await create_object_audit_log( + object_id=param_name, + action=action, + table_name=table_name, + before_value=_dump_redacted_config(before_value, redact_all_values=redact_all_values), + after_value=_dump_redacted_config(after_value, redact_all_values=redact_all_values), + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + litellm_proxy_admin_name=LITELLM_PROXY_ADMIN_NAME, + ) + + @router.get( "/config/field/info", tags=["config.yaml"], diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index e4f68a1e9db..1be17c86123 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -1,4 +1,5 @@ #### CRUD ENDPOINTS for UI Settings ##### +import asyncio import json from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union from urllib.parse import urlparse @@ -553,6 +554,7 @@ async def _update_litellm_setting( settings: Union[DefaultInternalUserParams, DefaultTeamSSOParams, MCPSemanticFilterSettings], settings_key: str, success_message: str, + user_api_key_dict: Optional[UserAPIKeyAuth] = None, ): """ Common utility function to update `litellm_settings` in both memory and config. @@ -561,8 +563,15 @@ async def _update_litellm_setting( settings: The settings object to update settings_key: The key in litellm_settings to update success_message: Message to return on success + user_api_key_dict: The acting admin, recorded as the audit-log actor. + Optional today so callers that have not been wired for auditing + keep working; the audit row is only written when an actor is passed. """ - from litellm.proxy.proxy_server import proxy_config, store_model_in_db + from litellm.proxy.proxy_server import ( + create_config_audit_log, + proxy_config, + store_model_in_db, + ) if store_model_in_db is not True: raise HTTPException( @@ -576,6 +585,7 @@ async def _update_litellm_setting( # because get_config() may overwrite litellm. with stale DB values # via LITELLM_SETTINGS_SAFE_DB_OVERRIDES. config = await proxy_config.get_config() + before_value = config.get("litellm_settings", {}).get(settings_key) # Update the in-memory settings (after get_config to avoid stale override) setattr(litellm, settings_key, in_memory_var) @@ -589,6 +599,21 @@ async def _update_litellm_setting( # Save the updated config await proxy_config.save_config(new_config=config) + if user_api_key_dict is not None: + # Fire-and-forget so an audit-log failure (transient DB blip, etc.) + # never surfaces as a 500 after save_config has already committed, + # matching the create_object_audit_log pattern used elsewhere + # (e.g. model_management_endpoints). + asyncio.create_task( + create_config_audit_log( + param_name=settings_key, + action="updated", + before_value=before_value, + after_value=in_memory_var, + user_api_key_dict=user_api_key_dict, + ) + ) + return { "message": success_message, "status": "success", @@ -619,6 +644,7 @@ async def update_internal_user_settings( settings=settings, settings_key="default_internal_user_params", success_message="Internal user settings updated successfully", + user_api_key_dict=user_api_key_dict, ) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 019a7dc90d2..dc35d71ccbd 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -8658,3 +8658,109 @@ def test_config_field_info_returns_raw_secrets_for_full_admin(monkeypatch): ) finally: app.dependency_overrides.clear() + + +def _fake_prisma_with_config(existing_param_value): + """MagicMock prisma whose litellm_config row returns existing_param_value and + whose litellm_auditlog.create records the written audit row.""" + fake = MagicMock() + config_row = MagicMock() + config_row.param_value = existing_param_value + fake.db.litellm_config.find_first = AsyncMock(return_value=config_row) + fake.db.litellm_config.upsert = AsyncMock(return_value=config_row) + fake.db.litellm_auditlog.create = AsyncMock() + return fake + + +def test_dump_redacted_config_redacts_secret_leaves(): + from litellm.proxy.proxy_server import _dump_redacted_config + + assert _dump_redacted_config(None) is None + + restored = json.loads( + _dump_redacted_config( + { + "api_key": "sk-leak", + "model": "gpt-4", + "nested": {"aws_secret_access_key": "abc", "region": "us-east-1"}, + } + ) + ) + assert restored["api_key"] == "REDACTED" + assert restored["model"] == "gpt-4" + assert restored["nested"]["aws_secret_access_key"] == "REDACTED" + assert restored["nested"]["region"] == "us-east-1" + + +@pytest.mark.asyncio +async def test_create_config_audit_log_writes_redacted_entry(monkeypatch): + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import LitellmTableNames + from litellm.proxy.proxy_server import create_config_audit_log + + fake = _fake_prisma_with_config({}) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + + caller = UserAPIKeyAuth(api_key="hashed-key-abc", user_id="admin-7") + await create_config_audit_log( + "router_settings", + "updated", + {"routing_strategy": "simple-shuffle", "api_key": "sk-old"}, + {"routing_strategy": "latency-based", "api_key": "sk-new"}, + caller, + ) + + fake.db.litellm_auditlog.create.assert_awaited_once() + written = fake.db.litellm_auditlog.create.call_args.kwargs["data"] + assert written["table_name"] == LitellmTableNames.CONFIG_TABLE_NAME.value + assert written["object_id"] == "router_settings" + assert written["action"] == "updated" + assert written["changed_by"] == "admin-7" + assert written["changed_by_api_key"] == "hashed-key-abc" + + before = json.loads(written["before_value"]) + after = json.loads(written["updated_values"]) + assert before["routing_strategy"] == "simple-shuffle" + assert after["routing_strategy"] == "latency-based" + assert "sk-old" not in written["before_value"] + assert "sk-new" not in written["updated_values"] + assert before["api_key"] != "sk-old" + assert after["api_key"] != "sk-new" + + +@pytest.mark.asyncio +async def test_create_config_audit_log_noop_when_store_audit_logs_disabled(monkeypatch): + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy.proxy_server import create_config_audit_log + + fake = _fake_prisma_with_config({}) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr(litellm, "store_audit_logs", False) + + await create_config_audit_log( + "router_settings", + "updated", + {}, + {"a": 1}, + UserAPIKeyAuth(api_key="k", user_id="u"), + ) + fake.db.litellm_auditlog.create.assert_not_called() + + +def test_dump_redacted_config_serializes_non_json_native_values(): + """YAML-loaded config can contain datetime/date/custom values that plain + json.dumps refuses. Without default=str the audit write turns into a 500 + after the config change has already committed; the sibling audit-log + serializers in team_endpoints.py use default=str for the same reason.""" + from datetime import datetime, timezone + + from litellm.proxy.proxy_server import _dump_redacted_config + + out = _dump_redacted_config({"updated_at": datetime(2026, 6, 30, tzinfo=timezone.utc)}) + assert out is not None + restored = json.loads(out) + assert "2026-06-30" in restored["updated_at"] + diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index ae217aca16e..7a586f758f4 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -1869,3 +1869,94 @@ class TestProxySettingEndpoints: assert "field_schema" in data assert "properties" in data["field_schema"] assert "role_mappings" in data["field_schema"]["properties"] + + +def test_update_internal_user_settings_writes_audit_log(mock_proxy_config, monkeypatch): + """Regression for the reported scenario: an admin changes Default User + Settings from the dashboard, which issues PATCH /update/internal_user_settings + (NOT /config/update). An audit row must record who changed it and what + changed.""" + from unittest.mock import AsyncMock, MagicMock + + import litellm + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + audit_create = AsyncMock() + fake_prisma = MagicMock() + fake_prisma.db.litellm_auditlog.create = audit_create + + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + monkeypatch.setattr(litellm, "default_internal_user_params", {}) + + async def _admin_auth(): + return UserAPIKeyAuth( + user_id="audit-admin", + api_key="hashed-admin-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + app.dependency_overrides[user_api_key_auth] = _admin_auth + try: + resp = client.patch( + "/update/internal_user_settings", + json={"max_budget": 999.0, "models": ["gpt-4"]}, + ) + assert resp.status_code == 200, resp.text + + audit_create.assert_awaited_once() + written = audit_create.await_args.kwargs["data"] + assert written["object_id"] == "default_internal_user_params" + assert written["action"] == "updated" + assert written["table_name"] == "LiteLLM_Config" + assert written["changed_by"] == "audit-admin" + assert written["changed_by_api_key"] == "hashed-admin-key" + + before = json.loads(written["before_value"]) + after = json.loads(written["updated_values"]) + assert before["max_budget"] == 100.0 + assert after["max_budget"] == 999.0 + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_update_internal_user_settings_returns_200_when_audit_write_raises( + mock_proxy_config, monkeypatch +): + """The settings change is already committed by save_config, so an + audit-log failure must never surface as a 500. Scheduling via + asyncio.create_task keeps the audit call off the request path; this + test asserts that contract by making the audit helper raise.""" + import litellm + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr(litellm, "default_internal_user_params", {}) + + async def _raise(**_kwargs): + raise RuntimeError("audit prisma blip") + + monkeypatch.setattr(proxy_server_module, "create_config_audit_log", _raise) + + async def _admin_auth(): + return UserAPIKeyAuth( + user_id="audit-admin", + api_key="hashed-admin-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + app.dependency_overrides[user_api_key_auth] = _admin_auth + try: + resp = client.patch( + "/update/internal_user_settings", json={"max_budget": 42.0} + ) + assert resp.status_code == 200, resp.text + assert resp.json()["status"] == "success" + finally: + app.dependency_overrides.pop(user_api_key_auth, None) From ada9ef88ac1a18d7c3073bad951cea9be4a3f981 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 30 Jun 2026 18:36:40 -0700 Subject: [PATCH 11/81] fix(websearch): websearch_interception agentic loop fixes for chat completions and anthropic messages (#31669) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(websearch): wire chat completion agentic loop to correct hooks maybe_run_chat_completion_agentic_loop was calling async_should_run_agentic_loop (Anthropic format) and async_run_agentic_loop (Anthropic path) instead of the chat-completion variants. This meant WebSearchInterceptionLogger never intercepted chat completion requests — the LLM returned a litellm_web_search tool_call but the agentic loop never executed, so the raw tool_calls response was returned to the caller. Fix: gate on async_should_run_chat_completion_agentic_loop override, call that hook and async_build_chat_completion_agentic_loop_plan / async_run_chat_completion_agentic_loop in the execution path. Regression test added. * fix(websearch): strip tool_choice from follow-up request When the original request forces tool_choice to litellm_web_search, the follow-up request after search execution inherited that tool_choice, causing the model to call the search tool again instead of synthesizing an answer from the results. * fix(websearch): inject api_key into agentic hook kwargs for anthropic messages Follow-up calls inside async_run_agentic_loop (e.g. websearch interception's synthesis call after executing Exa/Perplexity searches) were missing api_key because the named api_key param in async_anthropic_messages_handler was never merged into the kwargs dict forwarded downstream. Result: every /v1/messages websearch follow-up failed with "x-api-key header is required" and the caller received the raw tool_use response instead of the synthesized answer. * ci: trigger CI run * fix(websearch): support unified agentic hooks alongside chat-completion-specific hooks CodeInterpreterInterceptionLogger uses async_should_run_agentic_loop with _agentic_loop_api_surface to handle both surfaces from one hook. The chat completion loop must also check _gate_overridden so callbacks using the unified hook pattern still fire for chat completions. * fix(websearch): strip tool_choice from legacy chat completion follow-up call The _execute_chat_completion_agentic_loop path merged original optional_params (which includes forced tool_choice) into follow-up params without explicit removal. _build_chat_completion_request_patch already excluded tool_choice from its optional_params output, but dict.update() with a missing key leaves the original value intact. Explicit pop after the merge removes it. * fix(websearch): always strip tool_choice from plan-path follow-up params The tool_choice removal was gated on patch.tools is not None. WebSearch sets tools via patch.optional_params not patch.tools, so the gate was False and forced tool_choice from the original request survived into the synthesis call. Move the pop outside the patch.tools branch so it applies unconditionally. --- .../websearch_interception/handler.py | 31 +- .../chat_completion_agentic_loop.py | 18 +- litellm/llms/custom_httpx/llm_http_handler.py | 8 +- .../test_websearch_chat_completion.py | 281 ++++++++++++++---- .../test_chat_completion_agentic_loop.py | 18 +- .../custom_httpx/test_llm_http_handler.py | 67 +++++ 6 files changed, 325 insertions(+), 98 deletions(-) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 2e11405af3f..bfae6d5b7b0 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -31,6 +31,7 @@ from litellm.types.integrations.websearch_interception import ( WebSearchInterceptionConfig, ) from litellm.types.integrations.custom_logger import ( + CHAT_COMPLETION_AGENTIC_SURFACE, AgenticLoopPlan, AgenticLoopRequestPatch, ) @@ -440,12 +441,16 @@ class WebSearchInterceptionLogger(CustomLogger): custom_llm_provider: str, kwargs: Dict, ) -> Tuple[bool, Dict]: - """ - Check if WebSearch tool interception is needed for Anthropic Messages API. - - This is the legacy method for Anthropic-style responses. - For chat completions, use async_should_run_chat_completion_agentic_loop instead. - """ + if kwargs.get("_agentic_loop_api_surface") == CHAT_COMPLETION_AGENTIC_SURFACE: + return await self.async_should_run_chat_completion_agentic_loop( + response=response, + model=model, + messages=messages, + tools=tools, + stream=stream, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + ) verbose_logger.debug(f"WebSearchInterception: Hook called! provider={custom_llm_provider}, stream={stream}") verbose_logger.debug(f"WebSearchInterception: Response type: {type(response)}") @@ -629,6 +634,18 @@ class WebSearchInterceptionLogger(CustomLogger): stream: bool, kwargs: Dict, ) -> AgenticLoopPlan: + if kwargs.get("_agentic_loop_api_surface") == CHAT_COMPLETION_AGENTIC_SURFACE: + return await self.async_build_chat_completion_agentic_loop_plan( + tools=tools, + model=model, + messages=messages, + response=response, + optional_params=anthropic_messages_optional_request_params, + logging_obj=logging_obj, + stream=stream, + kwargs=kwargs, + ) + tool_calls = tools["tool_calls"] thinking_blocks = tools.get("thinking_blocks", []) request_patch, structured_results = await self._build_anthropic_request_patch( @@ -1088,6 +1105,7 @@ class WebSearchInterceptionLogger(CustomLogger): raise ValueError("WebSearchInterception: missing follow-up messages") params = dict(optional_params) params.update(request_patch.optional_params) + params.pop("tool_choice", None) return await litellm.acompletion( model=request_patch.model or model, messages=request_patch.messages, @@ -1203,6 +1221,7 @@ class WebSearchInterceptionLogger(CustomLogger): if k not in { "tools", + "tool_choice", "extra_body", "model_alias_map", "stream_response", diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index 828605d5ef8..b7262a42324 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -137,8 +137,8 @@ async def _execute_chat_completion_agentic_plan( optional_params_for_followup = {**optional_params, **patch.optional_params} if patch.tools is not None: optional_params_for_followup["tools"] = patch.tools - if "tool_choice" not in patch.optional_params: - optional_params_for_followup.pop("tool_choice", None) + if "tool_choice" not in patch.optional_params: + optional_params_for_followup.pop("tool_choice", None) kwargs_for_followup = _filter_followup_kwargs(kwargs) kwargs_for_followup.update( @@ -206,10 +206,11 @@ async def maybe_run_chat_completion_agentic_loop( for callback in callbacks: if not isinstance(callback, CustomLogger): continue + if not _gate_overridden(callback): continue - gate_kwargs = { + hook_kwargs = { **kwargs, "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE, "custom_llm_provider": custom_llm_provider, @@ -222,7 +223,7 @@ async def maybe_run_chat_completion_agentic_loop( tools=tools, stream=stream, custom_llm_provider=custom_llm_provider, - kwargs=gate_kwargs, + kwargs=hook_kwargs, ) except Exception as e: verbose_logger.exception( @@ -243,11 +244,6 @@ async def maybe_run_chat_completion_agentic_loop( ) try: - plan_kwargs = { - **kwargs, - "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE, - "custom_llm_provider": custom_llm_provider, - } if not _build_plan_overridden(callback): return await callback.async_run_agentic_loop( tools=tool_calls, @@ -258,7 +254,7 @@ async def maybe_run_chat_completion_agentic_loop( anthropic_messages_optional_request_params=optional_params, logging_obj=logging_obj, stream=stream, - kwargs=plan_kwargs, + kwargs=hook_kwargs, ) plan = await callback.async_build_agentic_loop_plan( @@ -270,7 +266,7 @@ async def maybe_run_chat_completion_agentic_loop( anthropic_messages_optional_request_params=optional_params, logging_obj=logging_obj, stream=stream, - kwargs=plan_kwargs, + kwargs=hook_kwargs, ) if plan.response_override is not None: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 9bb956d0808..3c10239f868 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2112,7 +2112,7 @@ class BaseLLMHTTPHandler: anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, - kwargs=kwargs, + kwargs={**kwargs, "api_key": api_key} if api_key else kwargs, ) return initial_response else: @@ -2122,6 +2122,10 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, ) + # Inject api_key into kwargs so follow-up calls in agentic hooks can + # authenticate. api_key is a named param here (not in kwargs), so + # _prepare_followup_kwargs would miss it otherwise. + kwargs_for_agentic = {**kwargs, "api_key": api_key} if api_key else kwargs # Call agentic completion hooks (non-streaming path only) final_response = await self._call_agentic_completion_hooks( response=initial_response, @@ -2132,7 +2136,7 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, stream=False, custom_llm_provider=custom_llm_provider, - kwargs=kwargs, + kwargs=kwargs_for_agentic, ) return self._maybe_wrap_in_fake_stream( diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_chat_completion.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_chat_completion.py index 34555d76554..7ef43e2eadf 100644 --- a/tests/test_litellm/integrations/websearch_interception/test_websearch_chat_completion.py +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_chat_completion.py @@ -6,7 +6,7 @@ litellm.acompletion() for transparent server-side web search execution. """ import os -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import MagicMock import pytest @@ -34,9 +34,7 @@ def mock_search_response(): @pytest.fixture def websearch_logger(): """Create a WebSearchInterceptionLogger instance""" - return WebSearchInterceptionLogger( - enabled_providers=[LlmProviders.OPENAI, LlmProviders.MINIMAX] - ) + return WebSearchInterceptionLogger(enabled_providers=[LlmProviders.OPENAI, LlmProviders.MINIMAX]) @pytest.mark.asyncio @@ -55,9 +53,7 @@ async def test_websearch_chat_completion_with_openai(): """ # Configure WebSearch interception original_callbacks = litellm.callbacks.copy() if litellm.callbacks else [] - websearch_logger = WebSearchInterceptionLogger( - enabled_providers=[LlmProviders.OPENAI] - ) + websearch_logger = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.OPENAI]) litellm.callbacks = [websearch_logger] try: @@ -100,9 +96,7 @@ async def test_websearch_chat_completion_with_openai(): if hasattr(response.choices[0].message, "tool_calls"): # If tool_calls exist, it means agentic loop didn't run # This could happen if search tool is not configured - pytest.skip( - "Agentic loop did not execute - search tool may not be configured" - ) + pytest.skip("Agentic loop did not execute - search tool may not be configured") # Verify we got a meaningful response assert response.choices[0].finish_reason in ["stop", "end_turn"] @@ -122,9 +116,7 @@ async def test_websearch_chat_completion_hook_detection(): Message, ) - websearch_logger = WebSearchInterceptionLogger( - enabled_providers=[LlmProviders.OPENAI] - ) + websearch_logger = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.OPENAI]) # Mock response with litellm_web_search tool call mock_response = ModelResponse( @@ -155,21 +147,19 @@ async def test_websearch_chat_completion_hook_detection(): ) # Test should_run_chat_completion_agentic_loop - should_run, tools_dict = ( - await websearch_logger.async_should_run_chat_completion_agentic_loop( - response=mock_response, - model="gpt-4o", - messages=[{"role": "user", "content": "What's the weather?"}], - tools=[ - { - "type": "function", - "function": {"name": "litellm_web_search"}, - } - ], - stream=False, - custom_llm_provider="openai", - kwargs={}, - ) + should_run, tools_dict = await websearch_logger.async_should_run_chat_completion_agentic_loop( + response=mock_response, + model="gpt-4o", + messages=[{"role": "user", "content": "What's the weather?"}], + tools=[ + { + "type": "function", + "function": {"name": "litellm_web_search"}, + } + ], + stream=False, + custom_llm_provider="openai", + kwargs={}, ) # Verify hook detected the tool call @@ -185,9 +175,7 @@ async def test_websearch_not_triggered_without_tool(): """Test that websearch hook is NOT triggered when no web search tool in request.""" from litellm.types.utils import Choices, Message - websearch_logger = WebSearchInterceptionLogger( - enabled_providers=[LlmProviders.OPENAI] - ) + websearch_logger = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.OPENAI]) mock_response = ModelResponse( id="test-123", @@ -208,21 +196,19 @@ async def test_websearch_not_triggered_without_tool(): ) # Test without web search tool - should_run, tools_dict = ( - await websearch_logger.async_should_run_chat_completion_agentic_loop( - response=mock_response, - model="gpt-4o", - messages=[{"role": "user", "content": "Hello"}], - tools=[ - { - "type": "function", - "function": {"name": "some_other_tool"}, - } - ], - stream=False, - custom_llm_provider="openai", - kwargs={}, - ) + should_run, tools_dict = await websearch_logger.async_should_run_chat_completion_agentic_loop( + response=mock_response, + model="gpt-4o", + messages=[{"role": "user", "content": "Hello"}], + tools=[ + { + "type": "function", + "function": {"name": "some_other_tool"}, + } + ], + stream=False, + custom_llm_provider="openai", + kwargs={}, ) # Verify hook did NOT trigger @@ -241,9 +227,7 @@ async def test_websearch_not_triggered_for_disabled_provider(): ) # Only enable bedrock - websearch_logger = WebSearchInterceptionLogger( - enabled_providers=[LlmProviders.BEDROCK] - ) + websearch_logger = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.BEDROCK]) mock_response = ModelResponse( id="test-123", @@ -273,21 +257,19 @@ async def test_websearch_not_triggered_for_disabled_provider(): ) # Test with OpenAI provider (not enabled) - should_run, tools_dict = ( - await websearch_logger.async_should_run_chat_completion_agentic_loop( - response=mock_response, - model="gpt-4o", - messages=[{"role": "user", "content": "test"}], - tools=[ - { - "type": "function", - "function": {"name": "litellm_web_search"}, - } - ], - stream=False, - custom_llm_provider="openai", # Not in enabled_providers - kwargs={}, - ) + should_run, tools_dict = await websearch_logger.async_should_run_chat_completion_agentic_loop( + response=mock_response, + model="gpt-4o", + messages=[{"role": "user", "content": "test"}], + tools=[ + { + "type": "function", + "function": {"name": "litellm_web_search"}, + } + ], + stream=False, + custom_llm_provider="openai", # Not in enabled_providers + kwargs={}, ) # Verify hook did NOT trigger @@ -341,8 +323,7 @@ async def test_websearch_json_serialization_fix(): @pytest.mark.asyncio @pytest.mark.skipif( - os.environ.get("OPENAI_API_KEY") is None - or os.environ.get("PERPLEXITY_API_KEY") is None, + os.environ.get("OPENAI_API_KEY") is None or os.environ.get("PERPLEXITY_API_KEY") is None, reason="OPENAI_API_KEY or PERPLEXITY_API_KEY not set", ) async def test_websearch_streaming_conversion(): @@ -395,6 +376,174 @@ async def test_websearch_streaming_conversion(): litellm.callbacks = [] +@pytest.mark.asyncio +async def test_maybe_run_chat_completion_agentic_loop_calls_chat_completion_hook(): + """Regression test: maybe_run_chat_completion_agentic_loop must call + async_should_run_chat_completion_agentic_loop, not async_should_run_agentic_loop. + + Before the fix, the function used the wrong gate check and wrong hook, + causing WebSearchInterceptionLogger to never intercept chat completion requests + even when the LLM returned a litellm_web_search tool call. + """ + from litellm.litellm_core_utils.chat_completion_agentic_loop import ( + maybe_run_chat_completion_agentic_loop, + ) + from litellm.types.utils import ( + ChatCompletionMessageToolCall, + Choices, + Function, + Message, + ) + + mock_response = ModelResponse( + id="test-regression-123", + choices=[ + Choices( + finish_reason="tool_calls", + index=0, + message=Message( + role="assistant", + content=None, + tool_calls=[ + ChatCompletionMessageToolCall( + id="call_abc", + type="function", + function=Function( + name="litellm_web_search", + arguments='{"query": "latest news"}', + ), + ) + ], + ), + ) + ], + model="gpt-4o", + object="chat.completion", + created=1234567890, + ) + + sentinel = ModelResponse( + id="sentinel-final", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message(role="assistant", content="Here is the news."), + ) + ], + model="gpt-4o", + object="chat.completion", + created=1234567890, + ) + + websearch_logger = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.OPENAI]) + + chat_completion_hook_called = False + + async def fake_should_run_chat_completion(response, model, messages, tools, stream, custom_llm_provider, kwargs): + nonlocal chat_completion_hook_called + chat_completion_hook_called = True + return True, { + "tool_calls": [{"id": "call_abc", "name": "litellm_web_search", "input": {"query": "latest news"}}], + "tool_type": "websearch", + "provider": "openai", + "response_format": "openai", + } + + async def fake_build_plan(tools, model, messages, response, optional_params, logging_obj, stream, kwargs): + from litellm.types.integrations.custom_logger import AgenticLoopPlan + + return AgenticLoopPlan(run_agentic_loop=False, response_override=sentinel) + + websearch_logger.async_should_run_chat_completion_agentic_loop = fake_should_run_chat_completion + websearch_logger.async_build_chat_completion_agentic_loop_plan = fake_build_plan + + import litellm as _litellm + + original_callbacks = _litellm.callbacks[:] + _litellm.callbacks = [websearch_logger] + + mock_logging_obj = MagicMock() + mock_logging_obj.dynamic_success_callbacks = None + + try: + result = await maybe_run_chat_completion_agentic_loop( + response=mock_response, + model="gpt-4o", + messages=[{"role": "user", "content": "Latest news?"}], + optional_params={ + "tools": [ + { + "type": "function", + "function": {"name": "litellm_web_search"}, + } + ] + }, + kwargs={}, + logging_obj=mock_logging_obj, + custom_llm_provider="openai", + stream=False, + ) + finally: + _litellm.callbacks = original_callbacks + + assert chat_completion_hook_called, ( + "async_should_run_chat_completion_agentic_loop was never called; " + "maybe_run_chat_completion_agentic_loop used the wrong hook" + ) + assert result is sentinel, "Expected agentic loop to return sentinel final response" + + +@pytest.mark.asyncio +async def test_execute_chat_completion_agentic_loop_strips_tool_choice(): + """Regression: _execute_chat_completion_agentic_loop must not forward tool_choice + from the original request into the follow-up synthesis call. + + When the original request forces tool_choice to litellm_web_search, merging + optional_params into the follow-up params without explicit removal causes the + model to call the search tool again instead of synthesizing an answer. + """ + from unittest.mock import patch + + websearch_logger = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.OPENAI]) + + captured_kwargs: dict = {} + + async def fake_acompletion(**kwargs): + captured_kwargs.update(kwargs) + return ModelResponse(id="followup", model="gpt-4o", object="chat.completion") + + async def fake_search(query): + return ("Bitcoin price is $60,000", None) + + with patch.object(websearch_logger, "_execute_search", side_effect=fake_search): + with patch("litellm.acompletion", side_effect=fake_acompletion): + await websearch_logger._execute_chat_completion_agentic_loop( + model="gpt-4o", + messages=[{"role": "user", "content": "What is Bitcoin price?"}], + tool_calls=[ + { + "id": "call_1", + "name": "litellm_web_search", + "input": {"query": "bitcoin price"}, + } + ], + optional_params={ + "tools": [{"type": "function", "function": {"name": "litellm_web_search"}}], + "tool_choice": {"type": "function", "function": {"name": "litellm_web_search"}}, + "max_tokens": 512, + }, + logging_obj=MagicMock(), + stream=False, + kwargs={}, + ) + + assert "tool_choice" not in captured_kwargs, ( + "tool_choice must not appear in follow-up acompletion kwargs; " + "it would force the model to call the search tool again instead of synthesizing" + ) + + if __name__ == "__main__": # Run with: pytest test_websearch_chat_completion.py -v -s pytest.main([__file__, "-v", "-s"]) diff --git a/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py index f1196ab4692..cc16ad558e4 100644 --- a/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py +++ b/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py @@ -181,8 +181,7 @@ async def test_internal_control_fields_never_leak_into_provider_body(restore_cal # The loop must have actually fired (sanity: two provider calls). assert create.await_count == 2, ( - "expected the agentic loop to issue a follow-up provider call; " - f"got {create.await_count} call(s)" + f"expected the agentic loop to issue a follow-up provider call; got {create.await_count} call(s)" ) for idx, call in enumerate(create.await_args_list): @@ -194,8 +193,7 @@ async def test_internal_control_fields_never_leak_into_provider_body(restore_cal f"top-level request body: {sorted(body.keys())}" ) assert field not in extra_body, ( - f"provider call #{idx}: internal field {field!r} leaked into " - f"extra_body: {sorted(extra_body.keys())}" + f"provider call #{idx}: internal field {field!r} leaked into extra_body: {sorted(extra_body.keys())}" ) # The native code_interpreter tool must have been swapped for the # function tool, never sent raw to OpenAI as a chat-completions request. @@ -254,9 +252,7 @@ class _GateOnlyLogger(CustomLogger): ) -> AgenticLoopPlan: return self._plan - async def async_agentic_loop_cleanup_hook( - self, plan: AgenticLoopPlan, kwargs: Dict[str, Any] - ) -> None: + async def async_agentic_loop_cleanup_hook(self, plan: AgenticLoopPlan, kwargs: Dict[str, Any]) -> None: self.cleanup_calls += 1 @@ -343,9 +339,7 @@ async def test_dispatcher_runs_followup_with_incremented_depth_and_patched_messa assert call_kwargs["max_agentic_loops"] >= 1 assert "_agentic_loop_fingerprints" in call_kwargs # Interception markers are mirrored into litellm_metadata for the follow-up. - assert ( - call_kwargs["litellm_metadata"]["_code_interpreter_interception_active"] is True - ) + assert call_kwargs["litellm_metadata"]["_code_interpreter_interception_active"] is True # The transient surface marker is NOT forwarded to the follow-up call. assert "_agentic_loop_api_surface" not in call_kwargs # Cleanup hook always runs. @@ -390,9 +384,7 @@ async def test_dispatcher_raises_on_repeated_tool_call_fingerprint(restore_callb # The dispatcher fingerprints the whole value the gate returns as its second # tuple element, so the seeded fingerprint must mirror that dict exactly. - gate_tool_calls = { - "tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}] - } + gate_tool_calls = {"tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}]} fingerprint = json.dumps(gate_tool_calls, sort_keys=True, default=str) logger = _GateOnlyLogger( diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index 8c934f9c21e..b18af060a20 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -1212,6 +1212,73 @@ def test_async_compact_handler_sends_json_when_not_signed(): assert "data" not in kwargs +@pytest.mark.asyncio +async def test_async_anthropic_messages_handler_passes_api_key_to_agentic_hooks(): + """ + Regression: async_anthropic_messages_handler must inject api_key into the + kwargs dict forwarded to _call_agentic_completion_hooks. + + Without this, follow-up calls made by agentic hooks (e.g. websearch + interception's second LLM call after executing searches) have no api_key + and fail with "x-api-key header is required". + """ + handler = BaseLLMHTTPHandler() + + mock_config = Mock() + mock_config.validate_anthropic_messages_environment = Mock( + return_value=({"x-api-key": "sk-test"}, "https://api.anthropic.com") + ) + mock_config.transform_anthropic_messages_request = Mock( + return_value={"model": "claude-haiku", "messages": [], "max_tokens": 16} + ) + mock_config.sign_request = Mock(return_value=({}, None)) + + fake_raw_response = {"id": "msg_1", "type": "message", "role": "assistant", "content": [], "stop_reason": "end_turn"} + mock_config.transform_anthropic_messages_response = Mock(return_value=fake_raw_response) + + mock_logging_obj = Mock() + mock_logging_obj.update_environment_variables = Mock() + mock_logging_obj.model_call_details = {} + mock_logging_obj.stream = False + mock_logging_obj.dynamic_success_callbacks = None + + captured_kwargs: dict = {} + sentinel_response = object() + + async def fake_agentic_hooks(**call_kwargs): + captured_kwargs.update(call_kwargs) + return sentinel_response + + mock_httpx_response = Mock() + mock_httpx_response.status_code = 200 + + with ( + patch.object(handler, "_async_post_anthropic_messages_with_http_error_retry", new=AsyncMock(return_value=mock_httpx_response)), + patch.object(handler, "_call_agentic_completion_hooks", side_effect=fake_agentic_hooks), + patch("litellm.llms.custom_httpx.llm_http_handler.get_async_httpx_client"), + patch("litellm.litellm_core_utils.get_provider_specific_headers.ProviderSpecificHeaderUtils.get_provider_specific_headers", return_value=None), + ): + result = await handler.async_anthropic_messages_handler( + model="claude-haiku", + messages=[{"role": "user", "content": "hi"}], + anthropic_messages_provider_config=mock_config, + anthropic_messages_optional_request_params={"stream": False}, + custom_llm_provider="anthropic", + litellm_params=GenericLiteLLMParams(api_key="sk-real-anthropic-key"), + logging_obj=mock_logging_obj, + api_key="sk-real-anthropic-key", + stream=False, + ) + + assert result is sentinel_response + assert "kwargs" in captured_kwargs, "_call_agentic_completion_hooks not called" + forwarded = captured_kwargs["kwargs"] + assert forwarded.get("api_key") == "sk-real-anthropic-key", ( + "api_key must be injected into kwargs passed to _call_agentic_completion_hooks " + "so follow-up calls in agentic hooks (e.g. websearch) can authenticate" + ) + + class _FakeWSExceptions: class WebSocketException(Exception): pass From 6c21029cb7e8fe827ea9f8d108f1ff18ae3b9e4b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 30 Jun 2026 18:58:09 -0700 Subject: [PATCH 12/81] feat(sandbox): reuse e2b container across requests when metadata.session_id is set (#31688) * feat(sandbox): reuse e2b container across requests when metadata.session_id is set When a client passes `metadata.session_id` in a /chat/completions request alongside a code_interpreter tool, the proxy now routes all requests sharing that session_id to the same sandbox container. State (variables, imports, installed packages) persists across requests within the session. Without a session_id the existing ephemeral behavior is unchanged: one container per agentic loop, deleted immediately after. The sandbox key is derived from session_id rather than a per-request UUID. The cleanup and post-loop hooks skip deletion for session-scoped containers. TTL-based pruning (15 min idle) still applies and refreshes on every use, so an active session never expires mid-use. The session_id-scoped key is registered in all_litellm_params and the proxy strip-list so it never leaks to the upstream LLM provider. * fix(sandbox): scope session sandbox key to API key identity; add per-identity LRU cap Two security issues addressed: 1. Cross-user sandbox isolation: the session_id supplied by the client is now combined with the server-minted user_api_key_hash to form the cache key (format: "{hash}:{session_id}" when authenticated, bare session_id for non-proxy use). Two tenants sharing the same session_id no longer share a sandbox. 2. Bounded session allocation: each API key identity is capped at _SESSION_SCOPED_PER_IDENTITY_CAP (10) live session-scoped containers. When a new session is opened beyond the cap, the least-recently-used entry for that identity is evicted and its sandbox deleted, preventing unbounded accumulation via rotating session IDs. The container cache tuple gains a fourth element (identity: str | None) so eviction can filter by identity without parsing key formats. Tests added for both properties. --- .../code_interpreter_interception/handler.py | 71 +++- litellm/proxy/dev_config.yaml | 15 + litellm/proxy/litellm_pre_call_utils.py | 1 + litellm/types/utils.py | 1 + qa_sticky_session.sh | 59 +++ .../test_handler.py | 369 ++++++++++++++---- 6 files changed, 421 insertions(+), 95 deletions(-) create mode 100755 qa_sticky_session.sh diff --git a/litellm/integrations/code_interpreter_interception/handler.py b/litellm/integrations/code_interpreter_interception/handler.py index cd7b211f1a5..759b2be3a84 100644 --- a/litellm/integrations/code_interpreter_interception/handler.py +++ b/litellm/integrations/code_interpreter_interception/handler.py @@ -40,9 +40,11 @@ from litellm.types.utils import ( LITELLM_CODE_EXECUTION_TOOL_NAME = "litellm_code_execution" _INTERCEPTION_ACTIVE_KEY = "_code_interpreter_interception_active" _SANDBOX_KEY = "_code_interpreter_interception_sandbox_key" +_SESSION_SCOPED_KEY = "_code_interpreter_interception_session_scoped" _CONVERTED_STREAM_KEY = "_code_interpreter_interception_converted_stream" _LITELLM_METADATA_KEY = "litellm_metadata" _CACHE_TTL_SECONDS = 15 * 60 +_SESSION_SCOPED_PER_IDENTITY_CAP = 10 class CodeExecutionToolCall(TypedDict, total=False): @@ -107,6 +109,20 @@ class ChatCompletionFunctionToolChoice(TypedDict): CodeExecutionFunctionToolChoice = ResponsesFunctionToolChoice | ChatCompletionFunctionToolChoice +def _extract_session_id(kwargs: dict[str, Any]) -> str | None: + for meta_key in ("metadata", "litellm_metadata"): + meta = kwargs.get(meta_key) + if isinstance(meta, dict): + sid = meta.get("session_id") + if sid and isinstance(sid, str): + return sid + return None + + +def _extract_identity(kwargs: dict[str, Any]) -> str: + return kwargs.get("user_api_key_hash") or "" + + def _resolve_sandbox_tool(sandbox_tool_name: str | None) -> dict[str, Any] | None: try: from litellm.sandbox.sandbox_tools import resolve_sandbox_tool @@ -140,7 +156,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger): self.enabled_providers = enabled_providers self.sandbox_tool_name = sandbox_tool_name self.sandbox_config = sandbox_config - self._container_cache: dict[str, tuple[Any, dict[str, Any] | None, float]] = {} + self._container_cache: dict[str, tuple[Any, dict[str, Any] | None, float, str | None]] = {} @classmethod def from_config_yaml(cls, config: CodeInterpreterInterceptionConfig) -> "CodeInterpreterInterceptionLogger": @@ -191,7 +207,13 @@ class CodeInterpreterInterceptionLogger(CustomLogger): return None kwargs[_INTERCEPTION_ACTIVE_KEY] = True - kwargs[_SANDBOX_KEY] = uuid.uuid4().hex + session_id = _extract_session_id(kwargs) + if session_id: + identity = _extract_identity(kwargs) + kwargs[_SANDBOX_KEY] = f"{identity}:{session_id}" if identity else session_id + kwargs[_SESSION_SCOPED_KEY] = True + else: + kwargs[_SANDBOX_KEY] = uuid.uuid4().hex if kwargs.get("stream"): kwargs["stream"] = False kwargs[_CONVERTED_STREAM_KEY] = True @@ -217,6 +239,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger): if not is_interception_internal_key(key) and not key.startswith("_agentic_loop") and key != "max_agentic_loops" + and key != _SESSION_SCOPED_KEY } if filtered_metadata: kwargs[_LITELLM_METADATA_KEY] = filtered_metadata @@ -227,7 +250,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger): def _write_interception_metadata(kwargs: dict[str, Any]) -> None: metadata = kwargs.get(_LITELLM_METADATA_KEY) metadata = dict(metadata) if isinstance(metadata, dict) else {} - for key in (_INTERCEPTION_ACTIVE_KEY, _SANDBOX_KEY, _CONVERTED_STREAM_KEY): + for key in (_INTERCEPTION_ACTIVE_KEY, _SANDBOX_KEY, _SESSION_SCOPED_KEY, _CONVERTED_STREAM_KEY): if key in kwargs: metadata[key] = kwargs[key] kwargs[_LITELLM_METADATA_KEY] = metadata @@ -347,7 +370,9 @@ class CodeInterpreterInterceptionLogger(CustomLogger): await self._prune_expired_cache() tool_calls = cast(list[CodeExecutionToolCall], tools.get("tool_calls", [])) sandbox_key = kwargs.get(_SANDBOX_KEY) - container, params = await self._get_or_create_container(cache_key=sandbox_key) + is_session = bool(kwargs.get(_SESSION_SCOPED_KEY)) + identity = _extract_identity(kwargs) if is_session else None + container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity) try: container_id = cast(str | None, getattr(container, "id", None)) @@ -404,6 +429,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger): metadata={ "tool_type": "code_interpreter", "sandbox_key": sandbox_key or "", + "is_session_scoped": bool(kwargs.get(_SESSION_SCOPED_KEY)), "code_interpreter_calls": code_interpreter_calls, }, ) @@ -419,7 +445,9 @@ class CodeInterpreterInterceptionLogger(CustomLogger): await self._prune_expired_cache() tool_calls = cast(list[CodeExecutionToolCall], tools.get("tool_calls", [])) sandbox_key = cast(str | None, kwargs.get(_SANDBOX_KEY)) - container, params = await self._get_or_create_container(cache_key=sandbox_key) + is_session = bool(kwargs.get(_SESSION_SCOPED_KEY)) + identity = _extract_identity(cast(dict[str, Any], kwargs)) if is_session else None + container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity) try: container_id = cast(str | None, getattr(container, "id", None)) @@ -455,6 +483,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger): metadata={ "tool_type": "code_interpreter", "sandbox_key": sandbox_key or "", + "is_session_scoped": bool(kwargs.get(_SESSION_SCOPED_KEY)), "code_interpreter_calls": code_interpreter_calls, "response_format": "openai", }, @@ -489,6 +518,8 @@ class CodeInterpreterInterceptionLogger(CustomLogger): async def async_agentic_loop_cleanup_hook(self, plan: AgenticLoopPlan, kwargs: dict) -> None: metadata = plan.metadata or {} if plan else {} + if metadata.get("is_session_scoped"): + return await self._delete_container_for_cache_key(metadata.get("sandbox_key")) @staticmethod @@ -520,7 +551,8 @@ class CodeInterpreterInterceptionLogger(CustomLogger): async def async_post_agentic_loop_response_hook(self, response: Any, plan: AgenticLoopPlan, kwargs: dict) -> Any: metadata = plan.metadata or {} if plan else {} - await self._delete_container_for_cache_key(metadata.get("sandbox_key")) + if not metadata.get("is_session_scoped"): + await self._delete_container_for_cache_key(metadata.get("sandbox_key")) calls = metadata.get("code_interpreter_calls") if not calls: @@ -565,17 +597,32 @@ class CodeInterpreterInterceptionLogger(CustomLogger): return f"[execution error] {message}" return getattr(result, "stdout", "") or "" - async def _get_or_create_container(self, cache_key: str | None) -> tuple[Any, dict[str, Any] | None]: + async def _get_or_create_container( + self, + cache_key: str | None, + identity: str | None = None, + ) -> tuple[Any, dict[str, Any] | None]: if cache_key: cached = self._container_cache.get(cache_key) if cached is not None: + self._container_cache[cache_key] = (cached[0], cached[1], time.time(), cached[3]) return cached[0], cached[1] container, params = await self._create_container() if cache_key: - self._container_cache[cache_key] = (container, params, time.time()) + if identity is not None: + await self._evict_lru_session_if_over_cap(identity) + self._container_cache[cache_key] = (container, params, time.time(), identity) return container, params + async def _evict_lru_session_if_over_cap(self, identity: str) -> None: + identity_entries = [(k, v) for k, v in self._container_cache.items() if v[3] == identity] + if len(identity_entries) < _SESSION_SCOPED_PER_IDENTITY_CAP: + return + lru_key, lru_entry = min(identity_entries, key=lambda item: item[1][2]) + self._container_cache.pop(lru_key, None) + await self._delete_container(container=lru_entry[0], params=lru_entry[1]) + async def _create_container(self) -> tuple[Any, dict[str, Any] | None]: if self.sandbox_config is not None: return await self.sandbox_config.acreate_sandbox(), None @@ -739,12 +786,8 @@ class CodeInterpreterInterceptionLogger(CustomLogger): now = time.time() expired = [ (cache_key, container, params) - for cache_key, ( - container, - params, - created_at, - ) in self._container_cache.items() - if now - created_at > _CACHE_TTL_SECONDS + for cache_key, (container, params, last_accessed, *_) in self._container_cache.items() + if now - last_accessed > _CACHE_TTL_SECONDS ] for cache_key, container, params in expired: self._container_cache.pop(cache_key, None) diff --git a/litellm/proxy/dev_config.yaml b/litellm/proxy/dev_config.yaml index e437ed7a118..6078161c780 100644 --- a/litellm/proxy/dev_config.yaml +++ b/litellm/proxy/dev_config.yaml @@ -182,10 +182,25 @@ model_list: litellm_params: model: openai/gpt-5.5 api_key: os.environ/OPENAI_API_KEY + - model_name: gpt-4o-mini + litellm_params: + model: openai/gpt-4o-mini + api_key: os.environ/OPENAI_API_KEY general_settings: master_key: sk-1234 +sandbox_tools: + - sandbox_tool_name: e2b_sandbox + litellm_params: + sandbox_provider: e2b + api_key: os.environ/E2B_API_KEY + litellm_settings: drop_params: True telemetry: False + code_interpreter_interception_params: + enabled: true + sandbox_tool_name: e2b_sandbox + callbacks: + - code_interpreter_interception diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 0ffb0337545..6f5c82530e3 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -153,6 +153,7 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS = ( "_code_interpreter_interception_active", "_code_interpreter_interception_converted_stream", "_code_interpreter_interception_sandbox_key", + "_code_interpreter_interception_session_scoped", "max_agentic_loops", ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index dff4e4af89e..4f0c0c21bce 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3062,6 +3062,7 @@ agentic_loop_internal_litellm_params = [ "max_agentic_loops", "_code_interpreter_interception_active", "_code_interpreter_interception_sandbox_key", + "_code_interpreter_interception_session_scoped", "_code_interpreter_interception_converted_stream", ] diff --git a/qa_sticky_session.sh b/qa_sticky_session.sh new file mode 100755 index 00000000000..326bb8117c7 --- /dev/null +++ b/qa_sticky_session.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# QA: code interpreter sandbox stickiness via metadata.session_id +# bash qa_sticky_session.sh +# LITELLM_BASE_URL=http://localhost:4000 LITELLM_KEY=sk-1234 bash qa_sticky_session.sh + +set -euo pipefail + +BASE="${LITELLM_BASE_URL:-http://localhost:4000}" +KEY="${LITELLM_KEY:-sk-1234}" +MODEL="${LITELLM_MODEL:-gpt-4o-mini}" +# proxy running at http://localhost:4000 (master key: sk-1234) +SESSION_A="qa-session-$(date +%s)-A" +SESSION_B="qa-session-$(date +%s)-B" + +content() { + echo "$1" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('choices',[{}])[0].get('message',{}).get('content',''))" +} + +call() { + local session="${1:-}" code="$2" meta="" + [[ -n "$session" ]] && meta=", \"metadata\": {\"session_id\": \"$session\"}" + curl -s -X POST "$BASE/chat/completions" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $KEY" \ + -d "{\"model\":\"$MODEL\"$meta,\"tools\":[{\"type\":\"code_interpreter\"}],\"messages\":[{\"role\":\"user\",\"content\":\"Run this Python code and tell me the result: $code\"}]}" +} + +assert_match() { + local label="$1" body="$2" pattern="$3" + if echo "$body" | grep -qiE "$pattern"; then + echo "PASS $label" + else + echo "FAIL $label (expected /$pattern/)" + echo " $(content "$body")" + exit 1 + fi +} + +echo "=== Sticky Session Sandbox QA ===" +echo "base: $BASE session A: $SESSION_A session B: $SESSION_B" +echo + +R=$(call "$SESSION_A" "x = 42; print(x)") +assert_match "same session_id reuses sandbox (set x=42)" "$R" "42" + +R=$(call "$SESSION_A" "print(x)") +assert_match "same session_id keeps state (x still 42)" "$R" "42" + +R=$(call "$SESSION_B" "print(x)") +assert_match "different session_id is isolated" "$R" "not defined|NameError|undefined|error" + +R=$(call "" "y = 99; print(y)") +assert_match "no session_id runs code" "$R" "99" + +R=$(call "" "print(y)") +assert_match "no session_id gets fresh sandbox each request" "$R" "not defined|NameError|undefined|error" + +echo +echo "All checks passed." diff --git a/tests/test_litellm/integrations/code_interpreter_interception/test_handler.py b/tests/test_litellm/integrations/code_interpreter_interception/test_handler.py index 7ff58ba6324..9d308ac1989 100644 --- a/tests/test_litellm/integrations/code_interpreter_interception/test_handler.py +++ b/tests/test_litellm/integrations/code_interpreter_interception/test_handler.py @@ -14,6 +14,7 @@ from litellm.integrations.code_interpreter_interception.handler import ( LITELLM_CODE_EXECUTION_TOOL_NAME, _INTERCEPTION_ACTIVE_KEY as _ACTIVE_KEY, _SANDBOX_KEY, + _SESSION_SCOPED_KEY, ) from litellm.types.integrations.custom_logger import ( CHAT_COMPLETION_AGENTIC_SURFACE, @@ -138,11 +139,7 @@ async def test_build_plan_runs_code_and_feeds_output_back(): assert sandbox.run_calls[0]["code"] == "print(40 + 2)" messages = _iter_messages(plan) - outputs = [ - m - for m in messages - if isinstance(m, dict) and m.get("type") == "function_call_output" - ] + outputs = [m for m in messages if isinstance(m, dict) and m.get("type") == "function_call_output"] assert outputs, "expected a function_call_output item appended" output_item = next(m for m in outputs if m.get("call_id") == "c1") assert "42" in str(output_item["output"]) @@ -160,9 +157,7 @@ async def test_pre_call_converts_code_interpreter_tool(): assert result is not None tools = result["tools"] - assert not any( - t.get("type") == "code_interpreter" for t in tools - ), "code_interpreter tool must be removed" + assert not any(t.get("type") == "code_interpreter" for t in tools), "code_interpreter tool must be removed" names = [t.get("name") or (t.get("function") or {}).get("name") for t in tools] assert LITELLM_CODE_EXECUTION_TOOL_NAME in names @@ -267,9 +262,7 @@ async def test_should_run_detects_only_matching_function_call(): logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox()) active_kwargs = {"_code_interpreter_interception_active": True} - match = FakeResponse( - output=[_function_call_item(name=LITELLM_CODE_EXECUTION_TOOL_NAME)] - ) + match = FakeResponse(output=[_function_call_item(name=LITELLM_CODE_EXECUTION_TOOL_NAME)]) should_run, payload = await logger.async_should_run_agentic_loop( response=match, model="gpt-5", @@ -331,9 +324,7 @@ async def test_container_reused_within_request_via_server_sandbox_key(): **common, ) - assert ( - len(sandbox.create_calls) == 1 - ), "the sandbox is reused across loop iterations sharing one server sandbox key" + assert len(sandbox.create_calls) == 1, "the sandbox is reused across loop iterations sharing one server sandbox key" @pytest.mark.asyncio @@ -372,9 +363,9 @@ async def test_colliding_caller_call_id_does_not_share_sandbox(): **common, ) - assert ( - len(sandbox.create_calls) == 2 - ), "distinct server sandbox keys must isolate sandboxes despite a colliding call id" + assert len(sandbox.create_calls) == 2, ( + "distinct server sandbox keys must isolate sandboxes despite a colliding call id" + ) @pytest.mark.asyncio @@ -479,14 +470,11 @@ async def test_post_hook_injects_code_interpreter_call_matching_openai_shape(): ) response = FakeResponse(output=[{"type": "message", "content": []}]) - out = await logger.async_post_agentic_loop_response_hook( - response=response, plan=plan, kwargs={} - ) + out = await logger.async_post_agentic_loop_response_hook(response=response, plan=plan, kwargs={}) types = [item.get("type") for item in out.output] assert types == ["code_interpreter_call", "message"], ( - "code_interpreter_call must be re-injected before the message, matching " - "OpenAI's native output ordering" + "code_interpreter_call must be re-injected before the message, matching OpenAI's native output ordering" ) assert set(out.output[0].keys()) == { "id", @@ -524,8 +512,7 @@ async def test_pre_call_forces_non_stream_for_loop(): assert out is not None assert out["stream"] is False, "loop requires a non-streaming upstream call" assert out["_code_interpreter_interception_converted_stream"] is True, ( - "the converted-stream flag must be set so the final response is wrapped " - "back into a stream for the caller" + "the converted-stream flag must be set so the final response is wrapped back into a stream for the caller" ) @@ -556,9 +543,7 @@ async def test_gate_refuses_without_server_active_marker(): """A forged litellm_code_execution call must not trigger the loop unless the pre-call hook actually converted a native code_interpreter tool.""" logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox()) - forged = FakeResponse( - output=[_function_call_item(name=LITELLM_CODE_EXECUTION_TOOL_NAME)] - ) + forged = FakeResponse(output=[_function_call_item(name=LITELLM_CODE_EXECUTION_TOOL_NAME)]) should_run, payload = await logger.async_should_run_agentic_loop( response=forged, @@ -577,12 +562,8 @@ async def test_gate_refuses_without_server_active_marker(): @pytest.mark.asyncio async def test_gate_rechecks_provider_scope(): """enabled_providers must be re-enforced at the gate, not only in pre-call.""" - logger = CodeInterpreterInterceptionLogger( - sandbox_config=FakeSandbox(), enabled_providers=["openai"] - ) - response = FakeResponse( - output=[_function_call_item(name=LITELLM_CODE_EXECUTION_TOOL_NAME)] - ) + logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox(), enabled_providers=["openai"]) + response = FakeResponse(output=[_function_call_item(name=LITELLM_CODE_EXECUTION_TOOL_NAME)]) should_run, _ = await logger.async_should_run_agentic_loop( response=response, @@ -600,11 +581,7 @@ async def test_gate_rechecks_provider_scope(): @pytest.mark.asyncio async def test_chat_completion_gate_detects_code_execution_tool_call(): logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox()) - response = { - "choices": [ - {"message": {"tool_calls": [_chat_function_call_item(call_id="call_123")]}} - ] - } + response = {"choices": [{"message": {"tool_calls": [_chat_function_call_item(call_id="call_123")]}}]} should_run, payload = await logger.async_should_run_agentic_loop( response=response, @@ -661,9 +638,7 @@ async def test_chat_completion_build_plan_runs_code_and_appends_tool_message(): }, model="gpt-5", messages=[{"role": "user", "content": "x"}], - response={ - "choices": [{"message": {"tool_calls": [_chat_function_call_item()]}}] - }, + response={"choices": [{"message": {"tool_calls": [_chat_function_call_item()]}}]}, anthropic_messages_provider_config=None, anthropic_messages_optional_request_params={ "tools": [native_chat_tool], @@ -738,8 +713,7 @@ async def test_pre_call_strips_client_forged_marker_on_initial_request(): await logger.async_pre_call_deployment_hook(kwargs, CallTypes.aresponses) assert _ACTIVE_KEY not in kwargs, ( - "no native code_interpreter tool was present, so a client-supplied " - "active marker must be cleared" + "no native code_interpreter tool was present, so a client-supplied active marker must be cleared" ) assert kwargs["litellm_metadata"] == {"safe_user_value": "kept"} @@ -774,8 +748,7 @@ async def test_pre_call_strips_forged_loop_controls_then_mints_own_markers(): assert metadata[_ACTIVE_KEY] is True assert metadata[_SANDBOX_KEY] == result[_SANDBOX_KEY] assert metadata[_SANDBOX_KEY] != "client-forged", ( - "the surviving sandbox key must be the server-minted one, not the forged " - "value the client supplied" + "the surviving sandbox key must be the server-minted one, not the forged value the client supplied" ) @@ -793,8 +766,7 @@ async def test_pre_call_preserves_marker_on_server_followup(): await logger.async_pre_call_deployment_hook(kwargs, CallTypes.aresponses) assert kwargs.get(_ACTIVE_KEY) is True, ( - "the server-set marker must survive followup requests so multi-round " - "code execution keeps working" + "the server-set marker must survive followup requests so multi-round code execution keeps working" ) @@ -805,9 +777,7 @@ async def test_sandbox_deleted_after_loop_completes(): plan = await _build_plan(logger, sandbox, call_id="k1") assert sandbox.create_calls, "sandbox must be created during the loop" - assert ( - not sandbox.delete_calls - ), "sandbox must outlive the loop until the final hook" + assert not sandbox.delete_calls, "sandbox must outlive the loop until the final hook" await logger.async_post_agentic_loop_response_hook( response=FakeResponse(output=[{"type": "message", "content": []}]), @@ -816,8 +786,7 @@ async def test_sandbox_deleted_after_loop_completes(): ) assert len(sandbox.delete_calls) == 1, ( - "the sandbox must be deleted once the final response is assembled, " - "otherwise it keeps running and billing" + "the sandbox must be deleted once the final response is assembled, otherwise it keeps running and billing" ) assert "sbxkey1" not in logger._container_cache @@ -829,16 +798,11 @@ async def test_post_hook_delete_is_idempotent_across_loop_levels(): plan = await _build_plan(logger, sandbox, call_id="k1") response = FakeResponse(output=[{"type": "message", "content": []}]) - await logger.async_post_agentic_loop_response_hook( - response=response, plan=plan, kwargs={} - ) - await logger.async_post_agentic_loop_response_hook( - response=response, plan=plan, kwargs={} - ) + await logger.async_post_agentic_loop_response_hook(response=response, plan=plan, kwargs={}) + await logger.async_post_agentic_loop_response_hook(response=response, plan=plan, kwargs={}) assert len(sandbox.delete_calls) == 1, ( - "deleting an already-removed container must be a no-op so unwinding " - "loop levels do not double-delete" + "deleting an already-removed container must be a no-op so unwinding loop levels do not double-delete" ) @@ -860,8 +824,7 @@ async def test_build_plan_deletes_sandbox_when_execution_raises(): assert len(sandbox.create_calls) == 1, "the sandbox must have been created" assert len(sandbox.delete_calls) == 1, ( - "a build failure must delete the cached sandbox so it does not keep " - "running and billing" + "a build failure must delete the cached sandbox so it does not keep running and billing" ) assert "sbxkey1" not in logger._container_cache @@ -875,8 +838,7 @@ async def test_cleanup_hook_deletes_sandbox(): await logger.async_agentic_loop_cleanup_hook(plan=plan, kwargs={}) assert len(sandbox.delete_calls) == 1, ( - "the cleanup hook must delete the sandbox so a rerun failure cannot " - "leak a running container" + "the cleanup hook must delete the sandbox so a rerun failure cannot leak a running container" ) assert "sbxkey1" not in logger._container_cache @@ -895,8 +857,7 @@ async def test_cleanup_hook_is_idempotent_with_post_hook(): await logger.async_agentic_loop_cleanup_hook(plan=plan, kwargs={}) assert len(sandbox.delete_calls) == 1, ( - "cleanup running in finally after the success-path post hook already " - "deleted the sandbox must not double-delete" + "cleanup running in finally after the success-path post hook already deleted the sandbox must not double-delete" ) @@ -923,9 +884,7 @@ async def test_responses_plan_cleans_up_sandbox_when_followup_raises(): plan = AgenticLoopPlan( run_agentic_loop=True, - request_patch=AgenticLoopRequestPatch( - model="gpt-5", messages=[{"role": "user", "content": "x"}] - ), + request_patch=AgenticLoopRequestPatch(model="gpt-5", messages=[{"role": "user", "content": "x"}]), metadata={"sandbox_key": "sbxkey1"}, ) @@ -995,9 +954,7 @@ async def test_run_code_does_not_re_resolve_registry(monkeypatch): sandbox_tools.clear_sandbox_tools() - stdout = await logger._run_tool_call( - container=container, params=params, arguments='{"code":"print(1)"}' - ) + stdout = await logger._run_tool_call(container=container, params=params, arguments='{"code":"print(1)"}') finally: sandbox_tools.clear_sandbox_tools() @@ -1013,9 +970,7 @@ async def test_run_tool_call_surfaces_execution_error(): class ErroringSandbox(FakeSandbox): async def arun_code(self, *, container, code, **kwargs): self.run_calls.append({"container": container, "code": code}) - return CodeExecutionResult( - stdout="", error={"name": "ValueError", "value": "boom"} - ) + return CodeExecutionResult(stdout="", error={"name": "ValueError", "value": "boom"}) sandbox = ErroringSandbox() logger = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) @@ -1036,9 +991,7 @@ async def test_run_tool_call_reports_unparseable_arguments(): logger = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) container = await logger._create_container() - stdout = await logger._run_tool_call( - container=container[0], params=None, arguments="not-json" - ) + stdout = await logger._run_tool_call(container=container[0], params=None, arguments="not-json") assert stdout == "[invalid tool arguments: could not parse code]" assert not sandbox.run_calls, "code must not run when arguments cannot be parsed" @@ -1048,9 +1001,7 @@ async def test_run_tool_call_reports_unparseable_arguments(): async def test_pre_call_skips_provider_outside_scope(): """enabled_providers must filter the pre-call conversion so a request to an out-of-scope provider is left untouched.""" - logger = CodeInterpreterInterceptionLogger( - sandbox_config=FakeSandbox(), enabled_providers=["openai"] - ) + logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox(), enabled_providers=["openai"]) kwargs = { "tools": [{"type": "code_interpreter", "container": {"type": "auto"}}], "custom_llm_provider": "anthropic", @@ -1119,6 +1070,7 @@ async def test_prune_expired_cache_deletes_underlying_container(): container, params, time.time() - handler_mod._CACHE_TTL_SECONDS - 1, + None, ) await logger._prune_expired_cache() @@ -1217,3 +1169,258 @@ async def test_extract_tool_calls_reads_object_attributes(): assert len(calls) == 1 assert calls[0]["call_id"] == "c9" assert calls[0]["arguments"] == '{"code":"print(1)"}' + + +# --------------------------------------------------------------------------- +# Sticky session tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_pre_call_uses_session_id_from_metadata_as_sandbox_key(): + """When session_id is in request metadata, it becomes the sandbox key so the + container is shared across requests in the same session.""" + logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox()) + session_id = "conv-abc-123" + kwargs = { + "tools": [{"type": "code_interpreter", "container": {"type": "auto"}}], + "custom_llm_provider": "openai", + "metadata": {"session_id": session_id}, + } + + result = await logger.async_pre_call_deployment_hook(kwargs, CallTypes.acompletion) + + assert result is not None + assert result[_SANDBOX_KEY] == session_id + assert result[_SESSION_SCOPED_KEY] is True + assert result["litellm_metadata"][_SANDBOX_KEY] == session_id + assert result["litellm_metadata"][_SESSION_SCOPED_KEY] is True + + +@pytest.mark.asyncio +async def test_pre_call_uses_session_id_from_litellm_metadata(): + """session_id in litellm_metadata also works as the sticky key.""" + logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox()) + session_id = "sess-xyz-789" + kwargs = { + "tools": [{"type": "code_interpreter", "container": {"type": "auto"}}], + "custom_llm_provider": "openai", + "litellm_metadata": {"session_id": session_id}, + } + + result = await logger.async_pre_call_deployment_hook(kwargs, CallTypes.acompletion) + + assert result is not None + assert result[_SANDBOX_KEY] == session_id + assert result[_SESSION_SCOPED_KEY] is True + + +@pytest.mark.asyncio +async def test_pre_call_without_session_id_still_mints_random_key(): + """Requests without a session_id still get a server-minted random sandbox key.""" + logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox()) + kwargs = { + "tools": [{"type": "code_interpreter", "container": {"type": "auto"}}], + "custom_llm_provider": "openai", + } + + result = await logger.async_pre_call_deployment_hook(kwargs, CallTypes.acompletion) + + assert result is not None + assert _SESSION_SCOPED_KEY not in result or result[_SESSION_SCOPED_KEY] is False + assert len(result[_SANDBOX_KEY]) >= 16 + + +@pytest.mark.asyncio +async def test_session_scoped_sandbox_survives_agentic_loop_cleanup(): + """A session-scoped sandbox must NOT be deleted by the cleanup or post hooks; + it needs to persist across requests within the same session.""" + sandbox = FakeSandbox(stdout="42") + logger = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) + session_id = "conv-persist-me" + + plan = await logger.async_build_agentic_loop_plan( + tools={ + "tool_calls": [ + { + "call_id": "c1", + "name": LITELLM_CODE_EXECUTION_TOOL_NAME, + "arguments": '{"code":"x = 10"}', + } + ] + }, + model="gpt-4o-mini", + messages=[{"role": "user", "content": "set x"}], + response=FakeResponse(output=[_function_call_item()]), + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={"tools": []}, + logging_obj=FakeLogging(litellm_call_id="k1"), + stream=False, + kwargs={ + "litellm_call_id": "k1", + _SANDBOX_KEY: session_id, + _SESSION_SCOPED_KEY: True, + }, + ) + + assert plan.metadata["is_session_scoped"] is True + + await logger.async_post_agentic_loop_response_hook( + response=FakeResponse(output=[{"type": "message", "content": []}]), + plan=plan, + kwargs={}, + ) + await logger.async_agentic_loop_cleanup_hook(plan=plan, kwargs={}) + + assert not sandbox.delete_calls, ( + "session-scoped sandbox must not be deleted after a single agentic loop; " + "it must persist for the next request in the session" + ) + assert session_id in logger._container_cache, "session-scoped container must remain in cache after loop ends" + + +@pytest.mark.asyncio +async def test_session_scoped_sandbox_reused_across_sequential_requests(): + """Two sequential requests with the same session_id must share one container, + confirming state (e.g. assigned variables) can persist across HTTP requests.""" + sandbox = FakeSandbox(stdout="42") + logger = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) + session_id = "conv-reuse-me" + + common_plan_args = dict( + tools={ + "tool_calls": [ + { + "call_id": "c1", + "name": LITELLM_CODE_EXECUTION_TOOL_NAME, + "arguments": '{"code":"print(1)"}', + } + ] + }, + model="gpt-4o-mini", + messages=[{"role": "user", "content": "x"}], + response=FakeResponse(output=[_function_call_item()]), + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={"tools": []}, + stream=False, + ) + session_kwargs = {_SANDBOX_KEY: session_id, _SESSION_SCOPED_KEY: True} + + plan1 = await logger.async_build_agentic_loop_plan( + logging_obj=FakeLogging(litellm_call_id="req1"), + kwargs={"litellm_call_id": "req1", **session_kwargs}, + **common_plan_args, + ) + await logger.async_post_agentic_loop_response_hook( + response=FakeResponse(output=[{"type": "message", "content": []}]), + plan=plan1, + kwargs={}, + ) + + plan2 = await logger.async_build_agentic_loop_plan( + logging_obj=FakeLogging(litellm_call_id="req2"), + kwargs={"litellm_call_id": "req2", **session_kwargs}, + **common_plan_args, + ) + await logger.async_post_agentic_loop_response_hook( + response=FakeResponse(output=[{"type": "message", "content": []}]), + plan=plan2, + kwargs={}, + ) + + assert len(sandbox.create_calls) == 1, ( + "a single container must serve both requests in the same session; " + "two creates means state cannot persist between requests" + ) + assert len(sandbox.delete_calls) == 0, "the session container must still be alive after both requests complete" + + +@pytest.mark.asyncio +async def test_non_session_sandbox_still_deleted_after_loop(): + """Without a session_id, the existing per-request ephemeral behavior is unchanged.""" + sandbox = FakeSandbox(stdout="42") + logger = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) + + plan = await _build_plan(logger, sandbox, call_id="k1") + await logger.async_post_agentic_loop_response_hook( + response=FakeResponse(output=[{"type": "message", "content": []}]), + plan=plan, + kwargs={}, + ) + + assert len(sandbox.delete_calls) == 1, "non-session sandbox must still be cleaned up after each request" + + +@pytest.mark.asyncio +async def test_sandbox_key_scoped_to_api_key_hash_isolates_users(): + """Two callers supplying the same session_id but different API key hashes must + each get their own sandbox; sharing across tenants would let one read or mutate + the other's interpreter state.""" + logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox()) + session_id = "same-session-id" + + result_a = await logger.async_pre_call_deployment_hook( + { + "tools": [{"type": "code_interpreter"}], + "custom_llm_provider": "openai", + "metadata": {"session_id": session_id}, + "user_api_key_hash": "hash-for-tenant-a", + }, + CallTypes.acompletion, + ) + result_b = await logger.async_pre_call_deployment_hook( + { + "tools": [{"type": "code_interpreter"}], + "custom_llm_provider": "openai", + "metadata": {"session_id": session_id}, + "user_api_key_hash": "hash-for-tenant-b", + }, + CallTypes.acompletion, + ) + + assert result_a is not None and result_b is not None + assert result_a[_SANDBOX_KEY] != result_b[_SANDBOX_KEY], ( + "same session_id from different API keys must yield different sandbox keys; " + "otherwise tenant A can read tenant B's sandbox state" + ) + assert "hash-for-tenant-a" in result_a[_SANDBOX_KEY] + assert "hash-for-tenant-b" in result_b[_SANDBOX_KEY] + + +@pytest.mark.asyncio +async def test_per_identity_cap_evicts_lru_session(): + """When a single identity holds the cap limit of session sandboxes and opens a + new one, the least-recently-used session is evicted so the allocation stays + bounded. Without this, rotating session IDs is an unbounded sandbox leak.""" + from litellm.integrations.code_interpreter_interception.handler import _SESSION_SCOPED_PER_IDENTITY_CAP + + sandbox = FakeSandbox(stdout="ok") + logger = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) + identity = "hash-for-identity-x" + + for i in range(_SESSION_SCOPED_PER_IDENTITY_CAP): + await logger._get_or_create_container( + cache_key=f"{identity}:session-{i}", + identity=identity, + ) + logger._container_cache[f"{identity}:session-{i}"] = ( + logger._container_cache[f"{identity}:session-{i}"][0], + logger._container_cache[f"{identity}:session-{i}"][1], + float(i), + identity, + ) + + assert len(logger._container_cache) == _SESSION_SCOPED_PER_IDENTITY_CAP + + await logger._get_or_create_container( + cache_key=f"{identity}:session-new", + identity=identity, + ) + + assert len(logger._container_cache) == _SESSION_SCOPED_PER_IDENTITY_CAP, ( + "adding a new session beyond the cap must evict one entry so total stays bounded" + ) + assert f"{identity}:session-0" not in logger._container_cache, ( + "the entry with the oldest last_accessed timestamp must be evicted first (LRU)" + ) + assert len(sandbox.delete_calls) == 1, "evicted sandbox must be deleted, not just removed from cache" From 846dbecbf2bcf3e11c60f56971c16c21f13f40fd Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 30 Jun 2026 19:02:00 -0700 Subject: [PATCH 13/81] feat(proxy): support object_permission in default_key_generate_params (#31776) * feat(proxy): support object_permission in default_key_generate_params default_key_generate_params filled in a fixed whitelist of scalar fields plus a full-replace for models/metadata, but never touched object_permission, so admins had no way to set a default (e.g. mcp_tool_search_enabled, vector_stores) applied to every new key. Merge object_permission field-by-field instead of replacing it wholesale, so a caller-supplied field (e.g. mcp_servers) is preserved alongside defaulted fields the caller left unset. * ci: retrigger proxy_pass_through_endpoint_tests (suspected flake, unrelated to this PR's diff) * fix(proxy): apply default object_permission after team-scope validation Injecting the default before validate_key_vector_stores_against_team / validate_key_search_tools_against_team ran meant a default containing a team-scoped field (e.g. vector_stores) looked like a caller-requested permission, turning ordinary non-admin personal key creation into a 403. Merge the default into data_json after those checks instead, and guard against a non-dict default value. --- .../key_management_endpoints.py | 17 ++ .../test_key_management_endpoints.py | 258 ++++++++++++++++++ 2 files changed, 275 insertions(+) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 4106eae606c..523cb9b74e5 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -965,6 +965,23 @@ async def _common_key_generation_helper( is_proxy_admin=_is_proxy_admin_caller, ) + # Merge default_key_generate_params.object_permission in *after* the team-scope + # checks above, so an admin-configured default (e.g. vector_stores, search_tools) + # is never mistaken for a caller-requested permission and rejected by those + # non-admin/no-team checks. Only fields the caller left unset are filled in. + _default_object_permission = ( + litellm.default_key_generate_params.get("object_permission") + if litellm.default_key_generate_params is not None + else None + ) + if isinstance(_default_object_permission, dict): + _caller_object_permission = data_json.get("object_permission") + if _caller_object_permission is None: + data_json["object_permission"] = dict(_default_object_permission) + elif isinstance(_caller_object_permission, dict): + for _op_field, _op_default_value in _default_object_permission.items(): + _caller_object_permission.setdefault(_op_field, _op_default_value) + data_json = await _set_object_permission( data_json=data_json, prisma_client=prisma_client, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 04048020e18..ae62be8799f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -7781,6 +7781,264 @@ async def test_default_key_generate_params_duration(monkeypatch): litellm.default_key_generate_params = original_value +async def test_default_key_generate_params_object_permission_applied_when_absent( + monkeypatch, +): + """ + default_key_generate_params.object_permission is applied to a key that + doesn't specify object_permission at all. + """ + import litellm + + mock_prisma_client = AsyncMock() + mock_insert_data = AsyncMock( + return_value=MagicMock( + token="hashed_token_123", litellm_budget_table=None, object_permission=None + ) + ) + mock_prisma_client.insert_data = mock_insert_data + mock_prisma_client.db = MagicMock() + mock_prisma_client.db.litellm_verificationtoken = MagicMock() + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=None + ) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[] + ) + mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) + mock_prisma_client.db.litellm_verificationtoken.update = AsyncMock( + return_value=MagicMock( + token="hashed_token_123", litellm_budget_table=None, object_permission=None + ) + ) + mock_prisma_client.db.litellm_objectpermissiontable = MagicMock() + mock_prisma_client.db.litellm_objectpermissiontable.create = AsyncMock( + return_value=MagicMock(object_permission_id="objperm-1") + ) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + original_value = litellm.default_key_generate_params + litellm.default_key_generate_params = { + "object_permission": {"vector_stores": ["default-vs"]} + } + + try: + request = GenerateKeyRequest() # No object_permission specified + await _common_key_generation_helper( + data=request, + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="1234", + ), + litellm_changed_by=None, + team_table=None, + ) + + created_data = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"] + assert created_data["vector_stores"] == ["default-vs"] + finally: + litellm.default_key_generate_params = original_value + + +async def test_default_key_generate_params_object_permission_merges_partial( + monkeypatch, +): + """ + default_key_generate_params.object_permission fills only the fields the + caller left unset - an explicitly supplied field (agents here) is + preserved alongside the defaulted field (vector_stores). + """ + import litellm + from litellm.proxy._types import LiteLLM_ObjectPermissionBase + + mock_prisma_client = AsyncMock() + mock_insert_data = AsyncMock( + return_value=MagicMock( + token="hashed_token_123", litellm_budget_table=None, object_permission=None + ) + ) + mock_prisma_client.insert_data = mock_insert_data + mock_prisma_client.db = MagicMock() + mock_prisma_client.db.litellm_verificationtoken = MagicMock() + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=None + ) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[] + ) + mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) + mock_prisma_client.db.litellm_verificationtoken.update = AsyncMock( + return_value=MagicMock( + token="hashed_token_123", litellm_budget_table=None, object_permission=None + ) + ) + mock_prisma_client.db.litellm_objectpermissiontable = MagicMock() + mock_prisma_client.db.litellm_objectpermissiontable.create = AsyncMock( + return_value=MagicMock(object_permission_id="objperm-2") + ) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + original_value = litellm.default_key_generate_params + litellm.default_key_generate_params = { + "object_permission": {"vector_stores": ["default-vs"]} + } + + try: + request = GenerateKeyRequest( + object_permission=LiteLLM_ObjectPermissionBase(agents=["agent-1"]) + ) + await _common_key_generation_helper( + data=request, + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="1234", + ), + litellm_changed_by=None, + team_table=None, + ) + + created_data = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"] + assert created_data["agents"] == ["agent-1"] + assert created_data["vector_stores"] == ["default-vs"] + finally: + litellm.default_key_generate_params = original_value + + +async def test_default_key_generate_params_object_permission_does_not_override_explicit( + monkeypatch, +): + """ + A field the caller explicitly set on object_permission must win over the + same field in default_key_generate_params. + """ + import litellm + from litellm.proxy._types import LiteLLM_ObjectPermissionBase + + mock_prisma_client = AsyncMock() + mock_insert_data = AsyncMock( + return_value=MagicMock( + token="hashed_token_123", litellm_budget_table=None, object_permission=None + ) + ) + mock_prisma_client.insert_data = mock_insert_data + mock_prisma_client.db = MagicMock() + mock_prisma_client.db.litellm_verificationtoken = MagicMock() + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=None + ) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[] + ) + mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) + mock_prisma_client.db.litellm_verificationtoken.update = AsyncMock( + return_value=MagicMock( + token="hashed_token_123", litellm_budget_table=None, object_permission=None + ) + ) + mock_prisma_client.db.litellm_objectpermissiontable = MagicMock() + mock_prisma_client.db.litellm_objectpermissiontable.create = AsyncMock( + return_value=MagicMock(object_permission_id="objperm-3") + ) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + original_value = litellm.default_key_generate_params + litellm.default_key_generate_params = { + "object_permission": {"vector_stores": ["default-vs"]} + } + + try: + request = GenerateKeyRequest( + object_permission=LiteLLM_ObjectPermissionBase( + vector_stores=["explicit-vs"] + ) + ) + await _common_key_generation_helper( + data=request, + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="1234", + ), + litellm_changed_by=None, + team_table=None, + ) + + created_data = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"] + assert created_data["vector_stores"] == ["explicit-vs"] + finally: + litellm.default_key_generate_params = original_value + + +async def test_default_key_generate_params_object_permission_not_rejected_for_non_admin_personal_key( + monkeypatch, +): + """ + Regression test: a default_key_generate_params.object_permission containing + a team-scoped field (vector_stores) must not turn ordinary non-admin + personal key creation into a 403. The default is merged in *after* the + caller-scope validation, so it is never mistaken for a caller-requested + permission. + """ + import litellm + + mock_prisma_client = AsyncMock() + mock_insert_data = AsyncMock( + return_value=MagicMock( + token="hashed_token_123", litellm_budget_table=None, object_permission=None + ) + ) + mock_prisma_client.insert_data = mock_insert_data + mock_prisma_client.db = MagicMock() + mock_prisma_client.db.litellm_verificationtoken = MagicMock() + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=None + ) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[] + ) + mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) + mock_prisma_client.db.litellm_verificationtoken.update = AsyncMock( + return_value=MagicMock( + token="hashed_token_123", litellm_budget_table=None, object_permission=None + ) + ) + mock_prisma_client.db.litellm_objectpermissiontable = MagicMock() + mock_prisma_client.db.litellm_objectpermissiontable.create = AsyncMock( + return_value=MagicMock(object_permission_id="objperm-4") + ) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + original_value = litellm.default_key_generate_params + litellm.default_key_generate_params = { + "object_permission": {"vector_stores": ["default-vs"]} + } + + try: + request = GenerateKeyRequest(user_id="alice") # No object_permission specified + response = await _common_key_generation_helper( + data=request, + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + api_key="sk-alice", + user_id="alice", + ), + litellm_changed_by=None, + team_table=None, + ) + + assert response is not None + created_data = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"] + assert created_data["vector_stores"] == ["default-vs"] + finally: + litellm.default_key_generate_params = original_value + + @pytest.mark.asyncio async def test_build_key_filter_member_team_service_accounts(): """ From 50b936c75e8d7066968b8b6e31b73969d95790ae Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 30 Jun 2026 19:19:34 -0700 Subject: [PATCH 14/81] feat(guardrails/headroom): add CCR (compress-cache-retrieve) via agentic loop (#31681) * feat(guardrails/headroom): add CCR (compress-cache-retrieve) support via agentic loop When Headroom's /v1/compress returns messages containing hash markers (hash=[a-f0-9]{24}), inject a headroom_retrieve tool into the request. When the LLM calls that tool, intercept via async_should_run_agentic_loop and async_build_agentic_loop_plan, call GET /v1/retrieve/{hash} on the Headroom sidecar, and replay the LLM with the original content as a tool result -- all transparent to the caller. * style: run ruff format on headroom guardrail and tests * fix(guardrails/headroom): detect headroom_retrieve calls in both OpenAI and Anthropic response formats * test(guardrails/headroom): add test for Anthropic content block format detection in CCR loop * ci: trigger CI checks * fix(guardrails/headroom): replace List/Dict with list/dict to fix UP006 ruff violations * fix(guardrails/headroom): replace except Exception with except ValueError to fix BLE001 * fix(guardrails/headroom): add Responses API output format detection for CCR tool calls * refactor(guardrails/headroom): extract format-specific helpers to fix C901 complexity * fix(guardrails/headroom): scope CCR retrieval to hashes produced by current request Previously any LLM-supplied hash in a headroom_retrieve tool call was forwarded to the Headroom retrieve API, letting a crafted tool call fetch arbitrary cached content. Validate the hash against the set produced by compressing the current request's messages before calling retrieve. * fix(guardrails/headroom): track issued hashes server-side, fix Responses API replay shape Hash validation now also checks an in-memory cache of hashes actually returned by /v1/compress, not just whether the hash text appears somewhere in the request's messages. The message-text check alone is forgeable: an attacker can plant a hash-shaped string in their own prompt and have it treated as valid. Responses API follow-up now emits function_call/function_call_output items keyed by call_id instead of chat-style assistant/tool messages, since the Responses API does not accept the latter as input. Also fixes call_id/id field priority when extracting tool calls from Responses API output, since call_id (not id) is what must match between the function_call and its output. * fix(guardrails/headroom): drop redundant quoted type annotations UP037 flags quotes on annotations that are already lazily evaluated via `from __future__ import annotations`. * test(guardrails/headroom): add missing pytest.mark.asyncio decorators Functional under asyncio_mode=auto, but every other async test in the file has the decorator for consistency. * fix(guardrails/headroom): scope CCR hashes per call_id, fix Anthropic replay shape Two real gaps found in review: 1. The instance-wide issued-hash cache combined with a message-text check did not actually scope retrieval to the request that produced the hash. A hash issued for request A stays in the shared cache until TTL expiry, and the message-text check is satisfied by any request whose own messages happen to echo that hash string. Request B could plant A's hash in its own prompt and retrieve A's content. Fixed by keying the issued-hash cache by litellm_call_id, matching the pattern already used in compression_interception: a hash is only honored when it was issued under the exact call_id resolving for the current request. 2. The Anthropic Messages replay path fell through to the chat-style assistant/tool-message builder, which Anthropic does not accept. Anthropic requires the tool_use block echoed in an assistant message paired with a tool_result block in a user message, keyed by tool_use_id. Added a dedicated branch for this shape. * docs: note proactive API-fragmentation helper convention Add a bullet to the coding-conventions list: look for or add a shared helper when logic branches on API surface (chat completions vs Anthropic Messages vs Responses API), instead of duplicating format-detection per module. * fix(guardrails/headroom): fix Anthropic tool-shape detection, extract shared cross-API tool util Live e2e testing against the real Anthropic API surfaced two bugs the mocked unit tests couldn't catch because they used MagicMock responses instead of realistic response shapes: 1. has_headroom_retrieve_tool only recognized OpenAI-shaped function tools. By the time an Anthropic Messages response reaches the agentic-loop gate, the tool this guardrail injected has already been transformed into Anthropic's native shape (type: "custom", top-level "name"), so the gate never fired for real Anthropic requests. 2. AnthropicMessagesResponse is a TypedDict, so real responses are plain dicts at runtime, not objects with attribute access. The extractors and format detectors used bare getattr(), which silently returns nothing for dict responses instead of reading the actual key. Extracted the cross-API-surface tool-call extraction and tool-presence check into litellm/litellm_core_utils/prompt_templates/factory.py (get_tool_calls_from_response, has_tool_with_name) so this format fragmentation is handled in one place instead of being duplicated per-guardrail, and reused the existing repair-aware parse_tool_call_arguments from common_utils instead of a naive json.loads. headroom.py now delegates to these shared helpers. Confirmed live against the real Anthropic API: the retrieve loop now fires and successfully retrieves the correct hash's content through the full compress -> tool-call -> retrieve -> replay round-trip. * fix(guardrails/headroom): fix ruff-strict UP006/I001 budget violations Use lowercase list/dict generics in the new factory.py tool-call helpers instead of typing.List/Dict, drop the now-unused Tuple import in headroom.py, and reorder the new factory import ahead of the llms.custom_httpx import to satisfy import sorting. * fix(guardrails/headroom): match Anthropic tools without a type field Anthropic's documented client tool format is just name + input_schema; type: "custom" is only one possible value, not a requirement. Match any non-OpenAI-shaped tool on its top-level name instead of requiring type == "custom". --- CLAUDE.md | 1 + .../prompt_templates/factory.py | 145 +++- .../guardrail_hooks/headroom/headroom.py | 355 ++++++++- tests/llm_translation/test_prompt_factory.py | 99 +++ .../guardrail_hooks/test_headroom.py | 741 +++++++++++++++++- 5 files changed, 1319 insertions(+), 22 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0cd1605b1b2..83bf3e22d27 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -69,6 +69,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega - No monster files or god objects - No file sprawl: deliberate file and folder structure - Standard over hand-rolled: use the official SDK or a library where one exists; where none does, follow industry standards instead of inventing local conventions +- API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module; if none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration Follow conventional commits for commit names and PR titles diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 1f0df51d7de..c2448430387 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -6,7 +6,7 @@ import mimetypes import re import xml.etree.ElementTree as ET from enum import Enum -from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast, overload +from typing import Any, Dict, List, Optional, Set, Tuple, TypedDict, Union, cast, overload from jinja2.sandbox import ImmutableSandboxedEnvironment @@ -5322,3 +5322,146 @@ def get_attribute_or_key(tool_or_function, attribute, default=None): if hasattr(tool_or_function, attribute): return getattr(tool_or_function, attribute) return tool_or_function.get(attribute, default) + + +class NormalizedToolCall(TypedDict): + id: Optional[str] + name: Optional[str] + arguments: dict[str, Any] + + +def _parse_tool_call_arguments(raw: Any, tool_name: Optional[str], context: str) -> dict[str, Any]: + # Anthropic's tool_use blocks already carry a parsed dict in "input"; + # chat completions and the Responses API carry a JSON string that may be + # truncated by the model, so route those through the repair-aware parser. + if isinstance(raw, dict): + return raw + if not isinstance(raw, str): + return {} + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + parse_tool_call_arguments, + ) + + try: + parsed = parse_tool_call_arguments(raw, tool_name=tool_name, context=context) + except ValueError as e: + verbose_logger.warning("Failed to parse tool call arguments: %s", e) + return {} + return parsed if isinstance(parsed, dict) else {} + + +def _tool_calls_from_chat_completion_response(response: Any) -> list[NormalizedToolCall]: + choices = get_attribute_or_key(response, "choices", None) + if not (isinstance(choices, list) and choices): + return [] + message = get_attribute_or_key(choices[0], "message", None) + tool_calls = get_attribute_or_key(message, "tool_calls", None) if message else None + if not isinstance(tool_calls, list): + return [] + result: list[NormalizedToolCall] = [] + for tc in tool_calls: + fn = get_attribute_or_key(tc, "function", None) + if fn is None: + continue + name = get_attribute_or_key(fn, "name") + result.append( + NormalizedToolCall( + id=get_attribute_or_key(tc, "id"), + name=name, + arguments=_parse_tool_call_arguments( + get_attribute_or_key(fn, "arguments", "{}"), + tool_name=name, + context="chat completions", + ), + ) + ) + return result + + +def _tool_calls_from_responses_api_response(response: Any) -> list[NormalizedToolCall]: + output = get_attribute_or_key(response, "output", None) + if not isinstance(output, list): + return [] + result: list[NormalizedToolCall] = [] + for item in output: + if get_attribute_or_key(item, "type") != "function_call": + continue + name = get_attribute_or_key(item, "name") + result.append( + NormalizedToolCall( + id=get_attribute_or_key(item, "call_id") or get_attribute_or_key(item, "id"), + name=name, + arguments=_parse_tool_call_arguments( + get_attribute_or_key(item, "arguments", "{}"), + tool_name=name, + context="responses API", + ), + ) + ) + return result + + +def _tool_calls_from_anthropic_messages_response(response: Any) -> list[NormalizedToolCall]: + content = get_attribute_or_key(response, "content", None) + if not isinstance(content, list): + return [] + result: list[NormalizedToolCall] = [] + for block in content: + if get_attribute_or_key(block, "type") != "tool_use": + continue + raw_input = get_attribute_or_key(block, "input", {}) + result.append( + NormalizedToolCall( + id=get_attribute_or_key(block, "id"), + name=get_attribute_or_key(block, "name"), + arguments=raw_input if isinstance(raw_input, dict) else {}, + ) + ) + return result + + +def get_tool_calls_from_response(response: Any) -> list[NormalizedToolCall]: + """ + Extract tool/function calls from a response object into a normalized + ``{"id", "name", "arguments"}`` shape, regardless of which API surface + produced it: chat completions (``choices[].message.tool_calls``), + the Responses API (``output`` items of type ``function_call``), or the + Anthropic Messages API (``content`` blocks of type ``tool_use``). + + Callers that only care about a specific tool should filter the result by + ``name`` themselves -- this returns every tool call found. + """ + for extractor in ( + _tool_calls_from_chat_completion_response, + _tool_calls_from_responses_api_response, + _tool_calls_from_anthropic_messages_response, + ): + tool_calls = extractor(response) + if tool_calls: + return tool_calls + return [] + + +def has_tool_with_name(tools: Any, tool_name: str) -> bool: + """ + Check whether a tools list (as sent to an LLM) includes a tool with the + given name, regardless of shape: OpenAI-style function tools + (``{"type": "function", "function": {"name": ...}}``) or Anthropic's + native tool shape (a top-level ``"name"``, e.g. + ``{"name": ..., "input_schema": ...}``). Anthropic's documented client + tool format doesn't require a ``"type"`` key at all -- ``"custom"`` is + only one of several possible values -- so any non-OpenAI-shaped tool is + matched on its top-level ``"name"``. + """ + if not isinstance(tools, list): + return False + for tool in tools: + if not isinstance(tool, dict): + continue + function = tool.get("function") + if tool.get("type") == "function" and isinstance(function, dict): + if function.get("name") == tool_name: + return True + elif tool.get("name") == tool_name: + return True + return False diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index 2228ccf3997..4badb48e2eb 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -1,6 +1,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Literal +import json +import re +import time +import uuid +from typing import TYPE_CHECKING, Any, Literal, Optional import httpx from fastapi import HTTPException @@ -12,12 +16,18 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.litellm_core_utils.prompt_templates.factory import ( + get_attribute_or_key, + get_tool_calls_from_response, + has_tool_with_name, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] httpxSpecialProvider, ) from litellm.secret_managers.main import get_secret_str from litellm.types.guardrails import GuardrailEventHooks, Mode +from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: @@ -25,6 +35,9 @@ if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel BYPASS_HEADER = "x-headroom-bypass" +HEADROOM_RETRIEVE_TOOL_NAME = "headroom_retrieve" +_HASH_PATTERN = re.compile(r"hash=([a-f0-9]{24})") +_HASH_CACHE_TTL_SECONDS = 15 * 60 def _is_str_object_dict(value: object) -> TypeGuard[dict[str, object]]: # guard-ok: isinstance narrows correctly; predicate is trivially correct # fmt: skip @@ -35,6 +48,163 @@ def _is_object_list(value: object) -> TypeGuard[list[object]]: # guard-ok: isin return isinstance(value, list) +def extract_hashes_from_messages(messages: list[dict[str, object]]) -> list[str]: + hashes: list[str] = [] + for msg in messages: + content = msg.get("content") + if isinstance(content, str): + hashes.extend(_HASH_PATTERN.findall(content)) + elif isinstance(content, list): + for block in content: + if isinstance(block, dict): + text = block.get("text") + if isinstance(text, str): + hashes.extend(_HASH_PATTERN.findall(text)) + return hashes + + +def _build_headroom_retrieve_tool() -> dict[str, object]: + return { + "type": "function", + "function": { + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "description": ( + "Retrieve original content that was compressed by Headroom. " + "Call this when you encounter a compression marker containing a hash." + ), + "parameters": { + "type": "object", + "properties": { + "hash": { + "type": "string", + "description": "The 24-character hex hash from the compression marker.", + }, + "query": { + "type": "string", + "description": "Optional search query for BM25-ranked retrieval.", + }, + }, + "required": ["hash"], + }, + }, + } + + +def _resolve_call_id(logging_obj: object, request_state: dict[str, object]) -> Optional[str]: + """Resolve the litellm_call_id shared by a request's pre-call hook and its + agentic-loop hooks, so CCR hash validation can be scoped per call instead + of trusting any hash-shaped string that shows up in message text.""" + logging_call_id = getattr(logging_obj, "litellm_call_id", None) + if isinstance(logging_call_id, str) and logging_call_id: + return logging_call_id + kwargs_call_id = request_state.get("litellm_call_id") + return kwargs_call_id if isinstance(kwargs_call_id, str) else None + + +def has_headroom_retrieve_tool(tools: object) -> bool: + return has_tool_with_name(tools, HEADROOM_RETRIEVE_TOOL_NAME) + + +def _extract_headroom_tool_calls(response: object) -> list[dict[str, object]]: + return [ + {"id": tc["id"], "type": "function", "name": tc["name"], "arguments": tc["arguments"]} + for tc in get_tool_calls_from_response(response) + if tc["name"] == HEADROOM_RETRIEVE_TOOL_NAME + ] + + +def _build_assistant_message_from_response(response: object) -> dict[str, object]: + choices = getattr(response, "choices", None) + if not isinstance(choices, list) or not choices: + return {"role": "assistant", "content": None, "tool_calls": []} + message = getattr(choices[0], "message", None) + if message is None: + return {"role": "assistant", "content": None, "tool_calls": []} + content = getattr(message, "content", None) + tool_calls = getattr(message, "tool_calls", None) + raw_tool_calls: list[dict[str, object]] = [] + if isinstance(tool_calls, list): + for tc in tool_calls: + fn = getattr(tc, "function", None) + raw_tool_calls.append( + { + "id": getattr(tc, "id", None), + "type": "function", + "function": { + "name": getattr(fn, "name", None) if fn else None, + "arguments": getattr(fn, "arguments", "{}") if fn else "{}", + }, + } + ) + return {"role": "assistant", "content": content, "tool_calls": raw_tool_calls} + + +def _is_responses_api_response(response: object) -> bool: + # Real response objects can be plain dicts at runtime (e.g. TypedDict-based + # response types), so getattr alone would silently miss the key -- use the + # same dict-or-object accessor as the tool-call extractors. + return isinstance(get_attribute_or_key(response, "output", None), list) + + +def _is_anthropic_messages_response(response: object) -> bool: + return isinstance(get_attribute_or_key(response, "content", None), list) + + +def _build_anthropic_followup_messages( + retrieved: list[tuple[dict[str, object], str]], +) -> list[dict[str, object]]: + """Build Anthropic Messages API follow-up messages for a tool round-trip. + + Anthropic requires the tool_use block to be echoed back in an assistant + message, paired with a tool_result block in a user message keyed by the + same tool_use_id -- it does not accept chat-style tool-role messages. + """ + assistant_message: dict[str, object] = { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": tool_call.get("id"), + "name": tool_call.get("name"), + "input": tool_call.get("arguments", {}), + } + for tool_call, _ in retrieved + ], + } + user_message: dict[str, object] = { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": tool_call.get("id"), "content": content} + for tool_call, content in retrieved + ], + } + return [assistant_message, user_message] + + +def _build_responses_followup_items( + retrieved: list[tuple[dict[str, object], str]], +) -> list[dict[str, object]]: + """Build Responses API input items for a tool round-trip. + + The Responses API does not accept chat-style assistant/tool messages as + follow-up input; it requires the model's function_call to be echoed back + paired with a function_call_output keyed by the same call_id. + """ + items: list[dict[str, object]] = [] + for tool_call, content in retrieved: + call_id = tool_call.get("id") + items.append( + { + "type": "function_call", + "call_id": call_id, + "name": tool_call.get("name"), + "arguments": json.dumps(tool_call.get("arguments", {})), + } + ) + items.append({"type": "function_call_output", "call_id": call_id, "output": content}) + return items + + class HeadroomGuardrail(CustomGuardrail): def __init__( self, @@ -56,6 +226,7 @@ class HeadroomGuardrail(CustomGuardrail): self.async_handler = get_async_httpx_client( llm_provider=httpxSpecialProvider.GuardrailCallback, ) + self._issued_hashes_by_call_id: dict[str, tuple[frozenset[str], float]] = {} super().__init__( # pyright: ignore[reportUnknownMemberType] guardrail_name=guardrail_name, event_hook=event_hook, @@ -72,6 +243,20 @@ class HeadroomGuardrail(CustomGuardrail): value = headers.get(BYPASS_HEADER) return str(value).lower() == "true" + def _request_headers(self) -> dict[str, str]: + headers: dict[str, str] = {"Content-Type": "application/json"} + if self.headroom_api_key: + headers["Authorization"] = f"Bearer {self.headroom_api_key}" + return headers + + def _prune_expired_hashes(self) -> None: + now = time.monotonic() + self._issued_hashes_by_call_id = { + call_id: (hashes, expiry) + for call_id, (hashes, expiry) in self._issued_hashes_by_call_id.items() + if expiry > now + } + async def _call_compress( self, messages: list[dict[str, object]], @@ -81,15 +266,11 @@ class HeadroomGuardrail(CustomGuardrail): if model: payload["model"] = model - request_headers: dict[str, str] = {"Content-Type": "application/json"} - if self.headroom_api_key: - request_headers["Authorization"] = f"Bearer {self.headroom_api_key}" - try: raw_response: HttpxResponse | None = await self.async_handler.post( # pyright: ignore[reportUnknownMemberType] url=f"{self.headroom_api_base}/v1/compress", json=payload, - headers=request_headers, + headers=self._request_headers(), ) except (httpx.ConnectError, httpx.TimeoutException, httpx.TransportError) as e: raise HTTPException( @@ -118,7 +299,7 @@ class HeadroomGuardrail(CustomGuardrail): try: body: object = response.json() - except Exception: + except ValueError: raise HTTPException( status_code=502, detail={ @@ -163,6 +344,44 @@ class HeadroomGuardrail(CustomGuardrail): ) return filtered + async def _call_retrieve(self, hash_value: str, query: str | None = None) -> str: + params: dict[str, str] = {} + if query: + params["query"] = query + + try: + raw_response: HttpxResponse | None = await self.async_handler.get( # pyright: ignore[reportUnknownMemberType] + url=f"{self.headroom_api_base}/v1/retrieve/{hash_value}", + params=params, + headers=self._request_headers(), + ) + except (httpx.ConnectError, httpx.TimeoutException, httpx.TransportError) as e: + verbose_proxy_logger.warning("Headroom: retrieve failed for hash=%s: %s", hash_value, e) + return f"[Headroom: retrieval failed for hash={hash_value}]" + + if raw_response is None or raw_response.status_code == 404: + return f"[Headroom: hash={hash_value} not found or expired]" + + if raw_response.status_code != 200: + verbose_proxy_logger.warning( + "Headroom: retrieve returned %s for hash=%s", + raw_response.status_code, + hash_value, + ) + return f"[Headroom: retrieval error {raw_response.status_code} for hash={hash_value}]" + + try: + body: object = raw_response.json() + except ValueError: + return raw_response.text + + if _is_str_object_dict(body): + original_content = body.get("original_content") + if isinstance(original_content, str): + return original_content + + return str(body) + @log_guardrail_information async def apply_guardrail( self, @@ -192,7 +411,127 @@ class HeadroomGuardrail(CustomGuardrail): model=model if isinstance(model, str) else None, ) - return {**inputs, "structured_messages": compressed} # pyright: ignore[reportReturnType] + hashes = extract_hashes_from_messages(compressed) + if not hashes: + return {**inputs, "structured_messages": compressed} # pyright: ignore[reportReturnType] + + self._prune_expired_hashes() + call_id = _resolve_call_id(logging_obj, request_data) + if not call_id: + call_id = str(uuid.uuid4()) + request_data["litellm_call_id"] = call_id + self._issued_hashes_by_call_id[call_id] = (frozenset(hashes), time.monotonic() + _HASH_CACHE_TTL_SECONDS) + + existing_tools = inputs.get("tools") + retrieve_tool = _build_headroom_retrieve_tool() + if isinstance(existing_tools, list) and not has_headroom_retrieve_tool(existing_tools): + merged_tools: list[object] = list(existing_tools) + [retrieve_tool] + elif existing_tools is None: + merged_tools = [retrieve_tool] + else: + merged_tools = list(existing_tools) if isinstance(existing_tools, list) else [retrieve_tool] + + return {**inputs, "structured_messages": compressed, "tools": merged_tools} # pyright: ignore[reportReturnType] + + async def async_should_run_agentic_loop( + self, + response: Any, + model: str, + messages: list[dict], + tools: Optional[list[dict]], + stream: bool, + custom_llm_provider: str, + kwargs: dict, + ) -> tuple[bool, dict]: + if not has_headroom_retrieve_tool(tools): + return False, {} + + tool_calls = _extract_headroom_tool_calls(response) + if not tool_calls: + return False, {} + + return True, {"tool_calls": tool_calls} + + async def async_build_agentic_loop_plan( + self, + tools: dict, + model: str, + messages: list[dict], + response: Any, + anthropic_messages_provider_config: Any, + anthropic_messages_optional_request_params: dict, + logging_obj: Any, + stream: bool, + kwargs: dict, + ) -> AgenticLoopPlan: + tool_calls: list[dict[str, object]] = tools.get("tool_calls", []) # type: ignore[assignment] + + self._prune_expired_hashes() + call_id = _resolve_call_id(logging_obj, kwargs) + valid_hashes = self._issued_hashes_by_call_id.get(call_id, (frozenset(), 0.0))[0] if call_id else frozenset() + + retrieved: list[tuple[dict[str, object], str]] = [] + for tc in tool_calls: + arguments = tc.get("arguments", {}) + hash_value = arguments.get("hash", "") if isinstance(arguments, dict) else "" + query = arguments.get("query") if isinstance(arguments, dict) else None + # A hash is only honored if it was issued by *this request's own* + # Headroom /v1/compress call, scoped by litellm_call_id. Scoping by + # message text alone is forgeable -- an attacker can plant a + # hash-shaped string in their own prompt, and a hash issued for one + # request would validate for any other request that echoes it back. + if str(hash_value) not in valid_hashes: + verbose_proxy_logger.warning( + "Headroom CCR: rejecting hash=%s not produced by current request compression", + hash_value, + ) + content = f"[Headroom: hash={hash_value} was not produced by the current request]" + else: + content = await self._call_retrieve( + hash_value=str(hash_value), + query=str(query) if query else None, + ) + verbose_proxy_logger.debug("Headroom CCR: retrieved hash=%s (%d chars)", hash_value, len(content)) + retrieved.append((tc, content)) + + if _is_responses_api_response(response): + follow_up_messages = list(messages) + _build_responses_followup_items(retrieved) + elif _is_anthropic_messages_response(response): + follow_up_messages = list(messages) + _build_anthropic_followup_messages(retrieved) + else: + assistant_message = _build_assistant_message_from_response(response) + tool_results = [ + {"role": "tool", "tool_call_id": tc.get("id"), "content": content} for tc, content in retrieved + ] + follow_up_messages = list(messages) + [assistant_message] + tool_results + + max_tokens: Optional[int] = anthropic_messages_optional_request_params.get("max_tokens") or kwargs.get( + "max_tokens" + ) + optional_params_without_max_tokens = { + k: v for k, v in anthropic_messages_optional_request_params.items() if k != "max_tokens" + } + + full_model_name = model + if logging_obj is not None: + agentic_params = getattr(logging_obj, "model_call_details", {}).get("agentic_loop_params", {}) + candidate = agentic_params.get("model", model) + if isinstance(candidate, str) and candidate: + full_model_name = candidate + + return AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + model=full_model_name, + messages=follow_up_messages, + max_tokens=max_tokens, + optional_params=optional_params_without_max_tokens, + kwargs={ + k: v for k, v in kwargs.items() if not k.startswith("_headroom") and k != "litellm_logging_obj" + }, + ), + metadata={"tool_type": "headroom_ccr"}, + ) @staticmethod def get_config_model() -> type[GuardrailConfigModel[object]] | None: diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index 05a58a135d2..ae215602e31 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -20,6 +20,8 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_anthropic_tool_invoke, convert_url_to_base64, create_anthropic_image_param, + get_tool_calls_from_response, + has_tool_with_name, llama_2_chat_pt, prompt_factory, ) @@ -2385,3 +2387,100 @@ def test_anthropic_messages_pt_list_content_with_thinking_preserves_order(): # Verify signatures preserved in correct positions assert content[0]["signature"] == "sig_1" assert content[3]["signature"] == "sig_2" + + +def test_get_tool_calls_from_response_chat_completions(): + response = MagicMock() + response.output = None + response.content = None + tool_call = MagicMock() + tool_call.id = "call_abc" + tool_call.function.name = "my_tool" + tool_call.function.arguments = '{"x": 1}' + response.choices = [MagicMock(message=MagicMock(tool_calls=[tool_call]))] + + result = get_tool_calls_from_response(response) + + assert result == [{"id": "call_abc", "name": "my_tool", "arguments": {"x": 1}}] + + +def test_get_tool_calls_from_response_responses_api(): + response = MagicMock() + response.choices = None + response.content = None + response.output = [ + { + "type": "function_call", + "id": "fc_1", + "call_id": "call_1", + "name": "my_tool", + "arguments": '{"x": 2}', + } + ] + + result = get_tool_calls_from_response(response) + + assert result == [{"id": "call_1", "name": "my_tool", "arguments": {"x": 2}}] + + +def test_get_tool_calls_from_response_anthropic_messages(): + response = MagicMock() + response.choices = None + response.output = None + response.content = [ + {"type": "tool_use", "id": "toolu_1", "name": "my_tool", "input": {"x": 3}}, + ] + + result = get_tool_calls_from_response(response) + + assert result == [{"id": "toolu_1", "name": "my_tool", "arguments": {"x": 3}}] + + +def test_get_tool_calls_from_response_anthropic_messages_plain_dict(): + # AnthropicMessagesResponse is a TypedDict -- real responses are plain + # dicts at runtime, not objects with attribute access. A MagicMock-only + # test would pass even if the extractor used bare getattr() and silently + # returned nothing for a real response. + response = { + "content": [ + {"type": "tool_use", "id": "toolu_1", "name": "my_tool", "input": {"x": 3}}, + ] + } + + result = get_tool_calls_from_response(response) + + assert result == [{"id": "toolu_1", "name": "my_tool", "arguments": {"x": 3}}] + + +def test_get_tool_calls_from_response_no_tool_calls(): + response = MagicMock() + response.choices = None + response.output = None + response.content = None + + assert get_tool_calls_from_response(response) == [] + + +def test_has_tool_with_name_openai_function_shape(): + tools = [{"type": "function", "function": {"name": "my_tool"}}] + assert has_tool_with_name(tools, "my_tool") + assert not has_tool_with_name(tools, "other_tool") + + +def test_has_tool_with_name_anthropic_custom_shape(): + tools = [{"type": "custom", "name": "my_tool", "input_schema": {}}] + assert has_tool_with_name(tools, "my_tool") + assert not has_tool_with_name(tools, "other_tool") + + +def test_has_tool_with_name_anthropic_shape_without_type_field(): + # Anthropic's documented client tool format is just name + input_schema; + # "type" isn't required at all (type: "custom" is only one possible value). + tools = [{"name": "my_tool", "input_schema": {}}] + assert has_tool_with_name(tools, "my_tool") + assert not has_tool_with_name(tools, "other_tool") + + +def test_has_tool_with_name_not_a_list(): + assert not has_tool_with_name(None, "my_tool") + assert not has_tool_with_name("not a list", "my_tool") diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py index 66395035384..f5ce6cedf64 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -8,15 +8,25 @@ Tests cover: - response-type input is passed through unchanged - /v1/compress HTTP error raises HTTPException - /v1/compress returning malformed JSON raises HTTPException +- CCR: headroom_retrieve tool injected when compressed messages contain hashes +- CCR: async_should_run_agentic_loop returns True when response has headroom_retrieve tool calls +- CCR: async_build_agentic_loop_plan calls retrieve endpoint and builds follow-up messages """ +import json +import time from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest from fastapi import HTTPException -from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import HeadroomGuardrail +from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import ( + HeadroomGuardrail, + extract_hashes_from_messages, + has_headroom_retrieve_tool, + HEADROOM_RETRIEVE_TOOL_NAME, +) from litellm.types.utils import GenericGuardrailAPIInputs FAKE_API_BASE = "https://headroom.example.com" @@ -30,6 +40,13 @@ COMPRESSED_MESSAGES = [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "A" * 500}, ] +COMPRESSED_MESSAGES_WITH_HASH = [ + {"role": "system", "content": "You are a helpful assistant."}, + { + "role": "user", + "content": "Summary. Retrieve more: hash=b573993006976af767214fac", + }, +] def _make_guardrail(**kwargs) -> HeadroomGuardrail: @@ -57,6 +74,36 @@ def _make_compress_response(messages: list, status: int = 200) -> MagicMock: return mock +def _make_retrieve_response(original_content: str, status: int = 200) -> MagicMock: + mock = MagicMock() + mock.status_code = status + mock.json.return_value = {"original_content": original_content} + mock.text = original_content + return mock + + +def _make_openai_response_with_tool_call(tool_name: str, arguments: dict, tool_id: str = "call_abc123") -> MagicMock: + fn = MagicMock() + fn.name = tool_name + fn.arguments = json.dumps(arguments) + + tc = MagicMock() + tc.id = tool_id + tc.type = "function" + tc.function = fn + + message = MagicMock() + message.content = None + message.tool_calls = [tc] + + choice = MagicMock() + choice.message = message + + response = MagicMock() + response.choices = [choice] + return response + + @pytest.fixture def guardrail() -> HeadroomGuardrail: return _make_guardrail() @@ -87,6 +134,567 @@ async def test_apply_guardrail_compresses_and_returns_structured_messages( assert result.get("structured_messages") == COMPRESSED_MESSAGES +@pytest.mark.asyncio +async def test_apply_guardrail_injects_retrieve_tool_when_hashes_present( + guardrail: HeadroomGuardrail, +): + inputs = GenericGuardrailAPIInputs( + texts=["A" * 5000], + structured_messages=ORIGINAL_MESSAGES, + ) + mock_response = _make_compress_response(COMPRESSED_MESSAGES_WITH_HASH) + + with patch.object( + guardrail.async_handler, + "post", + new_callable=AsyncMock, + return_value=mock_response, + ): + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data={"model": "gpt-4o"}, + input_type="request", + ) + + tools = result.get("tools") + assert tools is not None + assert has_headroom_retrieve_tool(tools) + + +@pytest.mark.asyncio +async def test_apply_guardrail_no_tool_injected_when_no_hashes( + guardrail: HeadroomGuardrail, +): + inputs = GenericGuardrailAPIInputs( + texts=["A" * 5000], + structured_messages=ORIGINAL_MESSAGES, + ) + mock_response = _make_compress_response(COMPRESSED_MESSAGES) + + with patch.object( + guardrail.async_handler, + "post", + new_callable=AsyncMock, + return_value=mock_response, + ): + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data={"model": "gpt-4o"}, + input_type="request", + ) + + tools = result.get("tools") + assert not has_headroom_retrieve_tool(tools or []) + + +@pytest.mark.asyncio +async def test_apply_guardrail_preserves_existing_tools_when_injecting( + guardrail: HeadroomGuardrail, +): + existing_tool = {"type": "function", "function": {"name": "my_tool", "parameters": {}}} + inputs = GenericGuardrailAPIInputs( + texts=["A" * 5000], + structured_messages=ORIGINAL_MESSAGES, + tools=[existing_tool], + ) + mock_response = _make_compress_response(COMPRESSED_MESSAGES_WITH_HASH) + + with patch.object( + guardrail.async_handler, + "post", + new_callable=AsyncMock, + return_value=mock_response, + ): + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data={"model": "gpt-4o"}, + input_type="request", + ) + + tools = result.get("tools") + assert tools is not None + assert isinstance(tools, list) + assert any(isinstance(t, dict) and t.get("function", {}).get("name") == "my_tool" for t in tools) + assert has_headroom_retrieve_tool(tools) + + +@pytest.mark.asyncio +async def test_async_should_run_agentic_loop_returns_true_for_retrieve_call( + guardrail: HeadroomGuardrail, +): + retrieve_tool_def = [{"type": "function", "function": {"name": HEADROOM_RETRIEVE_TOOL_NAME}}] + response = _make_openai_response_with_tool_call( + tool_name=HEADROOM_RETRIEVE_TOOL_NAME, + arguments={"hash": "b573993006976af767214fac"}, + ) + + should_run, ctx = await guardrail.async_should_run_agentic_loop( + response=response, + model="gpt-4o", + messages=[], + tools=retrieve_tool_def, + stream=False, + custom_llm_provider="openai", + kwargs={}, + ) + + assert should_run is True + assert len(ctx["tool_calls"]) == 1 + assert ctx["tool_calls"][0]["arguments"]["hash"] == "b573993006976af767214fac" + + +@pytest.mark.asyncio +async def test_async_should_run_agentic_loop_returns_false_without_retrieve_tool( + guardrail: HeadroomGuardrail, +): + other_tools = [{"type": "function", "function": {"name": "other_tool"}}] + response = _make_openai_response_with_tool_call( + tool_name="other_tool", + arguments={}, + ) + + should_run, _ = await guardrail.async_should_run_agentic_loop( + response=response, + model="gpt-4o", + messages=[], + tools=other_tools, + stream=False, + custom_llm_provider="openai", + kwargs={}, + ) + + assert should_run is False + + +@pytest.mark.asyncio +async def test_async_should_run_agentic_loop_returns_false_when_no_retrieve_calls( + guardrail: HeadroomGuardrail, +): + retrieve_tool_def = [{"type": "function", "function": {"name": HEADROOM_RETRIEVE_TOOL_NAME}}] + response = _make_openai_response_with_tool_call( + tool_name="some_other_function", + arguments={}, + ) + + should_run, _ = await guardrail.async_should_run_agentic_loop( + response=response, + model="gpt-4o", + messages=[], + tools=retrieve_tool_def, + stream=False, + custom_llm_provider="openai", + kwargs={}, + ) + + assert should_run is False + + +@pytest.mark.asyncio +async def test_async_build_agentic_loop_plan_calls_retrieve_and_builds_messages( + guardrail: HeadroomGuardrail, +): + original_content = "This is the full compressed content." + mock_retrieve = _make_retrieve_response(original_content) + + tool_calls = [ + { + "id": "call_abc123", + "type": "function", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "arguments": {"hash": "b573993006976af767214fac"}, + } + ] + response = _make_openai_response_with_tool_call( + tool_name=HEADROOM_RETRIEVE_TOOL_NAME, + arguments={"hash": "b573993006976af767214fac"}, + tool_id="call_abc123", + ) + messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}] + guardrail._issued_hashes_by_call_id["call-1"] = ( + frozenset({"b573993006976af767214fac"}), + time.monotonic() + 999, + ) + + with patch.object( + guardrail.async_handler, + "get", + new_callable=AsyncMock, + return_value=mock_retrieve, + ) as mock_get: + plan = await guardrail.async_build_agentic_loop_plan( + tools={"tool_calls": tool_calls}, + model="gpt-4o", + messages=messages, + response=response, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={}, + logging_obj=None, + stream=False, + kwargs={"litellm_call_id": "call-1"}, + ) + + assert plan.run_agentic_loop is True + assert plan.request_patch is not None + + follow_up = plan.request_patch.messages + assert follow_up is not None + + tool_result_message = next((m for m in follow_up if m.get("role") == "tool"), None) + assert tool_result_message is not None + assert tool_result_message["content"] == original_content + assert tool_result_message["tool_call_id"] == "call_abc123" + + mock_get.assert_called_once() + call_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args.args[0] + assert "b573993006976af767214fac" in call_url + + +@pytest.mark.asyncio +async def test_async_build_agentic_loop_plan_handles_retrieve_404( + guardrail: HeadroomGuardrail, +): + mock_retrieve = MagicMock() + mock_retrieve.status_code = 404 + + tool_calls = [ + { + "id": "call_xyz", + "type": "function", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "arguments": {"hash": "deadbeef000000000000dead"}, + } + ] + response = _make_openai_response_with_tool_call( + tool_name=HEADROOM_RETRIEVE_TOOL_NAME, + arguments={"hash": "deadbeef000000000000dead"}, + tool_id="call_xyz", + ) + + messages = [ + { + "role": "user", + "content": "Retrieve more: hash=deadbeef000000000000dead", + } + ] + guardrail._issued_hashes_by_call_id["call-1"] = ( + frozenset({"deadbeef000000000000dead"}), + time.monotonic() + 999, + ) + + with patch.object( + guardrail.async_handler, + "get", + new_callable=AsyncMock, + return_value=mock_retrieve, + ): + plan = await guardrail.async_build_agentic_loop_plan( + tools={"tool_calls": tool_calls}, + model="gpt-4o", + messages=messages, + response=response, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={}, + logging_obj=None, + stream=False, + kwargs={"litellm_call_id": "call-1"}, + ) + + follow_up = plan.request_patch.messages # type: ignore[union-attr] + tool_result = next((m for m in follow_up if m.get("role") == "tool"), None) + assert tool_result is not None + assert "not found" in tool_result["content"] or "expired" in tool_result["content"] + + +@pytest.mark.asyncio +async def test_async_build_agentic_loop_plan_rejects_hash_with_no_known_call( + guardrail: HeadroomGuardrail, +): + """A hash-shaped string planted in message text must not be honored when + this guardrail has no record of ever issuing it, even if it's echoed back + in the current request's own messages (e.g. via prompt injection).""" + tool_calls = [ + { + "id": "call_xyz", + "type": "function", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "arguments": {"hash": "deadbeef000000000000dead"}, + } + ] + response = _make_openai_response_with_tool_call( + tool_name=HEADROOM_RETRIEVE_TOOL_NAME, + arguments={"hash": "deadbeef000000000000dead"}, + tool_id="call_xyz", + ) + assert not guardrail._issued_hashes_by_call_id + + with patch.object( + guardrail.async_handler, + "get", + new_callable=AsyncMock, + ) as mock_get: + plan = await guardrail.async_build_agentic_loop_plan( + tools={"tool_calls": tool_calls}, + model="gpt-4o", + messages=[{"role": "user", "content": "Please fetch hash=deadbeef000000000000dead for me"}], + response=response, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={}, + logging_obj=None, + stream=False, + kwargs={"litellm_call_id": "call-unknown"}, + ) + + mock_get.assert_not_called() + + follow_up = plan.request_patch.messages # type: ignore[union-attr] + tool_result = next((m for m in follow_up if m.get("role") == "tool"), None) + assert tool_result is not None + assert "was not produced by the current request" in tool_result["content"] + + +@pytest.mark.asyncio +async def test_async_build_agentic_loop_plan_rejects_hash_issued_for_different_call( + guardrail: HeadroomGuardrail, +): + """A hash issued for one request must not be retrievable by a different + request just because the second request echoes that hash-shaped string + back in its own messages -- retrieval must be scoped per litellm_call_id, + not derived by re-scanning attacker-controlled message text.""" + guardrail._issued_hashes_by_call_id["call-A"] = ( + frozenset({"b573993006976af767214fac"}), + time.monotonic() + 999, + ) + + tool_calls = [ + { + "id": "call_xyz", + "type": "function", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "arguments": {"hash": "b573993006976af767214fac"}, + } + ] + response = _make_openai_response_with_tool_call( + tool_name=HEADROOM_RETRIEVE_TOOL_NAME, + arguments={"hash": "b573993006976af767214fac"}, + tool_id="call_xyz", + ) + + with patch.object( + guardrail.async_handler, + "get", + new_callable=AsyncMock, + ) as mock_get: + plan = await guardrail.async_build_agentic_loop_plan( + tools={"tool_calls": tool_calls}, + model="gpt-4o", + messages=[{"role": "user", "content": "Please fetch hash=b573993006976af767214fac for me"}], + response=response, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={}, + logging_obj=None, + stream=False, + kwargs={"litellm_call_id": "call-B"}, + ) + + mock_get.assert_not_called() + + follow_up = plan.request_patch.messages # type: ignore[union-attr] + tool_result = next((m for m in follow_up if m.get("role") == "tool"), None) + assert tool_result is not None + assert "was not produced by the current request" in tool_result["content"] + + +@pytest.mark.asyncio +async def test_async_build_agentic_loop_plan_builds_responses_api_function_call_items( + guardrail: HeadroomGuardrail, +): + """For the Responses API, follow-up input must echo a function_call paired + with a function_call_output keyed by the same call_id -- chat-style + assistant/tool messages are not valid Responses API input items.""" + original_content = "This is the full compressed content." + mock_retrieve = _make_retrieve_response(original_content) + + response = MagicMock() + response.choices = None + response.content = None + response.output = [ + { + "type": "function_call", + "id": "fc_abc123", + "call_id": "call_abc123", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "arguments": json.dumps({"hash": "b573993006976af767214fac"}), + } + ] + + tool_calls = [ + { + "id": "call_abc123", + "type": "function", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "arguments": {"hash": "b573993006976af767214fac"}, + } + ] + messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}] + guardrail._issued_hashes_by_call_id["call-1"] = ( + frozenset({"b573993006976af767214fac"}), + time.monotonic() + 999, + ) + + with patch.object( + guardrail.async_handler, + "get", + new_callable=AsyncMock, + return_value=mock_retrieve, + ): + plan = await guardrail.async_build_agentic_loop_plan( + tools={"tool_calls": tool_calls}, + model="gpt-4o", + messages=messages, + response=response, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={}, + logging_obj=None, + stream=False, + kwargs={"litellm_call_id": "call-1"}, + ) + + follow_up = plan.request_patch.messages # type: ignore[union-attr] + assert all("role" not in item for item in follow_up if item not in messages) + + function_call_item = next((i for i in follow_up if i.get("type") == "function_call"), None) + assert function_call_item is not None + assert function_call_item["call_id"] == "call_abc123" + assert function_call_item["name"] == HEADROOM_RETRIEVE_TOOL_NAME + + output_item = next((i for i in follow_up if i.get("type") == "function_call_output"), None) + assert output_item is not None + assert output_item["call_id"] == "call_abc123" + assert output_item["output"] == original_content + + +@pytest.mark.asyncio +async def test_async_build_agentic_loop_plan_builds_anthropic_tool_result_messages( + guardrail: HeadroomGuardrail, +): + """For the Anthropic Messages API, follow-up must echo a tool_use content + block in an assistant message paired with a tool_result content block in a + user message keyed by the same tool_use_id -- chat-style tool-role + messages are not valid Anthropic input. + + AnthropicMessagesResponse is a TypedDict, so real responses are plain + dicts at runtime; a MagicMock response here would pass even if branch + selection used bare getattr() and silently fell through to the + chat-completions replay shape for every real Anthropic response. + """ + original_content = "This is the full compressed content." + mock_retrieve = _make_retrieve_response(original_content) + + response = { + "content": [ + { + "type": "tool_use", + "id": "toolu_abc123", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "input": {"hash": "b573993006976af767214fac"}, + } + ] + } + + tool_calls = [ + { + "id": "toolu_abc123", + "type": "function", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "arguments": {"hash": "b573993006976af767214fac"}, + } + ] + messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}] + guardrail._issued_hashes_by_call_id["call-1"] = ( + frozenset({"b573993006976af767214fac"}), + time.monotonic() + 999, + ) + + with patch.object( + guardrail.async_handler, + "get", + new_callable=AsyncMock, + return_value=mock_retrieve, + ): + plan = await guardrail.async_build_agentic_loop_plan( + tools={"tool_calls": tool_calls}, + model="claude-sonnet-4-5", + messages=messages, + response=response, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={}, + logging_obj=None, + stream=False, + kwargs={"litellm_call_id": "call-1"}, + ) + + follow_up = plan.request_patch.messages # type: ignore[union-attr] + assert all(m.get("role") != "tool" for m in follow_up) + + assistant_message = next((m for m in follow_up if m.get("role") == "assistant"), None) + assert assistant_message is not None + tool_use_block = next((b for b in assistant_message["content"] if b.get("type") == "tool_use"), None) + assert tool_use_block is not None + assert tool_use_block["id"] == "toolu_abc123" + + user_message = follow_up[-1] + assert user_message["role"] == "user" + tool_result_block = next((b for b in user_message["content"] if b.get("type") == "tool_result"), None) + assert tool_result_block is not None + assert tool_result_block["tool_use_id"] == "toolu_abc123" + assert tool_result_block["content"] == original_content + + +def test_extract_hashes_from_messages_finds_hashes(): + messages = [ + {"role": "user", "content": "Retrieve more: hash=b573993006976af767214fac"}, + {"role": "assistant", "content": "Also: hash=aabbccdd001122334455aabb"}, + ] + hashes = extract_hashes_from_messages(messages) + assert "b573993006976af767214fac" in hashes + assert "aabbccdd001122334455aabb" in hashes + + +def test_extract_hashes_from_messages_ignores_short_hashes(): + messages = [{"role": "user", "content": "hash=tooshort"}] + hashes = extract_hashes_from_messages(messages) + assert not hashes + + +def test_extract_hashes_from_list_content_blocks(): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "hash=b573993006976af767214fac found here"}, + ], + } + ] + hashes = extract_hashes_from_messages(messages) + assert "b573993006976af767214fac" in hashes + + +def test_has_headroom_retrieve_tool_recognizes_anthropic_native_shape(): + """By the time an Anthropic Messages API response reaches the agentic-loop + gate, the OpenAI-shaped tool this guardrail injects (type: "function") + has already been transformed into Anthropic's native tool shape + (type: "custom", top-level "name", no nested "function" object).""" + anthropic_native_tools = [ + { + "type": "custom", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "input_schema": {"type": "object", "properties": {"hash": {"type": "string"}}}, + } + ] + assert has_headroom_retrieve_tool(anthropic_native_tools) + assert not has_headroom_retrieve_tool([{"type": "custom", "name": "some_other_tool"}]) + + @pytest.mark.asyncio async def test_apply_guardrail_bypass_header_skips_compression( guardrail: HeadroomGuardrail, @@ -97,9 +705,7 @@ async def test_apply_guardrail_bypass_header_skips_compression( ) request_data = {"proxy_server_request": {"headers": {"x-headroom-bypass": "true"}}} - with patch.object( - guardrail.async_handler, "post", new_callable=AsyncMock - ) as mock_post: + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post: result = await guardrail.apply_guardrail( inputs=inputs, request_data=request_data, @@ -119,9 +725,7 @@ async def test_apply_guardrail_response_type_passthrough( structured_messages=ORIGINAL_MESSAGES, ) - with patch.object( - guardrail.async_handler, "post", new_callable=AsyncMock - ) as mock_post: + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post: result = await guardrail.apply_guardrail( inputs=inputs, request_data={}, @@ -138,9 +742,7 @@ async def test_apply_guardrail_empty_structured_messages_passthrough( ): inputs = GenericGuardrailAPIInputs(texts=["hello"]) - with patch.object( - guardrail.async_handler, "post", new_callable=AsyncMock - ) as mock_post: + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post: result = await guardrail.apply_guardrail( inputs=inputs, request_data={}, @@ -277,9 +879,7 @@ def test_bypass_header_case_insensitive(): guardrail = _make_guardrail() for header_value in ("true", "True", "TRUE"): - data = { - "proxy_server_request": {"headers": {"x-headroom-bypass": header_value}} - } + data = {"proxy_server_request": {"headers": {"x-headroom-bypass": header_value}}} assert guardrail._should_bypass(data) is True data = {"proxy_server_request": {"headers": {"x-headroom-bypass": "false"}}} @@ -344,3 +944,118 @@ async def test_apply_guardrail_sends_model_from_request_data_when_no_config_mode call_kwargs = mock_post.call_args sent_payload = call_kwargs.kwargs.get("json") or call_kwargs.args[1] assert sent_payload.get("model") == "gpt-4o" + + +@pytest.mark.asyncio +async def test_async_should_run_agentic_loop_detects_anthropic_content_block_format( + guardrail: HeadroomGuardrail, +): + # Anthropic's native tool format (type: "custom", top-level "name") -- + # by the time a Messages API response reaches this gate, the OpenAI-shaped + # tool this guardrail injects has already been transformed into this shape. + retrieve_tool_def = [ + { + "type": "custom", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "input_schema": {"type": "object", "properties": {"hash": {"type": "string"}}}, + } + ] + + response = MagicMock() + response.choices = None + response.content = [ + { + "type": "tool_use", + "id": "toolu_abc", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "input": {"hash": "b573993006976af767214fac"}, + } + ] + + should_run, ctx = await guardrail.async_should_run_agentic_loop( + response=response, + model="claude-sonnet-4-6", + messages=[], + tools=retrieve_tool_def, + stream=False, + custom_llm_provider="anthropic", + kwargs={}, + ) + + assert should_run is True + assert len(ctx["tool_calls"]) == 1 + assert ctx["tool_calls"][0]["arguments"]["hash"] == "b573993006976af767214fac" + + +@pytest.mark.asyncio +async def test_async_should_run_agentic_loop_detects_anthropic_response_as_plain_dict( + guardrail: HeadroomGuardrail, +): + """AnthropicMessagesResponse is a TypedDict -- real Messages API responses + are plain dicts at runtime, not objects with attribute access. A + MagicMock-only test would pass even if detection used bare getattr() and + silently treated every real response as having no tool calls.""" + retrieve_tool_def = [ + { + "type": "custom", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "input_schema": {"type": "object", "properties": {"hash": {"type": "string"}}}, + } + ] + response = { + "content": [ + { + "type": "tool_use", + "id": "toolu_abc", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "input": {"hash": "b573993006976af767214fac"}, + } + ] + } + + should_run, ctx = await guardrail.async_should_run_agentic_loop( + response=response, + model="claude-sonnet-4-6", + messages=[], + tools=retrieve_tool_def, + stream=False, + custom_llm_provider="anthropic", + kwargs={}, + ) + + assert should_run is True + assert len(ctx["tool_calls"]) == 1 + assert ctx["tool_calls"][0]["arguments"]["hash"] == "b573993006976af767214fac" + + +@pytest.mark.asyncio +async def test_async_should_run_agentic_loop_detects_responses_api_output_format( + guardrail: HeadroomGuardrail, +): + retrieve_tool_def = [{"type": "function", "function": {"name": HEADROOM_RETRIEVE_TOOL_NAME}}] + + response = MagicMock() + response.choices = None + response.content = None + response.output = [ + { + "type": "function_call", + "id": "fc_abc123", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "arguments": json.dumps({"hash": "b573993006976af767214fac"}), + } + ] + + should_run, ctx = await guardrail.async_should_run_agentic_loop( + response=response, + model="gpt-4o", + messages=[], + tools=retrieve_tool_def, + stream=False, + custom_llm_provider="openai", + kwargs={}, + ) + + assert should_run is True + assert len(ctx["tool_calls"]) == 1 + assert ctx["tool_calls"][0]["arguments"]["hash"] == "b573993006976af767214fac" From 23af78465c877f5f7f02c53d9f04cf1af612f7a9 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 30 Jun 2026 19:31:51 -0700 Subject: [PATCH 15/81] feat: add cache control injection support for v1/messages endpoint (#31778) * feat: add cache control injection support for v1/messages endpoint Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix: normalize string content to list for Anthropic-native cache_control injection Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor: simplify cache control injection, fix system=[] bug, fix handler system type Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor: extract cache control logic into static helper on AnthropicCacheControlHook Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Krrish Dholakia Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../anthropic_cache_control_hook.py | 97 ++++- .../messages/handler.py | 16 +- .../test_anthropic_cache_control_hook.py | 400 +++++++++++------- 3 files changed, 356 insertions(+), 157 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 1314fd82255..608fdebc1d9 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -1,9 +1,12 @@ """ -This hook is used to inject cache control directives into the messages of a chat completion. +This hook is used to inject cache control directives into messages. Users can define - `cache_control_injection_points` in the completion params and litellm will inject the cache control directives into the messages at the specified injection points. +Supported for both `v1/chat/completions` (via the prompt-management hook) and +`v1/messages` (via `apply_to_anthropic_messages_request`). + """ import copy @@ -225,6 +228,98 @@ class AnthropicCacheControlHook(CustomPromptManagement): message_content[-1]["cache_control"] = control # type: ignore return message + @staticmethod + def apply_to_anthropic_messages_request( + messages: List[Dict], + system: str | list | None, + injection_points: List[CacheControlInjectionPoint], + ) -> Tuple[List[Dict], str | list | None, List[CacheControlInjectionPoint]]: + """Apply cache control injection for the Anthropic-native v1/messages endpoint. + + Returns (messages, system, remaining_non_message_points). + """ + if not injection_points: + return messages, system, [] + + processed_messages: List[Dict] = copy.deepcopy(messages) + processed_system = copy.deepcopy(system) if system is not None else None + + message_points: List[CacheControlMessageInjectionPoint] = [] + system_points: List[CacheControlMessageInjectionPoint] = [] + remaining_points: List[CacheControlInjectionPoint] = [] + + for point in injection_points: + if point.get("location") == "message": + msg_point = cast(CacheControlMessageInjectionPoint, point) + if msg_point.get("role") == "system": + system_points.append(msg_point) + else: + message_points.append(msg_point) + else: + remaining_points.append(point) + + reserved_blocks = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0 + max_blocks = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks + + used_blocks = sum( + AnthropicCacheControlHook._count_cache_control_blocks(cast(AllMessageValues, msg)) + for msg in processed_messages + ) + if isinstance(processed_system, list): + used_blocks += sum( + 1 for b in processed_system if isinstance(b, dict) and b.get("cache_control") is not None + ) + + if system_points and processed_system is not None and used_blocks < max_blocks: + system_already_has_cc = isinstance(processed_system, list) and any( + isinstance(b, dict) and b.get("cache_control") is not None for b in processed_system + ) + if not system_already_has_cc: + control = system_points[0].get("control") or ChatCompletionCachedContent(type="ephemeral") + if isinstance(processed_system, str): + processed_system = [{"type": "text", "text": processed_system, "cache_control": control}] + used_blocks += 1 + elif len(processed_system) > 0 and isinstance(processed_system[-1], dict): + processed_system[-1] = {**processed_system[-1], "cache_control": control} + used_blocks += 1 + + for i, msg in enumerate(processed_messages): + content = msg.get("content") + if isinstance(content, str): + processed_messages[i] = {**msg, "content": [{"type": "text", "text": content}]} + + processed_messages = AnthropicCacheControlHook._apply_message_injections( + points=message_points, + messages=cast(List[AllMessageValues], processed_messages), + max_blocks=max_blocks - used_blocks, + ) + + return processed_messages, processed_system, remaining_points + + @staticmethod + def maybe_inject_cache_control( + messages: List[Dict], + system: str | list | None, + kwargs: Dict[str, Any], + ) -> Tuple[List[Dict], str | list | None]: + """Extract cache_control_injection_points from kwargs and apply if present. + + Pops the key from kwargs; if remaining (non-message) points exist they + are written back so downstream transforms can handle them. + """ + injection_points = kwargs.pop("cache_control_injection_points", None) + if not injection_points: + return messages, system + + messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + messages=messages, + system=system, + injection_points=injection_points, + ) + if remaining: + kwargs["cache_control_injection_points"] = remaining + return messages, system + @property def integration_name(self) -> str: """Return the integration name for this hook.""" diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 547ddd9b8d3..effd7dda6a0 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -199,7 +199,7 @@ async def anthropic_messages( metadata: Optional[Dict] = None, stop_sequences: Optional[List[str]] = None, stream: Optional[bool] = False, - system: Optional[str] = None, + system: Optional[Union[str, list]] = None, temperature: Optional[float] = None, thinking: Optional[Dict] = None, tool_choice: Optional[Dict] = None, @@ -230,6 +230,12 @@ async def anthropic_messages( # ids like ``functions.Bash:0`` that violate Anthropic's id pattern. messages = sanitize_tool_use_ids_in_anthropic_messages(messages) + from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + ) + + messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(messages, system, kwargs) + original_stream = stream or kwargs.get("_websearch_interception_converted_stream", False) # Execute pre-request hooks to allow CustomLoggers to modify request. @@ -375,7 +381,7 @@ def anthropic_messages_handler( metadata: Optional[Dict] = None, stop_sequences: Optional[List[str]] = None, stream: Optional[bool] = False, - system: Optional[str] = None, + system: Optional[Union[str, list]] = None, temperature: Optional[float] = None, thinking: Optional[Dict] = None, tool_choice: Optional[Dict] = None, @@ -412,6 +418,12 @@ def anthropic_messages_handler( messages = strip_empty_text_blocks_from_anthropic_messages(messages) messages = sanitize_tool_use_ids_in_anthropic_messages(messages) + from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + ) + + messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(messages, system, kwargs) + metadata = validate_anthropic_api_metadata(metadata) local_vars = locals() diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 6afe5efc54d..4664cc86303 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -1,3 +1,4 @@ +import copy import datetime import json import os @@ -9,9 +10,7 @@ from unittest.mock import ANY, MagicMock, Mock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path +sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system-path import litellm from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -93,13 +92,9 @@ async def test_anthropic_cache_control_hook_system_message(): # Verify that cache control was applied (Bedrock transforms it to a separate item) cache_control_count = sum( - 1 - for item in request_body["system"] - if isinstance(item, dict) and "cachePoint" in item + 1 for item in request_body["system"] if isinstance(item, dict) and "cachePoint" in item ) - assert ( - cache_control_count == 1 - ), f"Expected exactly 1 cache control point, found {cache_control_count}" + assert cache_control_count == 1, f"Expected exactly 1 cache control point, found {cache_control_count}" @pytest.mark.asyncio @@ -171,9 +166,7 @@ async def test_anthropic_cache_control_hook_user_message(): print("request_body: ", json.dumps(request_body, indent=4)) # Verify the request body - assert request_body["messages"][1]["content"][1]["cachePoint"] == { - "type": "default" - } + assert request_body["messages"][1]["content"][1]["cachePoint"] == {"type": "default"} @pytest.mark.asyncio @@ -262,14 +255,10 @@ async def test_anthropic_cache_control_hook_negative_indices(): # Verify the last message (input index -1 -> request index 2) has cache control last_message_content = request_body["messages"][2]["content"] - assert isinstance( - last_message_content, list - ), "Last message content should be a list" - assert any( - "cachePoint" in item - for item in last_message_content - if isinstance(item, dict) - ), "CachePoint missing in last message" + assert isinstance(last_message_content, list), "Last message content should be a list" + assert any("cachePoint" in item for item in last_message_content if isinstance(item, dict)), ( + "CachePoint missing in last message" + ) # Note: Based on debug output, the hook correctly applies cache control to both messages, # but the Bedrock API transformation appears to only preserve cache control for user messages, @@ -278,30 +267,20 @@ async def test_anthropic_cache_control_hook_negative_indices(): # The second-to-last message (assistant) gets cache_control from the hook but loses it # during API transformation. This test documents this behavior. second_last_message_content = request_body["messages"][1]["content"] - assert isinstance( - second_last_message_content, list - ), "Second-to-last message content should be a list" + assert isinstance(second_last_message_content, list), "Second-to-last message content should be a list" # Check if assistant message cache control is preserved (currently it's not) assistant_has_cache_control = any( - "cachePoint" in item - for item in second_last_message_content - if isinstance(item, dict) - ) - print( - f"Assistant message has cache control in final request: {assistant_has_cache_control}" + "cachePoint" in item for item in second_last_message_content if isinstance(item, dict) ) + print(f"Assistant message has cache control in final request: {assistant_has_cache_control}") # Verify the first user message (request index 0) was NOT modified first_user_message_content = request_body["messages"][0]["content"] - assert isinstance( - first_user_message_content, list - ), "First user message content should be a list" - assert not any( - "cachePoint" in item - for item in first_user_message_content - if isinstance(item, dict) - ), "CachePoint unexpectedly found in first user message" + assert isinstance(first_user_message_content, list), "First user message content should be a list" + assert not any("cachePoint" in item for item in first_user_message_content if isinstance(item, dict)), ( + "CachePoint unexpectedly found in first user message" + ) @pytest.mark.asyncio @@ -342,9 +321,7 @@ async def test_anthropic_cache_control_hook_out_of_bounds_logging(): client = AsyncHTTPHandler() # Mock the verbose_logger to capture warning calls - with patch( - "litellm.integrations.anthropic_cache_control_hook.verbose_logger" - ) as mock_logger: + with patch("litellm.integrations.anthropic_cache_control_hook.verbose_logger") as mock_logger: with patch.object(client, "post", return_value=mock_response) as mock_post: messages = [ {"role": "user", "content": "Message 1"}, @@ -354,9 +331,7 @@ async def test_anthropic_cache_control_hook_out_of_bounds_logging(): await litellm.acompletion( model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", messages=messages, - cache_control_injection_points=[ - {"location": "message", "index": 10} - ], # Out of bounds index + cache_control_injection_points=[{"location": "message", "index": 10}], # Out of bounds index client=client, ) @@ -365,10 +340,7 @@ async def test_anthropic_cache_control_hook_out_of_bounds_logging(): warning_call = mock_logger.warning.call_args[0][0] # Check that the warning message contains the expected information - assert ( - "AnthropicCacheControlHook: Provided index 10 is out of bounds" - in warning_call - ) + assert "AnthropicCacheControlHook: Provided index 10 is out of bounds" in warning_call assert "message list of length 2" in warning_call assert "Targeted index was 10" in warning_call assert "Skipping cache control injection for this point" in warning_call @@ -411,9 +383,7 @@ async def test_anthropic_cache_control_hook_negative_out_of_bounds_logging(): client = AsyncHTTPHandler() # Mock the verbose_logger to capture warning calls - with patch( - "litellm.integrations.anthropic_cache_control_hook.verbose_logger" - ) as mock_logger: + with patch("litellm.integrations.anthropic_cache_control_hook.verbose_logger") as mock_logger: with patch.object(client, "post", return_value=mock_response) as mock_post: messages = [ {"role": "user", "content": "Single message"}, @@ -436,14 +406,9 @@ async def test_anthropic_cache_control_hook_negative_out_of_bounds_logging(): warning_call = mock_logger.warning.call_args[0][0] # Check that the warning message contains the original negative index - assert ( - "AnthropicCacheControlHook: Provided index -5 is out of bounds" - in warning_call - ) + assert "AnthropicCacheControlHook: Provided index -5 is out of bounds" in warning_call assert "message list of length 1" in warning_call - assert ( - "Targeted index was -4" in warning_call - ) # -5 + 1 = -4 (converted index) + assert "Targeted index was -4" in warning_call # -5 + 1 = -4 (converted index) assert "Skipping cache control injection for this point" in warning_call @@ -531,15 +496,11 @@ async def test_anthropic_cache_control_hook_multiple_user_messages(): # Count cache control points - should have 2 since both injection points were applied cache_control_count = sum( - 1 - for item in combined_message_content - if isinstance(item, dict) and "cachePoint" in item + 1 for item in combined_message_content if isinstance(item, dict) and "cachePoint" in item ) assert cache_control_count == 2 - print( - f"Found {cache_control_count} cache control points in the combined message" - ) + print(f"Found {cache_control_count} cache control points in the combined message") @pytest.mark.asyncio @@ -588,9 +549,7 @@ async def test_anthropic_cache_control_hook_out_of_bounds(bad_index): await litellm.acompletion( model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", messages=messages, - cache_control_injection_points=[ - {"location": "message", "index": bad_index} - ], + cache_control_injection_points=[{"location": "message", "index": bad_index}], client=client, ) @@ -601,19 +560,13 @@ async def test_anthropic_cache_control_hook_out_of_bounds(bad_index): for msg in request_body["messages"]: content = msg.get("content", []) if isinstance(content, list): - assert not any( - "cachePoint" in item - for item in content - if isinstance(item, dict) - ) + assert not any("cachePoint" in item for item in content if isinstance(item, dict)) @pytest.mark.asyncio @pytest.mark.parametrize( "message_list", - [ - [{"role": "user", "content": "Single message"}] - ], # Single message only - empty list will fail at API level + [[{"role": "user", "content": "Single message"}]], # Single message only - empty list will fail at API level ) async def test_anthropic_cache_control_hook_single_message(message_list): """ @@ -662,9 +615,7 @@ async def test_anthropic_cache_control_hook_single_message(message_list): # For the single message, verify cache control was applied content = request_body["messages"][0]["content"] assert isinstance(content, list) - assert any( - "cachePoint" in item for item in content if isinstance(item, dict) - ) + assert any("cachePoint" in item for item in content if isinstance(item, dict)) @pytest.mark.asyncio @@ -693,9 +644,7 @@ async def test_anthropic_cache_control_hook_empty_message_list(): await litellm.acompletion( model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", messages=[], - cache_control_injection_points=[ - {"location": "message", "index": -1} - ], + cache_control_injection_points=[{"location": "message", "index": -1}], client=client, ) @@ -755,11 +704,7 @@ async def test_anthropic_cache_control_hook_no_op(): for msg in request_body["messages"]: content = msg.get("content", []) if isinstance(content, list): - assert not any( - "cachePoint" in item - for item in content - if isinstance(item, dict) - ) + assert not any("cachePoint" in item for item in content if isinstance(item, dict)) @pytest.mark.asyncio @@ -827,14 +772,10 @@ async def test_anthropic_cache_control_hook_multiple_content_items_last_only(): message_content = request_body["messages"][0]["content"] assert isinstance(message_content, list) - cache_control_count = sum( - 1 - for item in message_content - if isinstance(item, dict) and "cachePoint" in item + cache_control_count = sum(1 for item in message_content if isinstance(item, dict) and "cachePoint" in item) + assert cache_control_count == 1, ( + f"Expected exactly 1 cache control point, found {cache_control_count}. This test verifies the fix for issue 15696 where cache_control was incorrectly applied to ALL content items." ) - assert ( - cache_control_count == 1 - ), f"Expected exactly 1 cache control point, found {cache_control_count}. This test verifies the fix for issue 15696 where cache_control was incorrectly applied to ALL content items." @pytest.mark.asyncio @@ -891,30 +832,22 @@ async def test_anthropic_cache_control_hook_document_analysis_multiple_pages(): ], } ], - cache_control_injection_points=[ - {"location": "message", "role": "user"} - ], + cache_control_injection_points=[{"location": "message", "role": "user"}], client=client, ) mock_post.assert_called_once() request_body = json.loads(mock_post.call_args.kwargs["data"]) - print( - "Document analysis request_body: ", json.dumps(request_body, indent=4) - ) + print("Document analysis request_body: ", json.dumps(request_body, indent=4)) message_content = request_body["messages"][0]["content"] assert isinstance(message_content, list) - cache_control_count = sum( - 1 - for item in message_content - if isinstance(item, dict) and "cachePoint" in item + cache_control_count = sum(1 for item in message_content if isinstance(item, dict) and "cachePoint" in item) + assert cache_control_count == 1, ( + f"Expected exactly 1 cache control point (last item only), found {cache_control_count}. Before fix, this would be 6 (one for each content item)." ) - assert ( - cache_control_count == 1 - ), f"Expected exactly 1 cache control point (last item only), found {cache_control_count}. Before fix, this would be 6 (one for each content item)." def test_gemini_cache_control_injection_points_detected(): @@ -1076,13 +1009,8 @@ async def test_anthropic_cache_control_hook_string_negative_index(): # The last user message should have cache control applied last_message = request_body["messages"][-1] last_message_content = last_message["content"] - assert isinstance( - last_message_content, list - ), f"Expected list content, got {type(last_message_content)}" - has_cache_point = any( - isinstance(item, dict) and "cachePoint" in item - for item in last_message_content - ) + assert isinstance(last_message_content, list), f"Expected list content, got {type(last_message_content)}" + has_cache_point = any(isinstance(item, dict) and "cachePoint" in item for item in last_message_content) assert has_cache_point, ( f"Expected cachePoint in last message content, got: {last_message_content}. " "String index '-1' was not parsed correctly (str.isdigit() returns False for negative strings)." @@ -1146,17 +1074,13 @@ def test_cache_control_hook_caps_at_four_blocks_with_client_cache_control(): _, processed, _ = hook.get_chat_completion_prompt( model="bedrock/us.anthropic.claude-opus-4-6-v1:0", messages=messages, - non_default_params={ - "cache_control_injection_points": _build_injection_points() - }, + non_default_params={"cache_control_injection_points": _build_injection_points()}, prompt_id=None, prompt_variables=None, dynamic_callback_params={}, ) - assert ( - _count_cache_control(processed) == 4 - ), "Hook must cap cache_control at Anthropic's limit of 4 blocks" + assert _count_cache_control(processed) == 4, "Hook must cap cache_control at Anthropic's limit of 4 blocks" # Client TTL on system blocks must be preserved (not overwritten by config). for i in range(4): @@ -1170,11 +1094,7 @@ def test_cache_control_hook_caps_at_four_blocks_with_client_cache_control(): assert user_message.get("cache_control") is None user_content = user_message.get("content") if isinstance(user_content, list): - assert all( - block.get("cache_control") is None - for block in user_content - if isinstance(block, dict) - ) + assert all(block.get("cache_control") is None for block in user_content if isinstance(block, dict)) def test_cache_control_hook_caps_at_four_blocks_without_client_cache_control(): @@ -1184,17 +1104,13 @@ def test_cache_control_hook_caps_at_four_blocks_without_client_cache_control(): """ hook = AnthropicCacheControlHook() - messages: List[AllMessageValues] = [ - {"role": "system", "content": f"System {i}"} for i in range(4) - ] + messages: List[AllMessageValues] = [{"role": "system", "content": f"System {i}"} for i in range(4)] messages.append({"role": "user", "content": "hello"}) _, processed, _ = hook.get_chat_completion_prompt( model="bedrock/us.anthropic.claude-opus-4-6-v1:0", messages=messages, - non_default_params={ - "cache_control_injection_points": _build_injection_points() - }, + non_default_params={"cache_control_injection_points": _build_injection_points()}, prompt_id=None, prompt_variables=None, dynamic_callback_params={}, @@ -1303,18 +1219,12 @@ async def test_cache_control_hook_bedrock_payload_caps_cachepoints_at_four(): request_body = json.loads(mock_post.call_args.kwargs["data"]) cache_points = sum( - 1 - for block in request_body.get("system", []) - if isinstance(block, dict) and "cachePoint" in block + 1 for block in request_body.get("system", []) if isinstance(block, dict) and "cachePoint" in block ) for msg in request_body.get("messages", []): content = msg.get("content", []) if isinstance(content, list): - cache_points += sum( - 1 - for block in content - if isinstance(block, dict) and "cachePoint" in block - ) + cache_points += sum(1 for block in content if isinstance(block, dict) and "cachePoint" in block) assert cache_points <= 4, ( f"Bedrock payload exceeded Anthropic's 4 cache_control block limit: " @@ -1331,9 +1241,7 @@ def test_cache_control_hook_reserves_slot_for_tool_config_point(): """ hook = AnthropicCacheControlHook() - messages: List[AllMessageValues] = [ - {"role": "system", "content": f"System {i}"} for i in range(4) - ] + messages: List[AllMessageValues] = [{"role": "system", "content": f"System {i}"} for i in range(4)] messages.append({"role": "user", "content": "hello"}) _, processed, non_default_params = hook.get_chat_completion_prompt( @@ -1356,9 +1264,7 @@ def test_cache_control_hook_reserves_slot_for_tool_config_point(): assert _count_cache_control(processed) == 3 # The tool_config point is passed through for the provider transform. - assert non_default_params["cache_control_injection_points"] == [ - {"location": "tool_config"} - ] + assert non_default_params["cache_control_injection_points"] == [{"location": "tool_config"}] @pytest.mark.asyncio @@ -1384,9 +1290,7 @@ async def test_cache_control_hook_bedrock_payload_caps_with_tool_config_point(): client = AsyncHTTPHandler() with patch.object(client, "post", return_value=mock_response) as mock_post: - messages = [ - {"role": "system", "content": f"System block {i}"} for i in range(4) - ] + messages = [{"role": "system", "content": f"System block {i}"} for i in range(4)] messages.append({"role": "user", "content": "What is the weather?"}) await litellm.acompletion( @@ -1421,18 +1325,12 @@ async def test_cache_control_hook_bedrock_payload_caps_with_tool_config_point(): request_body = json.loads(mock_post.call_args.kwargs["data"]) cache_points = sum( - 1 - for block in request_body.get("system", []) - if isinstance(block, dict) and "cachePoint" in block + 1 for block in request_body.get("system", []) if isinstance(block, dict) and "cachePoint" in block ) for msg in request_body.get("messages", []): content = msg.get("content", []) if isinstance(content, list): - cache_points += sum( - 1 - for block in content - if isinstance(block, dict) and "cachePoint" in block - ) + cache_points += sum(1 for block in content if isinstance(block, dict) and "cachePoint" in block) for tool in request_body.get("toolConfig", {}).get("tools", []): if isinstance(tool, dict) and "cachePoint" in tool: cache_points += 1 @@ -1441,3 +1339,197 @@ async def test_cache_control_hook_bedrock_payload_caps_with_tool_config_point(): f"Bedrock payload exceeded Anthropic's 4 cache_control block limit " f"when mixing message and tool_config injection: found {cache_points}" ) + + +class TestApplyToAnthropicMessagesRequest: + """Tests for apply_to_anthropic_messages_request (v1/messages cache control).""" + + def test_system_string_injection(self): + messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}] + system = "You are helpful" + injection_points = [{"location": "message", "role": "system"}] + + result_msgs, result_sys, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + messages=messages, + system=system, + injection_points=injection_points, + ) + + assert result_sys == [{"type": "text", "text": "You are helpful", "cache_control": {"type": "ephemeral"}}] + assert result_msgs == messages + assert remaining == [] + + def test_system_list_injection(self): + messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}] + system = [ + {"type": "text", "text": "Part 1"}, + {"type": "text", "text": "Part 2"}, + ] + injection_points = [{"location": "message", "role": "system"}] + + _, result_sys, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + messages=messages, + system=system, + injection_points=injection_points, + ) + + assert result_sys[0] == {"type": "text", "text": "Part 1"} + assert result_sys[1] == {"type": "text", "text": "Part 2", "cache_control": {"type": "ephemeral"}} + + def test_user_message_injection_by_role(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "First"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "Response"}]}, + {"role": "user", "content": [{"type": "text", "text": "Second"}]}, + ] + injection_points = [{"location": "message", "role": "user"}] + + result_msgs, _, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + messages=messages, + system=None, + injection_points=injection_points, + ) + + assert result_msgs[0]["content"][-1].get("cache_control") == {"type": "ephemeral"} + assert result_msgs[2]["content"][-1].get("cache_control") == {"type": "ephemeral"} + assert result_msgs[1]["content"][-1].get("cache_control") is None + + def test_message_injection_by_index(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "First"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "Response"}]}, + {"role": "user", "content": [{"type": "text", "text": "Second"}]}, + ] + injection_points = [{"location": "message", "index": -1}] + + result_msgs, _, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + messages=messages, + system=None, + injection_points=injection_points, + ) + + assert result_msgs[2]["content"][-1].get("cache_control") == {"type": "ephemeral"} + assert result_msgs[0]["content"][-1].get("cache_control") is None + assert result_msgs[1]["content"][-1].get("cache_control") is None + + def test_mixed_system_and_message_injection(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "Hi"}]}, + {"role": "user", "content": [{"type": "text", "text": "Question"}]}, + ] + system = "System prompt" + injection_points = [ + {"location": "message", "role": "system"}, + {"location": "message", "index": -1}, + ] + + result_msgs, result_sys, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + messages=messages, + system=system, + injection_points=injection_points, + ) + + assert result_sys[0]["cache_control"] == {"type": "ephemeral"} + assert result_msgs[2]["content"][-1].get("cache_control") == {"type": "ephemeral"} + + def test_respects_max_4_blocks(self): + messages = [{"role": "user", "content": [{"type": "text", "text": f"Msg {i}"}]} for i in range(6)] + system = "System" + injection_points = [ + {"location": "message", "role": "system"}, + {"location": "message", "role": "user"}, + ] + + result_msgs, result_sys, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + messages=messages, + system=system, + injection_points=injection_points, + ) + + sys_blocks = sum(1 for b in (result_sys or []) if isinstance(b, dict) and b.get("cache_control") is not None) + total_blocks = sys_blocks + sum(AnthropicCacheControlHook._count_cache_control_blocks(m) for m in result_msgs) + assert total_blocks <= 4 + + def test_tool_config_points_forwarded_as_remaining(self): + messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}] + injection_points = [ + {"location": "message", "role": "user"}, + {"location": "tool_config"}, + ] + + _, _, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + messages=messages, + system=None, + injection_points=injection_points, + ) + + assert remaining == [{"location": "tool_config"}] + + def test_no_injection_points_returns_unchanged(self): + messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}] + system = "System" + + result_msgs, result_sys, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + messages=messages, + system=system, + injection_points=[], + ) + + assert result_msgs == messages + assert result_sys == system + assert remaining == [] + + def test_does_not_mutate_input(self): + messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}] + system = [{"type": "text", "text": "System"}] + injection_points = [{"location": "message", "role": "system"}] + + original_system = copy.deepcopy(system) + original_messages = copy.deepcopy(messages) + + AnthropicCacheControlHook.apply_to_anthropic_messages_request( + messages=messages, + system=system, + injection_points=injection_points, + ) + + assert messages == original_messages + assert system == original_system + + def test_system_none_with_system_point_skipped(self): + messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}] + injection_points = [{"location": "message", "role": "system"}] + + result_msgs, result_sys, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + messages=messages, + system=None, + injection_points=injection_points, + ) + + assert result_sys is None + + def test_existing_cache_control_counted_toward_limit(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "A", "cache_control": {"type": "ephemeral"}}]}, + {"role": "assistant", "content": [{"type": "text", "text": "B", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "C", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "D"}]}, + {"role": "user", "content": [{"type": "text", "text": "E"}]}, + ] + system = "System" + injection_points = [ + {"location": "message", "role": "system"}, + {"location": "message", "index": 3}, + {"location": "message", "index": 4}, + ] + + result_msgs, result_sys, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + messages=messages, + system=system, + injection_points=injection_points, + ) + + sys_blocks = sum(1 for b in (result_sys or []) if isinstance(b, dict) and b.get("cache_control") is not None) + total_blocks = sys_blocks + sum(AnthropicCacheControlHook._count_cache_control_blocks(m) for m in result_msgs) + assert total_blocks <= 4 From bfb8ffccb8e42b69533d95605c5821d88324c870 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Tue, 30 Jun 2026 19:32:27 -0700 Subject: [PATCH 16/81] feat(proxy): audit remaining system-wide settings updates (#31754) * feat(proxy): audit remaining system-wide settings updates Extends the audit logging framework introduced in the parent PR to the rest of the LiteLLM_Config writers and the two adjacent settings tables: /config/update (general, environment_variables, litellm_settings, router_settings sections), /config/field/update, /config/field/delete, /config/callback/delete, /update/default_team_settings, /update/mcp_semantic_filter_settings, /add/allowed_ip, /delete/allowed_ip, /update/sso_settings, /update/ui_theme_settings, /update/ui_settings. Each writer records the actor, action, the affected config section, and a redacted before/after snapshot. SSO and UI settings rows use their own table_name (LiteLLM_SSOConfig, LiteLLM_UISettings). The /config/callback and /update/sso_settings audits fire BEFORE the proxy reload and the env cleanup step respectively, so a failure in either leaves the audit row intact. The audit-actor parameter on _update_litellm_setting is now required rather than optional; the chokepoint covers default_team and mcp_semantic_filter for free, and a future caller that forgets the actor fails loudly instead of silently skipping the audit. The two direct-calling tests pass a dummy actor. The environment_variables section redacts every value rather than relying on key-name matching, because it carries credentials under non-secret-looking uppercase keys (e.g. DATABASE_URL). * fix(proxy): capture redacted SSO before-snapshot in audit log Greptile review of #31754 flagged update_sso_settings as the one endpoint where before_value is permanently None, so the LiteLLM_SSOConfig audit trail has no pre-change state. An auditor reviewing a secret-rotation event could see what the SSO settings were changed to but not what they were before. Read the existing SSO row before the upsert, decrypt it via proxy_config._decrypt_db_variables, and pass it as before_value. create_config_audit_log's secret-name redaction then masks the *_client_secret fields, so neither the old nor the new plaintext secret lands in the audit row. Add a regression test asserting the before-snapshot reflects the pre-change values for non-secret fields (google_client_id) and is redacted for secret fields (google_client_secret). Mutation-checked against reverting to before_value=None. The pre-existing SSO tests now also mock litellm_ssoconfig.find_unique since the endpoint reads it; the read returns None for tests that do not care about the before-state. * fix: remove committed zero init migration * refactor(proxy): audit config writes via asyncio.create_task everywhere PR A's chokepoint audit call was refactored from a blocking await to asyncio.create_task so that a post-save audit-log failure could not surface as a 500 to the caller. The 12 other audit call sites added in this PR were still using await, reintroducing the exact 500-after-commit exposure at every sibling endpoint. Wrap them all in asyncio.create_task to match the model_management_endpoints / key_management_endpoints / hooks / config_override_endpoints / team_callback_endpoints / cache_settings_endpoints house pattern, so the codebase tells one story. The two direct-invocation tests (test_update_config_general_settings and test_delete_config_general_settings, which call the handler in-process rather than via TestClient) yield with `await asyncio.sleep(0)` after the handler returns so the scheduled audit task runs before the assertion. --------- Co-authored-by: Cursor Agent --- litellm/proxy/_types.py | 2 + litellm/proxy/proxy_server.py | 53 ++- .../proxy_setting_endpoints.py | 145 +++++-- .../scim/test_scim_v2_endpoints.py | 2 + .../test_team_default_params.py | 2 + tests/test_litellm/proxy/test_proxy_server.py | 275 ++++++++++++ .../test_proxy_setting_endpoints.py | 407 ++++++++++++++++++ 7 files changed, 861 insertions(+), 25 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index a6ef7de07ae..643f8d69300 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -190,6 +190,8 @@ class LitellmTableNames(str, enum.Enum): CACHE_CONFIG_TABLE_NAME = "LiteLLM_CacheConfig" CONFIG_OVERRIDES_TABLE_NAME = "LiteLLM_ConfigOverrides" CONFIG_TABLE_NAME = "LiteLLM_Config" + SSO_CONFIG_TABLE_NAME = "LiteLLM_SSOConfig" + UI_SETTINGS_TABLE_NAME = "LiteLLM_UISettings" class Litellm_EntityType(enum.Enum): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0158f601d32..2f6c48a751b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -13962,6 +13962,7 @@ async def update_config( # effect of auto-enabling slack alerting. if config_info.general_settings is not None: existing = await _read_section("general_settings") + before_general_settings = copy.deepcopy(existing) updates = config_info.general_settings.dict(exclude_none=True) for k, v in updates.items(): if k == "alert_to_webhook_url": @@ -13971,6 +13972,11 @@ async def update_config( existing["alerting"].append("slack") existing[k] = v await _upsert_section("general_settings", existing) + asyncio.create_task( + create_config_audit_log( + "general_settings", "updated", before_general_settings, existing, user_api_key_dict + ) + ) # environment_variables: idempotently encrypt the request values # (plaintext on first write, OR ciphertext the UI read back via @@ -13979,10 +13985,16 @@ async def update_config( # their stored ciphertext byte-for-byte. if config_info.environment_variables is not None: existing = await _read_section("environment_variables") + before_environment_variables = copy.deepcopy(existing) existing.update( proxy_config._encrypt_env_variables_for_db(environment_variables=config_info.environment_variables) ) await _upsert_section("environment_variables", existing) + asyncio.create_task( + create_config_audit_log( + "environment_variables", "updated", before_environment_variables, existing, user_api_key_dict + ) + ) # litellm_settings: merge existing + request, request wins (matching # router_settings semantics — the caller's value for any given key is @@ -13994,6 +14006,7 @@ async def update_config( # entries that delete_callback (lowercase lookup) cannot find. if config_info.litellm_settings is not None: existing = await _read_section("litellm_settings") + before_litellm_settings = copy.deepcopy(existing) updated_litellm_settings = dict(config_info.litellm_settings) incoming_cb = updated_litellm_settings.get("success_callback") @@ -14015,12 +14028,24 @@ async def update_config( merged["success_callback"] = list(set(incoming_cb)) await _upsert_section("litellm_settings", merged) + asyncio.create_task( + create_config_audit_log( + "litellm_settings", "updated", before_litellm_settings, merged, user_api_key_dict + ) + ) # router_settings: merge existing + request, request wins. if config_info.router_settings is not None: existing = await _read_section("router_settings") + before_router_settings = copy.deepcopy(existing) updates = config_info.router_settings.dict(exclude_none=True) - await _upsert_section("router_settings", {**existing, **updates}) + new_router_settings = {**existing, **updates} + await _upsert_section("router_settings", new_router_settings) + asyncio.create_task( + create_config_audit_log( + "router_settings", "updated", before_router_settings, new_router_settings, user_api_key_dict + ) + ) await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) @@ -14152,6 +14177,8 @@ async def update_config_general_settings( else: general_settings = dict(db_general_settings.param_value) + before_general_settings = copy.deepcopy(general_settings) + ## update db field_value = data.field_value @@ -14171,6 +14198,11 @@ async def update_config_general_settings( }, ) await invalidate_config_param("general_settings") + asyncio.create_task( + create_config_audit_log( + "general_settings", "updated", before_general_settings, general_settings, user_api_key_dict + ) + ) if data.field_name == "plugins": register_plugins_from_config(general_settings) @@ -14555,6 +14587,8 @@ async def delete_config_general_settings( else: general_settings = dict(db_general_settings.param_value) + before_general_settings = copy.deepcopy(general_settings) + ## update db general_settings.pop(data.field_name, None) @@ -14570,6 +14604,11 @@ async def delete_config_general_settings( }, ) await invalidate_config_param("general_settings") + asyncio.create_task( + create_config_audit_log( + "general_settings", "deleted", before_general_settings, general_settings, user_api_key_dict + ) + ) return response @@ -14627,6 +14666,8 @@ async def delete_callback( detail={"error": f"Callback '{callback_name}' not found in active configuration"}, ) + before_success_callbacks = list(success_callbacks) + # Remove callback from success_callback list success_callbacks.remove(callback_name) config.setdefault("litellm_settings", {})["success_callback"] = success_callbacks @@ -14634,6 +14675,16 @@ async def delete_callback( # Save the updated configuration await proxy_config.save_config(new_config=config) + asyncio.create_task( + create_config_audit_log( + "litellm_settings", + "deleted", + {"success_callback": before_success_callbacks}, + {"success_callback": success_callbacks}, + user_api_key_dict, + ) + ) + # Restart the proxy to apply changes await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 1be17c86123..0fc303737b4 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -323,8 +323,12 @@ async def get_allowed_ips(): tags=["Budget & Spend Tracking"], dependencies=[Depends(user_api_key_auth)], ) -async def add_allowed_ip(ip_address: IPAddress): +async def add_allowed_ip( + ip_address: IPAddress, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): from litellm.proxy.proxy_server import ( + create_config_audit_log, general_settings, prisma_client, proxy_config, @@ -356,11 +360,22 @@ async def add_allowed_ip(ip_address: IPAddress): if "allowed_ips" not in config["general_settings"]: config["general_settings"]["allowed_ips"] = [] + before_allowed_ips = list(config["general_settings"]["allowed_ips"]) if ip_address.ip not in config["general_settings"]["allowed_ips"]: config["general_settings"]["allowed_ips"].append(ip_address.ip) await proxy_config.save_config(new_config=config) + asyncio.create_task( + create_config_audit_log( + param_name="general_settings", + action="updated", + before_value={"allowed_ips": before_allowed_ips}, + after_value={"allowed_ips": config["general_settings"]["allowed_ips"]}, + user_api_key_dict=user_api_key_dict, + ) + ) + return { "message": f"IP {ip_address.ip} address added successfully", "status": "success", @@ -372,8 +387,15 @@ async def add_allowed_ip(ip_address: IPAddress): tags=["Budget & Spend Tracking"], dependencies=[Depends(user_api_key_auth)], ) -async def delete_allowed_ip(ip_address: IPAddress): - from litellm.proxy.proxy_server import general_settings, proxy_config +async def delete_allowed_ip( + ip_address: IPAddress, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + from litellm.proxy.proxy_server import ( + create_config_audit_log, + general_settings, + proxy_config, + ) _allowed_ips: List = general_settings.get("allowed_ips", []) if ip_address.ip in _allowed_ips: @@ -391,11 +413,22 @@ async def delete_allowed_ip(ip_address: IPAddress): if "allowed_ips" not in config["general_settings"]: config["general_settings"]["allowed_ips"] = [] + before_allowed_ips = list(config["general_settings"]["allowed_ips"]) if ip_address.ip in config["general_settings"]["allowed_ips"]: config["general_settings"]["allowed_ips"].remove(ip_address.ip) await proxy_config.save_config(new_config=config) + asyncio.create_task( + create_config_audit_log( + param_name="general_settings", + action="deleted", + before_value={"allowed_ips": before_allowed_ips}, + after_value={"allowed_ips": config["general_settings"]["allowed_ips"]}, + user_api_key_dict=user_api_key_dict, + ) + ) + return {"message": f"IP {ip_address.ip} deleted successfully", "status": "success"} @@ -554,7 +587,7 @@ async def _update_litellm_setting( settings: Union[DefaultInternalUserParams, DefaultTeamSSOParams, MCPSemanticFilterSettings], settings_key: str, success_message: str, - user_api_key_dict: Optional[UserAPIKeyAuth] = None, + user_api_key_dict: UserAPIKeyAuth, ): """ Common utility function to update `litellm_settings` in both memory and config. @@ -564,8 +597,6 @@ async def _update_litellm_setting( settings_key: The key in litellm_settings to update success_message: Message to return on success user_api_key_dict: The acting admin, recorded as the audit-log actor. - Optional today so callers that have not been wired for auditing - keep working; the audit row is only written when an actor is passed. """ from litellm.proxy.proxy_server import ( create_config_audit_log, @@ -599,20 +630,19 @@ async def _update_litellm_setting( # Save the updated config await proxy_config.save_config(new_config=config) - if user_api_key_dict is not None: - # Fire-and-forget so an audit-log failure (transient DB blip, etc.) - # never surfaces as a 500 after save_config has already committed, - # matching the create_object_audit_log pattern used elsewhere - # (e.g. model_management_endpoints). - asyncio.create_task( - create_config_audit_log( - param_name=settings_key, - action="updated", - before_value=before_value, - after_value=in_memory_var, - user_api_key_dict=user_api_key_dict, - ) + # Fire-and-forget so an audit-log failure (transient DB blip, etc.) + # never surfaces as a 500 after save_config has already committed, + # matching the create_object_audit_log pattern used elsewhere + # (e.g. model_management_endpoints). + asyncio.create_task( + create_config_audit_log( + param_name=settings_key, + action="updated", + before_value=before_value, + after_value=in_memory_var, + user_api_key_dict=user_api_key_dict, ) + ) return { "message": success_message, @@ -653,7 +683,10 @@ async def update_internal_user_settings( tags=["SSO Settings"], dependencies=[Depends(user_api_key_auth)], ) -async def update_default_team_settings(settings: DefaultTeamSSOParams): +async def update_default_team_settings( + settings: DefaultTeamSSOParams, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ Update the default team parameters for SSO users. These settings will be applied to new teams created from SSO. @@ -662,6 +695,7 @@ async def update_default_team_settings(settings: DefaultTeamSSOParams): settings=settings, settings_key="default_team_params", success_message="Default team settings updated successfully", + user_api_key_dict=user_api_key_dict, ) @@ -772,7 +806,10 @@ async def get_sso_settings(): tags=["SSO Settings"], dependencies=[Depends(user_api_key_auth)], ) -async def update_sso_settings(sso_config: SSOConfig): +async def update_sso_settings( + sso_config: SSOConfig, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ Update SSO configuration by saving to the dedicated SSO table. """ @@ -780,6 +817,7 @@ async def update_sso_settings(sso_config: SSOConfig): import os from litellm.proxy.proxy_server import ( + create_config_audit_log, prisma_client, proxy_config, store_model_in_db, @@ -812,6 +850,20 @@ async def update_sso_settings(sso_config: SSOConfig): "proxy_base_url": "PROXY_BASE_URL", } + # Read the existing SSO row first so the audit log captures a real + # before/after diff. Stored values are encrypted; decrypt them so the + # before-snapshot has the same shape as after_value, and rely on + # create_config_audit_log's secret-name redaction to mask the + # *_client_secret fields before the audit row is written. + existing_sso_record = await SSOConfigRepository(prisma_client).table.find_unique(where={"id": "sso_config"}) + before_sso_data: Optional[Dict[str, Any]] = None + if existing_sso_record and existing_sso_record.sso_settings: + stored = existing_sso_record.sso_settings + if isinstance(stored, str): + stored = json.loads(stored) + if isinstance(stored, dict): + before_sso_data = proxy_config._decrypt_db_variables(stored) + # Load existing config config = await proxy_config.get_config() @@ -850,6 +902,17 @@ async def update_sso_settings(sso_config: SSOConfig): }, ) + asyncio.create_task( + create_config_audit_log( + param_name="sso_config", + action="updated", + before_value=before_sso_data, + after_value=sso_data, + user_api_key_dict=user_api_key_dict, + table_name=LitellmTableNames.SSO_CONFIG_TABLE_NAME, + ) + ) + # Remove SSO-related env vars from config.environment_variables try: env_var_entry = await ConfigRepository(prisma_client).table.find_unique( @@ -943,14 +1006,21 @@ def _validate_public_image_url(value: Optional[str], field_name: str) -> None: tags=["UI Theme Settings"], dependencies=[Depends(user_api_key_auth)], ) -async def update_ui_theme_settings(theme_config: UIThemeConfig): +async def update_ui_theme_settings( + theme_config: UIThemeConfig, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ Update UI theme configuration. Updates logo settings for the admin UI. """ import os - from litellm.proxy.proxy_server import proxy_config, store_model_in_db + from litellm.proxy.proxy_server import ( + create_config_audit_log, + proxy_config, + store_model_in_db, + ) _validate_public_image_url(theme_config.logo_url, "logo_url") _validate_public_image_url(theme_config.favicon_url, "favicon_url") @@ -963,6 +1033,7 @@ async def update_ui_theme_settings(theme_config: UIThemeConfig): # Load existing config config = await proxy_config.get_config() + before_theme = config.get("litellm_settings", {}).get("ui_theme_config") # Update config with UI theme settings if "general_settings" not in config: @@ -1029,6 +1100,16 @@ async def update_ui_theme_settings(theme_config: UIThemeConfig): # Save the updated config await proxy_config.save_config(new_config=stored_config) + asyncio.create_task( + create_config_audit_log( + param_name="ui_theme_config", + action="updated", + before_value=before_theme, + after_value=theme_data, + user_api_key_dict=user_api_key_dict, + ) + ) + return { "message": "UI theme settings updated successfully.", "status": "success", @@ -1083,6 +1164,7 @@ async def update_mcp_semantic_filter_settings( settings=settings, settings_key="mcp_semantic_tool_filter", success_message="MCP Semantic Filter settings updated successfully. Changes will be applied across all pods within 10 seconds.", + user_api_key_dict=user_api_key_dict, ) try: from litellm.proxy.proxy_server import prisma_client, proxy_config @@ -1200,7 +1282,11 @@ async def update_ui_settings( Update UI-specific configuration flags. Only proxy admins are allowed to modify these settings. """ - from litellm.proxy.proxy_server import prisma_client, store_model_in_db + from litellm.proxy.proxy_server import ( + create_config_audit_log, + prisma_client, + store_model_in_db, + ) if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: raise HTTPException(status_code=403, detail="Only proxy admins can update UI settings.") @@ -1282,6 +1368,17 @@ async def update_ui_settings( sanitized = {k: v for k, v in ui_settings.items() if k in ALLOWED_UI_SETTINGS_FIELDS} await user_api_key_cache.async_set_cache(key=UI_SETTINGS_CACHE_KEY, value=sanitized, ttl=UI_SETTINGS_CACHE_TTL) + asyncio.create_task( + create_config_audit_log( + param_name="ui_settings", + action="updated", + before_value=existing, + after_value=ui_settings, + user_api_key_dict=user_api_key_dict, + table_name=LitellmTableNames.UI_SETTINGS_TABLE_NAME, + ) + ) + return { "message": "UI settings updated successfully", "status": "success", diff --git a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py index 7f5aee51f51..f39ff93cee7 100644 --- a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py @@ -258,6 +258,7 @@ async def test_scim_create_user_respects_default_role_set_via_ui(mocker, monkeyp ) import litellm + from litellm.proxy._types import UserAPIKeyAuth settings = DefaultInternalUserParams( user_role=LitellmUserRoles.INTERNAL_USER, @@ -266,6 +267,7 @@ async def test_scim_create_user_respects_default_role_set_via_ui(mocker, monkeyp settings=settings, settings_key="default_internal_user_params", success_message="ok", + user_api_key_dict=UserAPIKeyAuth(user_id="test-admin"), ) # Verify the in-memory variable was actually updated diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py b/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py index 443089b5f01..e0b90332ca0 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py @@ -426,6 +426,7 @@ class TestUpdateLitellmSettingOrdering: settings=new_settings, settings_key="default_team_params", success_message="Updated", + user_api_key_dict=UserAPIKeyAuth(user_id="test-admin"), ) # In-memory value should be the NEW value, not the stale one @@ -459,6 +460,7 @@ class TestUpdateLitellmSettingOrdering: settings=DefaultTeamSSOParams(max_budget=100.0), settings_key="default_team_params", success_message="Updated", + user_api_key_dict=UserAPIKeyAuth(user_id="test-admin"), ) assert exc_info.value.status_code == 500 diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index dc35d71ccbd..88d9ad0d968 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -8764,3 +8764,278 @@ def test_dump_redacted_config_serializes_non_json_native_values(): restored = json.loads(out) assert "2026-06-30" in restored["updated_at"] + +@pytest.mark.asyncio +async def test_update_config_general_settings_emits_audit_log(monkeypatch): + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import ConfigFieldUpdate + from litellm.proxy.proxy_server import update_config_general_settings + + existing = {"max_parallel_requests": 5, "some_api_key": "sk-stored-secret"} + fake = _fake_prisma_with_config(existing) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + + admin = UserAPIKeyAuth( + api_key="hashed-admin", + user_id="admin-1", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + await update_config_general_settings( + data=ConfigFieldUpdate( + field_name="max_parallel_requests", + field_value=42, + config_type="general_settings", + ), + user_api_key_dict=admin, + ) + # Audit is scheduled via asyncio.create_task; yield so it runs. + await asyncio.sleep(0) + + fake.db.litellm_auditlog.create.assert_awaited_once() + written = fake.db.litellm_auditlog.create.call_args.kwargs["data"] + assert written["table_name"] == "LiteLLM_Config" + assert written["object_id"] == "general_settings" + assert written["action"] == "updated" + assert written["changed_by"] == "admin-1" + + before = json.loads(written["before_value"]) + after = json.loads(written["updated_values"]) + assert before["max_parallel_requests"] == 5 + assert after["max_parallel_requests"] == 42 + assert "sk-stored-secret" not in written["before_value"] + assert "sk-stored-secret" not in written["updated_values"] + assert before["some_api_key"] != "sk-stored-secret" + + +@pytest.mark.asyncio +async def test_delete_config_general_settings_emits_deleted_audit_log(monkeypatch): + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import ConfigFieldDelete + from litellm.proxy.proxy_server import delete_config_general_settings + + existing = {"max_parallel_requests": 5} + fake = _fake_prisma_with_config(existing) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + + admin = UserAPIKeyAuth( + api_key="hashed-admin", + user_id="admin-1", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + await delete_config_general_settings( + data=ConfigFieldDelete( + field_name="max_parallel_requests", config_type="general_settings" + ), + user_api_key_dict=admin, + ) + # Audit is scheduled via asyncio.create_task; yield so it runs. + await asyncio.sleep(0) + + fake.db.litellm_auditlog.create.assert_awaited_once() + written = fake.db.litellm_auditlog.create.call_args.kwargs["data"] + assert written["object_id"] == "general_settings" + assert written["action"] == "deleted" + before = json.loads(written["before_value"]) + after = json.loads(written["updated_values"]) + assert before["max_parallel_requests"] == 5 + assert "max_parallel_requests" not in after + + +def test_update_config_audits_every_written_section(_update_config_setup, monkeypatch): + """/config/update must emit one audit row per section it writes, so each + of the four call sites (general_settings, environment_variables, + litellm_settings, router_settings) is mutation-protected. litellm_settings + is the row that holds default_internal_user_params ("default user settings").""" + import litellm.proxy.proxy_server as proxy_server_module + + client, prisma, restore = _update_config_setup( + initial_rows={"litellm_settings": {"drop_params": True}} + ) + audit_create = AsyncMock() + prisma.db.litellm_auditlog.create = audit_create + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + try: + resp = client.post( + "/config/update", + json={ + "general_settings": {"store_prompts_in_spend_logs": True}, + "environment_variables": {"FOO": "bar"}, + "litellm_settings": { + "default_internal_user_params": {"max_budget": 10} + }, + "router_settings": {"routing_strategy": "latency-based-routing"}, + }, + ) + assert resp.status_code == 200, resp.text + + audited = { + call.kwargs["data"]["object_id"]: call.kwargs["data"]["action"] + for call in audit_create.await_args_list + } + assert audited == { + "general_settings": "updated", + "environment_variables": "updated", + "litellm_settings": "updated", + "router_settings": "updated", + } + for call in audit_create.await_args_list: + assert call.kwargs["data"]["table_name"] == "LiteLLM_Config" + assert call.kwargs["data"]["changed_by"] == "test_admin" + + ls_call = next( + c + for c in audit_create.await_args_list + if c.kwargs["data"]["object_id"] == "litellm_settings" + ) + after = json.loads(ls_call.kwargs["data"]["updated_values"]) + assert after["default_internal_user_params"] == {"max_budget": 10} + finally: + restore() + + +def test_delete_callback_audits_litellm_settings_deletion( + _update_config_setup, monkeypatch +): + """/config/callback/delete must emit a deleted audit row for litellm_settings + capturing the success_callback list before and after removal.""" + import litellm.proxy.proxy_server as proxy_server_module + + client, prisma, restore = _update_config_setup() + audit_create = AsyncMock() + prisma.db.litellm_auditlog.create = audit_create + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + + from litellm.proxy.proxy_server import proxy_config as real_proxy_config + + monkeypatch.setattr( + real_proxy_config, + "get_config", + AsyncMock( + return_value={ + "litellm_settings": {"success_callback": ["langfuse", "datadog"]} + } + ), + ) + monkeypatch.setattr( + real_proxy_config, "save_config", AsyncMock(return_value=None) + ) + try: + resp = client.post( + "/config/callback/delete", json={"callback_name": "datadog"} + ) + assert resp.status_code == 200, resp.text + + audit_create.assert_awaited_once() + written = audit_create.await_args.kwargs["data"] + assert written["object_id"] == "litellm_settings" + assert written["action"] == "deleted" + before = json.loads(written["before_value"]) + after = json.loads(written["updated_values"]) + assert before["success_callback"] == ["langfuse", "datadog"] + assert after["success_callback"] == ["langfuse"] + finally: + restore() + + +def test_delete_callback_audits_before_reload_failure(_update_config_setup, monkeypatch): + import litellm.proxy.proxy_server as proxy_server_module + + client, prisma, restore = _update_config_setup() + audit_create = AsyncMock() + prisma.db.litellm_auditlog.create = audit_create + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + + from litellm.proxy.proxy_server import proxy_config as real_proxy_config + + monkeypatch.setattr( + real_proxy_config, + "get_config", + AsyncMock( + return_value={ + "litellm_settings": {"success_callback": ["langfuse", "datadog"]} + } + ), + ) + monkeypatch.setattr( + real_proxy_config, "save_config", AsyncMock(return_value=None) + ) + monkeypatch.setattr( + real_proxy_config, + "add_deployment", + AsyncMock(side_effect=RuntimeError("reload failed")), + ) + try: + resp = client.post( + "/config/callback/delete", json={"callback_name": "datadog"} + ) + assert resp.status_code == 500, resp.text + + audit_create.assert_awaited_once() + written = audit_create.await_args.kwargs["data"] + assert written["object_id"] == "litellm_settings" + assert written["action"] == "deleted" + finally: + restore() + + +def test_update_config_redacts_all_environment_variable_values( + _update_config_setup, monkeypatch +): + """environment_variables hold credentials under arbitrary uppercase keys + (DATABASE_URL) that key-name secret matching misses, so every value in the + section must be redacted before the audit row is written; a plaintext + secret must never reach LiteLLM_AuditLog.""" + import litellm.proxy.proxy_server as proxy_server_module + + # DATABASE_URL is the bug class: an uppercase env key that key-name secret + # matching does NOT flag, so only whole-section value redaction protects it. + client, prisma, restore = _update_config_setup( + initial_rows={ + "environment_variables": { + "DATABASE_URL": "enc:postgresql://OLDsecret@old.host:5432/db" + } + } + ) + audit_create = AsyncMock() + prisma.db.litellm_auditlog.create = audit_create + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + try: + resp = client.post( + "/config/update", + json={ + "environment_variables": { + "DATABASE_URL": "postgresql://u:p@db.internal:5432/litellm", + "LOG_LEVEL": "debug", + } + }, + ) + assert resp.status_code == 200, resp.text + + env_call = next( + c + for c in audit_create.await_args_list + if c.kwargs["data"]["object_id"] == "environment_variables" + ) + data = env_call.kwargs["data"] + + # the pre-existing secret must be redacted in the before snapshot + before = json.loads(data["before_value"]) + assert before == {"DATABASE_URL": "REDACTED"} + assert "OLDsecret" not in data["before_value"] + assert "old.host" not in data["before_value"] + + # the newly-written values must be redacted in the after snapshot + after = json.loads(data["updated_values"]) + assert after == {"DATABASE_URL": "REDACTED", "LOG_LEVEL": "REDACTED"} + assert "postgresql://" not in data["updated_values"] + assert "db.internal" not in data["updated_values"] + finally: + restore() diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index 7a586f758f4..cb77c42fe9b 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -396,6 +396,7 @@ class TestProxySettingEndpoints: # Mock the prisma client mock_prisma = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() mock_prisma.db.litellm_config = MagicMock() mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) @@ -466,6 +467,62 @@ class TestProxySettingEndpoints: create_sso_settings = json.loads(create_data["sso_settings"]) assert create_sso_settings["google_client_id"] == "new_google_client_id" + def test_update_sso_settings_audits_when_env_cleanup_fails( + self, mock_proxy_config, mock_auth, monkeypatch + ): + import json + from unittest.mock import AsyncMock, MagicMock + + monkeypatch.setenv("LITELLM_SALT_KEY", "test_salt_key") + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + + mock_prisma = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) + mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() + mock_prisma.db.litellm_config = MagicMock() + mock_prisma.db.litellm_config.find_unique = AsyncMock( + side_effect=ValueError("cleanup failed") + ) + mock_prisma.db.litellm_config.update = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + + from litellm.proxy.proxy_server import proxy_config + + monkeypatch.setattr( + proxy_config, + "_encrypt_env_variables", + lambda environment_variables: environment_variables, + ) + + create_config_audit_log = AsyncMock() + monkeypatch.setattr( + "litellm.proxy.proxy_server.create_config_audit_log", + create_config_audit_log, + ) + + response = client.patch( + "/update/sso_settings", + json={"google_client_id": "new_google_client_id"}, + ) + + assert response.status_code == 500 + assert mock_prisma.db.litellm_ssoconfig.upsert.called + create_config_audit_log.assert_awaited_once() + audit_log_kwargs = create_config_audit_log.await_args.kwargs + assert audit_log_kwargs["param_name"] == "sso_config" + assert ( + audit_log_kwargs["after_value"]["google_client_id"] + == "new_google_client_id" + ) + assert ( + json.loads( + mock_prisma.db.litellm_ssoconfig.upsert.call_args.kwargs["data"][ + "create" + ]["sso_settings"] + )["google_client_id"] + == "new_google_client_id" + ) + def test_update_sso_settings_with_null_values_clears_env_vars( self, mock_proxy_config, mock_auth, monkeypatch ): @@ -478,6 +535,7 @@ class TestProxySettingEndpoints: # Mock the prisma client mock_prisma = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() mock_prisma.db.litellm_config = MagicMock() @@ -557,6 +615,7 @@ class TestProxySettingEndpoints: # Mock the prisma client mock_prisma = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() mock_prisma.db.litellm_config = MagicMock() env_var_entry = MagicMock() @@ -627,6 +686,7 @@ class TestProxySettingEndpoints: # Mock the prisma client mock_prisma = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() mock_prisma.db.litellm_config = MagicMock() @@ -704,6 +764,7 @@ class TestProxySettingEndpoints: # Mock the prisma client mock_prisma = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() mock_prisma.db.litellm_config = MagicMock() mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) @@ -1350,6 +1411,7 @@ class TestProxySettingEndpoints: # Mock the prisma client mock_prisma = MagicMock() upsert_mock = AsyncMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = upsert_mock mock_prisma.db.litellm_config = MagicMock() mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) @@ -1429,6 +1491,7 @@ class TestProxySettingEndpoints: mock_prisma = MagicMock() mock_prisma.db = MagicMock() mock_prisma.db.litellm_ssoconfig = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() env_var_entry = MagicMock() @@ -1480,6 +1543,7 @@ class TestProxySettingEndpoints: mock_prisma = MagicMock() mock_prisma.db = MagicMock() mock_prisma.db.litellm_ssoconfig = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() env_var_entry = MagicMock() @@ -1651,6 +1715,7 @@ class TestProxySettingEndpoints: # Mock the prisma client mock_prisma = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() mock_prisma.db.litellm_config = MagicMock() mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) @@ -1960,3 +2025,345 @@ def test_update_internal_user_settings_returns_200_when_audit_write_raises( assert resp.json()["status"] == "success" finally: app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_update_sso_settings_writes_redacted_audit_log(mock_proxy_config, monkeypatch): + """Updating SSO settings must write an audit row to the SSO config table + with the client secret redacted.""" + from unittest.mock import AsyncMock, MagicMock + + import litellm + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + audit_create = AsyncMock() + fake_prisma = MagicMock() + fake_prisma.db.litellm_auditlog.create = audit_create + fake_prisma.db.litellm_ssoconfig.upsert = AsyncMock() + # No prior SSO row, so before_value resolves to None. + fake_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) + fake_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) + + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + monkeypatch.setattr( + proxy_server_module.proxy_config, + "_encrypt_env_variables", + lambda environment_variables: environment_variables, + ) + + async def _admin_auth(): + return UserAPIKeyAuth( + user_id="audit-admin", + api_key="hashed-admin-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + app.dependency_overrides[user_api_key_auth] = _admin_auth + try: + resp = client.patch( + "/update/sso_settings", + json={ + "google_client_id": "client-id-123", + "google_client_secret": "super-secret-xyz", + }, + ) + assert resp.status_code == 200, resp.text + + audit_create.assert_awaited_once() + written = audit_create.await_args.kwargs["data"] + assert written["object_id"] == "sso_config" + assert written["table_name"] == "LiteLLM_SSOConfig" + assert written["changed_by"] == "audit-admin" + + after = json.loads(written["updated_values"]) + assert after["google_client_id"] == "client-id-123" + assert after["google_client_secret"] == "REDACTED" + assert "super-secret-xyz" not in written["updated_values"] + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_update_sso_settings_audit_captures_redacted_before_snapshot( + mock_proxy_config, monkeypatch +): + """An auditor reviewing an SSO secret rotation needs to see a real + before/after diff in the audit row, not before_value=None. The endpoint + reads the existing (encrypted) SSO row, decrypts it, and lets the audit + helper redact the *_client_secret fields before persistence so neither + the old nor the new plaintext secret is recorded.""" + from unittest.mock import AsyncMock, MagicMock + + import litellm + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + audit_create = AsyncMock() + fake_prisma = MagicMock() + fake_prisma.db.litellm_auditlog.create = audit_create + fake_prisma.db.litellm_ssoconfig.upsert = AsyncMock() + + # Pre-existing SSO row contains the *prior* secret (would be ciphertext in + # production; the test patches _decrypt_db_variables to pass through). + existing_record = MagicMock() + existing_record.sso_settings = { + "google_client_id": "old-client-id", + "google_client_secret": "OLD-SUPER-SECRET", + } + fake_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=existing_record) + fake_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) + + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + monkeypatch.setattr( + proxy_server_module.proxy_config, + "_encrypt_env_variables", + lambda environment_variables: environment_variables, + ) + # Pretend the stored value is already plaintext for the test (production + # decrypts via Fernet); the audit helper still has to redact it. + monkeypatch.setattr( + proxy_server_module.proxy_config, + "_decrypt_db_variables", + lambda variables_dict: dict(variables_dict), + ) + + async def _admin_auth(): + return UserAPIKeyAuth( + user_id="audit-admin", + api_key="hashed-admin-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + app.dependency_overrides[user_api_key_auth] = _admin_auth + try: + resp = client.patch( + "/update/sso_settings", + json={ + "google_client_id": "new-client-id", + "google_client_secret": "NEW-SUPER-SECRET", + }, + ) + assert resp.status_code == 200, resp.text + + audit_create.assert_awaited_once() + written = audit_create.await_args.kwargs["data"] + before = json.loads(written["before_value"]) + after = json.loads(written["updated_values"]) + + # Non-secret field shows the diff + assert before["google_client_id"] == "old-client-id" + assert after["google_client_id"] == "new-client-id" + + # Secret field is redacted in BOTH snapshots — auditor sees the + # rotation event without ever seeing either plaintext secret. + assert before["google_client_secret"] == "REDACTED" + assert after["google_client_secret"] == "REDACTED" + assert "OLD-SUPER-SECRET" not in written["before_value"] + assert "NEW-SUPER-SECRET" not in written["updated_values"] + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_add_allowed_ip_writes_audit_log(mock_proxy_config, monkeypatch): + """Adding an allowed IP is a system-wide security setting change and must + be audited with the before and after IP list.""" + from unittest.mock import AsyncMock, MagicMock + + import litellm + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + audit_create = AsyncMock() + fake_prisma = MagicMock() + fake_prisma.db.litellm_auditlog.create = audit_create + + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr(proxy_server_module, "general_settings", {}) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + + async def _admin_auth(): + return UserAPIKeyAuth( + user_id="audit-admin", + api_key="hashed-admin-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + app.dependency_overrides[user_api_key_auth] = _admin_auth + try: + resp = client.post("/add/allowed_ip", json={"ip": "203.0.113.77"}) + assert resp.status_code == 200, resp.text + + audit_create.assert_awaited_once() + written = audit_create.await_args.kwargs["data"] + assert written["object_id"] == "general_settings" + assert written["action"] == "updated" + assert written["changed_by"] == "audit-admin" + + before = json.loads(written["before_value"]) + after = json.loads(written["updated_values"]) + assert "203.0.113.77" not in before["allowed_ips"] + assert "203.0.113.77" in after["allowed_ips"] + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_delete_allowed_ip_writes_deleted_audit_log(monkeypatch): + """Removing an allowed IP must be audited as a deletion, symmetric with the + add path.""" + from unittest.mock import AsyncMock, MagicMock + + import litellm + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + audit_create = AsyncMock() + fake_prisma = MagicMock() + fake_prisma.db.litellm_auditlog.create = audit_create + + config = {"general_settings": {"allowed_ips": ["203.0.113.77", "198.51.100.1"]}} + + async def _get_config(): + return config + + async def _save_config(new_config=None): + nonlocal config + if new_config is not None: + config = new_config + return config + + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr( + proxy_server_module, "general_settings", {"allowed_ips": ["203.0.113.77"]} + ) + monkeypatch.setattr(litellm, "store_audit_logs", True) + monkeypatch.setattr(proxy_server_module.proxy_config, "get_config", _get_config) + monkeypatch.setattr(proxy_server_module.proxy_config, "save_config", _save_config) + + async def _admin_auth(): + return UserAPIKeyAuth( + user_id="audit-admin", + api_key="hashed-admin-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + app.dependency_overrides[user_api_key_auth] = _admin_auth + try: + resp = client.post("/delete/allowed_ip", json={"ip": "203.0.113.77"}) + assert resp.status_code == 200, resp.text + + audit_create.assert_awaited_once() + written = audit_create.await_args.kwargs["data"] + assert written["object_id"] == "general_settings" + assert written["action"] == "deleted" + before = json.loads(written["before_value"]) + after = json.loads(written["updated_values"]) + assert "203.0.113.77" in before["allowed_ips"] + assert "203.0.113.77" not in after["allowed_ips"] + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_update_ui_theme_settings_writes_audit_log(mock_proxy_config, monkeypatch): + """Updating the UI theme must be audited under ui_theme_config.""" + from unittest.mock import AsyncMock, MagicMock + + import litellm + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + audit_create = AsyncMock() + fake_prisma = MagicMock() + fake_prisma.db.litellm_auditlog.create = audit_create + + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + monkeypatch.setattr( + proxy_server_module.proxy_config, + "_encrypt_env_variables", + lambda environment_variables: environment_variables, + ) + + async def _admin_auth(): + return UserAPIKeyAuth( + user_id="audit-admin", + api_key="hashed-admin-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + app.dependency_overrides[user_api_key_auth] = _admin_auth + try: + resp = client.patch( + "/update/ui_theme_settings", + json={"logo_url": "https://example.com/logo.png"}, + ) + assert resp.status_code == 200, resp.text + + audit_create.assert_awaited_once() + written = audit_create.await_args.kwargs["data"] + assert written["object_id"] == "ui_theme_config" + assert written["action"] == "updated" + assert written["changed_by"] == "audit-admin" + after = json.loads(written["updated_values"]) + assert after["logo_url"] == "https://example.com/logo.png" + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_update_ui_settings_writes_audit_log(monkeypatch): + """Updating UI settings must be audited under the UI settings table.""" + from unittest.mock import AsyncMock, MagicMock + + import litellm + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + audit_create = AsyncMock() + fake_prisma = MagicMock() + fake_prisma.db.litellm_auditlog.create = audit_create + fake_prisma.db.litellm_uisettings.find_unique = AsyncMock(return_value=None) + fake_prisma.db.litellm_uisettings.upsert = AsyncMock() + + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + + async def _admin_auth(): + return UserAPIKeyAuth( + user_id="audit-admin", + api_key="hashed-admin-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + app.dependency_overrides[user_api_key_auth] = _admin_auth + try: + resp = client.patch( + "/update/ui_settings", json={"disable_custom_api_keys": True} + ) + assert resp.status_code == 200, resp.text + + audit_create.assert_awaited_once() + written = audit_create.await_args.kwargs["data"] + assert written["object_id"] == "ui_settings" + assert written["table_name"] == "LiteLLM_UISettings" + assert written["changed_by"] == "audit-admin" + after = json.loads(written["updated_values"]) + assert after["disable_custom_api_keys"] is True + finally: + app.dependency_overrides.pop(user_api_key_auth, None) From c4a77bded7b3e21e0ca8bf52caaa75b9442f6145 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:44:02 +0800 Subject: [PATCH 17/81] fix(prometheus): expose project_alias in custom metadata labels (LIT-3741) (#31784) Include top-level scalar fields from standard_logging_metadata in the combined metadata dict used by custom_prometheus_metadata_labels. Previously only nested sub-dicts (requester_metadata, user_api_key_auth_metadata, spend_logs_metadata) were spread into combined_metadata, so fields like user_api_key_project_alias were inaccessible and always resolved to None. Co-authored-by: unknown <> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/prometheus.py | 5 + .../test_prometheus_spend_logs_metadata.py | 98 ++++++++++++++++++- 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 1f516e9dc93..8eb6eaa8e2b 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -3804,6 +3804,10 @@ def _get_combined_custom_metadata_from_standard_logging_payload( ) -> Dict[str, Any]: """ Combine the metadata sources that can supply custom Prometheus labels. + + Includes top-level scalar fields from the standard logging metadata (e.g. + user_api_key_project_alias, user_api_key_team_alias) so they are accessible + via custom_prometheus_metadata_labels configuration. """ if not isinstance(standard_logging_payload, dict): return {} @@ -3817,6 +3821,7 @@ def _get_combined_custom_metadata_from_standard_logging_payload( spend_logs_metadata = standard_logging_metadata.get("spend_logs_metadata") return { + **{k: v for k, v in standard_logging_metadata.items() if not isinstance(v, dict)}, **(requester_metadata if isinstance(requester_metadata, dict) else {}), **(user_api_key_auth_metadata if isinstance(user_api_key_auth_metadata, dict) else {}), **(spend_logs_metadata if isinstance(spend_logs_metadata, dict) else {}), diff --git a/tests/test_litellm/integrations/test_prometheus_spend_logs_metadata.py b/tests/test_litellm/integrations/test_prometheus_spend_logs_metadata.py index 31934e5fd8e..e2af6fd2daf 100644 --- a/tests/test_litellm/integrations/test_prometheus_spend_logs_metadata.py +++ b/tests/test_litellm/integrations/test_prometheus_spend_logs_metadata.py @@ -5,7 +5,10 @@ Verifies that metadata from x-litellm-spend-logs-metadata header is available in Prometheus custom labels via combined_metadata. """ -from litellm.integrations.prometheus import get_custom_labels_from_metadata +from litellm.integrations.prometheus import ( + _get_combined_custom_metadata_from_standard_logging_payload, + get_custom_labels_from_metadata, +) def test_get_custom_labels_includes_spend_logs_metadata(monkeypatch): @@ -109,3 +112,96 @@ def test_combined_metadata_with_none_spend_logs(monkeypatch): result = get_custom_labels_from_metadata(combined_metadata) assert result == {"metadata_foo": "bar"} + + +def test_combined_metadata_includes_top_level_fields(): + """ + Regression test for LIT-3741: user_api_key_project_alias (and other + top-level metadata fields) must be included in the combined metadata + so they can be referenced via custom_prometheus_metadata_labels. + """ + standard_logging_payload = { + "metadata": { + "user_api_key_hash": "sk-abc123", + "user_api_key_alias": "hotel-key", + "user_api_key_team_id": "team-1", + "user_api_key_team_alias": "hotel-team", + "user_api_key_project_id": "proj-1", + "user_api_key_project_alias": "hotel-recommendations", + "user_api_key_user_id": "user-1", + "user_api_key_user_email": "user@example.com", + "user_api_key_end_user_id": None, + "user_api_key_org_id": None, + "user_api_key_org_alias": None, + "user_api_key_request_route": "/v1/chat/completions", + "requester_metadata": {"custom_field": "custom_value"}, + "user_api_key_auth_metadata": {"auth_field": "auth_value"}, + "spend_logs_metadata": None, + } + } + + combined = _get_combined_custom_metadata_from_standard_logging_payload( + standard_logging_payload + ) + + assert combined["user_api_key_project_alias"] == "hotel-recommendations" + assert combined["user_api_key_project_id"] == "proj-1" + assert combined["user_api_key_team_alias"] == "hotel-team" + assert combined["user_api_key_request_route"] == "/v1/chat/completions" + assert combined["custom_field"] == "custom_value" + assert combined["auth_field"] == "auth_value" + + +def test_project_alias_accessible_via_custom_prometheus_labels(monkeypatch): + """ + Regression test for LIT-3741: configuring + custom_prometheus_metadata_labels with "metadata.user_api_key_project_alias" + should produce a label with the project's alias value. + """ + monkeypatch.setattr( + "litellm.custom_prometheus_metadata_labels", + ["metadata.user_api_key_project_alias"], + ) + + standard_logging_payload = { + "metadata": { + "user_api_key_project_alias": "hotel-recommendations", + "requester_metadata": None, + "user_api_key_auth_metadata": None, + "spend_logs_metadata": None, + } + } + + combined = _get_combined_custom_metadata_from_standard_logging_payload( + standard_logging_payload + ) + result = get_custom_labels_from_metadata(combined) + + assert result == {"metadata_user_api_key_project_alias": "hotel-recommendations"} + + +def test_project_alias_accessible_without_prefix(monkeypatch): + """ + user_api_key_project_alias should also be accessible without + the "metadata." prefix in custom_prometheus_metadata_labels config. + """ + monkeypatch.setattr( + "litellm.custom_prometheus_metadata_labels", + ["user_api_key_project_alias"], + ) + + standard_logging_payload = { + "metadata": { + "user_api_key_project_alias": "hotel-recommendations", + "requester_metadata": None, + "user_api_key_auth_metadata": None, + "spend_logs_metadata": None, + } + } + + combined = _get_combined_custom_metadata_from_standard_logging_payload( + standard_logging_payload + ) + result = get_custom_labels_from_metadata(combined) + + assert result == {"user_api_key_project_alias": "hotel-recommendations"} From cca71a07c20e065fe0bd2fa1857cba783946ac93 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 30 Jun 2026 20:03:59 -0700 Subject: [PATCH 18/81] feat(mcp): add mcp_tool_search virtual tools for large tool catalogs (#31777) * feat(mcp): add tool search virtual tools for large catalogs When mcp_tool_search_enabled is set on a key's object_permission, tools/list returns only mcp_tool_search and mcp_tool_call instead of the full catalog. The LLM searches by keyword then calls discovered tools by name, avoiding context bloat with 100+ tool deployments. * fix(mcp): persist mcp_tool_search_enabled and route tool_call by name The mcp_tool_search_enabled flag existed on the Pydantic models but the Prisma schema lacked the column, so keys generated with the flag never persisted it and tools/list kept returning the full catalog. Add the column across all three schema.prisma copies plus a migration. handle_mcp_tool_call passed server_name="" into call_tool, which built a malformed prefixed name ("-") and failed to resolve the server. Resolve the caller's allowed servers and dispatch through execute_mcp_tool instead, matching how the normal /tools/call path routes. * fix(mcp): filter list_tools to virtual tools on the protocol path The REST surface (/mcp-rest/tools/list) returned only the two virtual tools when mcp_tool_search_enabled was set, but the MCP protocol handler (handle_list_tools, used by real MCP clients over streamable-http/SSE) still returned the full catalog. Apply the same early return there so an actual MCP client sees mcp_tool_search and mcp_tool_call instead of every tool. call_tool was already intercepted on this path. * fix(mcp): enforce IP + server filtering on virtual tool search/call Review flagged that the virtual mcp_tool_search/mcp_tool_call path skipped access controls the normal MCP flow applies. mcp_tool_call resolved allowed servers from key permissions only, never applying IP filtering, so a caller on a public IP could invoke a tool on a server marked available_on_public_internet: false. mcp_tool_search listed the raw catalog via global_mcp_server_manager.list_tools, exposing tool names/schemas that /tools/list would hide and ignoring per-key/per-server tool filters. Route both virtual handlers through the same filtered paths used by the normal MCP flow: search now calls _list_mcp_tools and call resolves servers via _get_allowed_mcp_servers, both threaded with the request client IP so filter_server_ids_by_ip applies. execute_mcp_tool then enforces the server allowlist and per-key tool permissions. Thread client_ip through _list_mcp_tools/_get_tools_from_mcp_servers and pass it from the REST and SSE call sites. * fix(ci): ruff format server.py and sync dashboard API types ruff format normalizes the list_tools client_ip changes in server.py, and schema.d.ts gains the mcp_tool_search_enabled object-permission field so the generated dashboard types match the proxy OpenAPI spec. * style(mcp): drop quoted annotations and sort imports Clears UP037 on the virtual tool handler signatures (redundant with from __future__ import annotations) and I001 on the list_tools import block. * refactor(mcp): extract virtual-tool dispatch and host progress capture Pulls the mcp_tool_search/mcp_tool_call interception and the host progress-callback setup out of mcp_server_tool_call into helpers, keeping that handler under the strict cyclomatic-complexity ceiling after the client_ip threading. No behavior change. * test(mcp): cover SSE virtual-tool dispatch and host progress helpers Adds unit tests for _dispatch_virtual_mcp_tool (non-virtual passthrough, flag-disabled rejection, search/call routing with client_ip), _capture_host_progress_callback, and the protocol list_tools virtual early-return, covering the new server.py paths. * fix(mcp): forward per-request auth headers through virtual tool handlers The virtual mcp_tool_search/mcp_tool_call path intercepted the request before the normal header extraction ran, so client-supplied per-request auth (Authorization for upstream pass-through, x-mcp-auth-) was dropped and execute_mcp_tool/_list_mcp_tools received None. Thread mcp_auth_header, mcp_server_auth_headers, oauth2_headers, and raw_headers from both the REST and SSE call sites through the handlers so upstream MCP servers that require pass-through auth can be listed and called. * fix(mcp): preserve requested server scope in virtual tool calls A scoped MCP session (/mcp// or header-scoped) carries an mcp_servers scope that the normal call path passes into routing so the session can only reach that server. The virtual-tool branch dropped it and resolved with mcp_servers=None, letting a scoped session call mcp_tool_call for any server the key can access. Thread the context mcp_servers scope through _dispatch_virtual_mcp_tool into both handlers so search and call resolve against the same scoped server set. * fix(mcp): convert virtual tool errors to isError on the protocol path The virtual-tool dispatch ran before the protocol handler's HTTPException and guardrail handling, so a rejected virtual call (e.g. an out-of-scope 403 from execute_mcp_tool) raised out of mcp_server_tool_call and broke the MCP JSON-RPC stream instead of returning an isError CallToolResult. Move the dispatch inside the same try that wraps call_mcp_tool so virtual-tool errors get the same isError conversion as normal tool calls. * fix(mcp): spend-log virtual tool calls on the REST path The REST virtual-tool branch returned before common_processing_pre_call_logic, so execute_mcp_tool ran without a litellm_logging_obj and virtual mcp_tool_call invocations were not spend-logged or guardrail-checked like normal calls. Run the same pre-call pipeline in the call branch and thread the resulting litellm_logging_obj through handle_mcp_tool_call into execute_mcp_tool. * fix(mcp): reject virtual tool call when key has no accessible servers handle_mcp_tool_call passed an empty allowed_mcp_servers list into execute_mcp_tool; an unprefixed local tool name then fell through to the local registry, which has no server permission check, so a key with only mcp_tool_search_enabled and no server grants could run operator-configured local tools by name. Reject with 403 before dispatch when no servers are accessible, matching call_mcp_tool. * docs(mcp): document virtual tool_search module and parity rule in AGENTS.md * style(mcp): apply ruff format at repo line-length (120) * fix(mcp): add mcp_tool_search_enabled to ObjectPermissionDict and customer test fixture * chore: trigger CI * fix(mcp): mirror pre-call pipeline, guard imports, coerce top_k, honor include_disabled_tools - SSE mcp_tool_call now runs common_processing_pre_call_logic so it spend-logs and runs guardrails like the REST path (P1) - coerce_top_k avoids ValueError on non-integer top_k from clients (both REST and SSE) - guard mcp.types import in tool_search behind runtime/TYPE_CHECKING per package convention - admin list with include_disabled_tools returns the real catalog even when mcp_tool_search_enabled is set --- .../migration.sql | 2 + .../litellm_proxy_extras/schema.prisma | 1 + litellm/models/object_permission.py | 1 + .../proxy/_experimental/mcp_server/AGENTS.md | 6 + .../mcp_server/rest_endpoints.py | 86 +- .../proxy/_experimental/mcp_server/server.py | 228 ++++- .../_experimental/mcp_server/tool_search.py | 157 ++++ litellm/proxy/_types.py | 1 + litellm/proxy/schema.prisma | 1 + litellm/types/object_permission.py | 1 + schema.prisma | 1 + .../mcp_server/test_mcp_tool_search.py | 837 ++++++++++++++++++ .../test_customer_endpoints.py | 1 + .../test_object_permission_utils.py | 32 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 + 15 files changed, 1321 insertions(+), 38 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260626120000_add_mcp_tool_search_enabled/migration.sql create mode 100644 litellm/proxy/_experimental/mcp_server/tool_search.py create mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260626120000_add_mcp_tool_search_enabled/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260626120000_add_mcp_tool_search_enabled/migration.sql new file mode 100644 index 00000000000..542677426ba --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260626120000_add_mcp_tool_search_enabled/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN "mcp_tool_search_enabled" BOOLEAN; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index e21c0016491..5351e0a1470 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -279,6 +279,7 @@ model LiteLLM_ObjectPermissionTable { blocked_tools String[] @default([]) // Tool names blocked for any key/team/user with this permission mcp_toolsets String[] @default([]) // Toolset IDs granted to this key/team/user search_tools String[] @default([]) // search_tool_name values this key/team/user may call + mcp_tool_search_enabled Boolean? teams LiteLLM_TeamTable[] projects LiteLLM_ProjectTable[] verification_tokens LiteLLM_VerificationToken[] diff --git a/litellm/models/object_permission.py b/litellm/models/object_permission.py index 6c0d100046c..3052a2af459 100644 --- a/litellm/models/object_permission.py +++ b/litellm/models/object_permission.py @@ -24,3 +24,4 @@ class LiteLLM_ObjectPermissionTable(LiteLLMPydanticObjectBase): mcp_toolsets: Optional[List[str]] = None blocked_tools: Optional[List[str]] = [] search_tools: Optional[List[str]] = [] + mcp_tool_search_enabled: Optional[bool] = None diff --git a/litellm/proxy/_experimental/mcp_server/AGENTS.md b/litellm/proxy/_experimental/mcp_server/AGENTS.md index 8eebc3ea3b3..6e1d121c3be 100644 --- a/litellm/proxy/_experimental/mcp_server/AGENTS.md +++ b/litellm/proxy/_experimental/mcp_server/AGENTS.md @@ -41,6 +41,7 @@ litellm/proxy/_experimental/mcp_server/ sampling_handler.py # MCP sampling to LiteLLM completion flow elicitation_handler.py # MCP elicitation relay flow semantic_tool_filter.py # semantic filtering of available MCP tools + tool_search.py # opt-in virtual tools (mcp_tool_search + mcp_tool_call) for large catalogs guardrail_translation/ handler.py # MCP guardrail result translation sse_transport.py # SSE transport implementation @@ -79,6 +80,11 @@ module materially harder to understand. encryption need focused tests for both allowed and rejected paths. - Avoid adding comments to new code unless they explain non-obvious security or protocol behavior. Prefer clear names and small functions. +- The virtual tool path (`tool_search.py`, gated by `mcp_tool_search_enabled`) + must mirror the normal tool flow: IP filtering, server allowlist, per-key tool + permissions, no-accessible-server rejection, per-request auth headers, server + scope, error to `isError` conversion, and spend logging. Reuse `_list_mcp_tools` + and `execute_mcp_tool` rather than reimplementing any of these checks. ## Tests diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index d30d8af2af2..7ab7eb28147 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -569,6 +569,21 @@ if MCP_AVAILABLE: include_disabled_tools and user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN ) + if apply_tool_filters and getattr( + getattr(user_api_key_dict, "object_permission", None), + "mcp_tool_search_enabled", + False, + ): + from litellm.proxy._experimental.mcp_server.tool_search import ( + get_virtual_tool_definitions, + ) + + return { + "tools": get_virtual_tool_definitions(), + "error": None, + "message": "Successfully retrieved tools", + } + # Extract auth headers from request headers = request.headers raw_headers_from_request = dict(headers) @@ -727,6 +742,74 @@ if MCP_AVAILABLE: try: data = await request.json() + tool_name = data.get("name") + tool_arguments = data.get("arguments") or {} + + from litellm.proxy._experimental.mcp_server.tool_search import ( + MCP_TOOL_CALL_TOOL_NAME, + MCP_TOOL_SEARCH_TOOL_NAME, + coerce_top_k, + handle_mcp_tool_call, + handle_mcp_tool_search, + ) + + if tool_name in (MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME): + if not getattr( + getattr(user_api_key_dict, "object_permission", None), + "mcp_tool_search_enabled", + False, + ): + raise HTTPException( + status_code=403, + detail={ + "error": "forbidden", + "message": f"{tool_name} requires mcp_tool_search_enabled on the key", + }, + ) + rest_client_ip = IPAddressUtils.get_mcp_client_ip(request) + ( + virtual_mcp_auth_header, + virtual_mcp_server_auth_headers, + virtual_raw_headers, + ) = _extract_mcp_headers_from_request(request, MCPRequestHandler) + virtual_oauth2_headers = MCPRequestHandler._get_oauth2_headers_from_headers(request.headers) + if tool_name == MCP_TOOL_SEARCH_TOOL_NAME: + return await handle_mcp_tool_search( + query=tool_arguments.get("query", ""), + top_k=coerce_top_k(tool_arguments.get("top_k", 5)), + user_api_key_dict=user_api_key_dict, + client_ip=rest_client_ip, + mcp_auth_header=virtual_mcp_auth_header, + mcp_server_auth_headers=virtual_mcp_server_auth_headers, + oauth2_headers=virtual_oauth2_headers, + raw_headers=virtual_raw_headers, + ) + else: # MCP_TOOL_CALL_TOOL_NAME + # Run the same pre-call pipeline as the normal call path so the + # tool execution is spend-logged and guardrail-checked. + ( + _, + virtual_logging_obj, + ) = await ProxyBaseLLMRequestProcessing(data=data).common_processing_pre_call_logic( + request=request, + user_api_key_dict=user_api_key_dict, + proxy_config=proxy_config, + route_type=CallTypes.call_mcp_tool.value, + proxy_logging_obj=proxy_logging_obj, + general_settings=general_settings, + ) + return await handle_mcp_tool_call( + tool_name=tool_arguments.get("tool_name", ""), + arguments=tool_arguments.get("arguments") or {}, + user_api_key_dict=user_api_key_dict, + client_ip=rest_client_ip, + mcp_auth_header=virtual_mcp_auth_header, + mcp_server_auth_headers=virtual_mcp_server_auth_headers, + oauth2_headers=virtual_oauth2_headers, + raw_headers=virtual_raw_headers, + litellm_logging_obj=virtual_logging_obj, + ) + # Validate required parameters early server_id = data.get("server_id") if not server_id: @@ -738,7 +821,6 @@ if MCP_AVAILABLE: }, ) - tool_name = data.get("name") if not tool_name: raise HTTPException( status_code=400, @@ -748,8 +830,6 @@ if MCP_AVAILABLE: }, ) - tool_arguments = data.get("arguments") or {} - proxy_base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) ( data, diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 607e676524e..a65239b296f 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -10,8 +10,8 @@ import contextvars import hashlib import json import time -import types import traceback +import types import uuid from datetime import datetime from typing import ( @@ -37,13 +37,17 @@ from starlette.types import Message, Receive, Scope, Send from litellm._logging import verbose_logger from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, +) from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) -from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( get_request_base_url, ) +from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError from litellm.proxy._experimental.mcp_server.mcp_context import ( _mcp_active_toolset_id, _mcp_gateway_initialize_instructions, @@ -59,10 +63,6 @@ from litellm.proxy._experimental.mcp_server.utils import ( get_server_prefix, iter_known_server_prefixes, ) -from litellm.llms.custom_httpx.http_handler import ( - get_async_httpx_client, - httpxSpecialProvider, -) from litellm.proxy._types import ( ProxyException, SpecialMCPServerNames, @@ -122,9 +122,12 @@ def _write_byok_cred_cache(user_id: str, server_id: str, credential: Optional[st # TODO: Make this a util function for litellm client usage MCP_AVAILABLE: bool = True try: + import weakref + from mcp import ReadResourceResult, Resource from mcp.server import Server from mcp.server.lowlevel.helper_types import ReadResourceContents + from mcp.server.session import ServerSession as _McpServerSession from mcp.types import ( BlobResourceContents, GetPromptResult, @@ -132,8 +135,6 @@ try: TextResourceContents, Tool, ) - from mcp.server.session import ServerSession as _McpServerSession - import weakref # Robust auth lookup keyed by session_object. _session_obj_auth_storage: "weakref.WeakKeyDictionary[Any, MCPAuthenticatedUser]" = weakref.WeakKeyDictionary() @@ -303,14 +304,14 @@ def _proxy_exception_to_http_exception(exc: ProxyException) -> HTTPException: if MCP_AVAILABLE: from mcp.server import Server - from mcp.server.lowlevel.server import NotificationOptions - from mcp.server.models import InitializationOptions # Import auth context variables and middleware from mcp.server.auth.middleware.auth_context import ( AuthContextMiddleware, auth_context_var, ) + from mcp.server.lowlevel.server import NotificationOptions + from mcp.server.models import InitializationOptions try: from mcp.server.streamable_http_manager import StreamableHTTPSessionManager @@ -664,6 +665,19 @@ if MCP_AVAILABLE: verbose_logger.debug( f"MCP list_tools - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" ) + if getattr( + getattr(user_api_key_auth, "object_permission", None), + "mcp_tool_search_enabled", + False, + ): + from mcp.types import Tool + + from litellm.proxy._experimental.mcp_server.tool_search import ( + get_virtual_tool_definitions, + ) + + return [Tool(**d) for d in get_virtual_tool_definitions()] + # Get mcp_servers from context variable verbose_logger.debug("MCP list_tools - Calling _list_mcp_tools") tools = await _list_mcp_tools( @@ -688,6 +702,150 @@ if MCP_AVAILABLE: if _session_reset_token is not None: active_mcp_session_var.reset(_session_reset_token) + def _capture_host_progress_callback(host_server) -> Optional[Callable]: + """Return a progress-forwarding callback bound to the host MCP session. + + Returns ``None`` when the host did not supply a progress token. + """ + try: + host_ctx = host_server.request_context + except Exception as e: + verbose_logger.warning(f"Could not capture host progress context: {e}") + return None + + if not (host_ctx and hasattr(host_ctx, "meta") and host_ctx.meta): + return None + host_token = getattr(host_ctx.meta, "progressToken", None) + if not (host_token and hasattr(host_ctx, "session") and host_ctx.session): + return None + host_session = host_ctx.session + + async def forward_progress(progress: float, total: Optional[float]): + """Forward progress notifications from external MCP to Host""" + try: + await host_session.send_progress_notification( + progress_token=host_token, + progress=progress, + total=total, + ) + verbose_logger.debug(f"Forwarded progress {progress}/{total} to Host") + except Exception as e: + verbose_logger.error(f"Failed to forward progress to Host: {e}") + + verbose_logger.debug(f"Host progressToken captured: {host_token[:8]}...") + return forward_progress + + async def _build_virtual_call_logging_obj( + name: str, + arguments: dict[str, Any], + user_api_key_auth: UserAPIKeyAuth, + ) -> Optional[LiteLLMLoggingObj]: + """Run the pre-call pipeline (guardrails + logging setup) for a virtual + mcp_tool_call so the SSE path spend-logs like the REST path.""" + from fastapi import Request + + from litellm.proxy.common_request_processing import ( + ProxyBaseLLMRequestProcessing, + ) + from litellm.proxy.proxy_server import ( + general_settings, + proxy_config, + proxy_logging_obj, + ) + + request = Request( + scope={ + "type": "http", + "method": "POST", + "path": "/mcp/tools/call", + "headers": [(b"content-type", b"application/json")], + } + ) + _, virtual_logging_obj = await ProxyBaseLLMRequestProcessing( + data={"name": name, "arguments": arguments} + ).common_processing_pre_call_logic( + request=request, + user_api_key_dict=user_api_key_auth, + proxy_config=proxy_config, + route_type=CallTypes.call_mcp_tool.value, + proxy_logging_obj=proxy_logging_obj, + general_settings=general_settings, + ) + return virtual_logging_obj + + async def _dispatch_virtual_mcp_tool( + name: str, + arguments: Optional[dict[str, Any]], + user_api_key_auth: Optional[UserAPIKeyAuth], + client_ip: Optional[str], + mcp_servers: Optional[list[str]] = None, + mcp_auth_header: Optional[str] = None, + mcp_server_auth_headers: Optional[dict[str, dict[str, str]]] = None, + oauth2_headers: Optional[dict[str, str]] = None, + raw_headers: Optional[dict[str, str]] = None, + ) -> Optional[CallToolResult]: + """Handle the mcp_tool_search / mcp_tool_call virtual tools. + + Returns a CallToolResult when ``name`` is a virtual tool, else ``None`` so + the caller falls through to normal tool routing. + """ + from litellm.proxy._experimental.mcp_server.tool_search import ( + MCP_TOOL_CALL_TOOL_NAME, + MCP_TOOL_SEARCH_TOOL_NAME, + coerce_top_k, + handle_mcp_tool_call, + handle_mcp_tool_search, + ) + + if name not in (MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME): + return None + + if not getattr( + getattr(user_api_key_auth, "object_permission", None), + "mcp_tool_search_enabled", + False, + ): + return CallToolResult( + content=[ + TextContent( + type="text", + text=f"Tool {name} requires mcp_tool_search_enabled on the key", + ) + ], + isError=True, + ) + + args = arguments or {} + if name == MCP_TOOL_SEARCH_TOOL_NAME: + return await handle_mcp_tool_search( + query=args.get("query", ""), + top_k=coerce_top_k(args.get("top_k", 5)), + user_api_key_dict=user_api_key_auth, + client_ip=client_ip, + mcp_servers=mcp_servers, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + assert user_api_key_auth is not None # guaranteed by the flag check above + virtual_logging_obj = await _build_virtual_call_logging_obj( + name=name, arguments=args, user_api_key_auth=user_api_key_auth + ) + return await handle_mcp_tool_call( + tool_name=args.get("tool_name", ""), + arguments=args.get("arguments") or {}, + user_api_key_dict=user_api_key_auth, + client_ip=client_ip, + mcp_servers=mcp_servers, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + litellm_logging_obj=virtual_logging_obj, + ) + @server.call_tool() async def mcp_server_tool_call(name: str, arguments: Dict[str, Any] | None) -> CallToolResult: """ @@ -701,11 +859,12 @@ if MCP_AVAILABLE: HTTPException: If tool not found or arguments missing """ from fastapi import Request + from mcp.server.lowlevel.server import request_ctx + from mcp.types import CallToolResult + from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request from litellm.proxy.proxy_server import proxy_config - from mcp.types import CallToolResult - from mcp.server.lowlevel.server import request_ctx req_ctx = request_ctx.get(None) _session_reset_token = None @@ -730,31 +889,25 @@ if MCP_AVAILABLE: ) verbose_logger.debug(f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}") - host_progress_callback = None - try: - host_ctx = server.request_context - if host_ctx and hasattr(host_ctx, "meta") and host_ctx.meta: - host_token = getattr(host_ctx.meta, "progressToken", None) - if host_token and hasattr(host_ctx, "session") and host_ctx.session: - host_session = host_ctx.session - async def forward_progress(progress: float, total: Optional[float]): - """Forward progress notifications from external MCP to Host""" - try: - await host_session.send_progress_notification( - progress_token=host_token, - progress=progress, - total=total, - ) - verbose_logger.debug(f"Forwarded progress {progress}/{total} to Host") - except Exception as e: - verbose_logger.error(f"Failed to forward progress to Host: {e}") - - host_progress_callback = forward_progress - verbose_logger.debug(f"Host progressToken captured: {host_token[:8]}...") - except Exception as e: - verbose_logger.warning(f"Could not capture host progress context: {e}") try: + # Inside this try so virtual-tool errors convert to isError + # CallToolResult instead of raising out of the protocol handler. + virtual_tool_result = await _dispatch_virtual_mcp_tool( + name=name, + arguments=arguments, + user_api_key_auth=user_api_key_auth, + client_ip=_client_ip, + mcp_servers=mcp_servers, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + if virtual_tool_result is not None: + return virtual_tool_result + + host_progress_callback = _capture_host_progress_callback(server) # Create a body date for logging body_data = {"name": name, "arguments": arguments} # Set trace/session id from raw_headers so spend logs and logging_obj stay consistent (same as A2A) @@ -1528,6 +1681,7 @@ if MCP_AVAILABLE: log_list_tools_to_spendlogs: bool = False, list_tools_log_source: Optional[str] = None, litellm_trace_id: Optional[str] = None, + client_ip: Optional[str] = None, ) -> List[MCPTool]: """ Helper method to fetch tools from MCP servers based on server filtering criteria. @@ -1615,6 +1769,7 @@ if MCP_AVAILABLE: allowed_mcp_servers = await _get_allowed_mcp_servers( user_api_key_auth=user_api_key_auth, mcp_servers=mcp_servers, + client_ip=client_ip, ) # Pre-fetch OAuth credentials only when at least one server uses OAuth2, @@ -2024,6 +2179,7 @@ if MCP_AVAILABLE: raw_headers: Optional[Dict[str, str]] = None, log_list_tools_to_spendlogs: bool = False, list_tools_log_source: Optional[str] = None, + client_ip: Optional[str] = None, ) -> List[MCPTool]: """ List all available MCP tools. @@ -2033,6 +2189,7 @@ if MCP_AVAILABLE: mcp_auth_header: Optional auth header for MCP server (deprecated) mcp_servers: Optional list of server names/aliases to filter by mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value} + client_ip: Client IP for IP-based server access control Returns: List[MCPTool]: Combined list of tools from all accessible servers @@ -2056,6 +2213,7 @@ if MCP_AVAILABLE: raw_headers=raw_headers, log_list_tools_to_spendlogs=log_list_tools_to_spendlogs, list_tools_log_source=list_tools_log_source, + client_ip=client_ip, ) verbose_logger.debug(f"Successfully fetched {len(managed_tools)} tools from managed MCP servers") except Exception as e: diff --git a/litellm/proxy/_experimental/mcp_server/tool_search.py b/litellm/proxy/_experimental/mcp_server/tool_search.py new file mode 100644 index 00000000000..fa57a2b3eb2 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/tool_search.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import json +from datetime import datetime +from typing import TYPE_CHECKING, Any, Optional + +if TYPE_CHECKING: + from mcp.types import CallToolResult + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.proxy._types import UserAPIKeyAuth + +MCP_TOOL_SEARCH_TOOL_NAME: str = "mcp_tool_search" +MCP_TOOL_CALL_TOOL_NAME: str = "mcp_tool_call" + + +def coerce_top_k(value: Any, default: int = 5) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def search_tools(query: str, tools: list[dict[str, Any]], top_k: int = 5) -> list[dict[str, Any]]: + if not query: + return [] + tokens = query.lower().split() + + def _score(tool: dict[str, Any]) -> int: + haystack = (tool.get("name", "") + " " + tool.get("description", "")).lower() + return sum(1 for t in tokens if t in haystack) + + scored = ((s, tool) for tool in tools if (s := _score(tool)) > 0) + return [tool for _, tool in sorted(scored, key=lambda x: x[0], reverse=True)[:top_k]] + + +def get_virtual_tool_definitions() -> list[dict[str, Any]]: + return [ + { + "name": MCP_TOOL_SEARCH_TOOL_NAME, + "description": "Search for MCP tools by keyword. Returns top matching tools with names, descriptions, and input schemas.", + "inputSchema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Keywords to search for in tool names and descriptions.", + }, + "top_k": { + "type": "integer", + "description": "Maximum number of results to return.", + "default": 5, + }, + }, + "required": ["query"], + }, + }, + { + "name": MCP_TOOL_CALL_TOOL_NAME, + "description": "Call an MCP tool by name with the given arguments.", + "inputSchema": { + "type": "object", + "properties": { + "tool_name": { + "type": "string", + "description": "The exact name of the MCP tool to call.", + }, + "arguments": { + "type": "object", + "description": "Arguments to pass to the tool.", + }, + }, + "required": ["tool_name"], + }, + }, + ] + + +async def handle_mcp_tool_search( + query: str, + top_k: int, + user_api_key_dict: UserAPIKeyAuth, + client_ip: Optional[str] = None, + mcp_servers: Optional[list[str]] = None, + mcp_auth_header: Optional[str] = None, + mcp_server_auth_headers: Optional[dict[str, dict[str, str]]] = None, + oauth2_headers: Optional[dict[str, str]] = None, + raw_headers: Optional[dict[str, str]] = None, +) -> CallToolResult: + from mcp.types import CallToolResult, TextContent + + from litellm.proxy._experimental.mcp_server.server import _list_mcp_tools + + mcp_tools = await _list_mcp_tools( + user_api_key_auth=user_api_key_dict, + mcp_servers=mcp_servers, + client_ip=client_ip, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + tools = [ + { + "name": t.name, + "description": t.description or "", + "inputSchema": t.inputSchema, + } + for t in mcp_tools + ] + results = search_tools(query, tools, top_k) + return CallToolResult(content=[TextContent(type="text", text=json.dumps(results))], isError=False) + + +async def handle_mcp_tool_call( + tool_name: str, + arguments: dict[str, Any], + user_api_key_dict: UserAPIKeyAuth, + client_ip: Optional[str] = None, + mcp_servers: Optional[list[str]] = None, + mcp_auth_header: Optional[str] = None, + mcp_server_auth_headers: Optional[dict[str, dict[str, str]]] = None, + oauth2_headers: Optional[dict[str, str]] = None, + raw_headers: Optional[dict[str, str]] = None, + litellm_logging_obj: Optional[LiteLLMLoggingObj] = None, +) -> CallToolResult: + from litellm.proxy._experimental.mcp_server.server import ( + _get_allowed_mcp_servers, + execute_mcp_tool, + ) + + allowed_mcp_servers = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_dict, + mcp_servers=mcp_servers, + client_ip=client_ip, + ) + + # Reject before dispatch when the key has no accessible servers; otherwise an + # unprefixed local tool name would fall through to the local registry in + # execute_mcp_tool, which has no server permission check. + if not allowed_mcp_servers: + from fastapi import HTTPException + + raise HTTPException(status_code=403, detail="User not allowed to call this tool.") + + return await execute_mcp_tool( + name=tool_name, + arguments=arguments, + allowed_mcp_servers=allowed_mcp_servers, + start_time=datetime.now(), + user_api_key_auth=user_api_key_dict, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + litellm_logging_obj=litellm_logging_obj, + ) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 643f8d69300..12466b525d6 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1006,6 +1006,7 @@ class LiteLLM_ObjectPermissionBase(LiteLLMPydanticObjectBase): agent_access_groups: Optional[List[str]] = None models: Optional[List[str]] = None search_tools: Optional[List[str]] = None + mcp_tool_search_enabled: Optional[bool] = None from litellm.types.object_permission import ( # noqa: E402 diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index e21c0016491..5351e0a1470 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -279,6 +279,7 @@ model LiteLLM_ObjectPermissionTable { blocked_tools String[] @default([]) // Tool names blocked for any key/team/user with this permission mcp_toolsets String[] @default([]) // Toolset IDs granted to this key/team/user search_tools String[] @default([]) // search_tool_name values this key/team/user may call + mcp_tool_search_enabled Boolean? teams LiteLLM_TeamTable[] projects LiteLLM_ProjectTable[] verification_tokens LiteLLM_VerificationToken[] diff --git a/litellm/types/object_permission.py b/litellm/types/object_permission.py index ff932dccd5d..d0458173fbf 100644 --- a/litellm/types/object_permission.py +++ b/litellm/types/object_permission.py @@ -24,3 +24,4 @@ class ObjectPermissionDict(TypedDict, total=False): agent_access_groups: Optional[list[str]] models: Optional[list[str]] search_tools: Optional[list[str]] + mcp_tool_search_enabled: Optional[bool] diff --git a/schema.prisma b/schema.prisma index e21c0016491..5351e0a1470 100644 --- a/schema.prisma +++ b/schema.prisma @@ -279,6 +279,7 @@ model LiteLLM_ObjectPermissionTable { blocked_tools String[] @default([]) // Tool names blocked for any key/team/user with this permission mcp_toolsets String[] @default([]) // Toolset IDs granted to this key/team/user search_tools String[] @default([]) // search_tool_name values this key/team/user may call + mcp_tool_search_enabled Boolean? teams LiteLLM_TeamTable[] projects LiteLLM_ProjectTable[] verification_tokens LiteLLM_VerificationToken[] diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py new file mode 100644 index 00000000000..9c20808df67 --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py @@ -0,0 +1,837 @@ +""" +Tests for MCP tool search feature. + +Covers: +- search_tools() pure function +- get_virtual_tool_definitions() shape +- list_tool_rest_api returns only virtual tools when mcp_tool_search_enabled=True +- call_tool_rest_api intercepts mcp_tool_search calls +- call_tool_rest_api intercepts mcp_tool_call calls +""" + +import json +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from litellm.models.object_permission import LiteLLM_ObjectPermissionTable +from litellm.proxy._experimental.mcp_server.tool_search import ( + MCP_TOOL_CALL_TOOL_NAME, + MCP_TOOL_SEARCH_TOOL_NAME, + coerce_top_k, + get_virtual_tool_definitions, + search_tools, +) +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + + +def _make_tools(specs: list[tuple[str, str]]) -> list[dict[str, Any]]: + return [ + { + "name": name, + "description": desc, + "inputSchema": {"type": "object", "properties": {}}, + } + for name, desc in specs + ] + + +def _make_perm(**kwargs: Any) -> LiteLLM_ObjectPermissionTable: + return LiteLLM_ObjectPermissionTable(object_permission_id="test", **kwargs) + + +SAMPLE_TOOLS = _make_tools( + [ + ("github-create_issue", "Create a new issue in a GitHub repository"), + ("github-list_repos", "List all repositories for a GitHub user"), + ("slack-send_message", "Send a message to a Slack channel"), + ("slack-list_channels", "List all Slack channels in a workspace"), + ("notion-create_page", "Create a new page in Notion"), + ] +) + + +class TestCoerceTopK: + def test_int_passthrough(self) -> None: + assert coerce_top_k(3) == 3 + + def test_numeric_string_coerced(self) -> None: + assert coerce_top_k("7") == 7 + + def test_float_truncated(self) -> None: + assert coerce_top_k(3.9) == 3 + + def test_non_numeric_string_returns_default(self) -> None: + assert coerce_top_k("abc") == 5 + + def test_none_returns_default(self) -> None: + assert coerce_top_k(None) == 5 + + def test_custom_default(self) -> None: + assert coerce_top_k("nope", default=10) == 10 + + +class TestSearchTools: + def test_returns_matching_tools(self) -> None: + results = search_tools("github issue", SAMPLE_TOOLS) + names = [t["name"] for t in results] + assert "github-create_issue" in names + + def test_ranks_by_relevance(self) -> None: + results = search_tools("github", SAMPLE_TOOLS) + names = [t["name"] for t in results] + github_positions = [i for i, n in enumerate(names) if n.startswith("github")] + other_positions = [i for i, n in enumerate(names) if not n.startswith("github")] + assert all(g < o for g in github_positions for o in other_positions) + + def test_top_k_limits_results(self) -> None: + results = search_tools("a", SAMPLE_TOOLS, top_k=2) + assert len(results) <= 2 + + def test_empty_query_returns_empty(self) -> None: + assert search_tools("", SAMPLE_TOOLS) == [] + + def test_no_match_returns_empty(self) -> None: + assert search_tools("xyzzy_nonexistent_zzz", SAMPLE_TOOLS) == [] + + def test_matches_description_not_just_name(self) -> None: + results = search_tools("channel", SAMPLE_TOOLS) + names = [t["name"] for t in results] + assert "slack-list_channels" in names + + def test_case_insensitive(self) -> None: + lower = [t["name"] for t in search_tools("github", SAMPLE_TOOLS)] + upper = [t["name"] for t in search_tools("GITHUB", SAMPLE_TOOLS)] + assert lower == upper + + def test_result_tools_have_full_schema(self) -> None: + for tool in search_tools("github", SAMPLE_TOOLS): + assert "name" in tool + assert "description" in tool + assert "inputSchema" in tool + + +class TestGetVirtualToolDefinitions: + def test_returns_two_tools(self) -> None: + assert len(get_virtual_tool_definitions()) == 2 + + def test_has_mcp_tool_search(self) -> None: + names = [t["name"] for t in get_virtual_tool_definitions()] + assert MCP_TOOL_SEARCH_TOOL_NAME in names + + def test_has_mcp_tool_call(self) -> None: + names = [t["name"] for t in get_virtual_tool_definitions()] + assert MCP_TOOL_CALL_TOOL_NAME in names + + def test_mcp_tool_search_schema_has_query(self) -> None: + tools = get_virtual_tool_definitions() + search_tool = next(t for t in tools if t["name"] == MCP_TOOL_SEARCH_TOOL_NAME) + props = search_tool["inputSchema"]["properties"] + assert "query" in props + assert search_tool["inputSchema"]["required"] == ["query"] + + def test_mcp_tool_call_schema_has_tool_name_and_arguments(self) -> None: + tools = get_virtual_tool_definitions() + call_tool = next(t for t in tools if t["name"] == MCP_TOOL_CALL_TOOL_NAME) + props = call_tool["inputSchema"]["properties"] + assert "tool_name" in props + assert "arguments" in props + assert "tool_name" in call_tool["inputSchema"]["required"] + + def test_all_tools_have_description(self) -> None: + for tool in get_virtual_tool_definitions(): + assert tool.get("description"), f"{tool['name']} missing description" + + def test_definitions_construct_mcp_protocol_tool(self) -> None: + """The MCP protocol list_tools handler builds mcp.types.Tool(**d) from + each definition, so the dict keys must stay valid Tool fields.""" + from mcp.types import Tool + + built = [Tool(**d) for d in get_virtual_tool_definitions()] + assert {t.name for t in built} == { + MCP_TOOL_SEARCH_TOOL_NAME, + MCP_TOOL_CALL_TOOL_NAME, + } + + +class TestListToolRestApiWithToolSearch: + @pytest.mark.asyncio + async def test_returns_only_virtual_tools_when_flag_enabled(self) -> None: + from litellm.proxy._experimental.mcp_server.rest_endpoints import router + + user_api_key_dict = UserAPIKeyAuth( + api_key="test_key", + object_permission=_make_perm( + mcp_tool_search_enabled=True, + mcp_servers=["github", "slack"], + ), + ) + + mock_request = MagicMock() + mock_request.headers = {} + + list_fn = next( + r.endpoint + for r in router.routes + if hasattr(r, "path") and r.path.endswith("/tools/list") and hasattr(r, "methods") and "GET" in r.methods + ) + + result = await list_fn( + request=mock_request, + server_id=None, + include_disabled_tools=False, + user_api_key_dict=user_api_key_dict, + ) + + assert result["error"] is None + tool_names = [t["name"] for t in result["tools"]] + assert set(tool_names) == {MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME} + + @pytest.mark.asyncio + async def test_returns_full_catalog_when_flag_disabled(self) -> None: + from litellm.proxy._experimental.mcp_server.rest_endpoints import router + + user_api_key_dict = UserAPIKeyAuth( + api_key="test_key", + object_permission=_make_perm( + mcp_tool_search_enabled=False, + mcp_servers=["github"], + ), + ) + + mock_request = MagicMock() + mock_request.headers = {} + + fake_tools = [ + { + "name": "github-create_issue", + "description": "Create issue", + "inputSchema": {"type": "object"}, + } + ] + + list_fn = next( + r.endpoint + for r in router.routes + if hasattr(r, "path") and r.path.endswith("/tools/list") and hasattr(r, "methods") and "GET" in r.methods + ) + + with ( + patch( + "litellm.proxy._experimental.mcp_server.rest_endpoints.build_effective_auth_contexts", + new_callable=AsyncMock, + return_value=[user_api_key_dict], + ), + patch("litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager") as mock_manager, + patch( + "litellm.proxy._experimental.mcp_server.rest_endpoints._get_tools_for_single_server", + new_callable=AsyncMock, + return_value=fake_tools, + ), + patch("litellm.proxy._experimental.mcp_server.rest_endpoints.IPAddressUtils"), + patch( + "litellm.proxy._experimental.mcp_server.rest_endpoints._prefetch_user_oauth_creds", + new_callable=AsyncMock, + return_value={}, + ), + patch( + "litellm.proxy._experimental.mcp_server.rest_endpoints._get_oauth2_server_ids", + return_value=[], + ), + patch( + "litellm.proxy._experimental.mcp_server.rest_endpoints._get_server_auth_header", + return_value=None, + ), + patch( + "litellm.proxy._experimental.mcp_server.rest_endpoints._get_user_oauth_extra_headers", + new_callable=AsyncMock, + return_value=None, + ), + ): + mock_manager.get_allowed_mcp_servers = AsyncMock(return_value=["github"]) + mock_manager.filter_server_ids_by_ip_with_info = MagicMock(return_value=(["github"], 0)) + mock_manager.get_mcp_server_by_id = MagicMock(return_value=MagicMock(name="github", server_id="github")) + result = await list_fn( + request=mock_request, + server_id=None, + include_disabled_tools=False, + user_api_key_dict=user_api_key_dict, + ) + + tool_names = [t["name"] for t in result["tools"]] + assert MCP_TOOL_SEARCH_TOOL_NAME not in tool_names + assert "github-create_issue" in tool_names + + @pytest.mark.asyncio + async def test_admin_include_disabled_tools_bypasses_virtual_catalog(self) -> None: + """Regression: an admin listing with include_disabled_tools must see the + real catalog (to configure allowlists) even when mcp_tool_search_enabled is + set, instead of the two virtual tools.""" + from litellm.proxy._experimental.mcp_server.rest_endpoints import router + + user_api_key_dict = UserAPIKeyAuth( + api_key="admin_key", + user_role=LitellmUserRoles.PROXY_ADMIN, + object_permission=_make_perm( + mcp_tool_search_enabled=True, + mcp_servers=["github"], + ), + ) + + mock_request = MagicMock() + mock_request.headers = {} + + fake_tools = [ + { + "name": "github-create_issue", + "description": "Create issue", + "inputSchema": {"type": "object"}, + } + ] + + list_fn = next( + r.endpoint + for r in router.routes + if hasattr(r, "path") and r.path.endswith("/tools/list") and hasattr(r, "methods") and "GET" in r.methods + ) + + with ( + patch( + "litellm.proxy._experimental.mcp_server.rest_endpoints.build_effective_auth_contexts", + new_callable=AsyncMock, + return_value=[user_api_key_dict], + ), + patch("litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager") as mock_manager, + patch( + "litellm.proxy._experimental.mcp_server.rest_endpoints._get_tools_for_single_server", + new_callable=AsyncMock, + return_value=fake_tools, + ), + patch("litellm.proxy._experimental.mcp_server.rest_endpoints.IPAddressUtils"), + patch( + "litellm.proxy._experimental.mcp_server.rest_endpoints._prefetch_user_oauth_creds", + new_callable=AsyncMock, + return_value={}, + ), + patch( + "litellm.proxy._experimental.mcp_server.rest_endpoints._get_oauth2_server_ids", + return_value=[], + ), + patch( + "litellm.proxy._experimental.mcp_server.rest_endpoints._get_server_auth_header", + return_value=None, + ), + patch( + "litellm.proxy._experimental.mcp_server.rest_endpoints._get_user_oauth_extra_headers", + new_callable=AsyncMock, + return_value=None, + ), + ): + mock_manager.get_allowed_mcp_servers = AsyncMock(return_value=["github"]) + mock_manager.filter_server_ids_by_ip_with_info = MagicMock(return_value=(["github"], 0)) + mock_manager.get_mcp_server_by_id = MagicMock(return_value=MagicMock(name="github", server_id="github")) + result = await list_fn( + request=mock_request, + server_id=None, + include_disabled_tools=True, + user_api_key_dict=user_api_key_dict, + ) + + tool_names = [t["name"] for t in result["tools"]] + assert MCP_TOOL_SEARCH_TOOL_NAME not in tool_names + assert "github-create_issue" in tool_names + + +class TestCallToolRestApiVirtualTools: + def _make_request(self, body: dict[str, Any]) -> MagicMock: + mock_request = MagicMock() + mock_request.json = AsyncMock(return_value=body) + mock_request.headers = {} + mock_request.url = MagicMock() + mock_request.url.path = "/mcp-rest/tools/call" + return mock_request + + def _get_call_fn(self) -> Any: + from litellm.proxy._experimental.mcp_server.rest_endpoints import router + + return next( + r.endpoint + for r in router.routes + if hasattr(r, "path") and r.path.endswith("/tools/call") and hasattr(r, "methods") and "POST" in r.methods + ) + + @pytest.mark.asyncio + async def test_mcp_tool_search_call_returns_tool_defs(self) -> None: + user_api_key_dict = UserAPIKeyAuth( + api_key="test_key", + object_permission=_make_perm( + mcp_tool_search_enabled=True, + mcp_servers=["github"], + ), + ) + + request = self._make_request({"name": MCP_TOOL_SEARCH_TOOL_NAME, "arguments": {"query": "create issue"}}) + + mock_tool = MagicMock() + mock_tool.name = "github-create_issue" + mock_tool.description = "Create a GitHub issue" + mock_tool.inputSchema = {"type": "object", "properties": {}} + + with patch( + "litellm.proxy._experimental.mcp_server.server._list_mcp_tools", + new_callable=AsyncMock, + return_value=[mock_tool], + ): + result = await self._get_call_fn()( + request=request, + user_api_key_dict=user_api_key_dict, + ) + + assert result.content + assert result.content[0].type == "text" + returned_tools = json.loads(result.content[0].text) + assert isinstance(returned_tools, list) + assert any(t["name"] == "github-create_issue" for t in returned_tools) + + @pytest.mark.asyncio + async def test_mcp_tool_call_executes_discovered_tool(self) -> None: + from mcp.types import CallToolResult, TextContent + + user_api_key_dict = UserAPIKeyAuth( + api_key="test_key", + object_permission=_make_perm( + mcp_tool_search_enabled=True, + mcp_servers=["github"], + ), + ) + + request = self._make_request( + { + "name": MCP_TOOL_CALL_TOOL_NAME, + "arguments": { + "tool_name": "github-create_issue", + "arguments": {"title": "bug", "repo": "myrepo"}, + }, + } + ) + + fake_result = CallToolResult( + content=[TextContent(type="text", text="Issue created")], + isError=False, + ) + + with ( + patch( + "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + new_callable=AsyncMock, + return_value=[MagicMock()], + ), + patch( + "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool", + new_callable=AsyncMock, + return_value=fake_result, + ) as mock_execute, + ): + result = await self._get_call_fn()( + request=request, + user_api_key_dict=user_api_key_dict, + ) + + mock_execute.assert_awaited_once() + assert mock_execute.await_args.kwargs["name"] == "github-create_issue" + + assert result.isError is False + assert result.content[0].text == "Issue created" + + @pytest.mark.asyncio + async def test_mcp_tool_call_forwards_client_ip_for_ip_filtering(self) -> None: + """Regression: the virtual call path must resolve allowed servers with the + request's client IP so IP-restricted servers (available_on_public_internet: + false) cannot be reached from a public IP.""" + from mcp.types import CallToolResult, TextContent + + user_api_key_dict = UserAPIKeyAuth( + api_key="test_key", + object_permission=_make_perm(mcp_tool_search_enabled=True, mcp_servers=["github"]), + ) + request = self._make_request( + { + "name": MCP_TOOL_CALL_TOOL_NAME, + "arguments": {"tool_name": "github-create_issue", "arguments": {}}, + } + ) + + fake_result = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False) + + with ( + patch( + "litellm.proxy._experimental.mcp_server.rest_endpoints.IPAddressUtils.get_mcp_client_ip", + return_value="203.0.113.7", + ), + patch( + "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + new_callable=AsyncMock, + return_value=[MagicMock()], + ) as mock_allowed, + patch( + "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool", + new_callable=AsyncMock, + return_value=fake_result, + ), + ): + await self._get_call_fn()(request=request, user_api_key_dict=user_api_key_dict) + + mock_allowed.assert_awaited_once() + assert mock_allowed.await_args.kwargs["client_ip"] == "203.0.113.7" + + @pytest.mark.asyncio + async def test_mcp_tool_search_forwards_client_ip_for_ip_filtering(self) -> None: + """Search must list tools through the IP-filtered catalog, not the raw one.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="test_key", + object_permission=_make_perm(mcp_tool_search_enabled=True, mcp_servers=["github"]), + ) + request = self._make_request({"name": MCP_TOOL_SEARCH_TOOL_NAME, "arguments": {"query": "issue"}}) + + with ( + patch( + "litellm.proxy._experimental.mcp_server.rest_endpoints.IPAddressUtils.get_mcp_client_ip", + return_value="203.0.113.7", + ), + patch( + "litellm.proxy._experimental.mcp_server.server._list_mcp_tools", + new_callable=AsyncMock, + return_value=[], + ) as mock_list, + ): + await self._get_call_fn()(request=request, user_api_key_dict=user_api_key_dict) + + mock_list.assert_awaited_once() + assert mock_list.await_args.kwargs["client_ip"] == "203.0.113.7" + + @pytest.mark.asyncio + async def test_mcp_tool_search_requires_flag_enabled(self) -> None: + from fastapi import HTTPException + + user_api_key_dict = UserAPIKeyAuth( + api_key="test_key", + object_permission=_make_perm(mcp_tool_search_enabled=False), + ) + + request = self._make_request({"name": MCP_TOOL_SEARCH_TOOL_NAME, "arguments": {"query": "create issue"}}) + + with pytest.raises(HTTPException) as exc_info: + await self._get_call_fn()( + request=request, + user_api_key_dict=user_api_key_dict, + ) + + assert exc_info.value.status_code in (400, 403, 404) + + +class TestDispatchVirtualMcpTool: + """Covers the SSE/protocol-path interception helper in server.py.""" + + @pytest.mark.asyncio + async def test_returns_none_for_non_virtual_tool(self) -> None: + from litellm.proxy._experimental.mcp_server.server import ( + _dispatch_virtual_mcp_tool, + ) + + result = await _dispatch_virtual_mcp_tool( + name="github-create_issue", + arguments={}, + user_api_key_auth=UserAPIKeyAuth(api_key="k"), + client_ip=None, + ) + assert result is None + + @pytest.mark.asyncio + async def test_rejects_when_flag_disabled(self) -> None: + from litellm.proxy._experimental.mcp_server.server import ( + _dispatch_virtual_mcp_tool, + ) + + uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=False)) + result = await _dispatch_virtual_mcp_tool( + name=MCP_TOOL_SEARCH_TOOL_NAME, + arguments={"query": "x"}, + user_api_key_auth=uak, + client_ip=None, + ) + assert result is not None + assert result.isError is True + + @pytest.mark.asyncio + async def test_routes_search_with_client_ip(self) -> None: + from litellm.proxy._experimental.mcp_server import server as srv + + uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) + with patch( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", + new_callable=AsyncMock, + return_value="SEARCH_RESULT", + ) as mock_search: + result = await srv._dispatch_virtual_mcp_tool( + name=MCP_TOOL_SEARCH_TOOL_NAME, + arguments={"query": "q", "top_k": 3}, + user_api_key_auth=uak, + client_ip="203.0.113.9", + ) + + assert result == "SEARCH_RESULT" + assert mock_search.await_args.kwargs["client_ip"] == "203.0.113.9" + assert mock_search.await_args.kwargs["query"] == "q" + assert mock_search.await_args.kwargs["top_k"] == 3 + + @pytest.mark.asyncio + async def test_routes_call_with_client_ip(self) -> None: + from litellm.proxy._experimental.mcp_server import server as srv + + uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) + with patch( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", + new_callable=AsyncMock, + return_value="CALL_RESULT", + ) as mock_call: + result = await srv._dispatch_virtual_mcp_tool( + name=MCP_TOOL_CALL_TOOL_NAME, + arguments={"tool_name": "math-add", "arguments": {"a": 1, "b": 2}}, + user_api_key_auth=uak, + client_ip="203.0.113.9", + mcp_auth_header="bearer-xyz", + mcp_server_auth_headers={"github": {"Authorization": "Bearer gh"}}, + oauth2_headers={"Authorization": "Bearer oauth"}, + raw_headers={"x-mcp-auth": "tok"}, + ) + + assert result == "CALL_RESULT" + kw = mock_call.await_args.kwargs + assert kw["tool_name"] == "math-add" + assert kw["client_ip"] == "203.0.113.9" + assert kw["mcp_auth_header"] == "bearer-xyz" + assert kw["mcp_server_auth_headers"] == {"github": {"Authorization": "Bearer gh"}} + assert kw["oauth2_headers"] == {"Authorization": "Bearer oauth"} + assert kw["raw_headers"] == {"x-mcp-auth": "tok"} + + @pytest.mark.asyncio + async def test_call_builds_and_forwards_logging_obj(self) -> None: + """Regression: the SSE dispatch must run the pre-call pipeline and forward + the resulting logging object to handle_mcp_tool_call, otherwise mcp_tool_call + over /mcp/ skips spend logging and guardrails (unlike the REST path).""" + from litellm.proxy._experimental.mcp_server import server as srv + + uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) + sentinel_logging_obj = object() + with ( + patch.object( + srv, + "_build_virtual_call_logging_obj", + new_callable=AsyncMock, + return_value=sentinel_logging_obj, + ) as mock_build, + patch( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", + new_callable=AsyncMock, + return_value="CALL_RESULT", + ) as mock_call, + ): + await srv._dispatch_virtual_mcp_tool( + name=MCP_TOOL_CALL_TOOL_NAME, + arguments={"tool_name": "math-add", "arguments": {"a": 1}}, + user_api_key_auth=uak, + client_ip=None, + ) + + assert mock_build.await_count == 1 + assert mock_call.await_args.kwargs["litellm_logging_obj"] is sentinel_logging_obj + + @pytest.mark.asyncio + async def test_search_coerces_non_int_top_k(self) -> None: + """Regression: a non-integer top_k from an MCP client must not raise; it + falls back to the default instead of ValueError propagating out.""" + from litellm.proxy._experimental.mcp_server import server as srv + + uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) + with patch( + "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", + new_callable=AsyncMock, + return_value="SEARCH_RESULT", + ) as mock_search: + await srv._dispatch_virtual_mcp_tool( + name=MCP_TOOL_SEARCH_TOOL_NAME, + arguments={"query": "issue", "top_k": "not-a-number"}, + user_api_key_auth=uak, + client_ip=None, + ) + + assert mock_search.await_args.kwargs["top_k"] == 5 + + @pytest.mark.asyncio + async def test_call_handler_forwards_auth_headers_to_execute(self) -> None: + """Regression: per-request auth headers must reach execute_mcp_tool so + upstream MCP servers needing pass-through auth can be called.""" + from mcp.types import CallToolResult, TextContent + + from litellm.proxy._experimental.mcp_server.tool_search import ( + handle_mcp_tool_call, + ) + + uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) + fake = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False) + with ( + patch( + "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + new_callable=AsyncMock, + return_value=[MagicMock()], + ) as mock_allowed, + patch( + "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool", + new_callable=AsyncMock, + return_value=fake, + ) as mock_exec, + ): + sentinel_logging_obj = object() + await handle_mcp_tool_call( + tool_name="github-create_issue", + arguments={}, + user_api_key_dict=uak, + mcp_servers=["github"], + mcp_auth_header="bearer-xyz", + mcp_server_auth_headers={"github": {"Authorization": "Bearer gh"}}, + oauth2_headers={"Authorization": "Bearer oauth"}, + raw_headers={"x-mcp-auth": "tok"}, + litellm_logging_obj=sentinel_logging_obj, + ) + + kw = mock_exec.await_args.kwargs + assert kw["mcp_auth_header"] == "bearer-xyz" + assert kw["mcp_server_auth_headers"] == {"github": {"Authorization": "Bearer gh"}} + assert kw["oauth2_headers"] == {"Authorization": "Bearer oauth"} + assert kw["raw_headers"] == {"x-mcp-auth": "tok"} + # Spend logging: the logging object must reach execute_mcp_tool + assert kw["litellm_logging_obj"] is sentinel_logging_obj + # Scoped session: the requested mcp_servers scope must reach server resolution + assert mock_allowed.await_args.kwargs["mcp_servers"] == ["github"] + + @pytest.mark.asyncio + async def test_call_rejected_when_no_accessible_servers(self) -> None: + """Regression: a key with no accessible MCP servers must not reach + execute_mcp_tool, where an unprefixed local tool name would otherwise + run via the local registry without a server permission check.""" + from fastapi import HTTPException + + from litellm.proxy._experimental.mcp_server.tool_search import ( + handle_mcp_tool_call, + ) + + uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) + with ( + patch( + "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + new_callable=AsyncMock, + return_value=[], + ), + patch( + "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool", + new_callable=AsyncMock, + ) as mock_exec, + ): + with pytest.raises(HTTPException) as exc_info: + await handle_mcp_tool_call( + tool_name="local_secret_tool", + arguments={}, + user_api_key_dict=uak, + ) + + assert exc_info.value.status_code == 403 + mock_exec.assert_not_awaited() + + +class TestCaptureHostProgressCallback: + """Covers the host progress-forwarding helper extracted from the tool call path.""" + + def test_returns_none_when_request_context_unavailable(self) -> None: + from litellm.proxy._experimental.mcp_server.server import ( + _capture_host_progress_callback, + ) + + class _NoCtx: + @property + def request_context(self): # type: ignore[no-untyped-def] + raise RuntimeError("no context") + + assert _capture_host_progress_callback(_NoCtx()) is None + + def test_returns_none_when_no_progress_token(self) -> None: + from litellm.proxy._experimental.mcp_server.server import ( + _capture_host_progress_callback, + ) + + host = MagicMock() + host.request_context.meta.progressToken = None + assert _capture_host_progress_callback(host) is None + + def test_returns_callable_when_token_present(self) -> None: + from litellm.proxy._experimental.mcp_server.server import ( + _capture_host_progress_callback, + ) + + host = MagicMock() + host.request_context.meta.progressToken = "tok12345" + host.request_context.session = MagicMock() + assert callable(_capture_host_progress_callback(host)) + + +class TestHandleListToolsVirtual: + """Covers the protocol list_tools early-return when the flag is enabled.""" + + @pytest.mark.asyncio + async def test_returns_virtual_tools_when_flag_enabled(self) -> None: + from litellm.proxy._experimental.mcp_server import server as srv + + uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) + with patch( + "litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context", + new_callable=AsyncMock, + return_value=(uak, None, None, None, None, None, None), + ): + tools = await srv.handle_list_tools() + + assert {t.name for t in tools} == { + MCP_TOOL_SEARCH_TOOL_NAME, + MCP_TOOL_CALL_TOOL_NAME, + } + + +class TestMcpServerToolCallErrorHandling: + """The protocol tool-call handler must convert virtual-tool errors to an + isError CallToolResult instead of letting them raise out of the handler.""" + + @pytest.mark.asyncio + async def test_virtual_tool_error_returns_iserror_not_raised(self) -> None: + from fastapi import HTTPException + + from litellm.proxy._experimental.mcp_server import server as srv + + uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) + with ( + patch( + "litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context", + new_callable=AsyncMock, + return_value=(uak, None, None, None, None, None, None), + ), + patch( + "litellm.proxy._experimental.mcp_server.server._dispatch_virtual_mcp_tool", + new_callable=AsyncMock, + side_effect=HTTPException(status_code=403, detail="User not allowed to call this tool"), + ), + ): + result = await srv.mcp_server_tool_call( + name=MCP_TOOL_CALL_TOOL_NAME, + arguments={"tool_name": "other-server-tool", "arguments": {}}, + ) + + assert result.isError is True + assert "User not allowed to call this tool" in result.content[0].text diff --git a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py index d4089b23e81..5fbc3c4869b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py @@ -718,6 +718,7 @@ _EXPECTED_CUSTOMER = { "mcp_toolsets": None, "blocked_tools": [], "search_tools": [], + "mcp_tool_search_enabled": None, }, } diff --git a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py index 26c8c774812..0981c4239ee 100644 --- a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py +++ b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py @@ -87,6 +87,38 @@ async def test_set_object_permission(): assert result["models"] == ["gpt-4"] +@pytest.mark.asyncio +async def test_set_object_permission_persists_mcp_tool_search_enabled(): + """ + Regression: mcp_tool_search_enabled must be carried into the Prisma create + payload so it persists to LiteLLM_ObjectPermissionTable. The field was + present on the Pydantic models but missing from the create path, so keys + generated with mcp_tool_search_enabled=True silently lost the flag. + """ + mock_prisma_client = MagicMock() + mock_created_permission = MagicMock() + mock_created_permission.object_permission_id = "perm_id" + mock_prisma_client.db.litellm_objectpermissiontable.create = AsyncMock( + return_value=mock_created_permission + ) + + data_json = { + "object_permission": { + "mcp_servers": ["server_a"], + "mcp_tool_search_enabled": True, + }, + } + + await _set_object_permission(data_json=data_json, prisma_client=mock_prisma_client) + + created_data = ( + mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs[ + "data" + ] + ) + assert created_data["mcp_tool_search_enabled"] is True + + # ---- Tests for _extract_requested_mcp_server_ids ---- diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index f15eaf9ea1f..ddf2040cd04 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25107,6 +25107,8 @@ export interface components { mcp_tool_permissions?: { [key: string]: string[]; } | null; + /** Mcp Tool Search Enabled */ + mcp_tool_search_enabled?: boolean | null; /** Mcp Toolsets */ mcp_toolsets?: string[] | null; /** Models */ @@ -25150,6 +25152,8 @@ export interface components { mcp_tool_permissions?: { [key: string]: string[]; } | null; + /** Mcp Tool Search Enabled */ + mcp_tool_search_enabled?: boolean | null; /** Mcp Toolsets */ mcp_toolsets?: string[] | null; /** From 13b590c8ec8d5c0d69bdd6e6affe51a57976fd4b Mon Sep 17 00:00:00 2001 From: tin-berri Date: Tue, 30 Jun 2026 21:23:49 -0700 Subject: [PATCH 19/81] fix(proxy): hydrate MCP server registry from DB on startup when store_model_in_db is false (#31775) MCP servers created through the UI are persisted to the database independent of store_model_in_db, but the in-memory registry that GET /v1/mcp/server reads was hydrated from the database only through add_deployment, which runs solely when store_model_in_db is True. On a DB-backed single-instance proxy with store_model_in_db unset the registry started empty after a restart, so the MCP Servers page showed nothing until an add or edit triggered a reload. Hydrate the registry from the database on startup regardless of store_model_in_db via a new ProxyConfig.init_mcp_servers_from_db, honoring supported_db_objects. --- litellm/proxy/proxy_server.py | 7 +++ tests/test_litellm/proxy/test_proxy_server.py | 60 +++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 2f6c48a751b..6d64843fab0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6314,6 +6314,10 @@ class ProxyConfig: "litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db - {}".format(str(e)) ) + async def init_mcp_servers_from_db(self) -> None: + if self._should_load_db_object(object_type="mcp"): + await self._init_mcp_servers_in_db() + async def _init_agents_in_db(self, prisma_client: PrismaClient): from litellm.proxy.agent_endpoints.agent_registry import ( global_agent_registry as AGENT_REGISTRY, @@ -7561,6 +7565,9 @@ class ProxyStartupEvent: ) await proxy_config.get_credentials(prisma_client=prisma_client) + if store_model_in_db is not True: + await proxy_config.init_mcp_servers_from_db() + await cls._initialize_slack_alerting_jobs( scheduler=scheduler, general_settings=general_settings, diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 88d9ad0d968..0d6cd972459 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -754,6 +754,66 @@ async def test_initialize_scheduled_jobs_credentials(monkeypatch): assert len(mock_scheduler_calls) > 0 +@pytest.mark.asyncio +async def test_initialize_scheduled_jobs_hydrates_mcp_when_store_model_in_db_false(monkeypatch): + """ + Regression (LIT-4128): MCP servers created via the UI are persisted to the DB + regardless of store_model_in_db, but the in-memory registry that GET + /v1/mcp/server reads is hydrated from the DB only by the store_model_in_db + model-sync loop (add_deployment). On a DB-backed proxy with store_model_in_db + unset the registry must still be hydrated on startup so previously-added + servers survive a restart instead of showing an empty list until a write. + """ + monkeypatch.delenv("DISABLE_PRISMA_SCHEMA_UPDATE", raising=False) + monkeypatch.delenv("STORE_MODEL_IN_DB", raising=False) + from litellm.proxy.proxy_server import ProxyStartupEvent + from litellm.proxy.utils import ProxyLogging + + mock_prisma_client = MagicMock() + mock_proxy_logging = MagicMock(spec=ProxyLogging) + mock_proxy_logging.slack_alerting_instance = MagicMock() + mock_proxy_config = AsyncMock() + + with ( + patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", False), + ): + await ProxyStartupEvent.initialize_scheduled_background_jobs( + general_settings={}, + prisma_client=mock_prisma_client, + proxy_budget_rescheduler_min_time=1, + proxy_budget_rescheduler_max_time=2, + proxy_batch_write_at=5, + proxy_logging_obj=mock_proxy_logging, + ) + + mock_proxy_config.add_deployment.assert_not_called() + mock_proxy_config.init_mcp_servers_from_db.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_init_mcp_servers_from_db_respects_supported_db_objects(monkeypatch): + """ + init_mcp_servers_from_db hydrates MCP from the DB by default but skips it when + an explicit supported_db_objects allowlist omits "mcp". + """ + from litellm.proxy.proxy_server import ProxyConfig + + config = ProxyConfig() + with patch.object(config, "_init_mcp_servers_in_db", new=AsyncMock()) as mock_init: + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + await config.init_mcp_servers_from_db() + mock_init.assert_awaited_once() + + mock_init.reset_mock() + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"supported_db_objects": ["models"]}, + ) + await config.init_mcp_servers_from_db() + mock_init.assert_not_awaited() + + def test_update_config_fields_deep_merge_db_wins(): from litellm.proxy.proxy_server import ProxyConfig From e1415962049cfdc3f94522c697aaee4b25948e08 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 1 Jul 2026 08:12:35 -0700 Subject: [PATCH 20/81] refactor(lint): collapse type/lint budgets to a single per-rule limit (#31883) * chore(lint): raise basedpyright per-rule slack to 50% of baseline The per-rule ceilings in basedpyright-code-budget.json sat at roughly 10% slack over baseline, which several in-flight PRs are already bumping into. Raise the slack on every rule to at least 50% of its baseline so there is ample headroom for a long while, while never lowering any rule that already had more generous slack (e.g. reportReturnType stays at 100). Co-authored-by: Mateo Wang * refactor(lint): collapse type/lint budgets to a single per-rule limit The three non-frontend budget files (ruff-strict, type-discipline, basedpyright-code) tracked a per-rule baseline and slack whose sum was the ceiling. Nothing consumed the split beyond that sum, so this replaces both keys with a single limit equal to the old baseline + slack; the original baselines live in git history if anyone needs them. The gate scripts and the ratchet guard now read limit directly. lint-budget-update no longer re-captures raw counts; it ratchets each rule's limit down by the number of violations this branch cleared since its branch point (the merge-base), so the granted headroom shrinks by exactly what was fixed and a limit never rises. The ratchet guard reads either schema so it still compares correctly across the migration boundary. Co-authored-by: Mateo Wang * chore(lint): surface staged-vs-working parity for pre-commit and budget-update make pre-commit selects which checks to run from the staged index but runs the linters over the working tree, so unstaged edits to tracked files and untracked files skew a green/red away from what a commit of only the staged changes would produce. There is no safe in-place way to lint the index, so the script now warns when unstaged or untracked changes are present, and CLAUDE.md documents that you must stage everything first for both make pre-commit and make lint-budget-update to predict CI correctly. Co-authored-by: Mateo Wang * docs(lint): list type-discipline budget in lint-budget-update instruction --------- Co-authored-by: Cursor Agent Co-authored-by: Mateo Wang --- CLAUDE.md | 4 +- Makefile | 23 +- basedpyright-code-budget.json | 144 +++---- ruff-strict-budget.json | 366 ++++++------------ scripts/budget_ratchet_check.py | 89 ++--- scripts/pre_commit_lint.sh | 16 + scripts/ruff_strict_gate.py | 55 ++- scripts/type_check_gate.py | 100 +++-- scripts/type_discipline_gate.py | 68 ++-- .../test_litellm/test_budget_ratchet_check.py | 76 ++-- tests/test_litellm/test_ruff_strict_gate.py | 36 +- tests/test_litellm/test_type_check_gate.py | 67 ++-- .../test_litellm/test_type_discipline_gate.py | 36 +- type-discipline-budget.json | 24 +- 14 files changed, 515 insertions(+), 589 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 83bf3e22d27..86bd89156a3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -33,9 +33,9 @@ If you ever make public-facing PR descriptions, comments, issues, commit message Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs -Run tests before you commit. Also, run `make pre-commit` right before each commit, which generates types (as needed) and formats/lints your code. Any errors found must be fixed +Run tests before you commit. Also, run `make pre-commit` right before each commit, which generates types (as needed) and formats/lints your code. Any errors found must be fixed. For `make pre-commit` to work properly you must stage your changes first (git add): it reports CI red or green based on what would happen if you committed your staged changes, but it runs the linters over the working tree, so any unstaged edits to tracked files or untracked files are folded into the result and will skew it away from what CI (which only sees your commit) would report -When you fix violations gated by `ruff-strict-budget.json` or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered baselines so the ceilings ratchet down instead of leaving stale headroom +When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered limits so the ceilings ratchet down instead of leaving stale headroom. It lowers each rule's limit by the number of violations this branch cleared since its branch point and never raises one, measured against the working tree, so stage exactly the fixes you're committing before running it; crediting unstaged fixes you won't commit would over-tighten the limits and turn CI red once the committed subset is checked If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in diff --git a/Makefile b/Makefile index fb927148c80..c3fa21c156c 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \ test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \ info lint lint-dev format \ - lint-basedpyright lint-basedpyright-budget-update \ + lint-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \ lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \ install-dev install-proxy-dev install-test-deps install-hooks \ install-helm-unittest check-circular-imports check-import-safety pre-commit \ @@ -27,12 +27,12 @@ help: @echo " make lint - Run all linting (Ruff, basedpyright, format check, circular imports, import safety)" @echo " make lint-ruff - Run Ruff linting only" @echo " make lint-basedpyright - Run basedpyright strict, gated by per-rule error counts" - @echo " make lint-basedpyright-budget-update - Re-capture the basedpyright per-rule budget (ratchet)" + @echo " make lint-basedpyright-budget-update - Ratchet basedpyright limits down by what this branch fixed" @echo " make lint-format - Check ruff format formatting (matches CI)" - @echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its ceiling" + @echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its limit" @echo " make lint-gate - Strict ruff gate in CI-parity mode (fetches staging, simulates the merge)" - @echo " make lint-ruff-budget-update - Re-capture per-rule baselines in ruff-strict-budget.json (ratchet)" - @echo " make lint-budget-update - Re-capture all ratchet budgets (ruff + basedpyright)" + @echo " make lint-ruff-budget-update - Ratchet ruff-strict-budget.json limits down by what this branch fixed" + @echo " make lint-budget-update - Ratchet all budgets down (ruff + type-discipline + basedpyright)" @echo " make check-circular-imports - Check for circular imports" @echo " make check-import-safety - Check import safety" @echo " make test - Run all tests" @@ -164,7 +164,9 @@ lint-basedpyright: install-dev lint-fetch-base lint-type-discipline: install-dev lint-fetch-base $(UV_RUN) python scripts/type_discipline_gate.py --base origin/litellm_internal_staging -lint-basedpyright-budget-update: install-dev +# --update lowers each limit by what this branch fixed since its branch point, so +# it needs the base ref fetched to resolve the merge-base. +lint-basedpyright-budget-update: install-dev lint-fetch-base ($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --update lint-format: format-check @@ -177,11 +179,14 @@ lint-ruff-budget: install-dev lint-gate: install-dev lint-fetch-base $(UV_RUN) python scripts/ruff_strict_gate.py --base origin/litellm_internal_staging -lint-ruff-budget-update: install-dev +lint-ruff-budget-update: install-dev lint-fetch-base $(UV_RUN) python scripts/ruff_strict_gate.py --update -# Ratchet all budgets in one shot (ruff strict + basedpyright) -lint-budget-update: lint-ruff-budget-update lint-basedpyright-budget-update +lint-type-discipline-budget-update: install-dev lint-fetch-base + $(UV_RUN) python scripts/type_discipline_gate.py --update + +# Ratchet all budgets in one shot (ruff strict + type-discipline + basedpyright) +lint-budget-update: lint-ruff-budget-update lint-type-discipline-budget-update lint-basedpyright-budget-update check-circular-imports: install-dev cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd .. diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index f2b54e1f889..79e6af05978 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,194 +1,146 @@ { "reportAny": { - "baseline": 24989, - "slack": 2500 + "limit": 37484 }, "reportArgumentType": { - "baseline": 1814, - "slack": 180 + "limit": 2721 }, "reportAssignmentType": { - "baseline": 220, - "slack": 22 + "limit": 330 }, "reportAttributeAccessIssue": { - "baseline": 346, - "slack": 35 + "limit": 519 }, "reportCallIssue": { - "baseline": 87, - "slack": 10 + "limit": 131 }, "reportConstantRedefinition": { - "baseline": 39, - "slack": 4 + "limit": 59 }, "reportDeprecated": { - "baseline": 217, - "slack": 22 + "limit": 326 }, "reportDuplicateImport": { - "baseline": 28, - "slack": 3 + "limit": 42 }, "reportExplicitAny": { - "baseline": 6931, - "slack": 700 + "limit": 10397 }, "reportFunctionMemberAccess": { - "baseline": 7, - "slack": 3 + "limit": 11 }, "reportGeneralTypeIssues": { - "baseline": 151, - "slack": 15 + "limit": 227 }, "reportIncompatibleMethodOverride": { - "baseline": 52, - "slack": 5 + "limit": 78 }, "reportIncompatibleVariableOverride": { - "baseline": 8, - "slack": 3 + "limit": 12 }, "reportInconsistentOverload": { - "baseline": 12, - "slack": 3 + "limit": 18 }, "reportIndexIssue": { - "baseline": 26, - "slack": 3 + "limit": 39 }, "reportInvalidTypeForm": { - "baseline": 23, - "slack": 3 + "limit": 35 }, "reportInvalidTypeVarUse": { - "baseline": 2, - "slack": 3 + "limit": 5 }, "reportMatchNotExhaustive": { - "baseline": 1, - "slack": 0 + "limit": 2 }, "reportMissingParameterType": { - "baseline": 3933, - "slack": 390 + "limit": 5900 }, "reportMissingTypeArgument": { - "baseline": 10612, - "slack": 1000 + "limit": 15918 }, "reportMissingTypeStubs": { - "baseline": 27, - "slack": 10 + "limit": 41 }, "reportOperatorIssue": { - "baseline": 6, - "slack": 3 + "limit": 9 }, "reportOptionalCall": { - "baseline": 4, - "slack": 3 + "limit": 7 }, "reportOptionalIterable": { - "baseline": 3, - "slack": 3 + "limit": 6 }, "reportOptionalMemberAccess": { - "baseline": 724, - "slack": 72 + "limit": 1086 }, "reportOptionalOperand": { - "baseline": 3, - "slack": 3 + "limit": 6 }, "reportOptionalSubscript": { - "baseline": 11, - "slack": 3 + "limit": 17 }, "reportPossiblyUnboundVariable": { - "baseline": 52, - "slack": 10 + "limit": 78 }, "reportPrivateUsage": { - "baseline": 1625, - "slack": 160 + "limit": 2438 }, "reportRedeclaration": { - "baseline": 8, - "slack": 3 + "limit": 12 }, "reportReturnType": { - "baseline": 126, - "slack": 100 + "limit": 226 }, "reportTypedDictNotRequiredAccess": { - "baseline": 20, - "slack": 3 + "limit": 30 }, "reportUndefinedVariable": { - "baseline": 2, - "slack": 3 + "limit": 5 }, "reportUnknownArgumentType": { - "baseline": 30603, - "slack": 3000 + "limit": 45905 }, "reportUnknownLambdaType": { - "baseline": 75, - "slack": 10 + "limit": 113 }, "reportUnknownMemberType": { - "baseline": 27037, - "slack": 2500 + "limit": 40556 }, "reportUnknownParameterType": { - "baseline": 13612, - "slack": 1000 + "limit": 20418 }, "reportUnknownVariableType": { - "baseline": 21445, - "slack": 2000 + "limit": 32168 }, "reportUnnecessaryCast": { - "baseline": 118, - "slack": 10 + "limit": 177 }, "reportUnnecessaryComparison": { - "baseline": 683, - "slack": 100 + "limit": 1025 }, "reportUnnecessaryContains": { - "baseline": 4, - "slack": 3 + "limit": 7 }, "reportUnnecessaryIsInstance": { - "baseline": 808, - "slack": 80 + "limit": 1212 }, "reportUntypedBaseClass": { - "baseline": 110, - "slack": 11 + "limit": 165 }, "reportUntypedFunctionDecorator": { - "baseline": 22, - "slack": 3 + "limit": 33 }, "reportUnusedClass": { - "baseline": 22, - "slack": 3 + "limit": 33 }, "reportUnusedFunction": { - "baseline": 137, - "slack": 10 + "limit": 206 }, "reportUnusedImport": { - "baseline": 670, - "slack": 50 + "limit": 1005 }, "reportUnusedVariable": { - "baseline": 865, - "slack": 50 + "limit": 1298 } } diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 10c820324ea..be62f8a9d67 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -1,490 +1,368 @@ { "ANN001": { - "baseline": 2865, - "slack": 287 + "limit": 3152 }, "ANN002": { - "baseline": 64, - "slack": 5 + "limit": 69 }, "ANN003": { - "baseline": 759, - "slack": 76 + "limit": 835 }, "ANN201": { - "baseline": 1944, - "slack": 194 + "limit": 2138 }, "ANN202": { - "baseline": 858, - "slack": 86 + "limit": 944 }, "ANN204": { - "baseline": 658, - "slack": 66 + "limit": 724 }, "ANN205": { - "baseline": 117, - "slack": 10 + "limit": 127 }, "ANN206": { - "baseline": 120, - "slack": 10 + "limit": 130 }, "ANN401": { - "baseline": 1886, - "slack": 189 + "limit": 2075 }, "ASYNC230": { - "baseline": 11, - "slack": 3 + "limit": 14 }, "B004": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "B006": { - "baseline": 180, - "slack": 10 + "limit": 190 }, "B008": { - "baseline": 490, - "slack": 15 + "limit": 505 }, "B009": { - "baseline": 79, - "slack": 5 + "limit": 84 }, "B010": { - "baseline": 187, - "slack": 10 + "limit": 197 }, "B018": { - "baseline": 2, - "slack": 3 + "limit": 5 }, "B019": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "B021": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "B026": { - "baseline": 3, - "slack": 3 + "limit": 6 }, "B033": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "BLE001": { - "baseline": 2854, - "slack": 50 + "limit": 2904 }, "C401": { - "baseline": 8, - "slack": 3 + "limit": 11 }, "C404": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "C405": { - "baseline": 20, - "slack": 3 + "limit": 23 }, "C408": { - "baseline": 11, - "slack": 3 + "limit": 14 }, "C414": { - "baseline": 4, - "slack": 3 + "limit": 7 }, "C419": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "C901": { - "baseline": 301, - "slack": 15 + "limit": 316 }, "D419": { - "baseline": 6, - "slack": 3 + "limit": 9 }, "DTZ001": { - "baseline": 2, - "slack": 3 + "limit": 5 }, "DTZ003": { - "baseline": 30, - "slack": 3 + "limit": 33 }, "DTZ005": { - "baseline": 229, - "slack": 15 + "limit": 244 }, "DTZ006": { - "baseline": 10, - "slack": 3 + "limit": 13 }, "DTZ007": { - "baseline": 20, - "slack": 3 + "limit": 23 }, "DTZ011": { - "baseline": 3, - "slack": 3 + "limit": 6 }, "EXE001": { - "baseline": 4, - "slack": 3 + "limit": 7 }, "EXE002": { - "baseline": 3, - "slack": 3 + "limit": 6 }, "F401": { - "baseline": 20, - "slack": 3 + "limit": 23 }, "FURB136": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "FURB168": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "FURB188": { - "baseline": 49, - "slack": 3 + "limit": 52 }, "I001": { - "baseline": 258, - "slack": 15 + "limit": 273 }, "LOG015": { - "baseline": 5, - "slack": 3 + "limit": 8 }, "N999": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "PERF102": { - "baseline": 27, - "slack": 3 + "limit": 30 }, "PERF401": { - "baseline": 136, - "slack": 10 + "limit": 146 }, "PERF402": { - "baseline": 6, - "slack": 3 + "limit": 9 }, "PERF403": { - "baseline": 69, - "slack": 5 + "limit": 74 }, "PIE790": { - "baseline": 263, - "slack": 15 + "limit": 278 }, "PIE800": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "PIE804": { - "baseline": 21, - "slack": 3 + "limit": 24 }, "PIE810": { - "baseline": 41, - "slack": 3 + "limit": 44 }, "PLC0206": { - "baseline": 28, - "slack": 3 + "limit": 31 }, "PLC0208": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "PLC0414": { - "baseline": 35, - "slack": 3 + "limit": 38 }, "PLR0124": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "PLR0206": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "PLR0402": { - "baseline": 6, - "slack": 3 + "limit": 9 }, "PLR1704": { - "baseline": 3, - "slack": 3 + "limit": 6 }, "PLR1711": { - "baseline": 31, - "slack": 3 + "limit": 34 }, "PLR1714": { - "baseline": 252, - "slack": 15 + "limit": 267 }, "PLR1730": { - "baseline": 7, - "slack": 3 + "limit": 10 }, "PLR2044": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "PLW0127": { - "baseline": 41, - "slack": 3 + "limit": 44 }, "PLW0133": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "PLW0602": { - "baseline": 215, - "slack": 15 + "limit": 230 }, "PLW0603": { - "baseline": 183, - "slack": 10 + "limit": 193 }, "PLW1508": { - "baseline": 188, - "slack": 10 + "limit": 198 }, "PLW1510": { - "baseline": 2, - "slack": 3 + "limit": 5 }, "PYI030": { - "baseline": 2, - "slack": 3 + "limit": 5 }, "PYI036": { - "baseline": 2, - "slack": 3 + "limit": 5 }, "PYI041": { - "baseline": 9, - "slack": 3 + "limit": 12 }, "PYI064": { - "baseline": 2, - "slack": 3 + "limit": 5 }, "RET501": { - "baseline": 35, - "slack": 3 + "limit": 38 }, "RET504": { - "baseline": 702, - "slack": 20 + "limit": 722 }, "RUF010": { - "baseline": 844, - "slack": 30 + "limit": 874 }, "RUF012": { - "baseline": 158, - "slack": 10 + "limit": 168 }, "RUF015": { - "baseline": 8, - "slack": 3 + "limit": 11 }, "RUF019": { - "baseline": 38, - "slack": 3 + "limit": 41 }, "RUF022": { - "baseline": 80, - "slack": 5 + "limit": 85 }, "RUF023": { - "baseline": 2, - "slack": 3 + "limit": 5 }, "RUF046": { - "baseline": 5, - "slack": 3 + "limit": 8 }, "RUF051": { - "baseline": 3, - "slack": 3 + "limit": 6 }, "RUF059": { - "baseline": 69, - "slack": 5 + "limit": 74 }, "RUF100": { - "baseline": 465, - "slack": 15 + "limit": 480 }, "S110": { - "baseline": 222, - "slack": 15 + "limit": 237 }, "S112": { - "baseline": 21, - "slack": 3 + "limit": 24 }, "SIM101": { - "baseline": 58, - "slack": 5 + "limit": 63 }, "SIM102": { - "baseline": 311, - "slack": 15 + "limit": 326 }, "SIM103": { - "baseline": 119, - "slack": 10 + "limit": 129 }, "SIM113": { - "baseline": 3, - "slack": 3 + "limit": 6 }, "SIM114": { - "baseline": 103, - "slack": 10 + "limit": 113 }, "SIM115": { - "baseline": 2, - "slack": 3 + "limit": 5 }, "SIM117": { - "baseline": 7, - "slack": 3 + "limit": 10 }, "SIM118": { - "baseline": 104, - "slack": 10 + "limit": 114 }, "SIM201": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "SIM210": { - "baseline": 9, - "slack": 3 + "limit": 12 }, "SIM211": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "SIM222": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "SIM401": { - "baseline": 9, - "slack": 3 + "limit": 12 }, "TC004": { - "baseline": 5, - "slack": 3 + "limit": 8 }, "TC005": { - "baseline": 6, - "slack": 3 + "limit": 9 }, "TID251": { - "baseline": 2664, - "slack": 50 + "limit": 2714 }, "TRY002": { - "baseline": 528, - "slack": 20 + "limit": 548 }, "TRY004": { - "baseline": 93, - "slack": 5 + "limit": 98 }, "TRY201": { - "baseline": 409, - "slack": 15 + "limit": 424 }, "TRY203": { - "baseline": 113, - "slack": 10 + "limit": 123 }, "TRY300": { - "baseline": 853, - "slack": 30 + "limit": 883 }, "UP006": { - "baseline": 12941, - "slack": 100 + "limit": 13041 }, "UP007": { - "baseline": 2520, - "slack": 50 + "limit": 2570 }, "UP008": { - "baseline": 2, - "slack": 3 + "limit": 5 }, "UP012": { - "baseline": 4, - "slack": 3 + "limit": 7 }, "UP018": { - "baseline": 18, - "slack": 3 + "limit": 21 }, "UP024": { - "baseline": 12, - "slack": 3 + "limit": 15 }, "UP028": { - "baseline": 2, - "slack": 3 + "limit": 5 }, "UP031": { - "baseline": 2, - "slack": 3 + "limit": 5 }, "UP032": { - "baseline": 609, - "slack": 20 + "limit": 629 }, "UP034": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "UP035": { - "baseline": 2250, - "slack": 50 + "limit": 2300 }, "UP036": { - "baseline": 1, - "slack": 3 + "limit": 4 }, "UP037": { - "baseline": 100, - "slack": 5 + "limit": 105 }, "UP045": { - "baseline": 18417, - "slack": 100 + "limit": 18517 } } diff --git a/scripts/budget_ratchet_check.py b/scripts/budget_ratchet_check.py index df9815d6557..10a78483643 100644 --- a/scripts/budget_ratchet_check.py +++ b/scripts/budget_ratchet_check.py @@ -1,19 +1,16 @@ #!/usr/bin/env python3 -"""Non-gating ratchet guard: budget baselines and ceilings may only fall, never rise. +"""Non-gating ratchet guard: budget limits may only fall, never rise. Every `*-budget.json` file (ruff-strict, type-discipline, basedpyright-code) is a -one-way ratchet: each rule's ceiling is `baseline + slack`, and both the recorded -`baseline` (the live violation count) and that ceiling are meant to be driven DOWN -over time. This check compares every budget file against its own content at the -merge-base with the target branch and fails (exits 1, red) if: +one-way ratchet: each rule's ceiling is its `limit`, and that limit is meant to be +driven DOWN over time. This check compares every budget file against its own +content at the merge-base with the target branch and fails (exits 1, red) if: - * a rule's ceiling (`baseline + slack`) went up, - * a rule's `baseline` went up, even if `slack` was lowered to keep the ceiling - flat (a higher baseline bakes in more accepted debt and must be acknowledged), + * a rule's `limit` went up, * a rule was dropped from a budget (its ceiling effectively became infinite), or * an entire budget file was deleted. -New rules and lowered/equal baselines and ceilings are fine. +New rules and lowered/equal limits are fine. This is deliberately NOT a gating check. It should turn the run red so that a loosening is impossible to miss in review, but it must stay OUT of the @@ -89,19 +86,21 @@ def _load_base(rel: str, ref: str) -> dict | None: return json.loads(proc.stdout) -def _baselines(budget: dict) -> dict[str, int]: - """Map each rule to its recorded baseline; skip malformed specs.""" - return { - rule: int(spec.get("baseline", 0)) - for rule, spec in budget.items() - if isinstance(spec, dict) - } +def _ceiling(spec: dict) -> int: + """A rule's ceiling: its `limit`, or legacy `baseline + slack`. + + The base side of the diff can predate the `limit` migration, so a spec is read + under either schema and the two are compared on the same footing. + """ + if "limit" in spec: + return int(spec["limit"]) + return int(spec.get("baseline", 0)) + int(spec.get("slack", 0)) -def _caps(budget: dict) -> dict[str, int]: - """Map each rule to its ceiling (baseline + slack); skip malformed specs.""" +def _limits(budget: dict) -> dict[str, int]: + """Map each rule to its ceiling; skip malformed specs.""" return { - rule: int(spec.get("baseline", 0)) + int(spec.get("slack", 0)) + rule: _ceiling(spec) for rule, spec in budget.items() if isinstance(spec, dict) } @@ -109,54 +108,32 @@ def _caps(budget: dict) -> dict[str, int]: def _regression_detail( rule: str, - base_caps: dict[str, int], - head_caps: dict[str, int], - base_baselines: dict[str, int], - head_baselines: dict[str, int], + base_limits: dict[str, int], + head_limits: dict[str, int], ) -> str | None: """Why `rule` regressed vs base, or None when it held flat or fell. - A dropped rule is terminal; otherwise a raised ceiling and a raised baseline are - independent loosenings (the latter catches a baseline bump masked by a slack cut), - so both reasons are reported when both apply. + A dropped rule is terminal; otherwise the only loosening left is a raised limit. """ - base_cap = base_caps[rule] - if rule not in head_caps: - return f"rule dropped (ceiling {base_cap} -> removed)" - reasons = tuple( - message - for raised, message in ( - ( - head_caps[rule] > base_cap, - f"ceiling raised {base_cap} -> {head_caps[rule]}", - ), - ( - head_baselines[rule] > base_baselines[rule], - f"baseline raised {base_baselines[rule]} -> {head_baselines[rule]}", - ), - ) - if raised - ) - return "; ".join(reasons) or None + base_limit = base_limits[rule] + if rule not in head_limits: + return f"rule dropped (limit {base_limit} -> removed)" + if head_limits[rule] > base_limit: + return f"limit raised {base_limit} -> {head_limits[rule]}" + return None def regressions_for(rel: str, base: dict | None, head: dict | None) -> list[Regression]: if base is None: return [] # new budget file: nothing to ratchet against yet if head is None: - return [Regression(rel, "*", "budget file was deleted (every ceiling removed)")] + return [Regression(rel, "*", "budget file was deleted (every limit removed)")] - base_caps, head_caps = _caps(base), _caps(head) - base_baselines, head_baselines = _baselines(base), _baselines(head) + base_limits, head_limits = _limits(base), _limits(head) return [ Regression(rel, rule, detail) - for rule in sorted(base_caps) - if ( - detail := _regression_detail( - rule, base_caps, head_caps, base_baselines, head_baselines - ) - ) - is not None + for rule in sorted(base_limits) + if (detail := _regression_detail(rule, base_limits, head_limits)) is not None ] @@ -191,7 +168,7 @@ def main() -> int: if regressions: print( - f"FAIL: budget baseline(s)/ceiling(s) loosened vs base {args.base} (merge-base {ref[:12]}):" + f"FAIL: budget limit(s) loosened vs base {args.base} (merge-base {ref[:12]}):" ) for reg in regressions: print(f" {reg.budget} {reg.rule}: {reg.detail}") @@ -203,7 +180,7 @@ def main() -> int: return 1 suffix = f" ({', '.join(checked)})" if checked else "" - print(f"OK: no budget ceiling increased vs base {args.base}{suffix}") + print(f"OK: no budget limit increased vs base {args.base}{suffix}") return 0 diff --git a/scripts/pre_commit_lint.sh b/scripts/pre_commit_lint.sh index 0852e9e0ca2..d667d6758e1 100755 --- a/scripts/pre_commit_lint.sh +++ b/scripts/pre_commit_lint.sh @@ -36,6 +36,22 @@ spec_files=$(staged_match '^(litellm/(proxy|types)/.*|ui/litellm-dashboard/(scri ui_prettier_files=$(staged_match '^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs|json|css|scss|md|mdx|yml|yaml|html)$') ui_eslint_files=$(staged_match '^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs)$') +# CI lints the committed tree, so this script predicts CI for what you have STAGED +# (every trigger above reads `git diff --cached`). The tools it runs, though, read +# the working tree, so unstaged edits to tracked files and untracked files fold +# into the result and a green/red here won't match a commit of just the staged +# changes. There's no safe way to lint the index in place, so surface the gap +# instead of hiding it: stage everything you intend to commit before trusting a +# pass. This only warns; it never blocks or touches your changes. +unstaged=$(git diff --name-only) +untracked=$(git ls-files --others --exclude-standard) +if [ -n "$unstaged" ] || [ -n "$untracked" ]; then + echo "pre-commit: NOTE - unstaged/untracked changes are included in these checks but" >&2 + echo " won't be in a commit of only your staged changes, so this result may differ from" >&2 + echo " CI. Stage everything you intend to commit (git add) for an accurate prediction:" >&2 + printf '%s\n' "$unstaged" "$untracked" | sed '/^$/d' | sed 's/^/ /' >&2 +fi + lint_dashboard() { ( rc=0 diff --git a/scripts/ruff_strict_gate.py b/scripts/ruff_strict_gate.py index 5951a1215ed..5273e4805f6 100644 --- a/scripts/ruff_strict_gate.py +++ b/scripts/ruff_strict_gate.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 """Total-count gate for the strict ruff rules in ruff-strict.toml. -Each rule has a hard ceiling (baseline + slack) in ruff-strict-budget.json. The -gate counts each rule across the whole tree and fails when a rule is both over -its ceiling and higher than the base it merges into, so a change is blamed for -the violations it adds, never for drift that already exists in the base. +Each rule has a hard ``limit`` in ruff-strict-budget.json. The gate counts each +rule across the whole tree and fails when a rule is both over its limit and +higher than the base it merges into, so a change is blamed for the violations it +adds, never for drift that already exists in the base. ``--update`` ratchets each +rule's limit down by the number of violations this branch fixed relative to its +branch point (the merge-base). """ import argparse @@ -90,7 +92,7 @@ def base_counts(ref: str) -> dict: def evaluate(head: dict, base: dict, budget: dict) -> list: breaches = [] for rule, spec in budget.items(): - cap = spec["baseline"] + spec["slack"] + cap = spec["limit"] total = head.get(rule, 0) if total > cap and total > base.get(rule, 0): breaches.append(Breach(rule, total, cap, total - base.get(rule, 0))) @@ -128,26 +130,49 @@ def cmd_check(base: str) -> None: _run(["git", "diff", base_point, "--unified=0", "--no-color", "--", TARGET]) ), ) - print(f"FAIL: strict-rule totals exceed their ceiling (base {base}):") + print(f"FAIL: strict-rule totals exceed their limit (base {base}):") for breach in breaches: print( - f" {breach.rule}: total {breach.total} over cap {breach.cap} (this change added {breach.added})" + f" {breach.rule}: total {breach.total} over limit {breach.cap} (this change added {breach.added})" ) for violation in sorted(v for v in new if v.code == breach.rule): print(f" {violation.file}:{violation.line}") print( - "Reduce the new violations or remove an equal number elsewhere; the ceiling is baseline + slack in ruff-strict-budget.json." + "Reduce the new violations or remove an equal number elsewhere; the ceiling is the limit in ruff-strict-budget.json." ) raise SystemExit(1) -def cmd_update() -> None: +def ratcheted_budget(budget: dict, current: dict, base: dict) -> dict: + """Each rule's limit lowered by the violations `current` fixed vs `base`. + + `base` is the count at the branch point (the commit this branch diverged + from). The drop is clamped to what was actually cleared (a rule that grew + stays put), so the limit only ever falls. + """ + return { + rule: { + "limit": max(0, spec["limit"] - max(0, base.get(rule, 0) - current.get(rule, 0))) + } + for rule, spec in sorted(budget.items()) + } + + +def cmd_update(base_ref: str = DEFAULT_BASE) -> None: + """Ratchet each rule's limit down by the violations this branch fixed. + + The working-tree count is compared against a ruff pass over a detached + worktree at the branch point (the merge-base with `base_ref`), so a branch's + fixes tighten its own ceilings by exactly what they cleared since it diverged. + """ budget = json.loads(BUDGET_PATH.read_text()) - head = count_by_rule(head_violations()) - for rule in budget: - budget[rule]["baseline"] = head.get(rule, 0) - BUDGET_PATH.write_text(json.dumps(budget, indent=2, sort_keys=True) + "\n") - print("Re-captured per-rule baselines from the current tree") + base_point = _run(["git", "merge-base", base_ref, "HEAD"]).strip() or base_ref + updated = ratcheted_budget( + budget, count_by_rule(head_violations()), base_counts(base_point) + ) + BUDGET_PATH.write_text(json.dumps(updated, indent=2, sort_keys=True) + "\n") + cleared = sum(budget[rule]["limit"] - updated[rule]["limit"] for rule in updated) + print(f"Ratcheted strict-rule limits down by {cleared} violations this branch fixed") def main() -> None: @@ -155,7 +180,7 @@ def main() -> None: parser.add_argument("--base", default=DEFAULT_BASE) parser.add_argument("--update", action="store_true") args = parser.parse_args() - cmd_update() if args.update else cmd_check(args.base) + cmd_update(args.base) if args.update else cmd_check(args.base) if __name__ == "__main__": diff --git a/scripts/type_check_gate.py b/scripts/type_check_gate.py index 2ef332d91ea..256fc433d8d 100644 --- a/scripts/type_check_gate.py +++ b/scripts/type_check_gate.py @@ -3,20 +3,22 @@ basedpyright's ``--outputjson`` is reduced to a count of errors per *rule* (``reportAny``, ``reportArgumentType``, ...) and checked against a committed -budget of the form ``{rule: {baseline, slack}}``, the same shape as +budget of the form ``{rule: {limit}}``, the same shape as ``ruff-strict-budget.json``. A rule fails only when its codebase-wide total is -both over its ceiling (``baseline + slack``) *and* higher than the count on the -base it merges into, so a change is blamed for the errors it adds, never for -drift that already sits in the base. That ``> base`` guard is what stops an -unrelated PR from inheriting a red once two PRs each land near the ceiling and -their sum crosses it: the bystander's count equals its base, so it is spared, -while any PR that actually grows the rule past the cap still fails. +both over its ``limit`` *and* higher than the count on the base it merges into, +so a change is blamed for the errors it adds, never for drift that already sits +in the base. That ``> base`` guard is what stops an unrelated PR from inheriting +a red once two PRs each land near the limit and their sum crosses it: the +bystander's count equals its base, so it is spared, while any PR that actually +grows the rule past its limit still fails. Head counts are read from stdin (the caller runs basedpyright once and pipes ``--outputjson`` in); the base count is a second basedpyright pass over a detached worktree at the merge-base, run under the same environment so import -resolution matches. ``--update`` re-captures the absolute per-rule baselines for -the ratchet, preserving each rule's slack. +resolution matches. ``--update`` ratchets each rule's ``limit`` down by the +number of errors this branch fixed relative to its branch point (the merge-base), +so the headroom you were granted shrinks by exactly what you cleared and never +grows. ``--outputjson`` is used rather than text diagnostics because the latter wrap across lines, leaving the ``(reportRule)`` on a continuation line away from the @@ -44,10 +46,10 @@ DEFAULT_BASE = "origin/litellm_internal_staging" # Bucket for a basedpyright diagnostic with no `rule`. Counted so it's gated. UNCODED = "" -# Ceiling for a rule that shows up at HEAD but isn't in the budget at all -- a -# brand-new error category (new construct, or a tool/version change). baseline -# is treated as 0, so the rule fails once it clears this much slack. -DEFAULT_SLACK = 10 +# Limit for a rule that shows up at HEAD but isn't in the budget at all -- a +# brand-new error category (new construct, or a tool/version change). The rule +# fails once it clears this many errors. +DEFAULT_LIMIT = 10 class Breach(NamedTuple): @@ -57,13 +59,6 @@ class Breach(NamedTuple): added: int -def _seed_slack(baseline: int) -> int: - """Slack written for a rule first captured into a budget; busy rules get - more headroom, mirroring the tiering in ruff-strict-budget.json. Existing - rules keep whatever slack their JSON already declares.""" - return 10 if baseline >= 50 else 3 - - def _to_relative(raw: str, root: Path) -> str | None: path = Path(raw) absolute = path if path.is_absolute() else root / path @@ -142,7 +137,7 @@ def evaluate( breaches = [] for code, total in head.items(): spec = budget.get(code) - cap = spec["baseline"] + spec["slack"] if spec else DEFAULT_SLACK + cap = spec["limit"] if spec else DEFAULT_LIMIT prior = base.get(code, 0) if total > cap and total > prior: breaches.append(Breach(code, total, cap, total - prior)) @@ -155,24 +150,47 @@ def is_vacuous_run( """True when nothing was parsed but the budget expects errors -- the signature of a type checker that crashed or produced no output. The CI pipe swallows the tool's exit code (`tool || true`), so without this guard an - empty run would clear every ceiling and pass silently.""" - return not counts and any(spec["baseline"] for spec in budget.values()) + empty run would clear every limit and pass silently.""" + return not counts and any(spec["limit"] for spec in budget.values()) -def cmd_update(counts: Mapping[str, int]) -> None: - existing = json.loads(BUDGET_PATH.read_text()) if BUDGET_PATH.exists() else {} - budget = { +def ratcheted_budget( + budget: Mapping[str, Mapping[str, int]], + current: Mapping[str, int], + base: Mapping[str, int], +) -> dict[str, dict[str, int]]: + """Each rule's limit lowered by the errors `current` fixed vs `base`. + + `base` is the count at the branch point (the commit this branch diverged + from). The drop is clamped to what was actually cleared (a rule that grew + stays put), so the limit only ever falls. Rules absent from the budget are + dropped: a genuinely new error category is added to the JSON deliberately, + not on update. + """ + return { code: { - "baseline": count, - "slack": ( - existing[code]["slack"] if code in existing else _seed_slack(count) - ), + "limit": max(0, spec["limit"] - max(0, base.get(code, 0) - current.get(code, 0))) } - for code, count in sorted(counts.items()) + for code, spec in sorted(budget.items()) } - BUDGET_PATH.write_text(json.dumps(budget, indent=2, sort_keys=True) + "\n") + + +def cmd_update(current: Mapping[str, int], base_ref: str = DEFAULT_BASE) -> None: + """Ratchet each rule's limit down by the errors this branch fixed. + + `current` is the working-tree count (piped in); the reference count comes + from a second basedpyright pass over a detached worktree at the branch point + (the merge-base with `base_ref`), so a branch's fixes tighten its own ceilings + by exactly what they cleared since it diverged, and limits never rise. + """ + budget = json.loads(BUDGET_PATH.read_text()) if BUDGET_PATH.exists() else {} + base_point = _run(["git", "merge-base", base_ref, "HEAD"]).strip() or base_ref + updated = ratcheted_budget(budget, current, base_counts(base_point)) + BUDGET_PATH.write_text(json.dumps(updated, indent=2, sort_keys=True) + "\n") + cleared = sum(budget[code]["limit"] - updated[code]["limit"] for code in updated) print( - f"Re-captured basedpyright per-rule budget: {len(budget)} rules, {sum(counts.values())} errors total" + f"Ratcheted basedpyright limits down by {cleared} errors this branch fixed " + f"across {len(updated)} rules" ) @@ -180,10 +198,10 @@ def cmd_check(base_ref: str) -> None: budget = json.loads(BUDGET_PATH.read_text()) head = count_basedpyright(sys.stdin.read()) if is_vacuous_run(head, budget): - expected = sum(spec["baseline"] for spec in budget.values()) + expected = sum(spec["limit"] for spec in budget.values()) print( - f"FAIL: basedpyright produced no errors, but {BUDGET_PATH.name} expects " - f"~{expected}. The type checker almost certainly crashed or emitted " + f"FAIL: basedpyright produced no errors, but {BUDGET_PATH.name} allows " + f"up to ~{expected}. The type checker almost certainly crashed or emitted " f"nothing; refusing to certify a vacuous run." ) raise SystemExit(1) @@ -199,17 +217,17 @@ def cmd_check(base_ref: str) -> None: breaches = evaluate(head, base, budget) if not breaches: print( - f"OK: every rule is within its basedpyright ceiling or no higher than base ({sum(head.values())} errors total)" + f"OK: every rule is within its basedpyright limit or no higher than base ({sum(head.values())} errors total)" ) return - print("FAIL: basedpyright errors exceed the per-rule ceiling:") + print("FAIL: basedpyright errors exceed the per-rule limit:") for breach in breaches: print( - f" {breach.code}: total {breach.total} over cap {breach.cap} (this change added {breach.added})" + f" {breach.code}: total {breach.total} over limit {breach.cap} (this change added {breach.added})" ) print( "Reduce the new errors or remove an equal number elsewhere; the ceiling is " - "baseline + slack in basedpyright-code-budget.json." + "the limit in basedpyright-code-budget.json." ) summary = "; ".join(f"{b.code} {b.total}/{b.cap} (+{b.added})" for b in breaches) print(f"BREACHED RULES: {summary}") @@ -222,7 +240,7 @@ def main() -> None: parser.add_argument("--update", action="store_true") args = parser.parse_args() if args.update: - cmd_update(count_basedpyright(sys.stdin.read())) + cmd_update(count_basedpyright(sys.stdin.read()), args.base) else: cmd_check(args.base) diff --git a/scripts/type_discipline_gate.py b/scripts/type_discipline_gate.py index c111486e56a..bd63a42dcab 100644 --- a/scripts/type_discipline_gate.py +++ b/scripts/type_discipline_gate.py @@ -2,18 +2,19 @@ """Total-count gate for the LIT* rules in scripts/check_type_discipline.py. Sibling of scripts/ruff_strict_gate.py. Each rule listed in -type-discipline-budget.json has a hard ceiling (baseline + slack). The gate counts -each rule across the whole `litellm` tree and fails when a rule is both over its -ceiling and higher than the base it merges into, so a change is blamed for the -violations it adds, never for drift that already exists in the base. +type-discipline-budget.json has a hard ``limit``. The gate counts each rule +across the whole `litellm` tree and fails when a rule is both over its limit and +higher than the base it merges into, so a change is blamed for the violations it +adds, never for drift that already exists in the base. Rules not present in the budget are ignored, but today every rule the checker emits is gated: LIT001 (mutable collection in any annotation), LIT002 (mutable-collection construction), LIT003/LIT004 (noqa / ignore without codes or -reason), LIT006 (cast), and LIT008 (`**kwargs`) carry slack-buffered ceilings to -ratchet down; LIT005 (`*-ok` suppression without a reason) is frozen at slack 0 -so any net-new reasonless suppression trips the gate; and LIT007 (TypeGuard/TypeIs) -is a hard zero. Re-baseline with `--update` to ratchet a ceiling down. +reason), LIT006 (cast), and LIT008 (`**kwargs`) carry limits above their current +count to ratchet down; LIT005 (`*-ok` suppression without a reason) is frozen at +limit 0 so any net-new reasonless suppression trips the gate; and LIT007 +(TypeGuard/TypeIs) is a hard zero. ``--update`` ratchets a limit down by the +violations this branch fixed relative to its branch point (the merge-base). """ import argparse @@ -104,21 +105,21 @@ def base_counts(ref: str) -> dict: def over_ceiling(head: dict, budget: dict) -> frozenset: - """Rules whose head count already exceeds baseline + slack. + """Rules whose head count already exceeds their limit. - A rule can only breach when it is over its ceiling, so when none are the base + A rule can only breach when it is over its limit, so when none are the base comparison cannot change the verdict and the base worktree scan can be skipped. """ return frozenset( rule for rule, spec in budget.items() - if head.get(rule, 0) > spec["baseline"] + spec["slack"] + if head.get(rule, 0) > spec["limit"] ) def evaluate(head: dict, base: dict, budget: dict) -> list: breaches = [] for rule, spec in budget.items(): - cap = spec["baseline"] + spec["slack"] + cap = spec["limit"] total = head.get(rule, 0) if total > cap and total > base.get(rule, 0): breaches.append(Breach(rule, total, cap, total - base.get(rule, 0))) @@ -160,10 +161,10 @@ def cmd_check(base: str) -> None: _run(["git", "diff", base_point, "--unified=0", "--no-color", "--", TARGET]) ), ) - print(f"FAIL: LIT-rule totals exceed their ceiling (base {base}):") + print(f"FAIL: LIT-rule totals exceed their limit (base {base}):") for breach in breaches: print( - f" {breach.rule}: total {breach.total} over cap {breach.cap} (this change added {breach.added})" + f" {breach.rule}: total {breach.total} over limit {breach.cap} (this change added {breach.added})" ) for violation in sorted(v for v in new if v.code == breach.rule): print(f" {violation.file}:{violation.line}") @@ -171,19 +172,42 @@ def cmd_check(base: str) -> None: "Remove the new violations, give each a reason (`# noqa: XXX # `, " "`# pyright: ignore[rule] # `, `# mutable-ok: `, " "`# cast-ok: `, `# guard-ok: `, `# kwargs-ok: `), or " - "remove an equal number elsewhere; the ceiling is baseline + slack in " + "remove an equal number elsewhere; the ceiling is the limit in " "type-discipline-budget.json." ) raise SystemExit(1) -def cmd_update() -> None: +def ratcheted_budget(budget: dict, current: dict, base: dict) -> dict: + """Each rule's limit lowered by the violations `current` fixed vs `base`. + + `base` is the count at the branch point (the commit this branch diverged + from). The drop is clamped to what was actually cleared (a rule that grew + stays put), so the limit only ever falls. + """ + return { + rule: { + "limit": max(0, spec["limit"] - max(0, base.get(rule, 0) - current.get(rule, 0))) + } + for rule, spec in sorted(budget.items()) + } + + +def cmd_update(base_ref: str = DEFAULT_BASE) -> None: + """Ratchet each rule's limit down by the violations this branch fixed. + + The working-tree count is compared against a checker pass over a detached + worktree at the branch point (the merge-base with `base_ref`), so a branch's + fixes tighten its own ceilings by exactly what they cleared since it diverged. + """ budget = json.loads(BUDGET_PATH.read_text()) - head = count_by_rule(head_violations()) - for rule in budget: - budget[rule]["baseline"] = head.get(rule, 0) - BUDGET_PATH.write_text(json.dumps(budget, indent=2, sort_keys=True) + "\n") - print("Re-captured per-rule baselines from the current tree") + base_point = _run(["git", "merge-base", base_ref, "HEAD"]).strip() or base_ref + updated = ratcheted_budget( + budget, count_by_rule(head_violations()), base_counts(base_point) + ) + BUDGET_PATH.write_text(json.dumps(updated, indent=2, sort_keys=True) + "\n") + cleared = sum(budget[rule]["limit"] - updated[rule]["limit"] for rule in updated) + print(f"Ratcheted LIT-rule limits down by {cleared} violations this branch fixed") def main() -> None: @@ -191,7 +215,7 @@ def main() -> None: parser.add_argument("--base", default=DEFAULT_BASE) parser.add_argument("--update", action="store_true") args = parser.parse_args() - cmd_update() if args.update else cmd_check(args.base) + cmd_update(args.base) if args.update else cmd_check(args.base) if __name__ == "__main__": diff --git a/tests/test_litellm/test_budget_ratchet_check.py b/tests/test_litellm/test_budget_ratchet_check.py index 77cee8a485c..1972c1b6386 100644 --- a/tests/test_litellm/test_budget_ratchet_check.py +++ b/tests/test_litellm/test_budget_ratchet_check.py @@ -1,9 +1,8 @@ """Tests for scripts/budget_ratchet_check.py. -The guard's contract is "baselines and ceilings may only fall": a raised ceiling, a -raised baseline (even when slack is cut to keep the ceiling flat), a dropped rule, or -a deleted file is a regression, while a lowered/equal baseline and ceiling, a brand-new -rule, or a brand-new budget file is fine. Each branch is pinned here. +The guard's contract is "limits may only fall": a raised limit, a dropped rule, or +a deleted file is a regression, while a lowered/equal limit, a brand-new rule, or a +brand-new budget file is fine. Each branch is pinned here. """ import importlib.util @@ -19,69 +18,64 @@ ratchet = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(ratchet) -def _spec_of(baseline, slack): - return {"baseline": baseline, "slack": slack} +def _spec_of(limit): + return {"limit": limit} -def test_caps_sum_baseline_and_slack_and_skip_malformed(): - caps = ratchet._caps({"LIT006": _spec_of(1013, 10), "junk": 5}) - assert caps == {"LIT006": 1023} # malformed (non-dict) spec ignored +def test_limits_read_the_limit_and_skip_malformed(): + limits = ratchet._limits({"LIT006": _spec_of(1023), "junk": 5}) + assert limits == {"LIT006": 1023} # malformed (non-dict) spec ignored -def test_raised_ceiling_is_a_regression(): - base = {"LIT006": _spec_of(1013, 10)} - head = {"LIT006": _spec_of(1013, 11)} # cap 1023 -> 1024 +def test_limits_fall_back_to_legacy_baseline_plus_slack(): + # The base side of a diff can predate the `limit` migration; its ceiling is + # baseline + slack, read on the same footing as a new-schema `limit`. + assert ratchet._limits({"LIT006": {"baseline": 1013, "slack": 10}}) == {"LIT006": 1023} + + +def test_migration_from_legacy_schema_to_equal_limit_is_clean(): + # baseline+slack (1023) -> limit 1023 is the same ceiling, so no regression. + base = {"LIT006": {"baseline": 1013, "slack": 10}} + assert ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1023)}) == [] + # ...and a genuine raise across the migration is still caught. + regs = ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1024)}) + assert [r.rule for r in regs] == ["LIT006"] and "1023 -> 1024" in regs[0].detail + + +def test_raised_limit_is_a_regression(): + base = {"LIT006": _spec_of(1023)} + head = {"LIT006": _spec_of(1024)} regs = ratchet.regressions_for("b.json", base, head) assert [r.rule for r in regs] == ["LIT006"] assert "1023 -> 1024" in regs[0].detail -def test_lowered_or_equal_ceiling_is_clean(): - base = {"LIT006": _spec_of(1013, 10)} - # baseline drops, slack flat -> ceiling falls - assert ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1000, 10)}) == [] +def test_lowered_or_equal_limit_is_clean(): + base = {"LIT006": _spec_of(1023)} + # limit drops + assert ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1000)}) == [] # nothing changes - assert ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1013, 10)}) == [] - # slack cut while baseline holds -> ceiling falls, baseline flat - assert ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1013, 0)}) == [] - - -def test_raised_baseline_is_a_regression_even_when_ceiling_held_flat(): - # baseline 1013 -> 1023 with slack cut 10 -> 0 keeps the ceiling at 1023, but a - # higher baseline bakes in more accepted debt and must still surface as a regression - base = {"LIT006": _spec_of(1013, 10)} - regs = ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1023, 0)}) - assert [r.rule for r in regs] == ["LIT006"] - assert "baseline raised 1013 -> 1023" in regs[0].detail - assert "ceiling raised" not in regs[0].detail - - -def test_raised_baseline_and_ceiling_report_both_reasons(): - base = {"LIT006": _spec_of(1013, 10)} - regs = ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1100, 10)}) - assert [r.rule for r in regs] == ["LIT006"] - assert "ceiling raised 1023 -> 1110" in regs[0].detail - assert "baseline raised 1013 -> 1100" in regs[0].detail + assert ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1023)}) == [] def test_dropped_rule_is_a_regression(): - regs = ratchet.regressions_for("b.json", {"LIT007": _spec_of(0, 0)}, {}) + regs = ratchet.regressions_for("b.json", {"LIT007": _spec_of(0)}, {}) assert [r.rule for r in regs] == ["LIT007"] assert "dropped" in regs[0].detail def test_new_rule_in_head_is_clean(): - assert ratchet.regressions_for("b.json", {}, {"new-rule": _spec_of(5, 0)}) == [] + assert ratchet.regressions_for("b.json", {}, {"new-rule": _spec_of(5)}) == [] def test_deleted_budget_file_is_a_regression(): - regs = ratchet.regressions_for("b.json", {"LIT006": _spec_of(1, 0)}, None) + regs = ratchet.regressions_for("b.json", {"LIT006": _spec_of(1)}, None) assert [r.rule for r in regs] == ["*"] assert "deleted" in regs[0].detail def test_new_budget_file_has_nothing_to_ratchet(): - assert ratchet.regressions_for("b.json", None, {"LIT006": _spec_of(1, 0)}) == [] + assert ratchet.regressions_for("b.json", None, {"LIT006": _spec_of(1)}) == [] def test_default_budgets_watch_every_budget_file_in_the_repo(): diff --git a/tests/test_litellm/test_ruff_strict_gate.py b/tests/test_litellm/test_ruff_strict_gate.py index 22255f0555e..ec8f49730dd 100644 --- a/tests/test_litellm/test_ruff_strict_gate.py +++ b/tests/test_litellm/test_ruff_strict_gate.py @@ -11,16 +11,16 @@ _spec.loader.exec_module(gate) Violation = gate.Violation -def rule(name, baseline, slack): - return {name: {"baseline": baseline, "slack": slack}} +def rule(name, limit): + return {name: {"limit": limit}} def test_under_ceiling_passes(): - assert gate.evaluate({"ANN001": 100}, {"ANN001": 100}, rule("ANN001", 90, 20)) == [] + assert gate.evaluate({"ANN001": 100}, {"ANN001": 100}, rule("ANN001", 110)) == [] -def test_ceiling_is_baseline_plus_slack_boundary(): - budget = rule("ANN001", 90, 20) # cap 110 +def test_ceiling_is_the_limit_boundary(): + budget = rule("ANN001", 110) at = gate.evaluate({"ANN001": 110}, {"ANN001": 90}, budget) over = gate.evaluate({"ANN001": 111}, {"ANN001": 90}, budget) assert at == [] @@ -30,23 +30,23 @@ def test_ceiling_is_baseline_plus_slack_boundary(): def test_over_ceiling_and_change_added_fails(): - breaches = gate.evaluate({"C901": 11}, {"C901": 9}, rule("C901", 10, 0)) + breaches = gate.evaluate({"C901": 11}, {"C901": 9}, rule("C901", 10)) assert [b.rule for b in breaches] == ["C901"] assert breaches[0].added == 2 def test_base_already_over_ceiling_change_added_nothing_is_not_blamed(): - # drift safety: base is over cap, this change leaves the count where it is - assert gate.evaluate({"C901": 15}, {"C901": 15}, rule("C901", 10, 0)) == [] + # drift safety: base is over limit, this change leaves the count where it is + assert gate.evaluate({"C901": 15}, {"C901": 15}, rule("C901", 10)) == [] def test_change_that_reduces_an_over_ceiling_rule_is_not_blamed(): - # still over cap, but moving the right direction - assert gate.evaluate({"C901": 14}, {"C901": 16}, rule("C901", 10, 0)) == [] + # still over limit, but moving the right direction + assert gate.evaluate({"C901": 14}, {"C901": 16}, rule("C901", 10)) == [] def test_rules_are_independent(): - budget = {**rule("ANN001", 100, 50), **rule("C901", 10, 0)} + budget = {**rule("ANN001", 150), **rule("C901", 10)} breaches = gate.evaluate( {"ANN001": 130, "C901": 11}, {"ANN001": 100, "C901": 10}, budget ) @@ -54,7 +54,19 @@ def test_rules_are_independent(): def test_missing_rule_counts_as_zero(): - assert gate.evaluate({}, {}, rule("C901", 0, 0)) == [] + assert gate.evaluate({}, {}, rule("C901", 0)) == [] + + +def test_update_ratchets_limit_down_by_what_the_branch_fixed_never_up(): + budget = {**rule("ANN001", 150), **rule("C901", 10)} + # ANN001 fixed 20 (100 -> 80) so its limit falls 150 -> 130; C901 grew, so its + # limit holds flat at 10 (a fix must never loosen a ceiling). + current = {"ANN001": 80, "C901": 12} + base = {"ANN001": 100, "C901": 9} + assert gate.ratcheted_budget(budget, current, base) == { + "ANN001": {"limit": 130}, + "C901": {"limit": 10}, + } def test_parse_changed_lines_maps_added_lines_per_file(): diff --git a/tests/test_litellm/test_type_check_gate.py b/tests/test_litellm/test_type_check_gate.py index e99ad0a4f41..3faf46c87de 100644 --- a/tests/test_litellm/test_type_check_gate.py +++ b/tests/test_litellm/test_type_check_gate.py @@ -55,77 +55,98 @@ def test_paths_outside_repo_are_skipped(): def test_at_or_under_ceiling_passes(): - budget = {"no-any-return": {"baseline": 5, "slack": 0}} + budget = {"no-any-return": {"limit": 5}} assert gate.evaluate({"no-any-return": 5}, {}, budget) == [] def test_one_more_error_than_ceiling_fails(): - budget = {"no-any-return": {"baseline": 5, "slack": 0}} + budget = {"no-any-return": {"limit": 5}} assert gate.evaluate({"no-any-return": 6}, {}, budget) == [ gate.Breach("no-any-return", 6, 5, 6) ] -def test_slack_absorbs_small_increase_then_fails_past_it(): - budget = {"arg-type": {"baseline": 5, "slack": 5}} +def test_limit_absorbs_increase_up_to_it_then_fails_past_it(): + budget = {"arg-type": {"limit": 10}} assert gate.evaluate({"arg-type": 10}, {}, budget) == [] assert gate.evaluate({"arg-type": 11}, {}, budget) == [ gate.Breach("arg-type", 11, 10, 11) ] -def test_unbudgeted_new_code_uses_default_slack(): - assert gate.evaluate({"brand-new": gate.DEFAULT_SLACK}, {}, {}) == [] - assert gate.evaluate({"brand-new": gate.DEFAULT_SLACK + 1}, {}, {}) == [ +def test_unbudgeted_new_code_uses_default_limit(): + assert gate.evaluate({"brand-new": gate.DEFAULT_LIMIT}, {}, {}) == [] + assert gate.evaluate({"brand-new": gate.DEFAULT_LIMIT + 1}, {}, {}) == [ gate.Breach( "brand-new", - gate.DEFAULT_SLACK + 1, - gate.DEFAULT_SLACK, - gate.DEFAULT_SLACK + 1, + gate.DEFAULT_LIMIT + 1, + gate.DEFAULT_LIMIT, + gate.DEFAULT_LIMIT + 1, ) ] def test_drift_already_over_cap_in_base_is_not_blamed_on_a_flat_change(): - # The bystander case: a rule sits over its ceiling because two earlier PRs + # The bystander case: a rule sits over its limit because two earlier PRs # summed past it. A PR that branches off that base and adds nothing must pass - # -- total > cap but total == base, so the `> base` guard spares it. - budget = {"arg-type": {"baseline": 5, "slack": 5}} + # -- total > limit but total == base, so the `> base` guard spares it. + budget = {"arg-type": {"limit": 10}} assert gate.evaluate({"arg-type": 12}, {"arg-type": 12}, budget) == [] def test_change_that_grows_an_over_cap_rule_is_blamed_for_only_what_it_added(): - # Over cap AND above base: blamed, and `added` is the delta vs base, not the + # Over limit AND above base: blamed, and `added` is the delta vs base, not the # whole overage, so the message points at this change's contribution. - budget = {"arg-type": {"baseline": 5, "slack": 5}} + budget = {"arg-type": {"limit": 10}} assert gate.evaluate({"arg-type": 14}, {"arg-type": 12}, budget) == [ gate.Breach("arg-type", 14, 10, 2) ] def test_reducing_an_over_cap_rule_below_base_passes(): - budget = {"arg-type": {"baseline": 5, "slack": 5}} + budget = {"arg-type": {"limit": 10}} assert gate.evaluate({"arg-type": 11}, {"arg-type": 12}, budget) == [] def test_no_output_against_a_nonempty_budget_is_a_vacuous_run(): # A crashed type checker emits nothing; the gate must not certify it as clean. - budget = {"no-untyped-def": {"baseline": 4888, "slack": 10}} + budget = {"no-untyped-def": {"limit": 4898}} assert gate.is_vacuous_run({}, budget) is True def test_genuine_zero_and_empty_budget_are_not_vacuous(): assert gate.is_vacuous_run({}, {}) is False + assert gate.is_vacuous_run({}, {"no-untyped-def": {"limit": 0}}) is False assert ( - gate.is_vacuous_run({}, {"no-untyped-def": {"baseline": 0, "slack": 3}}) - is False - ) - assert ( - gate.is_vacuous_run({"arg-type": 1}, {"arg-type": {"baseline": 9, "slack": 1}}) - is False + gate.is_vacuous_run({"arg-type": 1}, {"arg-type": {"limit": 10}}) is False ) +def test_update_ratchets_a_limit_down_by_what_the_branch_fixed(): + # A rule that dropped from 40 (branch point) to 30 (current) fixed 10, so its + # limit of 100 falls to 90 -- the granted headroom (60) is preserved, not the + # raw count. + budget = {"reportAny": {"limit": 100}} + assert gate.ratcheted_budget(budget, {"reportAny": 30}, {"reportAny": 40}) == { + "reportAny": {"limit": 90} + } + + +def test_update_never_raises_a_limit_when_a_rule_grows(): + # Adding violations must not loosen the ceiling; the limit holds flat. + budget = {"reportAny": {"limit": 100}} + assert gate.ratcheted_budget(budget, {"reportAny": 55}, {"reportAny": 40}) == { + "reportAny": {"limit": 100} + } + + +def test_update_clamps_a_limit_at_zero_never_negative(): + budget = {"reportAny": {"limit": 5}} + assert gate.ratcheted_budget(budget, {"reportAny": 0}, {"reportAny": 40}) == { + "reportAny": {"limit": 0} + } + + def test_malformed_basedpyright_json_exits_loudly_not_as_zero_errors(): import pytest diff --git a/tests/test_litellm/test_type_discipline_gate.py b/tests/test_litellm/test_type_discipline_gate.py index d7d827685a6..8424d480fa6 100644 --- a/tests/test_litellm/test_type_discipline_gate.py +++ b/tests/test_litellm/test_type_discipline_gate.py @@ -14,27 +14,39 @@ gate = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(gate) -def _budget(baseline, slack): - return {"LIT006": {"baseline": baseline, "slack": slack}} +def _budget(limit): + return {"LIT006": {"limit": limit}} -def test_over_ceiling_flags_only_counts_above_baseline_plus_slack(): - budget = _budget(10, 2) # cap 12 - assert gate.over_ceiling({"LIT006": 12}, budget) == frozenset() # at cap - assert gate.over_ceiling({"LIT006": 13}, budget) == frozenset({"LIT006"}) # over cap +def test_over_ceiling_flags_only_counts_above_the_limit(): + budget = _budget(12) + assert gate.over_ceiling({"LIT006": 12}, budget) == frozenset() # at limit + assert gate.over_ceiling({"LIT006": 13}, budget) == frozenset({"LIT006"}) # over limit assert gate.over_ceiling({}, budget) == frozenset() # missing rule counts as zero def test_over_ceiling_is_independent_across_rules(): - budget = {"LIT001": {"baseline": 5, "slack": 0}, "LIT006": {"baseline": 10, "slack": 0}} + budget = {"LIT001": {"limit": 5}, "LIT006": {"limit": 10}} assert gate.over_ceiling({"LIT001": 6, "LIT006": 10}, budget) == frozenset({"LIT001"}) -def test_evaluate_blames_only_a_rule_over_cap_and_over_base(): - budget = _budget(10, 0) # cap 10 - # over cap and grown vs base -> breach +def test_evaluate_blames_only_a_rule_over_limit_and_over_base(): + budget = _budget(10) + # over limit and grown vs base -> breach assert [b.rule for b in gate.evaluate({"LIT006": 12}, {"LIT006": 9}, budget)] == ["LIT006"] - # over cap but flat vs base (pre-existing drift) -> not blamed + # over limit but flat vs base (pre-existing drift) -> not blamed assert gate.evaluate({"LIT006": 12}, {"LIT006": 12}, budget) == [] - # within cap -> not blamed regardless of base + # within limit -> not blamed regardless of base assert gate.evaluate({"LIT006": 10}, {"LIT006": 0}, budget) == [] + + +def test_update_ratchets_limit_down_by_what_the_branch_fixed_never_up(): + budget = {"LIT001": {"limit": 100}, "LIT006": {"limit": 10}} + # LIT001 fixed 15 (60 -> 45) so its limit falls 100 -> 85; LIT006 grew, so its + # limit holds flat at 10. + current = {"LIT001": 45, "LIT006": 12} + base = {"LIT001": 60, "LIT006": 9} + assert gate.ratcheted_budget(budget, current, base) == { + "LIT001": {"limit": 85}, + "LIT006": {"limit": 10}, + } diff --git a/type-discipline-budget.json b/type-discipline-budget.json index a6588ac89aa..aa16b30b215 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,34 +1,26 @@ { "LIT001": { - "baseline": 21452, - "slack": 2000 + "limit": 23452 }, "LIT002": { - "baseline": 25022, - "slack": 2500 + "limit": 27522 }, "LIT003": { - "baseline": 397, - "slack": 25 + "limit": 422 }, "LIT004": { - "baseline": 2515, - "slack": 50 + "limit": 2565 }, "LIT005": { - "baseline": 0, - "slack": 0 + "limit": 0 }, "LIT006": { - "baseline": 1013, - "slack": 100 + "limit": 1113 }, "LIT007": { - "baseline": 0, - "slack": 0 + "limit": 0 }, "LIT008": { - "baseline": 914, - "slack": 90 + "limit": 1004 } } From 3e0bd71ee933957610bd6faeda4e800420f5e8ed Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 1 Jul 2026 10:25:32 -0700 Subject: [PATCH 21/81] feat(ui): disclaim that the Update API Key modal only rotates api_key (#31805) * feat(ui): disclaim that the Update API Key modal only rotates api_key An adversarial review of the credential-rotation work noted the modal always writes litellm_params.api_key, so models that authenticate with an Azure AD token, AWS credentials, or a Vertex service-account JSON are not rotated by it. Adds a warning Alert to the modal so users are not misled into thinking those secrets were rotated; broadening the modal to those providers is a follow-up * Update ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * style(ui): prettier-format the credential modal --------- Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../src/components/update_model_credentials_modal.tsx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx b/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx index 238207a4aa8..b98f0ec3242 100644 --- a/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx +++ b/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx @@ -1,4 +1,4 @@ -import { Button, Form, Input, Modal, Typography } from "antd"; +import { Alert, Button, Form, Input, Modal, Typography } from "antd"; import { useState } from "react"; import { modelPatchUpdateCall } from "./networking"; import NotificationsManager from "./molecules/notifications_manager"; @@ -56,8 +56,15 @@ export default function UpdateModelCredentialsModal({ return ( - Rotate this model's API key. Only the new key is sent; the rest of the deployment is left untouched. + Update this model's API key. Only the new key is sent; the rest of the deployment configuration is left + untouched. +
From 1fe76dcedba6595fcb3b2e30c80f7d5973dc7c2d Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 1 Jul 2026 13:25:47 -0700 Subject: [PATCH 22/81] Revert "chore: remove _experimental/out (#31546)" This reverts commit 72bcb748b97179657a4c252230f6b249757d7e66. --- .gitignore | 11 +- litellm/proxy/_experimental/out/404.html | 1 + .../proxy/_experimental/out/404/index.html | 1 + .../out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt | 9 + .../out/__next.!KGRhc2hib2FyZCk.txt | 7 + .../proxy/_experimental/out/__next._full.txt | 30 ++ .../proxy/_experimental/out/__next._head.txt | 6 + .../proxy/_experimental/out/__next._index.txt | 9 + .../proxy/_experimental/out/__next._tree.txt | 4 + .../5rDiFx0t_mOGYmV_8kSkw/_buildManifest.js | 16 + .../_clientMiddlewareManifest.js | 1 + .../5rDiFx0t_mOGYmV_8kSkw/_ssgManifest.js | 1 + .../out/_next/static/chunks/0-3i_.uof35pm.js | 2 + .../out/_next/static/chunks/0-4tg9f~_a3b~.js | 12 + .../out/_next/static/chunks/0-85n.4jrc2vv.js | 1 + .../out/_next/static/chunks/0-dhh1_d1.b1u.js | 3 + .../out/_next/static/chunks/0-f.2po-pctaa.js | 1 + .../out/_next/static/chunks/0-ih8xcz_89nt.js | 1 + .../out/_next/static/chunks/0.4.bbjx7y007.js | 143 ++++++ .../out/_next/static/chunks/0.bx44y-6~tug.js | 10 + .../out/_next/static/chunks/0.yiw37jc_bvi.js | 1 + .../out/_next/static/chunks/00cy3g~l27g1y.js | 1 + .../out/_next/static/chunks/00jwo~_zp.35~.js | 420 ++++++++++++++++++ .../out/_next/static/chunks/00p.gft-l.6p..js | 3 + .../out/_next/static/chunks/00pl5r0.xdcua.js | 1 + .../out/_next/static/chunks/00q4mtjboprhm.js | 4 + .../out/_next/static/chunks/011mgw.-67gs_.js | 10 + .../out/_next/static/chunks/01_xjyxcb1uco.js | 1 + .../out/_next/static/chunks/01xm1xt.gmrff.js | 3 + .../out/_next/static/chunks/01y._o853f7le.js | 4 + .../out/_next/static/chunks/01~uswbzv7_90.js | 1 + .../out/_next/static/chunks/022.sz94ycw4x.js | 4 + .../out/_next/static/chunks/02813b2b-kz98.js | 8 + .../out/_next/static/chunks/02c1-r_khzb89.js | 1 + .../out/_next/static/chunks/02ihc5xweq16v.js | 1 + .../out/_next/static/chunks/02nrwvikmd-wf.js | 1 + .../out/_next/static/chunks/02oicwo.~e~ak.js | 1 + .../out/_next/static/chunks/036wlkuzplhfz.js | 1 + .../out/_next/static/chunks/038lmn5.g6myc.js | 8 + .../out/_next/static/chunks/03_wvlr03g~35.js | 1 + .../out/_next/static/chunks/03fia.h6j.gpu.js | 14 + .../out/_next/static/chunks/03iznh0~x-p5x.js | 1 + .../out/_next/static/chunks/03l9yp-0vdrvg.js | 1 + .../out/_next/static/chunks/03rcuw-pknh--.js | 1 + .../out/_next/static/chunks/03~yq9q893hmn.js | 1 + .../out/_next/static/chunks/043q3g5-5-aju.js | 55 +++ .../out/_next/static/chunks/04476udqypzuu.js | 1 + .../out/_next/static/chunks/04amwk-x_vjxu.js | 1 + .../out/_next/static/chunks/04jvxoid~vpxj.js | 1 + .../out/_next/static/chunks/04p5iour3skhn.js | 1 + .../out/_next/static/chunks/04~mux1g2xqfl.js | 10 + .../out/_next/static/chunks/05.uhnqp00zd5.js | 86 ++++ .../out/_next/static/chunks/058o-fyv9lb_l.js | 10 + .../out/_next/static/chunks/05btv.l5gro_..js | 10 + .../out/_next/static/chunks/05qmwjqau64bz.css | 1 + .../out/_next/static/chunks/05t1k89l9tc3s.js | 1 + .../out/_next/static/chunks/05w6e8.ake4_v.js | 11 + .../out/_next/static/chunks/05wzckn7dnk9_.js | 1 + .../out/_next/static/chunks/05z02g9s~8km0.js | 4 + .../out/_next/static/chunks/066hp9.940823.js | 1 + .../out/_next/static/chunks/0689o862~x~pg.js | 1 + .../out/_next/static/chunks/06x5y8ia4k1mc.js | 2 + .../out/_next/static/chunks/07.fwfv-sinb5.js | 4 + .../out/_next/static/chunks/07_~yky8gc9_m.js | 10 + .../out/_next/static/chunks/08b3bdf-s.-y4.js | 2 + .../out/_next/static/chunks/08is8lfgypp_2.js | 31 ++ .../out/_next/static/chunks/09dh.hm0vr~61.js | 3 + .../out/_next/static/chunks/09n64dqzn.le~.js | 13 + .../out/_next/static/chunks/0_cwbuh_om4s9.js | 91 ++++ .../out/_next/static/chunks/0_rk9sxkapt-r.js | 1 + .../out/_next/static/chunks/0_tak0mb5m-3k.js | 1 + .../out/_next/static/chunks/0_y-b9_d9dsuv.js | 14 + .../out/_next/static/chunks/0aj3r46j-.qsy.js | 1 + .../out/_next/static/chunks/0ajdq5~-z4-0o.js | 1 + .../out/_next/static/chunks/0au3mg4n33g_o.js | 12 + .../out/_next/static/chunks/0b5g~_decuer~.js | 1 + .../out/_next/static/chunks/0bqafy~83g2md.js | 8 + .../out/_next/static/chunks/0byy7z~x~srwc.js | 1 + .../out/_next/static/chunks/0c2apcdkbqq0o.js | 1 + .../out/_next/static/chunks/0c4pfjjue0uc-.js | 86 ++++ .../out/_next/static/chunks/0ceh~7zrbxj.y.js | 1 + .../out/_next/static/chunks/0d2qt-f_paso0.js | 2 + .../out/_next/static/chunks/0ecsfnbwne0sn.js | 1 + .../out/_next/static/chunks/0el08tticy_20.js | 3 + .../out/_next/static/chunks/0em0654rb513m.js | 4 + .../out/_next/static/chunks/0gj2~qks1xrx8.js | 1 + .../out/_next/static/chunks/0gtegjaljim2a.js | 1 + .../out/_next/static/chunks/0h274dbe8lloe.js | 1 + .../out/_next/static/chunks/0hsqxu.xbf.l5.js | 216 +++++++++ .../out/_next/static/chunks/0hzdsr8t0ksq..js | 2 + .../out/_next/static/chunks/0hzj3mfqun9q~.js | 8 + .../out/_next/static/chunks/0i77.0u.82o9u.css | 1 + .../out/_next/static/chunks/0ip1d_6ew-zr2.js | 179 ++++++++ .../out/_next/static/chunks/0ivj_wax-joap.js | 31 ++ .../out/_next/static/chunks/0j2~0jseuoube.js | 16 + .../out/_next/static/chunks/0jaa-io9cz430.js | 10 + .../out/_next/static/chunks/0jdm7x5soayfw.js | 1 + .../out/_next/static/chunks/0jib1e4hgitwz.css | 1 + .../out/_next/static/chunks/0jr8wo_7ak~7n.js | 1 + .../out/_next/static/chunks/0jzxuesytdzt0.js | 1 + .../out/_next/static/chunks/0k3aqiu733i3f.js | 1 + .../out/_next/static/chunks/0kqhn69~lkflo.js | 11 + .../out/_next/static/chunks/0kr3_6r.1wa_9.js | 1 + .../out/_next/static/chunks/0l7em-5kjv49e.js | 7 + .../out/_next/static/chunks/0lb0p7rh5znu_.js | 20 + .../out/_next/static/chunks/0ldurpg4iqx04.js | 1 + .../out/_next/static/chunks/0lg.6rbfsd-l9.js | 1 + .../out/_next/static/chunks/0lku60vnd9m1i.js | 1 + .../out/_next/static/chunks/0lstohw6r.qs..js | 1 + .../out/_next/static/chunks/0m._ijxus~ryi.js | 4 + .../out/_next/static/chunks/0m.pilqkjqyg3.js | 1 + .../out/_next/static/chunks/0m5k-5fv1ya8x.js | 3 + .../out/_next/static/chunks/0m6zdocif1gl4.js | 1 + .../out/_next/static/chunks/0mb3erwqomzal.js | 1 + .../out/_next/static/chunks/0md97r_057_33.js | 1 + .../out/_next/static/chunks/0mh1wnrvmv_y7.js | 4 + .../out/_next/static/chunks/0mmrbksvmhp.1.js | 1 + .../out/_next/static/chunks/0mspdfvjqoti_.js | 1 + .../out/_next/static/chunks/0mzw3maijoev6.js | 1 + .../out/_next/static/chunks/0n.a~e5dwfnkn.js | 1 + .../out/_next/static/chunks/0n028f.v-dhms.js | 1 + .../out/_next/static/chunks/0ngre0.s4-ej6.js | 1 + .../out/_next/static/chunks/0nnx~7-7e5t~1.js | 5 + .../out/_next/static/chunks/0ogm.~yq5rjmw.js | 179 ++++++++ .../out/_next/static/chunks/0ovmgshl9hfea.js | 10 + .../out/_next/static/chunks/0p.6bs58-_3lw.js | 2 + .../out/_next/static/chunks/0pd5zl~lciww9.js | 1 + .../out/_next/static/chunks/0pidya1qvuvx8.js | 1 + .../out/_next/static/chunks/0pu3ltw1cci2~.js | 35 ++ .../out/_next/static/chunks/0pwkd9r.mc_ee.js | 1 + .../out/_next/static/chunks/0pwrfxkkt~qfh.js | 50 +++ .../out/_next/static/chunks/0q2og72gex34u.js | 1 + .../out/_next/static/chunks/0q6~n4y84cejn.js | 1 + .../out/_next/static/chunks/0q9_qqi.nzx5l.js | 1 + .../out/_next/static/chunks/0ql_-8xthluga.js | 1 + .../out/_next/static/chunks/0r8_z31ow7vw9.js | 68 +++ .../out/_next/static/chunks/0rdv7_7_95b-1.js | 1 + .../out/_next/static/chunks/0rsh-mjgd1-1b.js | 11 + .../out/_next/static/chunks/0scfmfivwcppe.js | 10 + .../out/_next/static/chunks/0snrx6.._0zus.js | 8 + .../out/_next/static/chunks/0sx3mu2_l9g_y.js | 21 + .../out/_next/static/chunks/0sxgv7gc5lm3g.js | 1 + .../out/_next/static/chunks/0sylbcw3ha_ba.js | 11 + .../out/_next/static/chunks/0t4ig3ibz46ga.js | 1 + .../out/_next/static/chunks/0tbzoqict3-mi.js | 1 + .../out/_next/static/chunks/0teffxf7o_863.js | 1 + .../out/_next/static/chunks/0tgl~~_4hb1rp.js | 1 + .../out/_next/static/chunks/0u3_nka63vh6t.js | 1 + .../out/_next/static/chunks/0us_9w7qaihte.js | 1 + .../out/_next/static/chunks/0uu6lckpr0s15.js | 14 + .../out/_next/static/chunks/0uy6wzxw5oh5v.js | 1 + .../out/_next/static/chunks/0v1rxqc1hqmrl.js | 4 + .../out/_next/static/chunks/0vo11_94ear6l.js | 1 + .../out/_next/static/chunks/0w39dn9x3dp9g.js | 1 + .../out/_next/static/chunks/0whkizop7gd0~.js | 41 ++ .../out/_next/static/chunks/0x.73w57rn4ou.js | 1 + .../out/_next/static/chunks/0x6hmpiq7.b-x.js | 420 ++++++++++++++++++ .../out/_next/static/chunks/0ydd65iv6ffpl.js | 10 + .../out/_next/static/chunks/0ys10755n8os_.js | 1 + .../out/_next/static/chunks/0z4fh7pvzmoy8.js | 1 + .../out/_next/static/chunks/0zqdpz_rk5.wq.js | 14 + .../out/_next/static/chunks/0zrbitbm~0koh.js | 14 + .../out/_next/static/chunks/0~-ovi6c4wjt1.js | 1 + .../out/_next/static/chunks/0~0su3wi_7f6-.js | 1 + .../out/_next/static/chunks/0~tp1mbr_st8h.js | 1 + .../out/_next/static/chunks/0~~y94vmu8z5d.js | 1 + .../out/_next/static/chunks/101az3fsw7lje.js | 1 + .../out/_next/static/chunks/10e9lx.nawttb.js | 1 + .../out/_next/static/chunks/10jlu0mdcmzoi.js | 1 + .../out/_next/static/chunks/10sdqywhhhn7i.js | 1 + .../out/_next/static/chunks/10ybnll3qh-8s.js | 10 + .../out/_next/static/chunks/114pbx0696lkh.js | 1 + .../out/_next/static/chunks/11h.ntqd0jl3z.js | 1 + .../out/_next/static/chunks/11kowzys1c43t.js | 420 ++++++++++++++++++ .../out/_next/static/chunks/129bujhdmi9ce.js | 4 + .../out/_next/static/chunks/13c74.fwk0wmq.js | 1 + .../out/_next/static/chunks/13ln.k6r3lkv_.js | 167 +++++++ .../out/_next/static/chunks/13s0v9siktndj.js | 1 + .../out/_next/static/chunks/142-5lmjc6wc~.js | 1 + .../out/_next/static/chunks/14566-_ogh-19.js | 1 + .../out/_next/static/chunks/14_9gq.6yjjih.js | 2 + .../out/_next/static/chunks/15.9ylrtxojbj.js | 4 + .../out/_next/static/chunks/1560njdijg7fq.js | 48 ++ .../out/_next/static/chunks/15auqattd2wzv.js | 1 + .../out/_next/static/chunks/15hm8gokjq2uu.js | 13 + .../out/_next/static/chunks/15rg~y4h.lcrl.js | 1 + .../out/_next/static/chunks/15wqqcwhnlidr.js | 1 + .../out/_next/static/chunks/16.oisvgwzo8s.js | 56 +++ .../out/_next/static/chunks/169km.d7x9qr6.js | 1 + .../out/_next/static/chunks/16qfko21~_dn~.js | 10 + .../out/_next/static/chunks/1781p3yhsw7kp.js | 1 + .../out/_next/static/chunks/17b18lwgc39xm.js | 420 ++++++++++++++++++ .../out/_next/static/chunks/17cvpyw6fshd4.js | 1 + .../out/_next/static/chunks/17e1s6gkzjh5f.js | 1 + .../out/_next/static/chunks/17j1m89pizunk.js | 1 + .../out/_next/static/chunks/17jd5l9o~hzf3.js | 1 + .../out/_next/static/chunks/17n.qg70cy9.9.js | 1 + .../out/_next/static/chunks/184o99uxk88c7.js | 1 + .../static/chunks/turbopack-0a~tzicx4wgrt.js | 1 + .../1bffadaabf893a1e-s.16ipb6fqu393i.woff2 | Bin 0 -> 85272 bytes .../2bbe8d2671613f1f-s.067x_6k0k23tk.woff2 | Bin 0 -> 10280 bytes .../2c55a0e60120577a-s.0bjc5tiuqdqro.woff2 | Bin 0 -> 25844 bytes .../5476f68d60460930-s.0wxq9webf.ew4.woff2 | Bin 0 -> 19044 bytes .../83afe278b6a6bb3c-s.p.0q-301v4kxxnr.woff2 | Bin 0 -> 48432 bytes .../9c72aa0f40e4eef8-s.0m6w47a4e5dy9.woff2 | Bin 0 -> 18744 bytes .../ad66f9afd8947f86-s.11u06r12fd6v_.woff2 | Bin 0 -> 11272 bytes .../static/media/favicon.0~dgapwhi~75y.ico | Bin 0 -> 6387 bytes .../out/_not-found/__next._full.txt | 20 + .../out/_not-found/__next._head.txt | 6 + .../out/_not-found/__next._index.txt | 9 + .../_not-found/__next._not-found.__PAGE__.txt | 5 + .../out/_not-found/__next._not-found.txt | 5 + .../out/_not-found/__next._tree.txt | 3 + .../_experimental/out/_not-found/index.html | 1 + .../_experimental/out/_not-found/index.txt | 20 + ...KGRhc2hib2FyZCk.access-groups.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.access-groups.txt | 5 + .../access-groups/__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/access-groups/__next._full.txt | 33 ++ .../out/access-groups/__next._head.txt | 6 + .../out/access-groups/__next._index.txt | 9 + .../out/access-groups/__next._tree.txt | 4 + .../out/access-groups/index.html | 1 + .../_experimental/out/access-groups/index.txt | 33 ++ ....!KGRhc2hib2FyZCk.admin-panel.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.admin-panel.txt | 5 + .../admin-panel/__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/admin-panel/__next._full.txt | 33 ++ .../out/admin-panel/__next._head.txt | 6 + .../out/admin-panel/__next._index.txt | 9 + .../out/admin-panel/__next._tree.txt | 4 + .../_experimental/out/admin-panel/index.html | 1 + .../_experimental/out/admin-panel/index.txt | 33 ++ ..._next.!KGRhc2hib2FyZCk.agents.__PAGE__.txt | 9 + .../agents/__next.!KGRhc2hib2FyZCk.agents.txt | 5 + .../out/agents/__next.!KGRhc2hib2FyZCk.txt | 7 + .../_experimental/out/agents/__next._full.txt | 33 ++ .../_experimental/out/agents/__next._head.txt | 6 + .../out/agents/__next._index.txt | 9 + .../_experimental/out/agents/__next._tree.txt | 4 + .../proxy/_experimental/out/agents/index.html | 1 + .../proxy/_experimental/out/agents/index.txt | 33 ++ ...ext.!KGRhc2hib2FyZCk.api-keys.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.api-keys.txt | 5 + .../out/api-keys/__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/api-keys/__next._full.txt | 33 ++ .../out/api-keys/__next._head.txt | 6 + .../out/api-keys/__next._index.txt | 9 + .../out/api-keys/__next._tree.txt | 4 + .../_experimental/out/api-keys/index.html | 1 + .../_experimental/out/api-keys/index.txt | 33 ++ ...KGRhc2hib2FyZCk.api-reference.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.api-reference.txt | 5 + .../api-reference/__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/api-reference/__next._full.txt | 33 ++ .../out/api-reference/__next._head.txt | 6 + .../out/api-reference/__next._index.txt | 9 + .../out/api-reference/__next._tree.txt | 4 + .../out/api-reference/index.html | 1 + .../_experimental/out/api-reference/index.txt | 33 ++ .../out/assets/audit-logs-preview.png | Bin 0 -> 240654 bytes .../out/assets/logos/a2a_agent.png | Bin 0 -> 72568 bytes .../_experimental/out/assets/logos/ai21.svg | 1 + .../out/assets/logos/aim_logo.jpeg | Bin 0 -> 3754 bytes .../out/assets/logos/aim_security.jpeg | Bin 0 -> 3754 bytes .../out/assets/logos/aiml_api.svg | 1 + .../_experimental/out/assets/logos/akto.svg | 10 + .../out/assets/logos/anthropic.svg | 5 + .../_experimental/out/assets/logos/aporia.png | Bin 0 -> 2472 bytes .../_experimental/out/assets/logos/arize.png | Bin 0 -> 14249 bytes .../out/assets/logos/assemblyai_small.png | Bin 0 -> 414 bytes .../_experimental/out/assets/logos/aws.svg | 34 ++ .../out/assets/logos/azure_ai_foundry.png | Bin 0 -> 26316 bytes .../out/assets/logos/baseten.svg | 1 + .../out/assets/logos/bedrock.svg | 1 + .../out/assets/logos/braintrust.png | Bin 0 -> 10428 bytes .../out/assets/logos/cato_networks.svg | 4 + .../out/assets/logos/cerebras.svg | 89 ++++ .../_experimental/out/assets/logos/cisco.png | Bin 0 -> 1964 bytes .../out/assets/logos/cloudflare.svg | 1 + .../_experimental/out/assets/logos/cohere.svg | 1 + .../out/assets/logos/cometapi.svg | 1 + .../_experimental/out/assets/logos/cursor.svg | 1 + .../out/assets/logos/databricks.svg | 1 + .../out/assets/logos/datadog.png | Bin 0 -> 5213 bytes .../out/assets/logos/dataforseo.png | Bin 0 -> 139307 bytes .../out/assets/logos/deepgram.png | Bin 0 -> 1224 bytes .../out/assets/logos/deepinfra.png | Bin 0 -> 7014 bytes .../out/assets/logos/deepseek.svg | 25 ++ .../out/assets/logos/elevenlabs.png | Bin 0 -> 35410 bytes .../out/assets/logos/enkrypt_ai.avif | Bin 0 -> 2908 bytes .../_experimental/out/assets/logos/exa_ai.png | Bin 0 -> 40751 bytes .../_experimental/out/assets/logos/fal_ai.jpg | Bin 0 -> 8254 bytes .../out/assets/logos/featherless.svg | 1 + .../_experimental/out/assets/logos/figma.svg | 7 + .../out/assets/logos/fireworks.svg | 1 + .../out/assets/logos/friendli.svg | 1 + .../out/assets/logos/galileo.ico | Bin 0 -> 9714 bytes .../_experimental/out/assets/logos/github.svg | 1 + .../out/assets/logos/github_copilot.svg | 1 + .../_experimental/out/assets/logos/gitlab.svg | 8 + .../_experimental/out/assets/logos/gmail.svg | 3 + .../_experimental/out/assets/logos/google.svg | 2 + .../out/assets/logos/google_drive.svg | 6 + .../out/assets/logos/google_pse.png | Bin 0 -> 2392 bytes .../_experimental/out/assets/logos/groq.svg | 3 + .../out/assets/logos/guardrails_ai.jpeg | Bin 0 -> 9041 bytes .../out/assets/logos/hubspot.svg | 3 + .../out/assets/logos/huggingface.svg | 1 + .../out/assets/logos/hyperbolic.svg | 1 + .../out/assets/logos/infinity.png | Bin 0 -> 7377 bytes .../out/assets/logos/javelin.png | Bin 0 -> 1956 bytes .../_experimental/out/assets/logos/jina.png | Bin 0 -> 2758 bytes .../_experimental/out/assets/logos/jira.svg | 15 + .../_experimental/out/assets/logos/lago.svg | 11 + .../out/assets/logos/lakeraai.jpeg | Bin 0 -> 2617 bytes .../_experimental/out/assets/logos/lambda.svg | 1 + .../out/assets/logos/langflow.svg | 5 + .../out/assets/logos/langfuse.png | Bin 0 -> 10860 bytes .../out/assets/logos/langfuse.svg | 1 + .../out/assets/logos/langgraph.png | Bin 0 -> 5495 bytes .../out/assets/logos/langsmith.png | Bin 0 -> 5495 bytes .../_experimental/out/assets/logos/lasso.png | Bin 0 -> 4115 bytes .../_experimental/out/assets/logos/linear.svg | 3 + .../out/assets/logos/litellm.jpg | Bin 0 -> 24694 bytes .../out/assets/logos/litellm_logo.jpg | Bin 0 -> 9222 bytes .../out/assets/logos/llm_guard.png | Bin 0 -> 48665 bytes .../out/assets/logos/lmstudio.svg | 1 + .../out/assets/logos/mcp_logo.png | Bin 0 -> 3902 bytes .../out/assets/logos/meta_llama.svg | 1 + .../out/assets/logos/microsoft_azure.svg | 72 +++ .../_experimental/out/assets/logos/milvus.svg | 1 + .../out/assets/logos/minimax.svg | 1 + .../out/assets/logos/mistral.svg | 1 + .../out/assets/logos/moonshot.svg | 1 + .../_experimental/out/assets/logos/morph.svg | 1 + .../_experimental/out/assets/logos/nebius.svg | 1 + .../out/assets/logos/newrelic.png | Bin 0 -> 862 bytes .../out/assets/logos/noma_security.png | Bin 0 -> 3163 bytes .../_experimental/out/assets/logos/notion.svg | 3 + .../_experimental/out/assets/logos/novita.svg | 1 + .../out/assets/logos/nvidia_nim.svg | 1 + .../out/assets/logos/nvidia_triton.png | Bin 0 -> 5704 bytes .../_experimental/out/assets/logos/ollama.svg | 7 + .../out/assets/logos/openai_small.svg | 5 + .../out/assets/logos/openmeter.png | Bin 0 -> 1114 bytes .../out/assets/logos/openrouter.svg | 39 ++ .../_experimental/out/assets/logos/oracle.svg | 1 + .../_experimental/out/assets/logos/otel.png | Bin 0 -> 1949 bytes .../out/assets/logos/palo_alto_networks.jpeg | Bin 0 -> 5642 bytes .../_experimental/out/assets/logos/pangea.png | Bin 0 -> 31102 bytes .../out/assets/logos/parallel_ai.png | Bin 0 -> 2191 bytes .../out/assets/logos/perplexity-ai.svg | 16 + .../out/assets/logos/perplexity.png | Bin 0 -> 9615 bytes .../out/assets/logos/pillar.jpeg | Bin 0 -> 2554 bytes .../out/assets/logos/postgresql.svg | 1 + .../out/assets/logos/presidio.png | Bin 0 -> 62523 bytes .../out/assets/logos/prompt_security.png | Bin 0 -> 5695 bytes .../out/assets/logos/promptguard.svg | 95 ++++ .../out/assets/logos/pydantic.svg | 5 + .../_experimental/out/assets/logos/qohash.jpg | Bin 0 -> 11581 bytes .../_experimental/out/assets/logos/qwen.png | Bin 0 -> 49453 bytes .../out/assets/logos/recraft.svg | 1 + .../out/assets/logos/repelloai.png | Bin 0 -> 14323 bytes .../out/assets/logos/replicate.svg | 1 + .../_experimental/out/assets/logos/runway.png | Bin 0 -> 5165 bytes .../out/assets/logos/s3_vector.png | Bin 0 -> 191076 bytes .../out/assets/logos/salesforce.svg | 3 + .../out/assets/logos/sambanova.svg | 42 ++ .../_experimental/out/assets/logos/sap.png | Bin 0 -> 200176 bytes .../out/assets/logos/search1api.png | Bin 0 -> 1549 bytes .../out/assets/logos/secret_detect.png | Bin 0 -> 15590 bytes .../_experimental/out/assets/logos/sentry.svg | 3 + .../out/assets/logos/shopify.svg | 4 + .../_experimental/out/assets/logos/slack.svg | 6 + .../out/assets/logos/snowflake.svg | 9 + .../_experimental/out/assets/logos/soniox.svg | 1 + .../_experimental/out/assets/logos/stripe.svg | 3 + .../_experimental/out/assets/logos/tavily.png | Bin 0 -> 30986 bytes .../out/assets/logos/togetherai.svg | 14 + .../_experimental/out/assets/logos/topaz.svg | 1 + .../_experimental/out/assets/logos/twilio.svg | 3 + .../_experimental/out/assets/logos/v0.svg | 1 + .../_experimental/out/assets/logos/vercel.svg | 1 + .../_experimental/out/assets/logos/vllm.png | Bin 0 -> 1167 bytes .../out/assets/logos/volcengine.png | Bin 0 -> 36944 bytes .../out/assets/logos/voyage.webp | Bin 0 -> 2896 bytes .../out/assets/logos/watsonx.svg | 1 + .../_experimental/out/assets/logos/xai.svg | 28 ++ .../out/assets/logos/xecguard.svg | 4 + .../out/assets/logos/xinference.svg | 1 + .../_experimental/out/assets/logos/zapier.svg | 3 + .../out/assets/logos/zscaler.svg | 5 + ...next.!KGRhc2hib2FyZCk.budgets.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.budgets.txt | 5 + .../out/budgets/__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/budgets/__next._full.txt | 33 ++ .../out/budgets/__next._head.txt | 6 + .../out/budgets/__next._index.txt | 9 + .../out/budgets/__next._tree.txt | 4 + .../_experimental/out/budgets/index.html | 1 + .../proxy/_experimental/out/budgets/index.txt | 33 ++ ...next.!KGRhc2hib2FyZCk.caching.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.caching.txt | 5 + .../out/caching/__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/caching/__next._full.txt | 33 ++ .../out/caching/__next._head.txt | 6 + .../out/caching/__next._index.txt | 9 + .../out/caching/__next._tree.txt | 4 + .../_experimental/out/caching/index.html | 1 + .../proxy/_experimental/out/caching/index.txt | 33 ++ ...KGRhc2hib2FyZCk.cost-tracking.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.cost-tracking.txt | 5 + .../cost-tracking/__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/cost-tracking/__next._full.txt | 33 ++ .../out/cost-tracking/__next._head.txt | 6 + .../out/cost-tracking/__next._index.txt | 9 + .../out/cost-tracking/__next._tree.txt | 4 + .../out/cost-tracking/index.html | 1 + .../_experimental/out/cost-tracking/index.txt | 33 ++ litellm/proxy/_experimental/out/favicon.ico | Bin 0 -> 6387 bytes ...2hib2FyZCk.guardrails-monitor.__PAGE__.txt | 10 + ...xt.!KGRhc2hib2FyZCk.guardrails-monitor.txt | 5 + .../__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/guardrails-monitor/__next._full.txt | 34 ++ .../out/guardrails-monitor/__next._head.txt | 6 + .../out/guardrails-monitor/__next._index.txt | 9 + .../out/guardrails-monitor/__next._tree.txt | 5 + .../out/guardrails-monitor/index.html | 1 + .../out/guardrails-monitor/index.txt | 34 ++ ...t.!KGRhc2hib2FyZCk.guardrails.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.guardrails.txt | 5 + .../guardrails/__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/guardrails/__next._full.txt | 33 ++ .../out/guardrails/__next._head.txt | 6 + .../out/guardrails/__next._index.txt | 9 + .../out/guardrails/__next._tree.txt | 4 + .../_experimental/out/guardrails/index.html | 1 + .../_experimental/out/guardrails/index.txt | 33 ++ litellm/proxy/_experimental/out/index.html | 1 + litellm/proxy/_experimental/out/index.txt | 30 ++ ...2hib2FyZCk.logging-and-alerts.__PAGE__.txt | 9 + ...xt.!KGRhc2hib2FyZCk.logging-and-alerts.txt | 5 + .../__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/logging-and-alerts/__next._full.txt | 33 ++ .../out/logging-and-alerts/__next._head.txt | 6 + .../out/logging-and-alerts/__next._index.txt | 9 + .../out/logging-and-alerts/__next._tree.txt | 4 + .../out/logging-and-alerts/index.html | 1 + .../out/logging-and-alerts/index.txt | 33 ++ .../_experimental/out/login/__next._full.txt | 25 ++ .../_experimental/out/login/__next._head.txt | 6 + .../_experimental/out/login/__next._index.txt | 9 + .../_experimental/out/login/__next._tree.txt | 4 + .../out/login/__next.login.__PAGE__.txt | 9 + .../_experimental/out/login/__next.login.txt | 5 + .../proxy/_experimental/out/login/index.html | 1 + .../proxy/_experimental/out/login/index.txt | 25 ++ .../__next.!KGRhc2hib2FyZCk.logs.__PAGE__.txt | 10 + .../out/logs/__next.!KGRhc2hib2FyZCk.logs.txt | 5 + .../out/logs/__next.!KGRhc2hib2FyZCk.txt | 7 + .../_experimental/out/logs/__next._full.txt | 34 ++ .../_experimental/out/logs/__next._head.txt | 6 + .../_experimental/out/logs/__next._index.txt | 9 + .../_experimental/out/logs/__next._tree.txt | 5 + .../proxy/_experimental/out/logs/index.html | 1 + .../proxy/_experimental/out/logs/index.txt | 34 ++ ....!KGRhc2hib2FyZCk.mcp-servers.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.mcp-servers.txt | 5 + .../mcp-servers/__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/mcp-servers/__next._full.txt | 33 ++ .../out/mcp-servers/__next._head.txt | 6 + .../out/mcp-servers/__next._index.txt | 9 + .../out/mcp-servers/__next._tree.txt | 4 + .../_experimental/out/mcp-servers/index.html | 1 + .../_experimental/out/mcp-servers/index.txt | 33 ++ .../out/mcp/oauth/callback/__next._full.txt | 25 ++ .../out/mcp/oauth/callback/__next._head.txt | 6 + .../out/mcp/oauth/callback/__next._index.txt | 9 + .../out/mcp/oauth/callback/__next._tree.txt | 4 + .../__next.mcp.oauth.callback.__PAGE__.txt | 9 + .../callback/__next.mcp.oauth.callback.txt | 5 + .../mcp/oauth/callback/__next.mcp.oauth.txt | 5 + .../out/mcp/oauth/callback/__next.mcp.txt | 5 + .../out/mcp/oauth/callback/index.html | 1 + .../out/mcp/oauth/callback/index.txt | 25 ++ ..._next.!KGRhc2hib2FyZCk.memory.__PAGE__.txt | 9 + .../memory/__next.!KGRhc2hib2FyZCk.memory.txt | 5 + .../out/memory/__next.!KGRhc2hib2FyZCk.txt | 7 + .../_experimental/out/memory/__next._full.txt | 33 ++ .../_experimental/out/memory/__next._head.txt | 6 + .../out/memory/__next._index.txt | 9 + .../_experimental/out/memory/__next._tree.txt | 4 + .../proxy/_experimental/out/memory/index.html | 1 + .../proxy/_experimental/out/memory/index.txt | 33 ++ ...Rhc2hib2FyZCk.model-hub-table.__PAGE__.txt | 9 + ..._next.!KGRhc2hib2FyZCk.model-hub-table.txt | 5 + .../__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/model-hub-table/__next._full.txt | 33 ++ .../out/model-hub-table/__next._head.txt | 6 + .../out/model-hub-table/__next._index.txt | 9 + .../out/model-hub-table/__next._tree.txt | 4 + .../out/model-hub-table/index.html | 1 + .../out/model-hub-table/index.txt | 33 ++ .../out/model_hub/__next._full.txt | 28 ++ .../out/model_hub/__next._head.txt | 6 + .../out/model_hub/__next._index.txt | 9 + .../out/model_hub/__next._tree.txt | 4 + .../model_hub/__next.model_hub.__PAGE__.txt | 9 + .../out/model_hub/__next.model_hub.txt | 5 + .../_experimental/out/model_hub/index.html | 1 + .../_experimental/out/model_hub/index.txt | 28 ++ .../out/model_hub_table/__next._full.txt | 32 ++ .../out/model_hub_table/__next._head.txt | 6 + .../out/model_hub_table/__next._index.txt | 9 + .../out/model_hub_table/__next._tree.txt | 4 + .../__next.model_hub_table.__PAGE__.txt | 9 + .../__next.model_hub_table.txt | 5 + .../out/model_hub_table/index.html | 1 + .../out/model_hub_table/index.txt | 32 ++ ...ib2FyZCk.models-and-endpoints.__PAGE__.txt | 9 + ....!KGRhc2hib2FyZCk.models-and-endpoints.txt | 5 + .../__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/models-and-endpoints/__next._full.txt | 33 ++ .../out/models-and-endpoints/__next._head.txt | 6 + .../models-and-endpoints/__next._index.txt | 9 + .../out/models-and-endpoints/__next._tree.txt | 4 + .../out/models-and-endpoints/index.html | 1 + .../out/models-and-endpoints/index.txt | 33 ++ litellm/proxy/_experimental/out/next.svg | 1 + ...xt.!KGRhc2hib2FyZCk.old-usage.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.old-usage.txt | 5 + .../out/old-usage/__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/old-usage/__next._full.txt | 33 ++ .../out/old-usage/__next._head.txt | 6 + .../out/old-usage/__next._index.txt | 9 + .../out/old-usage/__next._tree.txt | 4 + .../_experimental/out/old-usage/index.html | 1 + .../_experimental/out/old-usage/index.txt | 33 ++ .../out/onboarding/__next._full.txt | 25 ++ .../out/onboarding/__next._head.txt | 6 + .../out/onboarding/__next._index.txt | 9 + .../out/onboarding/__next._tree.txt | 4 + .../onboarding/__next.onboarding.__PAGE__.txt | 9 + .../out/onboarding/__next.onboarding.txt | 5 + .../_experimental/out/onboarding/index.html | 1 + .../_experimental/out/onboarding/index.txt | 25 ++ ...KGRhc2hib2FyZCk.organizations.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.organizations.txt | 5 + .../organizations/__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/organizations/__next._full.txt | 33 ++ .../out/organizations/__next._head.txt | 6 + .../out/organizations/__next._index.txt | 9 + .../out/organizations/__next._tree.txt | 4 + .../out/organizations/index.html | 1 + .../_experimental/out/organizations/index.txt | 33 ++ ...t.!KGRhc2hib2FyZCk.playground.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.playground.txt | 5 + .../playground/__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/playground/__next._full.txt | 33 ++ .../out/playground/__next._head.txt | 6 + .../out/playground/__next._index.txt | 9 + .../out/playground/__next._tree.txt | 4 + .../_experimental/out/playground/index.html | 1 + .../_experimental/out/playground/index.txt | 33 ++ ...ext.!KGRhc2hib2FyZCk.policies.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.policies.txt | 5 + .../out/policies/__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/policies/__next._full.txt | 33 ++ .../out/policies/__next._head.txt | 6 + .../out/policies/__next._index.txt | 9 + .../out/policies/__next._tree.txt | 4 + .../_experimental/out/policies/index.html | 1 + .../_experimental/out/policies/index.txt | 33 ++ ...ext.!KGRhc2hib2FyZCk.projects.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.projects.txt | 5 + .../out/projects/__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/projects/__next._full.txt | 33 ++ .../out/projects/__next._head.txt | 6 + .../out/projects/__next._index.txt | 9 + .../out/projects/__next._tree.txt | 4 + .../_experimental/out/projects/index.html | 1 + .../_experimental/out/projects/index.txt | 33 ++ ...next.!KGRhc2hib2FyZCk.prompts.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.prompts.txt | 5 + .../out/prompts/__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/prompts/__next._full.txt | 33 ++ .../out/prompts/__next._head.txt | 6 + .../out/prompts/__next._index.txt | 9 + .../out/prompts/__next._tree.txt | 4 + .../_experimental/out/prompts/index.html | 1 + .../proxy/_experimental/out/prompts/index.txt | 33 ++ ...Rhc2hib2FyZCk.router-settings.__PAGE__.txt | 9 + ..._next.!KGRhc2hib2FyZCk.router-settings.txt | 5 + .../__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/router-settings/__next._full.txt | 33 ++ .../out/router-settings/__next._head.txt | 6 + .../out/router-settings/__next._index.txt | 9 + .../out/router-settings/__next._tree.txt | 4 + .../out/router-settings/index.html | 1 + .../out/router-settings/index.txt | 33 ++ ...!KGRhc2hib2FyZCk.search-tools.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.search-tools.txt | 5 + .../search-tools/__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/search-tools/__next._full.txt | 33 ++ .../out/search-tools/__next._head.txt | 6 + .../out/search-tools/__next._index.txt | 9 + .../out/search-tools/__next._tree.txt | 4 + .../_experimental/out/search-tools/index.html | 1 + .../_experimental/out/search-tools/index.txt | 33 ++ ..._next.!KGRhc2hib2FyZCk.skills.__PAGE__.txt | 9 + .../skills/__next.!KGRhc2hib2FyZCk.skills.txt | 5 + .../out/skills/__next.!KGRhc2hib2FyZCk.txt | 7 + .../_experimental/out/skills/__next._full.txt | 33 ++ .../_experimental/out/skills/__next._head.txt | 6 + .../out/skills/__next._index.txt | 9 + .../_experimental/out/skills/__next._tree.txt | 4 + .../proxy/_experimental/out/skills/index.html | 1 + .../proxy/_experimental/out/skills/index.txt | 33 ++ ...GRhc2hib2FyZCk.tag-management.__PAGE__.txt | 9 + ...__next.!KGRhc2hib2FyZCk.tag-management.txt | 5 + .../__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/tag-management/__next._full.txt | 33 ++ .../out/tag-management/__next._head.txt | 6 + .../out/tag-management/__next._index.txt | 9 + .../out/tag-management/__next._tree.txt | 4 + .../out/tag-management/index.html | 1 + .../out/tag-management/index.txt | 33 ++ ...__next.!KGRhc2hib2FyZCk.teams.__PAGE__.txt | 9 + .../teams/__next.!KGRhc2hib2FyZCk.teams.txt | 5 + .../out/teams/__next.!KGRhc2hib2FyZCk.txt | 7 + .../_experimental/out/teams/__next._full.txt | 33 ++ .../_experimental/out/teams/__next._head.txt | 6 + .../_experimental/out/teams/__next._index.txt | 9 + .../_experimental/out/teams/__next._tree.txt | 4 + .../proxy/_experimental/out/teams/index.html | 1 + .../proxy/_experimental/out/teams/index.txt | 33 ++ ...KGRhc2hib2FyZCk.tool-policies.__PAGE__.txt | 10 + .../__next.!KGRhc2hib2FyZCk.tool-policies.txt | 5 + .../tool-policies/__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/tool-policies/__next._full.txt | 34 ++ .../out/tool-policies/__next._head.txt | 6 + .../out/tool-policies/__next._index.txt | 9 + .../out/tool-policies/__next._tree.txt | 5 + .../out/tool-policies/index.html | 1 + .../_experimental/out/tool-policies/index.txt | 34 ++ ...c2hib2FyZCk.transform-request.__PAGE__.txt | 9 + ...ext.!KGRhc2hib2FyZCk.transform-request.txt | 5 + .../__next.!KGRhc2hib2FyZCk.txt | 7 + .../out/transform-request/__next._full.txt | 33 ++ .../out/transform-request/__next._head.txt | 6 + .../out/transform-request/__next._index.txt | 9 + .../out/transform-request/__next._tree.txt | 4 + .../out/transform-request/index.html | 1 + .../out/transform-request/index.txt | 33 ++ .../out/ui-theme/__next.!KGRhc2hib2FyZCk.txt | 7 + ...ext.!KGRhc2hib2FyZCk.ui-theme.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.ui-theme.txt | 5 + .../out/ui-theme/__next._full.txt | 33 ++ .../out/ui-theme/__next._head.txt | 6 + .../out/ui-theme/__next._index.txt | 9 + .../out/ui-theme/__next._tree.txt | 4 + .../_experimental/out/ui-theme/index.html | 1 + .../_experimental/out/ui-theme/index.txt | 33 ++ .../out/usage/__next.!KGRhc2hib2FyZCk.txt | 7 + ...__next.!KGRhc2hib2FyZCk.usage.__PAGE__.txt | 9 + .../usage/__next.!KGRhc2hib2FyZCk.usage.txt | 5 + .../_experimental/out/usage/__next._full.txt | 33 ++ .../_experimental/out/usage/__next._head.txt | 6 + .../_experimental/out/usage/__next._index.txt | 9 + .../_experimental/out/usage/__next._tree.txt | 4 + .../proxy/_experimental/out/usage/index.html | 1 + .../proxy/_experimental/out/usage/index.txt | 33 ++ .../out/users/__next.!KGRhc2hib2FyZCk.txt | 7 + ...__next.!KGRhc2hib2FyZCk.users.__PAGE__.txt | 9 + .../users/__next.!KGRhc2hib2FyZCk.users.txt | 5 + .../_experimental/out/users/__next._full.txt | 33 ++ .../_experimental/out/users/__next._head.txt | 6 + .../_experimental/out/users/__next._index.txt | 9 + .../_experimental/out/users/__next._tree.txt | 4 + .../proxy/_experimental/out/users/index.html | 1 + .../proxy/_experimental/out/users/index.txt | 33 ++ .../vector-stores/__next.!KGRhc2hib2FyZCk.txt | 7 + ...KGRhc2hib2FyZCk.vector-stores.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.vector-stores.txt | 5 + .../out/vector-stores/__next._full.txt | 33 ++ .../out/vector-stores/__next._head.txt | 6 + .../out/vector-stores/__next._index.txt | 9 + .../out/vector-stores/__next._tree.txt | 4 + .../out/vector-stores/index.html | 1 + .../_experimental/out/vector-stores/index.txt | 33 ++ litellm/proxy/_experimental/out/vercel.svg | 1 + .../out/workflows/__next.!KGRhc2hib2FyZCk.txt | 7 + ...xt.!KGRhc2hib2FyZCk.workflows.__PAGE__.txt | 9 + .../__next.!KGRhc2hib2FyZCk.workflows.txt | 5 + .../out/workflows/__next._full.txt | 33 ++ .../out/workflows/__next._head.txt | 6 + .../out/workflows/__next._index.txt | 9 + .../out/workflows/__next._tree.txt | 4 + .../_experimental/out/workflows/index.html | 1 + .../_experimental/out/workflows/index.txt | 33 ++ litellm/proxy/_new_new_secret_config.yaml | 14 + litellm/proxy/_new_secret_config.yaml | 83 ++++ litellm/proxy/_super_secret_config.yaml | 110 +++++ litellm/proxy/proxy_server.py | 61 ++- .../test-results/.last-run.json | 4 + tests/test_litellm/proxy/test_proxy_server.py | 14 +- ui/litellm-dashboard/build_ui.sh | 3 +- ui/litellm-dashboard/build_ui_custom_path.sh | 3 +- 709 files changed, 8998 insertions(+), 50 deletions(-) create mode 100644 litellm/proxy/_experimental/out/404.html create mode 100644 litellm/proxy/_experimental/out/404/index.html create mode 100644 litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_buildManifest.js create mode 100644 litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_clientMiddlewareManifest.js create mode 100644 litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_ssgManifest.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0-3i_.uof35pm.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0-4tg9f~_a3b~.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0-85n.4jrc2vv.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0-dhh1_d1.b1u.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0-f.2po-pctaa.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0-ih8xcz_89nt.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0.4.bbjx7y007.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0.bx44y-6~tug.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0.yiw37jc_bvi.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/00cy3g~l27g1y.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/00jwo~_zp.35~.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/00p.gft-l.6p..js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/00pl5r0.xdcua.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/00q4mtjboprhm.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/011mgw.-67gs_.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/01_xjyxcb1uco.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/01xm1xt.gmrff.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/01y._o853f7le.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/01~uswbzv7_90.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/022.sz94ycw4x.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/02813b2b-kz98.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/02c1-r_khzb89.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/02ihc5xweq16v.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/02nrwvikmd-wf.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/02oicwo.~e~ak.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/036wlkuzplhfz.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/038lmn5.g6myc.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/03_wvlr03g~35.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/03fia.h6j.gpu.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/03iznh0~x-p5x.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/03l9yp-0vdrvg.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/03rcuw-pknh--.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/03~yq9q893hmn.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/043q3g5-5-aju.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/04476udqypzuu.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/04amwk-x_vjxu.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/04jvxoid~vpxj.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/04p5iour3skhn.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/04~mux1g2xqfl.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/05.uhnqp00zd5.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/058o-fyv9lb_l.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/05btv.l5gro_..js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/05qmwjqau64bz.css create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/05t1k89l9tc3s.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/05w6e8.ake4_v.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/05wzckn7dnk9_.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/05z02g9s~8km0.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/066hp9.940823.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0689o862~x~pg.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/06x5y8ia4k1mc.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/07.fwfv-sinb5.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/07_~yky8gc9_m.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/08b3bdf-s.-y4.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/08is8lfgypp_2.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/09dh.hm0vr~61.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/09n64dqzn.le~.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0_cwbuh_om4s9.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0_rk9sxkapt-r.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0_tak0mb5m-3k.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0_y-b9_d9dsuv.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0aj3r46j-.qsy.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ajdq5~-z4-0o.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0au3mg4n33g_o.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0b5g~_decuer~.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0bqafy~83g2md.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0byy7z~x~srwc.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0c2apcdkbqq0o.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0c4pfjjue0uc-.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ceh~7zrbxj.y.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0d2qt-f_paso0.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ecsfnbwne0sn.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0el08tticy_20.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0em0654rb513m.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0gj2~qks1xrx8.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0gtegjaljim2a.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0h274dbe8lloe.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0hsqxu.xbf.l5.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0hzdsr8t0ksq..js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0hzj3mfqun9q~.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0i77.0u.82o9u.css create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ip1d_6ew-zr2.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ivj_wax-joap.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0j2~0jseuoube.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0jaa-io9cz430.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0jdm7x5soayfw.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0jib1e4hgitwz.css create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0jr8wo_7ak~7n.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0jzxuesytdzt0.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0k3aqiu733i3f.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0kqhn69~lkflo.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0kr3_6r.1wa_9.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0l7em-5kjv49e.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0lb0p7rh5znu_.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ldurpg4iqx04.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0lg.6rbfsd-l9.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0lku60vnd9m1i.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0lstohw6r.qs..js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0m._ijxus~ryi.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0m.pilqkjqyg3.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0m5k-5fv1ya8x.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0m6zdocif1gl4.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0mb3erwqomzal.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0md97r_057_33.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0mh1wnrvmv_y7.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0mmrbksvmhp.1.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0mspdfvjqoti_.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0mzw3maijoev6.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0n.a~e5dwfnkn.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0n028f.v-dhms.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ngre0.s4-ej6.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0nnx~7-7e5t~1.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ogm.~yq5rjmw.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ovmgshl9hfea.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0p.6bs58-_3lw.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0pd5zl~lciww9.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0pidya1qvuvx8.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0pu3ltw1cci2~.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0pwkd9r.mc_ee.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0pwrfxkkt~qfh.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0q2og72gex34u.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0q6~n4y84cejn.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0q9_qqi.nzx5l.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ql_-8xthluga.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0r8_z31ow7vw9.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0rdv7_7_95b-1.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0rsh-mjgd1-1b.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0scfmfivwcppe.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0snrx6.._0zus.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0sx3mu2_l9g_y.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0sxgv7gc5lm3g.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0sylbcw3ha_ba.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0t4ig3ibz46ga.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0tbzoqict3-mi.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0teffxf7o_863.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0tgl~~_4hb1rp.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0u3_nka63vh6t.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0us_9w7qaihte.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0uu6lckpr0s15.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0uy6wzxw5oh5v.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0v1rxqc1hqmrl.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0vo11_94ear6l.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0w39dn9x3dp9g.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0whkizop7gd0~.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0x.73w57rn4ou.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0x6hmpiq7.b-x.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ydd65iv6ffpl.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ys10755n8os_.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0z4fh7pvzmoy8.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0zqdpz_rk5.wq.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0zrbitbm~0koh.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0~-ovi6c4wjt1.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0~0su3wi_7f6-.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0~tp1mbr_st8h.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0~~y94vmu8z5d.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/101az3fsw7lje.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/10e9lx.nawttb.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/10jlu0mdcmzoi.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/10sdqywhhhn7i.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/10ybnll3qh-8s.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/114pbx0696lkh.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/11h.ntqd0jl3z.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/11kowzys1c43t.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/129bujhdmi9ce.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/13c74.fwk0wmq.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/13ln.k6r3lkv_.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/13s0v9siktndj.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/142-5lmjc6wc~.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/14566-_ogh-19.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/14_9gq.6yjjih.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/15.9ylrtxojbj.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1560njdijg7fq.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/15auqattd2wzv.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/15hm8gokjq2uu.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/15rg~y4h.lcrl.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/15wqqcwhnlidr.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/16.oisvgwzo8s.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/169km.d7x9qr6.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/16qfko21~_dn~.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1781p3yhsw7kp.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/17b18lwgc39xm.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/17cvpyw6fshd4.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/17e1s6gkzjh5f.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/17j1m89pizunk.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/17jd5l9o~hzf3.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/17n.qg70cy9.9.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/184o99uxk88c7.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/turbopack-0a~tzicx4wgrt.js create mode 100644 litellm/proxy/_experimental/out/_next/static/media/1bffadaabf893a1e-s.16ipb6fqu393i.woff2 create mode 100644 litellm/proxy/_experimental/out/_next/static/media/2bbe8d2671613f1f-s.067x_6k0k23tk.woff2 create mode 100644 litellm/proxy/_experimental/out/_next/static/media/2c55a0e60120577a-s.0bjc5tiuqdqro.woff2 create mode 100644 litellm/proxy/_experimental/out/_next/static/media/5476f68d60460930-s.0wxq9webf.ew4.woff2 create mode 100644 litellm/proxy/_experimental/out/_next/static/media/83afe278b6a6bb3c-s.p.0q-301v4kxxnr.woff2 create mode 100644 litellm/proxy/_experimental/out/_next/static/media/9c72aa0f40e4eef8-s.0m6w47a4e5dy9.woff2 create mode 100644 litellm/proxy/_experimental/out/_next/static/media/ad66f9afd8947f86-s.11u06r12fd6v_.woff2 create mode 100644 litellm/proxy/_experimental/out/_next/static/media/favicon.0~dgapwhi~75y.ico create mode 100644 litellm/proxy/_experimental/out/_not-found/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/_not-found/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/_not-found/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/_not-found/__next._not-found.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/_not-found/__next._not-found.txt create mode 100644 litellm/proxy/_experimental/out/_not-found/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/_not-found/index.html create mode 100644 litellm/proxy/_experimental/out/_not-found/index.txt create mode 100644 litellm/proxy/_experimental/out/access-groups/__next.!KGRhc2hib2FyZCk.access-groups.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/access-groups/__next.!KGRhc2hib2FyZCk.access-groups.txt create mode 100644 litellm/proxy/_experimental/out/access-groups/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/access-groups/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/access-groups/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/access-groups/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/access-groups/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/access-groups/index.html create mode 100644 litellm/proxy/_experimental/out/access-groups/index.txt create mode 100644 litellm/proxy/_experimental/out/admin-panel/__next.!KGRhc2hib2FyZCk.admin-panel.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/admin-panel/__next.!KGRhc2hib2FyZCk.admin-panel.txt create mode 100644 litellm/proxy/_experimental/out/admin-panel/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/admin-panel/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/admin-panel/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/admin-panel/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/admin-panel/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/admin-panel/index.html create mode 100644 litellm/proxy/_experimental/out/admin-panel/index.txt create mode 100644 litellm/proxy/_experimental/out/agents/__next.!KGRhc2hib2FyZCk.agents.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/agents/__next.!KGRhc2hib2FyZCk.agents.txt create mode 100644 litellm/proxy/_experimental/out/agents/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/agents/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/agents/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/agents/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/agents/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/agents/index.html create mode 100644 litellm/proxy/_experimental/out/agents/index.txt create mode 100644 litellm/proxy/_experimental/out/api-keys/__next.!KGRhc2hib2FyZCk.api-keys.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/api-keys/__next.!KGRhc2hib2FyZCk.api-keys.txt create mode 100644 litellm/proxy/_experimental/out/api-keys/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/api-keys/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/api-keys/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/api-keys/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/api-keys/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/api-keys/index.html create mode 100644 litellm/proxy/_experimental/out/api-keys/index.txt create mode 100644 litellm/proxy/_experimental/out/api-reference/__next.!KGRhc2hib2FyZCk.api-reference.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/api-reference/__next.!KGRhc2hib2FyZCk.api-reference.txt create mode 100644 litellm/proxy/_experimental/out/api-reference/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/api-reference/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/api-reference/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/api-reference/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/api-reference/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/api-reference/index.html create mode 100644 litellm/proxy/_experimental/out/api-reference/index.txt create mode 100644 litellm/proxy/_experimental/out/assets/audit-logs-preview.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/a2a_agent.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/ai21.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/aim_logo.jpeg create mode 100644 litellm/proxy/_experimental/out/assets/logos/aim_security.jpeg create mode 100644 litellm/proxy/_experimental/out/assets/logos/aiml_api.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/akto.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/anthropic.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/aporia.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/arize.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/assemblyai_small.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/aws.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/azure_ai_foundry.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/baseten.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/bedrock.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/braintrust.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/cato_networks.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/cerebras.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/cisco.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/cloudflare.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/cohere.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/cometapi.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/cursor.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/databricks.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/datadog.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/dataforseo.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/deepgram.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/deepinfra.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/deepseek.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/elevenlabs.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/enkrypt_ai.avif create mode 100644 litellm/proxy/_experimental/out/assets/logos/exa_ai.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/fal_ai.jpg create mode 100644 litellm/proxy/_experimental/out/assets/logos/featherless.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/figma.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/fireworks.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/friendli.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/galileo.ico create mode 100644 litellm/proxy/_experimental/out/assets/logos/github.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/github_copilot.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/gitlab.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/gmail.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/google.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/google_drive.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/google_pse.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/groq.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/guardrails_ai.jpeg create mode 100644 litellm/proxy/_experimental/out/assets/logos/hubspot.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/huggingface.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/hyperbolic.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/infinity.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/javelin.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/jina.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/jira.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/lago.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/lakeraai.jpeg create mode 100644 litellm/proxy/_experimental/out/assets/logos/lambda.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/langflow.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/langfuse.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/langfuse.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/langgraph.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/langsmith.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/lasso.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/linear.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/litellm.jpg create mode 100644 litellm/proxy/_experimental/out/assets/logos/litellm_logo.jpg create mode 100644 litellm/proxy/_experimental/out/assets/logos/llm_guard.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/lmstudio.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/mcp_logo.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/meta_llama.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/microsoft_azure.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/milvus.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/minimax.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/mistral.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/moonshot.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/morph.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/nebius.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/newrelic.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/noma_security.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/notion.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/novita.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/nvidia_nim.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/nvidia_triton.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/ollama.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/openai_small.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/openmeter.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/openrouter.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/oracle.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/otel.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/palo_alto_networks.jpeg create mode 100644 litellm/proxy/_experimental/out/assets/logos/pangea.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/parallel_ai.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/perplexity-ai.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/perplexity.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/pillar.jpeg create mode 100644 litellm/proxy/_experimental/out/assets/logos/postgresql.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/presidio.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/prompt_security.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/promptguard.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/pydantic.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/qohash.jpg create mode 100644 litellm/proxy/_experimental/out/assets/logos/qwen.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/recraft.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/repelloai.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/replicate.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/runway.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/s3_vector.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/salesforce.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/sambanova.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/sap.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/search1api.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/secret_detect.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/sentry.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/shopify.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/slack.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/snowflake.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/soniox.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/stripe.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/tavily.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/togetherai.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/topaz.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/twilio.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/v0.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/vercel.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/vllm.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/volcengine.png create mode 100644 litellm/proxy/_experimental/out/assets/logos/voyage.webp create mode 100644 litellm/proxy/_experimental/out/assets/logos/watsonx.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/xai.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/xecguard.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/xinference.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/zapier.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/zscaler.svg create mode 100644 litellm/proxy/_experimental/out/budgets/__next.!KGRhc2hib2FyZCk.budgets.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/budgets/__next.!KGRhc2hib2FyZCk.budgets.txt create mode 100644 litellm/proxy/_experimental/out/budgets/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/budgets/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/budgets/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/budgets/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/budgets/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/budgets/index.html create mode 100644 litellm/proxy/_experimental/out/budgets/index.txt create mode 100644 litellm/proxy/_experimental/out/caching/__next.!KGRhc2hib2FyZCk.caching.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/caching/__next.!KGRhc2hib2FyZCk.caching.txt create mode 100644 litellm/proxy/_experimental/out/caching/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/caching/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/caching/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/caching/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/caching/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/caching/index.html create mode 100644 litellm/proxy/_experimental/out/caching/index.txt create mode 100644 litellm/proxy/_experimental/out/cost-tracking/__next.!KGRhc2hib2FyZCk.cost-tracking.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/cost-tracking/__next.!KGRhc2hib2FyZCk.cost-tracking.txt create mode 100644 litellm/proxy/_experimental/out/cost-tracking/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/cost-tracking/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/cost-tracking/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/cost-tracking/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/cost-tracking/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/cost-tracking/index.html create mode 100644 litellm/proxy/_experimental/out/cost-tracking/index.txt create mode 100644 litellm/proxy/_experimental/out/favicon.ico create mode 100644 litellm/proxy/_experimental/out/guardrails-monitor/__next.!KGRhc2hib2FyZCk.guardrails-monitor.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/guardrails-monitor/__next.!KGRhc2hib2FyZCk.guardrails-monitor.txt create mode 100644 litellm/proxy/_experimental/out/guardrails-monitor/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/guardrails-monitor/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/guardrails-monitor/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/guardrails-monitor/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/guardrails-monitor/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/guardrails-monitor/index.html create mode 100644 litellm/proxy/_experimental/out/guardrails-monitor/index.txt create mode 100644 litellm/proxy/_experimental/out/guardrails/__next.!KGRhc2hib2FyZCk.guardrails.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/guardrails/__next.!KGRhc2hib2FyZCk.guardrails.txt create mode 100644 litellm/proxy/_experimental/out/guardrails/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/guardrails/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/guardrails/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/guardrails/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/guardrails/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/guardrails/index.html create mode 100644 litellm/proxy/_experimental/out/guardrails/index.txt create mode 100644 litellm/proxy/_experimental/out/index.html create mode 100644 litellm/proxy/_experimental/out/index.txt create mode 100644 litellm/proxy/_experimental/out/logging-and-alerts/__next.!KGRhc2hib2FyZCk.logging-and-alerts.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/logging-and-alerts/__next.!KGRhc2hib2FyZCk.logging-and-alerts.txt create mode 100644 litellm/proxy/_experimental/out/logging-and-alerts/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/logging-and-alerts/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/logging-and-alerts/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/logging-and-alerts/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/logging-and-alerts/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/logging-and-alerts/index.html create mode 100644 litellm/proxy/_experimental/out/logging-and-alerts/index.txt create mode 100644 litellm/proxy/_experimental/out/login/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/login/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/login/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/login/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/login/__next.login.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/login/__next.login.txt create mode 100644 litellm/proxy/_experimental/out/login/index.html create mode 100644 litellm/proxy/_experimental/out/login/index.txt create mode 100644 litellm/proxy/_experimental/out/logs/__next.!KGRhc2hib2FyZCk.logs.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/logs/__next.!KGRhc2hib2FyZCk.logs.txt create mode 100644 litellm/proxy/_experimental/out/logs/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/logs/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/logs/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/logs/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/logs/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/logs/index.html create mode 100644 litellm/proxy/_experimental/out/logs/index.txt create mode 100644 litellm/proxy/_experimental/out/mcp-servers/__next.!KGRhc2hib2FyZCk.mcp-servers.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/mcp-servers/__next.!KGRhc2hib2FyZCk.mcp-servers.txt create mode 100644 litellm/proxy/_experimental/out/mcp-servers/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/mcp-servers/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/mcp-servers/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/mcp-servers/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/mcp-servers/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/mcp-servers/index.html create mode 100644 litellm/proxy/_experimental/out/mcp-servers/index.txt create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/__next.mcp.oauth.callback.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/__next.mcp.oauth.callback.txt create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/__next.mcp.oauth.txt create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/__next.mcp.txt create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/index.html create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/index.txt create mode 100644 litellm/proxy/_experimental/out/memory/__next.!KGRhc2hib2FyZCk.memory.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/memory/__next.!KGRhc2hib2FyZCk.memory.txt create mode 100644 litellm/proxy/_experimental/out/memory/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/memory/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/memory/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/memory/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/memory/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/memory/index.html create mode 100644 litellm/proxy/_experimental/out/memory/index.txt create mode 100644 litellm/proxy/_experimental/out/model-hub-table/__next.!KGRhc2hib2FyZCk.model-hub-table.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/model-hub-table/__next.!KGRhc2hib2FyZCk.model-hub-table.txt create mode 100644 litellm/proxy/_experimental/out/model-hub-table/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/model-hub-table/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/model-hub-table/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/model-hub-table/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/model-hub-table/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/model-hub-table/index.html create mode 100644 litellm/proxy/_experimental/out/model-hub-table/index.txt create mode 100644 litellm/proxy/_experimental/out/model_hub/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/model_hub/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/model_hub/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/model_hub/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/model_hub/__next.model_hub.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/model_hub/__next.model_hub.txt create mode 100644 litellm/proxy/_experimental/out/model_hub/index.html create mode 100644 litellm/proxy/_experimental/out/model_hub/index.txt create mode 100644 litellm/proxy/_experimental/out/model_hub_table/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/model_hub_table/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/model_hub_table/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/model_hub_table/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/model_hub_table/__next.model_hub_table.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/model_hub_table/__next.model_hub_table.txt create mode 100644 litellm/proxy/_experimental/out/model_hub_table/index.html create mode 100644 litellm/proxy/_experimental/out/model_hub_table/index.txt create mode 100644 litellm/proxy/_experimental/out/models-and-endpoints/__next.!KGRhc2hib2FyZCk.models-and-endpoints.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/models-and-endpoints/__next.!KGRhc2hib2FyZCk.models-and-endpoints.txt create mode 100644 litellm/proxy/_experimental/out/models-and-endpoints/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/models-and-endpoints/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/models-and-endpoints/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/models-and-endpoints/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/models-and-endpoints/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/models-and-endpoints/index.html create mode 100644 litellm/proxy/_experimental/out/models-and-endpoints/index.txt create mode 100644 litellm/proxy/_experimental/out/next.svg create mode 100644 litellm/proxy/_experimental/out/old-usage/__next.!KGRhc2hib2FyZCk.old-usage.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/old-usage/__next.!KGRhc2hib2FyZCk.old-usage.txt create mode 100644 litellm/proxy/_experimental/out/old-usage/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/old-usage/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/old-usage/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/old-usage/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/old-usage/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/old-usage/index.html create mode 100644 litellm/proxy/_experimental/out/old-usage/index.txt create mode 100644 litellm/proxy/_experimental/out/onboarding/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/onboarding/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/onboarding/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/onboarding/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/onboarding/__next.onboarding.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/onboarding/__next.onboarding.txt create mode 100644 litellm/proxy/_experimental/out/onboarding/index.html create mode 100644 litellm/proxy/_experimental/out/onboarding/index.txt create mode 100644 litellm/proxy/_experimental/out/organizations/__next.!KGRhc2hib2FyZCk.organizations.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/organizations/__next.!KGRhc2hib2FyZCk.organizations.txt create mode 100644 litellm/proxy/_experimental/out/organizations/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/organizations/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/organizations/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/organizations/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/organizations/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/organizations/index.html create mode 100644 litellm/proxy/_experimental/out/organizations/index.txt create mode 100644 litellm/proxy/_experimental/out/playground/__next.!KGRhc2hib2FyZCk.playground.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/playground/__next.!KGRhc2hib2FyZCk.playground.txt create mode 100644 litellm/proxy/_experimental/out/playground/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/playground/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/playground/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/playground/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/playground/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/playground/index.html create mode 100644 litellm/proxy/_experimental/out/playground/index.txt create mode 100644 litellm/proxy/_experimental/out/policies/__next.!KGRhc2hib2FyZCk.policies.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/policies/__next.!KGRhc2hib2FyZCk.policies.txt create mode 100644 litellm/proxy/_experimental/out/policies/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/policies/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/policies/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/policies/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/policies/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/policies/index.html create mode 100644 litellm/proxy/_experimental/out/policies/index.txt create mode 100644 litellm/proxy/_experimental/out/projects/__next.!KGRhc2hib2FyZCk.projects.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/projects/__next.!KGRhc2hib2FyZCk.projects.txt create mode 100644 litellm/proxy/_experimental/out/projects/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/projects/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/projects/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/projects/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/projects/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/projects/index.html create mode 100644 litellm/proxy/_experimental/out/projects/index.txt create mode 100644 litellm/proxy/_experimental/out/prompts/__next.!KGRhc2hib2FyZCk.prompts.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/prompts/__next.!KGRhc2hib2FyZCk.prompts.txt create mode 100644 litellm/proxy/_experimental/out/prompts/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/prompts/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/prompts/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/prompts/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/prompts/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/prompts/index.html create mode 100644 litellm/proxy/_experimental/out/prompts/index.txt create mode 100644 litellm/proxy/_experimental/out/router-settings/__next.!KGRhc2hib2FyZCk.router-settings.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/router-settings/__next.!KGRhc2hib2FyZCk.router-settings.txt create mode 100644 litellm/proxy/_experimental/out/router-settings/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/router-settings/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/router-settings/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/router-settings/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/router-settings/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/router-settings/index.html create mode 100644 litellm/proxy/_experimental/out/router-settings/index.txt create mode 100644 litellm/proxy/_experimental/out/search-tools/__next.!KGRhc2hib2FyZCk.search-tools.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/search-tools/__next.!KGRhc2hib2FyZCk.search-tools.txt create mode 100644 litellm/proxy/_experimental/out/search-tools/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/search-tools/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/search-tools/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/search-tools/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/search-tools/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/search-tools/index.html create mode 100644 litellm/proxy/_experimental/out/search-tools/index.txt create mode 100644 litellm/proxy/_experimental/out/skills/__next.!KGRhc2hib2FyZCk.skills.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/skills/__next.!KGRhc2hib2FyZCk.skills.txt create mode 100644 litellm/proxy/_experimental/out/skills/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/skills/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/skills/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/skills/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/skills/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/skills/index.html create mode 100644 litellm/proxy/_experimental/out/skills/index.txt create mode 100644 litellm/proxy/_experimental/out/tag-management/__next.!KGRhc2hib2FyZCk.tag-management.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/tag-management/__next.!KGRhc2hib2FyZCk.tag-management.txt create mode 100644 litellm/proxy/_experimental/out/tag-management/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/tag-management/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/tag-management/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/tag-management/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/tag-management/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/tag-management/index.html create mode 100644 litellm/proxy/_experimental/out/tag-management/index.txt create mode 100644 litellm/proxy/_experimental/out/teams/__next.!KGRhc2hib2FyZCk.teams.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/teams/__next.!KGRhc2hib2FyZCk.teams.txt create mode 100644 litellm/proxy/_experimental/out/teams/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/teams/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/teams/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/teams/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/teams/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/teams/index.html create mode 100644 litellm/proxy/_experimental/out/teams/index.txt create mode 100644 litellm/proxy/_experimental/out/tool-policies/__next.!KGRhc2hib2FyZCk.tool-policies.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/tool-policies/__next.!KGRhc2hib2FyZCk.tool-policies.txt create mode 100644 litellm/proxy/_experimental/out/tool-policies/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/tool-policies/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/tool-policies/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/tool-policies/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/tool-policies/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/tool-policies/index.html create mode 100644 litellm/proxy/_experimental/out/tool-policies/index.txt create mode 100644 litellm/proxy/_experimental/out/transform-request/__next.!KGRhc2hib2FyZCk.transform-request.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/transform-request/__next.!KGRhc2hib2FyZCk.transform-request.txt create mode 100644 litellm/proxy/_experimental/out/transform-request/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/transform-request/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/transform-request/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/transform-request/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/transform-request/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/transform-request/index.html create mode 100644 litellm/proxy/_experimental/out/transform-request/index.txt create mode 100644 litellm/proxy/_experimental/out/ui-theme/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/ui-theme/__next.!KGRhc2hib2FyZCk.ui-theme.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/ui-theme/__next.!KGRhc2hib2FyZCk.ui-theme.txt create mode 100644 litellm/proxy/_experimental/out/ui-theme/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/ui-theme/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/ui-theme/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/ui-theme/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/ui-theme/index.html create mode 100644 litellm/proxy/_experimental/out/ui-theme/index.txt create mode 100644 litellm/proxy/_experimental/out/usage/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/usage/__next.!KGRhc2hib2FyZCk.usage.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/usage/__next.!KGRhc2hib2FyZCk.usage.txt create mode 100644 litellm/proxy/_experimental/out/usage/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/usage/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/usage/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/usage/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/usage/index.html create mode 100644 litellm/proxy/_experimental/out/usage/index.txt create mode 100644 litellm/proxy/_experimental/out/users/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/users/__next.!KGRhc2hib2FyZCk.users.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/users/__next.!KGRhc2hib2FyZCk.users.txt create mode 100644 litellm/proxy/_experimental/out/users/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/users/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/users/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/users/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/users/index.html create mode 100644 litellm/proxy/_experimental/out/users/index.txt create mode 100644 litellm/proxy/_experimental/out/vector-stores/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/vector-stores/__next.!KGRhc2hib2FyZCk.vector-stores.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/vector-stores/__next.!KGRhc2hib2FyZCk.vector-stores.txt create mode 100644 litellm/proxy/_experimental/out/vector-stores/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/vector-stores/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/vector-stores/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/vector-stores/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/vector-stores/index.html create mode 100644 litellm/proxy/_experimental/out/vector-stores/index.txt create mode 100644 litellm/proxy/_experimental/out/vercel.svg create mode 100644 litellm/proxy/_experimental/out/workflows/__next.!KGRhc2hib2FyZCk.txt create mode 100644 litellm/proxy/_experimental/out/workflows/__next.!KGRhc2hib2FyZCk.workflows.__PAGE__.txt create mode 100644 litellm/proxy/_experimental/out/workflows/__next.!KGRhc2hib2FyZCk.workflows.txt create mode 100644 litellm/proxy/_experimental/out/workflows/__next._full.txt create mode 100644 litellm/proxy/_experimental/out/workflows/__next._head.txt create mode 100644 litellm/proxy/_experimental/out/workflows/__next._index.txt create mode 100644 litellm/proxy/_experimental/out/workflows/__next._tree.txt create mode 100644 litellm/proxy/_experimental/out/workflows/index.html create mode 100644 litellm/proxy/_experimental/out/workflows/index.txt create mode 100644 litellm/proxy/_new_new_secret_config.yaml create mode 100644 litellm/proxy/_new_secret_config.yaml create mode 100644 litellm/proxy/_super_secret_config.yaml create mode 100644 tests/proxy_admin_ui_tests/test-results/.last-run.json diff --git a/.gitignore b/.gitignore index 5b7c6e5585b..59fa5803abe 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,8 @@ litellm/proxy/tests/package-lock.json ui/litellm-dashboard/.next ui/litellm-dashboard/node_modules ui/litellm-dashboard/next-env.d.ts +ui/litellm-dashboard/package.json +ui/litellm-dashboard/package-lock.json deploy/charts/litellm/*.tgz deploy/charts/litellm/charts/* deploy/charts/*.tgz @@ -85,12 +87,17 @@ litellm/proxy/db/migrations/* litellm/proxy/migrations/*config.yaml litellm/proxy/migrations/* litellm/proxy/to_delete_loadtest_work/* +config.yaml tests/litellm/litellm_core_utils/llm_cost_calc/log.txt tests/test_custom_dir/* +test.py +litellm_config.yaml +!.github/observatory/litellm_config.yaml .cursor litellm/proxy/to_delete_loadtest_work/* update_model_cost_map.py +tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py scripts/test_vertex_ai_search.py LAZY_LOADING_IMPROVEMENTS.md STABILIZATION_TODO.md @@ -123,7 +130,3 @@ crash.*.log # pytest coverage data .coverage - -# _experimental/out UI build output -# (both componentized and non-componentized build the UI on project release) -litellm/proxy/_experimental/out/ \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html new file mode 100644 index 00000000000..dd2a01991de --- /dev/null +++ b/litellm/proxy/_experimental/out/404.html @@ -0,0 +1 @@ +404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/404/index.html b/litellm/proxy/_experimental/out/404/index.html new file mode 100644 index 00000000000..dd2a01991de --- /dev/null +++ b/litellm/proxy/_experimental/out/404/index.html @@ -0,0 +1 @@ +404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt b/litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt new file mode 100644 index 00000000000..55b18876d5b --- /dev/null +++ b/litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt @@ -0,0 +1,9 @@ +1:"$Sreact.fragment" +2:I[347257,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientPageRoot"] +3:I[871135,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0whkizop7gd0~.js","/litellm-asset-prefix/_next/static/chunks/0-ih8xcz_89nt.js","/litellm-asset-prefix/_next/static/chunks/0pd5zl~lciww9.js","/litellm-asset-prefix/_next/static/chunks/02ihc5xweq16v.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0mzw3maijoev6.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/04amwk-x_vjxu.js","/litellm-asset-prefix/_next/static/chunks/0-dhh1_d1.b1u.js","/litellm-asset-prefix/_next/static/chunks/0pwkd9r.mc_ee.js","/litellm-asset-prefix/_next/static/chunks/011mgw.-67gs_.js","/litellm-asset-prefix/_next/static/chunks/0~-ovi6c4wjt1.js","/litellm-asset-prefix/_next/static/chunks/0_y-b9_d9dsuv.js","/litellm-asset-prefix/_next/static/chunks/0c2apcdkbqq0o.js","/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","/litellm-asset-prefix/_next/static/chunks/0ngre0.s4-ej6.js","/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","/litellm-asset-prefix/_next/static/chunks/05t1k89l9tc3s.js","/litellm-asset-prefix/_next/static/chunks/17n.qg70cy9.9.js","/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","/litellm-asset-prefix/_next/static/chunks/0-3i_.uof35pm.js","/litellm-asset-prefix/_next/static/chunks/14566-_ogh-19.js","/litellm-asset-prefix/_next/static/chunks/0w39dn9x3dp9g.js","/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","/litellm-asset-prefix/_next/static/chunks/0v1rxqc1hqmrl.js","/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js"],"default"] +6:I[897367,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"OutletBoundary"] +7:"$Sreact.suspense" +0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/011mgw.-67gs_.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0~-ovi6c4wjt1.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0_y-b9_d9dsuv.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0c2apcdkbqq0o.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0ngre0.s4-ej6.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/05t1k89l9tc3s.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/17n.qg70cy9.9.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/0-3i_.uof35pm.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/14566-_ogh-19.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/0w39dn9x3dp9g.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0v1rxqc1hqmrl.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"5rDiFx0t_mOGYmV_8kSkw"} +4:{} +5:"$0:rsc:props:children:0:props:serverProvidedParams:params" +8:null diff --git a/litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt b/litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt new file mode 100644 index 00000000000..55176f9118b --- /dev/null +++ b/litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt @@ -0,0 +1,7 @@ +1:"$Sreact.fragment" +2:I[92825,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientSegmentRoot"] +3:I[216370,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0whkizop7gd0~.js","/litellm-asset-prefix/_next/static/chunks/0-ih8xcz_89nt.js","/litellm-asset-prefix/_next/static/chunks/0pd5zl~lciww9.js","/litellm-asset-prefix/_next/static/chunks/02ihc5xweq16v.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0mzw3maijoev6.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/04amwk-x_vjxu.js","/litellm-asset-prefix/_next/static/chunks/0-dhh1_d1.b1u.js","/litellm-asset-prefix/_next/static/chunks/0pwkd9r.mc_ee.js"],"default"] +4:I[339756,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +5:I[837457,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0whkizop7gd0~.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0-ih8xcz_89nt.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0pd5zl~lciww9.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/02ihc5xweq16v.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0mzw3maijoev6.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/04amwk-x_vjxu.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0-dhh1_d1.b1u.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0pwkd9r.mc_ee.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"5rDiFx0t_mOGYmV_8kSkw"} +6:"$0:rsc:props:children:1:props:serverProvidedParams:params" diff --git a/litellm/proxy/_experimental/out/__next._full.txt b/litellm/proxy/_experimental/out/__next._full.txt new file mode 100644 index 00000000000..f1ff1ff8411 --- /dev/null +++ b/litellm/proxy/_experimental/out/__next._full.txt @@ -0,0 +1,30 @@ +1:"$Sreact.fragment" +2:I[867271,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +3:I[71195,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +4:I[557951,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"AuthProvider"] +5:I[339756,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +6:I[837457,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +7:I[92825,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientSegmentRoot"] +8:I[216370,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0whkizop7gd0~.js","/litellm-asset-prefix/_next/static/chunks/0-ih8xcz_89nt.js","/litellm-asset-prefix/_next/static/chunks/0pd5zl~lciww9.js","/litellm-asset-prefix/_next/static/chunks/02ihc5xweq16v.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0mzw3maijoev6.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/04amwk-x_vjxu.js","/litellm-asset-prefix/_next/static/chunks/0-dhh1_d1.b1u.js","/litellm-asset-prefix/_next/static/chunks/0pwkd9r.mc_ee.js"],"default"] +c:I[168027,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default",1] +:HL["/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","style"] +:HL["/litellm-asset-prefix/_next/static/chunks/0i77.0u.82o9u.css","style"] +:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.0q-301v4kxxnr.woff2","font",{"crossOrigin":"","type":"font/woff2"}] +0:{"P":null,"c":["",""],"q":"","i":false,"f":[[["",{"children":["(dashboard)",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",16],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/0i77.0u.82o9u.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L6",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0whkizop7gd0~.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0-ih8xcz_89nt.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0pd5zl~lciww9.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/02ihc5xweq16v.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0mzw3maijoev6.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/04amwk-x_vjxu.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0-dhh1_d1.b1u.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0pwkd9r.mc_ee.js","async":true,"nonce":"$undefined"}]],["$","$L7",null,{"Component":"$8","slots":{"children":["$","$L5",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L6",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:children:props:notFound:0:1:props:style","children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:1:props:style","children":404}],["$","div",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:style","children":["$","h2",null,{"style":"$0:f:0:1:0:props:children:1:props:children:props:children:props:children:props:children:props:children:props:notFound:0:1:props:children:props:children:2:props:children:props:style","children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]},"serverProvidedParams":{"params":{},"promises":["$@9"]}}]]}],{"children":["$La",{},null,false,null]},null,false,null]},null,false,null],"$Lb",false]],"m":"$undefined","G":["$c",["$Ld","$Le"]],"S":true,"h":null,"s":"$undefined","l":"$undefined","p":"$undefined","d":"$undefined","b":"5rDiFx0t_mOGYmV_8kSkw"} +f:I[347257,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ClientPageRoot"] +10:I[871135,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","/litellm-asset-prefix/_next/static/chunks/0whkizop7gd0~.js","/litellm-asset-prefix/_next/static/chunks/0-ih8xcz_89nt.js","/litellm-asset-prefix/_next/static/chunks/0pd5zl~lciww9.js","/litellm-asset-prefix/_next/static/chunks/02ihc5xweq16v.js","/litellm-asset-prefix/_next/static/chunks/0lg.6rbfsd-l9.js","/litellm-asset-prefix/_next/static/chunks/0mzw3maijoev6.js","/litellm-asset-prefix/_next/static/chunks/043q3g5-5-aju.js","/litellm-asset-prefix/_next/static/chunks/04amwk-x_vjxu.js","/litellm-asset-prefix/_next/static/chunks/0-dhh1_d1.b1u.js","/litellm-asset-prefix/_next/static/chunks/0pwkd9r.mc_ee.js","/litellm-asset-prefix/_next/static/chunks/011mgw.-67gs_.js","/litellm-asset-prefix/_next/static/chunks/0~-ovi6c4wjt1.js","/litellm-asset-prefix/_next/static/chunks/0_y-b9_d9dsuv.js","/litellm-asset-prefix/_next/static/chunks/0c2apcdkbqq0o.js","/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","/litellm-asset-prefix/_next/static/chunks/0ngre0.s4-ej6.js","/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","/litellm-asset-prefix/_next/static/chunks/05t1k89l9tc3s.js","/litellm-asset-prefix/_next/static/chunks/17n.qg70cy9.9.js","/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","/litellm-asset-prefix/_next/static/chunks/0-3i_.uof35pm.js","/litellm-asset-prefix/_next/static/chunks/14566-_ogh-19.js","/litellm-asset-prefix/_next/static/chunks/0w39dn9x3dp9g.js","/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","/litellm-asset-prefix/_next/static/chunks/0v1rxqc1hqmrl.js","/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js"],"default"] +13:I[897367,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"OutletBoundary"] +14:"$Sreact.suspense" +16:I[897367,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ViewportBoundary"] +18:I[897367,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"MetadataBoundary"] +a:["$","$1","c",{"children":[["$","$Lf",null,{"Component":"$10","serverProvidedParams":{"searchParams":{},"params":"$0:f:0:1:1:children:0:props:children:1:props:serverProvidedParams:params","promises":["$@11","$@12"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/011mgw.-67gs_.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0~-ovi6c4wjt1.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0_y-b9_d9dsuv.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0c2apcdkbqq0o.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0zrbitbm~0koh.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0sx3mu2_l9g_y.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0ngre0.s4-ej6.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0l7em-5kjv49e.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/05t1k89l9tc3s.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/17n.qg70cy9.9.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/00q4mtjboprhm.js","async":true,"nonce":"$undefined"}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0el08tticy_20.js","async":true,"nonce":"$undefined"}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/0-3i_.uof35pm.js","async":true,"nonce":"$undefined"}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/14566-_ogh-19.js","async":true,"nonce":"$undefined"}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/0w39dn9x3dp9g.js","async":true,"nonce":"$undefined"}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/0q6~n4y84cejn.js","async":true,"nonce":"$undefined"}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0v1rxqc1hqmrl.js","async":true,"nonce":"$undefined"}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/0c4pfjjue0uc-.js","async":true,"nonce":"$undefined"}]],["$","$L13",null,{"children":["$","$14",null,{"name":"Next.MetadataOutlet","children":"$@15"}]}]]}] +b:["$","$1","h",{"children":[null,["$","$L16",null,{"children":"$L17"}],["$","div",null,{"hidden":true,"children":["$","$L18",null,{"children":["$","$14",null,{"name":"Next.Metadata","children":"$L19"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}] +d:["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}] +e:["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/0i77.0u.82o9u.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}] +9:"$0:f:0:1:1:children:0:props:children:1:props:serverProvidedParams:params" +11:{} +12:"$0:f:0:1:1:children:0:props:children:1:props:serverProvidedParams:params" +17:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]] +1a:I[27201,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"IconMark"] +15:null +19:[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.0~dgapwhi~75y.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L1a","4",{}]] diff --git a/litellm/proxy/_experimental/out/__next._head.txt b/litellm/proxy/_experimental/out/__next._head.txt new file mode 100644 index 00000000000..27acd699792 --- /dev/null +++ b/litellm/proxy/_experimental/out/__next._head.txt @@ -0,0 +1,6 @@ +1:"$Sreact.fragment" +2:I[897367,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"ViewportBoundary"] +3:I[897367,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"MetadataBoundary"] +4:"$Sreact.suspense" +5:I[27201,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"IconMark"] +0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.0~dgapwhi~75y.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"5rDiFx0t_mOGYmV_8kSkw"} diff --git a/litellm/proxy/_experimental/out/__next._index.txt b/litellm/proxy/_experimental/out/__next._index.txt new file mode 100644 index 00000000000..9714fd22643 --- /dev/null +++ b/litellm/proxy/_experimental/out/__next._index.txt @@ -0,0 +1,9 @@ +1:"$Sreact.fragment" +2:I[867271,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +3:I[71195,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +4:I[557951,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"AuthProvider"] +5:I[339756,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +6:I[837457,["/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js"],"default"] +:HL["/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","style"] +:HL["/litellm-asset-prefix/_next/static/chunks/0i77.0u.82o9u.css","style"] +0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/0i77.0u.82o9u.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0n.a~e5dwfnkn.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0.4.bbjx7y007.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0pidya1qvuvx8.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"5rDiFx0t_mOGYmV_8kSkw"} diff --git a/litellm/proxy/_experimental/out/__next._tree.txt b/litellm/proxy/_experimental/out/__next._tree.txt new file mode 100644 index 00000000000..c8aadb1d1e2 --- /dev/null +++ b/litellm/proxy/_experimental/out/__next._tree.txt @@ -0,0 +1,4 @@ +:HL["/litellm-asset-prefix/_next/static/chunks/05qmwjqau64bz.css","style"] +:HL["/litellm-asset-prefix/_next/static/chunks/0i77.0u.82o9u.css","style"] +:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.0q-301v4kxxnr.woff2","font",{"crossOrigin":"","type":"font/woff2"}] +0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"5rDiFx0t_mOGYmV_8kSkw"} diff --git a/litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_buildManifest.js b/litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_buildManifest.js new file mode 100644 index 00000000000..d74e1661bbe --- /dev/null +++ b/litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_buildManifest.js @@ -0,0 +1,16 @@ +self.__BUILD_MANIFEST = { + "__rewrites": { + "afterFiles": [], + "beforeFiles": [ + { + "source": "/litellm-asset-prefix/_next/:path+", + "destination": "/_next/:path+" + } + ], + "fallback": [] + }, + "sortedPages": [ + "/_app", + "/_error" + ] +};self.__BUILD_MANIFEST_CB && self.__BUILD_MANIFEST_CB() \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_clientMiddlewareManifest.js b/litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_clientMiddlewareManifest.js new file mode 100644 index 00000000000..a8acaffa33a --- /dev/null +++ b/litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_clientMiddlewareManifest.js @@ -0,0 +1 @@ +self.__MIDDLEWARE_MATCHERS = [];self.__MIDDLEWARE_MATCHERS_CB && self.__MIDDLEWARE_MATCHERS_CB() \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_ssgManifest.js b/litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_ssgManifest.js new file mode 100644 index 00000000000..5b3ff592fd4 --- /dev/null +++ b/litellm/proxy/_experimental/out/_next/static/5rDiFx0t_mOGYmV_8kSkw/_ssgManifest.js @@ -0,0 +1 @@ +self.__SSG_MANIFEST=new Set([]);self.__SSG_MANIFEST_CB&&self.__SSG_MANIFEST_CB() \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/0-3i_.uof35pm.js b/litellm/proxy/_experimental/out/_next/static/chunks/0-3i_.uof35pm.js new file mode 100644 index 00000000000..aaaafac2d13 --- /dev/null +++ b/litellm/proxy/_experimental/out/_next/static/chunks/0-3i_.uof35pm.js @@ -0,0 +1,2 @@ +(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,184163,e=>{"use strict";e.i(247167);var t=e.i(931067),r=e.i(271645);let n={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M505.7 661a8 8 0 0012.6 0l112-141.7c4.1-5.2.4-12.9-6.3-12.9h-74.1V168c0-4.4-3.6-8-8-8h-60c-4.4 0-8 3.6-8 8v338.3H400c-6.7 0-10.4 7.7-6.3 12.9l112 141.8zM878 626h-60c-4.4 0-8 3.6-8 8v154H214V634c0-4.4-3.6-8-8-8h-60c-4.4 0-8 3.6-8 8v198c0 17.7 14.3 32 32 32h684c17.7 0 32-14.3 32-32V634c0-4.4-3.6-8-8-8z"}}]},name:"download",theme:"outlined"};var i=e.i(9583),s=r.forwardRef(function(e,s){return r.createElement(i.default,(0,t.default)({},e,{ref:s,icon:n}))});e.s(["default",0,s],184163)},309821,e=>{"use strict";e.i(247167);var t=e.i(271645),r=e.i(135551),n=e.i(201072),i=e.i(121229),s=e.i(726289),a=e.i(864517),l=e.i(343794),o=e.i(529681),c=e.i(242064),u=e.i(931067),d=e.i(209428),m=e.i(703923),f={percent:0,prefixCls:"rc-progress",strokeColor:"#2db7f5",strokeLinecap:"round",strokeWidth:1,trailColor:"#D9D9D9",trailWidth:1,gapPosition:"bottom"},h=function(){var e=(0,t.useRef)([]),r=(0,t.useRef)(null);return(0,t.useEffect)(function(){var t=Date.now(),n=!1;e.current.forEach(function(e){if(e){n=!0;var i=e.style;i.transitionDuration=".3s, .3s, .3s, .06s",r.current&&t-r.current<100&&(i.transitionDuration="0s, 0s")}}),n&&(r.current=Date.now())}),e.current},p=e.i(410160),g=e.i(392221),x=e.i(654310),v=0,y=(0,x.default)();let b=function(e){var r=t.useState(),n=(0,g.default)(r,2),i=n[0],s=n[1];return t.useEffect(function(){var e;s("rc_progress_".concat((y?(e=v,v+=1):e="TEST_OR_SSR",e)))},[]),e||i};var _=function(e){var r=e.bg,n=e.children;return t.createElement("div",{style:{width:"100%",height:"100%",background:r}},n)};function j(e,t){return Object.keys(e).map(function(r){var n=parseFloat(r),i="".concat(Math.floor(n*t),"%");return"".concat(e[r]," ").concat(i)})}var w=t.forwardRef(function(e,r){var n=e.prefixCls,i=e.color,s=e.gradientId,a=e.radius,l=e.style,o=e.ptg,c=e.strokeLinecap,u=e.strokeWidth,d=e.size,m=e.gapDegree,f=i&&"object"===(0,p.default)(i),h=d/2,g=t.createElement("circle",{className:"".concat(n,"-circle-path"),r:a,cx:h,cy:h,stroke:f?"#FFF":void 0,strokeLinecap:c,strokeWidth:u,opacity:+(0!==o),style:l,ref:r});if(!f)return g;var x="".concat(s,"-conic"),v=j(i,(360-m)/360),y=j(i,1),b="conic-gradient(from ".concat(m?"".concat(180+m/2,"deg"):"0deg",", ").concat(v.join(", "),")"),w="linear-gradient(to ".concat(m?"bottom":"top",", ").concat(y.join(", "),")");return t.createElement(t.Fragment,null,t.createElement("mask",{id:x},g),t.createElement("foreignObject",{x:0,y:0,width:d,height:d,mask:"url(#".concat(x,")")},t.createElement(_,{bg:w},t.createElement(_,{bg:b}))))}),k=function(e,t,r,n,i,s,a,l,o,c){var u=arguments.length>10&&void 0!==arguments[10]?arguments[10]:0,d=(100-n)/100*t;return"round"===o&&100!==n&&(d+=c/2)>=t&&(d=t-.01),{stroke:"string"==typeof l?l:void 0,strokeDasharray:"".concat(t,"px ").concat(e),strokeDashoffset:d+u,transform:"rotate(".concat(i+r/100*360*((360-s)/360)+(0===s?0:({bottom:0,top:180,left:90,right:-90})[a]),"deg)"),transformOrigin:"".concat(50,"px ").concat(50,"px"),transition:"stroke-dashoffset .3s ease 0s, stroke-dasharray .3s ease 0s, stroke .3s, stroke-width .06s ease .3s, opacity .3s ease 0s",fillOpacity:0}},C=["id","prefixCls","steps","strokeWidth","trailWidth","gapDegree","gapPosition","trailColor","strokeLinecap","style","className","strokeColor","percent"];function S(e){var t=null!=e?e:[];return Array.isArray(t)?t:[t]}let E=function(e){var r,n,i,s,a=(0,d.default)((0,d.default)({},f),e),o=a.id,c=a.prefixCls,g=a.steps,x=a.strokeWidth,v=a.trailWidth,y=a.gapDegree,_=void 0===y?0:y,j=a.gapPosition,E=a.trailColor,O=a.strokeLinecap,N=a.style,I=a.className,T=a.strokeColor,R=a.percent,P=(0,m.default)(a,C),D=b(o),$="".concat(D,"-gradient"),A=50-x/2,F=2*Math.PI*A,L=_>0?90+_/2:-90,M=(360-_)/360*F,B="object"===(0,p.default)(g)?g:{count:g,gap:2},U=B.count,z=B.gap,V=S(R),H=S(T),W=H.find(function(e){return e&&"object"===(0,p.default)(e)}),K=W&&"object"===(0,p.default)(W)?"butt":O,q=k(F,M,0,100,L,_,j,E,K,x),X=h();return t.createElement("svg",(0,u.default)({className:(0,l.default)("".concat(c,"-circle"),I),viewBox:"0 0 ".concat(100," ").concat(100),style:N,id:o,role:"presentation"},P),!U&&t.createElement("circle",{className:"".concat(c,"-circle-trail"),r:A,cx:50,cy:50,stroke:E,strokeLinecap:K,strokeWidth:v||x,style:q}),U?(r=Math.round(U*(V[0]/100)),n=100/U,i=0,Array(U).fill(null).map(function(e,s){var a=s<=r-1?H[0]:E,l=a&&"object"===(0,p.default)(a)?"url(#".concat($,")"):void 0,o=k(F,M,i,n,L,_,j,a,"butt",x,z);return i+=(M-o.strokeDashoffset+z)*100/M,t.createElement("circle",{key:s,className:"".concat(c,"-circle-path"),r:A,cx:50,cy:50,stroke:l,strokeWidth:x,opacity:1,style:o,ref:function(e){X[s]=e}})})):(s=0,V.map(function(e,r){var n=H[r]||H[H.length-1],i=k(F,M,s,e,L,_,j,n,K,x);return s+=e,t.createElement(w,{key:r,color:n,ptg:e,radius:A,prefixCls:c,gradientId:$,style:i,strokeLinecap:K,strokeWidth:x,gapDegree:_,ref:function(e){X[r]=e},size:100})}).reverse()))};var O=e.i(491816);e.i(765846);var N=e.i(896091);function I(e){return!e||e<0?0:e>100?100:e}function T({success:e,successPercent:t}){let r=t;return e&&"progress"in e&&(r=e.progress),e&&"percent"in e&&(r=e.percent),r}let R=(e,t,r)=>{var n,i,s,a;let l=-1,o=-1;if("step"===t){let t=r.steps,n=r.strokeWidth;"string"==typeof e||void 0===e?(l="small"===e?2:14,o=null!=n?n:8):"number"==typeof e?[l,o]=[e,e]:[l=14,o=8]=Array.isArray(e)?e:[e.width,e.height],l*=t}else if("line"===t){let t=null==r?void 0:r.strokeWidth;"string"==typeof e||void 0===e?o=t||("small"===e?6:8):"number"==typeof e?[l,o]=[e,e]:[l=-1,o=8]=Array.isArray(e)?e:[e.width,e.height]}else("circle"===t||"dashboard"===t)&&("string"==typeof e||void 0===e?[l,o]="small"===e?[60,60]:[120,120]:"number"==typeof e?[l,o]=[e,e]:Array.isArray(e)&&(l=null!=(i=null!=(n=e[0])?n:e[1])?i:120,o=null!=(a=null!=(s=e[0])?s:e[1])?a:120));return[l,o]},P=e=>{let{prefixCls:r,trailColor:n=null,strokeLinecap:i="round",gapPosition:s,gapDegree:a,width:o=120,type:c,children:u,success:d,size:m=o,steps:f}=e,[h,p]=R(m,"circle"),{strokeWidth:g}=e;void 0===g&&(g=Math.max(3/h*100,6));let x=t.useMemo(()=>a||0===a?a:"dashboard"===c?75:void 0,[a,c]),v=(({percent:e,success:t,successPercent:r})=>{let n=I(T({success:t,successPercent:r}));return[n,I(I(e)-n)]})(e),y="[object Object]"===Object.prototype.toString.call(e.strokeColor),b=(({success:e={},strokeColor:t})=>{let{strokeColor:r}=e;return[r||N.presetPrimaryColors.green,t||null]})({success:d,strokeColor:e.strokeColor}),_=(0,l.default)(`${r}-inner`,{[`${r}-circle-gradient`]:y}),j=t.createElement(E,{steps:f,percent:f?v[1]:v,strokeWidth:g,trailWidth:g,strokeColor:f?b[1]:b,strokeLinecap:i,trailColor:n,prefixCls:r,gapDegree:x,gapPosition:s||"dashboard"===c&&"bottom"||void 0}),w=h<=20,k=t.createElement("div",{className:_,style:{width:h,height:p,fontSize:.15*h+6}},j,!w&&u);return w?t.createElement(O.default,{title:u},k):k};e.i(296059);var D=e.i(694758),$=e.i(915654),A=e.i(183293),F=e.i(246422),L=e.i(838378);let M="--progress-line-stroke-color",B="--progress-percent",U=e=>{let t=e?"100%":"-100%";return new D.Keyframes(`antProgress${e?"RTL":"LTR"}Active`,{"0%":{transform:`translateX(${t}) scaleX(0)`,opacity:.1},"20%":{transform:`translateX(${t}) scaleX(0)`,opacity:.5},to:{transform:"translateX(0) scaleX(1)",opacity:0}})},z=(0,F.genStyleHooks)("Progress",e=>{let t=e.calc(e.marginXXS).div(2).equal(),r=(0,L.mergeToken)(e,{progressStepMarginInlineEnd:t,progressStepMinWidth:t,progressActiveMotionDuration:"2.4s"});return[(e=>{let{componentCls:t,iconCls:r}=e;return{[t]:Object.assign(Object.assign({},(0,A.resetComponent)(e)),{display:"inline-block","&-rtl":{direction:"rtl"},"&-line":{position:"relative",width:"100%",fontSize:e.fontSize},[`${t}-outer`]:{display:"inline-flex",alignItems:"center",width:"100%"},[`${t}-inner`]:{position:"relative",display:"inline-block",width:"100%",flex:1,overflow:"hidden",verticalAlign:"middle",backgroundColor:e.remainingColor,borderRadius:e.lineBorderRadius},[`${t}-inner:not(${t}-circle-gradient)`]:{[`${t}-circle-path`]:{stroke:e.defaultColor}},[`${t}-success-bg, ${t}-bg`]:{position:"relative",background:e.defaultColor,borderRadius:e.lineBorderRadius,transition:`all ${e.motionDurationSlow} ${e.motionEaseInOutCirc}`},[`${t}-layout-bottom`]:{display:"flex",flexDirection:"column",alignItems:"center",justifyContent:"center",[`${t}-text`]:{width:"max-content",marginInlineStart:0,marginTop:e.marginXXS}},[`${t}-bg`]:{overflow:"hidden","&::after":{content:'""',background:{_multi_value_:!0,value:["inherit",`var(${M})`]},height:"100%",width:`calc(1 / var(${B}) * 100%)`,display:"block"},[`&${t}-bg-inner`]:{minWidth:"max-content","&::after":{content:"none"},[`${t}-text-inner`]:{color:e.colorWhite,[`&${t}-text-bright`]:{color:"rgba(0, 0, 0, 0.45)"}}}},[`${t}-success-bg`]:{position:"absolute",insetBlockStart:0,insetInlineStart:0,backgroundColor:e.colorSuccess},[`${t}-text`]:{display:"inline-block",marginInlineStart:e.marginXS,color:e.colorText,lineHeight:1,width:"2em",whiteSpace:"nowrap",textAlign:"start",verticalAlign:"middle",wordBreak:"normal",[r]:{fontSize:e.fontSize},[`&${t}-text-outer`]:{width:"max-content"},[`&${t}-text-outer${t}-text-start`]:{width:"max-content",marginInlineStart:0,marginInlineEnd:e.marginXS}},[`${t}-text-inner`]:{display:"flex",justifyContent:"center",alignItems:"center",width:"100%",height:"100%",marginInlineStart:0,padding:`0 ${(0,$.unit)(e.paddingXXS)}`,[`&${t}-text-start`]:{justifyContent:"start"},[`&${t}-text-end`]:{justifyContent:"end"}},[`&${t}-status-active`]:{[`${t}-bg::before`]:{position:"absolute",inset:0,backgroundColor:e.colorBgContainer,borderRadius:e.lineBorderRadius,opacity:0,animationName:U(),animationDuration:e.progressActiveMotionDuration,animationTimingFunction:e.motionEaseOutQuint,animationIterationCount:"infinite",content:'""'}},[`&${t}-rtl${t}-status-active`]:{[`${t}-bg::before`]:{animationName:U(!0)}},[`&${t}-status-exception`]:{[`${t}-bg`]:{backgroundColor:e.colorError},[`${t}-text`]:{color:e.colorError}},[`&${t}-status-exception ${t}-inner:not(${t}-circle-gradient)`]:{[`${t}-circle-path`]:{stroke:e.colorError}},[`&${t}-status-success`]:{[`${t}-bg`]:{backgroundColor:e.colorSuccess},[`${t}-text`]:{color:e.colorSuccess}},[`&${t}-status-success ${t}-inner:not(${t}-circle-gradient)`]:{[`${t}-circle-path`]:{stroke:e.colorSuccess}}})}})(r),(e=>{let{componentCls:t,iconCls:r}=e;return{[t]:{[`${t}-circle-trail`]:{stroke:e.remainingColor},[`&${t}-circle ${t}-inner`]:{position:"relative",lineHeight:1,backgroundColor:"transparent"},[`&${t}-circle ${t}-text`]:{position:"absolute",insetBlockStart:"50%",insetInlineStart:0,width:"100%",margin:0,padding:0,color:e.circleTextColor,fontSize:e.circleTextFontSize,lineHeight:1,whiteSpace:"normal",textAlign:"center",transform:"translateY(-50%)",[r]:{fontSize:e.circleIconFontSize}},[`${t}-circle&-status-exception`]:{[`${t}-text`]:{color:e.colorError}},[`${t}-circle&-status-success`]:{[`${t}-text`]:{color:e.colorSuccess}}},[`${t}-inline-circle`]:{lineHeight:1,[`${t}-inner`]:{verticalAlign:"bottom"}}}})(r),(e=>{let{componentCls:t}=e;return{[t]:{[`${t}-steps`]:{display:"inline-block","&-outer":{display:"flex",flexDirection:"row",alignItems:"center"},"&-item":{flexShrink:0,minWidth:e.progressStepMinWidth,marginInlineEnd:e.progressStepMarginInlineEnd,backgroundColor:e.remainingColor,transition:`all ${e.motionDurationSlow}`,"&-active":{backgroundColor:e.defaultColor}}}}}})(r),(e=>{let{componentCls:t,iconCls:r}=e;return{[t]:{[`${t}-small&-line, ${t}-small&-line ${t}-text ${r}`]:{fontSize:e.fontSizeSM}}}})(r)]},e=>({circleTextColor:e.colorText,defaultColor:e.colorInfo,remainingColor:e.colorFillSecondary,lineBorderRadius:100,circleTextFontSize:"1em",circleIconFontSize:`${e.fontSize/e.fontSizeSM}em`}));var V=function(e,t){var r={};for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&0>t.indexOf(n)&&(r[n]=e[n]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var i=0,n=Object.getOwnPropertySymbols(e);it.indexOf(n[i])&&Object.prototype.propertyIsEnumerable.call(e,n[i])&&(r[n[i]]=e[n[i]]);return r};let H=e=>{let{prefixCls:r,direction:n,percent:i,size:s,strokeWidth:a,strokeColor:o,strokeLinecap:c="round",children:u,trailColor:d=null,percentPosition:m,success:f}=e,{align:h,type:p}=m,g=o&&"string"!=typeof o?((e,t)=>{let{from:r=N.presetPrimaryColors.blue,to:n=N.presetPrimaryColors.blue,direction:i="rtl"===t?"to left":"to right"}=e,s=V(e,["from","to","direction"]);if(0!==Object.keys(s).length){let e,t=(e=[],Object.keys(s).forEach(t=>{let r=Number.parseFloat(t.replace(/%/g,""));Number.isNaN(r)||e.push({key:r,value:s[t]})}),(e=e.sort((e,t)=>e.key-t.key)).map(({key:e,value:t})=>`${t} ${e}%`).join(", ")),r=`linear-gradient(${i}, ${t})`;return{background:r,[M]:r}}let a=`linear-gradient(${i}, ${r}, ${n})`;return{background:a,[M]:a}})(o,n):{[M]:o,background:o},x="square"===c||"butt"===c?0:void 0,[v,y]=R(null!=s?s:[-1,a||("small"===s?6:8)],"line",{strokeWidth:a}),b=Object.assign(Object.assign({width:`${I(i)}%`,height:y,borderRadius:x},g),{[B]:I(i)/100}),_=T(e),j={width:`${I(_)}%`,height:y,borderRadius:x,backgroundColor:null==f?void 0:f.strokeColor},w=t.createElement("div",{className:`${r}-inner`,style:{backgroundColor:d||void 0,borderRadius:x}},t.createElement("div",{className:(0,l.default)(`${r}-bg`,`${r}-bg-${p}`),style:b},"inner"===p&&u),void 0!==_&&t.createElement("div",{className:`${r}-success-bg`,style:j})),k="outer"===p&&"start"===h,C="outer"===p&&"end"===h;return"outer"===p&&"center"===h?t.createElement("div",{className:`${r}-layout-bottom`},w,u):t.createElement("div",{className:`${r}-outer`,style:{width:v<0?"100%":v}},k&&u,w,C&&u)},W=e=>{let{size:r,steps:n,rounding:i=Math.round,percent:s=0,strokeWidth:a=8,strokeColor:o,trailColor:c=null,prefixCls:u,children:d}=e,m=i(s/100*n),[f,h]=R(null!=r?r:["small"===r?2:14,a],"step",{steps:n,strokeWidth:a}),p=f/n,g=Array.from({length:n});for(let e=0;et.indexOf(n)&&(r[n]=e[n]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var i=0,n=Object.getOwnPropertySymbols(e);it.indexOf(n[i])&&Object.prototype.propertyIsEnumerable.call(e,n[i])&&(r[n[i]]=e[n[i]]);return r};let q=["normal","exception","active","success"],X=t.forwardRef((e,u)=>{let d,{prefixCls:m,className:f,rootClassName:h,steps:p,strokeColor:g,percent:x=0,size:v="default",showInfo:y=!0,type:b="line",status:_,format:j,style:w,percentPosition:k={}}=e,C=K(e,["prefixCls","className","rootClassName","steps","strokeColor","percent","size","showInfo","type","status","format","style","percentPosition"]),{align:S="end",type:E="outer"}=k,O=Array.isArray(g)?g[0]:g,N="string"==typeof g||Array.isArray(g)?g:void 0,D=t.useMemo(()=>{if(O){let e="string"==typeof O?O:Object.values(O)[0];return new r.FastColor(e).isLight()}return!1},[g]),$=t.useMemo(()=>{var t,r;let n=T(e);return Number.parseInt(void 0!==n?null==(t=null!=n?n:0)?void 0:t.toString():null==(r=null!=x?x:0)?void 0:r.toString(),10)},[x,e.success,e.successPercent]),A=t.useMemo(()=>!q.includes(_)&&$>=100?"success":_||"normal",[_,$]),{getPrefixCls:F,direction:L,progress:M}=t.useContext(c.ConfigContext),B=F("progress",m),[U,V,X]=z(B),Q="line"===b,J=Q&&!p,Y=t.useMemo(()=>{let r;if(!y)return null;let o=T(e),c=j||(e=>`${e}%`),u=Q&&D&&"inner"===E;return"inner"===E||j||"exception"!==A&&"success"!==A?r=c(I(x),I(o)):"exception"===A?r=Q?t.createElement(s.default,null):t.createElement(a.default,null):"success"===A&&(r=Q?t.createElement(n.default,null):t.createElement(i.default,null)),t.createElement("span",{className:(0,l.default)(`${B}-text`,{[`${B}-text-bright`]:u,[`${B}-text-${S}`]:J,[`${B}-text-${E}`]:J}),title:"string"==typeof r?r:void 0},r)},[y,x,$,A,b,B,j]);"line"===b?d=p?t.createElement(W,Object.assign({},e,{strokeColor:N,prefixCls:B,steps:"object"==typeof p?p.count:p}),Y):t.createElement(H,Object.assign({},e,{strokeColor:O,prefixCls:B,direction:L,percentPosition:{align:S,type:E}}),Y):("circle"===b||"dashboard"===b)&&(d=t.createElement(P,Object.assign({},e,{strokeColor:O,prefixCls:B,progressStatus:A}),Y));let G=(0,l.default)(B,`${B}-status-${A}`,{[`${B}-${"dashboard"===b&&"circle"||b}`]:"line"!==b,[`${B}-inline-circle`]:"circle"===b&&R(v,"circle")[0]<=20,[`${B}-line`]:J,[`${B}-line-align-${S}`]:J,[`${B}-line-position-${E}`]:J,[`${B}-steps`]:p,[`${B}-show-info`]:y,[`${B}-${v}`]:"string"==typeof v,[`${B}-rtl`]:"rtl"===L},null==M?void 0:M.className,f,h,V,X);return U(t.createElement("div",Object.assign({ref:u,style:Object.assign(Object.assign({},null==M?void 0:M.style),w),className:G,role:"progressbar","aria-valuenow":$,"aria-valuemin":0,"aria-valuemax":100},(0,o.default)(C,["trailColor","strokeWidth","width","gapDegree","gapPosition","strokeLinecap","success","successPercent"])),d))});e.s(["default",0,X],309821)},519756,e=>{"use strict";e.i(247167);var t=e.i(931067),r=e.i(271645);let n={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M400 317.7h73.9V656c0 4.4 3.6 8 8 8h60c4.4 0 8-3.6 8-8V317.7H624c6.7 0 10.4-7.7 6.3-12.9L518.3 163a8 8 0 00-12.6 0l-112 141.7c-4.1 5.3-.4 13 6.3 13zM878 626h-60c-4.4 0-8 3.6-8 8v154H214V634c0-4.4-3.6-8-8-8h-60c-4.4 0-8 3.6-8 8v198c0 17.7 14.3 32 32 32h684c17.7 0 32-14.3 32-32V634c0-4.4-3.6-8-8-8z"}}]},name:"upload",theme:"outlined"};var i=e.i(9583),s=r.forwardRef(function(e,s){return r.createElement(i.default,(0,t.default)({},e,{ref:s,icon:n}))});e.s(["UploadOutlined",0,s],519756)},233538,e=>{"use strict";e.s(["isDisabledReactIssue7711",0,function(e){let t=e.parentElement,r=null;for(;t&&!(t instanceof HTMLFieldSetElement);)t instanceof HTMLLegendElement&&(r=t),t=t.parentElement;let n=(null==t?void 0:t.getAttribute("disabled"))==="";return!(n&&function(e){if(!e)return!1;let t=e.previousElementSibling;for(;null!==t;){if(t instanceof HTMLLegendElement)return!1;t=t.previousElementSibling}return!0}(r))&&n}])},83733,233137,e=>{"use strict";let t,r;var n,i,s=e.i(247167),a=e.i(271645),l=e.i(544508),o=e.i(746725),c=e.i(835696);void 0!==s.default&&"u">typeof globalThis&&"u">typeof Element&&(null==(n=null==s.default?void 0:s.default.env)?void 0:n.NODE_ENV)==="test"&&void 0===(null==(i=null==Element?void 0:Element.prototype)?void 0:i.getAnimations)&&(Element.prototype.getAnimations=function(){return console.warn(["Headless UI has polyfilled `Element.prototype.getAnimations` for your tests.","Please install a proper polyfill e.g. `jsdom-testing-mocks`, to silence these warnings.","","Example usage:","```js","import { mockAnimationsApi } from 'jsdom-testing-mocks'","mockAnimationsApi()","```"].join(` +`)),[]});var u=((t=u||{})[t.None=0]="None",t[t.Closed=1]="Closed",t[t.Enter=2]="Enter",t[t.Leave=4]="Leave",t);e.s(["transitionDataAttributes",0,function(e){let t={};for(let r in e)!0===e[r]&&(t[`data-${r}`]="");return t},"useTransition",0,function(e,t,r,n){let[i,s]=(0,a.useState)(r),{hasFlag:u,addFlag:d,removeFlag:m}=function(e=0){let[t,r]=(0,a.useState)(e),n=(0,a.useCallback)(e=>r(e),[t]),i=(0,a.useCallback)(e=>r(t=>t|e),[t]),s=(0,a.useCallback)(e=>(t&e)===e,[t]);return{flags:t,setFlag:n,addFlag:i,hasFlag:s,removeFlag:(0,a.useCallback)(e=>r(t=>t&~e),[r]),toggleFlag:(0,a.useCallback)(e=>r(t=>t^e),[r])}}(e&&i?3:0),f=(0,a.useRef)(!1),h=(0,a.useRef)(!1),p=(0,o.useDisposables)();return(0,c.useIsoMorphicEffect)(()=>{var i;if(e){if(r&&s(!0),!t){r&&d(3);return}return null==(i=null==n?void 0:n.start)||i.call(n,r),function(e,{prepare:t,run:r,done:n,inFlight:i}){let s=(0,l.disposables)();return function(e,{inFlight:t,prepare:r}){if(null!=t&&t.current)return r();let n=e.style.transition;e.style.transition="none",r(),e.offsetHeight,e.style.transition=n}(e,{prepare:t,inFlight:i}),s.nextFrame(()=>{r(),s.requestAnimationFrame(()=>{s.add(function(e,t){var r,n;let i=(0,l.disposables)();if(!e)return i.dispose;let s=!1;i.add(()=>{s=!0});let a=null!=(n=null==(r=e.getAnimations)?void 0:r.call(e).filter(e=>e instanceof CSSTransition))?n:[];return 0===a.length?t():Promise.allSettled(a.map(e=>e.finished)).then(()=>{s||t()}),i.dispose}(e,n))})}),s.dispose}(t,{inFlight:f,prepare(){h.current?h.current=!1:h.current=f.current,f.current=!0,h.current||(r?(d(3),m(4)):(d(4),m(2)))},run(){h.current?r?(m(3),d(4)):(m(4),d(3)):r?m(1):d(1)},done(){var e;h.current&&"function"==typeof t.getAnimations&&t.getAnimations().length>0||(f.current=!1,m(7),r||s(!1),null==(e=null==n?void 0:n.end)||e.call(n,r))}})}},[e,r,t,p]),e?[i,{closed:u(1),enter:u(2),leave:u(4),transition:u(2)||u(4)}]:[r,{closed:void 0,enter:void 0,leave:void 0,transition:void 0}]}],83733);let d=(0,a.createContext)(null);d.displayName="OpenClosedContext";var m=((r=m||{})[r.Open=1]="Open",r[r.Closed=2]="Closed",r[r.Closing=4]="Closing",r[r.Opening=8]="Opening",r);e.s(["OpenClosedProvider",0,function({value:e,children:t}){return a.default.createElement(d.Provider,{value:e},t)},"ResetOpenClosedProvider",0,function({children:e}){return a.default.createElement(d.Provider,{value:null},e)},"State",0,m,"useOpenClosed",0,function(){return(0,a.useContext)(d)}],233137)},677667,674175,886148,543086,e=>{"use strict";let t,r;var n,i=e.i(290571),s=e.i(783222),a=e.i(433336),l=e.i(271645),o=e.i(394487),c=e.i(914189),u=e.i(144279),d=e.i(294316),m=e.i(83733);let f=(0,l.createContext)(()=>{});function h({value:e,children:t}){return l.default.createElement(f.Provider,{value:e},t)}e.s(["CloseProvider",0,h],674175);var p=e.i(233137),g=e.i(233538),x=e.i(397701),v=e.i(402155),y=e.i(700020);let b=null!=(n=l.default.startTransition)?n:function(e){e()};var _=e.i(998348),j=((t=j||{})[t.Open=0]="Open",t[t.Closed=1]="Closed",t),w=((r=w||{})[r.ToggleDisclosure=0]="ToggleDisclosure",r[r.CloseDisclosure=1]="CloseDisclosure",r[r.SetButtonId=2]="SetButtonId",r[r.SetPanelId=3]="SetPanelId",r[r.SetButtonElement=4]="SetButtonElement",r[r.SetPanelElement=5]="SetPanelElement",r);let k={0:e=>({...e,disclosureState:(0,x.match)(e.disclosureState,{0:1,1:0})}),1:e=>1===e.disclosureState?e:{...e,disclosureState:1},2:(e,t)=>e.buttonId===t.buttonId?e:{...e,buttonId:t.buttonId},3:(e,t)=>e.panelId===t.panelId?e:{...e,panelId:t.panelId},4:(e,t)=>e.buttonElement===t.element?e:{...e,buttonElement:t.element},5:(e,t)=>e.panelElement===t.element?e:{...e,panelElement:t.element}},C=(0,l.createContext)(null);function S(e){let t=(0,l.useContext)(C);if(null===t){let t=Error(`<${e} /> is missing a parent component.`);throw Error.captureStackTrace&&Error.captureStackTrace(t,S),t}return t}C.displayName="DisclosureContext";let E=(0,l.createContext)(null);E.displayName="DisclosureAPIContext";let O=(0,l.createContext)(null);function N(e,t){return(0,x.match)(t.type,k,e,t)}O.displayName="DisclosurePanelContext";let I=l.Fragment,T=y.RenderFeatures.RenderStrategy|y.RenderFeatures.Static,R=Object.assign((0,y.forwardRefWithAs)(function(e,t){let{defaultOpen:r=!1,...n}=e,i=(0,l.useRef)(null),s=(0,d.useSyncRefs)(t,(0,d.optionalRef)(e=>{i.current=e},void 0===e.as||e.as===l.Fragment)),a=(0,l.useReducer)(N,{disclosureState:+!r,buttonElement:null,panelElement:null,buttonId:null,panelId:null}),[{disclosureState:o,buttonId:u},m]=a,f=(0,c.useEvent)(e=>{m({type:1});let t=(0,v.getOwnerDocument)(i);if(!t||!u)return;let r=e?e instanceof HTMLElement?e:e.current instanceof HTMLElement?e.current:t.getElementById(u):t.getElementById(u);null==r||r.focus()}),g=(0,l.useMemo)(()=>({close:f}),[f]),b=(0,l.useMemo)(()=>({open:0===o,close:f}),[o,f]),_=(0,y.useRender)();return l.default.createElement(C.Provider,{value:a},l.default.createElement(E.Provider,{value:g},l.default.createElement(h,{value:f},l.default.createElement(p.OpenClosedProvider,{value:(0,x.match)(o,{0:p.State.Open,1:p.State.Closed})},_({ourProps:{ref:s},theirProps:n,slot:b,defaultTag:I,name:"Disclosure"})))))}),{Button:(0,y.forwardRefWithAs)(function(e,t){let r=(0,l.useId)(),{id:n=`headlessui-disclosure-button-${r}`,disabled:i=!1,autoFocus:m=!1,...f}=e,[h,p]=S("Disclosure.Button"),x=(0,l.useContext)(O),v=null!==x&&x===h.panelId,b=(0,l.useRef)(null),j=(0,d.useSyncRefs)(b,t,(0,c.useEvent)(e=>{if(!v)return p({type:4,element:e})}));(0,l.useEffect)(()=>{if(!v)return p({type:2,buttonId:n}),()=>{p({type:2,buttonId:null})}},[n,p,v]);let w=(0,c.useEvent)(e=>{var t;if(v){if(1===h.disclosureState)return;switch(e.key){case _.Keys.Space:case _.Keys.Enter:e.preventDefault(),e.stopPropagation(),p({type:0}),null==(t=h.buttonElement)||t.focus()}}else switch(e.key){case _.Keys.Space:case _.Keys.Enter:e.preventDefault(),e.stopPropagation(),p({type:0})}}),k=(0,c.useEvent)(e=>{e.key===_.Keys.Space&&e.preventDefault()}),C=(0,c.useEvent)(e=>{var t;(0,g.isDisabledReactIssue7711)(e.currentTarget)||i||(v?(p({type:0}),null==(t=h.buttonElement)||t.focus()):p({type:0}))}),{isFocusVisible:E,focusProps:N}=(0,s.useFocusRing)({autoFocus:m}),{isHovered:I,hoverProps:T}=(0,a.useHover)({isDisabled:i}),{pressed:R,pressProps:P}=(0,o.useActivePress)({disabled:i}),D=(0,l.useMemo)(()=>({open:0===h.disclosureState,hover:I,active:R,disabled:i,focus:E,autofocus:m}),[h,I,R,E,i,m]),$=(0,u.useResolveButtonType)(e,h.buttonElement),A=v?(0,y.mergeProps)({ref:j,type:$,disabled:i||void 0,autoFocus:m,onKeyDown:w,onClick:C},N,T,P):(0,y.mergeProps)({ref:j,id:n,type:$,"aria-expanded":0===h.disclosureState,"aria-controls":h.panelElement?h.panelId:void 0,disabled:i||void 0,autoFocus:m,onKeyDown:w,onKeyUp:k,onClick:C},N,T,P);return(0,y.useRender)()({ourProps:A,theirProps:f,slot:D,defaultTag:"button",name:"Disclosure.Button"})}),Panel:(0,y.forwardRefWithAs)(function(e,t){let r=(0,l.useId)(),{id:n=`headlessui-disclosure-panel-${r}`,transition:i=!1,...s}=e,[a,o]=S("Disclosure.Panel"),{close:u}=function e(t){let r=(0,l.useContext)(E);if(null===r){let r=Error(`<${t} /> is missing a parent component.`);throw Error.captureStackTrace&&Error.captureStackTrace(r,e),r}return r}("Disclosure.Panel"),[f,h]=(0,l.useState)(null),g=(0,d.useSyncRefs)(t,(0,c.useEvent)(e=>{b(()=>o({type:5,element:e}))}),h);(0,l.useEffect)(()=>(o({type:3,panelId:n}),()=>{o({type:3,panelId:null})}),[n,o]);let x=(0,p.useOpenClosed)(),[v,_]=(0,m.useTransition)(i,f,null!==x?(x&p.State.Open)===p.State.Open:0===a.disclosureState),j=(0,l.useMemo)(()=>({open:0===a.disclosureState,close:u}),[a.disclosureState,u]),w={ref:g,id:n,...(0,m.transitionDataAttributes)(_)},k=(0,y.useRender)();return l.default.createElement(p.ResetOpenClosedProvider,null,l.default.createElement(O.Provider,{value:a.panelId},k({ourProps:w,theirProps:s,slot:j,defaultTag:"div",features:T,visible:v,name:"Disclosure.Panel"})))})});e.s(["Disclosure",0,R],886148);let P=(0,l.createContext)(void 0);var D=e.i(444755);let $=(0,e.i(673706).makeClassName)("Accordion"),A=(0,l.createContext)({isOpen:!1}),F=l.default.forwardRef((e,t)=>{var r;let{defaultOpen:n=!1,children:s,className:a}=e,o=(0,i.__rest)(e,["defaultOpen","children","className"]),c=null!=(r=(0,l.useContext)(P))?r:(0,D.tremorTwMerge)("rounded-tremor-default border");return l.default.createElement(R,Object.assign({as:"div",ref:t,className:(0,D.tremorTwMerge)($("root"),"overflow-hidden","bg-tremor-background border-tremor-border","dark:bg-dark-tremor-background dark:border-dark-tremor-border",c,a),defaultOpen:n},o),({open:e})=>l.default.createElement(A.Provider,{value:{isOpen:e}},s))});F.displayName="Accordion",e.s(["OpenContext",0,A,"default",0,F],543086),e.s(["Accordion",0,F],677667)},130643,e=>{"use strict";var t=e.i(290571),r=e.i(271645),n=e.i(886148),i=e.i(444755);let s=(0,e.i(673706).makeClassName)("AccordionBody"),a=r.default.forwardRef((e,a)=>{let{children:l,className:o}=e,c=(0,t.__rest)(e,["children","className"]);return r.default.createElement(n.Disclosure.Panel,Object.assign({ref:a,className:(0,i.tremorTwMerge)(s("root"),"w-full text-tremor-default px-4 pb-3","text-tremor-content","dark:text-dark-tremor-content",o)},c),l)});a.displayName="AccordionBody",e.s(["AccordionBody",0,a],130643)},898667,e=>{"use strict";var t=e.i(290571),r=e.i(271645),n=e.i(886148);let i=e=>{var n=(0,t.__rest)(e,[]);return r.default.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor"},n),r.default.createElement("path",{d:"M11.9999 10.8284L7.0502 15.7782L5.63599 14.364L11.9999 8L18.3639 14.364L16.9497 15.7782L11.9999 10.8284Z"}))};var s=e.i(543086),a=e.i(444755);let l=(0,e.i(673706).makeClassName)("AccordionHeader"),o=r.default.forwardRef((e,o)=>{let{children:c,className:u}=e,d=(0,t.__rest)(e,["children","className"]),{isOpen:m}=(0,r.useContext)(s.OpenContext);return r.default.createElement(n.Disclosure.Button,Object.assign({ref:o,className:(0,a.tremorTwMerge)(l("root"),"w-full flex items-center justify-between px-4 py-3","text-tremor-content-emphasis","dark:text-dark-tremor-content-emphasis",u)},d),r.default.createElement("div",{className:(0,a.tremorTwMerge)(l("children"),"flex flex-1 text-inherit mr-4")},c),r.default.createElement("div",null,r.default.createElement(i,{className:(0,a.tremorTwMerge)(l("arrowIcon"),"h-5 w-5 -mr-1","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle",m?"transition-all":"transition-all -rotate-180")})))});o.displayName="AccordionHeader",e.s(["AccordionHeader",0,o],898667)},220508,e=>{"use strict";var t=e.i(271645);let r=t.forwardRef(function(e,r){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:r},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M9 12l2 2 4-4m6 2a9 9 0 11-18 0 9 9 0 0118 0z"}))});e.s(["CheckCircleIcon",0,r],220508)},503269,214520,601893,694421,140721,942803,35889,722678,e=>{"use strict";var t=e.i(271645),r=e.i(914189);e.s(["useControllable",0,function(e,n,i){let[s,a]=(0,t.useState)(i),l=void 0!==e,o=(0,t.useRef)(l),c=(0,t.useRef)(!1),u=(0,t.useRef)(!1);return!l||o.current||c.current?l||!o.current||u.current||(u.current=!0,o.current=l,console.error("A component is changing from controlled to uncontrolled. This may be caused by the value changing from a defined value to undefined, which should not happen.")):(c.current=!0,o.current=l,console.error("A component is changing from uncontrolled to controlled. This may be caused by the value changing from undefined to a defined value, which should not happen.")),[l?e:s,(0,r.useEvent)(e=>(l||a(e),null==n?void 0:n(e)))]}],503269),e.s(["useDefaultValue",0,function(e){let[r]=(0,t.useState)(e);return r}],214520);let n=(0,t.createContext)(void 0);function i(){return(0,t.useContext)(n)}e.s(["useDisabled",0,i],601893);var s=e.i(174080),a=e.i(746725);function l(e={},t=null,r=[]){for(let[n,i]of Object.entries(e))!function e(t,r,n){if(Array.isArray(n))for(let[i,s]of n.entries())e(t,o(r,i.toString()),s);else n instanceof Date?t.push([r,n.toISOString()]):"boolean"==typeof n?t.push([r,n?"1":"0"]):"string"==typeof n?t.push([r,n]):"number"==typeof n?t.push([r,`${n}`]):null==n?t.push([r,""]):l(n,r,t)}(r,o(t,n),i);return r}function o(e,t){return e?e+"["+t+"]":t}e.s(["attemptSubmit",0,function(e){var t,r;let n=null!=(t=null==e?void 0:e.form)?t:e.closest("form");if(n){for(let t of n.elements)if(t!==e&&("INPUT"===t.tagName&&"submit"===t.type||"BUTTON"===t.tagName&&"submit"===t.type||"INPUT"===t.nodeName&&"image"===t.type))return void t.click();null==(r=n.requestSubmit)||r.call(n)}},"objectToFormEntries",0,l],694421);var c=e.i(700020),u=e.i(2788);let d=(0,t.createContext)(null);function m({children:e}){let r=(0,t.useContext)(d);if(!r)return t.default.createElement(t.default.Fragment,null,e);let{target:n}=r;return n?(0,s.createPortal)(t.default.createElement(t.default.Fragment,null,e),n):null}function f({setForm:e,formId:r}){return(0,t.useEffect)(()=>{if(r){let t=document.getElementById(r);t&&e(t)}},[e,r]),r?null:t.default.createElement(u.Hidden,{features:u.HiddenFeatures.Hidden,as:"input",type:"hidden",hidden:!0,readOnly:!0,ref:t=>{if(!t)return;let r=t.closest("form");r&&e(r)}})}e.s(["FormFields",0,function({data:e,form:r,disabled:n,onReset:i,overrides:s}){let[o,d]=(0,t.useState)(null),h=(0,a.useDisposables)();return(0,t.useEffect)(()=>{if(i&&o)return h.addEventListener(o,"reset",i)},[o,r,i]),t.default.createElement(m,null,t.default.createElement(f,{setForm:d,formId:r}),l(e).map(([e,i])=>t.default.createElement(u.Hidden,{features:u.HiddenFeatures.Hidden,...(0,c.compact)({key:e,as:"input",type:"hidden",hidden:!0,readOnly:!0,form:r,disabled:n,name:e,value:i,...s})})))}],140721);let h=(0,t.createContext)(void 0);function p(){return(0,t.useContext)(h)}e.s(["useProvidedId",0,p],942803);var g=e.i(835696),x=e.i(294316);let v=(0,t.createContext)(null);v.displayName="DescriptionContext";let y=Object.assign((0,c.forwardRefWithAs)(function(e,r){let n=(0,t.useId)(),s=i(),{id:a=`headlessui-description-${n}`,...l}=e,o=function e(){let r=(0,t.useContext)(v);if(null===r){let t=Error("You used a component, but it is not inside a relevant parent.");throw Error.captureStackTrace&&Error.captureStackTrace(t,e),t}return r}(),u=(0,x.useSyncRefs)(r);(0,g.useIsoMorphicEffect)(()=>o.register(a),[a,o.register]);let d=s||!1,m=(0,t.useMemo)(()=>({...o.slot,disabled:d}),[o.slot,d]),f={ref:u,...o.props,id:a};return(0,c.useRender)()({ourProps:f,theirProps:l,slot:m,defaultTag:"p",name:o.name||"Description"})}),{});e.s(["Description",0,y,"useDescribedBy",0,function(){var e,r;return null!=(r=null==(e=(0,t.useContext)(v))?void 0:e.value)?r:void 0},"useDescriptions",0,function(){let[e,n]=(0,t.useState)([]);return[e.length>0?e.join(" "):void 0,(0,t.useMemo)(()=>function(e){let i=(0,r.useEvent)(e=>(n(t=>[...t,e]),()=>n(t=>{let r=t.slice(),n=r.indexOf(e);return -1!==n&&r.splice(n,1),r}))),s=(0,t.useMemo)(()=>({register:i,slot:e.slot,name:e.name,props:e.props,value:e.value}),[i,e.slot,e.name,e.props,e.value]);return t.default.createElement(v.Provider,{value:s},e.children)},[n])]}],35889);let b=(0,t.createContext)(null);function _(e){var r,n,i;let s=null!=(n=null==(r=(0,t.useContext)(b))?void 0:r.value)?n:void 0;return(null!=(i=null==e?void 0:e.length)?i:0)>0?[s,...e].filter(Boolean).join(" "):s}b.displayName="LabelContext";let j=Object.assign((0,c.forwardRefWithAs)(function(e,n){var s;let a=(0,t.useId)(),l=function e(){let r=(0,t.useContext)(b);if(null===r){let t=Error("You used a