From 35592323e24a3740ed2531a26d71145903ac9f2d Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 30 Jul 2026 16:12:08 -0700 Subject: [PATCH 01/50] fix(tag-management): drop unsupported prisma select kwarg from key lookup /tag/list returned HTTP 500 for every internal user with "LiteLLM_VerificationTokenActions.find_many() got an unexpected keyword argument 'select'". The non-admin branch scopes the tag list to keys owned by the caller, and that lookup passed select={"token": True}; prisma-client-py 0.11.0 has no select kwarg on find_many, so the call raised TypeError and the handler's except block turned it into a 500. Since the Admin UI calls /tag/list on load, Tags was broken for every non-admin user. /tag/daily/activity shares the same helper and was failing the same way The kwarg is dropped rather than replaced; the generated client has no projection API, and a user's key set is small enough that selecting all columns is not worth working around The reason this shipped green is that the existing test asserted the call was made with select={"token": True} against an AsyncMock, which accepts any keyword. The verification-token table double now binds each call against the real find_many signature, so an unsupported kwarg raises the same TypeError production does --- .../tag_management_endpoints.py | 2 - .../test_tag_management_endpoints.py | 109 +++++++++++++++--- 2 files changed, 91 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/management_endpoints/tag_management_endpoints.py b/litellm/proxy/management_endpoints/tag_management_endpoints.py index 47a8670e26f..ac53f254981 100644 --- a/litellm/proxy/management_endpoints/tag_management_endpoints.py +++ b/litellm/proxy/management_endpoints/tag_management_endpoints.py @@ -96,7 +96,6 @@ class _VerificationTokenTableClient(Protocol): async def find_many( self, where: Mapping[str, object] | None = None, - select: Mapping[str, object] | None = None, ) -> "Sequence[PrismaVerificationToken]": ... @@ -157,7 +156,6 @@ async def _get_internal_user_api_keys( key_records = await _table(VerificationTokenRepository(prisma_client)).find_many( where={"user_id": user_id}, - select={"token": True}, ) user_api_keys.update(key_record.token for key_record in key_records if getattr(key_record, "token", None)) diff --git a/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py index 76ba0e3dc67..9927c56b847 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py @@ -1,11 +1,13 @@ +import inspect import json import os import sys -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional import pytest from fastapi import HTTPException from fastapi.testclient import TestClient +from prisma.actions import LiteLLM_VerificationTokenActions sys.path.insert( 0, os.path.abspath("../../../..") @@ -21,6 +23,28 @@ from litellm.types.tag_management import TagDeleteRequest, TagInfoRequest, TagNe client = TestClient(app) +class FakeVerificationTokenTable: + """Stand-in for ``prisma_client.db.litellm_verificationtoken``. + + ``AsyncMock`` swallows any keyword argument, so a plain mock cannot catch a + call that the generated prisma client would reject at runtime. This double + binds every call against the real ``find_many`` signature, so passing an + unsupported kwarg (e.g. ``select``) raises the same ``TypeError`` the proxy + surfaces as an HTTP 500. + """ + + def __init__(self, records: List[Any]): + self._records = records + self.calls: List[Dict[str, Any]] = [] + + async def find_many(self, **kwargs: Any) -> List[Any]: + inspect.signature(LiteLLM_VerificationTokenActions.find_many).bind( + self, **kwargs + ) + self.calls.append(kwargs) + return self._records + + @pytest.mark.asyncio async def test_create_and_get_tag(): """ @@ -380,6 +404,7 @@ async def test_list_tags_no_dynamic_tags(): app.dependency_overrides.clear() +@pytest.mark.asyncio async def test_internal_user_list_tags_only_returns_tags_used_by_their_keys(): """ Internal users can view tag usage, but the tag list must be scoped to tags @@ -404,9 +429,8 @@ async def test_internal_user_list_tags_only_returns_tags_used_by_their_keys(): owned_key_record = Mock() owned_key_record.token = "owned-key" - mock_db.litellm_verificationtoken.find_many = AsyncMock( - return_value=[owned_key_record] - ) + fake_token_table = FakeVerificationTokenTable([owned_key_record]) + mock_db.litellm_verificationtoken = fake_token_table mock_db.litellm_dailytagspend.group_by = AsyncMock( return_value=[ @@ -446,10 +470,9 @@ async def test_internal_user_list_tags_only_returns_tags_used_by_their_keys(): "stored-owned-tag", "dynamic-owned-tag", ] - mock_db.litellm_verificationtoken.find_many.assert_awaited_once_with( - where={"user_id": "internal-user-123"}, - select={"token": True}, - ) + assert fake_token_table.calls == [ + {"where": {"user_id": "internal-user-123"}} + ] mock_db.litellm_dailytagspend.group_by.assert_awaited_once_with( by=["tag"], where={ @@ -468,6 +491,54 @@ async def test_internal_user_list_tags_only_returns_tags_used_by_their_keys(): app.dependency_overrides.clear() +@pytest.mark.asyncio +async def test_internal_user_list_tags_does_not_500_on_unsupported_prisma_kwarg(): + """ + Regression: /tag/list returned 500 for every internal user because the + non-admin branch looked up the caller's keys with + ``find_many(select={"token": True})``, and the generated prisma client has no + ``select`` kwarg. This reproduces the reported case exactly: a freshly created + internal user with no tag spend yet, which must get an empty 200 rather than + "LiteLLM_VerificationTokenActions.find_many() got an unexpected keyword + argument 'select'". + """ + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + mock_user_auth = UserAPIKeyAuth( + api_key="new-user-key", + user_id="brand-new-internal-user", + user_role=LitellmUserRoles.INTERNAL_USER, + ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + + try: + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + mock_db = Mock() + mock_prisma.db = mock_db + + key_record = Mock() + key_record.token = "new-user-key" + fake_token_table = FakeVerificationTokenTable([key_record]) + mock_db.litellm_verificationtoken = fake_token_table + + mock_db.litellm_dailytagspend.group_by = AsyncMock(return_value=[]) + mock_db.litellm_tagtable.find_many = AsyncMock(return_value=[]) + + response = client.get( + "/tag/list", headers={"Authorization": "Bearer new-user-key"} + ) + + assert response.status_code == 200, response.text + assert response.json() == [] + assert fake_token_table.calls == [ + {"where": {"user_id": "brand-new-internal-user"}} + ] + finally: + app.dependency_overrides.clear() + + @pytest.mark.asyncio async def test_list_tags_with_date_range_filters_dynamic_tags(): """ @@ -537,9 +608,8 @@ async def test_internal_user_tag_daily_activity_is_scoped_to_their_keys(): owned_key_record = Mock() owned_key_record.token = "owned-key" - mock_db.litellm_verificationtoken.find_many = AsyncMock( - return_value=[owned_key_record] - ) + fake_token_table = FakeVerificationTokenTable([owned_key_record]) + mock_db.litellm_verificationtoken = fake_token_table mock_get_daily_activity.return_value = "daily-activity-response" result = await get_tag_daily_activity( @@ -549,6 +619,7 @@ async def test_internal_user_tag_daily_activity_is_scoped_to_their_keys(): ) assert result == "daily-activity-response" + assert fake_token_table.calls == [{"where": {"user_id": "internal-user-123"}}] mock_get_daily_activity.assert_awaited_once() assert mock_get_daily_activity.await_args.kwargs["api_key"] == ["owned-key"] @@ -583,9 +654,8 @@ async def test_internal_user_tag_daily_activity_rejects_unowned_api_key_filter() owned_key_record = Mock() owned_key_record.token = "owned-key" - mock_db.litellm_verificationtoken.find_many = AsyncMock( - return_value=[owned_key_record] - ) + fake_token_table = FakeVerificationTokenTable([owned_key_record]) + mock_db.litellm_verificationtoken = fake_token_table result = await get_tag_daily_activity( start_date="2025-01-01", end_date="2025-01-31", @@ -593,6 +663,7 @@ async def test_internal_user_tag_daily_activity_rejects_unowned_api_key_filter() user_api_key_dict=mock_user_auth, ) + assert fake_token_table.calls == [{"where": {"user_id": "internal-user-123"}}] assert result.results == [] assert result.metadata.total_spend == 0 assert result.metadata.total_api_requests == 0 @@ -626,7 +697,8 @@ async def test_internal_user_tag_daily_activity_scopes_to_current_key_without_us ): mock_db = Mock() mock_prisma.db = mock_db - mock_db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + fake_token_table = FakeVerificationTokenTable([]) + mock_db.litellm_verificationtoken = fake_token_table mock_get_daily_activity.return_value = "daily-activity-response" result = await get_tag_daily_activity( @@ -636,7 +708,7 @@ async def test_internal_user_tag_daily_activity_scopes_to_current_key_without_us ) assert result == "daily-activity-response" - mock_db.litellm_verificationtoken.find_many.assert_not_awaited() + assert fake_token_table.calls == [] mock_get_daily_activity.assert_awaited_once() assert mock_get_daily_activity.await_args.kwargs["api_key"] == [ "current-owned-key" @@ -669,7 +741,8 @@ async def test_internal_user_tag_daily_activity_without_any_scoped_keys_returns_ ): mock_db = Mock() mock_prisma.db = mock_db - mock_db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + fake_token_table = FakeVerificationTokenTable([]) + mock_db.litellm_verificationtoken = fake_token_table result = await get_tag_daily_activity( start_date="2025-01-01", @@ -680,7 +753,7 @@ async def test_internal_user_tag_daily_activity_without_any_scoped_keys_returns_ assert result.results == [] assert result.metadata.total_spend == 0 assert result.metadata.total_api_requests == 0 - mock_db.litellm_verificationtoken.find_many.assert_not_awaited() + assert fake_token_table.calls == [] mock_get_daily_activity.assert_not_awaited() From 1d40f2a707a96be7b4d91587bb7ba0e834cd805d Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 30 Jul 2026 19:22:22 -0700 Subject: [PATCH 02/50] feat(ui): add sorting, filtering and search to the budgets page Move the budgets table onto the paged management list route so sorting, filtering and search happen server-side instead of over whichever rows happened to be in memory. Adds useResourceList, a generic hook that owns page, page_size, sort, q and filters for a server-driven table, folds them into one JSON:API query and returns exactly the props DataTable's server modes want. Budgets is its first consumer. The budget id column now renders in full with a copy button instead of a fixed-width cell, and the table gains Reset and Created columns. --- ui/litellm-dashboard/eslint-suppressions.json | 5 - .../budgets/_components/BudgetTable.test.tsx | 160 +++++-- .../budgets/_components/BudgetTable.tsx | 246 ++++++++++- .../_components/BudgetTableColumns.tsx | 62 ++- .../budgets/_components/budget_panel.test.tsx | 413 ++++++++++-------- .../budgets/_components/budget_panel.tsx | 31 +- .../hooks/budgets/budgetFilters.test.ts | 59 +++ .../hooks/budgets/budgetFilters.ts | 91 ++++ .../(dashboard)/hooks/budgets/useBudgets.ts | 52 ++- .../hooks/common/useResourceList.test.tsx | 161 +++++++ .../hooks/common/useResourceList.ts | 142 ++++++ 11 files changed, 1144 insertions(+), 278 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/budgets/budgetFilters.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/budgets/budgetFilters.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/common/useResourceList.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/common/useResourceList.ts diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 6819b2851f5..229d6c797c2 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -126,11 +126,6 @@ "count": 2 } }, - "src/app/(dashboard)/budgets/_components/budget_panel.test.tsx": { - "unused-imports/no-unused-imports": { - "count": 2 - } - }, "src/app/(dashboard)/budgets/_components/budget_panel.tsx": { "local/filename-pascal-case": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx index 2b97bcbc072..0c485adf4f2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx @@ -1,22 +1,58 @@ -import { screen, within } from "@testing-library/react"; +import { screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { renderWithProviders } from "@/../tests/test-utils"; +import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; import BudgetTable from "./BudgetTable"; -import { budgetItem } from "@/app/(dashboard)/hooks/budgets/useBudgets"; +import type { budgetItem } from "@/app/(dashboard)/hooks/budgets/useBudgets"; +import type { ResourceListResult } from "@/app/(dashboard)/hooks/common/useResourceList"; +import { ApiError } from "@/lib/http/client"; + +const { copyToClipboardMock } = vi.hoisted(() => ({ copyToClipboardMock: vi.fn() })); + +vi.mock("@/utils/dataUtils", async (importOriginal) => ({ + ...(await importOriginal()), + copyToClipboard: copyToClipboardMock, +})); const makeBudget = (overrides: Partial = {}): budgetItem => ({ budget_id: "budget-1", max_budget: 100, + soft_budget: null, tpm_limit: 1000, rpm_limit: 10, + budget_duration: "30d", + budget_reset_at: null, + created_at: "2024-01-01T00:00:00Z", updated_at: "2024-01-01T00:00:00Z", ...overrides, }); -const defaultProps = { - budgets: [makeBudget()], +const makeList = (overrides: Partial> = {}): ResourceListResult => ({ + rows: [makeBudget()], + rowCount: 1, isLoading: false, + isFetching: false, + error: null, + refetch: vi.fn(), + sorting: [{ id: "created_at", desc: true }], + onSortingChange: vi.fn(), + pagination: { pageIndex: 0, pageSize: 50 }, + onPaginationChange: vi.fn(), + columnFilters: [], + onColumnFiltersChange: vi.fn(), + searchValue: "", + onSearchChange: vi.fn(), + ...overrides, +}); + +const FORBIDDEN_PROBLEM = { + type: "about:blank", + title: "Forbidden", + status: 403, + detail: "Only proxy admins can view budgets", +}; + +const defaultProps = { canModify: true, onEditClick: vi.fn(), onDeleteClick: vi.fn(), @@ -25,72 +61,134 @@ const defaultProps = { describe("BudgetTable", () => { beforeEach(() => { vi.clearAllMocks(); + testQueryClient.clear(); }); it("should display budget information", () => { - renderWithProviders(); + renderWithProviders(); expect(screen.getByText("budget-1")).toBeInTheDocument(); expect(screen.getByText("$100.00")).toBeInTheDocument(); expect(screen.getByText("1000")).toBeInTheDocument(); expect(screen.getByText("10")).toBeInTheDocument(); }); - it("should render the budget id without a fixed character-count clamp", () => { + it("should render the reset column with the friendly duration label", () => { + renderWithProviders(); + expect(screen.getByText("monthly")).toBeInTheDocument(); + }); + + it("should render 'Not set' when a budget has no reset duration", () => { + const list = makeList({ rows: [makeBudget({ budget_duration: null })] }); + renderWithProviders(); + expect(screen.getByText("Not set")).toBeInTheDocument(); + }); + + it("should render the budget id in full, with no truncation", () => { const budgetId = "ecc1869c-6231-4380-a56d-1a0be457477d"; - renderWithProviders(); + const list = makeList({ rows: [makeBudget({ budget_id: budgetId })] }); + renderWithProviders(); const idCell = screen.getByText(budgetId); + expect(idCell.className).not.toContain("truncate"); expect(idCell.className).not.toMatch(/max-w-\[\d+(ch|rem|px)\]/); - expect(idCell.className).toContain("max-w-full"); - expect(idCell.className).toContain("truncate"); + }); + + it("should keep the budget id on a single line", () => { + const budgetId = "ecc1869c-6231-4380-a56d-1a0be457477d"; + const list = makeList({ rows: [makeBudget({ budget_id: budgetId })] }); + renderWithProviders(); + expect(screen.getByText(budgetId).className).toContain("whitespace-nowrap"); + }); + + it("should copy the budget id from the cell's copy button", async () => { + const user = userEvent.setup(); + const budgetId = "ecc1869c-6231-4380-a56d-1a0be457477d"; + const list = makeList({ rows: [makeBudget({ budget_id: budgetId })] }); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: "Copy ID" })); + expect(copyToClipboardMock).toHaveBeenCalledWith(budgetId); + }); + + it("should offer sorting on every backend-sortable column", async () => { + renderWithProviders(); + for (const field of ["budget_id", "max_budget", "tpm_limit", "rpm_limit", "created_at"]) { + expect(screen.getByTestId(`sort-header-${field}`)).toBeInTheDocument(); + } + }); + + it("should not make the reset column sortable", () => { + renderWithProviders(); + expect(screen.queryByTestId("sort-header-budget_duration")).not.toBeInTheDocument(); + expect(screen.getByText("Reset")).toBeInTheDocument(); + }); + + it("should ask the list for a new sort when a sortable header is clicked", async () => { + const user = userEvent.setup(); + const onSortingChange = vi.fn(); + renderWithProviders(); + await user.click(screen.getByTestId("sort-header-max_budget")); + expect(onSortingChange).toHaveBeenCalled(); }); it("should show n/a for missing rate limits and Unlimited for a missing max budget", () => { - renderWithProviders( - , - ); + const list = makeList({ rows: [makeBudget({ max_budget: null, tpm_limit: null, rpm_limit: null })] }); + renderWithProviders(); expect(screen.getAllByText("n/a")).toHaveLength(2); expect(screen.getByText("Unlimited")).toBeInTheDocument(); }); - it("should sort budgets by updated_at descending", () => { - const budgets = [ - makeBudget({ budget_id: "budget-old", updated_at: "2024-01-01T00:00:00Z" }), - makeBudget({ budget_id: "budget-new", updated_at: "2024-06-01T00:00:00Z" }), - ]; - renderWithProviders(); - const rows = screen.getAllByRole("row").slice(1); - expect(within(rows[0]).getByText("budget-new")).toBeInTheDocument(); - expect(within(rows[1]).getByText("budget-old")).toBeInTheDocument(); - }); - it("should call onEditClick from the actions menu", async () => { const user = userEvent.setup(); - renderWithProviders(); + const list = makeList(); + renderWithProviders(); await user.click(screen.getByTestId("budget-actions-budget-1")); await user.click(await screen.findByTestId("budget-action-edit")); - expect(defaultProps.onEditClick).toHaveBeenCalledWith(defaultProps.budgets[0]); + expect(defaultProps.onEditClick).toHaveBeenCalledWith(list.rows[0]); }); it("should call onDeleteClick from the actions menu", async () => { const user = userEvent.setup(); - renderWithProviders(); + const list = makeList(); + renderWithProviders(); await user.click(screen.getByTestId("budget-actions-budget-1")); await user.click(await screen.findByTestId("budget-action-delete")); - expect(defaultProps.onDeleteClick).toHaveBeenCalledWith(defaultProps.budgets[0]); + expect(defaultProps.onDeleteClick).toHaveBeenCalledWith(list.rows[0]); }); it("should not render the actions menu when the user cannot modify budgets", () => { - renderWithProviders(); + renderWithProviders(); expect(screen.queryByTestId("budget-actions-budget-1")).not.toBeInTheDocument(); }); it("should show skeleton rows when loading", () => { - renderWithProviders(); + renderWithProviders(); expect(screen.getAllByTestId("skeleton-row").length).toBeGreaterThan(0); }); it("should show the empty state when there are no budgets", () => { - renderWithProviders(); + renderWithProviders(); expect(screen.getByText("No budgets yet")).toBeInTheDocument(); }); + + it("should tell the user their search matched nothing rather than that no budgets exist", () => { + const list = makeList({ rows: [], rowCount: 0, searchValue: "nope" }); + renderWithProviders(); + expect(screen.getByText("No matching budgets")).toBeInTheDocument(); + }); + + it("should render an access-denied state for a 403 instead of an empty table", () => { + const error = new ApiError("Only proxy admins can view budgets", 403, FORBIDDEN_PROBLEM); + const list = makeList({ rows: [], rowCount: 0, error }); + const { container } = renderWithProviders(); + expect(screen.getByText("You do not have access to budgets")).toBeInTheDocument(); + expect(screen.queryByText("No budgets yet")).not.toBeInTheDocument(); + expect(container.querySelector(".lucide-shield-alert")).not.toBeNull(); + }); + + it("should surface the problem detail for a non-403 failure", () => { + const error = new ApiError("budget store unavailable", 500, null); + const list = makeList({ rows: [], rowCount: 0, error }); + renderWithProviders(); + expect(screen.getByText("Could not load budgets")).toBeInTheDocument(); + expect(screen.getByText("budget store unavailable")).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.tsx index 4bc06425f80..76355c874f8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.tsx @@ -1,55 +1,269 @@ "use client"; -import { Inbox } from "lucide-react"; -import React, { useMemo } from "react"; +import { Inbox, ShieldAlert } from "lucide-react"; +import React, { useMemo, useState } from "react"; -import { DataTable } from "@/components/shared/DataTable"; -import { budgetItem } from "@/app/(dashboard)/hooks/budgets/useBudgets"; +import { + BUDGET_DURATION_FILTER_OPTIONS, + BUDGET_DURATION_UNSET, + type CreatedAtFilterValue, + type MaxBudgetFilterValue, +} from "@/app/(dashboard)/hooks/budgets/budgetFilters"; +import type { budgetItem } from "@/app/(dashboard)/hooks/budgets/useBudgets"; +import type { ResourceListResult } from "@/app/(dashboard)/hooks/common/useResourceList"; +import { + DataTable, + DataTableFilterDrawer, + DataTableFilterField, + DataTableToolbar, + type FilterDraft, +} from "@/components/shared/DataTable"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { ApiError } from "@/lib/http/client"; import { getBudgetTableColumns } from "./BudgetTableColumns"; interface BudgetTableProps { - budgets: budgetItem[]; - isLoading: boolean; + list: ResourceListResult; canModify: boolean; onEditClick: (budget: budgetItem) => void; onDeleteClick: (budget: budgetItem) => void; } -function EmptyState() { +const PAGE_SIZE_OPTIONS = [25, 50, 100]; + +const FILTER_LABELS: Record = { + budget_duration: "Reset", + max_budget: "Max Budget", + created_at: "Created", +}; + +const durationLabel = (value: string): string => + BUDGET_DURATION_FILTER_OPTIONS.find((option) => option.value === value)?.label ?? value; + +const formatFilterValue = (columnId: string, value: unknown): string => { + if (columnId === "budget_duration") { + return (Array.isArray(value) ? value : []).map((entry) => durationLabel(String(entry))).join(", "); + } + if (columnId === "max_budget") { + const { min, max, unlimitedOnly } = (value ?? {}) as MaxBudgetFilterValue; + return unlimitedOnly === true ? "Unlimited only" : `${min ? `$${min}` : "any"} to ${max ? `$${max}` : "any"}`; + } + if (columnId === "created_at") { + const { from, to } = (value ?? {}) as CreatedAtFilterValue; + return `${from || "any"} to ${to || "any"}`; + } + return String(value); +}; + +/** The drawer keeps any non-empty object as an active filter, so collapse a blank draft to nothing. */ +const normalizeMaxBudget = (draft: MaxBudgetFilterValue): MaxBudgetFilterValue | undefined => { + if (draft.unlimitedOnly === true) { + return { unlimitedOnly: true }; + } + const min = draft.min?.trim() ?? ""; + const max = draft.max?.trim() ?? ""; + if (min === "" && max === "") { + return undefined; + } + return { ...(min === "" ? {} : { min }), ...(max === "" ? {} : { max }) }; +}; + +const normalizeCreatedAt = (draft: CreatedAtFilterValue): CreatedAtFilterValue | undefined => { + const from = draft.from ?? ""; + const to = draft.to ?? ""; + if (from === "" && to === "") { + return undefined; + } + return { ...(from === "" ? {} : { from }), ...(to === "" ? {} : { to }) }; +}; + +function EmptyState({ hasQuery }: { hasQuery: boolean }) { return (
-
No budgets yet
+
{hasQuery ? "No matching budgets" : "No budgets yet"}
- Create a budget to set spend, TPM and RPM limits for customers. + {hasQuery + ? "No budget matches your search or filters." + : "Create a budget to set spend, TPM and RPM limits for customers."}
); } -const BudgetTable: React.FC = ({ budgets, isLoading, canModify, onEditClick, onDeleteClick }) => { - const rows = useMemo( - () => [...budgets].sort((a, b) => new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime()), - [budgets], +function ErrorState({ error }: { error: Error }) { + const forbidden = error instanceof ApiError && error.status === 403; + return ( +
+
+ +
+
+ {forbidden ? "You do not have access to budgets" : "Could not load budgets"} +
+
+ {forbidden ? "Ask a proxy admin to grant you the admin viewer role." : error.message} +
+
); +} + +/** "Not set" and the concrete durations are exclusive; see serializeBudgetFilters for why. */ +function DurationFilter({ selected, onChange }: { selected: string[]; onChange: (selected: string[]) => void }) { + const toggle = (value: string, checked: boolean): void => { + if (!checked) { + onChange(selected.filter((entry) => entry !== value)); + return; + } + const kept = value === BUDGET_DURATION_UNSET ? [] : selected.filter((entry) => entry !== BUDGET_DURATION_UNSET); + onChange([...kept, value]); + }; + + return ( +
+ {BUDGET_DURATION_FILTER_OPTIONS.map((option) => ( + + ))} +
+ ); +} + +function BudgetFilterFields({ get, set }: FilterDraft) { + const maxBudget = (get("max_budget") as MaxBudgetFilterValue | undefined) ?? {}; + const created = (get("created_at") as CreatedAtFilterValue | undefined) ?? {}; + const unlimitedOnly = maxBudget.unlimitedOnly === true; + + return ( + <> + + set("budget_duration", selected)} + /> + + +
+ set("max_budget", normalizeMaxBudget({ ...maxBudget, min: event.target.value }))} + placeholder="Min" + aria-label="Minimum max budget" + data-testid="budget-filter-max-budget-min" + /> + set("max_budget", normalizeMaxBudget({ ...maxBudget, max: event.target.value }))} + placeholder="Max" + aria-label="Maximum max budget" + data-testid="budget-filter-max-budget-max" + /> +
+ + + +
+ set("created_at", normalizeCreatedAt({ ...created, from: event.target.value }))} + aria-label="Created from" + data-testid="budget-filter-created-from" + /> + set("created_at", normalizeCreatedAt({ ...created, to: event.target.value }))} + aria-label="Created to" + data-testid="budget-filter-created-to" + /> +
+ + + ); +} + +const BudgetTable: React.FC = ({ list, canModify, onEditClick, onDeleteClick }) => { + const [filtersOpen, setFiltersOpen] = useState(false); const columns = useMemo( () => getBudgetTableColumns({ canModify, onEditClick, onDeleteClick }), [canModify, onEditClick, onDeleteClick], ); + const hasQuery = list.searchValue.trim() !== "" || list.columnFilters.length > 0; + const emptyMessage = list.error === null ? : ; + return ( budget.budget_id || String(index)} - isLoading={isLoading} + sortingMode="server" + sorting={list.sorting} + onSortingChange={list.onSortingChange} + paginationMode="server" + pagination={list.pagination} + onPaginationChange={list.onPaginationChange} + rowCount={list.rowCount} + pageSizeOptions={PAGE_SIZE_OPTIONS} + filterMode="server" + columnFilters={list.columnFilters} + onColumnFiltersChange={list.onColumnFiltersChange} + isLoading={list.isLoading} loadingMessage="Loading budgets…" - noDataMessage={} + noDataMessage={emptyMessage} size="compact" + toolbar={(table) => ( + <> + setFiltersOpen(true)} + onRefresh={list.refetch} + isRefreshing={list.isFetching} + filterLabels={FILTER_LABELS} + formatFilterValue={formatFilterValue} + /> + + {(draft) => } + + + )} /> ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx index e3fbc9dba08..8cca2caf214 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx @@ -1,11 +1,13 @@ "use client"; -import { ColumnDef } from "@tanstack/react-table"; +import { ColumnDef, FilterFn } from "@tanstack/react-table"; import { MoreHorizontal, Pencil, Trash2 } from "lucide-react"; -import { IdCell, MoneyCell } from "@/components/shared/table_cells"; -import { budgetItem } from "@/app/(dashboard)/hooks/budgets/useBudgets"; +import { DataTableSortHeader } from "@/components/shared/DataTable"; +import { DateCell, IdCell, MoneyCell } from "@/components/shared/table_cells"; +import type { budgetItem } from "@/app/(dashboard)/hooks/budgets/useBudgets"; import { buttonVariants } from "@/components/ui/button"; +import { getBudgetDurationLabel } from "@/components/common_components/budget_duration_dropdown"; import { DropdownMenu, DropdownMenuContent, @@ -15,6 +17,15 @@ import { } from "@/components/ui/dropdown-menu"; import { cn } from "@/lib/cva.config"; +/** + * Filtering happens on the server, so this never runs as a predicate. It exists to override + * TanStack's auto-remove heuristic, which infers a filter shape from the column's first cell + * and silently discards a filter whose value is not that shape (a range object on a numeric + * column, for instance). + */ +const serverFilter: FilterFn = () => true; +serverFilter.autoRemove = () => false; + function RateLimitCell({ value }: { value: number | null }) { if (value == null) { return n/a; @@ -22,6 +33,13 @@ function RateLimitCell({ value }: { value: number | null }) { return {value}; } +function BudgetDurationCell({ value }: { value: string | null }) { + if (!value) { + return Not set; + } + return {getBudgetDurationLabel(value)}; +} + interface BudgetRowActionsProps { budget: budgetItem; onEditClick: (budget: budgetItem) => void; @@ -72,38 +90,56 @@ export const getBudgetTableColumns = ({ id: "budget_id", accessorKey: "budget_id", meta: { title: "Budget ID" }, - header: "Budget ID", - size: 220, - enableSorting: false, - cell: ({ row }) => , + header: ({ column }) => , + cell: ({ row }) => ( + + ), }, { id: "max_budget", accessorKey: "max_budget", + filterFn: serverFilter, meta: { title: "Max Budget", numeric: true }, - header: "Max Budget", + header: ({ column }) => , size: 120, - enableSorting: false, cell: ({ row }) => , }, { id: "tpm_limit", accessorKey: "tpm_limit", meta: { title: "TPM", numeric: true }, - header: "TPM", + header: ({ column }) => , size: 100, - enableSorting: false, cell: ({ row }) => , }, { id: "rpm_limit", accessorKey: "rpm_limit", meta: { title: "RPM", numeric: true }, - header: "RPM", + header: ({ column }) => , size: 100, - enableSorting: false, cell: ({ row }) => , }, + { + id: "budget_duration", + accessorKey: "budget_duration", + filterFn: serverFilter, + meta: { title: "Reset" }, + // "7d"/"30d" sort lexicographically, not chronologically, so the route does not offer it. + enableSorting: false, + header: ({ column }) => , + size: 110, + cell: ({ row }) => , + }, + { + id: "created_at", + accessorKey: "created_at", + filterFn: serverFilter, + meta: { title: "Created" }, + header: ({ column }) => , + size: 160, + cell: ({ row }) => , + }, ...(canModify ? [ { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.test.tsx index 392616f1935..46f72cd8886 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.test.tsx @@ -1,217 +1,254 @@ -import { fireEvent, render, waitFor, screen } from "@testing-library/react"; -import { act } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import React from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { ApiError } from "@/lib/http/client"; + import BudgetPanel from "./budget_panel"; -const mockBudgets = [ - { - budget_id: "budget-1", - max_budget: 100, - rpm_limit: 10, - tpm_limit: 1000, - updated_at: "2024-01-01T00:00:00Z", - }, -]; - -vi.mock("@/app/(dashboard)/hooks/budgets/useBudgets", () => ({ - useBudgets: vi.fn().mockReturnValue({ data: [], isLoading: false }), - useDeleteBudget: vi.fn().mockReturnValue({ mutateAsync: vi.fn(), isPending: false }), - useCreateBudget: vi.fn().mockReturnValue({ mutateAsync: vi.fn() }), - useUpdateBudget: vi.fn().mockReturnValue({ mutateAsync: vi.fn() }), +const { getMock, budgetDeleteMock } = vi.hoisted(() => ({ + getMock: vi.fn(), + budgetDeleteMock: vi.fn(), })); -import { - useBudgets, - useDeleteBudget, - useCreateBudget, - useUpdateBudget, -} from "@/app/(dashboard)/hooks/budgets/useBudgets"; +vi.mock("@/components/networking", () => ({ + apiClient: { get: getMock }, + budgetCreateCall: vi.fn(), + budgetUpdateCall: vi.fn(), + budgetDeleteCall: budgetDeleteMock, + getProxyBaseUrl: () => "", +})); -const createQueryClient = () => - new QueryClient({ - defaultOptions: { queries: { retry: false, gcTime: 0 } }, - }); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => ({ accessToken: "sk-test", userRole: "Admin", userId: "u1" }), +})); -function renderWithProviders(ui: React.ReactElement) { - const qc = createQueryClient(); - return render({ui}); +vi.mock("@/components/molecules/notifications_manager", () => ({ + default: { success: vi.fn(), info: vi.fn(), fromBackend: vi.fn() }, +})); + +interface BudgetSeed { + budget_id: string; + max_budget: number | null; + budget_duration: string | null; } +const budgetRow = (seed: BudgetSeed) => ({ + soft_budget: null, + tpm_limit: 1000, + rpm_limit: 10, + budget_reset_at: null, + created_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-01T00:00:00Z", + ...seed, +}); + +const FORBIDDEN_PROBLEM = { + type: "about:blank", + title: "Forbidden", + status: 403, + detail: "Only proxy admins can view budgets", +}; + +const DEFAULT_ROWS = [ + budgetRow({ budget_id: "ecc1869c-6231-4380-a56d-1a0be457477d", max_budget: 100, budget_duration: "30d" }), +]; + +const respondWith = (rows: ReturnType[], totalCount: number) => { + getMock.mockResolvedValue({ + data: rows, + meta: { total_count: totalCount, page: 1, page_size: 50, total_pages: Math.ceil(totalCount / 50) }, + }); +}; + +type QueryRecord = Record; + +const queries = (): QueryRecord[] => getMock.mock.calls.map((call) => (call[1] as { query: QueryRecord }).query); +const lastQuery = (): QueryRecord => queries()[queries().length - 1]; +const paths = (): string[] => getMock.mock.calls.map((call) => String(call[0])); + +const renderPanel = () => { + const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }); + return render( + + + , + ); +}; + +const openFilters = async (user: ReturnType) => { + await user.click(screen.getByTestId("datatable-filters-trigger")); + await screen.findByTestId("filter-drawer-body"); +}; + describe("Budget Panel", () => { - afterEach(() => { + beforeEach(() => { vi.clearAllMocks(); + respondWith(DEFAULT_ROWS, 1); }); - it("should render the budget panel and load budgets", async () => { - vi.mocked(useBudgets).mockReturnValue({ - data: mockBudgets, - isLoading: false, - } as any); - - renderWithProviders(); - - await waitFor(() => { - expect(screen.getByText("Create a budget to assign to customers.")).toBeInTheDocument(); - expect(screen.getByText("budget-1")).toBeInTheDocument(); - }); + it("loads the first page of budgets, newest first", async () => { + renderPanel(); + await waitFor(() => expect(getMock).toHaveBeenCalled()); + expect(paths()[0]).toBe("/management/v1/budgets"); + expect(queries()[0]).toEqual({ page: 1, page_size: 50, sort: "-created_at" }); + expect(await screen.findByText("ecc1869c-6231-4380-a56d-1a0be457477d")).toBeInTheDocument(); }); - it("should open delete modal from the actions menu", async () => { + it("asks the server to sort when a sortable header is clicked", async () => { const user = userEvent.setup(); - vi.mocked(useBudgets).mockReturnValue({ - data: [ - { - budget_id: "budget-to-delete", - max_budget: 200, - rpm_limit: 20, - tpm_limit: 2000, - updated_at: "2024-01-02T00:00:00Z", - }, - ], - isLoading: false, - } as any); + renderPanel(); + await waitFor(() => expect(getMock).toHaveBeenCalled()); - renderWithProviders(); + await user.click(screen.getByTestId("sort-header-max_budget")); + await waitFor(() => expect(lastQuery().sort).toBe("-max_budget")); - await waitFor(() => { - expect(screen.getByText("budget-to-delete")).toBeInTheDocument(); - }); + await user.click(screen.getByTestId("sort-header-max_budget")); + await waitFor(() => expect(lastQuery().sort).toBe("max_budget")); - await user.click(screen.getByTestId("budget-actions-budget-to-delete")); + await user.click(screen.getByTestId("sort-header-budget_id")); + await waitFor(() => expect(lastQuery().sort).toBe("budget_id")); + }); + + it("searches on budget_id with a debounced q", async () => { + const user = userEvent.setup(); + renderPanel(); + await waitFor(() => expect(getMock).toHaveBeenCalled()); + + await user.type(screen.getByTestId("datatable-search"), "ecc"); + await waitFor(() => expect(lastQuery().q).toBe("ecc")); + expect(queries().some((query) => query.q === "e" || query.q === "ec")).toBe(false); + }); + + it("filters by reset duration and clears it again", async () => { + const user = userEvent.setup(); + renderPanel(); + await waitFor(() => expect(getMock).toHaveBeenCalled()); + + await openFilters(user); + await user.click(screen.getByTestId("budget-filter-duration-7d")); + await user.click(screen.getByTestId("budget-filter-duration-30d")); + await user.click(screen.getByTestId("filter-drawer-apply")); + + await waitFor(() => expect(lastQuery()["filter[budget_duration][in]"]).toBe("7d,30d")); + + await user.click(screen.getByTestId("filter-chip-remove-budget_duration")); + await waitFor(() => expect(lastQuery()).not.toHaveProperty("filter[budget_duration][in]")); + }); + + it("filters by budgets with no reset duration", async () => { + const user = userEvent.setup(); + renderPanel(); + await waitFor(() => expect(getMock).toHaveBeenCalled()); + + await openFilters(user); + await user.click(screen.getByTestId("budget-filter-duration-__unset__")); + await user.click(screen.getByTestId("filter-drawer-apply")); + + await waitFor(() => expect(lastQuery()["filter[budget_duration][is_null]"]).toBe("true")); + expect(lastQuery()).not.toHaveProperty("filter[budget_duration][in]"); + }); + + it("filters by a max budget range and clears it again", async () => { + const user = userEvent.setup(); + renderPanel(); + await waitFor(() => expect(getMock).toHaveBeenCalled()); + + await openFilters(user); + await user.type(screen.getByTestId("budget-filter-max-budget-min"), "10"); + await user.type(screen.getByTestId("budget-filter-max-budget-max"), "500"); + await user.click(screen.getByTestId("filter-drawer-apply")); + + await waitFor(() => expect(lastQuery()["filter[max_budget][gte]"]).toBe("10")); + expect(lastQuery()["filter[max_budget][lte]"]).toBe("500"); + + await user.click(screen.getByTestId("datatable-clear-filters")); + await waitFor(() => expect(lastQuery()).not.toHaveProperty("filter[max_budget][gte]")); + expect(lastQuery()).not.toHaveProperty("filter[max_budget][lte]"); + }); + + it("filters to unlimited budgets only", async () => { + const user = userEvent.setup(); + renderPanel(); + await waitFor(() => expect(getMock).toHaveBeenCalled()); + + await openFilters(user); + await user.type(screen.getByTestId("budget-filter-max-budget-min"), "10"); + await user.click(screen.getByTestId("budget-filter-max-budget-unlimited")); + await user.click(screen.getByTestId("filter-drawer-apply")); + + await waitFor(() => expect(lastQuery()["filter[max_budget][is_null]"]).toBe("true")); + expect(lastQuery()).not.toHaveProperty("filter[max_budget][gte]"); + }); + + it("filters by a created date range covering whole local days", async () => { + const user = userEvent.setup(); + renderPanel(); + await waitFor(() => expect(getMock).toHaveBeenCalled()); + + await openFilters(user); + await user.type(screen.getByTestId("budget-filter-created-from"), "2026-01-05"); + await user.type(screen.getByTestId("budget-filter-created-to"), "2026-01-06"); + await user.click(screen.getByTestId("filter-drawer-apply")); + + await waitFor(() => + expect(lastQuery()["filter[created_at][gte]"]).toBe(new Date("2026-01-05T00:00:00.000").toISOString()), + ); + expect(lastQuery()["filter[created_at][lte]"]).toBe(new Date("2026-01-06T23:59:59.999").toISOString()); + }); + + it("pages through the results and changes page size", async () => { + const user = userEvent.setup(); + respondWith(DEFAULT_ROWS, 400); + renderPanel(); + await waitFor(() => expect(getMock).toHaveBeenCalled()); + + await user.click(screen.getByTestId("pagination-next")); + await waitFor(() => expect(lastQuery().page).toBe(2)); + expect(lastQuery().page_size).toBe(50); + + await user.click(screen.getByTestId("pagination-page-size")); + await user.click(await screen.findByRole("option", { name: "25" })); + await waitFor(() => expect(lastQuery().page_size).toBe(25)); + }); + + it("renders an access-denied state when the route rejects the caller", async () => { + getMock.mockRejectedValue(new ApiError("Only proxy admins can view budgets", 403, FORBIDDEN_PROBLEM)); + renderPanel(); + expect(await screen.findByText("You do not have access to budgets")).toBeInTheDocument(); + expect(screen.queryByText("No budgets yet")).not.toBeInTheDocument(); + }); + + it("deletes a budget from the actions menu", async () => { + const user = userEvent.setup(); + budgetDeleteMock.mockResolvedValue(undefined); + renderPanel(); + await screen.findByText("ecc1869c-6231-4380-a56d-1a0be457477d"); + + await user.click(screen.getByTestId("budget-actions-ecc1869c-6231-4380-a56d-1a0be457477d")); await user.click(await screen.findByTestId("budget-action-delete")); + await screen.findByText("Delete Budget?"); + await user.click(screen.getByRole("button", { name: /^delete$/i })); - await waitFor(() => { - expect(screen.getByText("Delete Budget?")).toBeInTheDocument(); - }); + await waitFor(() => + expect(budgetDeleteMock).toHaveBeenCalledWith("sk-test", "ecc1869c-6231-4380-a56d-1a0be457477d"), + ); }); - it("should successfully delete a budget", async () => { + it("refetches the current page after a delete", async () => { const user = userEvent.setup(); - const deleteMutateAsync = vi.fn().mockResolvedValue(undefined); - vi.mocked(useBudgets).mockReturnValue({ - data: [ - { - budget_id: "budget-to-delete", - max_budget: 200, - rpm_limit: 20, - tpm_limit: 2000, - updated_at: "2024-01-02T00:00:00Z", - }, - ], - isLoading: false, - } as any); - vi.mocked(useDeleteBudget).mockReturnValue({ - mutateAsync: deleteMutateAsync, - isPending: false, - } as any); + budgetDeleteMock.mockResolvedValue(undefined); + renderPanel(); + await screen.findByText("ecc1869c-6231-4380-a56d-1a0be457477d"); + const before = getMock.mock.calls.length; - renderWithProviders(); - - await waitFor(() => { - expect(screen.getByText("budget-to-delete")).toBeInTheDocument(); - }); - - await user.click(screen.getByTestId("budget-actions-budget-to-delete")); + await user.click(screen.getByTestId("budget-actions-ecc1869c-6231-4380-a56d-1a0be457477d")); await user.click(await screen.findByTestId("budget-action-delete")); + await screen.findByText("Delete Budget?"); + await user.click(screen.getByRole("button", { name: /^delete$/i })); - await waitFor(() => { - expect(screen.getByText("Delete Budget?")).toBeInTheDocument(); - }); - - const confirmButton = screen.getByRole("button", { name: /delete/i }); - act(() => { - fireEvent.click(confirmButton); - }); - - await waitFor(() => { - expect(deleteMutateAsync).toHaveBeenCalledWith("budget-to-delete"); - }); - }); - - it("should render empty state without crashing", async () => { - vi.mocked(useBudgets).mockReturnValue({ - data: [], - isLoading: false, - } as any); - - renderWithProviders(); - - await waitFor(() => { - expect(screen.getByText("Create a budget to assign to customers.")).toBeInTheDocument(); - }); - }); - - it("should handle delete error", async () => { - const user = userEvent.setup(); - const deleteMutateAsync = vi.fn().mockRejectedValue(new Error("Delete failed")); - vi.mocked(useBudgets).mockReturnValue({ - data: [ - { - budget_id: "budget-to-delete", - max_budget: 200, - rpm_limit: 20, - tpm_limit: 2000, - updated_at: "2024-01-02T00:00:00Z", - }, - ], - isLoading: false, - } as any); - vi.mocked(useDeleteBudget).mockReturnValue({ - mutateAsync: deleteMutateAsync, - isPending: false, - } as any); - - renderWithProviders(); - - await waitFor(() => { - expect(screen.getByText("budget-to-delete")).toBeInTheDocument(); - }); - - await user.click(screen.getByTestId("budget-actions-budget-to-delete")); - await user.click(await screen.findByTestId("budget-action-delete")); - - await waitFor(() => { - expect(screen.getByText("Delete Budget?")).toBeInTheDocument(); - }); - - const confirmButton = screen.getByRole("button", { name: /delete/i }); - act(() => { - fireEvent.click(confirmButton); - }); - - await waitFor(() => { - expect(deleteMutateAsync).toHaveBeenCalledWith("budget-to-delete"); - }); - }); - - it("should open edit modal from the actions menu", async () => { - const user = userEvent.setup(); - vi.mocked(useBudgets).mockReturnValue({ - data: [ - { - budget_id: "budget-to-edit", - max_budget: 300, - rpm_limit: 30, - tpm_limit: 3000, - updated_at: "2024-01-03T00:00:00Z", - }, - ], - isLoading: false, - } as any); - - renderWithProviders(); - - await waitFor(() => { - expect(screen.getByText("budget-to-edit")).toBeInTheDocument(); - }); - - await user.click(screen.getByTestId("budget-actions-budget-to-edit")); - await user.click(await screen.findByTestId("budget-action-edit")); - - await waitFor(() => { - expect(screen.getByText("Edit Budget")).toBeInTheDocument(); - }); + await waitFor(() => expect(getMock.mock.calls.length).toBeGreaterThan(before)); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx index 2cf2a4c06ec..78c2c0ca74a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx @@ -3,13 +3,13 @@ * */ -import React, { useState } from "react"; +import React, { useCallback, useState } from "react"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { Button } from "@/components/ui/button"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; import NotificationsManager from "@/components/molecules/notifications_manager"; -import { useBudgets, useDeleteBudget, budgetItem } from "@/app/(dashboard)/hooks/budgets/useBudgets"; +import { useBudgetList, useDeleteBudget, budgetItem } from "@/app/(dashboard)/hooks/budgets/useBudgets"; import BudgetModal from "./budget_modal"; import BudgetTable from "./BudgetTable"; import EditBudgetModal from "./edit_budget_modal"; @@ -31,21 +31,25 @@ const BudgetPanel: React.FC = ({ accessToken }) => { // Admin Viewer follows the read-parity rule: see budgets, no writes. const canModify = isProxyAdminRole(userRole ?? ""); - const { data: budgetList = [], isLoading } = useBudgets(); + const budgetList = useBudgetList(); const deleteBudget = useDeleteBudget(); - const handleEditCall = async (budget: budgetItem) => { - if (accessToken == null) { - return; - } - setSelectedBudget(budget); - setIsEditModalVisible(true); - }; + // Stable identities keep the memoized column defs stable; new ones remount every header and cell. + const handleEditCall = useCallback( + (budget: budgetItem) => { + if (accessToken == null) { + return; + } + setSelectedBudget(budget); + setIsEditModalVisible(true); + }, + [accessToken], + ); - const handleDeleteClick = (budget: budgetItem) => { + const handleDeleteClick = useCallback((budget: budgetItem) => { setSelectedBudget(budget); setIsDeleteModalVisible(true); - }; + }, []); const handleDeleteConfirm = async () => { if (!selectedBudget || accessToken == null) { @@ -99,8 +103,7 @@ const BudgetPanel: React.FC = ({ accessToken }) => { )}

Create a budget to assign to customers.

{ + it("sends nothing when no filter is active", () => { + expect(serializeBudgetFilters([])).toEqual({}); + }); + + it("maps selected durations onto the in operator", () => { + expect(serializeBudgetFilters([{ id: "budget_duration", value: ["7d", "30d"] }])).toEqual({ + "filter[budget_duration][in]": "7d,30d", + }); + }); + + it("maps 'Not set' onto is_null instead of in", () => { + expect(serializeBudgetFilters([{ id: "budget_duration", value: [BUDGET_DURATION_UNSET] }])).toEqual({ + "filter[budget_duration][is_null]": "true", + }); + }); + + it("never sends in alongside is_null for the same field", () => { + const params = serializeBudgetFilters([{ id: "budget_duration", value: ["7d", BUDGET_DURATION_UNSET] }]); + expect(params["filter[budget_duration][in]"]).toBeUndefined(); + expect(params["filter[budget_duration][is_null]"]).toBe("true"); + }); + + it("maps a max budget range onto gte and lte", () => { + expect(serializeBudgetFilters([{ id: "max_budget", value: { min: "10", max: "250.5" } }])).toEqual({ + "filter[max_budget][gte]": "10", + "filter[max_budget][lte]": "250.5", + }); + }); + + it("sends only the bound that was filled in", () => { + expect(serializeBudgetFilters([{ id: "max_budget", value: { min: "10", max: "" } }])).toEqual({ + "filter[max_budget][gte]": "10", + }); + }); + + it("maps 'Unlimited only' onto is_null and drops the range", () => { + const params = serializeBudgetFilters([{ id: "max_budget", value: { min: "10", unlimitedOnly: true } }]); + expect(params).toEqual({ "filter[max_budget][is_null]": "true" }); + }); + + it("widens a created-at day range to cover the whole local days", () => { + const params = serializeBudgetFilters([{ id: "created_at", value: { from: "2026-01-05", to: "2026-01-06" } }]); + expect(params["filter[created_at][gte]"]).toBe(new Date("2026-01-05T00:00:00.000").toISOString()); + expect(params["filter[created_at][lte]"]).toBe(new Date("2026-01-06T23:59:59.999").toISOString()); + }); + + it("ignores an unparseable date rather than sending a broken bound", () => { + expect(serializeBudgetFilters([{ id: "created_at", value: { from: "not-a-date" } }])).toEqual({}); + }); + + it("ignores filter ids the route does not declare", () => { + expect(serializeBudgetFilters([{ id: "spend", value: "5" }])).toEqual({}); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/budgets/budgetFilters.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/budgets/budgetFilters.ts new file mode 100644 index 00000000000..f54eddb3913 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/budgets/budgetFilters.ts @@ -0,0 +1,91 @@ +import type { ColumnFilter, ColumnFiltersState } from "@tanstack/react-table"; + +export const BUDGET_DURATION_UNSET = "__unset__"; + +export const BUDGET_DURATION_FILTER_OPTIONS: readonly { value: string; label: string }[] = [ + { value: "1h", label: "hourly" }, + { value: "24h", label: "daily" }, + { value: "7d", label: "weekly" }, + { value: "30d", label: "monthly" }, + { value: BUDGET_DURATION_UNSET, label: "Not set" }, +]; + +export interface MaxBudgetFilterValue { + min?: string; + max?: string; + unlimitedOnly?: boolean; +} + +export interface CreatedAtFilterValue { + from?: string; + to?: string; +} + +type QueryEntry = readonly [string, string]; + +const entries = (key: string, value: string): QueryEntry[] => (value === "" ? [] : [[key, value]]); + +const asStringArray = (value: unknown): string[] => + Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : []; + +const asRecord = (value: unknown): Record => + typeof value === "object" && value !== null ? (value as Record) : {}; + +const asTrimmed = (value: unknown): string => (typeof value === "string" ? value.trim() : ""); + +/** The date inputs give a calendar day; the route wants an instant, so widen to the viewer's whole local day. */ +const isoAt = (day: string, time: string): string => { + if (day === "") { + return ""; + } + const parsed = new Date(`${day}T${time}`); + return Number.isNaN(parsed.getTime()) ? "" : parsed.toISOString(); +}; + +/** + * "Not set" is exclusive with the concrete durations. The route's contract does not say how it + * combines `in` with `is_null` on one field, and under AND semantics that pair can only match + * nothing, so we never send both. + */ +const durationParams = (value: unknown): QueryEntry[] => { + const selected = asStringArray(value); + if (selected.includes(BUDGET_DURATION_UNSET)) { + return [["filter[budget_duration][is_null]", "true"]]; + } + return entries("filter[budget_duration][in]", selected.join(",")); +}; + +const maxBudgetParams = (value: unknown): QueryEntry[] => { + const draft = asRecord(value); + if (draft.unlimitedOnly === true) { + return [["filter[max_budget][is_null]", "true"]]; + } + return [ + ...entries("filter[max_budget][gte]", asTrimmed(draft.min)), + ...entries("filter[max_budget][lte]", asTrimmed(draft.max)), + ]; +}; + +const createdAtParams = (value: unknown): QueryEntry[] => { + const draft = asRecord(value); + return [ + ...entries("filter[created_at][gte]", isoAt(asTrimmed(draft.from), "00:00:00.000")), + ...entries("filter[created_at][lte]", isoAt(asTrimmed(draft.to), "23:59:59.999")), + ]; +}; + +const filterParams = (filter: ColumnFilter): QueryEntry[] => { + switch (filter.id) { + case "budget_duration": + return durationParams(filter.value); + case "max_budget": + return maxBudgetParams(filter.value); + case "created_at": + return createdAtParams(filter.value); + default: + return []; + } +}; + +export const serializeBudgetFilters = (filters: ColumnFiltersState): Readonly> => + Object.fromEntries(filters.flatMap(filterParams)); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/budgets/useBudgets.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/budgets/useBudgets.ts index 0d8d94f2369..e5f24e5412d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/budgets/useBudgets.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/budgets/useBudgets.ts @@ -1,28 +1,58 @@ -import { useQuery, useMutation, useQueryClient, UseQueryResult } from "@tanstack/react-query"; -import { createQueryKeys } from "../common/queryKeysFactory"; -import { getBudgetList, budgetCreateCall, budgetUpdateCall, budgetDeleteCall } from "@/components/networking"; +"use client"; + +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import type { SortingState } from "@tanstack/react-table"; +import { useCallback } from "react"; + import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { apiClient, budgetCreateCall, budgetUpdateCall, budgetDeleteCall } from "@/components/networking"; + +import { createQueryKeys } from "../common/queryKeysFactory"; +import { + useResourceList, + type ResourceListPage, + type ResourceListQuery, + type ResourceListResult, +} from "../common/useResourceList"; +import { serializeBudgetFilters } from "./budgetFilters"; export interface budgetItem { budget_id: string; max_budget: number | null; + soft_budget: number | null; rpm_limit: number | null; tpm_limit: number | null; + budget_duration: string | null; + budget_reset_at: string | null; + created_at: string; updated_at: string; } +export const BUDGET_LIST_PATH = "/management/v1/budgets"; + export const budgetKeys = createQueryKeys("budgets"); -export const useBudgets = (): UseQueryResult => { +const DEFAULT_PAGE_SIZE = 50; +const DEFAULT_SORTING: SortingState = [{ id: "created_at", desc: true }]; + +export const useBudgetList = (): ResourceListResult => { const { accessToken } = useAuthorized(); - return useQuery({ - queryKey: budgetKeys.list({}), - queryFn: async () => { - const data = await getBudgetList(accessToken!); - return (data ?? []).filter((item: budgetItem | null): item is budgetItem => item != null); - }, + + const fetchPage = useCallback( + (query: ResourceListQuery, signal: AbortSignal): Promise> => + apiClient.get>(BUDGET_LIST_PATH, { accessToken, query, signal }), + [accessToken], + ); + + const listOptions = { + queryKey: budgetKeys.lists(), + fetchPage, + serializeFilters: serializeBudgetFilters, + defaultSorting: DEFAULT_SORTING, + defaultPageSize: DEFAULT_PAGE_SIZE, enabled: Boolean(accessToken), - }); + }; + return useResourceList(listOptions); }; export const useCreateBudget = () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/common/useResourceList.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/hooks/common/useResourceList.test.tsx new file mode 100644 index 00000000000..3ca67b082ec --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/common/useResourceList.test.tsx @@ -0,0 +1,161 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import type { ColumnFiltersState } from "@tanstack/react-table"; +import { act, renderHook, waitFor } from "@testing-library/react"; +import React, { type PropsWithChildren } from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { + toSortParam, + useResourceList, + type ResourceListPage, + type ResourceListQuery, + type UseResourceListOptions, +} from "./useResourceList"; + +interface Row { + id: string; +} + +const page = (rows: Row[], totalCount: number): ResourceListPage => ({ + data: rows, + meta: { total_count: totalCount, page: 1, page_size: 50, total_pages: 1 }, +}); + +const noFilters = (): Readonly> => ({}); + +const calls: ResourceListQuery[] = []; + +const renderList = (overrides: Partial> = {}) => { + const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }); + const wrapper = ({ children }: PropsWithChildren) => ( + {children} + ); + const fetchPage = vi.fn((query: ResourceListQuery) => { + calls.push(query); + return Promise.resolve(page([{ id: "a" }], 3)); + }); + const options: UseResourceListOptions = { + queryKey: ["widgets", "list"], + fetchPage, + serializeFilters: noFilters, + defaultSorting: [{ id: "created_at", desc: true }], + defaultPageSize: 50, + enabled: true, + ...overrides, + }; + return renderHook(() => useResourceList(options), { wrapper }); +}; + +const lastCall = (): ResourceListQuery => calls[calls.length - 1]; + +describe("toSortParam", () => { + it("prefixes descending fields with a minus and joins with commas", () => { + expect(toSortParam([{ id: "created_at", desc: true }])).toBe("-created_at"); + expect(toSortParam([{ id: "max_budget", desc: false }])).toBe("max_budget"); + expect( + toSortParam([ + { id: "a", desc: false }, + { id: "b", desc: true }, + ]), + ).toBe("a,-b"); + }); +}); + +describe("useResourceList", () => { + beforeEach(() => { + calls.length = 0; + }); + + it("requests the first page with the default sort", async () => { + const { result } = renderList(); + await waitFor(() => expect(result.current.rowCount).toBe(3)); + expect(lastCall()).toEqual({ page: 1, page_size: 50, sort: "-created_at" }); + }); + + it("exposes the returned rows and total count", async () => { + const { result } = renderList(); + await waitFor(() => expect(result.current.rows).toEqual([{ id: "a" }])); + expect(result.current.rowCount).toBe(3); + }); + + it("does not fetch while disabled", async () => { + const { result } = renderList({ enabled: false }); + await waitFor(() => expect(result.current.isLoading).toBe(false)); + expect(calls).toHaveLength(0); + }); + + it("sends the new sort and returns to the first page", async () => { + const { result } = renderList(); + await waitFor(() => expect(calls).toHaveLength(1)); + + act(() => result.current.onPaginationChange({ pageIndex: 2, pageSize: 50 })); + await waitFor(() => expect(lastCall().page).toBe(3)); + + act(() => result.current.onSortingChange([{ id: "max_budget", desc: false }])); + await waitFor(() => expect(lastCall().sort).toBe("max_budget")); + expect(lastCall().page).toBe(1); + }); + + it("omits sort entirely when nothing is sorted", async () => { + const { result } = renderList({ defaultSorting: [] }); + await waitFor(() => expect(calls).toHaveLength(1)); + expect(result.current.sorting).toEqual([]); + expect(lastCall()).not.toHaveProperty("sort"); + }); + + it("debounces the search into a single trimmed q and returns to the first page", async () => { + const { result } = renderList(); + await waitFor(() => expect(calls).toHaveLength(1)); + + act(() => result.current.onPaginationChange({ pageIndex: 1, pageSize: 50 })); + await waitFor(() => expect(lastCall().page).toBe(2)); + + act(() => result.current.onSearchChange("bud")); + act(() => result.current.onSearchChange("budg ")); + + await waitFor(() => expect(lastCall().q).toBe("budg")); + expect(lastCall().page).toBe(1); + expect(calls.some((call) => call.q === "bud")).toBe(false); + }); + + it("stops sending q once the search box is cleared", async () => { + const { result } = renderList(); + act(() => result.current.onSearchChange("budget")); + await waitFor(() => expect(lastCall().q).toBe("budget")); + + act(() => result.current.onSearchChange("")); + await waitFor(() => expect(lastCall()).not.toHaveProperty("q")); + }); + + it("merges serialized filters into the request and returns to the first page", async () => { + const serializeFilters = (filters: ColumnFiltersState): Readonly> => + filters.length === 0 ? {} : { "filter[colour][in]": String(filters[0].value) }; + const { result } = renderList({ serializeFilters }); + await waitFor(() => expect(calls).toHaveLength(1)); + + act(() => result.current.onPaginationChange({ pageIndex: 3, pageSize: 50 })); + await waitFor(() => expect(lastCall().page).toBe(4)); + + act(() => result.current.onColumnFiltersChange([{ id: "colour", value: "red" }])); + await waitFor(() => expect(lastCall()["filter[colour][in]"]).toBe("red")); + expect(lastCall().page).toBe(1); + + act(() => result.current.onColumnFiltersChange([])); + await waitFor(() => expect(lastCall()).not.toHaveProperty("filter[colour][in]")); + }); + + it("sends the requested page size", async () => { + const { result } = renderList(); + await waitFor(() => expect(calls).toHaveLength(1)); + + act(() => result.current.onPaginationChange({ pageIndex: 0, pageSize: 25 })); + await waitFor(() => expect(lastCall().page_size).toBe(25)); + }); + + it("surfaces a failed page as an error instead of empty rows", async () => { + const fetchPage = vi.fn(() => Promise.reject(new Error("boom"))); + const { result } = renderList({ fetchPage }); + await waitFor(() => expect(result.current.error?.message).toBe("boom")); + expect(result.current.rows).toEqual([]); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/common/useResourceList.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/common/useResourceList.ts new file mode 100644 index 00000000000..fb40d108234 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/common/useResourceList.ts @@ -0,0 +1,142 @@ +"use client"; + +import { useDebouncedValue } from "@tanstack/react-pacer/debouncer"; +import { useQuery, type UseQueryOptions } from "@tanstack/react-query"; +import type { ColumnFiltersState, OnChangeFn, PaginationState, SortingState } from "@tanstack/react-table"; +import { useCallback, useMemo, useState } from "react"; + +import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants"; + +export type ResourceListQuery = Readonly>; + +export interface ResourceListMeta { + total_count: number; + page: number; + page_size: number; + total_pages: number; +} + +export interface ResourceListPage { + data: TRow[]; + meta: ResourceListMeta; +} + +export interface UseResourceListOptions { + /** Prefix every list variant hangs off, so invalidating the resource root refetches whichever page is on screen. */ + queryKey: readonly unknown[]; + fetchPage: (query: ResourceListQuery, signal: AbortSignal) => Promise>; + /** Must be referentially stable; it feeds the query key. */ + serializeFilters: (filters: ColumnFiltersState) => Readonly>; + defaultSorting: SortingState; + defaultPageSize: number; + enabled: boolean; +} + +export interface ResourceListResult { + rows: TRow[]; + rowCount: number; + isLoading: boolean; + isFetching: boolean; + error: Error | null; + refetch: () => void; + + sorting: SortingState; + onSortingChange: OnChangeFn; + pagination: PaginationState; + onPaginationChange: OnChangeFn; + columnFilters: ColumnFiltersState; + onColumnFiltersChange: OnChangeFn; + searchValue: string; + onSearchChange: (value: string) => void; +} + +/** JSON:API sort form: comma separated fields, `-` prefix for descending. */ +export const toSortParam = (sorting: SortingState): string => + sorting.map((entry) => (entry.desc ? `-${entry.id}` : entry.id)).join(","); + +/** + * State container for a table whose sorting, paging, search and filtering all run + * on the server. It owns those four pieces of state, folds them into one JSON:API + * query, and returns the exact props DataTable's server modes want. + * + * Empty parameters are dropped rather than sent blank because the management + * routes reject query params they do not declare. + */ +export function useResourceList(options: UseResourceListOptions): ResourceListResult { + const { queryKey, fetchPage, serializeFilters, defaultSorting, defaultPageSize, enabled } = options; + + const [sorting, setSorting] = useState(defaultSorting); + const [pagination, setPagination] = useState({ pageIndex: 0, pageSize: defaultPageSize }); + const [columnFilters, setColumnFilters] = useState([]); + const [searchValue, setSearchValue] = useState(""); + const [debouncedSearch] = useDebouncedValue(searchValue, { wait: DEBOUNCE_WAIT_MS }); + + const query = useMemo(() => { + const sort = toSortParam(sorting); + const search = debouncedSearch.trim(); + return { + page: pagination.pageIndex + 1, + page_size: pagination.pageSize, + ...(sort === "" ? {} : { sort }), + ...(search === "" ? {} : { q: search }), + ...serializeFilters(columnFilters), + }; + }, [sorting, pagination.pageIndex, pagination.pageSize, debouncedSearch, columnFilters, serializeFilters]); + + const queryOptions: UseQueryOptions, Error, ResourceListPage, readonly unknown[]> = { + queryKey: [...queryKey, query], + queryFn: ({ signal }) => fetchPage(query, signal), + enabled, + placeholderData: (previous) => previous, + }; + const { data, isLoading, isFetching, error, refetch: refetchQuery } = useQuery(queryOptions); + + const toFirstPage = useCallback(() => setPagination((previous) => ({ ...previous, pageIndex: 0 })), []); + + const onSortingChange = useCallback>( + (updater) => { + setSorting(updater); + toFirstPage(); + }, + [toFirstPage], + ); + + const onColumnFiltersChange = useCallback>( + (updater) => { + setColumnFilters(updater); + toFirstPage(); + }, + [toFirstPage], + ); + + const onSearchChange = useCallback( + (value: string) => { + setSearchValue(value); + toFirstPage(); + }, + [toFirstPage], + ); + + const refetch = useCallback(() => { + void refetchQuery(); + }, [refetchQuery]); + + const rows = useMemo(() => data?.data ?? [], [data]); + + return { + rows, + rowCount: data?.meta.total_count ?? 0, + isLoading, + isFetching, + error, + refetch, + sorting, + onSortingChange, + pagination, + onPaginationChange: setPagination, + columnFilters, + onColumnFiltersChange, + searchValue, + onSearchChange, + }; +} From a685cc1511387149285dca3ea623fa6db4de7873 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 30 Jul 2026 19:38:24 -0700 Subject: [PATCH 03/50] feat(proxy): add a generic list contract for management/v1 entity lists Paging, sorting, filtering and search for an entity collection, declared once as a ListSpec and served by handle_list. The route injects a ListExecutor that owns its table, so this module never imports Prisma. The caller's scope is derived from the caller alone and ANDed with whatever they filtered on, so a query parameter can only narrow what they may read. This is the shared half of the budgets list; it lands here so the endpoint has something to register against, and drops out when the framework arrives on its own branch. --- .../management_v1/list_framework.py | 308 ++++++++++++++++++ .../management_endpoints/management_v1.py | 33 +- 2 files changed, 340 insertions(+), 1 deletion(-) create mode 100644 litellm/proxy/management_endpoints/management_v1/list_framework.py diff --git a/litellm/proxy/management_endpoints/management_v1/list_framework.py b/litellm/proxy/management_endpoints/management_v1/list_framework.py new file mode 100644 index 00000000000..6e800d295f9 --- /dev/null +++ b/litellm/proxy/management_endpoints/management_v1/list_framework.py @@ -0,0 +1,308 @@ +"""Generic paging/sorting/filtering contract for `/management/v1` entity lists. + +Prisma-free by construction: a route declares a `ListSpec` and injects a +`ListExecutor` that owns the table, so the parsing, scoping and envelope rules +stay in one place and every entity list answers the same way. +""" + +from __future__ import annotations + +import math +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime +from typing import Generic, Literal, Protocol, TypeAlias, TypeVar +from urllib.parse import urlencode + +from fastapi import Request +from pydantic import JsonValue + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.management_endpoints.management_v1.common import ( + PROBLEM_TYPE_BASE, + ManagementProblem, +) +from litellm.types.proxy.management_endpoints.management_v1 import ( + ListLinks, + ListMeta, + ListResponse, + ProblemDetail, +) + +FilterOp: TypeAlias = Literal["eq", "in", "gte", "lte", "contains", "is_null"] +FilterType: TypeAlias = Literal["string", "number", "datetime"] + +# Quoted so the recursive alias parses under the repo's 3.10 floor, where neither +# the `type` statement nor a forward reference inside a `|` expression exists. +WhereLeaf: TypeAlias = "str | int | float | bool | datetime | None" +WhereValue: TypeAlias = "WhereLeaf | Sequence[WhereLeaf] | Where | Sequence[Where]" +Where: TypeAlias = "Mapping[str, WhereValue]" +OrderBy: TypeAlias = "Sequence[Mapping[str, Literal['asc', 'desc']]]" + +RowT = TypeVar("RowT") + +PAGINATION_PARAMS = frozenset({"page", "page_size", "sort", "q"}) + + +@dataclass(frozen=True, slots=True) +class FilterSpec: + type: FilterType + ops: frozenset[FilterOp] + + +@dataclass(frozen=True, slots=True) +class SortKey: + field: str + descending: bool + + +@dataclass(frozen=True, slots=True) +class ScopeAll: + """The caller may read every row.""" + + +@dataclass(frozen=True, slots=True) +class ScopeWhere: + """The caller may read only rows matching `where`.""" + + where: Where + + +@dataclass(frozen=True, slots=True) +class ScopeDenied: + """The caller may not read the collection at all.""" + + detail: str + + +Scope: TypeAlias = "ScopeAll | ScopeWhere | ScopeDenied" + + +class ListExecutor(Protocol, Generic[RowT]): + """The table half of a list, injected so the framework never imports Prisma.""" + + async def count(self, where: Where) -> int: ... + + async def find_many(self, where: Where, order: OrderBy, skip: int, take: int) -> Sequence[RowT]: ... + + +@dataclass(frozen=True, slots=True) +class ListSpec(Generic[RowT]): + resource: str + sortable: frozenset[str] + searchable: frozenset[str] + filters: Mapping[str, FilterSpec] + default_sort: tuple[SortKey, ...] + default_page_size: int + max_page_size: int + scope: Callable[[UserAPIKeyAuth], Scope] + serialize: Callable[[RowT], Mapping[str, JsonValue]] + tiebreaker: str + + +@dataclass(frozen=True, slots=True) +class QueryPlan: + where: Where + order: OrderBy + skip: int + take: int + page: int + page_size: int + + +def _problem(slug: str, title: str, detail: str, allowed: Sequence[str] | None = None) -> ManagementProblem: + return ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}{slug}", + title=title, + status=400, + detail=detail, + allowed=list(allowed) if allowed is not None else None, + ) + ) + + +def _allowed_params(filters: Mapping[str, FilterSpec]) -> frozenset[str]: + return PAGINATION_PARAMS | frozenset( + f"filter[{field}][{op}]" for field, filter_spec in filters.items() for op in filter_spec.ops + ) + + +def _reject_unknown_params(request: Request, filters: Mapping[str, FilterSpec]) -> None: + allowed = _allowed_params(filters) + unknown = tuple(sorted(name for name in request.query_params if name not in allowed)) + if not unknown: + return + raise _problem( + "unknown-query-parameter", + "Unknown query parameter", + f"Unrecognized query parameter(s): {', '.join(unknown)}.", + sorted(allowed), + ) + + +def _positive_int(raw: str | None, default: int, name: str) -> int: + if raw is None: + return default + try: + value = int(raw) + except ValueError: + raise _problem("invalid-query-parameter", "Invalid query parameter", f"{name} must be an integer.") + if value < 1: + raise _problem("invalid-query-parameter", "Invalid query parameter", f"{name} must be at least 1.") + return value + + +def _parse_sort(raw: str | None, sortable: frozenset[str], default_sort: tuple[SortKey, ...]) -> tuple[SortKey, ...]: + if raw is None: + return default_sort + keys = tuple( + SortKey(field=token.removeprefix("-"), descending=token.startswith("-")) + for token in (part.strip() for part in raw.split(",")) + if token + ) + unknown = tuple(key.field for key in keys if key.field not in sortable) + if unknown: + raise _problem( + "invalid-sort-field", + "Invalid sort field", + f"Cannot sort on: {', '.join(unknown)}.", + sorted(sortable), + ) + return keys or default_sort + + +def _order_by(keys: Sequence[SortKey], tiebreaker: str) -> OrderBy: + tail = () if any(key.field == tiebreaker for key in keys) else (SortKey(field=tiebreaker, descending=False),) + return tuple({key.field: ("desc" if key.descending else "asc")} for key in (*keys, *tail)) + + +def _coerce(value: str, filter_type: FilterType, param: str) -> WhereLeaf: + if filter_type == "number": + try: + return float(value) + except ValueError: + raise _problem("invalid-filter-value", "Invalid filter value", f"{param} must be a number.") + if filter_type == "datetime": + try: + return datetime.fromisoformat(value) + except ValueError: + raise _problem("invalid-filter-value", "Invalid filter value", f"{param} must be an ISO-8601 timestamp.") + return value + + +def _bool(value: str, param: str) -> bool: + if value.lower() in ("true", "1"): + return True + if value.lower() in ("false", "0"): + return False + raise _problem("invalid-filter-value", "Invalid filter value", f"{param} must be true or false.") + + +def _condition(field: str, op: FilterOp, raw: str, filter_type: FilterType, param: str) -> Where: + if op == "is_null": + return {field: None} if _bool(raw, param) else {field: {"not": None}} + if op == "in": + return {field: {"in": tuple(_coerce(part, filter_type, param) for part in raw.split(",") if part)}} + if op == "contains": + return {field: {"contains": raw, "mode": "insensitive"}} + if op == "eq": + return {field: _coerce(raw, filter_type, param)} + return {field: {op: _coerce(raw, filter_type, param)}} + + +def _filter_conditions(request: Request, filters: Mapping[str, FilterSpec]) -> tuple[Where, ...]: + return tuple( + _condition(field, op, request.query_params[f"filter[{field}][{op}]"], spec.type, f"filter[{field}][{op}]") + for field, spec in filters.items() + for op in sorted(spec.ops) + if f"filter[{field}][{op}]" in request.query_params + ) + + +def _search_condition(raw: str | None, searchable: frozenset[str]) -> tuple[Where, ...]: + if not raw or not searchable: + return () + return ({"OR": tuple({field: {"contains": raw, "mode": "insensitive"}} for field in sorted(searchable))},) + + +def build_query_plan(request: Request, spec: ListSpec[RowT], scope: Scope) -> QueryPlan: + """Turn the query string into the executor's arguments, or raise a 400 problem. + + `scope` is derived from the caller, never from the query string, and is ANDed + with the caller's filters so a filter can only ever narrow what they may read. + """ + _reject_unknown_params(request, spec.filters) + + page = _positive_int(request.query_params.get("page"), 1, "page") + page_size = min( + _positive_int(request.query_params.get("page_size"), spec.default_page_size, "page_size"), + spec.max_page_size, + ) + keys = _parse_sort(request.query_params.get("sort"), spec.sortable, spec.default_sort) + + scope_conditions: tuple[Where, ...] = (scope.where,) if isinstance(scope, ScopeWhere) else () + conditions = ( + scope_conditions + + _filter_conditions(request, spec.filters) + + _search_condition(request.query_params.get("q"), spec.searchable) + ) + + return QueryPlan( + where={"AND": conditions} if conditions else {}, + order=_order_by(keys, spec.tiebreaker), + skip=(page - 1) * page_size, + take=page_size, + page=page, + page_size=page_size, + ) + + +def _page_url(request: Request, page: int) -> str: + others = tuple((key, value) for key, value in request.query_params.multi_items() if key != "page") + return f"{request.url.path}?{urlencode((*others, ('page', page)))}" + + +def _links(request: Request, page: int, last_page: int) -> ListLinks: + return ListLinks( + self_link=_page_url(request, page), + first=_page_url(request, 1), + prev=_page_url(request, page - 1) if page > 1 else None, + next=_page_url(request, page + 1) if page < last_page else None, + last=_page_url(request, last_page), + ) + + +async def handle_list( + request: Request, + spec: ListSpec[RowT], + executor: ListExecutor[RowT], + caller: UserAPIKeyAuth, +) -> ListResponse: + """Serve one page of `spec.resource` under the caller's scope.""" + scope = spec.scope(caller) + if isinstance(scope, ScopeDenied): + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}forbidden", + title="Forbidden", + status=403, + detail=scope.detail, + ) + ) + + plan = build_query_plan(request, spec, scope) + total_count = await executor.count(plan.where) + rows = await executor.find_many(where=plan.where, order=plan.order, skip=plan.skip, take=plan.take) + total_pages = math.ceil(total_count / plan.page_size) + + return ListResponse( + data=tuple(spec.serialize(row) for row in rows), + meta=ListMeta( + page=plan.page, + page_size=plan.page_size, + total_count=total_count, + total_pages=total_pages, + ), + links=_links(request, plan.page, max(total_pages, 1)), + ) diff --git a/litellm/types/proxy/management_endpoints/management_v1.py b/litellm/types/proxy/management_endpoints/management_v1.py index 2aecc54f114..a7427bc7590 100644 --- a/litellm/types/proxy/management_endpoints/management_v1.py +++ b/litellm/types/proxy/management_endpoints/management_v1.py @@ -1,6 +1,8 @@ """Shared response shapes for the `/management/v1` control-plane surface.""" -from pydantic import BaseModel, ConfigDict, Field +from collections.abc import Mapping + +from pydantic import BaseModel, ConfigDict, Field, JsonValue class ProblemDetail(BaseModel): @@ -37,3 +39,32 @@ class FacetListResponse(BaseModel): data: list[str] meta: PageMeta links: PageLinks + + +class ListMeta(BaseModel): + """An entity list can afford the COUNT(*) a facet cannot, so it reports a real total.""" + + page: int + page_size: int + total_count: int + total_pages: int + + +class ListLinks(BaseModel): + """Hypermedia for an entity list. `first`/`last` exist here because `total_pages` is known.""" + + model_config = ConfigDict(populate_by_name=True) + + self_link: str = Field(alias="self") + first: str + prev: str | None = None + next: str | None = None + last: str + + +class ListResponse(BaseModel): + """One page of an entity collection. Rows are flat: no `{type, id, attributes}` wrapper.""" + + data: tuple[Mapping[str, JsonValue], ...] + meta: ListMeta + links: ListLinks From f0866d0446a76ee84bda688b93881f95d3ade9a8 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 30 Jul 2026 19:38:32 -0700 Subject: [PATCH 04/50] feat(proxy): add GET /management/v1/budgets The Budgets page reads /budget/list, which returns the whole table as a bare array with no way to page, sort or filter it. A customer with enough budgets to fill the page has no way to find one. Registers LiteLLM_BudgetTable against the management/v1 list contract: sortable on budget_id, max_budget, tpm_limit, rpm_limit and created_at, default order newest-first with budget_id breaking ties, search on budget_id, and filters for budget_duration, max_budget and created_at. budget_duration is deliberately not sortable; the column holds "7d"/"30d" strings, so a lexicographic ORDER BY puts "30d" ahead of "7d". tpm_limit and rpm_limit are BigInt? in Prisma, so rows validate through a pydantic model on the way out and serialize as JSON numbers. A caller without admin view is refused 403 as a problem document rather than served an empty page. /budget/list is untouched. --- litellm/proxy/_types.py | 1 + .../management_v1/__init__.py | 4 + .../management_v1/budgets.py | 195 ++++++++ tests/e2e/coverage_registry/mgmt.yaml | 2 + .../test_budget_customer_user_org_e2e.py | 166 +++++- .../auth/test_admin_viewer_handler_access.py | 9 + .../proxy/auth/test_route_checks.py | 1 + .../management_v1/test_budgets.py | 471 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 100 ++++ 9 files changed, 946 insertions(+), 3 deletions(-) create mode 100644 litellm/proxy/management_endpoints/management_v1/budgets.py create mode 100644 tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9e98cb46b9a..9f3b32328c5 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -840,6 +840,7 @@ class LiteLLMRoutes(enum.Enum): "/config/list", "/config/field/info", "/budget/list", + "/management/v1/budgets", "/budget/settings", # Invitation viewing (admin viewer cannot create/delete; can read). "/invitation/info", diff --git a/litellm/proxy/management_endpoints/management_v1/__init__.py b/litellm/proxy/management_endpoints/management_v1/__init__.py index 257de66130b..a06c6b2591c 100644 --- a/litellm/proxy/management_endpoints/management_v1/__init__.py +++ b/litellm/proxy/management_endpoints/management_v1/__init__.py @@ -2,11 +2,15 @@ from fastapi import APIRouter +from litellm.proxy.management_endpoints.management_v1.budgets import ( + router as budgets_router, +) from litellm.proxy.management_endpoints.management_v1.spend_logs import ( router as spend_logs_router, ) router = APIRouter() +router.include_router(budgets_router) router.include_router(spend_logs_router) __all__ = ["router"] diff --git a/litellm/proxy/management_endpoints/management_v1/budgets.py b/litellm/proxy/management_endpoints/management_v1/budgets.py new file mode 100644 index 00000000000..79c3876c7ab --- /dev/null +++ b/litellm/proxy/management_endpoints/management_v1/budgets.py @@ -0,0 +1,195 @@ +"""`GET /management/v1/budgets`.""" + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime +from typing import Annotated + +from fastapi import APIRouter, Depends, Request +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import ( + CommonProxyErrors, + UserAPIKeyAuth, + user_api_key_has_admin_view, +) +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.management_endpoints.management_v1.common import ( + MANAGEMENT_V1_PREFIX, + PROBLEM_TYPE_BASE, + ManagementProblem, +) +from litellm.proxy.management_endpoints.management_v1.list_framework import ( + FilterSpec, + ListSpec, + OrderBy, + Scope, + ScopeAll, + ScopeDenied, + SortKey, + Where, + handle_list, +) +from litellm.proxy.utils import PrismaClient +from litellm.types.proxy.management_endpoints.management_v1 import ( + ListResponse, + ProblemDetail, +) + +router = APIRouter(prefix=MANAGEMENT_V1_PREFIX) + + +class BudgetRow(BaseModel): + """The `LiteLLM_BudgetTable` columns this list serves. + + Validating the untyped Prisma row through here is what makes `tpm_limit` / + `rpm_limit` ints: they are `BigInt?` in the schema, which the query engine can + hand back as a decimal string. + """ + + model_config = ConfigDict(from_attributes=True) + + budget_id: str + max_budget: float | None = None + soft_budget: float | None = None + tpm_limit: int | None = None + rpm_limit: int | None = None + budget_duration: str | None = None + budget_reset_at: datetime | None = None + created_at: datetime + updated_at: datetime + + +_BUDGET_ROWS = TypeAdapter(tuple[BudgetRow, ...]) + + +@dataclass(frozen=True, slots=True) +class PrismaBudgetListExecutor: + """The `ListExecutor` half of the budgets list: everything Prisma-shaped lives here.""" + + prisma_client: PrismaClient + + async def count(self, where: Where) -> int: + return int(await self.prisma_client.db.litellm_budgettable.count(where=dict(where))) + + async def find_many(self, where: Where, order: OrderBy, skip: int, take: int) -> Sequence[BudgetRow]: + rows = await self.prisma_client.db.litellm_budgettable.find_many( + where=dict(where), order=list(order), skip=skip, take=take + ) + return _BUDGET_ROWS.validate_python(rows) + + +def _iso(value: datetime | None) -> str | None: + return value.isoformat() if value is not None else None + + +def _serialize(row: BudgetRow) -> Mapping[str, JsonValue]: + return { + "budget_id": row.budget_id, + "max_budget": row.max_budget, + "soft_budget": row.soft_budget, + "tpm_limit": row.tpm_limit, + "rpm_limit": row.rpm_limit, + "budget_duration": row.budget_duration, + "budget_reset_at": _iso(row.budget_reset_at), + "created_at": _iso(row.created_at), + "updated_at": _iso(row.updated_at), + } + + +def _scope(caller: UserAPIKeyAuth) -> Scope: + if user_api_key_has_admin_view(caller): + return ScopeAll() + return ScopeDenied( + detail="Only proxy admins can list budgets, your role={}".format(caller.user_role), + ) + + +# budget_duration is deliberately absent from `sortable`: the column holds strings +# like "7d" and "30d", so a lexicographic ORDER BY puts "30d" ahead of "7d". +BUDGETS_LIST_SPEC: ListSpec[BudgetRow] = ListSpec( + resource="budgets", + sortable=frozenset({"budget_id", "max_budget", "tpm_limit", "rpm_limit", "created_at"}), + searchable=frozenset({"budget_id"}), + filters={ + "budget_duration": FilterSpec(type="string", ops=frozenset({"in", "is_null"})), + "max_budget": FilterSpec(type="number", ops=frozenset({"gte", "lte", "is_null"})), + "created_at": FilterSpec(type="datetime", ops=frozenset({"gte", "lte"})), + }, + default_sort=(SortKey(field="created_at", descending=True), SortKey(field="budget_id", descending=False)), + default_page_size=50, + max_page_size=100, + scope=_scope, + serialize=_serialize, + tiebreaker="budget_id", +) + + +@router.get( + "/budgets", + tags=["budget management"], + dependencies=[Depends(user_api_key_auth)], + response_model=ListResponse, +) +async def list_budgets( + request: Request, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> ListResponse: + """ + The budgets defined on this proxy, paged, sortable and filterable, for the + Budgets page. + + Readable by a proxy admin or an admin viewer; anyone else is refused 403. The + older `/budget/list` answers with the whole table as a bare array and has no + way to page, sort or filter it. + + `sort` takes a comma-separated list of `budget_id`, `max_budget`, `tpm_limit`, + `rpm_limit` or `created_at`, each optionally prefixed with `-` for descending, + and defaults to `-created_at,budget_id`. `q` is a case-insensitive substring + match on `budget_id`. `page_size` defaults to 50 and is capped at 100. + Filters are `filter[budget_duration][in|is_null]`, + `filter[max_budget][gte|lte|is_null]` and `filter[created_at][gte|lte]`. + + Example curl: + ``` + curl --location --globoff 'http://0.0.0.0:4000/management/v1/budgets?sort=-max_budget&filter[budget_duration][in]=7d,30d&page_size=25' \ + --header 'Authorization: Bearer sk-1234' + ``` + """ + try: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}database-not-connected", + title="Database not connected", + status=503, + detail=CommonProxyErrors.db_not_connected_error.value, + ) + ) + + return await handle_list( + request=request, + spec=BUDGETS_LIST_SPEC, + executor=PrismaBudgetListExecutor(prisma_client=prisma_client), + caller=user_api_key_dict, + ) + + except ManagementProblem: + raise + except Exception as e: # noqa: BLE001 # a driver error answers as a problem document, not the OpenAI error shape + verbose_proxy_logger.exception( + "litellm.proxy.management_endpoints.management_v1.budgets.list_budgets(): Exception occured - {}".format( + str(e) + ) + ) + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}internal-server-error", + title="Internal server error", + status=500, + detail="Failed to list budgets.", + ) + ) diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index 68c5ef6b31d..2a0fc5c9f29 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -57,6 +57,8 @@ - {id: mgmt.budget.update.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "budget_management_endpoints.py:155", rationale: "Limit changes apply"} - {id: mgmt.budget.delete.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "budget_management_endpoints.py:280", rationale: "Clears limits"} - {id: mgmt.budget.list.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "budget_management_endpoints.py:215", rationale: "Budget enumeration"} +- {id: mgmt.budget.list_v1.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "management_v1/budgets.py:129", rationale: "Budget enumeration the Budgets page can page, sort and filter"} +- {id: mgmt.budget.list_v1.admin_only, module: mgmt, tier: P1, surface: api, assertions: [admin_only], source: "management_v1/budgets.py:129", rationale: "A caller without admin view is refused, not served an empty page"} - {id: mgmt.callback.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "callback_management_endpoints.py", rationale: "Callback config (smoke)"} - {id: mgmt.cache_settings.update.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "cache_settings_endpoints.py", rationale: "Cache config (smoke). Deliberately uncovered: the previous test read the live settings and wrote them back, which proves nothing (identical values in, so a no-op POST still passes) while being able to break the deployment. /cache/settings persists what it receives and that row outranks YAML cache_params, re-applied on a timer, so a write that omits ssl or redis_startup_nodes turns a TLS cluster into a plaintext standalone node and every later Redis call hangs. That took out 60 of 72 tests on 2026-07-25. GET cannot round-trip it either: it resolves the stored row overlaid with REDIS_* env and never reads YAML, so on a fresh deploy it cannot see YAML ssl to echo back. A safe test needs an isolated proxy, or LIT-4816 fixed so a partial write cannot downgrade transport. Do not re-add a read-then-write-back test against a shared proxy."} - {id: mgmt.cost_tracking.estimate.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "cost_tracking_settings.py", rationale: "Cost estimate (smoke)"} diff --git a/tests/e2e/management/test_budget_customer_user_org_e2e.py b/tests/e2e/management/test_budget_customer_user_org_e2e.py index 54cc18b228b..12372bb7cc1 100644 --- a/tests/e2e/management/test_budget_customer_user_org_e2e.py +++ b/tests/e2e/management/test_budget_customer_user_org_e2e.py @@ -19,10 +19,10 @@ import time from collections.abc import Callable import pytest -from pydantic import BaseModel, RootModel +from pydantic import BaseModel, Field, RootModel from e2e_config import unique_marker -from e2e_http import NoBody, unwrap +from e2e_http import NoBody, Success, UnauthorizedError, UnknownApiError, unwrap from lifecycle import ResourceManager from management_client import ManagementClient from models import KeyGenerateBody, OrgInfoParams, OrgNewBody, UserNewBody @@ -44,9 +44,11 @@ def _poll[T](client: ManagementClient, attempt: Callable[[], T | None], failure: class BudgetNewBody(BaseModel): - max_budget: float + max_budget: float | None = None soft_budget: float | None = None budget_duration: str | None = None + budget_id: str | None = None + tpm_limit: int | None = None class BudgetNewResponse(BaseModel): @@ -204,6 +206,164 @@ class TestBudgetManagement: ) +# ---------- /management/v1/budgets ---------- + +_BUDGETS_V1 = "/management/v1/budgets" + + +class BudgetPageParams(BaseModel): + """Query for GET /management/v1/budgets. The filter fields serialize to the + bracketed keys the route reads them under, so nothing here is a raw dict.""" + + q: str | None = None + sort: str | None = None + page: int | None = None + page_size: int | None = None + duration_in: str | None = Field(default=None, serialization_alias="filter[budget_duration][in]") + max_budget_is_null: bool | None = Field(default=None, serialization_alias="filter[max_budget][is_null]") + not_a_parameter: str | None = Field(default=None, serialization_alias="filter[budget_id][eq]") + + +class BudgetPageMeta(BaseModel): + page: int + page_size: int + total_count: int + total_pages: int + + +class BudgetPageLinks(BaseModel): + first: str + prev: str | None = None + next: str | None = None + last: str + + +class BudgetPageRow(BaseModel): + budget_id: str + max_budget: float | None = None + tpm_limit: int | None = None + budget_duration: str | None = None + + +class BudgetPageResponse(BaseModel): + data: list[BudgetPageRow] + meta: BudgetPageMeta + links: BudgetPageLinks + + +def _list_budgets(client: ManagementClient, params: BudgetPageParams) -> BudgetPageResponse: + return unwrap( + client.proxy.transport.get( + _BUDGETS_V1, + headers=client.proxy.transport.master, + params=params, + response_type=BudgetPageResponse, + ) + ) + + +def _list_budget_ids(client: ManagementClient, params: BudgetPageParams) -> tuple[str, ...]: + return tuple(row.budget_id for row in _list_budgets(client, params).data) + + +def _list_status(client: ManagementClient, params: BudgetPageParams, key: str | None = None) -> int: + headers = client.proxy.transport.master if key is None else client.proxy.transport.bearer(key) + outcome = client.proxy.transport.get( + _BUDGETS_V1, headers=headers, params=params, response_type=BudgetPageResponse + ) + match outcome: + case Success(status_code=status_code): + return status_code + case UnauthorizedError(): + return 401 + case UnknownApiError(status_code=status_code): + return status_code + case _: + raise AssertionError(outcome) + + +class TestBudgetListV1: + """The paged, sorted, filtered budget list the Budgets page reads. + + Every test tags its own budgets with a marker in the budget_id and searches on + it, so budgets left behind by other suites cannot move the assertions. + """ + + @pytest.mark.covers("mgmt.budget.list_v1.happy_path") + def test_sorts_pages_and_filters_the_budgets_it_created( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + marker = unique_marker() + small, medium, large = (f"{marker}-small", f"{marker}-medium", f"{marker}-large") + for budget_id, max_budget, duration in ( + (small, 1.0, "7d"), + (medium, 2.0, "30d"), + (large, 3.0, "30d"), + ): + _create_budget( + client, + resources, + BudgetNewBody( + budget_id=budget_id, max_budget=max_budget, budget_duration=duration, tpm_limit=60000 + ), + ) + + mine = BudgetPageParams(q=marker, sort="-max_budget") + _ = _poll( + client, + lambda: mine if len(_list_budget_ids(client, mine)) == 3 else None, + f"{_BUDGETS_V1} never listed all three budgets tagged {marker}", + ) + + assert _list_budget_ids(client, mine) == (large, medium, small) + + page_two = _list_budgets(client, BudgetPageParams(q=marker, sort="-max_budget", page=2, page_size=1)) + assert [row.budget_id for row in page_two.data] == [medium] + assert page_two.meta.total_count == 3 + assert page_two.meta.total_pages == 3 + assert page_two.meta.page_size == 1 + assert page_two.links.prev is not None and page_two.links.next is not None + + assert set(_list_budget_ids(client, BudgetPageParams(q=marker, duration_in="30d"))) == {medium, large} + + limits = _list_budgets(client, BudgetPageParams(q=marker, sort="budget_id")).data + assert [row.tpm_limit for row in limits] == [60000, 60000, 60000] + + @pytest.mark.covers("mgmt.budget.list_v1.happy_path") + def test_is_null_finds_the_budget_left_uncapped( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + marker = unique_marker() + uncapped = _create_budget(client, resources, BudgetNewBody(budget_id=f"{marker}-uncapped")) + _ = _create_budget(client, resources, BudgetNewBody(budget_id=f"{marker}-capped", max_budget=4.0)) + + params = BudgetPageParams(q=marker, max_budget_is_null=True) + found = _poll( + client, + lambda: params if _list_budget_ids(client, params) == (uncapped,) else None, + f"{_BUDGETS_V1} never isolated the uncapped budget {uncapped}", + ) + + assert [row.max_budget for row in _list_budgets(client, found).data] == [None] + + @pytest.mark.covers("mgmt.budget.list_v1.happy_path") + def test_refuses_a_sort_field_and_a_parameter_it_does_not_support(self, client: ManagementClient) -> None: + assert _list_status(client, BudgetPageParams(sort="budget_duration")) == 400 + assert _list_status(client, BudgetPageParams(not_a_parameter="b-1")) == 400 + + @pytest.mark.covers("mgmt.budget.list_v1.admin_only") + def test_is_refused_for_a_non_admin_key(self, client: ManagementClient, resources: ResourceManager) -> None: + key = client.proxy.generate_key(KeyGenerateBody()) + resources.defer(lambda: client.proxy.delete_key(key)) + + status = _list_status(client, BudgetPageParams(), key=key) + + assert status in (401, 403), ( + f"a non-admin key listing budgets must be refused 401/403, got {status}. Serving 200 with an " + f"empty page would read as 'this proxy has no budgets'" + ) + + # ---------- customer / end-user ---------- diff --git a/tests/test_litellm/proxy/auth/test_admin_viewer_handler_access.py b/tests/test_litellm/proxy/auth/test_admin_viewer_handler_access.py index b0d2595e48c..9f4a801eb83 100644 --- a/tests/test_litellm/proxy/auth/test_admin_viewer_handler_access.py +++ b/tests/test_litellm/proxy/auth/test_admin_viewer_handler_access.py @@ -51,6 +51,7 @@ def admin_viewer_client(monkeypatch): mock_budget_table = MagicMock() mock_budget_table.find_many = AsyncMock(return_value=[]) mock_budget_table.find_first = AsyncMock(return_value=None) + mock_budget_table.count = AsyncMock(return_value=0) mock_invitation_table = MagicMock() mock_invitation_table.find_unique = AsyncMock(return_value=None) @@ -106,6 +107,14 @@ def test_budget_list_allows_admin_viewer(admin_viewer_client): assert resp.status_code == 200, resp.text +def test_management_v1_budgets_allows_admin_viewer(admin_viewer_client): + """`/management/v1/budgets` is the paged/sortable budget list; same read tier as + `/budget/list`, and it answers 403 rather than an empty page when it refuses.""" + resp = admin_viewer_client.get("/management/v1/budgets") + _assert_not_role_blocked(resp) + assert resp.status_code == 200, resp.text + + def test_budget_settings_allows_admin_viewer(admin_viewer_client): """`/budget/settings` describes a budget's fields; read-only.""" resp = admin_viewer_client.get("/budget/settings", params={"budget_id": "b1"}) diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index a6d4dc63697..06764139eda 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -1960,6 +1960,7 @@ ADMIN_VIEWER_SETTINGS_ROUTES = [ "/config/field/info", # Budgets page "/budget/list", + "/management/v1/budgets", "/budget/settings", # Invitation viewing (admin viewer cannot create/delete; can read) "/invitation/info", diff --git a/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py b/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py new file mode 100644 index 00000000000..500c072bc7d --- /dev/null +++ b/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py @@ -0,0 +1,471 @@ +from datetime import datetime, timezone +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.testclient import TestClient + +from litellm.proxy._types import LiteLLMRoutes, LitellmUserRoles +from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth +from litellm.proxy.management_endpoints.management_v1 import router +from litellm.proxy.management_endpoints.management_v1.budgets import BUDGETS_LIST_SPEC +from litellm.proxy.management_endpoints.management_v1.common import ( + MANAGEMENT_V1_PREFIX, + PROBLEM_TYPE_BASE, + ManagementProblem, + problem_response, +) +from litellm.proxy.management_endpoints.management_v1.list_framework import ( + ScopeWhere, + build_query_plan, +) +from litellm.types.proxy.management_endpoints.management_v1 import ProblemDetail + +app = FastAPI() + + +@app.exception_handler(ManagementProblem) +async def management_problem_exception_handler(request: Request, exc: ManagementProblem): + return problem_response(exc.problem) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + return problem_response( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}invalid-query-parameter", + title="Invalid query parameter", + status=400, + detail="The request query parameters are invalid.", + ) + ) + + +app.include_router(router) +client = TestClient(app) + +BUDGETS_PATH = f"{MANAGEMENT_V1_PREFIX}/budgets" +SORTABLE = ["budget_id", "created_at", "max_budget", "rpm_limit", "tpm_limit"] + + +def _row(budget_id: str, **overrides: Any) -> dict[str, Any]: + return { + "budget_id": budget_id, + "max_budget": 10.0, + "soft_budget": None, + "tpm_limit": None, + "rpm_limit": None, + "budget_duration": "30d", + "budget_reset_at": None, + "created_at": datetime(2026, 7, 20, 12, 0, tzinfo=timezone.utc), + "updated_at": datetime(2026, 7, 21, 12, 0, tzinfo=timezone.utc), + **overrides, + } + + +@pytest.fixture +def budget_table(monkeypatch): + table = MagicMock() + table.count = AsyncMock(return_value=0) + table.find_many = AsyncMock(return_value=[]) + prisma_client = MagicMock() + prisma_client.db.litellm_budgettable = table + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + return table + + +@pytest.fixture +def as_proxy_admin(): + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN + ) + yield + app.dependency_overrides.clear() + + +def _serve(budget_table, rows: list[dict[str, Any]], total: int | None = None) -> None: + budget_table.find_many = AsyncMock(return_value=rows) + budget_table.count = AsyncMock(return_value=len(rows) if total is None else total) + + +def _as_role(role: LitellmUserRoles): + original = app.dependency_overrides.copy() + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="u", user_role=role) + return original + + +def _get(query: str = ""): + suffix = f"?{query}" if query else "" + return client.get(f"{BUDGETS_PATH}{suffix}", headers={"Authorization": "Bearer k"}) + + +def test_returns_flat_rows_in_the_control_plane_envelope(budget_table, as_proxy_admin): + """`{data, meta, links}` with flat rows; no JSON:API `{type, id, attributes}` wrapper.""" + _serve(budget_table, [_row("b-1")]) + + response = _get() + + assert response.status_code == 200 + body = response.json() + assert set(body) == {"data", "meta", "links"} + assert body["data"][0]["budget_id"] == "b-1" + assert "attributes" not in body["data"][0] + + +def test_serves_the_columns_the_budgets_page_renders(budget_table, as_proxy_admin): + _serve(budget_table, [_row("b-1", soft_budget=5.0, budget_reset_at=datetime(2026, 8, 1, tzinfo=timezone.utc))]) + + row = _get().json()["data"][0] + + assert set(row) == { + "budget_id", + "max_budget", + "soft_budget", + "tpm_limit", + "rpm_limit", + "budget_duration", + "budget_reset_at", + "created_at", + "updated_at", + } + assert row["soft_budget"] == 5.0 + assert row["budget_reset_at"].startswith("2026-08-01T00:00:00") + + +def test_defaults_to_newest_first_with_budget_id_breaking_ties(budget_table, as_proxy_admin): + """Two budgets created in the same transaction share a created_at; without the + tiebreaker their relative order is undefined and pages can repeat or drop rows.""" + _serve(budget_table, []) + + _get() + + assert budget_table.find_many.call_args.kwargs["order"] == [ + {"created_at": "desc"}, + {"budget_id": "asc"}, + ] + + +def test_appends_the_tiebreaker_to_an_explicit_sort(budget_table, as_proxy_admin): + _serve(budget_table, []) + + _get("sort=-max_budget") + + assert budget_table.find_many.call_args.kwargs["order"] == [ + {"max_budget": "desc"}, + {"budget_id": "asc"}, + ] + + +def test_does_not_duplicate_the_tiebreaker_when_it_is_sorted_on(budget_table, as_proxy_admin): + _serve(budget_table, []) + + _get("sort=-budget_id") + + assert budget_table.find_many.call_args.kwargs["order"] == [{"budget_id": "desc"}] + + +def test_refuses_to_sort_on_budget_duration(budget_table, as_proxy_admin): + """The column holds "7d"/"30d", so a lexicographic ORDER BY would put "30d" + before "7d" and silently mis-order the page.""" + _serve(budget_table, []) + + response = _get("sort=budget_duration") + + assert response.status_code == 400 + assert response.headers["content-type"].startswith("application/problem+json") + body = response.json() + assert "budget_duration" in body["detail"] + assert body["allowed"] == SORTABLE + budget_table.find_many.assert_not_called() + + +def test_the_advertised_sort_fields_are_the_ones_that_work(budget_table, as_proxy_admin): + """Guards the rejection above against drifting from what the spec actually accepts.""" + _serve(budget_table, []) + + for field in SORTABLE: + assert _get(f"sort={field}").status_code == 200, field + assert sorted(BUDGETS_LIST_SPEC.sortable) == SORTABLE + + +def test_rejects_an_unknown_query_parameter(budget_table, as_proxy_admin): + """A silently ignored filter over-returns budgets, which is worse than a rejected request.""" + _serve(budget_table, []) + + response = _get("filtre[max_budget][gte]=5") + + assert response.status_code == 400 + assert response.headers["content-type"].startswith("application/problem+json") + body = response.json() + assert "filtre[max_budget][gte]" in body["detail"] + assert "filter[max_budget][gte]" in body["allowed"] + budget_table.count.assert_not_called() + budget_table.find_many.assert_not_called() + + +def test_rejects_an_operator_the_filter_does_not_declare(budget_table, as_proxy_admin): + """`max_budget` takes ranges, not `in`; accepting an undeclared operator is how a + filter starts meaning something the query planner never checked.""" + _serve(budget_table, []) + + assert _get("filter[max_budget][in]=5,10").status_code == 400 + assert _get("filter[created_at][is_null]=true").status_code == 400 + + +def test_omitted_page_size_serves_fifty(budget_table, as_proxy_admin): + _serve(budget_table, []) + + body = _get().json() + + assert body["meta"]["page_size"] == 50 + assert budget_table.find_many.call_args.kwargs["take"] == 50 + + +def test_clamps_an_oversized_page_size_to_a_hundred(budget_table, as_proxy_admin): + """Unclamped, one request can ask the proxy to serialize the whole budget table.""" + _serve(budget_table, []) + + body = _get("page_size=500").json() + + assert body["meta"]["page_size"] == 100 + assert budget_table.find_many.call_args.kwargs["take"] == 100 + + +def test_offsets_by_page(budget_table, as_proxy_admin): + _serve(budget_table, []) + + _get("page=3&page_size=25") + + assert budget_table.find_many.call_args.kwargs["skip"] == 50 + assert budget_table.find_many.call_args.kwargs["take"] == 25 + + +@pytest.mark.parametrize( + "role", + [ + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + LitellmUserRoles.TEAM, + ], +) +def test_refuses_a_caller_without_admin_view(budget_table, role): + """Budgets are proxy-wide, so a caller who cannot read all of them must be told + so. Answering 200 with an empty list would read as "there are no budgets".""" + _serve(budget_table, [_row("b-1")]) + original = _as_role(role) + try: + response = _get() + finally: + app.dependency_overrides = original + + assert response.status_code == 403 + assert response.headers["content-type"].startswith("application/problem+json") + assert response.json()["status"] == 403 + budget_table.count.assert_not_called() + budget_table.find_many.assert_not_called() + + +@pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY]) +def test_admins_and_admin_viewers_may_read_every_budget(budget_table, role): + _serve(budget_table, [_row("b-1")]) + original = _as_role(role) + try: + response = _get() + finally: + app.dependency_overrides = original + + assert response.status_code == 200 + assert [row["budget_id"] for row in response.json()["data"]] == ["b-1"] + + +def test_a_denied_caller_stays_denied_whatever_they_filter_on(budget_table): + """The scope decision reads the caller, never the query string.""" + _serve(budget_table, [_row("b-1")]) + original = _as_role(LitellmUserRoles.INTERNAL_USER) + try: + response = _get("filter[max_budget][gte]=0&q=b-") + finally: + app.dependency_overrides = original + + assert response.status_code == 403 + + +def test_a_filter_narrows_the_scope_predicate_instead_of_replacing_it(budget_table, as_proxy_admin): + """A filter is ANDed in. Assigning it over the scope clause is what would let a + caller widen their own read.""" + _serve(budget_table, []) + + _get("filter[max_budget][gte]=5") + + where = budget_table.find_many.call_args.kwargs["where"] + assert {"max_budget": {"gte": 5.0}} in where["AND"] + + +def test_a_scoped_caller_keeps_their_scope_clause_alongside_their_filter(): + """Same spec, driven through the planner with a row-scoped caller: the scope + clause has to survive next to whatever the caller filtered on.""" + request = Request( + { + "type": "http", + "method": "GET", + "path": BUDGETS_PATH, + "headers": [], + "query_string": b"filter[max_budget][gte]=5", + } + ) + + plan = build_query_plan(request, BUDGETS_LIST_SPEC, ScopeWhere(where={"budget_id": {"in": ("b-1",)}})) + + assert {"budget_id": {"in": ("b-1",)}} in plan.where["AND"] + assert {"max_budget": {"gte": 5.0}} in plan.where["AND"] + + +def test_q_matches_budget_id_case_insensitively(budget_table, as_proxy_admin): + """budget_id is the only text identity on the row; matching anything else would + return budgets whose ids do not contain what the user typed.""" + _serve(budget_table, []) + + _get("q=Prod") + + where = budget_table.find_many.call_args.kwargs["where"] + assert {"OR": ({"budget_id": {"contains": "Prod", "mode": "insensitive"}},)} in where["AND"] + + +def test_q_does_not_search_any_other_column(budget_table, as_proxy_admin): + _serve(budget_table, []) + + _get("q=30d") + + searched = budget_table.find_many.call_args.kwargs["where"]["AND"][0]["OR"] + assert [next(iter(clause)) for clause in searched] == ["budget_id"] + assert BUDGETS_LIST_SPEC.searchable == frozenset({"budget_id"}) + + +def test_is_null_selects_the_unlimited_budgets(budget_table, as_proxy_admin): + """"Unlimited" is max_budget IS NULL; `max_budget = 0` would be a hard zero cap.""" + _serve(budget_table, [_row("b-unlimited", max_budget=None)]) + + body = _get("filter[max_budget][is_null]=true").json() + + assert {"max_budget": None} in budget_table.find_many.call_args.kwargs["where"]["AND"] + assert body["data"][0]["max_budget"] is None + + +def test_is_null_false_selects_the_capped_budgets(budget_table, as_proxy_admin): + _serve(budget_table, []) + + _get("filter[max_budget][is_null]=false") + + assert {"max_budget": {"not": None}} in budget_table.find_many.call_args.kwargs["where"]["AND"] + + +def test_in_filter_splits_the_requested_durations(budget_table, as_proxy_admin): + _serve(budget_table, []) + + _get("filter[budget_duration][in]=7d,30d") + + assert {"budget_duration": {"in": ("7d", "30d")}} in budget_table.find_many.call_args.kwargs["where"]["AND"] + + +def test_created_at_range_is_read_as_a_timestamp(budget_table, as_proxy_admin): + _serve(budget_table, []) + + _get("filter[created_at][gte]=2026-07-01T00:00:00%2B00:00") + + assert { + "created_at": {"gte": datetime(2026, 7, 1, tzinfo=timezone.utc)} + } in budget_table.find_many.call_args.kwargs["where"]["AND"] + + +def test_rejects_a_filter_value_that_is_not_of_the_declared_type(budget_table, as_proxy_admin): + _serve(budget_table, []) + + assert _get("filter[max_budget][gte]=lots").status_code == 400 + assert _get("filter[created_at][gte]=yesterday").status_code == 400 + + +def test_reports_the_total_and_links_every_page_on_a_middle_page(budget_table, as_proxy_admin): + """The Budgets page renders a page count, so the total has to be the match total, + not the length of the page it just received.""" + _serve(budget_table, [_row("b-3"), _row("b-4")], total=7) + + body = _get("page=2&page_size=2").json() + + assert body["meta"] == {"page": 2, "page_size": 2, "total_count": 7, "total_pages": 4} + links = body["links"] + assert "page=1" in links["first"] and "page_size=2" in links["first"] + assert "page=1" in links["prev"] + assert "page=3" in links["next"] + assert "page=4" in links["last"] + assert "page=2" in links["self"] + + +def test_the_last_page_has_no_next(budget_table, as_proxy_admin): + _serve(budget_table, [_row("b-5")], total=5) + + links = _get("page=3&page_size=2").json()["links"] + + assert links["next"] is None + assert "page=2" in links["prev"] + + +def test_the_first_page_has_no_prev(budget_table, as_proxy_admin): + _serve(budget_table, [_row("b-1")], total=5) + + links = _get("page_size=2").json()["links"] + + assert links["prev"] is None + assert "page=2" in links["next"] + + +def test_an_empty_table_still_links_a_first_and_last_page(budget_table, as_proxy_admin): + _serve(budget_table, [], total=0) + + body = _get().json() + + assert body["meta"]["total_count"] == 0 + assert body["meta"]["total_pages"] == 0 + assert "page=1" in body["links"]["first"] and "page=1" in body["links"]["last"] + + +def test_counts_over_the_same_predicate_it_pages(budget_table, as_proxy_admin): + """A total counted without the caller's filter would page through rows the + filter excluded.""" + _serve(budget_table, [], total=0) + + _get("filter[budget_duration][in]=30d") + + assert budget_table.count.call_args.kwargs["where"] == budget_table.find_many.call_args.kwargs["where"] + + +def test_bigint_limits_serialize_as_json_numbers(budget_table, as_proxy_admin): + """tpm_limit/rpm_limit are BigInt? in Prisma; the query engine can hand them back + as decimal strings, and a quoted "60000" breaks arithmetic in the dashboard.""" + _serve(budget_table, [_row("b-1", tpm_limit="60000", rpm_limit=1200)]) + + row = _get().json()["data"][0] + + assert row["tpm_limit"] == 60000 + assert row["rpm_limit"] == 1200 + assert isinstance(row["tpm_limit"], int) and not isinstance(row["tpm_limit"], bool) + assert '"tpm_limit": "60000"' not in _get().text + + +def test_reports_a_missing_database_as_a_problem_document(monkeypatch, as_proxy_admin): + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + + response = _get() + + assert response.status_code == 503 + assert response.headers["content-type"].startswith("application/problem+json") + + +def test_is_reachable_by_the_roles_that_can_open_the_budgets_page(): + """Route-level auth gate, which the dependency_overrides above bypass. The handler's + admin-view check is dead code if RouteChecks rejects the role first.""" + assert BUDGETS_PATH in LiteLLMRoutes.admin_viewer_routes.value + assert ("/budget/list" in LiteLLMRoutes.admin_viewer_routes.value) == ( + BUDGETS_PATH in LiteLLMRoutes.admin_viewer_routes.value + ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 380b6545da8..752b762e773 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -7169,6 +7169,43 @@ export interface paths { patch?: never; trace?: never; }; + "/management/v1/budgets": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * List Budgets + * @description The budgets defined on this proxy, paged, sortable and filterable, for the + * Budgets page. + * + * Readable by a proxy admin or an admin viewer; anyone else is refused 403. The + * older `/budget/list` answers with the whole table as a bare array and has no + * way to page, sort or filter it. + * + * `sort` takes a comma-separated list of `budget_id`, `max_budget`, `tpm_limit`, + * `rpm_limit` or `created_at`, each optionally prefixed with `-` for descending, + * and defaults to `-created_at,budget_id`. `q` is a case-insensitive substring + * match on `budget_id`. `page_size` defaults to 50 and is capped at 100. + * Filters are `filter[budget_duration][in|is_null]`, + * `filter[max_budget][gte|lte|is_null]` and `filter[created_at][gte|lte]`. + * + * Example curl: + * ``` + * curl --location --globoff 'http://0.0.0.0:4000/management/v1/budgets?sort=-max_budget&filter[budget_duration][in]=7d,30d&page_size=25' --header 'Authorization: Bearer sk-1234' + * ``` + */ + get: operations["list_budgets_management_v1_budgets_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/management/v1/spend_logs/end_users": { parameters: { query?: never; @@ -24771,6 +24808,7 @@ export interface components { /** Updated By */ updated_by?: string | null; }; + JsonValue: unknown; /** KeyHealthResponse */ KeyHealthResponse: { /** @@ -24922,6 +24960,36 @@ export interface components { /** Guardrails */ guardrails: components["schemas"]["GuardrailInfoResponse"][]; }; + /** + * ListLinks + * @description Hypermedia for an entity list. `first`/`last` exist here because `total_pages` is known. + */ + ListLinks: { + /** First */ + first: string; + /** Last */ + last: string; + /** Next */ + next?: string | null; + /** Prev */ + prev?: string | null; + /** Self */ + self: string; + }; + /** + * ListMeta + * @description An entity list can afford the COUNT(*) a facet cannot, so it reports a real total. + */ + ListMeta: { + /** Page */ + page: number; + /** Page Size */ + page_size: number; + /** Total Count */ + total_count: number; + /** Total Pages */ + total_pages: number; + }; /** * ListPluginsResponse * @description Response from listing plugins. @@ -24937,6 +25005,18 @@ export interface components { /** Prompts */ prompts: components["schemas"]["PromptSpec"][]; }; + /** + * ListResponse + * @description One page of an entity collection. Rows are flat: no `{type, id, attributes}` wrapper. + */ + ListResponse: { + /** Data */ + data: { + [key: string]: components["schemas"]["JsonValue"]; + }[]; + links: components["schemas"]["ListLinks"]; + meta: components["schemas"]["ListMeta"]; + }; /** * ListRunsResponse * @description Response from listing runs @@ -43416,6 +43496,26 @@ export interface operations { }; }; }; + list_budgets_management_v1_budgets_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ListResponse"]; + }; + }; + }; + }; list_spend_log_end_users_management_v1_spend_logs_end_users_get: { parameters: { query: { From 86da406f998d42980b283106de674f4b52193c9e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:07:51 -0700 Subject: [PATCH 05/50] fix(type-discipline): exempt values frozen in place by tuple/frozenset/MappingProxyType from LIT002 --- scripts/check_type_discipline.py | 34 +++++++++++++++++-- .../test_check_type_discipline.py | 16 +++++++++ type-discipline-budget.json | 2 +- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/scripts/check_type_discipline.py b/scripts/check_type_discipline.py index 809dc141eb8..88679f28190 100644 --- a/scripts/check_type_discipline.py +++ b/scripts/check_type_discipline.py @@ -20,7 +20,10 @@ LIT002 Mutable-collection *construction*: a list/dict/set literal or comprehens generator (`tuple(f(x) for x in xs)`), a tuple literal, or a frozen dataclass / NamedTuple / ReadOnly TypedDict. Generator expressions and `tuple`/`frozenset` calls are not construction and pass. Annotation-internal lists (`Callable[[int], - str]`) are exempt. Suppress with `# mutable-ok: `. + str]`) are exempt, as is a value passed directly to a freezing wrapper + (`tuple(...)`, `frozenset(...)`, `MappingProxyType(...)`): it is frozen before + it can escape, though anything mutable nested inside it still counts. + Suppress with `# mutable-ok: `. LIT003 noqa suppression without rule codes or without a reason. Required shape: `# noqa: TID251 # ` LIT004 pyright/mypy ignore without bracketed codes or without a reason. @@ -90,6 +93,7 @@ MUTABLE_CONSTRUCTORS = frozenset(( # are common methods (e.g. pydantic's `model.dict()`), not collection construction. A # qualified `collections.deque(...)` still counts. QUALIFIED_CONSTRUCTORS = MUTABLE_CONSTRUCTORS - frozenset(("dict", "list", "set")) +FREEZING_WRAPPERS = frozenset(("tuple", "frozenset", "MappingProxyType")) UNSAFE_GUARDS = frozenset(("TypeGuard", "TypeIs")) MIN_REASON_LEN = 3 @@ -382,6 +386,31 @@ def _annotation_node_ids(tree: ast.AST) -> frozenset[int]: ) +def _callable_name(func: ast.expr) -> str | None: + if isinstance(func, ast.Name): + return func.id + if isinstance(func, ast.Attribute): + return func.attr + return None + + +def _frozen_argument_ids(tree: ast.AST) -> frozenset[int]: + """ids() of every expression passed directly to a freezing wrapper. + + `MappingProxyType({...})`, `frozenset({...})`, and `tuple([...])` freeze their + argument before it can escape, so the literal inside is a one-shot build, not a + mutable value anyone can grow later. Only the argument itself is exempt; a + mutable collection nested inside it still trips LIT002. + """ + return frozenset( + id(node.args[0]) + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and len(node.args) == 1 + and _callable_name(node.func) in FREEZING_WRAPPERS + ) + + def _construction_kind(node: ast.expr) -> str | None: """Human label if `node` builds a mutable collection, else None.""" if isinstance(node, ast.List): @@ -407,8 +436,9 @@ def _construction_kind(node: ast.expr) -> str | None: def iter_construction_violations(path: Path, tree: ast.AST, comments: Comments) -> Iterator[Violation]: in_annotation = _annotation_node_ids(tree) + frozen_arguments = _frozen_argument_ids(tree) for node in ast.walk(tree): - if not isinstance(node, ast.expr) or id(node) in in_annotation: + if not isinstance(node, ast.expr) or id(node) in in_annotation or id(node) in frozen_arguments: continue kind = _construction_kind(node) if kind is None or node.lineno in comments.mutable_ok_lines: diff --git a/tests/test_litellm/test_check_type_discipline.py b/tests/test_litellm/test_check_type_discipline.py index 13edf6d1a95..f624eb926d1 100644 --- a/tests/test_litellm/test_check_type_discipline.py +++ b/tests/test_litellm/test_check_type_discipline.py @@ -152,6 +152,22 @@ def test_qualified_collections_constructors_still_count(tmp_path): assert "LIT002" in _codes(tmp_path, "import collections\nm = collections.defaultdict(list)\n") +def test_value_frozen_by_wrapper_is_exempt(tmp_path): + assert "LIT002" not in _codes(tmp_path, "from types import MappingProxyType\nm = MappingProxyType({'a': 1})\n") + assert "LIT002" not in _codes(tmp_path, "import types\nm = types.MappingProxyType({'a': 1})\n") + assert "LIT002" not in _codes(tmp_path, "from types import MappingProxyType\nm = MappingProxyType(dict(a=1))\n") + assert "LIT002" not in _codes(tmp_path, "f = frozenset({1, 2})\n") + assert "LIT002" not in _codes(tmp_path, "t = tuple([1, 2])\n") + + +def test_mutable_nested_inside_frozen_wrapper_still_counts(tmp_path): + assert "LIT002" in _codes(tmp_path, "from types import MappingProxyType\nm = MappingProxyType({'a': []})\n") + + +def test_unfrozen_literal_still_counts(tmp_path): + assert "LIT002" in _codes(tmp_path, "from types import MappingProxyType\nd = {'a': 1}\nm = MappingProxyType(d)\n") + + def test_mutable_ok_with_reason_suppresses_both_rules(tmp_path): codes = _codes(tmp_path, "x: dict[str, int] = {} # mutable-ok: in-place buffer mutated hot path\n") assert "LIT001" not in codes diff --git a/type-discipline-budget.json b/type-discipline-budget.json index c9a1b59cc06..2d5e4dd3a50 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -3,7 +3,7 @@ "limit": 23253 }, "LIT002": { - "limit": 27427 + "limit": 27280 }, "LIT003": { "limit": 292 From 089a4fa228db8bcf28f78db56b6a37e61f061bb6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:26:05 -0700 Subject: [PATCH 06/50] fix(lint): restrict freezing-wrapper match to bare names and types.MappingProxyType --- scripts/check_type_discipline.py | 21 +++++++++++-------- .../test_check_type_discipline.py | 6 ++++++ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/scripts/check_type_discipline.py b/scripts/check_type_discipline.py index 88679f28190..43a4cb66484 100644 --- a/scripts/check_type_discipline.py +++ b/scripts/check_type_discipline.py @@ -386,12 +386,15 @@ def _annotation_node_ids(tree: ast.AST) -> frozenset[int]: ) -def _callable_name(func: ast.expr) -> str | None: +def _is_freezing_wrapper(func: ast.expr) -> bool: if isinstance(func, ast.Name): - return func.id - if isinstance(func, ast.Attribute): - return func.attr - return None + return func.id in FREEZING_WRAPPERS + return ( + isinstance(func, ast.Attribute) + and func.attr == "MappingProxyType" + and isinstance(func.value, ast.Name) + and func.value.id == "types" + ) def _frozen_argument_ids(tree: ast.AST) -> frozenset[int]: @@ -400,14 +403,14 @@ def _frozen_argument_ids(tree: ast.AST) -> frozenset[int]: `MappingProxyType({...})`, `frozenset({...})`, and `tuple([...])` freeze their argument before it can escape, so the literal inside is a one-shot build, not a mutable value anyone can grow later. Only the argument itself is exempt; a - mutable collection nested inside it still trips LIT002. + mutable collection nested inside it still trips LIT002. Only bare names (plus + `types.MappingProxyType`) qualify, so an unrelated method that happens to share + a wrapper's name cannot exempt its argument. """ return frozenset( id(node.args[0]) for node in ast.walk(tree) - if isinstance(node, ast.Call) - and len(node.args) == 1 - and _callable_name(node.func) in FREEZING_WRAPPERS + if isinstance(node, ast.Call) and len(node.args) == 1 and _is_freezing_wrapper(node.func) ) diff --git a/tests/test_litellm/test_check_type_discipline.py b/tests/test_litellm/test_check_type_discipline.py index f624eb926d1..53d672fc4a8 100644 --- a/tests/test_litellm/test_check_type_discipline.py +++ b/tests/test_litellm/test_check_type_discipline.py @@ -160,6 +160,12 @@ def test_value_frozen_by_wrapper_is_exempt(tmp_path): assert "LIT002" not in _codes(tmp_path, "t = tuple([1, 2])\n") +def test_same_named_method_does_not_exempt_its_argument(tmp_path): + assert "LIT002" in _codes(tmp_path, "t = obj.tuple([1, 2])\n") + assert "LIT002" in _codes(tmp_path, "f = obj.frozenset({1, 2})\n") + assert "LIT002" in _codes(tmp_path, "m = obj.MappingProxyType({'a': 1})\n") + + def test_mutable_nested_inside_frozen_wrapper_still_counts(tmp_path): assert "LIT002" in _codes(tmp_path, "from types import MappingProxyType\nm = MappingProxyType({'a': []})\n") From ffd6ac52c5ddd1321b07761cd1f3d204ddb3cdc8 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 00:08:20 -0700 Subject: [PATCH 07/50] fix(deps): raise aiohttp floor to 3.14.2 to clear pooled-connection timeouts aiohttp 3.14.0 and 3.14.1 re-arm the sock_read timer on a keep-alive connection after it has already been returned to the idle pool. The stray timer stamps a SocketTimeoutError on the pooled connection without closing it, so the pool keeps handing it out and the next request to pick it up fails instantly on an error left behind by an earlier, unrelated request. Because a single pool is shared across providers, the failures appear simultaneously across Vertex AI, Bedrock, Anthropic and OpenAI-compatible deployments as sub-millisecond "Connection timed out" errors. uv.lock resolved aiohttp 3.14.1 and the published images install via `uv sync --frozen`, so every image built from that lock shipped the regression. The wheel's own metadata declared `aiohttp>=3.10,<4.0`, which also left pip consumers free to resolve into the same broken window, so both the runtime floor and the uv constraint move to >=3.14.2. Upstream fixed this in aio-libs/aiohttp#12954, released in aiohttp 3.14.2; the lock now resolves 3.14.3. Raising the floor rather than capping below 3.14 keeps the advisories that the existing 3.14.1 floor cleared, so no osv-scanner ignores are needed. litellm requires Python >=3.10 and aiohttp 3.14.2 requires >=3.10, so no supported interpreter loses support. Both new tests fail on the previous pins and pass on these. --- pyproject.toml | 4 +- .../test_basic_python_version.py | 71 +++++ uv.lock | 246 +++++++++--------- 3 files changed, 196 insertions(+), 125 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 93fb32da464..678e7384a05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "tokenizers>=0.21.0,<1.0", "click>=8.0.0,<9.0", "jinja2>=3.1.6,<4.0", - "aiohttp>=3.10,<4.0", + "aiohttp>=3.14.2,<4.0", "pydantic>=2.10.0,<3.0.0", "jsonschema>=4.0.0,<5.0", ] @@ -277,7 +277,7 @@ exclude = [ [tool.uv] constraint-dependencies = [ "tornado>=6.5.6", - "aiohttp>=3.14.1,<4.0", + "aiohttp>=3.14.2,<4.0", "packaging>=24.0", "soupsieve>=2.8.4", "httplib2>=0.32.0", diff --git a/tests/local_testing/test_basic_python_version.py b/tests/local_testing/test_basic_python_version.py index e31c3953714..1f260f86eeb 100644 --- a/tests/local_testing/test_basic_python_version.py +++ b/tests/local_testing/test_basic_python_version.py @@ -142,6 +142,77 @@ def test_cli_extra_is_a_thin_client_install(): assert not leaked, f"`cli` extra leaks proxy-server deps onto laptops: {leaked}" +AIOHTTP_POOL_POISONING_RANGE = ">=3.14.0,<3.14.2" +AIOHTTP_POOL_POISONING_RELEASES = ("3.14.0", "3.14.1") + + +def _load_toml(path): + try: + import tomllib as tomli + except ImportError: + try: + import tomli + except ImportError: + pytest.skip("tomli/tomllib not available - skipping dependency check") + + with open(path, "rb") as f: + return tomli.load(f) + + +def _declared_aiohttp_specifier(): + from packaging.requirements import Requirement + + pyproject = _load_toml(os.path.join(PROJECT_ROOT, "pyproject.toml")) + for requirement in pyproject["project"]["dependencies"]: + parsed = Requirement(requirement) + if parsed.name.lower() == "aiohttp": + return parsed.specifier + pytest.fail("aiohttp is no longer a declared runtime dependency of litellm") + + +def _locked_aiohttp_version(): + lock = _load_toml(os.path.join(PROJECT_ROOT, "uv.lock")) + for package in lock["package"]: + if package["name"].lower() == "aiohttp": + return package["version"] + pytest.fail("aiohttp is missing from uv.lock") + + +def test_declared_aiohttp_floor_excludes_pool_poisoning_releases(): + """aiohttp 3.14.0/3.14.1 re-arm the sock_read timer on a keep-alive connection + after it is back in the idle pool, so the next request to reuse it fails + instantly with a bogus timeout (aio-libs/aiohttp#12953, fixed in 3.14.2). + + The wheel's own metadata is what pip resolves against, so the floor declared + here - not just the lockfile - has to exclude that range. + """ + specifier = _declared_aiohttp_specifier() + + admitted = [v for v in AIOHTTP_POOL_POISONING_RELEASES if specifier.contains(v)] + assert not admitted, ( + f"litellm declares aiohttp{specifier}, which still admits {admitted}. " + "Those releases poison pooled keep-alive connections and cause " + "cross-provider sub-millisecond 'Connection timed out' failures; " + "keep the floor at >=3.14.2." + ) + + +def test_locked_aiohttp_version_is_not_pool_poisoning(): + """uv.lock is what the published Docker images install (uv sync --frozen), so a + lock that drifts back onto 3.14.0/3.14.1 ships the regression regardless of + what pyproject.toml declares. + """ + from packaging.specifiers import SpecifierSet + + locked = _locked_aiohttp_version() + + assert not SpecifierSet(AIOHTTP_POOL_POISONING_RANGE).contains(locked), ( + f"uv.lock resolves aiohttp {locked}, which is inside the pool-poisoning " + f"range {AIOHTTP_POOL_POISONING_RANGE} (aio-libs/aiohttp#12953). " + "Re-run `uv lock` against an aiohttp>=3.14.2 floor." + ) + + import os import subprocess import time diff --git a/uv.lock b/uv.lock index d30f2df0a0e..fa7652c67ec 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-07-27T18:40:42.08538Z" +exclude-newer = "2026-07-28T06:59:32.050819Z" exclude-newer-span = "P3D" [manifest] @@ -20,7 +20,7 @@ members = [ "litellm-proxy-extras", ] constraints = [ - { name = "aiohttp", specifier = ">=3.14.1,<4.0" }, + { name = "aiohttp", specifier = ">=3.14.2,<4.0" }, { name = "httplib2", specifier = ">=0.32.0" }, { name = "packaging", specifier = ">=24.0" }, { name = "setuptools", specifier = ">=83.0.0" }, @@ -82,7 +82,7 @@ wheels = [ [[package]] name = "aiohttp" -version = "3.14.1" +version = "3.14.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohappyeyeballs" }, @@ -95,126 +95,126 @@ dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.13'" }, { name = "yarl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/82/78/8ea7308cac6934de8c74a14f3d5f65d1c89287426688be79538d0e5c013d/aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035", size = 7955794, upload-time = "2026-06-07T21:09:35.529Z" } +sdist = { url = "https://files.pythonhosted.org/packages/58/d9/22ce5786ac0c1653ae8b6c23bded02c1686d11f0dbb45b31ce128e0df985/aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc", size = 7971213, upload-time = "2026-07-23T01:57:27.037Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/67/58ded4b3f2e10f94972d8928050c85330e249a31dd45a0e5f3c0e9c3fa05/aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e", size = 766140, upload-time = "2026-06-07T21:05:37.471Z" }, - { url = "https://files.pythonhosted.org/packages/18/68/4ae5b4e08943f316594bb68da89957d3baf5760588fa09509594bd777e4b/aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491", size = 519430, upload-time = "2026-06-07T21:05:40.751Z" }, - { url = "https://files.pythonhosted.org/packages/cb/c1/316c8f3549dbe5245f92bfd523ec6f32dd4d98cafe21df3f6a19b1184c75/aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce", size = 514406, upload-time = "2026-06-07T21:05:42.111Z" }, - { url = "https://files.pythonhosted.org/packages/5a/ee/fb0ac28684e8d753b83c8a4eebc19a5846912aa0a4daaabb6a9936363840/aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3", size = 1703649, upload-time = "2026-06-07T21:05:43.427Z" }, - { url = "https://files.pythonhosted.org/packages/3b/57/aa2beab673331f111885db8a7b69dfe3ab0e53e446a0ace18ca694b4dc58/aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505", size = 1675126, upload-time = "2026-06-07T21:05:44.897Z" }, - { url = "https://files.pythonhosted.org/packages/47/ea/dad128abe365e79be03b16ed464198ac73e0d257e8260c6f7d6f31cbef26/aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521", size = 1771558, upload-time = "2026-06-07T21:05:46.405Z" }, - { url = "https://files.pythonhosted.org/packages/63/f3/b5b4e10327cb85d34d24232c6b71b64602f190b3ccb238a043ac6b187dac/aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd", size = 1856631, upload-time = "2026-06-07T21:05:47.844Z" }, - { url = "https://files.pythonhosted.org/packages/2b/9d/93294c3045775c708ac8310eb3d3622a11d2951345ad590d532d62a1faa4/aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb", size = 1714139, upload-time = "2026-06-07T21:05:49.982Z" }, - { url = "https://files.pythonhosted.org/packages/29/c4/93067c85a0373492ce8e577435203c5947c454af074ac48ed4f3a1b9dd4a/aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42", size = 1588321, upload-time = "2026-06-07T21:05:51.431Z" }, - { url = "https://files.pythonhosted.org/packages/c4/39/9ff91aaf02af8b7b8222a987466da539f154c3e01732c22b5f5a20a8ee66/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b", size = 1670375, upload-time = "2026-06-07T21:05:53.109Z" }, - { url = "https://files.pythonhosted.org/packages/aa/e4/77452a3676b8d99ac1375f77691d6bf65ea6e9f4b201b82ef77c916dc767/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192", size = 1690933, upload-time = "2026-06-07T21:05:54.902Z" }, - { url = "https://files.pythonhosted.org/packages/7d/84/b0059a7c7fc05ea23f3bc1596ba91c12f79588b9450564a24cac37536d0a/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05", size = 1740798, upload-time = "2026-06-07T21:05:56.458Z" }, - { url = "https://files.pythonhosted.org/packages/8f/3a/e2a513ecbfc362591caa51a7f7e011b3bfc8938b388ae44cd95560d36999/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe", size = 1576412, upload-time = "2026-06-07T21:05:57.953Z" }, - { url = "https://files.pythonhosted.org/packages/a1/10/08f1654f538f93d36dcac66310a06eefce4641cdafca83f9f0a5317be254/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d", size = 1750199, upload-time = "2026-06-07T21:05:59.488Z" }, - { url = "https://files.pythonhosted.org/packages/99/e4/d91b70c57d8b8e9611e4a2e52238ca3698d3dc1c2efe25b7a9bf594ac584/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966", size = 1699356, upload-time = "2026-06-07T21:06:01.131Z" }, - { url = "https://files.pythonhosted.org/packages/3d/f1/15340176f35ff61b95dbe34020bcf43f9e624a2d7bbac934715ff97d2033/aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6", size = 458939, upload-time = "2026-06-07T21:06:02.86Z" }, - { url = "https://files.pythonhosted.org/packages/c3/c2/a2f1ec5b37f903109e43ae2862268cfe4a67a60c1b2cf43169fcdff5995f/aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df", size = 482583, upload-time = "2026-06-07T21:06:04.666Z" }, - { url = "https://files.pythonhosted.org/packages/d0/7a/7b56f6732ef79530afaa72aa335d41b67c8d79b946995f0b11ad72985435/aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c", size = 453470, upload-time = "2026-06-07T21:06:06.322Z" }, - { url = "https://files.pythonhosted.org/packages/26/dd/bf526e6f0a1120dd6f2df2e97bacfe4d358f13d17a0ff5847301a1375a51/aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2", size = 765225, upload-time = "2026-06-07T21:06:07.957Z" }, - { url = "https://files.pythonhosted.org/packages/8f/e1/a2872aa55495a70f61310d411541c6ee23812d9a884e000c716e1bc3edbf/aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f", size = 518743, upload-time = "2026-06-07T21:06:09.749Z" }, - { url = "https://files.pythonhosted.org/packages/5b/e7/c60c7b209e509cc787de3cea0550a518538cfc08003e1c1e14c1c63fff71/aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8", size = 514139, upload-time = "2026-06-07T21:06:11.26Z" }, - { url = "https://files.pythonhosted.org/packages/5b/8d/614ace2f579702c9840ab1e1447fd8509e35b0b904f7196418fa2f57b25d/aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04", size = 1784088, upload-time = "2026-06-07T21:06:12.887Z" }, - { url = "https://files.pythonhosted.org/packages/49/e0/726e90f99542bf292f81a96a12cc4847deb86f3ccf62c6f4014a201f4d33/aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8", size = 1737835, upload-time = "2026-06-07T21:06:14.564Z" }, - { url = "https://files.pythonhosted.org/packages/0b/4b/d176d5c4db9d33dacf0543102ea59503bc1d528af4cfd0b719949ca49389/aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6", size = 1842801, upload-time = "2026-06-07T21:06:16.228Z" }, - { url = "https://files.pythonhosted.org/packages/dc/d6/5a99b563690ea0cbed912ae94a2ce33993a5709a651a3a4fe761e7dd973a/aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af", size = 1929992, upload-time = "2026-06-07T21:06:17.947Z" }, - { url = "https://files.pythonhosted.org/packages/76/7f/a987b14a3859094b3cea3f4825219c3e5536242564af6e3f9c2f6c994eb2/aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730", size = 1786989, upload-time = "2026-06-07T21:06:19.677Z" }, - { url = "https://files.pythonhosted.org/packages/f1/1a/420e5c85a3e73349372ed22ce0b6af86bfa6ce16a4b20a64a2e94608c781/aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621", size = 1640129, upload-time = "2026-06-07T21:06:22.558Z" }, - { url = "https://files.pythonhosted.org/packages/a7/80/18a592ed3be0a402cc03670bd72ee1f8563ddbe1d8d5542dbf868f274136/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee", size = 1756576, upload-time = "2026-06-07T21:06:24.8Z" }, - { url = "https://files.pythonhosted.org/packages/ec/0b/8b3d5713373858ff71a617daf6e3b0e81ad63e79d09a3cf2f6b6b983939c/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573", size = 1754668, upload-time = "2026-06-07T21:06:26.528Z" }, - { url = "https://files.pythonhosted.org/packages/9f/49/fd564575cf225821d7ba5a117cb8bc27213d8a7e1811162afb43ae077039/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7", size = 1817019, upload-time = "2026-06-07T21:06:28.297Z" }, - { url = "https://files.pythonhosted.org/packages/ed/1b/e850c9ae6fc91356552ae668bb6c51e93fa29c8aef13398a10b56678557f/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf", size = 1631638, upload-time = "2026-06-07T21:06:30.242Z" }, - { url = "https://files.pythonhosted.org/packages/eb/94/3c337ba72451a89806ace6f75bddc92bafc5b8d53d90115a512858024b63/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85", size = 1835660, upload-time = "2026-06-07T21:06:31.943Z" }, - { url = "https://files.pythonhosted.org/packages/2b/9c/9c18cf367a0498212d9ba7daf990b504a5e8ae064cda4b504e2647c89c03/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3", size = 1775698, upload-time = "2026-06-07T21:06:33.72Z" }, - { url = "https://files.pythonhosted.org/packages/b5/63/a251a9d2a6cb45065b2ddc0bde2b3dd10108740a9a42f632c66405a761a2/aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126", size = 458386, upload-time = "2026-06-07T21:06:35.279Z" }, - { url = "https://files.pythonhosted.org/packages/17/ca/69274c51dcd6e8947d77b2806cf47a4a15f2c846e2cbeb1882547d3da283/aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5", size = 483406, upload-time = "2026-06-07T21:06:36.824Z" }, - { url = "https://files.pythonhosted.org/packages/2c/8a/c25904f77690c3688ec140f87591ef11a0cfe36bf3d5c0f1f38056fb62b3/aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b", size = 452987, upload-time = "2026-06-07T21:06:38.371Z" }, - { url = "https://files.pythonhosted.org/packages/1d/21/151624b51cd92553d95424daf4bf19f19ce9be9002d19253e7e7ce67197b/aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480", size = 757402, upload-time = "2026-06-07T21:06:40.311Z" }, - { url = "https://files.pythonhosted.org/packages/c2/82/280619e0bd7bf2454987e19282616e84762255dd9c8468f62382e8c191f1/aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d", size = 512310, upload-time = "2026-06-07T21:06:42.207Z" }, - { url = "https://files.pythonhosted.org/packages/55/b2/2aac325583aaa1353045f96dffa586d8a34e8322e14a7ba49cffeb103ab4/aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2", size = 512448, upload-time = "2026-06-07T21:06:43.813Z" }, - { url = "https://files.pythonhosted.org/packages/8a/72/a60607cb849faa8af8a356c9329ea2eb6f395d49e82cc82ccba1fd8deb8f/aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2", size = 1766854, upload-time = "2026-06-07T21:06:45.391Z" }, - { url = "https://files.pythonhosted.org/packages/b5/d3/d9fe1c9ec7557ab4d0d82bebaa728c6418f0b93295ec2f4ab015f7710cc7/aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a", size = 1740884, upload-time = "2026-06-07T21:06:47.413Z" }, - { url = "https://files.pythonhosted.org/packages/c1/dc/f2cecfaf9337ba3e63f181500814ff502aa3d00d9c7ec93a9d23d10a27b2/aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264", size = 1810034, upload-time = "2026-06-07T21:06:50.165Z" }, - { url = "https://files.pythonhosted.org/packages/66/d7/2ff65c5e65c0d7476daf7e15c032e0805e36811185b9623e3238ad6c763e/aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842", size = 1904054, upload-time = "2026-06-07T21:06:52.035Z" }, - { url = "https://files.pythonhosted.org/packages/20/9c/d445818389df371f56d141d881153ba23183c4735a03f7356ffb43f7757d/aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c", size = 1790278, upload-time = "2026-06-07T21:06:54.049Z" }, - { url = "https://files.pythonhosted.org/packages/4d/aa/bf04cb4d865fc6101c2229a294ad744973b72e513fdc5a6b791e6983d72a/aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95", size = 1591795, upload-time = "2026-06-07T21:06:55.911Z" }, - { url = "https://files.pythonhosted.org/packages/dc/b4/4dac0038960427ba832f6609dfb4ea5437d7fd80c72001b9e48f834f428b/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199", size = 1728397, upload-time = "2026-06-07T21:06:57.777Z" }, - { url = "https://files.pythonhosted.org/packages/2b/f9/7cd4e8ad7aa3b75f17d56bb5498dd604a93d4e6eece822ba0568c413fff0/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817", size = 1766504, upload-time = "2026-06-07T21:07:00.009Z" }, - { url = "https://files.pythonhosted.org/packages/f9/df/fc01d9fcad0f73fed3f3d361f1f94f975947b50dff82919f6dc2bf4316cc/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a", size = 1777806, upload-time = "2026-06-07T21:07:02.064Z" }, - { url = "https://files.pythonhosted.org/packages/41/09/47e2d090bddcc8fb4ccb4c314aadc32d7c5d9bb55f50f6ad1c92fc15d501/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4", size = 1580707, upload-time = "2026-06-07T21:07:03.942Z" }, - { url = "https://files.pythonhosted.org/packages/3d/36/f1a4ce904ae0b6930cfe9afc96d0896f7ec1a620c400405d63783bb95a9c/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087", size = 1798121, upload-time = "2026-06-07T21:07:05.987Z" }, - { url = "https://files.pythonhosted.org/packages/70/0a/e0075ce9ca0279ee1d4f0c0b85f54fea02ebc83c3007651a72bece658fec/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3", size = 1767580, upload-time = "2026-06-07T21:07:07.873Z" }, - { url = "https://files.pythonhosted.org/packages/3e/61/a0c0a8f327a9c52095cdd8e312391b00d3ed64ab6c72bb5c33d8ec251cf7/aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4", size = 452771, upload-time = "2026-06-07T21:07:09.669Z" }, - { url = "https://files.pythonhosted.org/packages/df/d9/ea367c75f16ac9c6cdc8febb25e8318fa21a2b1bc8d6514d4b2d890bface/aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271", size = 479873, upload-time = "2026-06-07T21:07:11.538Z" }, - { url = "https://files.pythonhosted.org/packages/03/64/8d96784a7851156db8a4c6c3f6f91042fdf39fb15a4cc38c8b3c14833c45/aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847", size = 448073, upload-time = "2026-06-07T21:07:13.637Z" }, - { url = "https://files.pythonhosted.org/packages/bc/97/bd137012dd97e1649162b099135a80e1fd59aaa807b2430fc448d1029aff/aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178", size = 506882, upload-time = "2026-06-07T21:07:15.501Z" }, - { url = "https://files.pythonhosted.org/packages/ef/79/e5cc690e9d922a66887ceeaca53a8ffd5a7b0be3816142b7abc433742d89/aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf", size = 515270, upload-time = "2026-06-07T21:07:17.53Z" }, - { url = "https://files.pythonhosted.org/packages/fe/22/a73ccbf9dbd6e26dda0b24d5fd5db7da92ee3383a79f47677ffb834c5c5b/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd", size = 485841, upload-time = "2026-06-07T21:07:19.555Z" }, - { url = "https://files.pythonhosted.org/packages/3b/b9/57ed8eaf596321c2ad747bd480fb1700dbd7177c60dfc9e4c187f629662e/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe", size = 492088, upload-time = "2026-06-07T21:07:21.581Z" }, - { url = "https://files.pythonhosted.org/packages/78/c0/5ebe5270a7c140d7c6f79dcb018640225f14d406c149e4eec04a7d82fe71/aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4", size = 501564, upload-time = "2026-06-07T21:07:23.388Z" }, - { url = "https://files.pythonhosted.org/packages/75/7f/8cdaa24fc7983865e0915153b96a9ac5bcdd3548d64c5a27d17cecccad2d/aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876", size = 751998, upload-time = "2026-06-07T21:07:25.046Z" }, - { url = "https://files.pythonhosted.org/packages/b2/f4/c4227aacfacc5cb0cc2d119b65301d177912a6842cd64e120c47af76064f/aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da", size = 510918, upload-time = "2026-06-07T21:07:27.28Z" }, - { url = "https://files.pythonhosted.org/packages/ab/01/a2d5f96cd4e74424864d30bc0a7e44d0a12dacdcfa91b5b2d1bd3dca6bf3/aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6", size = 508657, upload-time = "2026-06-07T21:07:29.252Z" }, - { url = "https://files.pythonhosted.org/packages/e8/ed/3c0fb5c500fdd8e7ebc10d1889c04384fffa1a9163eac1356088ca9da1b1/aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96", size = 1757907, upload-time = "2026-06-07T21:07:31.03Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ab/d4c924d9bd5be3050c226612413ce68cb54c70d2c31b661bfc8d9a5b6a70/aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f", size = 1737565, upload-time = "2026-06-07T21:07:33.031Z" }, - { url = "https://files.pythonhosted.org/packages/19/2a/37326821ff779084020cdc33224d20b19f42f4183a500ff92022a739eda7/aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296", size = 1799018, upload-time = "2026-06-07T21:07:35.003Z" }, - { url = "https://files.pythonhosted.org/packages/b3/4f/6e947ba73e4ce09070761c05ed3a8ceb7c21f5e46798671d8b2aac0e4626/aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa", size = 1894416, upload-time = "2026-06-07T21:07:36.956Z" }, - { url = "https://files.pythonhosted.org/packages/9d/6e/dbf1d0625dc711fb2851f4f3c3055c39ed58bae92082d8c627dbe6013736/aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451", size = 1783881, upload-time = "2026-06-07T21:07:39.063Z" }, - { url = "https://files.pythonhosted.org/packages/44/c2/5e25098a67268ed369483ae7d1a58bd0a13d03aab860d2a0e4a6eb25b046/aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c", size = 1587572, upload-time = "2026-06-07T21:07:41.058Z" }, - { url = "https://files.pythonhosted.org/packages/2a/bd/cf9cee17e140f942a3de73e658a543aa8fbf35a5fc67a9d2538d52d77f0b/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca", size = 1722137, upload-time = "2026-06-07T21:07:43.014Z" }, - { url = "https://files.pythonhosted.org/packages/89/6d/5684f8c59045c96f81a18cefbc1fbbd79d25b88f1c622f2a5c5c08fcb632/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09", size = 1755953, upload-time = "2026-06-07T21:07:45.933Z" }, - { url = "https://files.pythonhosted.org/packages/a8/40/35caf3170f8359760740a7d9aa0fff2e344bef98e1d1186f5a0f6dec17e6/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397", size = 1766479, upload-time = "2026-06-07T21:07:48.047Z" }, - { url = "https://files.pythonhosted.org/packages/6d/a1/b0c61e7a137f0d81de49a82023a6df73c3c16d6fefb0f8e4a93d21639002/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080", size = 1580077, upload-time = "2026-06-07T21:07:50.069Z" }, - { url = "https://files.pythonhosted.org/packages/0b/41/194ea4623693009fcefebef7aef63c141754f153e9cd0d39d3b9e36c175c/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345", size = 1791688, upload-time = "2026-06-07T21:07:52.106Z" }, - { url = "https://files.pythonhosted.org/packages/ba/45/4de841f005cfe1fd63e2a2fe011262c515e2a62aa6994b15947e7d717ac9/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588", size = 1761094, upload-time = "2026-06-07T21:07:54.113Z" }, - { url = "https://files.pythonhosted.org/packages/e4/ae/dbce10533d3896d544d5053939ed75b7dc31a1b0973d959b1b5ae21028d6/aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780", size = 452662, upload-time = "2026-06-07T21:07:56.06Z" }, - { url = "https://files.pythonhosted.org/packages/7b/d9/0bf1a19362c32f06229da5e7ddfcec91f93474d6307f7a2d3135e9c674dc/aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a", size = 479748, upload-time = "2026-06-07T21:07:58.319Z" }, - { url = "https://files.pythonhosted.org/packages/22/0a/62e7232dc9484fbec112ceb32efb6a624cc7994ec6e2b019286f17c4e8f2/aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8", size = 447723, upload-time = "2026-06-07T21:08:00.154Z" }, - { url = "https://files.pythonhosted.org/packages/c4/a1/5fafa04e1ca91ddb47608699d60649c1c6db3cf41c99e78fc4056f9513db/aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15", size = 508531, upload-time = "2026-06-07T21:08:02.093Z" }, - { url = "https://files.pythonhosted.org/packages/fa/2e/bfa02f699d87ffc86d5959270b28f1cb410add3ccaced8ed2e0b8a5238fc/aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8", size = 514718, upload-time = "2026-06-07T21:08:04.476Z" }, - { url = "https://files.pythonhosted.org/packages/85/a5/9594ad6289eebbc97d167c44213d557807f90e59115caad24de21ad2c3b1/aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3", size = 487918, upload-time = "2026-06-07T21:08:06.377Z" }, - { url = "https://files.pythonhosted.org/packages/b4/61/16a32c36c3c49edec122a3dc811f2057df2f94d3b14aa107c8017d981618/aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba", size = 494014, upload-time = "2026-06-07T21:08:08.263Z" }, - { url = "https://files.pythonhosted.org/packages/9b/89/3ebcf96ed99c05bec9c434aaac6963fd3cbab4a786ae739908a144d9ce44/aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397", size = 502398, upload-time = "2026-06-07T21:08:10.244Z" }, - { url = "https://files.pythonhosted.org/packages/fd/3d/b74870a0c2d40c355928cd5b96c7a11fa821b8a40fc41365e64479b151fb/aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448", size = 758018, upload-time = "2026-06-07T21:08:12.447Z" }, - { url = "https://files.pythonhosted.org/packages/d3/66/f42f5c984d99e49c6cff5f26f590750f2e2f7ef1fcfb99966ab5be1b632e/aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004", size = 512462, upload-time = "2026-06-07T21:08:14.624Z" }, - { url = "https://files.pythonhosted.org/packages/e9/a7/248e1aebe0c7810b0271e021a0f2a5eb6e78a051885b3c9df49f42a5802d/aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983", size = 512824, upload-time = "2026-06-07T21:08:16.572Z" }, - { url = "https://files.pythonhosted.org/packages/26/97/2aa0e5ba0727dc3bd5aaebb7ccbc510f7dfb7fb961ec87497cd496635ab1/aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe", size = 1749898, upload-time = "2026-06-07T21:08:18.635Z" }, - { url = "https://files.pythonhosted.org/packages/00/8d/e97f6c96c891d457c8479d92a514ba194d0412f981d72c70341ee18488ed/aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333", size = 1710114, upload-time = "2026-06-07T21:08:20.892Z" }, - { url = "https://files.pythonhosted.org/packages/6f/e6/aa8d7e863048c8fceb5cd6ce74017311cec3ead07847387e12265fb4444e/aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0", size = 1802541, upload-time = "2026-06-07T21:08:23.044Z" }, - { url = "https://files.pythonhosted.org/packages/83/a8/72193137de57fda4ebfae4563182d082c8856e3b6e9871d0b46f028fb369/aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a", size = 1875776, upload-time = "2026-06-07T21:08:25.288Z" }, - { url = "https://files.pythonhosted.org/packages/a0/18/938441025db6769a3464596b2410af3afde0b21eb2f204c6f766f68af4bd/aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602", size = 1760329, upload-time = "2026-06-07T21:08:27.363Z" }, - { url = "https://files.pythonhosted.org/packages/60/29/bf2496b4065e76e09fe48015aaffe5ce161d8f089b06ac6982070f653076/aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca", size = 1587293, upload-time = "2026-06-07T21:08:29.805Z" }, - { url = "https://files.pythonhosted.org/packages/49/a2/2136674d52123b1354bd05dd5753c318db47dc0c927cc70b27bab3755456/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35", size = 1714756, upload-time = "2026-06-07T21:08:32.094Z" }, - { url = "https://files.pythonhosted.org/packages/a7/b9/e5fd2e6f915503081c0f9b1e8540947037929c70c191da2e4d54b31a21a1/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844", size = 1721052, upload-time = "2026-06-07T21:08:34.167Z" }, - { url = "https://files.pythonhosted.org/packages/63/5a/2833e324a2263e104e31e2e91bc5bbee81bc499afd32203faee048a883f0/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496", size = 1766888, upload-time = "2026-06-07T21:08:36.95Z" }, - { url = "https://files.pythonhosted.org/packages/57/fa/dea6511870913162f3b2e8c42a7614eb203a4540b8c2da43e0bfb0548f3c/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5", size = 1581679, upload-time = "2026-06-07T21:08:39.292Z" }, - { url = "https://files.pythonhosted.org/packages/14/bd/3cf0d55e71784b33534e9710a67d382d900598b4787fbce6cc7317f8c42a/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95", size = 1782021, upload-time = "2026-06-07T21:08:41.407Z" }, - { url = "https://files.pythonhosted.org/packages/c1/af/14bb5843eccbe234f4dfb78ab73e549d99727247e62ae5d62cbd22eaf5b0/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444", size = 1742574, upload-time = "2026-06-07T21:08:43.795Z" }, - { url = "https://files.pythonhosted.org/packages/f2/1e/fbeb7af9210a67ac0f9c9bec0f8f4568497924e33137a3d5b48e1cf85f3f/aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0", size = 457773, upload-time = "2026-06-07T21:08:46.168Z" }, - { url = "https://files.pythonhosted.org/packages/f0/2b/13e8d741a9ec5db7d900c060554cf8352ab85e44e2a4469ebb9d377bda17/aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719", size = 485001, upload-time = "2026-06-07T21:08:48.401Z" }, - { url = "https://files.pythonhosted.org/packages/df/30/491acfa2c4d6c3ff59c49a14fc1b50be3241e25bbb0c84c09e2da4d11395/aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec", size = 453809, upload-time = "2026-06-07T21:08:50.7Z" }, - { url = "https://files.pythonhosted.org/packages/34/e3/19dbe1a1f4cc6230eb9e314de7fe68053b0992f9302b27d12141a0b5db53/aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2", size = 793320, upload-time = "2026-06-07T21:08:52.775Z" }, - { url = "https://files.pythonhosted.org/packages/7f/20/1b7182219ba1b108430d6e4dc53d25ae02dcfcf5a045b33af4e8c5167527/aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340", size = 529077, upload-time = "2026-06-07T21:08:55Z" }, - { url = "https://files.pythonhosted.org/packages/b9/c8/14ce60ec31a2e5f5274bb17d383a6f7a3aabca31ac04eee05585bbadab16/aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d", size = 532476, upload-time = "2026-06-07T21:08:57.176Z" }, - { url = "https://files.pythonhosted.org/packages/7e/02/9ac85e081e53da2e061b02fa7758fe0a12d17b8ce2d1f5e6c7cb76730328/aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94", size = 1922347, upload-time = "2026-06-07T21:08:59.563Z" }, - { url = "https://files.pythonhosted.org/packages/c0/3e/d3ba07a0ab38b5389e10bec4362d21e10a4f667cba2d79ba30837b3a5059/aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc", size = 1786465, upload-time = "2026-06-07T21:09:01.909Z" }, - { url = "https://files.pythonhosted.org/packages/0b/cb/e2ee978a00cfb2df829704a69528b18154eba5939f45bc1efa8f33aee4c5/aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251", size = 1909423, upload-time = "2026-06-07T21:09:04.357Z" }, - { url = "https://files.pythonhosted.org/packages/73/5d/1430334858b1022b58ae50399a918f0bd6fe8fa7fa183598d657ff61e040/aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1", size = 2001906, upload-time = "2026-06-07T21:09:06.722Z" }, - { url = "https://files.pythonhosted.org/packages/66/4e/560c7472d3d198a23aa5c8b19a5115bf6a9b77b7d3e4bb363da320430ad2/aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3", size = 1877095, upload-time = "2026-06-07T21:09:09.011Z" }, - { url = "https://files.pythonhosted.org/packages/0d/f1/4745806578d447db4a784a8591e2dae3afdfc2bcb96f8f81271b13df6543/aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c", size = 1676222, upload-time = "2026-06-07T21:09:11.461Z" }, - { url = "https://files.pythonhosted.org/packages/6a/c9/48255813cca749a229ef0ab476004ec623728ad79a9c0840616f6c076325/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203", size = 1842922, upload-time = "2026-06-07T21:09:14.118Z" }, - { url = "https://files.pythonhosted.org/packages/3d/c0/bbd054e2bee909f529523a5af3891052606af5143c09f5f183ec3b234676/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1", size = 1825035, upload-time = "2026-06-07T21:09:16.447Z" }, - { url = "https://files.pythonhosted.org/packages/a8/ae/90395d4376deceb74e09ec26b6adf7d2015a6f8802d6d84446af860fef04/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665", size = 1849512, upload-time = "2026-06-07T21:09:18.742Z" }, - { url = "https://files.pythonhosted.org/packages/93/bd/fb25f3049957553d4ce0ba6ae480aa2f592a6985497fca590837d16c1be0/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1", size = 1668571, upload-time = "2026-06-07T21:09:21.458Z" }, - { url = "https://files.pythonhosted.org/packages/3f/22/7f73303d64dd567ff3addca90b556690ed1233a47b8f55d242fb90af3681/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d", size = 1881159, upload-time = "2026-06-07T21:09:23.813Z" }, - { url = "https://files.pythonhosted.org/packages/44/be/0474c5a8b5640e1e4aa1923430a91f4151be82e511373fe764189b89aef5/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c", size = 1841409, upload-time = "2026-06-07T21:09:26.207Z" }, - { url = "https://files.pythonhosted.org/packages/7b/3c/bb4a7cba26956cb3da4553cc2056cf67be5b5ff6e6d8fa4fbdff73bfb7ae/aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365", size = 494166, upload-time = "2026-06-07T21:09:28.505Z" }, - { url = "https://files.pythonhosted.org/packages/8a/84/ec80c2c1f66a952555a9f86df6b33af65108a6febfa0471b69013a12f807/aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9", size = 530255, upload-time = "2026-06-07T21:09:30.843Z" }, - { url = "https://files.pythonhosted.org/packages/2a/71/6e22be134a4061ada85a92951b842f2657f17d926b727f3f94c56ae963d6/aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6", size = 469640, upload-time = "2026-06-07T21:09:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/2d/4d/4a99fb425c5e0cad715eea7bd190aff46f38b959a0a2dadb993705d34b26/aiohttp-3.14.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b", size = 765848, upload-time = "2026-07-23T01:52:08.217Z" }, + { url = "https://files.pythonhosted.org/packages/74/e8/43b85dc55b8e950dc644babe762add781319ea881b57b33d2cce12017d12/aiohttp-3.14.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a", size = 517476, upload-time = "2026-07-23T01:52:10.846Z" }, + { url = "https://files.pythonhosted.org/packages/7f/9e/73b582c4dbbc3c12ef4473822475effaabf1f934b56f14f5b03fe5d3a2af/aiohttp-3.14.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5", size = 515334, upload-time = "2026-07-23T01:52:12.636Z" }, + { url = "https://files.pythonhosted.org/packages/79/03/e98c3c9e05a5bdf97defe5ff9169baba4f0ec9a901f2d60e0f060c2f051e/aiohttp-3.14.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f", size = 1708830, upload-time = "2026-07-23T01:52:14.538Z" }, + { url = "https://files.pythonhosted.org/packages/d7/2c/26e60b694844dfd2176c57f913a22d0cd6a16f9ff202cbda7580d0328b98/aiohttp-3.14.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43", size = 1674012, upload-time = "2026-07-23T01:52:16.486Z" }, + { url = "https://files.pythonhosted.org/packages/38/65/672df92e3172cd876aacfa97a952ac560877eb169384b2991ac5b273de4c/aiohttp-3.14.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9", size = 1767015, upload-time = "2026-07-23T01:52:18.28Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c5/228dec7bfec1c373cc2217cdeb47d6456dcd7a13a4c55144930a75ae3851/aiohttp-3.14.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8", size = 1858700, upload-time = "2026-07-23T01:52:20.08Z" }, + { url = "https://files.pythonhosted.org/packages/bd/ff/cb36724e8c8d17f90ada567a9ff3efe1d6e9b549fba697a242aece180f21/aiohttp-3.14.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479", size = 1714075, upload-time = "2026-07-23T01:52:22.071Z" }, + { url = "https://files.pythonhosted.org/packages/9f/3a/296a4135c6366376263aeef54b15caca1f07676c2ae0c525d7832f2f808a/aiohttp-3.14.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b", size = 1588234, upload-time = "2026-07-23T01:52:23.757Z" }, + { url = "https://files.pythonhosted.org/packages/7d/81/9d5d853ef892dc066d1eb6db0e87a47348b920c1c879aa554612fdbd9d79/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d", size = 1677300, upload-time = "2026-07-23T01:52:25.861Z" }, + { url = "https://files.pythonhosted.org/packages/68/96/021d386ae32d9b26d4b88df2e794546232ff56bb6be952bf6be227c0bbc7/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d", size = 1691501, upload-time = "2026-07-23T01:52:28Z" }, + { url = "https://files.pythonhosted.org/packages/29/9f/af66adce26a14af135c003cbd0f44ccaa68cebd30ff8ac99ca47fb4958f7/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2", size = 1735113, upload-time = "2026-07-23T01:52:29.995Z" }, + { url = "https://files.pythonhosted.org/packages/2f/90/28c390d4c9851effe52ac25b5a2e1d92246acd00728b4fc7975dafb67484/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48", size = 1577486, upload-time = "2026-07-23T01:52:31.937Z" }, + { url = "https://files.pythonhosted.org/packages/db/c2/00e23a1bf2abb70dd353f6987db7e7f2491d0261f7363997738c71c98f95/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f", size = 1751353, upload-time = "2026-07-23T01:52:33.688Z" }, + { url = "https://files.pythonhosted.org/packages/6e/7d/d51a706a8cbfa57f0611127daf61ab3ae02ab8420b0407412079227d1c65/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32", size = 1698681, upload-time = "2026-07-23T01:52:38.167Z" }, + { url = "https://files.pythonhosted.org/packages/ec/b0/90bd5cd9fdd9787cb4211d284d1fb8401339a933cb0227a15b71e789232f/aiohttp-3.14.3-cp310-cp310-win32.whl", hash = "sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e", size = 456733, upload-time = "2026-07-23T01:52:41.823Z" }, + { url = "https://files.pythonhosted.org/packages/d8/15/fe5b8f6a71ae112bc677163d0b0701bda5dc15005249582258ede0eb88c7/aiohttp-3.14.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c", size = 480460, upload-time = "2026-07-23T01:52:43.905Z" }, + { url = "https://files.pythonhosted.org/packages/54/00/45e98b6645cd7f00a4b78b749ebd309094b0eaeb2d2e96157eadbc0d0050/aiohttp-3.14.3-cp310-cp310-win_arm64.whl", hash = "sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb", size = 453479, upload-time = "2026-07-23T01:52:46.075Z" }, + { url = "https://files.pythonhosted.org/packages/f8/5c/b3e4ff8ad43a8afef9602c5e90285936da1beaea8b029016b793891f03c3/aiohttp-3.14.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3", size = 764250, upload-time = "2026-07-23T01:52:48.525Z" }, + { url = "https://files.pythonhosted.org/packages/0e/da/f1b384465e51449d844056b75070461da03a9a23e6c1747003695bf4172a/aiohttp-3.14.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a", size = 516281, upload-time = "2026-07-23T01:52:51.047Z" }, + { url = "https://files.pythonhosted.org/packages/b9/3f/01264f820ee2e3712a827892b1cd6ff80f3300c1fcbffbb45714a915d47a/aiohttp-3.14.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8", size = 514742, upload-time = "2026-07-23T01:52:53.779Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8d/a71c6f2db52ac1ed142b133f7feddaa6b70539c3f4de24d7e226c95b794c/aiohttp-3.14.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239", size = 1780613, upload-time = "2026-07-23T01:52:56.948Z" }, + { url = "https://files.pythonhosted.org/packages/a5/11/3dd9b3fb3a170f6ec9011b5291d876a6fab4086714c9e158600edf01b4fd/aiohttp-3.14.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f", size = 1737688, upload-time = "2026-07-23T01:52:59.294Z" }, + { url = "https://files.pythonhosted.org/packages/6d/3e/834c26918be7d88068822b40e0db30fca50b5f4fe79104aa16a93f1d74e6/aiohttp-3.14.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06", size = 1845742, upload-time = "2026-07-23T01:53:01.641Z" }, + { url = "https://files.pythonhosted.org/packages/cc/c9/49ab8572df7d66bc13d11e31f781292badb04180dd87ba98733066c6aed7/aiohttp-3.14.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929", size = 1928412, upload-time = "2026-07-23T01:53:04.018Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b9/2b8f0c0ce09c87a1daf80fd483431b56b1435d3f62789bc86f572e1245de/aiohttp-3.14.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db", size = 1786220, upload-time = "2026-07-23T01:53:06.481Z" }, + { url = "https://files.pythonhosted.org/packages/85/00/9c45f81de11710460edfa1dc81317b6e882703b160926c879a9d20da9fcc/aiohttp-3.14.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce", size = 1637231, upload-time = "2026-07-23T01:53:10.258Z" }, + { url = "https://files.pythonhosted.org/packages/19/ce/967d628e910756f3539c6107cb7844a1b69440dcb3029a5ee7871b09ab63/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c", size = 1753161, upload-time = "2026-07-23T01:53:13.817Z" }, + { url = "https://files.pythonhosted.org/packages/11/b2/0c3d4114f0aee4f580f5b3b4eb71b24d7a23b834ea506a4dfebe76513f35/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15", size = 1756356, upload-time = "2026-07-23T01:53:16.211Z" }, + { url = "https://files.pythonhosted.org/packages/63/5d/99e7d91c82f1399d1ae2a854e080bd1493fbc31e5e959dbc4ec33dac3bec/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c", size = 1819846, upload-time = "2026-07-23T01:53:18.289Z" }, + { url = "https://files.pythonhosted.org/packages/ad/05/d5e1cb6480eeffd3f901d40a2c5e2d1e7effdc797837da3b490272699f13/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae", size = 1628531, upload-time = "2026-07-23T01:53:23.86Z" }, + { url = "https://files.pythonhosted.org/packages/c9/90/b934682bcaefae18a9e04f3dff5b68522ba810906358ae5029b68110ea3b/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910", size = 1832712, upload-time = "2026-07-23T01:53:27.551Z" }, + { url = "https://files.pythonhosted.org/packages/21/df/6061679faaf81fac746e7307c7adb71e858071a5d34c27583afefc64f543/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7", size = 1775014, upload-time = "2026-07-23T01:53:30.223Z" }, + { url = "https://files.pythonhosted.org/packages/8a/1d/f854878bbc69b88faefe924b619a34a6f59ec05fd387c77690667eaa75eb/aiohttp-3.14.3-cp311-cp311-win32.whl", hash = "sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa", size = 456006, upload-time = "2026-07-23T01:53:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/73/0c/2af9d1674baccd1dbd47282a93d660a22e57ef6167c856deb24b4214fbab/aiohttp-3.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d", size = 481069, upload-time = "2026-07-23T01:53:39.673Z" }, + { url = "https://files.pythonhosted.org/packages/8e/76/88401ff3fc95e85c5fc38d588f36f55e61ecb64343b2bc8d69326f453cc0/aiohttp-3.14.3-cp311-cp311-win_arm64.whl", hash = "sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39", size = 453021, upload-time = "2026-07-23T01:53:43.749Z" }, + { url = "https://files.pythonhosted.org/packages/18/d4/eb96299230e20acf2efae207cb8d69051f1f68e357e5ea5e479bf6fb097a/aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5", size = 754690, upload-time = "2026-07-23T01:53:47.332Z" }, + { url = "https://files.pythonhosted.org/packages/88/11/e7a70a209eb9a067c0d3212b518a0134e3484f5178c7533878b6b514d469/aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228", size = 509484, upload-time = "2026-07-23T01:53:51.159Z" }, + { url = "https://files.pythonhosted.org/packages/30/07/4bbc222cc8dbe31d4c3e8a5baad2286e4d42026ac0c570027b89afce6344/aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee", size = 511949, upload-time = "2026-07-23T01:53:55.083Z" }, + { url = "https://files.pythonhosted.org/packages/54/b9/42e74c46b7b7c794b995bbc1f573fb48950c38b19d8600c62a6804ee2d67/aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a", size = 1765282, upload-time = "2026-07-23T01:53:59.662Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ed/62bc4d74363ad346d518e0720363a949f63e2e23439a79eb5813d4d29bb3/aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b", size = 1741511, upload-time = "2026-07-23T01:54:04.063Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9f/181e8a8bc79e47d13c7fc4540bd7a3b729d9505609c61f392a8dd2fbfe55/aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529", size = 1810680, upload-time = "2026-07-23T01:54:09.882Z" }, + { url = "https://files.pythonhosted.org/packages/5c/9a/dec94d6ad694552fe3424e3f1928d7a606a5d9d9433a04e7ecdd9d38ae7f/aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787", size = 1905646, upload-time = "2026-07-23T01:54:13.475Z" }, + { url = "https://files.pythonhosted.org/packages/52/b7/7cd31f29d6055bd711ae6e669367fba6f5ae9de463910a793e30556a8db7/aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42", size = 1792122, upload-time = "2026-07-23T01:54:15.752Z" }, + { url = "https://files.pythonhosted.org/packages/66/73/10b1ef93afa61f4963c746257b70ced619cf31a4798671de5fdb2608501d/aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b", size = 1591127, upload-time = "2026-07-23T01:54:19.489Z" }, + { url = "https://files.pythonhosted.org/packages/49/ed/3b203fa6de1b338c14acdc06bf6ca9b043b7944f005966958c2ced932cde/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043", size = 1725210, upload-time = "2026-07-23T01:54:24.129Z" }, + { url = "https://files.pythonhosted.org/packages/28/b7/1c2aab8c706436dcc28598452488ac9cd7c409da815237c28c27d58993e6/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427", size = 1764848, upload-time = "2026-07-23T01:54:27.973Z" }, + { url = "https://files.pythonhosted.org/packages/54/50/94c28f08b131c4bf10984ea2c7a536c9920608bb2d6e7f95642c30cc87b7/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d", size = 1777102, upload-time = "2026-07-23T01:54:31.775Z" }, + { url = "https://files.pythonhosted.org/packages/13/d4/e7d09ba7d345fb2d74440fd2fa033c5e079fac05552927705986f41a364f/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0", size = 1580205, upload-time = "2026-07-23T01:54:34.518Z" }, + { url = "https://files.pythonhosted.org/packages/a3/84/072a91d68e1e1eb587985b54baab94221277f877e8ef274fc213a0ceae28/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d", size = 1797219, upload-time = "2026-07-23T01:54:36.995Z" }, + { url = "https://files.pythonhosted.org/packages/e0/eb/aad34e897e668424d6e995da5dff8a4a09af93363d3392488772957a63aa/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19", size = 1768629, upload-time = "2026-07-23T01:54:40.103Z" }, + { url = "https://files.pythonhosted.org/packages/b6/2b/6bb88ddba0fecd9122aa3ebcad25996cf6c083a4a7040dbb3a4f97972af6/aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559", size = 451481, upload-time = "2026-07-23T01:54:42.547Z" }, + { url = "https://files.pythonhosted.org/packages/76/9b/f2f8f108da17ecef2cc3efc424e8b7ad3782b1a8360f7b8eae8ced84f6ea/aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a", size = 476845, upload-time = "2026-07-23T01:54:44.853Z" }, + { url = "https://files.pythonhosted.org/packages/3e/44/28dac80a8941b604f4da10ce21097614ca1bf905ce93dca28d8d7de9c1e7/aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c", size = 448050, upload-time = "2026-07-23T01:54:47.087Z" }, + { url = "https://files.pythonhosted.org/packages/57/be/5afd201cc0ab139029aadb75392efe85a293403d9dd3a3226161c21ce00c/aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86", size = 506269, upload-time = "2026-07-23T01:54:49.075Z" }, + { url = "https://files.pythonhosted.org/packages/22/09/dec8189d62b45ade009f6792a2264b942a90cb88aeaf181239933cd72c3c/aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627", size = 515166, upload-time = "2026-07-23T01:54:51.894Z" }, + { url = "https://files.pythonhosted.org/packages/28/24/2854869d29ed8a8b19d74f9ec6629515f7e04d02dd329d9d179201e58e47/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82", size = 486263, upload-time = "2026-07-23T01:54:54.223Z" }, + { url = "https://files.pythonhosted.org/packages/d4/dd/57187c8be2a35aea65eaee3bd2c3dcbbcf0204f5106c89637e3610380cd1/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c", size = 492299, upload-time = "2026-07-23T01:54:56.236Z" }, + { url = "https://files.pythonhosted.org/packages/b9/11/06ae6ed8f0d414edf4068861e233d8fe23ee699bfd4b3ceb8663db948a62/aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f", size = 502235, upload-time = "2026-07-23T01:54:58.377Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a3/559639c34a345d2cf7c52dff6838119f2eaf29eb508227b5b83f573af813/aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80", size = 750883, upload-time = "2026-07-23T01:55:00.65Z" }, + { url = "https://files.pythonhosted.org/packages/91/cd/41e131f13afd1e7b0172a9d9eda085ef90eb8439f41f0d279db81ed3ae60/aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0", size = 508473, upload-time = "2026-07-23T01:55:02.945Z" }, + { url = "https://files.pythonhosted.org/packages/bc/6b/e7f13410d391c6e55b4c007a8de024355389d7d459e3d64c42b2d33617e5/aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf", size = 509190, upload-time = "2026-07-23T01:55:05.173Z" }, + { url = "https://files.pythonhosted.org/packages/97/21/6464573e53d69672cc1eada3e5c5cb2d2efa82701e8305a0f2047a576967/aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd", size = 1761478, upload-time = "2026-07-23T01:55:07.383Z" }, + { url = "https://files.pythonhosted.org/packages/1a/81/d217043a4c17fbce360905e3b2bdd20139ebc9a2de836d035d179c4da006/aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807", size = 1735092, upload-time = "2026-07-23T01:55:09.803Z" }, + { url = "https://files.pythonhosted.org/packages/a1/66/e13a02d0eeb1a9a502402a977abb4e4abff9fe4051c26f80558c57a7c975/aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8", size = 1800546, upload-time = "2026-07-23T01:55:12.012Z" }, + { url = "https://files.pythonhosted.org/packages/26/5e/57d42fca1d18cb5acc1cad945d017fabc5d6ae71d8a08ad66be8dc3ee544/aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24", size = 1895250, upload-time = "2026-07-23T01:55:14.357Z" }, + { url = "https://files.pythonhosted.org/packages/ca/1c/7da8d08e74d56f00070822f9638ff3f1c563f8ad87d1efa996c87bfc8644/aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5", size = 1789289, upload-time = "2026-07-23T01:55:16.668Z" }, + { url = "https://files.pythonhosted.org/packages/cd/0f/cf16bcf56896981c1a0319f5d5db9337994b5165730c48a8fa07e9b34be6/aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4", size = 1586706, upload-time = "2026-07-23T01:55:18.913Z" }, + { url = "https://files.pythonhosted.org/packages/fe/6f/76eac12a7f2480e1e304f842efdb07db33256b0d9165b866b6ef0806c202/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9", size = 1724652, upload-time = "2026-07-23T01:55:21.296Z" }, + { url = "https://files.pythonhosted.org/packages/39/b6/19c8c592baeeb94b75f966547d40c02ac7590902306ec5863d5c027cf506/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1", size = 1756239, upload-time = "2026-07-23T01:55:23.705Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c9/4e9383150296f97f873b680c4de8fb2cd88608fb9f48c79edcb111611abc/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371", size = 1769161, upload-time = "2026-07-23T01:55:26.082Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1e/147bdc6cc5de5f3ab011be8bf5d6e786633249f22c20bae06f85e45f5387/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde", size = 1578759, upload-time = "2026-07-23T01:55:28.846Z" }, + { url = "https://files.pythonhosted.org/packages/fd/31/78388a9d6040ece2e11df62ea229a822cf5e52d238374b220ae9975b2623/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e", size = 1792025, upload-time = "2026-07-23T01:55:31.457Z" }, + { url = "https://files.pythonhosted.org/packages/03/51/a3d29fdf2c25d796746af8ad6fe56a45d6256c38b0a8a2ed752e1160b3a2/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71", size = 1768477, upload-time = "2026-07-23T01:55:33.87Z" }, + { url = "https://files.pythonhosted.org/packages/29/a6/442e18b5afeade534d877a2dc3c3e392aff8d49787890b0cf84790410267/aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0", size = 451069, upload-time = "2026-07-23T01:55:36.121Z" }, + { url = "https://files.pythonhosted.org/packages/9d/69/3d876ac02659f271cf7f6769f14a8e3de5b6e888ed8b5a7e998086a4cec8/aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883", size = 476518, upload-time = "2026-07-23T01:55:38.303Z" }, + { url = "https://files.pythonhosted.org/packages/b2/0e/50d6e6471cd31edce8b282bdec59375a3a69124d8a989a0b1313355cae52/aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2", size = 447676, upload-time = "2026-07-23T01:55:40.451Z" }, + { url = "https://files.pythonhosted.org/packages/c8/20/887fdcf832326571b370ffc347b3e70abe101096f3720126aac161b1d872/aiohttp-3.14.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062", size = 509067, upload-time = "2026-07-23T01:55:42.618Z" }, + { url = "https://files.pythonhosted.org/packages/ad/a3/92cec936f78cc4bf0fa5554ebe593b73459d94e3c62303e1902a4cccb6f7/aiohttp-3.14.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6", size = 514774, upload-time = "2026-07-23T01:55:44.937Z" }, + { url = "https://files.pythonhosted.org/packages/29/ba/2a0c38df3fc557620b6a5acd98364af050053b6285b4dc7ee74100c63c18/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919", size = 488134, upload-time = "2026-07-23T01:55:47.135Z" }, + { url = "https://files.pythonhosted.org/packages/48/d6/d51b7d4bf309af3693940d8ffd2b9ed0b682434ef85959b7c9c137f60cf8/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7", size = 494201, upload-time = "2026-07-23T01:55:49.451Z" }, + { url = "https://files.pythonhosted.org/packages/3f/5a/8f624384e5f1efabb5229b94157eb966b021e97bdb188c62860c2ae243c2/aiohttp-3.14.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0", size = 502766, upload-time = "2026-07-23T01:55:51.656Z" }, + { url = "https://files.pythonhosted.org/packages/a6/26/4ff0164370deec18fb19254ee4ab10b7a73304ac0c860b13f5f84663759b/aiohttp-3.14.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924", size = 756557, upload-time = "2026-07-23T01:55:53.964Z" }, + { url = "https://files.pythonhosted.org/packages/97/a3/7056b86dc0d9ec709ea9777eae3b0161428f943372f8b98c01c11593b682/aiohttp-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646", size = 510168, upload-time = "2026-07-23T01:55:56.22Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/0357a015892fd68058bf2d39d3fd1958e459b997a7db30aaa6aaa434ae96/aiohttp-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b", size = 512957, upload-time = "2026-07-23T01:55:58.437Z" }, + { url = "https://files.pythonhosted.org/packages/47/d1/8aba53f15ccb2238405f5e9d30e2a8ca44f93878c26e7165ade00d374b1c/aiohttp-3.14.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30", size = 1750149, upload-time = "2026-07-23T01:56:00.856Z" }, + { url = "https://files.pythonhosted.org/packages/49/bd/40c3fee327529284375c6701cbb0fa4600cc2e8432af1378f897e2ef7d3a/aiohttp-3.14.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9", size = 1707685, upload-time = "2026-07-23T01:56:03.371Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a3/ca0cc6724cca8114b05694abd916060758c79894c3aa5b012cdadc1bc28e/aiohttp-3.14.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f", size = 1803911, upload-time = "2026-07-23T01:56:05.817Z" }, + { url = "https://files.pythonhosted.org/packages/95/b5/85b099c299c3ffd38ad9b3e43694c8a346934e4a30c88c4fd5a841234f77/aiohttp-3.14.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d", size = 1876929, upload-time = "2026-07-23T01:56:08.413Z" }, + { url = "https://files.pythonhosted.org/packages/d5/b7/1da684a04175473fa4cddbf9a2f572e79514c3fd27a74597f43057d4f3da/aiohttp-3.14.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147", size = 1761112, upload-time = "2026-07-23T01:56:10.918Z" }, + { url = "https://files.pythonhosted.org/packages/d1/16/bc4b55e3e5cb175fd69c53c90d60d2f47797cb343da5106e23863dc4dba4/aiohttp-3.14.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c", size = 1583500, upload-time = "2026-07-23T01:56:13.613Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e8/13a9d957a1ee40837f46aa30f0f4c657e673ad86a2e6362a9f9be20d26d9/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a", size = 1713940, upload-time = "2026-07-23T01:56:15.969Z" }, + { url = "https://files.pythonhosted.org/packages/38/05/d33c680c1bcf1c7e130f9cbfc1fc02fe8bb0c4af2a94a53dd5fb56131e5c/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0", size = 1724413, upload-time = "2026-07-23T01:56:18.591Z" }, + { url = "https://files.pythonhosted.org/packages/85/1d/af798d306f7a74b6a632dbcabcf62a4c91391b7582d2a8c6d7712e2cc54e/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661", size = 1770748, upload-time = "2026-07-23T01:56:21.074Z" }, + { url = "https://files.pythonhosted.org/packages/a8/92/ad720d472556a995049206867765e9410969684f86ee09423ff9969044c1/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22", size = 1577564, upload-time = "2026-07-23T01:56:23.475Z" }, + { url = "https://files.pythonhosted.org/packages/60/ad/0ed7586cbef7a884e23a752fa2bb987a122e6a5dd50dab109258d0a95193/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41", size = 1782080, upload-time = "2026-07-23T01:56:25.994Z" }, + { url = "https://files.pythonhosted.org/packages/97/ea/dbaed0d73e8a69aad653b045dab451c67c2454bb731a37b45a86593e9422/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf", size = 1745813, upload-time = "2026-07-23T01:56:28.604Z" }, + { url = "https://files.pythonhosted.org/packages/81/1b/6893d4bc57e434fc93a6c9217c637d967a0b651d989f6e3265179375754a/aiohttp-3.14.3-cp314-cp314-win32.whl", hash = "sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da", size = 455872, upload-time = "2026-07-23T01:56:31.031Z" }, + { url = "https://files.pythonhosted.org/packages/f5/8b/c7baa1ba1eda4db6989baefe5de6d99834921b84ebd7918624febcb9f290/aiohttp-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100", size = 481030, upload-time = "2026-07-23T01:56:33.365Z" }, + { url = "https://files.pythonhosted.org/packages/22/8c/c29d067df825a2df88ca432db848aa2fe8199598359cc06c12b09320cac9/aiohttp-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc", size = 453669, upload-time = "2026-07-23T01:56:35.731Z" }, + { url = "https://files.pythonhosted.org/packages/6a/a4/9c033beb355d39b6147980597ec9645e4729243f686ee4dc73945de72030/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b", size = 791403, upload-time = "2026-07-23T01:56:37.972Z" }, + { url = "https://files.pythonhosted.org/packages/80/ca/87c32a0a7704583cfc49660bd817889bae5b830bf53b5dcb4e92145ac2da/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0", size = 526413, upload-time = "2026-07-23T01:56:40.523Z" }, + { url = "https://files.pythonhosted.org/packages/9e/d8/8ec0e471248c500acdce2be3f46db8fb62b5eb60efef072529cc85ee1d26/aiohttp-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e", size = 532135, upload-time = "2026-07-23T01:56:42.876Z" }, + { url = "https://files.pythonhosted.org/packages/fe/45/f8919fd936e8b79fcd9bda7b6d8e62613462a713f4f17987fd7c34399142/aiohttp-3.14.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716", size = 1922742, upload-time = "2026-07-23T01:56:45.528Z" }, + { url = "https://files.pythonhosted.org/packages/f6/ec/9ca76b28a27525b0cc53e20842e0228b022f301ce1f436b7d814b4aaf2df/aiohttp-3.14.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f", size = 1787371, upload-time = "2026-07-23T01:56:48.045Z" }, + { url = "https://files.pythonhosted.org/packages/b1/04/6acdbf17315f7b55f1937e3387acb89a3cddeb4995689553d064af8e92ab/aiohttp-3.14.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553", size = 1912623, upload-time = "2026-07-23T01:56:50.605Z" }, + { url = "https://files.pythonhosted.org/packages/86/e6/438b0c79ca6f45eb9fd9817dd4c01a91919a38c0de5ee9e05e2b4dc0ece7/aiohttp-3.14.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100", size = 2005515, upload-time = "2026-07-23T01:56:53.153Z" }, + { url = "https://files.pythonhosted.org/packages/bb/6b/62cbd6577758699525f5c712d1ddef57d9875fbab0ae8d5f5a202fd598f8/aiohttp-3.14.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85", size = 1879906, upload-time = "2026-07-23T01:56:55.818Z" }, + { url = "https://files.pythonhosted.org/packages/00/95/18bcbf830a21dc3aae24d8f6b6feaf3db1d2090242d00a7868db2ffb0b67/aiohttp-3.14.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33", size = 1675849, upload-time = "2026-07-23T01:56:58.861Z" }, + { url = "https://files.pythonhosted.org/packages/a9/19/47f4968659c5e23606c3790c80fc624e691c153d036148449ee84d31b287/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f", size = 1843496, upload-time = "2026-07-23T01:57:01.591Z" }, + { url = "https://files.pythonhosted.org/packages/64/af/38c33c4dd82fddcb4e56c4653b6f1072a8edbc6b7fa15809f14932c41e2d/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0", size = 1827746, upload-time = "2026-07-23T01:57:05.131Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9d/0537cda4885ac8f5b7053d164dd06312f4c483a4edcb8ee5b8aaf2a989bf/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098", size = 1853810, upload-time = "2026-07-23T01:57:08.043Z" }, + { url = "https://files.pythonhosted.org/packages/19/fe/26f9c5e6458385aa86497836b0dea6fb2f027827d63f37c7856cce9286ee/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25", size = 1668895, upload-time = "2026-07-23T01:57:10.837Z" }, + { url = "https://files.pythonhosted.org/packages/ec/4c/618b1db9b9ba079b8875d2cdf78e7c4a3bf72903bd5850fee7dd9544600a/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9", size = 1883833, upload-time = "2026-07-23T01:57:13.672Z" }, + { url = "https://files.pythonhosted.org/packages/94/c6/bd959bd1e4771f9fd944e9e436224c48c77b018b73b519b5aad346335bcc/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb", size = 1844251, upload-time = "2026-07-23T01:57:16.593Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/08d41839658bdd44a0ed2480f3891705ecb487ce28c0dde62c9040c997e0/aiohttp-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963", size = 474180, upload-time = "2026-07-23T01:57:19.306Z" }, + { url = "https://files.pythonhosted.org/packages/99/5d/3cd6ef0a2b2851f7ab913b5b079334781bd50ff56a323e4454063377a080/aiohttp-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b", size = 500528, upload-time = "2026-07-23T01:57:21.762Z" }, + { url = "https://files.pythonhosted.org/packages/a4/37/cfd1ed540a4d318da025590d96b728e63713c09e9377950fc655dadeb856/aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7", size = 469280, upload-time = "2026-07-23T01:57:24.241Z" }, ] [[package]] @@ -4327,7 +4327,7 @@ proxy-dev = [ [package.metadata] requires-dist = [ { name = "a2a-sdk", marker = "extra == 'extra-proxy'", specifier = ">=1.1.0,<2.0" }, - { name = "aiohttp", specifier = ">=3.10,<4.0" }, + { name = "aiohttp", specifier = ">=3.14.2,<4.0" }, { name = "anthropic", extras = ["vertex"], marker = "extra == 'proxy-runtime'", specifier = ">=0.84.0,<1.0" }, { name = "apscheduler", marker = "extra == 'proxy'", specifier = ">=3.11.2,<4.0" }, { name = "audioread", marker = "extra == 'stt-nvidia-riva'", specifier = ">=3.0.1" }, From d9f53258e9f3a3281706b4692a42c660f01e4e21 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 09:52:46 -0700 Subject: [PATCH 08/50] refactor(test): tighten typing on the tag list verification token double Replaces the double's Any annotations and List/Dict aliases with concrete types, matching the equivalent double in the tool policy tests: kwargs are object, records are Sequence[Mock] held as a tuple, and the call log is list[dict[str, object]]. Behaviour is unchanged; the double still binds every call against the real generated prisma action signature, verified by reintroducing the select kwarg and watching the regression tests fail --- .../test_tag_management_endpoints.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py index 9927c56b847..4fe1b54694f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py @@ -2,7 +2,8 @@ import inspect import json import os import sys -from typing import Any, Dict, List, Optional +from collections.abc import Sequence +from typing import Optional import pytest from fastapi import HTTPException @@ -13,7 +14,7 @@ sys.path.insert( 0, os.path.abspath("../../../..") ) # Adds the parent directory to the system path -from unittest.mock import patch +from unittest.mock import Mock, patch import litellm from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth @@ -33,11 +34,11 @@ class FakeVerificationTokenTable: surfaces as an HTTP 500. """ - def __init__(self, records: List[Any]): - self._records = records - self.calls: List[Dict[str, Any]] = [] + def __init__(self, records: Sequence[Mock]): + self._records = tuple(records) + self.calls: list[dict[str, object]] = [] - async def find_many(self, **kwargs: Any) -> List[Any]: + async def find_many(self, **kwargs: object) -> tuple[Mock, ...]: inspect.signature(LiteLLM_VerificationTokenActions.find_many).bind( self, **kwargs ) From 78c756dff94554435b2912cd416022ebb9c10291 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 10:19:24 -0700 Subject: [PATCH 09/50] fix(proxy): rework the budgets list onto the merged list contract PR #35308 landed a different shape than this branch was written against: `where` is a tuple of frozen predicates rather than a Prisma-shaped mapping, `ListSpec` carries both the row and the wire type, and `where_sql` / `order_by_sql` render for a raw-SQL executor. The budgets executor now queries through `query_raw` the way the spend logs facet does, selecting only the columns it serves. Also casts datetime binds in `where_sql`. They cross into the query engine as JSON, so an uncast placeholder arrives as text and Postgres refuses `timestamp >= text` outright; every `filter[created_at][gte|lte]` was answering 500. The cast reads the bind as an instant and drops it to naive UTC to match Prisma's TIMESTAMP(3) column, the same one /spend/logs/ui applies. --- .../management_v1/budgets.py | 25 +++++--- .../management_v1/list_framework.py | 15 ++++- .../management_v1/test_budgets.py | 15 ++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 61 ++++++++++++++----- 4 files changed, 87 insertions(+), 29 deletions(-) diff --git a/litellm/proxy/management_endpoints/management_v1/budgets.py b/litellm/proxy/management_endpoints/management_v1/budgets.py index d9104eb5da2..bc1521caf0b 100644 --- a/litellm/proxy/management_endpoints/management_v1/budgets.py +++ b/litellm/proxy/management_endpoints/management_v1/budgets.py @@ -1,8 +1,9 @@ """`GET /management/v1/budgets`.""" -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import datetime +from types import MappingProxyType from typing import Annotated from fastapi import APIRouter, Depends, Request @@ -112,15 +113,19 @@ def _scope(caller: UserAPIKeyAuth) -> Scope: # budget_duration is deliberately absent from `sortable`: the column holds strings # like "7d" and "30d", so a lexicographic ORDER BY puts "30d" ahead of "7d". +BUDGET_FILTERS: Mapping[str, FilterSpec] = MappingProxyType( + { # mutable-ok: an immutable mapping has no literal form; MappingProxyType freezes this one and it never escapes + "budget_duration": FilterSpec(type=str, ops=frozenset(("in", "is_null"))), + "max_budget": FilterSpec(type=float, ops=frozenset(("gte", "lte", "is_null"))), + "created_at": FilterSpec(type=datetime, ops=frozenset(("gte", "lte"))), + } +) + BUDGETS_LIST_SPEC: ListSpec[BudgetListItem, BudgetListItem] = ListSpec( resource="budgets", - sortable=frozenset({"budget_id", "max_budget", "tpm_limit", "rpm_limit", "created_at"}), - searchable=frozenset({"budget_id"}), - filters={ - "budget_duration": FilterSpec(type=str, ops=frozenset({"in", "is_null"})), - "max_budget": FilterSpec(type=float, ops=frozenset({"gte", "lte", "is_null"})), - "created_at": FilterSpec(type=datetime, ops=frozenset({"gte", "lte"})), - }, + sortable=frozenset(("budget_id", "max_budget", "tpm_limit", "rpm_limit", "created_at")), + searchable=frozenset(("budget_id",)), + filters=BUDGET_FILTERS, default_sort=(SortKey(field="created_at", descending=True),), default_page_size=50, max_page_size=100, @@ -132,8 +137,8 @@ BUDGETS_LIST_SPEC: ListSpec[BudgetListItem, BudgetListItem] = ListSpec( @router.get( "/budgets", - tags=["budget management"], - dependencies=[Depends(user_api_key_auth)], + tags=("budget management",), + dependencies=(Depends(user_api_key_auth),), response_model=ListResponse[BudgetListItem], ) async def list_budgets( diff --git a/litellm/proxy/management_endpoints/management_v1/list_framework.py b/litellm/proxy/management_endpoints/management_v1/list_framework.py index 3e4b9131d1e..e2bddefab83 100644 --- a/litellm/proxy/management_endpoints/management_v1/list_framework.py +++ b/litellm/proxy/management_endpoints/management_v1/list_framework.py @@ -213,12 +213,23 @@ def _sql_operator(op: ComparisonOp) -> str: assert_never(op) +def _placeholder(index: int, value: FilterValue) -> str: + """`$n`, cast when the bind is a datetime. + + Binds cross into the query engine as JSON, so a datetime arrives as text and + Postgres refuses `timestamp >= text` outright. Prisma stores DateTime as a naive + `TIMESTAMP(3)` holding UTC, so the bind is read as an instant and then dropped to + naive UTC to match the column, the same cast `/spend/logs/ui` applies. + """ + return f"${index}::timestamptz AT TIME ZONE 'UTC'" if isinstance(value, datetime) else f"${index}" + + def _render(predicate: Predicate, index: int) -> tuple[str, tuple[object, ...]]: match predicate: case IsNull(field=field, negated=negated): return f'"{field}" IS {"NOT NULL" if negated else "NULL"}', () case Within(field=field, values=values): - placeholders = ", ".join(f"${index + offset}" for offset in range(len(values))) + placeholders = ", ".join(_placeholder(index + offset, value) for offset, value in enumerate(values)) return f'"{field}" IN ({placeholders})', values case AnyOf(clauses=clauses): rendered, params = _render_all(clauses, index) @@ -226,7 +237,7 @@ def _render(predicate: Predicate, index: int) -> tuple[str, tuple[object, ...]]: case Compare(field=field, op="contains", value=value): return f"\"{field}\" ILIKE ${index} ESCAPE '\\'", (f"%{escape_like(str(value))}%",) case Compare(field=field, op=op, value=value): - return f'"{field}" {_sql_operator(op)} ${index}', (value,) + return f'"{field}" {_sql_operator(op)} {_placeholder(index, value)}', (value,) case _: assert_never(predicate) diff --git a/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py b/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py index fe6d0289e53..f98286985b7 100644 --- a/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py +++ b/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py @@ -371,15 +371,28 @@ def test_in_filter_binds_each_requested_duration(query_raw, as_proxy_admin): def test_created_at_range_is_bound_as_a_timestamp(query_raw, as_proxy_admin): + """The bind crosses into the query engine as JSON, so an uncast placeholder reaches + Postgres as text and `timestamp >= text` is a hard error, not a wrong answer.""" _serve(query_raw, []) _get("filter[created_at][gte]=2026-07-01T00:00:00Z") sql, *params = _select_call(query_raw) - assert '"created_at" >= $1' in sql + assert "\"created_at\" >= $1::timestamptz AT TIME ZONE 'UTC'" in sql assert params[0] == datetime(2026, 7, 1, tzinfo=timezone.utc) +def test_a_non_datetime_bind_is_not_cast(query_raw, as_proxy_admin): + """Guards the cast above from being applied to every placeholder.""" + _serve(query_raw, []) + + _get("filter[max_budget][gte]=5") + + sql = _select_call(query_raw)[0] + assert '"max_budget" >= $1' in sql + assert "timestamptz" not in sql + + def test_an_offsetless_created_at_bound_is_read_as_utc(query_raw, as_proxy_admin): """The dashboard sends 'YYYY-MM-DDTHH:MM:SS' with no offset. Left naive, Postgres would compare it in the session timezone and shift the window off the rows shown.""" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 46a158bbac7..9a301f1c474 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -7187,10 +7187,11 @@ export interface paths { * * `sort` takes a comma-separated list of `budget_id`, `max_budget`, `tpm_limit`, * `rpm_limit` or `created_at`, each optionally prefixed with `-` for descending, - * and defaults to `-created_at,budget_id`. `q` is a case-insensitive substring - * match on `budget_id`. `page_size` defaults to 50 and is capped at 100. - * Filters are `filter[budget_duration][in|is_null]`, - * `filter[max_budget][gte|lte|is_null]` and `filter[created_at][gte|lte]`. + * and defaults to `-created_at`. `budget_id` is appended to every sort as the + * tiebreaker. `q` is a case-insensitive substring match on `budget_id`. + * `page_size` defaults to 50 and is capped at 100. Filters are + * `filter[budget_duration][in|is_null]`, `filter[max_budget][gte|lte|is_null]` + * and `filter[created_at][gte|lte]`. * * Example curl: * ``` @@ -21570,6 +21571,40 @@ export interface components { /** Reset At */ reset_at?: string | null; }; + /** + * BudgetListItem + * @description One budget as the Budgets page reads it, and as it comes back off the table. + * + * Validating the raw row through here is what makes `tpm_limit` / `rpm_limit` + * numbers: they are `BigInt?` in the schema, which the query engine hands back as + * decimal strings, and a quoted "60000" breaks arithmetic in the dashboard. + */ + BudgetListItem: { + /** Budget Duration */ + budget_duration?: string | null; + /** Budget Id */ + budget_id: string; + /** Budget Reset At */ + budget_reset_at?: string | null; + /** + * Created At + * Format: date-time + */ + created_at: string; + /** Max Budget */ + max_budget?: number | null; + /** Rpm Limit */ + rpm_limit?: number | null; + /** Soft Budget */ + soft_budget?: number | null; + /** Tpm Limit */ + tpm_limit?: number | null; + /** + * Updated At + * Format: date-time + */ + updated_at: string; + }; /** BudgetNewRequest */ BudgetNewRequest: { /** @@ -24814,7 +24849,6 @@ export interface components { /** Updated By */ updated_by?: string | null; }; - JsonValue: unknown; /** KeyHealthResponse */ KeyHealthResponse: { /** @@ -24968,7 +25002,7 @@ export interface components { }; /** * ListLinks - * @description Hypermedia for an entity list. `first`/`last` exist here because `total_pages` is known. + * @description Page-mode counterpart to `PageLinks`. `first`/`last` are knowable here because the total count is. */ ListLinks: { /** First */ @@ -24984,7 +25018,7 @@ export interface components { }; /** * ListMeta - * @description An entity list can afford the COUNT(*) a facet cannot, so it reports a real total. + * @description Page-mode counterpart to `PageMeta`: an entity list pays for the COUNT(*) so the table can show a page count. */ ListMeta: { /** Page */ @@ -25011,15 +25045,10 @@ export interface components { /** Prompts */ prompts: components["schemas"]["PromptSpec"][]; }; - /** - * ListResponse - * @description One page of an entity collection. Rows are flat: no `{type, id, attributes}` wrapper. - */ - ListResponse: { + /** ListResponse[BudgetListItem] */ + ListResponse_BudgetListItem_: { /** Data */ - data: { - [key: string]: components["schemas"]["JsonValue"]; - }[]; + data: components["schemas"]["BudgetListItem"][]; links: components["schemas"]["ListLinks"]; meta: components["schemas"]["ListMeta"]; }; @@ -43574,7 +43603,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["ListResponse"]; + "application/json": components["schemas"]["ListResponse_BudgetListItem_"]; }; }; }; From 858ba174308c0ccfd290640ea7cfb46014084e59 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 10:43:28 -0700 Subject: [PATCH 10/50] refactor(proxy): fold the predicate renderer instead of recursing recursive_detector flags `_render_all`, and the flag is fair: it recursed once per predicate, so the stack grew with the number of filters on the request for no reason. Walking a predicate list is a running bind index, which is a fold. `_render` still re-enters for `AnyOf`, but its clauses are plain comparisons built by `?q=`, so that nesting is one level deep and no caller can drive it deeper. --- .../management_v1/list_framework.py | 25 +++++++++++++++---- .../management_v1/test_budgets.py | 14 +++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/management_endpoints/management_v1/list_framework.py b/litellm/proxy/management_endpoints/management_v1/list_framework.py index e2bddefab83..8f25c45016b 100644 --- a/litellm/proxy/management_endpoints/management_v1/list_framework.py +++ b/litellm/proxy/management_endpoints/management_v1/list_framework.py @@ -15,6 +15,7 @@ raw-SQL executor with every caller-supplied value bound to a placeholder. from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone +from functools import partial, reduce from math import ceil from typing import Generic, Literal, Protocol, TypeVar @@ -242,12 +243,26 @@ def _render(predicate: Predicate, index: int) -> tuple[str, tuple[object, ...]]: assert_never(predicate) +def _render_one( + rendered: tuple[tuple[str, ...], tuple[object, ...]], + predicate: Predicate, + first_index: int, +) -> tuple[tuple[str, ...], tuple[object, ...]]: + """Append one predicate, numbering it after the binds already consumed.""" + clauses, params = rendered + clause, clause_params = _render(predicate, first_index + len(params)) + return (*clauses, clause), (*params, *clause_params) + + def _render_all(predicates: tuple[Predicate, ...], index: int) -> tuple[tuple[str, ...], tuple[object, ...]]: - if not predicates: - return (), () - head, head_params = _render(predicates[0], index) - tail, tail_params = _render_all(predicates[1:], index + len(head_params)) - return (head, *tail), head_params + tail_params + """Render every predicate, numbering placeholders continuously across them. + + Folded rather than self-recursive: walking a predicate list is a running index, and + recursing per predicate grew the stack with the filter count for nothing. `_render` + still re-enters here for `AnyOf`, whose clauses are plain `Compare`s from `?q=`, so + that nesting is one level deep and cannot be driven deeper by a caller. + """ + return reduce(partial(_render_one, first_index=index), predicates, ((), ())) def where_sql(where: tuple[Predicate, ...], first_index: int = 1) -> tuple[str, tuple[object, ...]]: diff --git a/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py b/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py index f98286985b7..40473f1a25a 100644 --- a/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py +++ b/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py @@ -382,6 +382,20 @@ def test_created_at_range_is_bound_as_a_timestamp(query_raw, as_proxy_admin): assert params[0] == datetime(2026, 7, 1, tzinfo=timezone.utc) +def test_numbers_placeholders_continuously_across_predicates(query_raw, as_proxy_admin): + """Each predicate is numbered after the binds the ones before it consumed. Restart + the count and `$1` gets read as the duration while the search string goes unbound.""" + _serve(query_raw, []) + + _get("filter[budget_duration][in]=7d,30d&filter[max_budget][gte]=5&q=prod") + + sql, *params = _select_call(query_raw) + assert '"budget_duration" IN ($1, $2)' in sql + assert '"max_budget" >= $3' in sql + assert '"budget_id" ILIKE $4' in sql + assert params[:4] == ["7d", "30d", 5.0, "%prod%"] + + def test_a_non_datetime_bind_is_not_cast(query_raw, as_proxy_admin): """Guards the cast above from being applied to every placeholder.""" _serve(query_raw, []) From 6a327aee6585760925603915ceb6e217279db9f8 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 11:42:06 -0700 Subject: [PATCH 11/50] refactor(ui): type the budgets list against the generated API schema The management list route now exists, so budgetItem, the list envelope and the response type come from schema.d.ts instead of being hand-written against the contract. The optional fields widen accordingly, so the rate limit and reset cells accept undefined alongside null. --- .../_components/BudgetTableColumns.tsx | 4 +-- .../(dashboard)/hooks/budgets/useBudgets.ts | 26 +++++-------------- .../hooks/common/useResourceList.ts | 9 +++---- 3 files changed, 12 insertions(+), 27 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx index 8cca2caf214..d1192468359 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx @@ -26,14 +26,14 @@ import { cn } from "@/lib/cva.config"; const serverFilter: FilterFn = () => true; serverFilter.autoRemove = () => false; -function RateLimitCell({ value }: { value: number | null }) { +function RateLimitCell({ value }: { value: number | null | undefined }) { if (value == null) { return n/a; } return {value}; } -function BudgetDurationCell({ value }: { value: string | null }) { +function BudgetDurationCell({ value }: { value: string | null | undefined }) { if (!value) { return Not set; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/budgets/useBudgets.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/budgets/useBudgets.ts index e5f24e5412d..50151e59d3d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/budgets/useBudgets.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/budgets/useBudgets.ts @@ -6,27 +6,15 @@ import { useCallback } from "react"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { apiClient, budgetCreateCall, budgetUpdateCall, budgetDeleteCall } from "@/components/networking"; +import type { components } from "@/lib/http/schema"; import { createQueryKeys } from "../common/queryKeysFactory"; -import { - useResourceList, - type ResourceListPage, - type ResourceListQuery, - type ResourceListResult, -} from "../common/useResourceList"; +import { useResourceList, type ResourceListQuery, type ResourceListResult } from "../common/useResourceList"; import { serializeBudgetFilters } from "./budgetFilters"; -export interface budgetItem { - budget_id: string; - max_budget: number | null; - soft_budget: number | null; - rpm_limit: number | null; - tpm_limit: number | null; - budget_duration: string | null; - budget_reset_at: string | null; - created_at: string; - updated_at: string; -} +export type budgetItem = components["schemas"]["BudgetListItem"]; + +type BudgetListResponse = components["schemas"]["ListResponse_BudgetListItem_"]; export const BUDGET_LIST_PATH = "/management/v1/budgets"; @@ -39,8 +27,8 @@ export const useBudgetList = (): ResourceListResult => { const { accessToken } = useAuthorized(); const fetchPage = useCallback( - (query: ResourceListQuery, signal: AbortSignal): Promise> => - apiClient.get>(BUDGET_LIST_PATH, { accessToken, query, signal }), + (query: ResourceListQuery, signal: AbortSignal): Promise => + apiClient.get(BUDGET_LIST_PATH, { accessToken, query, signal }), [accessToken], ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/common/useResourceList.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/common/useResourceList.ts index fb40d108234..8a6376b2248 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/common/useResourceList.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/common/useResourceList.ts @@ -5,16 +5,13 @@ import { useQuery, type UseQueryOptions } from "@tanstack/react-query"; import type { ColumnFiltersState, OnChangeFn, PaginationState, SortingState } from "@tanstack/react-table"; import { useCallback, useMemo, useState } from "react"; +import type { components } from "@/lib/http/schema"; import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants"; export type ResourceListQuery = Readonly>; -export interface ResourceListMeta { - total_count: number; - page: number; - page_size: number; - total_pages: number; -} +/** The management list envelope. The generated response models are monomorphic, so only `data` is generic here. */ +export type ResourceListMeta = components["schemas"]["ListMeta"]; export interface ResourceListPage { data: TRow[]; From c0cab45350dc2f7ace66fdc375c92cb4e672d65f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 31 Jul 2026 19:14:42 +0000 Subject: [PATCH 12/50] chore(typing): replace Any kwargs unpacking with validated model parsing Swap `Model(**payload)` for `Model.model_validate(payload)` at the seams where the payload comes back untyped, so basedpyright stops widening every target field to Any. None of the models involved override `__init__`, so validation goes through the same core validator either way. Also route UserRepository through its own typed helpers (find_many, update, find_by_id) instead of the raw Prisma table, drop the redundant `_to_model` override signature, and call generate_key_helper_fn with explicit arguments in the SSO callback rather than splatting an untyped dict. Whole-tree basedpyright: reportAny 21481 -> 20834, reportExplicitAny 7258 -> 7252, with every other rule unchanged or lower. --- basedpyright-code-budget.json | 8 +-- .../proxy/hooks/managed_files.py | 8 +-- .../litellm_core_utils/streaming_handler.py | 4 +- .../llms/azure/responses/transformation.py | 4 +- litellm/llms/custom_httpx/llm_http_handler.py | 10 ++-- .../llms/manus/responses/transformation.py | 4 +- .../hooks/user_management_event_hooks.py | 13 ++--- litellm/proxy/management_endpoints/ui_sso.py | 26 +++++---- .../openai_files_endpoints/common_utils.py | 2 +- litellm/repositories/user_repository.py | 57 ++++++++----------- litellm/router.py | 12 ++-- .../complexity_router/complexity_router.py | 2 +- ruff-strict-budget.json | 4 +- type-discipline-budget.json | 6 +- 14 files changed, 76 insertions(+), 84 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 65142091712..43df27ea2e2 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 31903 + "limit": 31256 }, "reportArgumentType": { "limit": 2645 @@ -24,7 +24,7 @@ "limit": 42 }, "reportExplicitAny": { - "limit": 10214 + "limit": 10208 }, "reportFunctionMemberAccess": { "limit": 11 @@ -33,7 +33,7 @@ "limit": 227 }, "reportIncompatibleMethodOverride": { - "limit": 78 + "limit": 77 }, "reportIncompatibleVariableOverride": { "limit": 12 @@ -99,7 +99,7 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 45366 + "limit": 45357 }, "reportUnknownLambdaType": { "limit": 113 diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 8821736d0ff..d57c1a78f3d 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -215,7 +215,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) if result: - return LiteLLM_ManagedFileTable(**result) + return LiteLLM_ManagedFileTable.model_validate(result) ## CHECK DB db_object = await self.prisma_client.db.litellm_managedfiletable.find_first( @@ -223,7 +223,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) if db_object: - return LiteLLM_ManagedFileTable(**db_object.model_dump()) + return LiteLLM_ManagedFileTable.model_validate(db_object.model_dump()) return None async def delete_unified_file_id( @@ -349,7 +349,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if isinstance(batch.file_object, str) else batch.file_object ) - batch_obj = LiteLLMBatch(**batch_data) + batch_obj = LiteLLMBatch.model_validate(batch_data) batch_obj.id = batch.unified_object_id batch_objects.append(batch_obj) @@ -382,7 +382,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): "flat_model_file_ids": {"hasSome": model_object_ids}, } ) - return [OpenAIFileObject(**file_object.file_object) for file_object in file_ids] + return [OpenAIFileObject.model_validate(file_object.file_object) for file_object in file_ids] async def check_managed_file_id_access( self, data: Dict, user_api_key_dict: UserAPIKeyAuth diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 60dbf7c644a..0ac22b5bb1e 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -131,8 +131,8 @@ class CustomStreamWrapper: self.sent_last_chunk = False self._stream_created_time: float = time.time() - litellm_params: GenericLiteLLMParams = GenericLiteLLMParams( - **self.logging_obj.model_call_details.get("litellm_params", {}) + litellm_params: GenericLiteLLMParams = GenericLiteLLMParams.model_validate( + dict(**self.logging_obj.model_call_details.get("litellm_params", {})) ) self.merge_reasoning_content_in_choices: bool = litellm_params.merge_reasoning_content_in_choices or False self.sent_first_thinking_block = False diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index d0b0dbb070d..1a860cca5a9 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -66,7 +66,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): ) # Create ResponseReasoningItem object from the item data - reasoning_item = ResponseReasoningItem(**item_data) + reasoning_item = ResponseReasoningItem.model_validate(item_data) # Convert back to dict with exclude_none=True to exclude None fields dict_reasoning_item = reasoning_item.model_dump(exclude_none=True) @@ -346,4 +346,4 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIError raise AzureOpenAIError(message=raw_response.text, status_code=raw_response.status_code) - return ResponsesAPIResponse(**raw_response_json) + return ResponsesAPIResponse.model_validate(raw_response_json) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ec1301e5923..d6acbaae434 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6184,10 +6184,12 @@ class BaseLLMHTTPHandler: import websockets from websockets.asyncio.client import ClientConnection - litellm_params = GenericLiteLLMParams( - api_base=api_base, - api_key=api_key, - **kwargs, + litellm_params = GenericLiteLLMParams.model_validate( + { + "api_base": api_base, + "api_key": api_key, + **kwargs, + } ) headers = responses_api_provider_config.validate_environment( headers={}, diff --git a/litellm/llms/manus/responses/transformation.py b/litellm/llms/manus/responses/transformation.py index 0db53f90330..e6fbbac0563 100644 --- a/litellm/llms/manus/responses/transformation.py +++ b/litellm/llms/manus/responses/transformation.py @@ -217,7 +217,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}" try: - response = ResponsesAPIResponse(**raw_response_json) + response = ResponsesAPIResponse.model_validate(raw_response_json) except Exception: verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") response = ResponsesAPIResponse.model_construct(**raw_response_json) @@ -305,7 +305,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}" try: - response = ResponsesAPIResponse(**raw_response_json) + response = ResponsesAPIResponse.model_validate(raw_response_json) except Exception: verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") response = ResponsesAPIResponse.model_construct(**raw_response_json) diff --git a/litellm/proxy/hooks/user_management_event_hooks.py b/litellm/proxy/hooks/user_management_event_hooks.py index 6122f0594e8..d8cfae5dab0 100644 --- a/litellm/proxy/hooks/user_management_event_hooks.py +++ b/litellm/proxy/hooks/user_management_event_hooks.py @@ -6,8 +6,6 @@ import asyncio from datetime import datetime, timezone from typing import Optional -from pydantic import BaseModel - import litellm from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid @@ -16,7 +14,6 @@ from litellm.proxy._types import ( CommonProxyErrors, LiteLLM_AuditLogs, Litellm_EntityType, - LiteLLM_UserTable, LitellmTableNames, NewUserRequest, NewUserResponse, @@ -58,11 +55,11 @@ class UserManagementEventHooks: try: if prisma_client is None: raise Exception(CommonProxyErrors.db_not_connected_error.value) - user_row: BaseModel = await UserRepository(prisma_client).table.find_first( - where={"user_id": response.user_id} - ) - - user_row_litellm_typed = LiteLLM_UserTable(**user_row.model_dump(exclude_none=True)) + if response.user_id is None: + raise Exception("no user_id returned for the newly created user") + user_row_litellm_typed = await UserRepository(prisma_client).find_by_id(response.user_id) + if user_row_litellm_typed is None: + raise Exception(f"no user row found for user_id={response.user_id}") asyncio.create_task( UserManagementEventHooks.create_internal_user_audit_log( user_id=user_row_litellm_typed.user_id, diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 8682b61f910..110e5883485 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -3137,7 +3137,7 @@ class SSOAuthenticationHandler: _default_team_params = deepcopy(litellm.default_team_params) _new_team_request = team_request.model_dump() _new_team_request.update(_default_team_params) - team_request = NewTeamRequest(**_new_team_request) + team_request = NewTeamRequest.model_validate(_new_team_request) return team_request @staticmethod @@ -3271,14 +3271,6 @@ class SSOAuthenticationHandler: # User might not be already created on first generation of key # But if it is, we want their models preferences - default_ui_key_values: Dict[str, Any] = { - "duration": LITELLM_UI_SESSION_DURATION, - "key_max_budget": litellm.max_ui_session_budget, - "aliases": {}, - "config": {}, - "spend": 0, - "team_id": "litellm-dashboard", - } user_defined_values: Optional[SSOUserDefinedValues] = None if user_custom_sso is not None: @@ -3338,10 +3330,20 @@ class SSOAuthenticationHandler: verbose_proxy_logger.info(f"user_defined_values for creating ui key: {user_defined_values}") - default_ui_key_values.update(user_defined_values) - default_ui_key_values["request_type"] = "key" response = await generate_key_helper_fn( - **default_ui_key_values, # type: ignore + request_type="key", + duration=LITELLM_UI_SESSION_DURATION, + key_max_budget=litellm.max_ui_session_budget, + aliases={}, + config={}, + spend=0, + team_id="litellm-dashboard", + models=user_defined_values["models"], + user_id=user_defined_values["user_id"], + user_email=user_defined_values["user_email"], + user_role=user_defined_values["user_role"], + max_budget=user_defined_values["max_budget"], + budget_duration=user_defined_values["budget_duration"], table_name="key", ) diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index b2e36188681..2960b031cd3 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -1035,7 +1035,7 @@ async def get_batch_from_database( if isinstance(db_batch_object.file_object, str) else db_batch_object.file_object ) - response = LiteLLMBatch(**batch_data) + response = LiteLLMBatch.model_validate(batch_data) response.id = batch_id # The stored batch object has the raw provider input_file_id. Resolve to unified ID. diff --git a/litellm/repositories/user_repository.py b/litellm/repositories/user_repository.py index f0b5dfd8bc5..e93991463f8 100644 --- a/litellm/repositories/user_repository.py +++ b/litellm/repositories/user_repository.py @@ -3,61 +3,56 @@ User repository for database operations on LiteLLM_UserTable. """ import json -from typing import Any, Dict, List, Optional, Type +from typing import Any, Dict, List, Mapping, Optional, Type from litellm.models.user import LiteLLM_UserTable -from litellm.repositories.base_repository import BaseRepository +from litellm.repositories.base_repository import BaseRepository, DbRecord, record_to_dict + +_JSON_ENCODED_COLUMNS = frozenset({"metadata", "model_spend", "model_max_budget"}) class UserRepository(BaseRepository[LiteLLM_UserTable]): """Repository for user database operations.""" @property - def table(self) -> Any: + def table(self) -> Any: # any-ok: Prisma table actions are reached through the untyped client wrapper return self.prisma_client.db.litellm_usertable @property def model_class(self) -> Type[LiteLLM_UserTable]: return LiteLLM_UserTable - def _to_model(self, record: Any) -> Optional[LiteLLM_UserTable]: + def _to_model(self, record: Optional[DbRecord]) -> Optional[LiteLLM_UserTable]: """Convert a database record to a User model.""" if record is None: return None - data = record.dict() if hasattr(record, "dict") else dict(record) + return LiteLLM_UserTable.model_validate( + { + column: json.loads(value) if column in _JSON_ENCODED_COLUMNS and isinstance(value, str) else value + for column, value in record_to_dict(record).items() + } + ) - json_fields = ["metadata", "model_spend", "model_max_budget"] - for field in json_fields: - if isinstance(data.get(field), str): - data[field] = json.loads(data[field]) - - return LiteLLM_UserTable(**data) - - async def find_by_id(self, user_id: str, id_field: str = "user_id") -> Optional[LiteLLM_UserTable]: - return await super().find_by_id(user_id, id_field) + async def find_by_id(self, id_value: str, id_field: str = "user_id") -> Optional[LiteLLM_UserTable]: + return await super().find_by_id(id_value, id_field) async def find_by_email(self, user_email: str) -> Optional[LiteLLM_UserTable]: """Find a user by email.""" - records = await self.table.find_many(where={"user_email": user_email}) - if records: - return self._to_model(records[0]) - return None + records = await self.find_many(where={"user_email": user_email}) + return records[0] if records else None async def find_by_sso_id(self, sso_user_id: str) -> Optional[LiteLLM_UserTable]: """Find a user by SSO ID.""" - record = await self.table.find_unique(where={"sso_user_id": sso_user_id}) - return self._to_model(record) + return await self.find_by_id(sso_user_id, id_field="sso_user_id") async def find_by_organization_id(self, organization_id: str) -> List[LiteLLM_UserTable]: """Find all users in an organization.""" - records = await self.table.find_many(where={"organization_id": organization_id}) - return self._to_model_list(records) + return await self.find_many(where={"organization_id": organization_id}) async def find_by_team_id(self, team_id: str) -> List[LiteLLM_UserTable]: """Find all users in a team.""" - records = await self.table.find_many(where={"teams": {"has": team_id}}) - return self._to_model_list(records) + return await self.find_many(where={"teams": {"has": team_id}}) async def count_billable_users(self) -> int: """Number of users that count toward the license seat limit. @@ -86,7 +81,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): max_budget: Optional[float] = None, user_email: Optional[str] = None, models: Optional[List[str]] = None, - metadata: Optional[Dict[str, Any]] = None, + metadata: Optional[Mapping[str, object]] = None, max_parallel_requests: Optional[int] = None, tpm_limit: Optional[int] = None, rpm_limit: Optional[int] = None, @@ -96,7 +91,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): object_permission_id: Optional[str] = None, ) -> LiteLLM_UserTable: """Create a new user.""" - data: Dict[str, Any] = {"user_id": user_id} + data: Dict[str, object] = {"user_id": user_id} if user_alias is not None: data["user_alias"] = user_alias if team_id is not None: @@ -149,7 +144,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): max_budget: Optional[float] = None, user_email: Optional[str] = None, models: Optional[List[str]] = None, - metadata: Optional[Dict[str, Any]] = None, + metadata: Optional[Mapping[str, object]] = None, max_parallel_requests: Optional[int] = None, tpm_limit: Optional[int] = None, rpm_limit: Optional[int] = None, @@ -159,7 +154,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): object_permission_id: Optional[str] = None, ) -> Optional[LiteLLM_UserTable]: """Update a user.""" - data: Dict[str, Any] = {} + data: Dict[str, object] = {} if user_alias is not None: data["user_alias"] = user_alias if team_id is not None: @@ -212,11 +207,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): if not await self.exists(user_id, id_field="user_id"): return None - record = await self.table.update( - where={"user_id": user_id}, - data={"teams": {"push": team_id}}, - ) - return self._to_model(record) + return await self.update(user_id, {"teams": {"push": team_id}}, id_field="user_id") async def remove_from_team(self, user_id: str, team_id: str) -> Optional[LiteLLM_UserTable]: """Remove a user from a team. diff --git a/litellm/router.py b/litellm/router.py index ac00cdbc2b0..399bfcdf3e1 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8613,9 +8613,9 @@ class Router: deployment = self.get_deployment(model_id=model_id) if deployment is None or self._is_deployment_blocked(deployment): return None - return CredentialLiteLLMParams(**deployment.litellm_params.model_dump(exclude_none=True)).model_dump( - exclude_none=True - ) + return CredentialLiteLLMParams.model_validate( + deployment.litellm_params.model_dump(exclude_none=True) + ).model_dump(exclude_none=True) def get_deployment_by_model_group_name(self, model_group_name: str) -> Optional[Deployment]: """ @@ -8755,9 +8755,9 @@ class Router: return None # Get basic credentials - credentials = CredentialLiteLLMParams(**deployment.litellm_params.model_dump(exclude_none=True)).model_dump( - exclude_none=True - ) + credentials = CredentialLiteLLMParams.model_validate( + deployment.litellm_params.model_dump(exclude_none=True) + ).model_dump(exclude_none=True) # Resolve litellm_credential_name to actual credentials if deployment.litellm_params.litellm_credential_name is not None: diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index b43fe0da4ca..1c6a83c6cd9 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -329,7 +329,7 @@ class ComplexityRouter(CustomLogger): # Parse config - always create a new instance to avoid singleton mutation if complexity_router_config: - self.config = ComplexityRouterConfig(**complexity_router_config) + self.config = ComplexityRouterConfig.model_validate(complexity_router_config) else: self.config = ComplexityRouterConfig() diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index dfdc4efe800..6fb3ed748b6 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -24,7 +24,7 @@ "limit": 130 }, "ANN401": { - "limit": 2010 + "limit": 2009 }, "ASYNC230": { "limit": 14 @@ -324,7 +324,7 @@ "limit": 879 }, "UP006": { - "limit": 12138 + "limit": 12135 }, "UP007": { "limit": 2526 diff --git a/type-discipline-budget.json b/type-discipline-budget.json index c9a1b59cc06..8d5161ec7ea 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 23253 + "limit": 23250 }, "LIT002": { - "limit": 27427 + "limit": 27425 }, "LIT003": { "limit": 292 @@ -24,6 +24,6 @@ "limit": 1004 }, "LIT009": { - "limit": 2474 + "limit": 2473 } } From 2769fbe37b0de1b0941a05f7e97183cbba5f48b5 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 12:22:09 -0700 Subject: [PATCH 13/50] feat(ui): give the budgets page a standard header and default column set Matches the Virtual Keys layout: a page header with the wallet icon, the create button directly beneath it, and the tab bar below that, on the same page padding Teams and Access Groups use so the table no longer sits against the window edge. Reset and Created start hidden, so the table opens on the four columns it has always shown and the two new ones are opt-in from the Columns menu. --- .../budgets/_components/BudgetTable.test.tsx | 30 ++++++++++++++++--- .../budgets/_components/BudgetTable.tsx | 3 +- .../_components/BudgetTableColumns.tsx | 6 ++++ .../budgets/_components/budget_panel.tsx | 19 ++++++++---- 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx index 0c485adf4f2..f78ba34770d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx @@ -52,6 +52,11 @@ const FORBIDDEN_PROBLEM = { detail: "Only proxy admins can view budgets", }; +const showColumn = async (user: ReturnType, columnId: string) => { + await user.click(screen.getByTestId("view-options-trigger")); + await user.click(await screen.findByTestId(`view-option-${columnId}`)); +}; + const defaultProps = { canModify: true, onEditClick: vi.fn(), @@ -72,14 +77,26 @@ describe("BudgetTable", () => { expect(screen.getByText("10")).toBeInTheDocument(); }); - it("should render the reset column with the friendly duration label", () => { + it("should open on the four columns the page has always shown, with reset and created off", () => { renderWithProviders(); + const headers = screen.getAllByRole("columnheader").map((header) => header.textContent); + expect(headers).toEqual(expect.arrayContaining(["Budget ID", "Max Budget", "TPM", "RPM"])); + expect(headers).not.toContain("Reset"); + expect(headers).not.toContain("Created"); + }); + + it("should render the reset column with the friendly duration label once it is turned on", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await showColumn(user, "budget_duration"); expect(screen.getByText("monthly")).toBeInTheDocument(); }); - it("should render 'Not set' when a budget has no reset duration", () => { + it("should render 'Not set' when a budget has no reset duration", async () => { + const user = userEvent.setup(); const list = makeList({ rows: [makeBudget({ budget_duration: null })] }); renderWithProviders(); + await showColumn(user, "budget_duration"); expect(screen.getByText("Not set")).toBeInTheDocument(); }); @@ -109,16 +126,21 @@ describe("BudgetTable", () => { }); it("should offer sorting on every backend-sortable column", async () => { + const user = userEvent.setup(); renderWithProviders(); + await showColumn(user, "created_at"); for (const field of ["budget_id", "max_budget", "tpm_limit", "rpm_limit", "created_at"]) { expect(screen.getByTestId(`sort-header-${field}`)).toBeInTheDocument(); } }); - it("should not make the reset column sortable", () => { + it("should not make the reset column sortable", async () => { + const user = userEvent.setup(); renderWithProviders(); + await showColumn(user, "budget_duration"); + const headers = screen.getAllByRole("columnheader").map((header) => header.textContent); + expect(headers).toContain("Reset"); expect(screen.queryByTestId("sort-header-budget_duration")).not.toBeInTheDocument(); - expect(screen.getByText("Reset")).toBeInTheDocument(); }); it("should ask the list for a new sort when a sortable header is clicked", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.tsx index 76355c874f8..88e40dfba2f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.tsx @@ -23,7 +23,7 @@ import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { ApiError } from "@/lib/http/client"; -import { getBudgetTableColumns } from "./BudgetTableColumns"; +import { BUDGET_TABLE_HIDDEN_COLUMNS, getBudgetTableColumns } from "./BudgetTableColumns"; interface BudgetTableProps { list: ResourceListResult; @@ -225,6 +225,7 @@ const BudgetTable: React.FC = ({ list, canModify, onEditClick, data={list.rows} columns={columns} getRowId={(budget, index) => budget.budget_id || String(index)} + defaultColumnVisibility={BUDGET_TABLE_HIDDEN_COLUMNS} sortingMode="server" sorting={list.sorting} onSortingChange={list.onSortingChange} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx index d1192468359..e5cd9043492 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx @@ -75,6 +75,12 @@ function BudgetRowActions({ budget, onEditClick, onDeleteClick }: BudgetRowActio ); } +/** Off by default so the table opens on the four columns it has always shown; the Columns menu turns them on. */ +export const BUDGET_TABLE_HIDDEN_COLUMNS: Record = { + budget_duration: false, + created_at: false, +}; + interface BudgetTableColumnsDeps { canModify: boolean; onEditClick: (budget: budgetItem) => void; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx index 78c2c0ca74a..fbcbd501313 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx @@ -3,8 +3,10 @@ * */ +import { Plus, Wallet } from "lucide-react"; import React, { useCallback, useState } from "react"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; +import { PageHeader } from "@/components/shared/PageHeader"; import { Button } from "@/components/ui/button"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; @@ -76,11 +78,19 @@ const BudgetPanel: React.FC = ({ accessToken }) => { }; return ( -
+
+ } + title="Budgets" + subtitle="Spend, TPM and RPM limits you can assign to customers." + /> {canModify && ( - +
+ +
)} @@ -101,7 +111,6 @@ const BudgetPanel: React.FC = ({ accessToken }) => { existingBudget={selectedBudget} /> )} -

Create a budget to assign to customers.

Date: Fri, 31 Jul 2026 19:26:04 +0000 Subject: [PATCH 14/50] test(repositories): lock in UserRepository JSON column decoding Covers the columns UserRepository._to_model decodes so a narrower or wider column set fails instead of silently changing what callers read back. --- .../repositories/test_repositories.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_litellm/repositories/test_repositories.py b/tests/test_litellm/repositories/test_repositories.py index c923b722991..8253308f393 100644 --- a/tests/test_litellm/repositories/test_repositories.py +++ b/tests/test_litellm/repositories/test_repositories.py @@ -1905,6 +1905,28 @@ class TestUserRepositoryExtended: ) assert updated.user_email == "new@example.com" + @pytest.mark.asyncio + async def test_find_by_id_decodes_only_the_json_encoded_columns(self, repo): + repo._prisma_client.db.litellm_usertable._records["user-json"] = { + "user_id": "user-json", + "user_email": "json@example.com", + "user_role": '{"not": "json"}', + "teams": [], + "models": [], + "metadata": '{"department": "engineering"}', + "model_spend": '{"gpt-4": 10.5}', + "model_max_budget": '{"gpt-4": 100.0}', + } + + user = await repo.find_by_id("user-json") + + assert user is not None + assert user.metadata == {"department": "engineering"} + assert user.model_spend == {"gpt-4": 10.5} + assert user.model_max_budget == {"gpt-4": 100.0} + assert user.user_email == "json@example.com" + assert user.user_role == '{"not": "json"}' + class TestProjectRepositoryExtended: @pytest.fixture From 26ace642be31d25c8826c27947fef76060ece375 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 31 Jul 2026 19:48:30 +0000 Subject: [PATCH 15/50] test(proxy): cover the user-created audit hook's database read-back The hook resolves the newly created user through UserRepository and builds the audit entry from that row. Pin both halves: the entry carries the persisted row's fields rather than the /user/new response, and a user id that resolves to nothing produces no entry at all. --- .../hooks/test_user_management_event_hooks.py | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 tests/test_litellm/proxy/hooks/test_user_management_event_hooks.py diff --git a/tests/test_litellm/proxy/hooks/test_user_management_event_hooks.py b/tests/test_litellm/proxy/hooks/test_user_management_event_hooks.py new file mode 100644 index 00000000000..102430ab985 --- /dev/null +++ b/tests/test_litellm/proxy/hooks/test_user_management_event_hooks.py @@ -0,0 +1,99 @@ +import asyncio +import json +import sys +from types import SimpleNamespace +from typing import Any, Dict, List, Optional +from unittest.mock import AsyncMock, patch + +import pytest + +import litellm +from litellm.proxy._types import NewUserRequest, NewUserResponse, UserAPIKeyAuth +from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks + + +class FakeUserTable: + def __init__(self, rows: List[Dict[str, Any]]): + self._rows = rows + + async def find_unique(self, where: Dict[str, Any]) -> Optional[Dict[str, Any]]: + return next( + (row for row in self._rows if all(row.get(key) == value for key, value in where.items())), + None, + ) + + +class FakePrismaClient: + def __init__(self, rows: List[Dict[str, Any]]): + self.db = SimpleNamespace(litellm_usertable=FakeUserTable(rows)) + + +async def _run_created_hook(prisma_client: FakePrismaClient, audit_log: AsyncMock) -> None: + proxy_server = SimpleNamespace( + prisma_client=prisma_client, + litellm_proxy_admin_name="admin-user", + ) + with ( + patch.dict(sys.modules, {"litellm.proxy.proxy_server": proxy_server}), + patch.object(litellm, "store_audit_logs", True), + patch( + "litellm.proxy.hooks.user_management_event_hooks.create_audit_log_for_update", + audit_log, + ), + patch.object( + UserManagementEventHooks, + "async_send_user_invitation_email", + AsyncMock(), + ), + ): + await UserManagementEventHooks.async_user_created_hook( + data=NewUserRequest(user_email="new@example.com", send_invite_email=False), + response=NewUserResponse( + user_id="user-1", + user_email="new@example.com", + key="sk-test", + ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-user", api_key="sk-admin"), + ) + await asyncio.sleep(0) + + +@pytest.mark.asyncio +async def test_created_hook_audit_logs_the_row_read_back_from_the_database(): + """The audit entry must describe the persisted user row, not the /user/new response.""" + prisma_client = FakePrismaClient( + [ + { + "user_id": "user-1", + "user_email": "new@example.com", + "user_role": "proxy_admin", + "models": ["gpt-4"], + "teams": ["team-a"], + "metadata": '{"source": "api"}', + } + ] + ) + audit_log = AsyncMock() + + await _run_created_hook(prisma_client, audit_log) + + audit_log.assert_awaited_once() + request_data = audit_log.await_args.kwargs["request_data"] + assert request_data.object_id == "user-1" + assert request_data.action == "created" + + updated_values = json.loads(request_data.updated_values) + assert updated_values["user_role"] == "proxy_admin" + assert updated_values["models"] == ["gpt-4"] + assert updated_values["teams"] == ["team-a"] + assert updated_values["metadata"] == {"source": "api"} + + +@pytest.mark.asyncio +async def test_created_hook_skips_the_audit_log_when_no_user_row_exists(): + """A user id that resolves to nothing must not produce an audit entry.""" + audit_log = AsyncMock() + + await _run_created_hook(FakePrismaClient([]), audit_log) + + audit_log.assert_not_awaited() From 79b2a5e56e4a2eb05a26bc5e7a66638b15825006 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 12:55:16 -0700 Subject: [PATCH 16/50] feat(ui): put the budgets CTA in the tab bar and scroll the rows, not the page The create button now sits in the tab bar beside the tabs, the way Teams lays it out, with one divider between them and the rule running the full width underneath. Adds a fillHeight mode to DataTable that treats the parent's height as a ceiling rather than a target, so the table still sizes to its rows and a short one keeps its footer under the last row, while a long one scrolls its rows under a sticky header instead of scrolling the page. This replaces the hardcoded viewport-height caps those tables would otherwise need. Two details the mode has to fix: the Table primitive's own overflow container would capture the sticky header, and rows would show through the semi-transparent header tint. --- .../budgets/_components/BudgetTable.tsx | 1 + .../budgets/_components/budget_panel.tsx | 46 ++++++++++--------- .../shared/DataTable/DataTable.test.tsx | 38 +++++++++++++++ .../components/shared/DataTable/DataTable.tsx | 34 ++++++++++---- .../src/components/shared/DataTable/types.ts | 6 +++ 5 files changed, 96 insertions(+), 29 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.tsx index 88e40dfba2f..80872c58d28 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.tsx @@ -226,6 +226,7 @@ const BudgetTable: React.FC = ({ list, canModify, onEditClick, columns={columns} getRowId={(budget, index) => budget.budget_id || String(index)} defaultColumnVisibility={BUDGET_TABLE_HIDDEN_COLUMNS} + fillHeight sortingMode="server" sorting={list.sorting} onSortingChange={list.onSortingChange} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx index fbcbd501313..e49f0ffc722 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx @@ -7,6 +7,7 @@ import { Plus, Wallet } from "lucide-react"; import React, { useCallback, useState } from "react"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { PageHeader } from "@/components/shared/PageHeader"; +import { ToolbarSeparator } from "@/components/shared/ToolbarSeparator"; import { Button } from "@/components/ui/button"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; @@ -78,31 +79,34 @@ const BudgetPanel: React.FC = ({ accessToken }) => { }; return ( -
+
} title="Budgets" subtitle="Spend, TPM and RPM limits you can assign to customers." /> - {canModify && ( -
- + +
+ {canModify && ( + <> + + + + )} + + + Budgets + + + Examples + +
- )} - - - - Budgets - - - Examples - - - -
+ +
{selectedBudget && ( = ({ accessToken }) => { />
- -
+ +

How to use budget id

diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx index 60547415908..8555a10c326 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx @@ -625,6 +625,44 @@ describe("DataTable layout", () => { const scroller = container.querySelector('[data-slot="table-container"]')?.parentElement as HTMLElement; expect(scroller.style.maxHeight).toBe("240px"); }); + + it("caps fillHeight at the parent's height instead of stretching to it, so a short table stays short", () => { + const { container } = render(); + const scroller = container.querySelector('[data-slot="table-container"]')?.parentElement as HTMLElement; + const frame = scroller.parentElement as HTMLElement; + const outer = frame.parentElement as HTMLElement; + + // A ceiling, not a stretch: flex-1 here would hold the footer at the bottom on a two-row table. + expect(outer.className).toContain("max-h-full"); + expect(outer.className).not.toContain("flex-1"); + expect(frame.className).not.toContain("flex-1"); + expect(scroller.className).not.toContain("flex-1"); + + expect(outer.className).toContain("flex-col"); + expect(frame.className).toContain("flex-col"); + expect(scroller.className).toContain("min-h-0"); + expect(scroller.className).toContain("overflow-auto"); + expect(scroller.style.maxHeight).toBe(""); + // Without this the Table primitive's own overflow container captures the sticky header. + expect(scroller.className).toContain("[&_[data-slot=table-container]]:overflow-visible"); + + const thead = container.querySelector("thead") as HTMLElement; + expect(thead.className).toContain("sticky"); + // Rows pass under the header, so the semi-transparent row tint alone would let them show through. + expect(thead.className).toContain("bg-background"); + }); + + it("leaves the default layout untouched when neither height mode is set", () => { + const { container } = render(); + const scroller = container.querySelector('[data-slot="table-container"]')?.parentElement as HTMLElement; + + expect(scroller.className).toContain("overflow-x-auto"); + expect(scroller.className).not.toContain("min-h-0"); + expect(scroller.style.maxHeight).toBe(""); + expect((scroller.parentElement as HTMLElement).className).not.toContain("flex-col"); + expect(container.querySelector("thead")?.className).not.toContain("sticky"); + expect(container.querySelector("thead")?.className).not.toContain("bg-background"); + }); }); describe("DataTable misconfiguration guards", () => { diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx index c4799594465..8cd0e25dfc4 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx @@ -48,6 +48,22 @@ const INTERACTIVE_SELECTOR = "button, a, input, select, textarea, [role=checkbox const noop = () => {}; +/** + * Height-filling mode. The table still sizes to its rows; the parent's height is only a ceiling, so + * a short table keeps its footer under the last row and a long one scrolls its rows instead of the + * page. `table-container` is the Table primitive's own overflow-x wrapper; left as a scroll box it + * captures the sticky header and the header scrolls away with the rows. And rows pass under that + * header, which the semi-transparent header row tint alone would not hide. + */ +const FILL_CLASSES = { + outer: "flex max-h-full min-h-0 flex-col", + frame: "flex min-h-0 flex-col", + body: "min-h-0 [&_[data-slot=table-container]]:overflow-visible", + header: "bg-background", +} as const; + +const NO_FILL_CLASSES = { outer: "", frame: "", body: "", header: "" } as const; + export class DataTableConfigError extends Error { constructor(messages: readonly string[]) { super(`DataTable misconfiguration:\n- ${messages.join("\n- ")}`); @@ -538,6 +554,7 @@ export function DataTable(props: DataTableProps(props: DataTableProps { @@ -604,15 +622,15 @@ export function DataTable(props: DataTableProps -
- {toolbar !== undefined &&
{toolbar(table)}
} +
+
+ {toolbar !== undefined &&
{toolbar(table)}
}
- + {table.getHeaderGroups().map((headerGroup) => ( {headerGroup.headers.map((header) => ( @@ -631,7 +649,7 @@ export function DataTable(props: DataTableProps{footer(table)}}
- {paginationNode !== null &&
{paginationNode}
} + {paginationNode !== null &&
{paginationNode}
}
); diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/types.ts b/ui/litellm-dashboard/src/components/shared/DataTable/types.ts index 40f3a4df204..dd578f4df45 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/types.ts +++ b/ui/litellm-dashboard/src/components/shared/DataTable/types.ts @@ -69,6 +69,12 @@ export interface DataTableProps { rowClassName?: (row: Row) => string; maxBodyHeight?: number | string; + /** + * Scroll the rows inside whatever height the parent gives the table, rather than growing the page. + * The table becomes a flex column, so the parent must be a height-constrained flex container; without + * one it degrades to the normal auto-height layout. Use instead of `maxBodyHeight` to avoid a magic number. + */ + fillHeight?: boolean; size?: DataTableSize; toolbar?: (table: Table) => React.ReactNode; From 3083c55ffcf4fd5456489526a2214801997a7ace Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:34:42 -0700 Subject: [PATCH 17/50] fix(ui): nest source object in Claude Code marketplace settings snippet (#35322) Co-authored-by: milan Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../claude_code_plugins/helpers.test.ts | 16 +++++++++ .../components/claude_code_plugins/helpers.ts | 21 +++++++++++ .../claude_code_plugins/skill_detail.tsx | 35 ++++--------------- 3 files changed, 44 insertions(+), 28 deletions(-) diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.test.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.test.ts index 4c84db2a97d..c16eba24f7b 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.test.ts +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.test.ts @@ -17,9 +17,25 @@ import { formatKeywords, parseSkillSource, isValidSubPath, + buildMarketplaceSettingsSnippet, } from "./helpers"; import { MarketplacePluginEntry, PluginSource } from "./types"; +describe("buildMarketplaceSettingsSnippet", () => { + it("nests the url under a source object so Claude Code accepts the marketplace", () => { + expect(JSON.parse(buildMarketplaceSettingsSnippet("https://proxy.example.com"))).toEqual({ + extraKnownMarketplaces: { + "my-org": { + source: { + source: "url", + url: "https://proxy.example.com/claude-code/marketplace.json", + }, + }, + }, + }); + }); +}); + describe("formatInstallCommand", () => { it("formats github source with repo", () => { const source: PluginSource = { source: "github", repo: "org/repo" }; diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.ts index cab3c5cba3c..a4e70f78af1 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.ts +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.ts @@ -176,6 +176,27 @@ export const parseSkillSource = (rawUrl: string, subPath?: string): SkillSourceP return parseRawGitSource(url, subPath); }; +/** + * Build the `~/.claude/settings.json` snippet that registers the proxy as a marketplace. + * Claude Code expects `extraKnownMarketplaces..source` to be a source object, not a + * bare `"url"` string, so the url/source pair is nested one level deeper. + */ +export const buildMarketplaceSettingsSnippet = (proxyOrigin: string): string => + JSON.stringify( + { + extraKnownMarketplaces: { + "my-org": { + source: { + source: "url", + url: `${proxyOrigin}/claude-code/marketplace.json`, + }, + }, + }, + }, + null, + 2, + ); + /** * Generate install command for Claude Code CLI * Format: /plugin marketplace add org/repo OR /plugin marketplace add url diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx b/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx index 8b7537c1988..fe001641135 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx @@ -1,6 +1,6 @@ import React, { useState } from "react"; import { ArrowLeftOutlined, CopyOutlined, CheckOutlined, LinkOutlined } from "@ant-design/icons"; -import { formatInstallCommand } from "./helpers"; +import { buildMarketplaceSettingsSnippet, formatInstallCommand } from "./helpers"; import { Plugin } from "./types"; interface SkillDetailProps { @@ -31,6 +31,10 @@ const SkillDetail: React.FC = ({ skill, onBack }) => { const installCommand = formatInstallCommand(skill); + const settingsSnippet = buildMarketplaceSettingsSnippet( + typeof window !== "undefined" ? window.location.origin : "", + ); + const detailRows = [ ...(skill.category ? [{ property: "Category", value: skill.category }] : []), ...(skill.domain ? [{ property: "Domain", value: skill.domain }] : []), @@ -298,21 +302,7 @@ const SkillDetail: React.FC = ({ skill, onBack }) => { > ~/.claude/settings.json
From 546640227463d3af563a796022fc8d49626401c1 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 31 Jul 2026 16:36:32 -0700 Subject: [PATCH 18/50] fix(ui): keep the session view open when selecting a log inside it (#35399) Opening a session from the logs table stored no ?session_id (row clicks called openLog, which deletes it), so session mode was derived from the clicked row's session_total_count. Rows fetched by the session drawer come from /spend/logs/session/ui, which does not enrich that field, so selecting any log inside the session view swapped in an unenriched row and collapsed the drawer to a single-log Trace view Row clicks on a multi-call session's row now call openSession, and selectLog writes ?session_id when the session view is active, so session mode is anchored in the URL instead of derived from row data --- .../view_logs/RequestLogsPanel.test.tsx | 35 +++++++++++++++++++ .../components/view_logs/RequestLogsPanel.tsx | 12 ++++--- .../components/view_logs/logDetailRouting.ts | 7 ++-- 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.test.tsx index 49870e51c2b..6a66e845675 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.test.tsx @@ -403,6 +403,41 @@ describe("RequestLogsPanel", () => { expect(drawer()).toHaveAttribute("data-session-id", "sess-1"); }); }); + + it("clicking a multi-call session's row writes ?session_id= alongside ?log_id=", async () => { + const user = userEvent.setup(); + respondWith([ + logEntry({ request_id: "req-llm", call_type: "acompletion", session_id: "sess-1", session_total_count: 3 }), + logEntry({ request_id: "req-llm-2", call_type: "acompletion", session_id: "sess-1", session_total_count: 3 }), + ]); + renderWithProviders(); + + await waitFor(() => expect(row("req-llm")).not.toBeNull()); + await user.click(row("req-llm") as HTMLElement); + + const params = new URLSearchParams(window.location.search); + expect(params.get("session_id")).toBe("sess-1"); + expect(params.get("log_id")).toBe("req-llm"); + await waitFor(() => expect(drawer()).toHaveAttribute("data-session-id", "sess-1")); + }); + + it("selecting another log while a session view is open keeps the session open", async () => { + const user = userEvent.setup(); + window.history.replaceState(null, "", "/logs/?log_id=req-llm"); + respondWith([ + logEntry({ request_id: "req-llm", call_type: "acompletion", session_id: "sess-1", session_total_count: 3 }), + logEntry({ request_id: "req-unenriched" }), + ]); + renderWithProviders(); + + await waitFor(() => expect(drawer()).toHaveAttribute("data-session-id", "sess-1")); + + await user.click(screen.getByRole("button", { name: "select-next-log" })); + + await waitFor(() => expect(drawer()).toHaveAttribute("data-log-id", "req-unenriched")); + expect(new URLSearchParams(window.location.search).get("session_id")).toBe("sess-1"); + expect(drawer()).toHaveAttribute("data-session-id", "sess-1"); + }); }); describe("live tail", () => { diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx index 06c8ca26a7e..aa3fa32bdc8 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx @@ -235,9 +235,13 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID, const handleRowClick = useCallback( (log: LogEntry) => { setSelectedLog(log); - openLog(log.request_id); + if (log.session_id && (log.session_total_count || 1) > 1) { + openSession(log.session_id, log.request_id); + } else { + openLog(log.request_id); + } }, - [openLog], + [openLog, openSession], ); const handleSessionClick = useCallback( @@ -253,9 +257,9 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID, const handleSelectLog = useCallback( (log: LogEntry) => { setSelectedLog(log); - selectLog(log.request_id); + selectLog(log.request_id, displaySessionId); }, - [selectLog], + [selectLog, displaySessionId], ); const handleKeyHashClick = useCallback((keyHash: string) => { diff --git a/ui/litellm-dashboard/src/components/view_logs/logDetailRouting.ts b/ui/litellm-dashboard/src/components/view_logs/logDetailRouting.ts index 5b311c94627..b37a4604585 100644 --- a/ui/litellm-dashboard/src/components/view_logs/logDetailRouting.ts +++ b/ui/litellm-dashboard/src/components/view_logs/logDetailRouting.ts @@ -11,7 +11,7 @@ export interface LogDetailRouting { sessionId: string | null; openLog: (requestId: string) => void; openSession: (sessionId: string, requestId: string | null) => void; - selectLog: (requestId: string) => void; + selectLog: (requestId: string, sessionId?: string | null) => void; close: () => void; } @@ -36,9 +36,12 @@ export function useLogDetailRouting(): LogDetailRouting { }); }, []); - const selectLog = useCallback((requestId: string) => { + const selectLog = useCallback((requestId: string, sessionId?: string | null) => { navigateWithParams((params) => { params.set(LOG_ID_QUERY_PARAM, requestId); + if (sessionId) { + params.set(SESSION_ID_QUERY_PARAM, sessionId); + } }, "replace"); }, []); From f8375780fe9eaa05c52716c9e78b18a120199f72 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Fri, 31 Jul 2026 16:47:01 -0700 Subject: [PATCH 19/50] fix(mcp): enforce tool entitlements on every MCP tool dispatch path (#35156) Tool-level MCP entitlements are enforced in one place, check_tool_permission_for_key_team, reached from pre_call_tool_check. Two dispatch paths reached a tool handler without passing through it. execute_mcp_tool's legacy fallback dispatched into the local tool registry after retrying the unprefixed name, with no allowed/banned-tool check, no key/team/org tool permissions and no parameter validation. It now runs the same gate, and only when something can actually dispatch: when the unprefixed name is absent from the local registry too, the existing 404 stands rather than becoming a misleading "server unavailable". The server the tool-level checks need is available even though the tool name is not in the tool -> server mapping: a non-empty prefix has already been compared against the caller's allowed_mcp_servers by exact name, so the named server is in that list. It is resolved from allowed_mcp_servers rather than from the manager's registry, because the registry can return a server the caller holds no grant for, and matching on anything other than name would accept a server the server-level check never validated. The remaining case is a prefix segment that is empty, which the server-level check skips entirely because it is gated on a non-empty server name; that now fails closed with 503 instead of dispatching for a caller holding no server grant at all. An entitled caller's legacy call therefore still dispatches, so a configuration that worked before keeps working; only the unentitled call is refused, now with the entitlement gate's own 403. call_tool ran pre_call_tool_check inside `if proxy_logging_obj:`, so an absent logging object would have skipped authorization silently. This half is defensive with no live hole: all four call sites source the module-level ProxyLogging singleton from proxy_server.py, which is never None. The shape was still wrong. pre_call_tool_check now runs its three authorization checks unconditionally and only the guardrail hooks, which are dispatched through the logger, depend on one being present. A third reported path, where allow_all_keys, BYOM-submitted and upstream-delegated servers are unioned in after the resolver's ceilings, was investigated and found not to be a defect. The widening is real, but a server's tool surface is already boundable for every caller at registration through MCPServer.allowed_tools / disallowed_tools, enforced by check_allowed_or_banned_tools ahead of the entitlement check, and per-caller narrowing plus the org tool ceiling remain available. Nothing here changes that path. Resolves LIT-4956 --- .../mcp_server/mcp_server_manager.py | 36 +-- .../proxy/_experimental/mcp_server/server.py | 48 ++++ .../mcp_server/test_mcp_server_manager.py | 75 ++++++ .../mcp_server/test_openapi_tool_auth.py | 246 +++++++++++++++++- 4 files changed, 389 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index db80c0f76ee..e3f7352e8ca 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -4436,13 +4436,18 @@ class MCPServerManager: arguments: dict[str, Any], server_name: str, user_api_key_auth: Optional[UserAPIKeyAuth], - proxy_logging_obj: ProxyLogging, + proxy_logging_obj: ProxyLogging | None, server: MCPServer, raw_headers: Optional[dict[str, str]] = None, ) -> dict[str, Any]: """ Run pre-call checks and guardrail hooks for an MCP tool call. + Authorization runs unconditionally; only the guardrail hooks, which are + dispatched through ``proxy_logging_obj``, depend on a logger being + present. An absent logger must never be able to turn an authorization + decision into a no-op. + Returns a dict that may contain: - "arguments": hook-modified tool arguments (only if changed) - "extra_headers": headers injected by pre_mcp_call guardrail hooks @@ -4470,6 +4475,10 @@ class MCPServerManager: server=server, ) + hook_result: dict[str, Any] = {} + if proxy_logging_obj is None: + return hook_result + # Extract incoming Bearer token from raw request headers so # guardrails like MCPJWTSigner can verify + re-sign it (FR-5). normalized_raw = {k.lower(): v for k, v in (raw_headers or {}).items()} @@ -4499,7 +4508,6 @@ class MCPServerManager: # Convert to LLM format for existing guardrail compatibility synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(mcp_request_obj, pre_hook_kwargs) - hook_result: dict[str, Any] = {} try: # Use standard pre_call_hook modified_data = await proxy_logging_obj.pre_call_hook( @@ -5125,19 +5133,17 @@ class MCPServerManager: # Allow validation and modification of tool calls before execution # Using standard pre_call_hook ######################################################### - hook_result: dict[str, Any] = {} - if proxy_logging_obj: - hook_result = await self.pre_call_tool_check( - name=name, - arguments=arguments, - server_name=server_name, - user_api_key_auth=user_api_key_auth, - proxy_logging_obj=proxy_logging_obj, - server=mcp_server, - raw_headers=raw_headers, - ) - if "arguments" in hook_result: - arguments = hook_result["arguments"] + hook_result: dict[str, Any] = await self.pre_call_tool_check( + name=name, + arguments=arguments, + server_name=server_name, + user_api_key_auth=user_api_key_auth, + proxy_logging_obj=proxy_logging_obj, + server=mcp_server, + raw_headers=raw_headers, + ) + if "arguments" in hook_result: + arguments = hook_result["arguments"] # Prepare tasks for during hooks tasks = [] diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 48effdb0f6e..d8431b9b3bb 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -2902,6 +2902,54 @@ if MCP_AVAILABLE: # Deprecated: Local MCP Server Tool ######################################################### else: + # Gate only what can actually dispatch. When the unprefixed name is + # not in the registry either, `_handle_local_mcp_tool` below reports + # 404 and nothing runs, so demanding a server here would turn every + # unknown tool name into a misleading 503. + if global_mcp_tool_registry.get_tool(original_tool_name) is not None: + # `mcp_server` is None here because the tool name is not in the + # tool -> server mapping, but the name still carries a prefix + # that the server-level check above compared against the + # caller's `allowed_mcp_servers` by exact `name`. So the named + # server is in that list and can carry the tool-level checks, + # even with the mapping cold. Resolve it from + # `allowed_mcp_servers` rather than the registry: the registry + # would happily return a server the caller holds no grant for, + # and matching anything other than `name` would accept a server + # the check never validated. + prefix_server = next( + (candidate for candidate in allowed_mcp_servers if candidate.name == server_name), + None, + ) + if prefix_server is None: + # A non-empty prefix that passed the server-level check + # always matches here, so this arm only fires when the + # prefix was empty, which is exactly the case that check + # skips. Fail closed rather than dispatch with no server to + # evaluate a tool ceiling against. + raise HTTPException( + status_code=503, + detail=( + f"MCP server for tool '{original_tool_name}' is not available; " + "refusing to dispatch without authorization checks. " + "Retry once the server is registered." + ), + ) + + from litellm.proxy.proxy_server import proxy_logging_obj + + hook_result = await global_mcp_server_manager.pre_call_tool_check( + name=original_tool_name, + arguments=arguments, + server_name=server_name, + user_api_key_auth=user_api_key_auth, + proxy_logging_obj=proxy_logging_obj, + server=prefix_server, + raw_headers=raw_headers, + ) + if "arguments" in hook_result: + arguments = hook_result["arguments"] # pyright: ignore[reportAny] # hook returns untyped args + local_content = await _handle_local_mcp_tool(original_tool_name, arguments) response = CallToolResult(content=cast(Any, local_content), isError=False) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index f6753e28a66..54fd5242d5f 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -46,10 +46,13 @@ from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( ) from litellm.proxy._types import ( LiteLLM_MCPServerTable, + LiteLLM_ObjectPermissionTable, + LitellmUserRoles, MCPApprovalStatus, MCPEnvVar, MCPEnvVarScope, MCPTransport, + UserAPIKeyAuth, ) from litellm.types.mcp import MCPAuth, MCPAuthType from litellm.types.mcp_server.mcp_server_manager import MCPOAuthMetadata, MCPServer @@ -9718,3 +9721,75 @@ class TestOpenAPIRegistryKeyMatchesRegistration: assert result.isError is True assert "not found in registry" in result.content[0].text + + +class TestToolAuthorizationIsNotConditionalOnLogging: + """`call_tool` used to run `pre_call_tool_check` — the only place tool-level + MCP entitlements are enforced — inside `if proxy_logging_obj:`, so a caller + reached with no logging object got no authorization decision at all. Both + production call sites pass the module-level `ProxyLogging` singleton, which + is never None, so this was not a live hole; the invariant being restored is + that an authorization decision cannot be skipped by an absent logger. + """ + + @staticmethod + def _manager_with_scoped_server() -> tuple[MCPServerManager, UserAPIKeyAuth]: + manager = MCPServerManager() + manager.registry["srv-gated"] = MCPServer( + server_id="srv-gated", + name="gated_server", + server_name="gated_server", + alias="gated_server", + url="http://127.0.0.1:1/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.none, + ) + for tool_name in ("read_only_tool", "delete_everything"): + manager.tool_name_to_mcp_server_name_mapping[tool_name] = "gated_server" + user = UserAPIKeyAuth( + api_key="sk-caller", + user_id="alice", + user_role=LitellmUserRoles.INTERNAL_USER.value, + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="op-gated", + mcp_servers=["srv-gated"], + mcp_tool_permissions={"srv-gated": ["read_only_tool"]}, + ), + ) + return manager, user + + @pytest.mark.asyncio + async def test_unentitled_tool_refused_without_proxy_logging_obj(self): + manager, user = self._manager_with_scoped_server() + upstream = AsyncMock(return_value=CallToolResult(content=[], isError=False)) + + with patch.object(manager, "_call_regular_mcp_tool", new=upstream): + with pytest.raises(HTTPException) as exc: + await manager.call_tool( + server_name="gated_server", + name="delete_everything", + arguments={}, + user_api_key_auth=user, + proxy_logging_obj=None, + ) + + assert exc.value.status_code == 403 + upstream.assert_not_awaited() + + @pytest.mark.asyncio + async def test_entitled_tool_still_dispatches_without_proxy_logging_obj(self): + """The gate must refuse only what the entitlement excludes; an allowed + tool still reaches the upstream when there is no logging object.""" + manager, user = self._manager_with_scoped_server() + upstream = AsyncMock(return_value=CallToolResult(content=[], isError=False)) + + with patch.object(manager, "_call_regular_mcp_tool", new=upstream): + await manager.call_tool( + server_name="gated_server", + name="read_only_tool", + arguments={}, + user_api_key_auth=user, + proxy_logging_obj=None, + ) + + upstream.assert_awaited_once() diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py index c4b3c7f5f67..63473bf3cf5 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py @@ -9,7 +9,13 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy._types import ( + LiteLLM_ObjectPermissionTable, + LitellmUserRoles, + UserAPIKeyAuth, +) +from litellm.types.mcp import MCPAuth, MCPTransport +from litellm.types.mcp_server.mcp_server_manager import MCPServer @pytest.mark.asyncio @@ -305,3 +311,241 @@ async def test_openapi_local_tool_injects_resolved_oauth_token(): assert captured["resolved"] == {"Authorization": "Bearer stored-user-token"} assert _request_resolved_auth_headers.get() is None + + + +LEGACY_SERVER_ID = "srv-legacy-petstore" +LEGACY_SERVER_NAME = "legacy_petstore" +LEGACY_TOOL = "dump_secrets" + + +@pytest.fixture +def legacy_local_tool(): + """A bare `mcp_tools`-style handler plus a registered server whose tools were + never listed, which is what leaves `tool_name_to_mcp_server_name_mapping` + cold and routes `{server}-{tool}` into `execute_mcp_tool`'s legacy fallback. + + Yields the server and the list the handler appends to, so a test can tell + "refused" from "dispatched" by whether the handler actually ran. + """ + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.proxy._experimental.mcp_server.tool_registry import ( + global_mcp_tool_registry, + ) + + executed: list[dict] = [] + server = MCPServer( + server_id=LEGACY_SERVER_ID, + name=LEGACY_SERVER_NAME, + server_name=LEGACY_SERVER_NAME, + alias=LEGACY_SERVER_NAME, + url="http://127.0.0.1:1/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.none, + ) + global_mcp_tool_registry.register_tool( + name=LEGACY_TOOL, + description="bare tool registered from the mcp_tools config block", + input_schema={"type": "object", "properties": {}}, + handler=lambda **kwargs: executed.append(kwargs) or "legacy local tool ran", + ) + global_mcp_server_manager.registry[LEGACY_SERVER_ID] = server + assert ( + global_mcp_server_manager._get_mcp_server_from_tool_name( + f"{LEGACY_SERVER_NAME}-{LEGACY_TOOL}" + ) + is None + ), "fixture precondition: the prefixed name must resolve to no server" + try: + yield server, executed + finally: + global_mcp_tool_registry.tools.pop(LEGACY_TOOL, None) + global_mcp_server_manager.registry.pop(LEGACY_SERVER_ID, None) + + +def _caller_entitled_to(tools: list[str]) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="sk-caller", + user_id="alice", + user_role=LitellmUserRoles.INTERNAL_USER.value, + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="op-legacy-fallback", + mcp_servers=[LEGACY_SERVER_ID], + mcp_tool_permissions={LEGACY_SERVER_ID: tools}, + ), + ) + + +@pytest.mark.asyncio +async def test_legacy_local_tool_fallback_refuses_unentitled_caller(legacy_local_tool): + """The legacy fallback dispatched into the local tool registry with no + tool-level authorization at all: no allowed/banned check, no key/team/org + tool permissions, no parameter validation. It must now run the same gate, + so a caller whose entitlement excludes the tool is refused and the handler + never runs. + + Nothing is mocked: the real registries and the real entitlement gate decide. + """ + from fastapi import HTTPException + + from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + MCPRequestHandler, + ) + + server, executed = legacy_local_tool + user = _caller_entitled_to(["list_pets"]) + + # The gate answers "no" for this caller/tool pair, so a dispatch below would + # be an entitlement bypass rather than a routing quirk. + assert ( + await MCPRequestHandler.is_tool_allowed_for_server( + tool_name=LEGACY_TOOL, + server_id=LEGACY_SERVER_ID, + user_api_key_auth=user, + ) + is False + ) + + with pytest.raises(HTTPException) as exc: + await mcp_module.execute_mcp_tool( + name=f"{LEGACY_SERVER_NAME}-{LEGACY_TOOL}", + arguments={}, + allowed_mcp_servers=[server], + start_time=datetime.now(timezone.utc), + user_api_key_auth=user, + ) + + assert exc.value.status_code == 403 + # Pin the refusal to the ENTITLEMENT gate. The server-level check earlier in + # execute_mcp_tool also raises 403 (with a plain-string detail), and the + # allowed/banned-tools check raises a dict naming the server rather than the + # key/team, so asserting on the status alone would pass for the wrong reason. + detail = exc.value.detail + assert isinstance(detail, dict), detail + assert "not allowed for your key/team" in detail["error"], detail + assert executed == [] + + +@pytest.mark.asyncio +async def test_legacy_local_tool_fallback_still_dispatches_entitled_caller( + legacy_local_tool, +): + """The gate must do per-tool work rather than disabling the fallback: the + same shape of call, from a caller entitled to the tool, still dispatches. + + This is the backwards-compatibility half. Refusing this call would trade an + authorization hole for an outage on a configuration that worked before. + """ + from litellm.proxy._experimental.mcp_server import server as mcp_module + + server, executed = legacy_local_tool + user = _caller_entitled_to([LEGACY_TOOL]) + + result = await mcp_module.execute_mcp_tool( + name=f"{LEGACY_SERVER_NAME}-{LEGACY_TOOL}", + arguments={}, + allowed_mcp_servers=[server], + start_time=datetime.now(timezone.utc), + user_api_key_auth=user, + ) + + assert result.isError is False + assert executed == [{}] + assert "legacy local tool ran" in result.content[0].text + + +@pytest.mark.asyncio +async def test_legacy_local_tool_fallback_fails_closed_on_empty_prefix( + legacy_local_tool, +): + """An empty prefix segment skips the server-level check outright: + `split_server_prefix_from_name` yields an empty `server_name`, and `execute_mcp_tool` + only runs `is_tool_allowed` `if server_name`. The legacy fallback then dispatched for a + caller holding no server grant at all, so this arm of the guard is reachable rather than + defensive. Nothing is patched here; the empty prefix segment is the whole of it. + """ + from fastapi import HTTPException + + from litellm.proxy._experimental.mcp_server import server as mcp_module + + _server, executed = legacy_local_tool + + with pytest.raises(HTTPException) as exc: + await mcp_module.execute_mcp_tool( + name=f"-{LEGACY_TOOL}", + arguments={}, + allowed_mcp_servers=[], + start_time=datetime.now(timezone.utc), + user_api_key_auth=_caller_entitled_to([LEGACY_TOOL]), + ) + + assert exc.value.status_code == 503 + assert executed == [] + + +@pytest.mark.asyncio +async def test_legacy_local_tool_fallback_fails_closed_when_prefix_names_no_server( + legacy_local_tool, +): + """Second arm of the same guard: a non-empty prefix that named a server the caller does + hold, but which is absent from `allowed_mcp_servers` by the time dispatch runs. Patching + the server-level check (which would otherwise refuse first) is what makes the arm + observable, so a later refactor cannot make the branch dispatch with no server to + evaluate a tool ceiling against. + """ + from fastapi import HTTPException + + from litellm.proxy._experimental.mcp_server import server as mcp_module + + _server, executed = legacy_local_tool + other_server = MCPServer( + server_id="srv-unrelated", + name="unrelated_server", + server_name="unrelated_server", + alias="unrelated_server", + url="http://127.0.0.1:1/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.none, + ) + + with patch( + "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed", + return_value=True, + ): + with pytest.raises(HTTPException) as exc: + await mcp_module.execute_mcp_tool( + name=f"{LEGACY_SERVER_NAME}-{LEGACY_TOOL}", + arguments={}, + allowed_mcp_servers=[other_server], + start_time=datetime.now(timezone.utc), + user_api_key_auth=_caller_entitled_to([LEGACY_TOOL]), + ) + + assert exc.value.status_code == 503 + assert executed == [] + + +@pytest.mark.asyncio +async def test_unknown_tool_name_still_reports_not_found(): + """The guard must gate dispatch, not existence. An unprefixed name that no registry + knows cannot dispatch anything, so it has to keep reporting 404 rather than collapsing + into the guard's 503; every typo'd tool name takes this branch. + """ + from fastapi import HTTPException + + from litellm.proxy._experimental.mcp_server import server as mcp_module + + with pytest.raises(HTTPException) as exc: + await mcp_module.execute_mcp_tool( + name="tool_no_registry_knows", + arguments={}, + allowed_mcp_servers=[], + start_time=datetime.now(timezone.utc), + user_api_key_auth=_caller_entitled_to([LEGACY_TOOL]), + ) + + assert exc.value.status_code == 404 + assert "not found" in str(exc.value.detail) From 594d0d7a0a5513d31e280811dd2eac9603d2f036 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 17:03:57 -0700 Subject: [PATCH 20/50] feat(proxy)!: gate all mock testing request params behind a single config flag Handling of the client-supplied mock testing params was split across three places with different behavior for each. Three were dropped from every proxy request, two reached the router untouched, and a request that asked for a synthetic failure came back as an ordinary success with nothing to indicate that no failure had been injected Put all six behind one opt-in, general_settings. dangerously_allow_mock_testing_request_params, and reject rather than drop when it is unset, so a fallback drill cannot report a pass for a test that never ran. The rejection names the params it saw and the config key to set, which is also the answer for anyone following the older docs The flag is config-file only. It is deliberately absent from ConfigGeneralSettings, and that absence is what makes /config/update drop it on parse and /config/field/update reject it; the tests pin both so the field cannot be added back for tidiness without the reason surfacing. Enabling it logs a startup warning naming every param it unlocks BREAKING CHANGE: mock_timeout and mock_testing_rate_limit_error now require general_settings.dangerously_allow_mock_testing_request_params to be set in config.yaml. Previously they were accepted unconditionally --- litellm/proxy/proxy_server.py | 33 ++++ litellm/proxy/route_llm_request.py | 61 ++++++-- tests/test_litellm/proxy/test_proxy_server.py | 80 ++++++++++ .../proxy/test_route_llm_request.py | 148 +++++++++++++++--- 4 files changed, 291 insertions(+), 31 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a60ea2da019..06e9cc1fbee 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1041,6 +1041,8 @@ async def proxy_startup_event(app: FastAPI): redis_usage_cache=transaction_buffer_redis_cache, ) + ProxyStartupEvent._warn_if_mock_testing_params_enabled(general_settings=general_settings) + ## SEMANTIC TOOL FILTER ## # Read litellm_settings from config for semantic filter initialization try: @@ -7712,6 +7714,37 @@ class ProxyStartupEvent: proxy_logging_obj.startup_event(llm_router=llm_router, redis_usage_cache=redis_usage_cache) + @staticmethod + def _warn_if_mock_testing_params_enabled(general_settings: dict) -> None: + """Announce, loudly, that any caller may inject synthetic failures.""" + from litellm.proxy.route_llm_request import ( + GATED_MOCK_PARAM_NAMES, + MOCK_TESTING_CONFIG_KEY, + ) + + if general_settings.get(MOCK_TESTING_CONFIG_KEY, False) is not True: + return + + verbose_proxy_logger.warning( + "\n%s\n" + " DANGEROUS SETTING ENABLED\n" + " general_settings.%s = true\n" + "\n" + " Any caller with a valid key on this proxy can now inject synthetic\n" + " failures and latency into their own requests using these body params:\n" + "%s\n" + "\n" + " A request using them consumes a connection and a concurrency slot\n" + " without reaching a provider, and returns an error the caller chose.\n" + "\n" + " Intended for testing fallback chains. Do not leave enabled.\n" + "%s", + "=" * 72, + MOCK_TESTING_CONFIG_KEY, + "\n".join(f" {name}" for name in GATED_MOCK_PARAM_NAMES), + "=" * 72, + ) + @staticmethod def _validate_redis_transaction_buffer_config( general_settings: dict, diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 1f5aacc2115..aebdbca86ad 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -8,18 +8,24 @@ import litellm from litellm.proxy._types import UserAPIKeyAuth from litellm.router_utils.common_utils import _is_proxy_admin_request -# Router-internal mock_testing_* flag names — kept in sync with -# ``litellm.types.router.MockRouterTestingParams`` by the test -# ``test_mock_testing_kwarg_names_matches_dataclass``. Hardcoding (rather -# than deriving via ``dataclasses.fields(MockRouterTestingParams)`` at +# Client-supplied params that make the router or the call path fabricate a +# failure or a delay instead of calling the provider. The ``mock_testing_*`` +# names are kept in sync with ``litellm.types.router.MockRouterTestingParams`` +# by ``test_gated_mock_params_cover_mock_router_testing_params``. Hardcoding +# (rather than deriving via ``dataclasses.fields(MockRouterTestingParams)`` at # import time) avoids a cyclic import: ``litellm.types.router`` imports # back into proxy modules before this module finishes loading. -_MOCK_TESTING_KWARG_NAMES: tuple = ( +GATED_MOCK_PARAM_NAMES: tuple[str, ...] = ( "mock_testing_fallbacks", "mock_testing_context_fallbacks", "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + "mock_timeout", + "mock_delay", ) +MOCK_TESTING_CONFIG_KEY = "dangerously_allow_mock_testing_request_params" + if TYPE_CHECKING: from litellm.router import Router as _Router @@ -169,6 +175,41 @@ def raise_if_required_body_param_missing(route_type: str, data: Mapping[str, obj ) +class MockTestingParamsDisabledError(HTTPException): + def __init__(self, params: tuple[str, ...]): + super().__init__( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ # mutable-ok: HTTPException.detail has no immutable form; same shape as the sibling errors here + "error": ( + f"Mock testing request params are disabled on this proxy: {', '.join(params)}. " + f"An admin can enable them by setting `general_settings.{MOCK_TESTING_CONFIG_KEY}: true` " + "in config.yaml. This setting cannot be changed from the Admin UI or the API." + ) + }, + ) + + +def raise_if_mock_testing_params_disallowed(data: Mapping[str, object], *, allowed: bool) -> None: + """Reject client-supplied mock testing params unless an admin opted in. + + Rejecting (rather than silently dropping) keeps a request that asked for a + synthetic failure from returning a normal success, which reads as a passing + fallback test that never ran. + """ + if allowed: + return + present = tuple(name for name in GATED_MOCK_PARAM_NAMES if name in data) + if present: + raise MockTestingParamsDisabledError(params=present) + + +def mock_testing_params_allowed() -> bool: + """Read the opt-in from the running proxy's ``general_settings``.""" + import litellm.proxy.proxy_server as proxy_server + + return proxy_server.general_settings.get(MOCK_TESTING_CONFIG_KEY, False) is True + + def get_team_id_from_data(data: dict) -> Optional[str]: """ Get the team id from the data's metadata or litellm_metadata params. @@ -381,12 +422,10 @@ async def route_request( await add_shared_session_to_data(data) - # Strip router-internal mock_testing_* flags. Combined with an - # unauthorized fallback in ``router_settings_override`` they let a - # caller deterministically execute requests against restricted - # models. VERIA-44. - for _key in _MOCK_TESTING_KWARG_NAMES: - data.pop(_key, None) + # Gated here rather than alongside the sibling untrusted-param checks in + # ``litellm_pre_call_utils``: the Responses WebSocket route calls + # ``route_request`` directly and never runs ``add_litellm_data_to_request``. + raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed()) data.pop("enable_tag_filtering", None) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index b9a33bd2cef..192c674771f 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -10678,3 +10678,83 @@ async def test_async_data_generator_forwards_usage_chunk_without_strip_marker(): assert len(data_frames) == 4 assert any('"usage"' in frame and '"completion_tokens":188' in frame.replace(" ", "") for frame in data_frames) assert frames[-1] == "data: [DONE]\n\n" + + +@pytest.mark.asyncio +async def test_config_field_update_rejects_mock_testing_flag(): + """The mock-testing opt-in is deliberately absent from + ``ConfigGeneralSettings`` so that ``/config/field/update`` refuses it. If + someone later adds the field for tidiness, this test fails and tells them + they have just opened an API write path into a config-file-only setting.""" + from fastapi import HTTPException + + from litellm.proxy._types import ConfigFieldUpdate + from litellm.proxy.proxy_server import update_config_general_settings + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + admin = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-test", + ) + + with patch.object(proxy_server_module, "prisma_client", MagicMock()): + with pytest.raises(HTTPException) as exc_info: + await update_config_general_settings( + data=ConfigFieldUpdate( + field_name=MOCK_TESTING_CONFIG_KEY, + field_value=True, + config_type="general_settings", + ), + user_api_key_dict=admin, + ) + + assert exc_info.value.status_code == 400 + + +def test_config_update_body_drops_mock_testing_flag(): + """``/config/update`` parses its body as ``ConfigYAML``, whose + ``general_settings`` is a ``ConfigGeneralSettings``. Undeclared keys are + dropped on parse, so the flag never reaches the DB by that route either.""" + from litellm.proxy._types import ConfigYAML + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + parsed = ConfigYAML.model_validate({"general_settings": {MOCK_TESTING_CONFIG_KEY: True}}) + + assert parsed.general_settings is not None + assert MOCK_TESTING_CONFIG_KEY not in parsed.general_settings.model_dump(exclude_none=True) + + +def test_startup_warns_when_mock_testing_params_enabled(caplog): + """Enabling the opt-in must announce itself, naming every param it + unlocks — the config key says ``mock_testing`` but the gate also covers + ``mock_timeout`` and ``mock_delay``, so coverage cannot be inferred from + the name alone.""" + import logging + + from litellm.proxy.proxy_server import ProxyStartupEvent + from litellm.proxy.route_llm_request import ( + GATED_MOCK_PARAM_NAMES, + MOCK_TESTING_CONFIG_KEY, + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + ProxyStartupEvent._warn_if_mock_testing_params_enabled( + general_settings={MOCK_TESTING_CONFIG_KEY: True} + ) + + assert MOCK_TESTING_CONFIG_KEY in caplog.text + for param_name in GATED_MOCK_PARAM_NAMES: + assert param_name in caplog.text + + +def test_startup_is_silent_when_mock_testing_params_disabled(caplog): + """A proxy that never set the opt-in must not emit the warning.""" + import logging + + from litellm.proxy.proxy_server import ProxyStartupEvent + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + ProxyStartupEvent._warn_if_mock_testing_params_enabled(general_settings={}) + + assert MOCK_TESTING_CONFIG_KEY not in caplog.text diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index 93b3ef1cce8..e74693a4f6c 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -8,6 +8,8 @@ sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to from unittest.mock import MagicMock +from fastapi import HTTPException + from litellm.proxy.route_llm_request import ProxyModelNotFoundError, route_request @@ -471,38 +473,147 @@ async def test_route_request_with_router_settings_override_preserves_existing(): assert call_kwargs["timeout"] == 30 -def test_mock_testing_kwarg_names_matches_dataclass(): - """``_MOCK_TESTING_KWARG_NAMES`` is hardcoded to avoid a cyclic import - against ``litellm.types.router``. This test guards against drift — - if a new ``mock_testing_*`` field is added to ``MockRouterTestingParams`` - the strip list must be updated to keep covering it.""" +def test_gated_mock_params_cover_mock_router_testing_params(): + """``GATED_MOCK_PARAM_NAMES`` is hardcoded to avoid a cyclic import + against ``litellm.types.router``. This test guards against drift — if a + new ``mock_testing_*`` field is added to ``MockRouterTestingParams`` the + gate must be updated to keep covering it. The gate is a superset: it also + covers params consumed outside that dataclass.""" from dataclasses import fields - from litellm.proxy.route_llm_request import _MOCK_TESTING_KWARG_NAMES + from litellm.proxy.route_llm_request import GATED_MOCK_PARAM_NAMES from litellm.types.router import MockRouterTestingParams - assert set(_MOCK_TESTING_KWARG_NAMES) == {f.name for f in fields(MockRouterTestingParams)} + assert {f.name for f in fields(MockRouterTestingParams)} <= set(GATED_MOCK_PARAM_NAMES) + assert {"mock_testing_rate_limit_error", "mock_timeout", "mock_delay"} <= set(GATED_MOCK_PARAM_NAMES) -@pytest.mark.asyncio @pytest.mark.parametrize( - "mock_flag", + "mock_param", [ "mock_testing_fallbacks", "mock_testing_context_fallbacks", "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + "mock_timeout", + "mock_delay", ], ) -async def test_route_request_strips_mock_testing_flags(mock_flag): - """VERIA-44: router-internal testing flags must not survive a - user-supplied request body. Without this strip, an attacker can - combine ``mock_testing_fallbacks=true`` with an unauthorized fallback - in ``router_settings_override`` to deterministically execute requests - against restricted models.""" +def test_mock_params_rejected_when_not_allowed(mock_param): + """Every gated param must be rejected by name when the proxy has not + opted in, and the error must point the caller at the config key.""" + from litellm.proxy.route_llm_request import ( + MOCK_TESTING_CONFIG_KEY, + raise_if_mock_testing_params_disallowed, + ) + + data = {"model": "gpt-3.5-turbo", mock_param: True} + + with pytest.raises(HTTPException) as exc_info: + raise_if_mock_testing_params_disallowed(data, allowed=False) + + assert exc_info.value.status_code == 400 + error_message = exc_info.value.detail["error"] + assert mock_param in error_message + assert MOCK_TESTING_CONFIG_KEY in error_message + + +@pytest.mark.parametrize( + "mock_param", + [ + "mock_testing_fallbacks", + "mock_testing_context_fallbacks", + "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + "mock_timeout", + "mock_delay", + ], +) +def test_mock_params_pass_through_when_allowed(mock_param): + """With the opt-in set, gated params must survive untouched — a gate that + rejects correctly but strips anyway would leave the feature unusable.""" + from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed + + data = {"model": "gpt-3.5-turbo", mock_param: True} + + raise_if_mock_testing_params_disallowed(data, allowed=True) + + assert data[mock_param] is True + + +def test_mock_param_gate_reports_every_param_present(): + """A request carrying several gated params must name all of them, so a + caller fixing one is not surprised by the next.""" + from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed + + data = { + "model": "gpt-3.5-turbo", + "mock_testing_fallbacks": True, + "mock_delay": 30, + } + + with pytest.raises(HTTPException) as exc_info: + raise_if_mock_testing_params_disallowed(data, allowed=False) + + error_message = exc_info.value.detail["error"] + assert "mock_testing_fallbacks" in error_message + assert "mock_delay" in error_message + + +def test_ordinary_request_is_not_rejected_by_the_mock_param_gate(): + """The gate must not fire on a request that carries no gated param.""" + from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed + data = { "model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello"}], - mock_flag: True, + "mock_response": "hi", + } + + raise_if_mock_testing_params_disallowed(data, allowed=False) + + +@pytest.mark.asyncio +async def test_route_request_rejects_mock_params_by_default(monkeypatch): + """End-to-end through ``route_request``: with no opt-in configured the + request is rejected before it ever reaches the router.""" + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", {}, raising=False) + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "mock_testing_fallbacks": True, + } + llm_router = MagicMock() + + with pytest.raises(HTTPException) as exc_info: + await route_request(data, llm_router, None, "acompletion") + + assert exc_info.value.status_code == 400 + llm_router.acompletion.assert_not_called() + + +@pytest.mark.asyncio +async def test_route_request_forwards_mock_params_when_opted_in(monkeypatch): + """End-to-end through ``route_request``: with the opt-in set the param + reaches the router, which is what makes a fallback drill possible.""" + import litellm.proxy.proxy_server as proxy_server + + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + monkeypatch.setattr( + proxy_server, + "general_settings", + {MOCK_TESTING_CONFIG_KEY: True}, + raising=False, + ) + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "mock_testing_fallbacks": True, } llm_router = MagicMock() llm_router.acompletion.return_value = "ok" @@ -510,10 +621,7 @@ async def test_route_request_strips_mock_testing_flags(mock_flag): await route_request(data, llm_router, None, "acompletion") call_kwargs = llm_router.acompletion.call_args[1] - assert mock_flag not in call_kwargs - # The flag is also gone from the original data dict so any subsequent - # processing (e.g. logging) doesn't see it either. - assert mock_flag not in data + assert call_kwargs["mock_testing_fallbacks"] is True @pytest.mark.parametrize("route_type", ["agenerate_content", "agenerate_content_stream"]) From cee78d61eadadfb2b0a3c25b723db357d1e40687 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 1 Aug 2026 00:38:58 +0000 Subject: [PATCH 21/50] chore(typing): re-ratchet lint budgets after merging staging --- type-discipline-budget.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type-discipline-budget.json b/type-discipline-budget.json index b79aae63b0b..05499e83c42 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -3,7 +3,7 @@ "limit": 23250 }, "LIT002": { - "limit": 27280 + "limit": 27277 }, "LIT003": { "limit": 292 From 764b2337701f972ecafd172c9d0705ed976189b6 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 17:52:01 -0700 Subject: [PATCH 22/50] refactor(proxy): drop an inaccurate comment on the mock testing gate The comment said the Responses WebSocket route never runs add_litellm_data_to_request. It does, via common_processing_pre_call_logic, so the note recorded a request-flow constraint that does not hold The gate stays in route_request, which is the dispatch chokepoint and where the previous handling lived --- litellm/proxy/route_llm_request.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index aebdbca86ad..12e9857219e 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -422,9 +422,6 @@ async def route_request( await add_shared_session_to_data(data) - # Gated here rather than alongside the sibling untrusted-param checks in - # ``litellm_pre_call_utils``: the Responses WebSocket route calls - # ``route_request`` directly and never runs ``add_litellm_data_to_request``. raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed()) data.pop("enable_tag_filtering", None) From d640ace6d88a08d4e1bef6dc68e9fbce54d6c035 Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Fri, 31 Jul 2026 18:05:50 -0700 Subject: [PATCH 23/50] fix(rate-limit): enforce token limits when the pre-call increment is zero The atomic check-and-increment path skipped any counter whose increment was <= 0. The dynamic rate limiter always passes a zero token increment pre-call because usage lands on the counters post-response, so on a model configured with only tpm the limiter evaluated no counters at all: no model-wide TPM cap and no priority reservation, in either generous or strict mode. Regressed in dd57ae6691 when the pre-call flow moved off the read-only should_rate_limit check, which did evaluate token limits. Keep zero-increment counters in the payload so they act as a pure check (current + 0 > limit), matching the pre-regression semantics in both the Lua and in-memory paths. Adds unit regressions at the primitive and hook level plus a live e2e covering the priority_generous/priority_strict registry rows. --- .../hooks/parallel_request_limiter_v3.py | 2 +- tests/e2e/models.py | 3 + .../test_dynamic_rate_limit_priority_e2e.py | 242 ++++++++++++++++++ .../hooks/test_dynamic_rate_limiter_v3.py | 92 +++++++ .../hooks/test_parallel_request_limiter_v3.py | 57 +++++ 5 files changed, 395 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/quota_management/ratelimit/test_dynamic_rate_limit_priority_e2e.py diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index b04ef5f7087..486e3cf88a4 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -1341,7 +1341,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): else: limit_value = rate_limit.get("tokens_per_unit") inc_amount = int(increment_amounts.get("tokens", 0) or 0) - if limit_value is None or inc_amount <= 0: + if limit_value is None: continue counter_key = self.create_rate_limit_keys(descriptor_key, descriptor_value, rlt) # Counter-key TTL and window_size are conceptually distinct diff --git a/tests/e2e/models.py b/tests/e2e/models.py index af695acaa5e..f1c0ede0e85 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -43,6 +43,7 @@ class KeyLoggingCallback(BaseModel): class KeyMetadata(BaseModel): logging: list[KeyLoggingCallback] | None = None + priority: str | None = None class ObjectPermission(BaseModel): @@ -97,6 +98,7 @@ class LiteLLMBudgetTable(BaseModel): class KeyInfo(BaseModel): key_alias: str | None = None + metadata: KeyMetadata | None = None models: list[str] = [] tpm_limit: int | None = None rpm_limit: int | None = None @@ -694,6 +696,7 @@ class LiteLLMParamsBody(BaseModel): complexity_router_config: dict[str, object] | None = None mock_response: str | None = None timeout: float | None = None + tpm: int | None = None ModelMode = Literal["batch", "realtime", "image_generation"] diff --git a/tests/e2e/quota_management/ratelimit/test_dynamic_rate_limit_priority_e2e.py b/tests/e2e/quota_management/ratelimit/test_dynamic_rate_limit_priority_e2e.py new file mode 100644 index 00000000000..06b4aa1a0b5 --- /dev/null +++ b/tests/e2e/quota_management/ratelimit/test_dynamic_rate_limit_priority_e2e.py @@ -0,0 +1,242 @@ +"""Live e2e: the v3 dynamic rate limiter's saturation-aware priority reservation. + +Covers quota_management.ratelimit.priority_generous / priority_strict: with +`dynamic_rate_limiter_v3` enabled, a model's TPM capacity is split into priority +reservations, but a reservation is only enforced once the model is saturated. + +- Generous mode (recorded usage below the saturation threshold): a key whose + priority reserves 25% of capacity keeps serving past its reservation, + borrowing the idle capacity (priority_generous.picks_under_tpm) +- Strict mode (recorded usage at/over the threshold): the over-reservation key + is blocked with the priority-flavored 429 while a key of a different priority, + still inside its own reservation, is served (priority_strict.picks_under_tpm) + +The proxy under test must run with this config (and LITELLM_LICENSE set, since +priority reservation is a premium feature): + + litellm_settings: + callbacks: ["dynamic_rate_limiter_v3"] + priority_reservation: + prod: 0.5 + dev: 0.25 + priority_reservation_settings: + saturation_threshold: 0.5 + saturation_check_cache_ttl: 1 + +The constants below mirror those values; if the proxy runs different ones the +tests fail with a message naming the required config rather than skipping. + +The limiter counts a request against the model-wide window pre-call, but tokens +only land on the counters after each response completes (there is no pre-call +token reservation at the model level), so recorded saturation always trails the +traffic that produced it. The tests therefore drive spend by summing each +body's usage.total_tokens (the counter can never be ahead of that sum) and poll +for the strict-mode block instead of expecting it on an exact call. Each test +creates its own /model/new deployment so its 60s rate-limit window and counters +are isolated from concurrent runs. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass + +import pytest +from pydantic import BaseModel, ConfigDict, ValidationError + +from e2e_config import unique_marker +from e2e_http import StreamingResponse, require_successful_call +from lifecycle import ResourceManager +from models import KeyGenerateBody, KeyMetadata, LiteLLMParamsBody +from quota_client import QuotaClient + +pytestmark = pytest.mark.e2e + +BACKEND = "anthropic/claude-haiku-4-5-20251001" +MODEL_TPM = 400 +DEV_PRIORITY = "dev" +PROD_PRIORITY = "prod" +DEV_RESERVED_TOKENS = int(MODEL_TPM * 0.25) +SATURATION_TOKENS = int(MODEL_TPM * 0.5) +CHAT_MAX_TOKENS = 16 +WINDOW_SECONDS = 60 +WINDOW_MARGIN_SECONDS = 10 +STRICT_POLL_SPEND_CEILING = int(MODEL_TPM * 0.7) + +REQUIRED_CONFIG_HINT = ( + "the proxy must run litellm_settings.callbacks=['dynamic_rate_limiter_v3'] with " + "priority_reservation {prod: 0.5, dev: 0.25} and priority_reservation_settings " + "{saturation_threshold: 0.5, saturation_check_cache_ttl: 1}; see this module's docstring" +) + + +class _ChatUsage(BaseModel): + model_config = ConfigDict(extra="ignore") + + total_tokens: int + + +class _ChatBodyWithUsage(BaseModel): + model_config = ConfigDict(extra="ignore") + + usage: _ChatUsage + + +def _total_tokens(outcome: StreamingResponse) -> int: + try: + return _ChatBodyWithUsage.model_validate_json(outcome.body).usage.total_tokens + except ValidationError: + pytest.fail(f"successful chat body must report usage.total_tokens, got: {outcome.body[:300]}") + + +@dataclass(frozen=True, slots=True) +class _Fixture: + model: str + dev_key: str + prod_key: str + + +def _dynamic_limited_model(client: QuotaClient, resources: ResourceManager, label: str) -> _Fixture: + model = f"e2e-dynpri-{label}-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody(model=BACKEND, api_key="os.environ/ANTHROPIC_API_KEY", tpm=MODEL_TPM), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + def _priority_key(priority: str) -> str: + key = client.proxy.generate_key( + KeyGenerateBody( + models=[model], + metadata=KeyMetadata(priority=priority), + key_alias=f"e2e-dynpri-{label}-{priority}-{unique_marker()}", + ) + ) + resources.defer(lambda: client.proxy.delete_key(key)) + return key + + return _Fixture(model=model, dev_key=_priority_key(DEV_PRIORITY), prod_key=_priority_key(PROD_PRIORITY)) + + +def _chat(client: QuotaClient, key: str, model: str) -> StreamingResponse: + return client.chat(key, model, f"reply with one word {unique_marker()}", max_tokens=CHAT_MAX_TOKENS) + + +@dataclass(frozen=True, slots=True) +class _FirstOk: + sent_at: float + response: StreamingResponse + + +def _first_ok(client: QuotaClient, key: str, model: str) -> _FirstOk: + """First successful call on a fresh key opens the model's rate-limit window; + `sent_at` (captured before the winning send) is a lower bound on the window + start. A fresh key may briefly 401 until the auth cache picks it up, so + retry 401s to a deadline; a 401 never reaches the limiter, so only the + successful call consumes budget.""" + deadline = time.monotonic() + client.proxy.poll_timeout + while True: + sent_at = time.monotonic() + outcome = _chat(client, key, model) + if outcome.ok: + return _FirstOk(sent_at=sent_at, response=outcome) + if outcome.status_code != 401 or time.monotonic() >= deadline: + require_successful_call(outcome) + time.sleep(client.proxy.poll_interval) + + +def _window_guard(first: _FirstOk, spent: int) -> None: + assert time.monotonic() < first.sent_at + WINDOW_SECONDS - WINDOW_MARGIN_SECONDS, ( + f"only {spent} tokens of spend landed before the {WINDOW_SECONDS}s rate-limit window could " + "roll; this test needs every call inside one window" + ) + + +class TestDynamicRateLimitPriority: + @pytest.mark.covers( + "quota_management.ratelimit.priority_generous.picks_under_tpm", + exercised_on=["chat_completions"], + ) + def test_generous_mode_lets_priority_borrow_past_reservation( + self, client: QuotaClient, resources: ResourceManager + ) -> None: + fixture = _dynamic_limited_model(client, resources, "generous") + + info = client.proxy.key_info(fixture.dev_key) + assert info.metadata is not None and info.metadata.priority == DEV_PRIORITY, ( + f"/key/info must echo the key's priority metadata, got {info.metadata}" + ) + + first = _first_ok(client, fixture.dev_key, fixture.model) + spent = _total_tokens(first.response) + while spent <= DEV_RESERVED_TOKENS: + _window_guard(first, spent) + assert spent < SATURATION_TOKENS, ( + f"spend reached the saturation threshold ({spent} of {SATURATION_TOKENS}) before " + f"crossing the dev reservation ({DEV_RESERVED_TOKENS}); shrink per-call spend to " + "keep the borrowing claim observable" + ) + outcome = _chat(client, fixture.dev_key, fixture.model) + assert outcome.status_code != 429, ( + f"dev key was blocked at {spent} recorded tokens, under the saturation threshold " + f"({SATURATION_TOKENS} of {MODEL_TPM}); generous mode must let it borrow past its " + f"{DEV_RESERVED_TOKENS}-token reservation. If the limiter is missing entirely, " + f"{REQUIRED_CONFIG_HINT}. 429 body: {outcome.body[:300]}" + ) + require_successful_call(outcome) + spent += _total_tokens(outcome) + + assert spent > DEV_RESERVED_TOKENS + + @pytest.mark.covers( + "quota_management.ratelimit.priority_strict.picks_under_tpm", + exercised_on=["chat_completions"], + ) + def test_strict_mode_blocks_saturated_priority_but_serves_the_other( + self, client: QuotaClient, resources: ResourceManager + ) -> None: + fixture = _dynamic_limited_model(client, resources, "strict") + + prod_warmup = _first_ok(client, fixture.prod_key, fixture.model) + first = _first_ok(client, fixture.dev_key, fixture.model) + prod_spent = _total_tokens(prod_warmup.response) + dev_spent = _total_tokens(first.response) + + while prod_spent + dev_spent < SATURATION_TOKENS: + _window_guard(prod_warmup, prod_spent + dev_spent) + outcome = _chat(client, fixture.dev_key, fixture.model) + assert outcome.status_code != 429, ( + f"dev key was blocked at {prod_spent + dev_spent} recorded tokens, before the " + f"saturation threshold ({SATURATION_TOKENS} of {MODEL_TPM}); strict enforcement " + f"must not engage early. 429 body: {outcome.body[:300]}" + ) + require_successful_call(outcome) + dev_spent += _total_tokens(outcome) + + while True: + _window_guard(prod_warmup, prod_spent + dev_spent) + assert prod_spent + dev_spent < STRICT_POLL_SPEND_CEILING, ( + f"dev key was still served at {prod_spent + dev_spent} tokens, past the saturation " + f"threshold ({SATURATION_TOKENS}) and {DEV_RESERVED_TOKENS}-token dev reservation; " + f"strict priority enforcement never engaged. Check that {REQUIRED_CONFIG_HINT}" + ) + outcome = _chat(client, fixture.dev_key, fixture.model) + if outcome.status_code == 429: + assert "Priority-based rate limit exceeded" in outcome.body, ( + f"the saturated dev key must get the priority-flavored 429, got: {outcome.body[:300]}" + ) + assert outcome.headers.get("x-litellm-priority") == DEV_PRIORITY, ( + f"the 429 must attribute the blocked priority, headers: " + f"{ {k: v for k, v in outcome.headers.items() if 'litellm' in k} }" + ) + break + require_successful_call(outcome) + dev_spent += _total_tokens(outcome) + + prod_outcome = _chat(client, fixture.prod_key, fixture.model) + require_successful_call(prod_outcome) + prod_spent += _total_tokens(prod_outcome) + assert prod_spent < int(MODEL_TPM * 0.5), ( + f"the prod fairness claim needs prod spend ({prod_spent}) inside its reservation " + f"({int(MODEL_TPM * 0.5)}); shrink per-call spend" + ) diff --git a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py index c8176ca6337..6c717d6f71c 100644 --- a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -1771,3 +1771,95 @@ async def test_priority_429_includes_model_name_and_configured_limits(): assert "Priority: prod" in error_msg, error_msg assert "Rate limit type: tokens" in error_msg, error_msg assert "Model saturation:" in error_msg, error_msg + + +@pytest.mark.asyncio +async def test_tpm_only_model_enforces_priority_and_model_capacity(): + """Regression: a model configured with ONLY tpm (no rpm) must still be + rate limited. + + The atomic check-and-increment path used to drop any counter whose + pre-call increment was zero. Token increments are always zero pre-call + (usage lands on the counters post-call), so on a TPM-only model the + limiter evaluated no counters at all: no model-wide cap, no priority + reservation, in either mode. This test drives the real pre-call -> + log-success -> pre-call flow with no limiter internals mocked. + """ + from fastapi import HTTPException + + from litellm.types.utils import ModelResponse, Usage + + os.environ["LITELLM_LICENSE"] = "test-license-key" + litellm.priority_reservation = {"dev": 0.25, "prod": 0.5} + + dual_cache = DualCache() + handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache) + + model = "tpm-only-model" + llm_router = Router( + model_list=[ + { + "model_name": model, + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "test-key", + "api_base": "test-base", + "tpm": 400, + }, + } + ] + ) + handler.update_variables(llm_router=llm_router) + + dev_user = UserAPIKeyAuth() + dev_user.metadata = {"priority": "dev"} + prod_user = UserAPIKeyAuth() + prod_user.metadata = {"priority": "prod"} + + async def record_usage(priority: str, total_tokens: int) -> None: + await handler.async_log_success_event( + kwargs={ + "standard_logging_object": { + "metadata": {"user_api_key_auth_metadata": {"priority": priority}}, + }, + "litellm_params": {"metadata": {"model_group": model}}, + }, + response_obj=ModelResponse( + model=model, + usage=Usage(prompt_tokens=0, completion_tokens=total_tokens, total_tokens=total_tokens), + ), + start_time=None, + end_time=None, + ) + + assert ( + await handler.async_pre_call_hook( + user_api_key_dict=dev_user, cache=dual_cache, data={"model": model}, call_type="completion" + ) + is None + ) + + await record_usage("dev", 250) + + with pytest.raises(HTTPException) as dev_blocked: + await handler.async_pre_call_hook( + user_api_key_dict=dev_user, cache=dual_cache, data={"model": model}, call_type="completion" + ) + assert dev_blocked.value.status_code == 429 + assert "Priority-based rate limit exceeded" in dev_blocked.value.detail["error"] + + assert ( + await handler.async_pre_call_hook( + user_api_key_dict=prod_user, cache=dual_cache, data={"model": model}, call_type="completion" + ) + is None + ) + + await record_usage("prod", 200) + + with pytest.raises(HTTPException) as capacity_blocked: + await handler.async_pre_call_hook( + user_api_key_dict=prod_user, cache=dual_cache, data={"model": model}, call_type="completion" + ) + assert capacity_blocked.value.status_code == 429 + assert "Model capacity reached" in capacity_blocked.value.detail["error"] diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 56bfd1829b5..84e41711176 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -4998,3 +4998,60 @@ async def test_split_usage_still_respects_the_configured_limit_type(monkeypatch) token_operations = [op for op in captured_operations if op["key"].endswith(":tokens")] assert token_operations assert all(op["increment_value"] == 7 for op in token_operations) + + +@pytest.mark.asyncio +async def test_atomic_check_with_zero_increment_still_enforces_token_limit(): + """Regression: a zero token increment must still CHECK the token limit. + + The dynamic rate limiter calls atomic_check_and_increment_by_n with + {"requests": 1, "tokens": 0} because tokens land on the counter post-call. + The payload builder used to skip any counter whose increment was <= 0, so a + TPM-only descriptor produced zero counters to evaluate and the call + returned OK with empty statuses; TPM limits were never enforced at all. + """ + from litellm.proxy.hooks.parallel_request_limiter_v3 import RateLimitDescriptor + + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache()) + ) + descriptor = RateLimitDescriptor( + key="model_saturation_check", + value="tpm-only-model", + rate_limit={"tokens_per_unit": 100, "window_size": 60}, + ) + zero_token_increment: Dict[str, int] = {"requests": 1, "tokens": 0} + + under_limit = await handler.atomic_check_and_increment_by_n( + descriptors=[descriptor], + increments=[zero_token_increment], + ) + assert under_limit["overall_code"] == "OK" + assert [s["rate_limit_type"] for s in under_limit["statuses"]] == ["tokens"] + + counter_key = handler.create_rate_limit_keys( + "model_saturation_check", "tpm-only-model", "tokens" + ) + await handler.async_increment_tokens_with_ttl_preservation( + pipeline_operations=[ + RedisPipelineIncrementOperation( + key=counter_key, increment_value=150, ttl=60 + ) + ], + ) + + over_limit = await handler.atomic_check_and_increment_by_n( + descriptors=[descriptor], + increments=[zero_token_increment], + ) + assert over_limit["overall_code"] == "OVER_LIMIT" + blocked = over_limit["statuses"][0] + assert blocked["rate_limit_type"] == "tokens" + assert blocked["current_limit"] == 100 + + assert ( + await handler.internal_usage_cache.async_get_cache( + key=counter_key, litellm_parent_otel_span=None, local_only=True + ) + == 150 + ) From 836bd927b68a7fbabfb1afcdcbdc6b31dd13b8fc Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Fri, 31 Jul 2026 18:05:50 -0700 Subject: [PATCH 24/50] fix(rate-limit): skip negative increments in the atomic payload builder Review feedback: the relaxed predicate admitted negative increments, which both atomic backends would apply as decrements. Restrict the new behavior to zero-valued pure checks and assert negatives neither check nor mutate counters. --- litellm/proxy/hooks/parallel_request_limiter_v3.py | 2 +- .../hooks/test_parallel_request_limiter_v3.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 486e3cf88a4..89559ff72ae 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -1341,7 +1341,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): else: limit_value = rate_limit.get("tokens_per_unit") inc_amount = int(increment_amounts.get("tokens", 0) or 0) - if limit_value is None: + if limit_value is None or inc_amount < 0: continue counter_key = self.create_rate_limit_keys(descriptor_key, descriptor_value, rlt) # Counter-key TTL and window_size are conceptually distinct diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 84e41711176..7b3f00a55a8 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -5055,3 +5055,17 @@ async def test_atomic_check_with_zero_increment_still_enforces_token_limit(): ) == 150 ) + + negative_increment: Dict[str, int] = {"requests": -1, "tokens": -50} + refund_attempt = await handler.atomic_check_and_increment_by_n( + descriptors=[descriptor], + increments=[negative_increment], + ) + assert refund_attempt["overall_code"] == "OK" + assert refund_attempt["statuses"] == [] + assert ( + await handler.internal_usage_cache.async_get_cache( + key=counter_key, litellm_parent_otel_span=None, local_only=True + ) + == 150 + ) From b4ff05be8edb669c0cb033776c74cfc749e9407c Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 31 Jul 2026 18:10:48 -0700 Subject: [PATCH 25/50] fix(proxy): stop model writes 500ing on another pod's delete (#35400) * fix(proxy): stop model writes 500ing on another pod's delete A model write judges the reload it triggers by diffing this pod's router before and after, and reports anything that stopped serving as damage. On a pod that has not yet polled a delete another pod made, the snapshot still lists that model; the reload then evicts it because the db no longer has it, and the guard reads its own correct reconcile as degradation. The row is written and served, but the caller gets a 500. Since propagation between pods is a 30s db poll, any delete followed by a create inside that window can land on a pod that has not caught up, so a delete-then-create pair returns 500 whenever the two requests hit different pods. _delete_deployment already computes exactly the set that settles it: the ids the db and config still want. Thread it up through _update_llm_router, add_deployment and clear_cache to the verdict, and intersect the drop set with it so an id the db no longer has stops counting as collateral. Where no reconcile ran the set is None and every drop is still reported, so a genuinely broken reload is caught as before. _delete_deployment now returns that set instead of a delete count; the count had no callers in the proxy, and the tests asserting it already assert the eviction calls. * test(proxy): fold reload-verdict test commentary into docstrings and assertions Greptile flagged the inline comments against the repo's no-new-comments rule. The case-by-case context moves into the test docstring, and the two return-contract assertions carry their reasoning as failure messages instead. * test: fix clear_cache mock return type in model block/unblock tests --- .../model_management_endpoints.py | 48 ++++++++++--- litellm/proxy/proxy_server.py | 39 +++++++---- .../test_model_management_endpoints.py | 70 ++++++++++++++++++- .../proxy/proxy_server/test_proxy_config.py | 6 +- tests/test_litellm/proxy/test_proxy_server.py | 19 +++-- .../test_update_llm_router_resilience.py | 15 ++-- .../test_litellm/test_model_block_unblock.py | 2 +- 7 files changed, 158 insertions(+), 41 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index b6422d7f5ae..41904e3883b 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -324,7 +324,7 @@ async def patch_model( # Clear cache and reload models (uses config setting or defaults to preserving config models for DB updates) live_before_reload = live_model_ids_snapshot() - await clear_cache() + still_desired_ids = await clear_cache() ## CREATE AUDIT LOG ## asyncio.create_task( @@ -344,6 +344,7 @@ async def patch_model( before=live_before_reload, written_models=[(model_id, getattr(updated_model, "model_info", None))], action="update", + still_desired=still_desired_ids, ) return updated_model @@ -429,7 +430,7 @@ async def _set_model_blocked_status( ) live_before_reload = live_model_ids_snapshot() - await clear_cache() + still_desired_ids = await clear_cache() asyncio.create_task( create_object_audit_log( @@ -450,6 +451,7 @@ async def _set_model_blocked_status( before=live_before_reload, written_models=[(data.model_id, getattr(updated_model, "model_info", None))], action=action, + still_desired=still_desired_ids, ) return updated_model @@ -1355,6 +1357,7 @@ async def add_new_model( """ live_before_reload = live_model_ids_snapshot() + still_desired_ids: frozenset[str] | None = None try: _original_litellm_model_name = model_params.model_name if model_params.model_info.team_id is None: @@ -1369,7 +1372,9 @@ async def add_new_model( user_api_key_dict=user_api_key_dict, prisma_client=prisma_client, ) - await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) + still_desired_ids = await proxy_config.add_deployment( + prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj + ) # don't let failed slack alert block the /model/new response _alerting = general_settings.get("alerting", []) or [] if "slack" in _alerting: @@ -1414,6 +1419,7 @@ async def add_new_model( before=live_before_reload, written_models=[(model_response.model_id, getattr(model_response, "model_info", None))], action="create", + still_desired=still_desired_ids, ) return model_response @@ -1542,7 +1548,7 @@ async def update_model( # Clear cache and reload models (uses config setting or defaults to preserving config models for DB updates) live_before_reload = live_model_ids_snapshot() - await clear_cache() + still_desired_ids = await clear_cache() ## CREATE AUDIT LOG ## asyncio.create_task( create_object_audit_log( @@ -1569,6 +1575,7 @@ async def update_model( before=live_before_reload, written_models=[(_model_id, getattr(model_response, "model_info", None))], action="update", + still_desired=still_desired_ids, ) return model_response @@ -1814,6 +1821,7 @@ def reload_serving_verdict( before: frozenset[str], written_models: Sequence[tuple[str, object]], written_must_serve: bool, + still_desired: frozenset[str] | None = None, ) -> tuple[tuple[str, ...], tuple[str, ...]]: """Judge a write-triggered reload by diffing the router's serving state instead of trusting any layer of the reload stack to report its own failure. @@ -1828,10 +1836,14 @@ def reload_serving_verdict( this write and blaming it would block unrelated metadata fixes - not written but live before and gone now: collateral degradation of this pod caused by the reload this request triggered (a wholesale re-add failure, or a - newly introduced conflict), always reported + newly introduced conflict), reported only when the db still wants that id + + ``still_desired`` is the db + config id set the reload just reconciled against. An + id absent from it was deleted on purpose, most often by another pod this one had not + yet polled, so the reload dropping it is the reconcile working rather than damage. + Without it (no reconcile ran) every drop is reported, which is the safe direction. Returns (written ids violating their obligation, collateral ids no longer served). - Best effort under concurrent admin writes: the snapshot spans only this request. """ now = live_model_ids_snapshot() written_ids = frozenset(model_id for model_id, _ in written_models) @@ -1843,7 +1855,8 @@ def reload_serving_verdict( ) else: missing = tuple(model_id for model_id, _ in written_models if model_id in before and model_id not in now) - collateral = tuple(sorted(before - now - written_ids)) + dropped = before - now - written_ids + collateral = tuple(sorted(dropped if still_desired is None else dropped & still_desired)) return (missing, collateral) @@ -1851,12 +1864,18 @@ def raise_if_reload_degraded_serving( before: frozenset[str], written_models: Sequence[tuple[str, object]], action: str, + still_desired: frozenset[str] | None = None, ) -> None: """The caller-visible error this pod's model-write endpoints owe their caller when the model they wrote is not being served after the reload they triggered. The DB write is durable either way and every other pod reloads on its own interval; this speaks only for the handling pod.""" - missing, collateral = reload_serving_verdict(before=before, written_models=written_models, written_must_serve=True) + missing, collateral = reload_serving_verdict( + before=before, + written_models=written_models, + written_must_serve=True, + still_desired=still_desired, + ) if not missing and not collateral: return missing_clause = ( @@ -1882,9 +1901,12 @@ def raise_if_reload_degraded_serving( ) -async def clear_cache(): +async def clear_cache() -> frozenset[str] | None: """ Clear router caches and reload models. + + Returns the db + config id set the reload reconciled against, or None when no + reload ran, so callers can pass it to raise_if_reload_degraded_serving. """ from litellm.proxy.proxy_server import ( llm_router, @@ -1896,7 +1918,7 @@ async def clear_cache(): if llm_router is None or prisma_client is None: verbose_proxy_logger.debug("llm_router or prisma_client is None, skipping cache clear") - return + return None try: # Only clear DB models, preserve config models @@ -1943,10 +1965,14 @@ async def clear_cache(): llm_router.quality_routers.pop(model_name, None) # Reload only DB models - await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) + still_desired_ids = await proxy_config.add_deployment( + prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj + ) verbose_proxy_logger.debug( f"Cleared {len(db_model_ids)} DB models, preserved {len(config_models)} config models" ) + return still_desired_ids except Exception as e: verbose_proxy_logger.exception(f"Failed to clear cache and reload models. Due to error - {str(e)}") + return None diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a60ea2da019..15d977b75f2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5300,7 +5300,7 @@ class ProxyConfig: _model_info = RouterModelInfo(id=model.model_id, db_model=db_model) return _model_info - async def _delete_deployment(self, db_models: list) -> int: + async def _delete_deployment(self, db_models: list) -> frozenset[str] | None: """ (Helper function of add deployment) -> combined to reduce prisma db calls @@ -5309,14 +5309,16 @@ class ProxyConfig: - Remove any that are missing Return: - - int - returns number of deleted deployments + - frozenset[str] - the ids the db + config say should be served after this + reconcile, so a caller can tell an id this evicted on purpose from one that + went missing. None when no reconcile ran and that set is therefore unknown. """ global user_config_file_path, llm_router combined_id_list = [] ## BASE CASES ## if llm_router is None: - return 0 + return None # NOTE: db_models may be legitimately empty when all DB models have been deleted. # Do NOT short-circuit on len(db_models) == 0 — we must still evict any # DB-sourced deployments that are no longer in the DB. The caller @@ -5337,7 +5339,7 @@ class ProxyConfig: "Skipping deployment cleanup to avoid removing valid models.", str(e), ) - return 0 + return None model_list = config.get("model_list", None) if model_list: for model in model_list: @@ -5361,13 +5363,10 @@ class ProxyConfig: router_model_ids = llm_router.get_model_ids() # Check for model IDs in llm_router not present in combined_id_list and delete them - deleted_deployments = 0 for model_id in router_model_ids: if model_id not in combined_id_list: - is_deleted = llm_router.delete_deployment(id=model_id) - if is_deleted is not None: - deleted_deployments += 1 - return deleted_deployments + llm_router.delete_deployment(id=model_id) + return frozenset(combined_id_list) def _resolve_db_litellm_param(self, key: str, value: object) -> object: if not isinstance(value, str): @@ -5452,9 +5451,11 @@ class ProxyConfig: self, new_models: Optional[Json], proxy_logging_obj: ProxyLogging, - ): + ) -> frozenset[str] | None: global llm_router, llm_model_list, master_key, general_settings + still_desired_ids: frozenset[str] | None = None + # Load config separately so a timeout here doesn't block model loading config_data: dict = {} search_tools = None @@ -5501,7 +5502,7 @@ class ProxyConfig: if search_tools is not None and llm_router is not None: llm_router.search_tools = search_tools ## DELETE MODEL LOGIC - await self._delete_deployment(db_models=models_list) + still_desired_ids = await self._delete_deployment(db_models=models_list) ## ADD MODEL LOGIC self._add_deployment(db_models=models_list) @@ -5527,6 +5528,8 @@ class ProxyConfig: proxy_logging_obj=proxy_logging_obj, ) + return still_desired_ids + def _add_callback_from_db_to_in_memory_litellm_callbacks( self, callback: str, @@ -6159,14 +6162,20 @@ class ProxyConfig: self, prisma_client: PrismaClient, proxy_logging_obj: ProxyLogging, - ): + ) -> frozenset[str] | None: """ - Check db for new models - Check if model id's in router already - If not, add to router + + Returns the ids the db + config say should be served after the reconcile, or + None when no reconcile ran. Callers that judge their own reload need it to tell + a deliberate eviction from a deployment that went missing. """ global llm_router, llm_model_list, master_key, general_settings + still_desired_ids: frozenset[str] | None = None + try: # warm the config cache so the per-param reads below all hit await prefetch_config_params( @@ -6186,7 +6195,9 @@ class ProxyConfig: new_models = await self._get_models_from_db(prisma_client=prisma_client) # update llm router - await self._update_llm_router(new_models=new_models, proxy_logging_obj=proxy_logging_obj) + still_desired_ids = await self._update_llm_router( + new_models=new_models, proxy_logging_obj=proxy_logging_obj + ) db_general_settings = await get_config_param(prisma_client, "general_settings") @@ -6204,6 +6215,8 @@ class ProxyConfig: "litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - {}".format(str(e)) ) + return still_desired_ids + async def _init_non_llm_objects_in_db(self, prisma_client: PrismaClient): """ Use this to read non-llm objects from the db and initialize them diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 1e8add52f74..8dbf38555b3 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -839,7 +839,7 @@ class TestUpdateModel: ), patch( "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache", - new=AsyncMock(return_value=True), + new=AsyncMock(return_value=None), ) as mock_clear_cache, ): await update_model( @@ -3223,7 +3223,7 @@ class TestPatchModelBlockedAuthGate: ), patch( "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache", - new=AsyncMock(return_value=True), + new=AsyncMock(return_value=None), ), ): result = await patch_model( @@ -3314,6 +3314,72 @@ class TestWriteSurfacesReloadDrop: before=frozenset({"m-live", "m-collateral"}), written_models=[("m-live", None)], action="update" ) + def test_a_model_the_db_no_longer_has_is_not_collateral(self, monkeypatch): + """Another pod deleting a model is not this pod's reload breaking. + + A pod that has not yet polled the delete still lists the id when the write + snapshots `before`; the reload it triggers then evicts the id because the db no + longer has it. That eviction is the reconcile working, so it must not fail the + write. `still_desired` is the db + config set the reload reconciled against, so + an id missing from it drops out of the collateral diff. + + The cases below, in order: an id the db no longer wants is not collateral and the + write succeeds; an id the db still wants that stopped serving is real degradation + and still raises, so a genuinely broken reload is caught; and with no reconcile at + all the desired set is unknown, so every drop is reported. + """ + import litellm + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + raise_if_reload_degraded_serving, + reload_serving_verdict, + ) + + live_router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4o", + "litellm_params": {"model": "gpt-4o"}, + "model_info": {"id": "m-live", "db_model": True}, + } + ] + ) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", live_router) + + _, collateral = reload_serving_verdict( + before=frozenset({"m-live", "m-deleted-elsewhere"}), + written_models=[("m-live", None)], + written_must_serve=True, + still_desired=frozenset({"m-live"}), + ) + assert collateral == () + + assert ( + raise_if_reload_degraded_serving( + before=frozenset({"m-live", "m-deleted-elsewhere"}), + written_models=[("m-live", None)], + action="create", + still_desired=frozenset({"m-live"}), + ) + is None + ) + + with pytest.raises(ProxyException, match="m-should-be-serving"): + raise_if_reload_degraded_serving( + before=frozenset({"m-live", "m-should-be-serving"}), + written_models=[("m-live", None)], + action="create", + still_desired=frozenset({"m-live", "m-should-be-serving"}), + ) + + with pytest.raises(ProxyException, match="m-deleted-elsewhere"): + raise_if_reload_degraded_serving( + before=frozenset({"m-live", "m-deleted-elsewhere"}), + written_models=[("m-live", None)], + action="create", + still_desired=None, + ) + class TestModelInfoAsMapping: """The model_info column reaches consumers as a dict or as its JSON string; this is diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 8d1d8185e4d..99287b92b8f 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -1404,12 +1404,12 @@ def test_ProxyConfig_get_model_info_with_id_missing_model_id_raises(monkeypatch) @pytest.mark.asyncio -async def test_ProxyConfig__delete_deployment_empty_returns_zero(monkeypatch): +async def test_ProxyConfig__delete_deployment_no_router_returns_none(monkeypatch): monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None) pc = ProxyConfig() result = await pc._delete_deployment(db_models=[]) - snapshot = {"deleted": result, "router_was": "none", "empty_db_models": True} - assert snapshot == {"deleted": 0, "router_was": "none", "empty_db_models": True} + snapshot = {"still_desired": result, "router_was": "none", "empty_db_models": True} + assert snapshot == {"still_desired": None, "router_was": "none", "empty_db_models": True} @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index b9a33bd2cef..1ecf7e8b83a 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -2459,13 +2459,11 @@ async def test_delete_deployment_type_mismatch(): patch("litellm.proxy.proxy_server.user_config_file_path", "test_config.yaml"), ): # Call the function under test - deleted_count = await pc._delete_deployment(db_models=[]) + still_desired = await pc._delete_deployment(db_models=[]) # The two SHA-hash models have no corresponding entry in combined_id_list # and must be evicted. - assert ( - deleted_count == 2 - ), f"Expected 2 deletions (SHA-hash models), got {deleted_count}" + assert len(deleted_ids) == 2, f"Expected 2 deletions (SHA-hash models), got {deleted_ids}" assert ( "a96e12e76b36a57cfae57a41288eb41567629cac89b4828c6f7074afc3534695" in deleted_ids @@ -2485,6 +2483,12 @@ async def test_delete_deployment_type_mismatch(): "12345679" not in deleted_ids ), f"Model 12345679 should NOT be deleted. Deleted IDs: {deleted_ids}" + assert still_desired is not None + assert {"12345678", "12345679"} <= still_desired, ( + "the int-keyed config models must come back as strings in the desired set, so a " + f"caller judging its own reload reads them as wanted rather than evicted; got {still_desired}" + ) + @pytest.mark.asyncio async def test_get_config_from_file(tmp_path, monkeypatch): @@ -9119,10 +9123,13 @@ class TestDeleteDeploymentSync: with patch.object( proxy_config, "get_config", AsyncMock(return_value={"model_list": []}) ): - count = await proxy_config._delete_deployment(db_models=[]) + still_desired = await proxy_config._delete_deployment(db_models=[]) mock_router.delete_deployment.assert_called_once_with(id="model-id-to-evict") - assert count == 1 + assert still_desired == frozenset(), ( + "an empty db and an empty config want nothing, which must stay distinct from " + f"the None returned when no reconcile ran at all; got {still_desired}" + ) @pytest.mark.asyncio async def test_update_llm_router_skips_update_on_db_fetch_failure(self): diff --git a/tests/test_litellm/proxy/test_update_llm_router_resilience.py b/tests/test_litellm/proxy/test_update_llm_router_resilience.py index fd0df4805e6..a7dc9c1783e 100644 --- a/tests/test_litellm/proxy/test_update_llm_router_resilience.py +++ b/tests/test_litellm/proxy/test_update_llm_router_resilience.py @@ -115,8 +115,10 @@ class TestDeleteDeploymentResilience: """Test _delete_deployment handles get_config failures gracefully.""" @pytest.mark.asyncio - async def test_returns_zero_when_get_config_times_out(self): - """Should return 0 (no deletions) when get_config fails, not raise.""" + async def test_returns_none_when_get_config_times_out(self): + """Should return None (no reconcile ran, desired set unknown) when get_config + fails, not raise. A caller judging its own reload must not read that as "the db + wants nothing" and blame the reload for every model it serves.""" proxy_config = ProxyConfig() db_models = [_make_db_model("gpt-5.1", "db-id-1")] @@ -136,8 +138,8 @@ class TestDeleteDeploymentResilience: ): result = await proxy_config._delete_deployment(db_models=db_models) - # Should safely return 0 instead of raising - assert result == 0 + # Should safely return None instead of raising + assert result is None # Should NOT have deleted any deployments mock_router.delete_deployment.assert_not_called() @@ -175,5 +177,8 @@ class TestDeleteDeploymentResilience: result = await proxy_config._delete_deployment(db_models=db_models) # "stale-id" should have been deleted (not in db_models or config) - assert result == 1 mock_router.delete_deployment.assert_called_once_with(id="stale-id") + assert result == frozenset({"db-id-1", "config-id-1"}), ( + "the returned set must be what the db + config still want, so a caller can " + f"tell that eviction apart from a deployment that went missing; got {result}" + ) diff --git a/tests/test_litellm/test_model_block_unblock.py b/tests/test_litellm/test_model_block_unblock.py index dc0098e405e..ff66bedf0dc 100644 --- a/tests/test_litellm/test_model_block_unblock.py +++ b/tests/test_litellm/test_model_block_unblock.py @@ -36,7 +36,7 @@ def _setup_model_block_mocks(monkeypatch, *, updated_blocked: bool): mock_router = MagicMock() mock_router.get_model_ids.return_value = [model_id] - mock_clear_cache = AsyncMock(return_value=True) + mock_clear_cache = AsyncMock(return_value=None) mock_audit_log = AsyncMock(return_value=None) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) From e38df02d859457e0d1003587e2d2e711c21cca24 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2026 18:18:48 -0700 Subject: [PATCH 26/50] fix(ui): show pass through route selections and match team id substrings in team search (#35319) * fix(ui): show pass through route selections in team/key forms and match team id substrings in team search Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * perf(teams): keep team id search index-friendly with a prefix match Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(teams): keep /v2/team/list search id matching exact by default and add an opt-in prefix mode Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: milan Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management_endpoints/team_endpoints.py | 13 ++++- .../management_endpoints/team_endpoints.py | 4 +- .../test_team_endpoints.py | 57 +++++++++++++++++-- .../app/(dashboard)/hooks/teams/useTeams.ts | 3 + ui/litellm-dashboard/src/components/Teams.tsx | 36 ++++++------ .../components/TeamsPage/TeamsTable.test.tsx | 13 +++++ .../src/components/TeamsPage/TeamsTable.tsx | 1 + .../PassThroughRoutesSelector.tsx | 2 +- .../organisms/create_key_button.tsx | 2 - .../src/components/team/TeamInfo.test.tsx | 51 +++++++++++++++++ .../src/components/team/TeamInfo.tsx | 35 ++++++------ .../components/templates/key_edit_view.tsx | 37 ++++++------ ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +- 13 files changed, 189 insertions(+), 69 deletions(-) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index c35c17aa359..fa01f43d049 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -131,6 +131,7 @@ from litellm.types.proxy.management_endpoints.team_endpoints import ( BulkUpdateTeamMemberPermissionsRequest, BulkUpdateTeamMemberPermissionsResponse, GetTeamMemberPermissionsResponse, + TeamIdSearchMatch, TeamListItem, TeamListResponse, TeamMemberAddResult, @@ -3973,6 +3974,7 @@ async def _build_team_list_where_conditions( user_id: Optional[str], use_deleted_table: bool, search: Optional[str] = None, + search_team_id_match: TeamIdSearchMatch = "exact", org_admin_org_ids: Optional[List[str]] = None, user_api_key_cache: Optional[Any] = None, proxy_logging_obj: Optional[Any] = None, @@ -3996,7 +3998,7 @@ async def _build_team_list_where_conditions( if search: where_conditions["OR"] = [ - {"team_id": search}, + ({"team_id": {"startsWith": search}} if search_team_id_match == "prefix" else {"team_id": search}), {"team_alias": {"contains": search, "mode": "insensitive"}}, ] @@ -4230,8 +4232,14 @@ async def list_team_v2( ), search: Optional[str] = fastapi.Query( default=None, - description="Combined search: matches teams whose 'team_id' equals the value OR whose 'team_alias' contains it (case-insensitive).", + description="Combined search: matches teams whose 'team_id' matches the value OR whose 'team_alias' contains it (case-insensitive).", ), + search_team_id_match: Annotated[ + TeamIdSearchMatch, + fastapi.Query( + description="How 'search' matches 'team_id': 'exact' (default) or 'prefix' for a case-sensitive prefix match." + ), + ] = "exact", page: int = fastapi.Query(default=1, description="Page number for pagination", ge=1), page_size: int = fastapi.Query(default=10, description="Number of teams per page", ge=1, le=100), sort_by: Optional[str] = fastapi.Query( @@ -4308,6 +4316,7 @@ async def list_team_v2( user_id=user_id, use_deleted_table=use_deleted_table, search=search, + search_team_id_match=search_team_id_match, org_admin_org_ids=org_admin_org_ids, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, diff --git a/litellm/types/proxy/management_endpoints/team_endpoints.py b/litellm/types/proxy/management_endpoints/team_endpoints.py index 0e555535874..2d9387da956 100644 --- a/litellm/types/proxy/management_endpoints/team_endpoints.py +++ b/litellm/types/proxy/management_endpoints/team_endpoints.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union from pydantic import BaseModel @@ -11,6 +11,8 @@ from litellm.proxy._types import ( Member, ) +TeamIdSearchMatch = Literal["exact", "prefix"] + class GetTeamMemberPermissionsRequest(BaseModel): """Request to get the team member permissions for a team""" diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index a658a4b7353..25a0ff644f8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -3627,9 +3627,9 @@ async def test_list_team_v2_with_invalid_status(): @pytest.mark.asyncio async def test_list_team_v2_search_builds_or_clause(): """ - `search` should be passed as a Prisma OR across team_id (exact) and - team_alias (case-insensitive contains), so the UI can hit a single - backend filter with either a UUID or a name fragment. + `search` should be passed as a Prisma OR across an exact team_id match and a + case-insensitive team_alias contains, so the UI needs one backend filter. + Exact id matching is the documented default and must not change. """ from unittest.mock import AsyncMock, Mock, patch @@ -3671,6 +3671,54 @@ async def test_list_team_v2_search_builds_or_clause(): } +@pytest.mark.asyncio +async def test_list_team_v2_search_team_id_match_prefix(): + """ + Opting into `search_team_id_match="prefix"` should widen the team_id side of + the search OR to an index-friendly prefix match, so the first characters of a + team id quoted in a proxy error find the team. + """ + from unittest.mock import AsyncMock, Mock, patch + + from fastapi import Request + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import list_team_v2 + + mock_request = Mock(spec=Request) + mock_admin = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma_client: + mock_db = Mock() + mock_prisma_client.db = mock_db + mock_db.litellm_teamtable.find_many = AsyncMock(return_value=[]) + mock_db.litellm_teamtable.count = AsyncMock(return_value=0) + + await list_team_v2( + http_request=mock_request, + user_id=None, + organization_id=None, + team_id=None, + team_alias=None, + search="66c432fa", + search_team_id_match="prefix", + user_api_key_dict=mock_admin, + page=1, + page_size=10, + status=None, + ) + + find_many_kwargs = mock_db.litellm_teamtable.find_many.call_args.kwargs + assert find_many_kwargs["where"] == { + "OR": [ + {"team_id": {"startsWith": "66c432fa"}}, + {"team_alias": {"contains": "66c432fa", "mode": "insensitive"}}, + ] + } + + @pytest.mark.asyncio async def test_list_team_v2_search_composes_with_user_id_filter(): """ @@ -3724,6 +3772,7 @@ async def test_list_team_v2_search_composes_with_user_id_filter(): team_id=None, team_alias=None, search="team_a", + search_team_id_match="prefix", user_api_key_dict=mock_user_api_key_dict, page=1, page_size=10, @@ -3733,7 +3782,7 @@ async def test_list_team_v2_search_composes_with_user_id_filter(): find_many_kwargs = mock_db.litellm_teamtable.find_many.call_args.kwargs where = find_many_kwargs["where"] assert where["OR"] == [ - {"team_id": "team_a"}, + {"team_id": {"startsWith": "team_a"}}, {"team_alias": {"contains": "team_a", "mode": "insensitive"}}, ] assert where["team_id"] == {"in": ["team_a", "team_b"]} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts index f532c44ffd7..4061026b94d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts @@ -24,6 +24,7 @@ export interface TeamListCallOptions { teamID?: string | null; team_alias?: string | null; search?: string | null; + searchTeamIdMatch?: "exact" | "prefix" | null; userID?: string | null; sortBy?: string | null; sortOrder?: string | null; @@ -48,6 +49,7 @@ export const teamListCall = async ( organization_id: options.organizationID, team_alias: options.team_alias, search: options.search, + search_team_id_match: options.searchTeamIdMatch, user_id: options.userID, page, page_size: pageSize, @@ -213,6 +215,7 @@ const deletedTeamListCall = async ( organization_id: options.organizationID, team_alias: options.team_alias, search: options.search, + search_team_id_match: options.searchTeamIdMatch, user_id: options.userID, page, page_size: pageSize, diff --git a/ui/litellm-dashboard/src/components/Teams.tsx b/ui/litellm-dashboard/src/components/Teams.tsx index 0a3c7fc736d..582f75578ed 100644 --- a/ui/litellm-dashboard/src/components/Teams.tsx +++ b/ui/litellm-dashboard/src/components/Teams.tsx @@ -957,25 +957,23 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser placeholder="Select vector stores (optional)" /> - - - form.setFieldValue("allowed_passthrough_routes", values)} - value={form.getFieldValue("allowed_passthrough_routes")} - accessToken={accessToken || ""} - placeholder="Select pass through routes (optional)" - disabled={!premiumUser || !isProxyAdminRole(userRole || "")} - /> - + + diff --git a/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.test.tsx b/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.test.tsx index 9469be12128..09bd3e9f245 100644 --- a/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.test.tsx +++ b/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.test.tsx @@ -196,6 +196,19 @@ describe("server-side filtering maps controls to the right query params", () => expect(mockUseTeamsTable).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ search: "platform" })); }); }); + + it("opts into team id prefix matching so a partial id from a proxy error finds the team", async () => { + renderTable(); + fireEvent.change(screen.getByTestId("datatable-search"), { target: { value: "66c432fa" } }); + + await waitFor(() => { + expect(mockUseTeamsTable).toHaveBeenLastCalledWith( + 1, + 50, + expect.objectContaining({ search: "66c432fa", searchTeamIdMatch: "prefix" }), + ); + }); + }); }); describe("non-admin scoping", () => { diff --git a/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.tsx b/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.tsx index 3b75db52b16..5f5a6f26c0a 100644 --- a/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.tsx +++ b/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.tsx @@ -66,6 +66,7 @@ export function TeamsTable({ userRole, userID, onSelectTeam, onEditTeam, onDelet team_alias: getFilterValue("alias"), teamID: getFilterValue("team_id"), search: searchQuery.trim() || undefined, + searchTeamIdMatch: "prefix" as const, userID: isAdminView ? undefined : userID ?? undefined, sortBy: sorting[0]?.id, sortOrder: toSortOrder(sorting), diff --git a/ui/litellm-dashboard/src/components/common_components/PassThroughRoutesSelector.tsx b/ui/litellm-dashboard/src/components/common_components/PassThroughRoutesSelector.tsx index 5cea88b2af2..e02125dea56 100644 --- a/ui/litellm-dashboard/src/components/common_components/PassThroughRoutesSelector.tsx +++ b/ui/litellm-dashboard/src/components/common_components/PassThroughRoutesSelector.tsx @@ -3,7 +3,7 @@ import { Select } from "antd"; import { getPassThroughEndpointsCall } from "../networking"; interface PassThroughRoutesSelectorProps { - onChange: (selectedRoutes: string[]) => void; + onChange?: (selectedRoutes: string[]) => void; value?: string[]; className?: string; accessToken: string; diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 711652eb783..00f36f016b7 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -1385,8 +1385,6 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp } > form.setFieldValue("allowed_passthrough_routes", values)} - value={form.getFieldValue("allowed_passthrough_routes")} accessToken={accessToken} placeholder={ !premiumUser diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index 25365d26a12..712cff80649 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -18,6 +18,7 @@ vi.mock("@/components/networking", () => ({ getTeamPermissionsCall: vi.fn(), organizationInfoCall: vi.fn(), getRouterSettingsCall: vi.fn().mockResolvedValue({ fields: [] }), + getPassThroughEndpointsCall: vi.fn(), })); vi.mock("@/components/utils/dataUtils", () => ({ @@ -1142,4 +1143,54 @@ describe("TeamInfoView", () => { expect(within(dropdown).getByTitle("opt-in")).toBeInTheDocument(); }); }); + + describe("allowed pass through routes", () => { + beforeEach(() => { + testQueryClient.clear(); + vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData({ models: ["gpt-4"] })); + vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: {}, team_id: "123" } as any); + vi.mocked(networking.getPassThroughEndpointsCall).mockResolvedValue({ + endpoints: [{ path: "/bedrock-passthrough", methods: ["POST"] }], + }); + }); + + it("should show a route picked from the dropdown in the field and save it", async () => { + const user = userEvent.setup({ delay: null }); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.queryAllByText("Test Team").length).toBeGreaterThan(0); + }); + + await user.click(screen.getByRole("tab", { name: "Settings" })); + await user.click(await screen.findByRole("button", { name: /edit settings/i })); + + const routesLabel = await screen.findByText("Allowed Pass Through Routes"); + const routesFormItem = routesLabel.closest(".ant-form-item") as HTMLElement; + + await user.click(within(routesFormItem).getByRole("combobox")); + + const option = await screen.findByTitle("POST /bedrock-passthrough"); + await user.click(option); + + await waitFor(() => { + expect(within(routesFormItem).getByText(/\/bedrock-passthrough/)).toBeInTheDocument(); + }); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => { + expect(networking.teamUpdateCall).toHaveBeenCalledWith( + "test-token", + expect.objectContaining({ + team_id: "123", + metadata: expect.objectContaining({ + allowed_passthrough_routes: ["/bedrock-passthrough"], + }), + }), + ); + }); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index acd5a8966a7..34570043f52 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -1361,25 +1361,22 @@ const TeamInfoView: React.FC = ({ /> - - - form.setFieldValue("allowed_passthrough_routes", values)} - value={form.getFieldValue("allowed_passthrough_routes")} - accessToken={accessToken || ""} - placeholder="Select pass through routes" - disabled={!premiumUser || !is_proxy_admin} - /> - + + diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index 962c6bc3568..4b6c266eb8f 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -689,26 +689,23 @@ export function KeyEditView({ - - - form.setFieldValue("allowed_passthrough_routes", values)} - value={form.getFieldValue("allowed_passthrough_routes")} - accessToken={accessToken || ""} - placeholder={ - !premiumUser - ? "Premium feature - Upgrade to set allowed pass through routes by key" - : Array.isArray(keyData.metadata?.allowed_passthrough_routes) && - keyData.metadata.allowed_passthrough_routes.length > 0 - ? `Current: ${keyData.metadata.allowed_passthrough_routes.join(", ")}` - : "Select or enter allowed pass through routes" - } - disabled={!premiumUser} - /> - + + 0 + ? `Current: ${keyData.metadata.allowed_passthrough_routes.join(", ")}` + : "Select or enter allowed pass through routes" + } + disabled={!premiumUser} + /> diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 9a301f1c474..109d638fb9c 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -58372,8 +58372,10 @@ export interface operations { team_id?: string | null; /** @description Only return teams which this 'team_alias' belongs to. Supports partial matching. */ team_alias?: string | null; - /** @description Combined search: matches teams whose 'team_id' equals the value OR whose 'team_alias' contains it (case-insensitive). */ + /** @description Combined search: matches teams whose 'team_id' matches the value OR whose 'team_alias' contains it (case-insensitive). */ search?: string | null; + /** @description How 'search' matches 'team_id': 'exact' (default) or 'prefix' for a case-sensitive prefix match. */ + search_team_id_match?: "exact" | "prefix"; /** @description Page number for pagination */ page?: number; /** @description Number of teams per page */ From 7c388b1fd97a88d11e7afacc8a4b6b056ae8ffeb Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:39:35 +0000 Subject: [PATCH 27/50] refactor(proxy): rename the audit hook's user row variable The typed-vs-raw distinction the _litellm_typed suffix marked is gone now that the repository returns LiteLLM_UserTable directly. --- litellm/proxy/hooks/user_management_event_hooks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/hooks/user_management_event_hooks.py b/litellm/proxy/hooks/user_management_event_hooks.py index d8cfae5dab0..e40db7f3f1f 100644 --- a/litellm/proxy/hooks/user_management_event_hooks.py +++ b/litellm/proxy/hooks/user_management_event_hooks.py @@ -57,18 +57,18 @@ class UserManagementEventHooks: raise Exception(CommonProxyErrors.db_not_connected_error.value) if response.user_id is None: raise Exception("no user_id returned for the newly created user") - user_row_litellm_typed = await UserRepository(prisma_client).find_by_id(response.user_id) - if user_row_litellm_typed is None: + user_row = await UserRepository(prisma_client).find_by_id(response.user_id) + if user_row is None: raise Exception(f"no user row found for user_id={response.user_id}") asyncio.create_task( UserManagementEventHooks.create_internal_user_audit_log( - user_id=user_row_litellm_typed.user_id, + user_id=user_row.user_id, action="created", litellm_changed_by=user_api_key_dict.user_id, user_api_key_dict=user_api_key_dict, litellm_proxy_admin_name=litellm_proxy_admin_name, before_value=None, - after_value=user_row_litellm_typed.model_dump_json(exclude_none=True), + after_value=user_row.model_dump_json(exclude_none=True), ) ) except Exception as e: From e8e2e07ef67010685839f79c0b0ad18dd9ec7ced Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 19:57:52 -0700 Subject: [PATCH 28/50] fix(proxy): align team member add with existing user provisioning rules Adding a team member by a user_id with no user row created that row as a side effect for any caller permitted to add members, while creating users directly is restricted to proxy admins. Restrict that path to proxy admins too; adding an existing user, and inviting a new one by user_email (where the user_id is allocated server-side), are unchanged. Also record the membership change, and any user row it creates, in the audit log, matching /team/update, /user/new and /key/*. --- .../management_endpoints/team_endpoints.py | 147 +++++++++++ .../test_team_endpoints.py | 236 ++++++++++++++++++ 2 files changed, 383 insertions(+) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index fa01f43d049..7b5ce33cb8b 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -13,6 +13,7 @@ import asyncio import json import math import traceback +from collections.abc import Sequence from datetime import datetime, timezone from typing import Annotated, Any, Dict, List, Mapping, Optional, Tuple, Union, cast @@ -2426,6 +2427,123 @@ def _emit_team_members_metric(team: LiteLLM_TeamTable) -> None: verbose_proxy_logger.debug("Prometheus: failed to emit team members metric: %s", str(e)) +async def _resolve_existing_member_user_ids( + members: Sequence[Member], + prisma_client: PrismaClient, +) -> frozenset[str]: + """Return the caller-supplied user_ids that already have a user row.""" + user_repository = UserRepository(prisma_client) + found = await asyncio.gather( + *(user_repository.find_by_id(member.user_id) for member in members if member.user_id is not None) + ) + return frozenset(user.user_id for user in found if user is not None and user.user_id is not None) + + +def _pre_existing_user_ids( + members: Sequence[Member], + caller_supplied_user_ids: frozenset[str], + existing_user_ids: frozenset[str], +) -> frozenset[str]: + """Return the user_ids that already had a user row before this request. + + Combines the caller-supplied ids that resolved to a user with the ids + ``_validate_and_populate_member_user_info`` filled in, which it only does + from a matched user row. Deriving it that way keeps this in step with the + email matching that resolution performs, rather than repeating it here. + """ + populated_user_ids = frozenset( + member.user_id + for member in members + if member.user_id is not None and member.user_id not in caller_supplied_user_ids + ) + return existing_user_ids | populated_user_ids + + +def _validate_member_user_id_provisioning( + members: Sequence[Member], + existing_user_ids: frozenset[str], + user_api_key_dict: UserAPIKeyAuth, +) -> None: + """Restrict adding a caller-chosen user_id that has no user row yet to proxy admins. + + Team and org admins keep the ability to add users that already exist and to + invite new ones by user_email, where the user_id is allocated server-side. + """ + if user_api_key_dict.user_role in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN.value, + ): + return + + unknown_user_ids = tuple( + member.user_id for member in members if member.user_id is not None and member.user_id not in existing_user_ids + ) + if not unknown_user_ids: + return + + raise HTTPException( + status_code=403, + detail={ # mutable-ok: HTTPException detail must be a plain mapping to keep this route's {"error": ...} response shape + "error": ( + "Only proxy admins can add a user_id that does not exist yet: {}. " + "Add the member by user_email to invite a new user, or ask a proxy admin " + "to create the user first.".format(", ".join(unknown_user_ids)) + ) + }, + ) + + +def _members_audit_value(members: Sequence[Member]) -> str: + """Serialize a team's member list for an audit-log value. + + The audit-log columns hold a JSON object, so the member list is nested + under a key rather than serialized as a top-level array. + """ + return safe_dumps( + { # mutable-ok: the audit-log JSON column rejects a top-level array, so this value must be an object + "members_with_roles": tuple(member.model_dump() for member in members) + } + ) + + +async def _create_team_member_add_audit_logs( + team_id: str, + updated_users: Sequence[LiteLLM_UserTable], + existing_user_ids: frozenset[str], + before_members: Sequence[Member], + after_members: Sequence[Member], + user_api_key_dict: UserAPIKeyAuth, + litellm_proxy_admin_name: str, +) -> None: + """Record the membership change, and any user row it created, in the audit log.""" + from litellm.proxy.management_helpers.audit_logs import create_object_audit_log + + for user in updated_users: + if user.user_id is None or user.user_id in existing_user_ids: + continue + await create_object_audit_log( + object_id=user.user_id, + action="created", + litellm_changed_by=None, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + table_name=LitellmTableNames.USER_TABLE_NAME, + before_value=None, + after_value=safe_dumps(user.model_dump(exclude_none=True)), + ) + + await create_object_audit_log( + object_id=team_id, + action="updated", + litellm_changed_by=None, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + table_name=LitellmTableNames.TEAM_TABLE_NAME, + before_value=_members_audit_value(before_members), + after_value=_members_audit_value(after_members), + ) + + async def _validate_and_populate_member_user_info( member: Member, prisma_client: PrismaClient, @@ -2606,6 +2724,19 @@ async def team_member_add( data=data, ) + requested_members = tuple(data.member) if isinstance(data.member, list) else (data.member,) + caller_supplied_user_ids = frozenset(member.user_id for member in requested_members if member.user_id is not None) + existing_user_ids = await _resolve_existing_member_user_ids( + members=requested_members, + prisma_client=prisma_client, + ) + _validate_member_user_id_provisioning( + members=requested_members, + existing_user_ids=existing_user_ids, + user_api_key_dict=user_api_key_dict, + ) + members_before_add = tuple(complete_team_data.members_with_roles) + # Validate and populate user_email/user_id for members before processing if isinstance(data.member, Member): await _validate_and_populate_member_user_info( @@ -2619,6 +2750,12 @@ async def team_member_add( prisma_client=prisma_client, ) + pre_existing_user_ids = _pre_existing_user_ids( + members=requested_members, + caller_supplied_user_ids=caller_supplied_user_ids, + existing_user_ids=existing_user_ids, + ) + ( updated_team, updated_users, @@ -2637,6 +2774,16 @@ async def team_member_add( _emit_team_members_metric(complete_team_data) + await _create_team_member_add_audit_logs( + team_id=data.team_id, + updated_users=updated_users, + existing_user_ids=pre_existing_user_ids, + before_members=members_before_add, + after_members=tuple(complete_team_data.members_with_roles), + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ) + return TeamAddMemberResponse.model_validate( { **updated_team.model_dump(), diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 25a0ff644f8..b1438447e4f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -10338,3 +10338,239 @@ async def test_list_available_teams_filters_joined_and_validates_rows(monkeypatc assert result[0].team_alias == "open team" find_many_kwargs = mock_prisma_client.db.litellm_teamtable.find_many.call_args.kwargs assert find_many_kwargs["where"] == {"team_id": {"in": ["team-open"]}} + + +def _provisioning_caller(role: LitellmUserRoles) -> UserAPIKeyAuth: + return UserAPIKeyAuth(user_id="caller-1", user_role=role) + + +def test_validate_member_user_id_provisioning_allows_proxy_admin(): + """Proxy admins may add a user_id that has no user row yet.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _validate_member_user_id_provisioning, + ) + + _validate_member_user_id_provisioning( + members=[Member(user_id="brand-new", role="user")], + existing_user_ids=frozenset(), + user_api_key_dict=_provisioning_caller(LitellmUserRoles.PROXY_ADMIN), + ) + + +def test_validate_member_user_id_provisioning_rejects_unknown_user_id_for_non_proxy_admin(): + """A non-proxy-admin cannot add a user_id that has no user row yet.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _validate_member_user_id_provisioning, + ) + + with pytest.raises(HTTPException) as exc_info: + _validate_member_user_id_provisioning( + members=[Member(user_id="brand-new", role="user")], + existing_user_ids=frozenset(), + user_api_key_dict=_provisioning_caller(LitellmUserRoles.INTERNAL_USER), + ) + + assert exc_info.value.status_code == 403 + assert "brand-new" in str(exc_info.value.detail) + + +def test_validate_member_user_id_provisioning_allows_existing_user_id_for_non_proxy_admin(): + """A non-proxy-admin may still add a user that already exists.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _validate_member_user_id_provisioning, + ) + + _validate_member_user_id_provisioning( + members=[Member(user_id="already-here", role="user")], + existing_user_ids=frozenset({"already-here"}), + user_api_key_dict=_provisioning_caller(LitellmUserRoles.INTERNAL_USER), + ) + + +def test_validate_member_user_id_provisioning_allows_email_only_member_for_non_proxy_admin(): + """Inviting by user_email stays open to non-proxy-admins; the user_id is server-allocated.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _validate_member_user_id_provisioning, + ) + + _validate_member_user_id_provisioning( + members=[Member(user_email="invitee@example.com", role="user")], + existing_user_ids=frozenset(), + user_api_key_dict=_provisioning_caller(LitellmUserRoles.INTERNAL_USER), + ) + + +def test_validate_member_user_id_provisioning_rejects_unknown_user_id_paired_with_email(): + """Supplying a user_email alongside an unknown user_id does not lift the restriction.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _validate_member_user_id_provisioning, + ) + + with pytest.raises(HTTPException) as exc_info: + _validate_member_user_id_provisioning( + members=[Member(user_id="chosen-id", user_email="invitee@example.com", role="user")], + existing_user_ids=frozenset(), + user_api_key_dict=_provisioning_caller(LitellmUserRoles.INTERNAL_USER), + ) + + assert exc_info.value.status_code == 403 + + +def test_validate_member_user_id_provisioning_reports_every_unknown_member(): + """A bulk add names each unknown user_id rather than only the first.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _validate_member_user_id_provisioning, + ) + + with pytest.raises(HTTPException) as exc_info: + _validate_member_user_id_provisioning( + members=[ + Member(user_id="known", role="user"), + Member(user_id="unknown-a", role="user"), + Member(user_id="unknown-b", role="user"), + ], + existing_user_ids=frozenset({"known"}), + user_api_key_dict=_provisioning_caller(LitellmUserRoles.INTERNAL_USER), + ) + + detail = str(exc_info.value.detail) + assert "unknown-a" in detail + assert "unknown-b" in detail + + +@pytest.mark.asyncio +async def test_resolve_existing_member_user_ids_matches_caller_supplied_user_ids(): + """Only caller-supplied user_ids are looked up; unknown ones resolve to nothing.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _resolve_existing_member_user_ids, + ) + + prisma_client = MagicMock() + + async def find_by_id(user_id): + if user_id == "by-id": + return LiteLLM_UserTable(user_id="by-id", max_budget=None, spend=0.0, user_email=None, models=[]) + return None + + with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: + repo.return_value.find_by_id = AsyncMock(side_effect=find_by_id) + + resolved = await _resolve_existing_member_user_ids( + members=[ + Member(user_id="by-id", role="user"), + Member(user_id="missing", role="user"), + Member(user_email="someone@example.com", role="user"), + ], + prisma_client=prisma_client, + ) + + assert resolved == frozenset({"by-id"}) + + +def test_pre_existing_user_ids_counts_ids_filled_in_by_member_resolution(): + """An id the member-resolution step filled in came from a matched row, so it pre-existed. + + This is what keeps a case-variant email invite of an existing user from being + recorded as a newly created user. + """ + from litellm.proxy.management_endpoints.team_endpoints import _pre_existing_user_ids + + # member arrived email-only; resolution matched an existing row and filled in the id + resolved_member = Member(user_id="matched-existing", user_email="Someone@Example.com", role="user") + + assert _pre_existing_user_ids( + members=[resolved_member], + caller_supplied_user_ids=frozenset(), + existing_user_ids=frozenset(), + ) == frozenset({"matched-existing"}) + + +def test_pre_existing_user_ids_excludes_caller_supplied_ids_that_do_not_exist(): + """A caller-supplied id that resolved to nothing is genuinely new, so it stays out.""" + from litellm.proxy.management_endpoints.team_endpoints import _pre_existing_user_ids + + assert _pre_existing_user_ids( + members=[Member(user_id="brand-new", role="user"), Member(user_id="already-here", role="user")], + caller_supplied_user_ids=frozenset({"brand-new", "already-here"}), + existing_user_ids=frozenset({"already-here"}), + ) == frozenset({"already-here"}) + + +def test_members_audit_value_serializes_to_a_json_object(): + """The audit-log columns hold a JSON object; a top-level array is rejected by the DB.""" + from litellm.proxy.management_endpoints.team_endpoints import _members_audit_value + + payload = json.loads(_members_audit_value([Member(user_id="u1", role="admin"), Member(user_id="u2", role="user")])) + + assert isinstance(payload, dict) + assert [m["user_id"] for m in payload["members_with_roles"]] == ["u1", "u2"] + + +@pytest.mark.asyncio +async def test_team_member_add_audits_a_user_created_from_a_list_payload(monkeypatch): + """A user created by a list payload must still be reported as newly created. + + For a list payload the member-list reconciliation back-fills the caller's own + Member objects with the ids of users this request just created. The set of + pre-existing ids therefore has to be captured before that runs, otherwise a + freshly created user looks like it was already there and no creation is recorded. + """ + from litellm.proxy._types import TeamMemberAddRequest + from litellm.proxy.management_endpoints.team_endpoints import team_member_add + + team_id = "team-list-audit" + created_user_id = "generated-uuid-for-new-invitee" + member = Member(user_email="invitee@example.com", role="user") + + mock_prisma_client = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + monkeypatch.setattr("litellm.proxy.proxy_server.litellm_proxy_admin_name", "default_user_id") + + team_row = LiteLLM_TeamTable(team_id=team_id, members_with_roles=[]) + created_user = LiteLLM_UserTable( + user_id=created_user_id, user_email="invitee@example.com", max_budget=None, spend=0.0, models=[] + ) + updated_team = MagicMock() + updated_team.model_dump.return_value = {"team_id": team_id, "members_with_roles": []} + + async def fake_add_team_members_to_team(**kwargs): + # mirrors _update_team_members_list: the list branch mutates the caller's Member in place + member.user_id = created_user_id + return updated_team, [created_user], [] + + with ( + patch( + "litellm.proxy.management_endpoints.team_endpoints.get_team_object", + new_callable=AsyncMock, + return_value=team_row, + ), + patch( + "litellm.proxy.management_endpoints.team_endpoints._validate_team_member_add_permissions", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.team_endpoints._validate_and_populate_member_user_info", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.team_endpoints._resolve_existing_member_user_ids", + new_callable=AsyncMock, + return_value=frozenset(), + ), + patch( + "litellm.proxy.management_endpoints.team_endpoints._add_team_members_to_team", + side_effect=fake_add_team_members_to_team, + ), + patch( + "litellm.proxy.management_endpoints.team_endpoints._create_team_member_add_audit_logs", + new_callable=AsyncMock, + ) as mock_audit, + ): + await team_member_add( + data=TeamMemberAddRequest(team_id=team_id, member=[member]), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin-1"), + ) + + mock_audit.assert_called_once() + assert created_user_id not in mock_audit.call_args.kwargs["existing_user_ids"] From 629d58443ea77bd6156db09f0dfa1280acd6fd20 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 31 Jul 2026 20:08:02 -0700 Subject: [PATCH 29/50] feat(proxy): push config sync to pods via redis pub/sub After any management write to a DB-backed config table, publish an invalidation event on the coordination Redis; every pod runs a subscriber that debounces, jitters, and triggers an immediate add_deployment plus get_credentials resync. The interval polls stay as slow reconciliation fallback and behavior without Redis is unchanged since publish and subscribe both no-op. --- .../proxy/common_utils/config_sync_pubsub.py | 258 ++++++++ .../key_management_endpoints.py | 5 + .../model_management_endpoints.py | 7 + litellm/proxy/proxy_server.py | 27 +- .../proxy_setting_endpoints.py | 2 + litellm/proxy/utils.py | 11 +- .../repositories/credentials_repository.py | 6 +- litellm/repositories/model_repository.py | 6 +- litellm/repositories/table_repositories.py | 7 +- ruff.toml | 2 +- .../common_utils/test_config_sync_pubsub.py | 600 ++++++++++++++++++ .../repositories/test_repositories.py | 58 +- 12 files changed, 978 insertions(+), 11 deletions(-) create mode 100644 litellm/proxy/common_utils/config_sync_pubsub.py create mode 100644 tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py diff --git a/litellm/proxy/common_utils/config_sync_pubsub.py b/litellm/proxy/common_utils/config_sync_pubsub.py new file mode 100644 index 00000000000..4292c67c8c2 --- /dev/null +++ b/litellm/proxy/common_utils/config_sync_pubsub.py @@ -0,0 +1,258 @@ +import asyncio +import json +import random +from collections.abc import Awaitable, Callable +from dataclasses import asdict, dataclass +from typing import TYPE_CHECKING, Protocol, cast # noqa: TID251 # untyped prisma/redis boundary needs cast + +from litellm._logging import verbose_proxy_logger + +if TYPE_CHECKING: + from litellm.caching.redis_cache import RedisCache + + +class _ConfigSyncPubSub(Protocol): + def subscribe(self, *channels: str) -> Awaitable[object]: ... + + def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Awaitable[object]: ... + + def aclose(self) -> Awaitable[object]: ... + + +class _ConfigSyncPubSubClient(Protocol): + def publish(self, channel: str, message: str) -> Awaitable[int]: ... + + def pubsub(self) -> _ConfigSyncPubSub: ... + + +CONFIG_SYNC_CHANNEL = "litellm_proxy.config_change" +CONFIG_SYNC_DEBOUNCE_SECONDS = 1.0 +CONFIG_SYNC_JITTER_MAX_SECONDS = 5.0 +_POLL_TIMEOUT_SECONDS = 1.0 +_BACKOFF_INITIAL_SECONDS = 5.0 +_BACKOFF_MAX_SECONDS = 60.0 + +_WRITE_ACTION_NAMES: frozenset[str] = frozenset( + {"create", "create_many", "update", "update_many", "upsert", "delete", "delete_many"} +) + +_CONFIG_SYNCED_TABLE_NAMES: frozenset[str] = frozenset( + { + "litellm_proxymodeltable", + "litellm_credentialstable", + "litellm_guardrailstable", + "litellm_policytable", + "litellm_policyattachmenttable", + "litellm_managedvectorstorestable", + "litellm_managedvectorstoreindextable", + "litellm_mcpservertable", + "litellm_agentstable", + "litellm_prompttable", + "litellm_searchtoolstable", + "litellm_ssoconfig", + "litellm_cacheconfig", + "litellm_configoverrides", + } +) + + +def coordination_redis_cache() -> "RedisCache | None": + from litellm.proxy.proxy_server import redis_usage_cache + + return redis_usage_cache + + +def config_sync_channel(redis_cache: "RedisCache") -> str: + if redis_cache.namespace is None: + return CONFIG_SYNC_CHANNEL + return f"{redis_cache.namespace}:{CONFIG_SYNC_CHANNEL}" + + +def _raw_async_client(redis_cache: "RedisCache") -> object: + return cast( # cast-ok: redis-py generics leave the client type partially unknown + object, + redis_cache.init_async_client(), # pyright: ignore[reportUnknownMemberType] # redis generics + ) + + +def _pubsub_capable_client(redis_cache: "RedisCache") -> _ConfigSyncPubSubClient | None: + from redis.asyncio import Redis + + client = _raw_async_client(redis_cache) + if isinstance(client, Redis): + return cast(_ConfigSyncPubSubClient, client) # cast-ok: protocol view of the standalone redis client + return None + + +@dataclass(frozen=True, slots=True) +class _ConfigChangeMessage: + object_type: str + + +def _config_change_message_json(object_type: str) -> str: + return json.dumps(asdict(_ConfigChangeMessage(object_type=object_type))) + + +async def publish_config_change(redis_cache: "RedisCache | None", object_type: str) -> None: + if redis_cache is None: + return + try: + client = _pubsub_capable_client(redis_cache) + if client is None: + verbose_proxy_logger.debug( + "config sync publish for %s skipped: cluster redis client has no pub/sub support", + object_type, + ) + return + await client.publish(config_sync_channel(redis_cache), _config_change_message_json(object_type)) + except Exception as e: # noqa: BLE001 # best-effort publish; writes must never fail on redis errors + verbose_proxy_logger.warning("config sync publish for %s failed: %s", object_type, e) + + +async def publish_config_change_for_object_type(object_type: str) -> None: + await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type) + + +class _PublishOnWriteActions: + __slots__ = ("_actions", "_object_type", "_publish") + + def __init__(self, actions: object, object_type: str, publish: Callable[[str], Awaitable[None]]) -> None: + self._actions = actions + self._object_type = object_type + self._publish = publish + + def __getattr__(self, name: str) -> object: + attribute = cast(object, getattr(self._actions, name)) # cast-ok: getattr on dynamic prisma actions + if name not in _WRITE_ACTION_NAMES: + return attribute + write_action = cast(Callable[..., Awaitable[object]], attribute) # cast-ok: prisma actions are untyped + object_type = self._object_type + publish = self._publish + + async def _write_then_publish( + *args: object, + **kwargs: object, # kwargs-ok: transparent passthrough to untyped prisma action + ) -> object: + result = await write_action(*args, **kwargs) + await publish(object_type) + return result + + return _write_then_publish + + +def wrap_table_actions_for_config_sync( + actions: object, + table_name: str, + publish: Callable[[str], Awaitable[None]] = publish_config_change_for_object_type, +) -> object: + if table_name not in _CONFIG_SYNCED_TABLE_NAMES: + return actions + return _PublishOnWriteActions(actions=actions, object_type=table_name, publish=publish) + + +class ConfigSyncSubscriber: + __slots__ = ( + "_backoff_initial_seconds", + "_backoff_max_seconds", + "_debounce_seconds", + "_jitter_max_seconds", + "_redis_cache", + "_resync_callbacks", + "_rng", + "_sleep", + "_task", + ) + + def __init__( + self, + redis_cache: "RedisCache", + resync_callbacks: tuple[Callable[[], Awaitable[None]], ...], + debounce_seconds: float = CONFIG_SYNC_DEBOUNCE_SECONDS, + jitter_max_seconds: float = CONFIG_SYNC_JITTER_MAX_SECONDS, + backoff_initial_seconds: float = _BACKOFF_INITIAL_SECONDS, + backoff_max_seconds: float = _BACKOFF_MAX_SECONDS, + rng: random.Random | None = None, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + ) -> None: + self._redis_cache = redis_cache + self._resync_callbacks = resync_callbacks + self._debounce_seconds = debounce_seconds + self._jitter_max_seconds = jitter_max_seconds + self._backoff_initial_seconds = backoff_initial_seconds + self._backoff_max_seconds = backoff_max_seconds + self._rng = rng if rng is not None else random.Random() + self._sleep = sleep + self._task: asyncio.Task[None] | None = None + + def start(self) -> None: + if self._task is not None: + return + self._task = asyncio.create_task(self._run()) + + async def stop(self) -> None: + task = self._task + if task is None: + return + self._task = None + _ = task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + async def _run(self) -> None: + backoff_seconds = self._backoff_initial_seconds + while True: + try: + client = _pubsub_capable_client(self._redis_cache) + if client is None: + verbose_proxy_logger.warning( + "config sync subscriber disabled: cluster redis client has no pub/sub support; " + "interval polling remains the only sync mechanism" + ) + return + pubsub = client.pubsub() + try: + await pubsub.subscribe(config_sync_channel(self._redis_cache)) + backoff_seconds = self._backoff_initial_seconds + await self._consume(pubsub) + finally: + await self._close_pubsub(pubsub) + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 # any redis failure falls through to backoff and reconnect + verbose_proxy_logger.warning( + "config sync subscriber redis error: %s; reconnecting in %.0fs", + e, + backoff_seconds, + ) + await self._sleep(backoff_seconds) + backoff_seconds = min(backoff_seconds * 2, self._backoff_max_seconds) + + async def _consume(self, pubsub: _ConfigSyncPubSub) -> None: + while True: + message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=_POLL_TIMEOUT_SECONDS) + if message is None: + continue + await self._sleep(self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds)) + await self._drain_pending(pubsub) + await self._run_resync_callbacks() + + @staticmethod + async def _drain_pending(pubsub: _ConfigSyncPubSub) -> None: + while await pubsub.get_message(ignore_subscribe_messages=True, timeout=0) is not None: + pass + + async def _run_resync_callbacks(self) -> None: + for callback in self._resync_callbacks: + try: + await callback() + except Exception as e: # noqa: BLE001 # one failing resync callback must not kill the subscriber + verbose_proxy_logger.warning("config sync resync callback failed: %s", e) + + @staticmethod + async def _close_pubsub(pubsub: _ConfigSyncPubSub) -> None: + try: + await pubsub.aclose() + except Exception as e: # noqa: BLE001 # best-effort close of a possibly-broken connection + verbose_proxy_logger.debug("config sync pubsub close failed: %s", e) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index a94a75fdfa3..1fbfda8bf8a 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -60,6 +60,10 @@ from litellm.proxy.common_utils.callback_utils import ( decrypt_callback_vars, encrypt_callback_vars, ) +from litellm.proxy.common_utils.config_sync_pubsub import ( + coordination_redis_cache, + publish_config_change, +) from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache @@ -4240,6 +4244,7 @@ async def _rotate_master_key( await tx.litellm_proxymodeltable.create_many( data=new_models, ) + await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable") # 3. process config table try: config = await ConfigRepository(prisma_client).table.find_many() diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index b6422d7f5ae..21395658dac 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -38,6 +38,10 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.config_sync_pubsub import ( + coordination_redis_cache, + publish_config_change, +) from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.management_endpoints.common_utils import _is_user_team_admin from litellm.proxy.management_endpoints.team_endpoints import ( @@ -809,6 +813,9 @@ async def delete_team_models( await tx.litellm_proxymodeltable.delete_many(where={"model_id": {"in": model_ids}}) deleted_model_ids.extend(model_ids) + if deleted_model_ids: + await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable") + if llm_router is not None: for model_id in deleted_model_ids: llm_router.delete_deployment(id=model_id) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a60ea2da019..6bb0032b38c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -301,6 +301,7 @@ from litellm.proxy.common_request_processing import ( create_response, ) from litellm.proxy.common_utils.callback_utils import initialize_callbacks_on_proxy +from litellm.proxy.common_utils.config_sync_pubsub import ConfigSyncSubscriber from litellm.proxy.common_utils.debug_utils import init_verbose_loggers from litellm.proxy.common_utils.debug_utils import router as debugging_endpoints_router from litellm.proxy.common_utils.encrypt_decrypt_utils import ( @@ -546,6 +547,7 @@ from litellm.proxy.utils import ( _get_redoc_url, _is_projected_spend_over_limit, _is_valid_team_configs, + evict_config_param, get_config_param, get_custom_url, get_error_message_str, @@ -1151,6 +1153,12 @@ async def proxy_startup_event(app: FastAPI): except Exception as e: verbose_proxy_logger.error(f"Error stopping DB health watchdog task: {e}") + if proxy_config.config_sync_subscriber is not None: + try: + await proxy_config.config_sync_subscriber.stop() + except Exception as e: + verbose_proxy_logger.error(f"Error stopping config sync subscriber: {e}") + await proxy_shutdown_event() # type: ignore[reportGeneralTypeIssues] @@ -3837,6 +3845,7 @@ class ProxyConfig: self._last_semantic_filter_config: Optional[Dict[str, Any]] = None self._last_hashicorp_vault_config: Optional[Dict[str, Any]] = None self.worker_registry: List["WorkerRegistryEntry"] = [] + self.config_sync_subscriber: ConfigSyncSubscriber | None = None def is_yaml(self, config_file_path: str) -> bool: if not os.path.isfile(config_file_path): @@ -6495,7 +6504,7 @@ class ProxyConfig: }, }, ) - await invalidate_config_param("model_cost_map_reload_config") + await evict_config_param("model_cost_map_reload_config") verbose_proxy_logger.info( f"Model cost map reloaded successfully. Models count: {len(new_model_cost_map) if new_model_cost_map else 0}" @@ -6590,7 +6599,7 @@ class ProxyConfig: }, }, ) - await invalidate_config_param("anthropic_beta_headers_reload_config") + await evict_config_param("anthropic_beta_headers_reload_config") # Count providers in config provider_count = sum(1 for k in new_config.keys() if k != "provider_aliases" and k != "description") @@ -8165,6 +8174,20 @@ class ProxyStartupEvent: ) await proxy_config.get_credentials(prisma_client=prisma_client) + if redis_usage_cache is not None and proxy_config.config_sync_subscriber is None: + + async def _resync_config_from_db() -> None: + await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) + + async def _resync_credentials_from_db() -> None: + await proxy_config.get_credentials(prisma_client=prisma_client) + + proxy_config.config_sync_subscriber = ConfigSyncSubscriber( + redis_cache=redis_usage_cache, + resync_callbacks=(_resync_config_from_db, _resync_credentials_from_db), + ) + proxy_config.config_sync_subscriber.start() + if store_model_in_db is not True: await proxy_config.init_mcp_servers_from_db() if prisma_client is not None: diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 8f3f8ad1bfc..e4872a4b6b5 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -21,6 +21,7 @@ from litellm.proxy.config_resolvers.sso import ( SSO_SECRET_FIELDS, resolve_sso_config, ) +from litellm.proxy.utils import invalidate_config_param from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.table_repositories import ( SSOConfigRepository, @@ -971,6 +972,7 @@ async def update_sso_settings( "param_value": json.dumps(filtered_env_vars, default=str), }, ) + await invalidate_config_param("environment_variables") except Exception as e: raise HTTPException( status_code=500, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 2ca251a3211..fbcca73e779 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -119,6 +119,10 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.route_checks import RouteChecks +from litellm.proxy.common_utils.config_sync_pubsub import ( + coordination_redis_cache, + publish_config_change, +) from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.db.create_views import ( create_missing_views, @@ -2971,9 +2975,14 @@ async def get_config_param(prisma_client: Any, param_name: str) -> Optional[Any] return row +async def evict_config_param(param_name: str) -> None: + await litellm_config_cache.async_delete_cache(_config_cache_key(param_name)) + + async def invalidate_config_param(param_name: str) -> None: """Evict from both cache layers; call after every LiteLLM_Config write.""" - await litellm_config_cache.async_delete_cache(_config_cache_key(param_name)) + await evict_config_param(param_name) + await publish_config_change(redis_cache=coordination_redis_cache(), object_type=param_name) async def prefetch_config_params(prisma_client: Any, param_names: List[str]) -> None: diff --git a/litellm/repositories/credentials_repository.py b/litellm/repositories/credentials_repository.py index b5a315d233c..8e4b9ac0be7 100644 --- a/litellm/repositories/credentials_repository.py +++ b/litellm/repositories/credentials_repository.py @@ -9,6 +9,7 @@ so reads return the stored values verbatim. from typing import Any, Dict, Optional from litellm.models.credentials import CredentialItem +from litellm.proxy.common_utils.config_sync_pubsub import wrap_table_actions_for_config_sync class CredentialsRepository: @@ -25,7 +26,10 @@ class CredentialsRepository: @property def table(self) -> Any: - return self.prisma_client.db.litellm_credentialstable + return wrap_table_actions_for_config_sync( + actions=self.prisma_client.db.litellm_credentialstable, + table_name="litellm_credentialstable", + ) @staticmethod def _to_model(record: Any) -> Optional[CredentialItem]: diff --git a/litellm/repositories/model_repository.py b/litellm/repositories/model_repository.py index 0da51519964..50a50cc60d6 100644 --- a/litellm/repositories/model_repository.py +++ b/litellm/repositories/model_repository.py @@ -7,6 +7,7 @@ from typing import Any, Dict, List, Optional, Type from litellm.models.model import LiteLLM_ProxyModelTable from litellm.repositories.base_repository import BaseRepository +from litellm.proxy.common_utils.config_sync_pubsub import wrap_table_actions_for_config_sync from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, @@ -22,7 +23,10 @@ class ModelRepository(BaseRepository[LiteLLM_ProxyModelTable]): @property def table(self) -> Any: - return self.prisma_client.db.litellm_proxymodeltable + return wrap_table_actions_for_config_sync( + actions=self.prisma_client.db.litellm_proxymodeltable, + table_name="litellm_proxymodeltable", + ) @property def model_class(self) -> Type[LiteLLM_ProxyModelTable]: diff --git a/litellm/repositories/table_repositories.py b/litellm/repositories/table_repositories.py index 54008c0950c..af8be986831 100644 --- a/litellm/repositories/table_repositories.py +++ b/litellm/repositories/table_repositories.py @@ -9,6 +9,8 @@ methods; richer repositories live in their own modules. from typing import Any +from litellm.proxy.common_utils.config_sync_pubsub import wrap_table_actions_for_config_sync + class PrismaTableRepository: """Base for repositories that expose a single Prisma table.""" @@ -26,7 +28,10 @@ class PrismaTableRepository: @property def table(self) -> Any: - return getattr(self.prisma_client.db, self.table_name) + return wrap_table_actions_for_config_sync( + actions=getattr(self.prisma_client.db, self.table_name), + table_name=self.table_name, + ) class PolicyRepository(PrismaTableRepository): diff --git a/ruff.toml b/ruff.toml index 2ea9d7260fb..b652e206f41 100644 --- a/ruff.toml +++ b/ruff.toml @@ -6,7 +6,7 @@ lint.extend-select = ["T20", "PGH004", "RUF008", "RUF009", "RUF100"] # litellm's own ruff config both rely on suppressions this config can't see. lint.external = [ # Enforced by the strict-rule gate (scripts/ruff_strict_gate.py + ruff-strict.toml) - "C901", + "C901", "TID251", # Enforced by upstream litellm's ruff config, but not run in this repo's CI "PLC0415", "E402", "BLE001", "ARG002", "S102", "S324", "S606", "D401", "F403", "F405", ] diff --git a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py new file mode 100644 index 00000000000..8a8ced8bc41 --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py @@ -0,0 +1,600 @@ +import asyncio +import json +import random +from typing import Callable, Coroutine, Iterable, List, Optional, Tuple +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from redis.asyncio import Redis + +import litellm +from litellm.proxy.common_utils.config_sync_pubsub import ( + CONFIG_SYNC_CHANNEL, + CONFIG_SYNC_JITTER_MAX_SECONDS, + ConfigSyncSubscriber, + _CONFIG_SYNCED_TABLE_NAMES, + _PublishOnWriteActions, + _WRITE_ACTION_NAMES, + publish_config_change, + wrap_table_actions_for_config_sync, +) + +_EXPECTED_WRITE_ACTION_NAMES = ( + "create", + "create_many", + "delete", + "delete_many", + "update", + "update_many", + "upsert", +) + +_EXPECTED_CONFIG_SYNCED_TABLE_NAMES = frozenset( + { + "litellm_agentstable", + "litellm_cacheconfig", + "litellm_configoverrides", + "litellm_credentialstable", + "litellm_guardrailstable", + "litellm_managedvectorstoreindextable", + "litellm_managedvectorstorestable", + "litellm_mcpservertable", + "litellm_policyattachmenttable", + "litellm_policytable", + "litellm_prompttable", + "litellm_proxymodeltable", + "litellm_searchtoolstable", + "litellm_ssoconfig", + } +) + + +class _RecordingRedisClient(Redis): + def __init__(self) -> None: + self.published: List[Tuple[str, str]] = [] + + async def publish(self, channel: str, message: str) -> int: + self.published.append((channel, message)) + return 1 + + +class _FailingPublishRedisClient(Redis): + def __init__(self) -> None: + pass + + async def publish(self, channel: str, message: str) -> int: + raise ConnectionError("redis down") + + +class _NotRedisClient: + def __init__(self) -> None: + self.published: List[Tuple[str, str]] = [] + + async def publish(self, channel: str, message: str) -> int: + self.published.append((channel, message)) + return 1 + + +class _QueuePubSub: + def __init__(self, initial_messages: Iterable[str] = ()) -> None: + self.queue: "asyncio.Queue[str]" = asyncio.Queue() + for message in initial_messages: + self.queue.put_nowait(message) + self.subscribed_channels: List[str] = [] + self.closed = False + + async def subscribe(self, *channels: str) -> None: + self.subscribed_channels.extend(channels) + + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[str]: + if timeout == 0: + try: + return self.queue.get_nowait() + except asyncio.QueueEmpty: + return None + try: + return await asyncio.wait_for(self.queue.get(), timeout) + except asyncio.TimeoutError: + return None + + async def aclose(self) -> None: + self.closed = True + + +class _BrokenPubSub(_QueuePubSub): + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[str]: + raise ConnectionError("connection lost") + + +class _ScriptedPubSubRedisClient(Redis): + def __init__(self, pubsubs: Iterable[_QueuePubSub]) -> None: + self._scripted_pubsubs = iter(pubsubs) + + def pubsub(self) -> _QueuePubSub: + return next(self._scripted_pubsubs) + + +class _FakeRedisCache: + def __init__(self, client: object, namespace: Optional[str] = None) -> None: + self._client = client + self.namespace = namespace + + def init_async_client(self) -> object: + return self._client + + +class _ExplodingRedisCache: + namespace: Optional[str] = None + + def init_async_client(self) -> object: + raise ConnectionError("cannot connect") + + +def _recording_callback( + events: List[str], name: str, fired: asyncio.Event +) -> Callable[[], Coroutine[None, None, None]]: + async def callback() -> None: + events.append(name) + fired.set() + + return callback + + +async def test_publish_noops_when_redis_cache_is_none() -> None: + await publish_config_change(redis_cache=None, object_type="litellm_proxymodeltable") + + +async def test_publish_sends_object_type_json_on_channel() -> None: + client = _RecordingRedisClient() + cache = _FakeRedisCache(client) + + await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable") + + assert len(client.published) == 1 + channel, message = client.published[0] + assert channel == "litellm_proxy.config_change" + assert json.loads(message) == {"object_type": "litellm_proxymodeltable"} + + +async def test_publish_uses_namespaced_channel() -> None: + client = _RecordingRedisClient() + cache = _FakeRedisCache(client, namespace="prod-eu") + + await publish_config_change(redis_cache=cache, object_type="litellm_credentialstable") + + assert client.published[0][0] == "prod-eu:litellm_proxy.config_change" + + +async def test_publish_swallows_redis_publish_errors() -> None: + cache = _FakeRedisCache(_FailingPublishRedisClient()) + + await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable") + + +async def test_publish_swallows_client_init_errors() -> None: + await publish_config_change(redis_cache=_ExplodingRedisCache(), object_type="litellm_proxymodeltable") + + +async def test_publish_skips_clients_without_pubsub_support() -> None: + client = _NotRedisClient() + cache = _FakeRedisCache(client) + + await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable") + + assert client.published == [] + + +async def test_subscriber_runs_injected_callbacks_in_order_on_message() -> None: + pubsub = _QueuePubSub() + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + events: List[str] = [] + fired = asyncio.Event() + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=( + _recording_callback(events, "add_deployment", asyncio.Event()), + _recording_callback(events, "get_credentials", fired), + ), + debounce_seconds=0.01, + jitter_max_seconds=0.0, + ) + + subscriber.start() + pubsub.queue.put_nowait(json.dumps({"object_type": "litellm_proxymodeltable"})) + await asyncio.wait_for(fired.wait(), timeout=5) + await subscriber.stop() + + assert events == ["add_deployment", "get_credentials"] + assert pubsub.subscribed_channels == [CONFIG_SYNC_CHANNEL] + assert pubsub.closed is True + + +async def test_burst_within_debounce_window_coalesces_into_one_resync() -> None: + burst = [json.dumps({"object_type": "litellm_proxymodeltable"}) for _ in range(5)] + pubsub = _QueuePubSub(initial_messages=burst) + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + resyncs: List[str] = [] + fired = asyncio.Event() + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(_recording_callback(resyncs, "resync", fired),), + debounce_seconds=0.05, + jitter_max_seconds=0.0, + ) + + subscriber.start() + await asyncio.wait_for(fired.wait(), timeout=5) + await asyncio.sleep(0.3) + await subscriber.stop() + + assert resyncs == ["resync"] + assert pubsub.queue.empty() + + +async def test_subscriber_subscribes_on_namespaced_channel_and_resyncs() -> None: + pubsub = _QueuePubSub() + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub]), namespace="prod-eu") + resyncs: List[str] = [] + fired = asyncio.Event() + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(_recording_callback(resyncs, "resync", fired),), + debounce_seconds=0.01, + jitter_max_seconds=0.0, + ) + + subscriber.start() + pubsub.queue.put_nowait(json.dumps({"object_type": "litellm_proxymodeltable"})) + await asyncio.wait_for(fired.wait(), timeout=5) + await subscriber.stop() + + assert pubsub.subscribed_channels == ["prod-eu:litellm_proxy.config_change"] + assert resyncs == ["resync"] + + +class _MaxJitterRandom(random.Random): + def uniform(self, a: float, b: float) -> float: + return b + + +async def test_debounce_sleep_adds_jitter_from_injected_rng() -> None: + pubsub = _QueuePubSub(initial_messages=[json.dumps({"object_type": "litellm_proxymodeltable"})]) + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + sleeps: List[float] = [] + fired = asyncio.Event() + + async def recording_sleep(seconds: float) -> None: + sleeps.append(seconds) + + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(_recording_callback([], "resync", fired),), + debounce_seconds=1.0, + jitter_max_seconds=4.0, + rng=_MaxJitterRandom(), + sleep=recording_sleep, + ) + + subscriber.start() + await asyncio.wait_for(fired.wait(), timeout=5) + await subscriber.stop() + + assert sleeps == [5.0] + + +def test_default_jitter_window_is_nonzero() -> None: + assert CONFIG_SYNC_JITTER_MAX_SECONDS > 0 + + +async def test_redis_error_leads_to_backoff_and_resubscribe() -> None: + broken = _BrokenPubSub() + healthy = _QueuePubSub(initial_messages=[json.dumps({"object_type": "litellm_credentialstable"})]) + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([broken, healthy])) + resyncs: List[str] = [] + fired = asyncio.Event() + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(_recording_callback(resyncs, "resync", fired),), + debounce_seconds=0.01, + jitter_max_seconds=0.0, + backoff_initial_seconds=0.02, + backoff_max_seconds=0.05, + ) + + subscriber.start() + await asyncio.wait_for(fired.wait(), timeout=5) + task = subscriber._task + assert task is not None + assert task.done() is False + await subscriber.stop() + + assert broken.subscribed_channels == [CONFIG_SYNC_CHANNEL] + assert broken.closed is True + assert healthy.subscribed_channels == [CONFIG_SYNC_CHANNEL] + assert resyncs == ["resync"] + + +async def test_failing_resync_callback_does_not_kill_subscriber() -> None: + pubsub = _QueuePubSub() + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + resyncs: List[str] = [] + fired = asyncio.Event() + + async def failing_callback() -> None: + raise RuntimeError("resync exploded") + + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(failing_callback, _recording_callback(resyncs, "resync", fired)), + debounce_seconds=0.01, + jitter_max_seconds=0.0, + ) + + subscriber.start() + pubsub.queue.put_nowait("change") + await asyncio.wait_for(fired.wait(), timeout=5) + fired.clear() + pubsub.queue.put_nowait("change") + await asyncio.wait_for(fired.wait(), timeout=5) + await subscriber.stop() + + assert resyncs == ["resync", "resync"] + + +async def test_stop_cancels_subscriber_cleanly() -> None: + pubsub = _QueuePubSub() + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + subscriber = ConfigSyncSubscriber(redis_cache=cache, resync_callbacks=(), debounce_seconds=0.01) + + subscriber.start() + await asyncio.sleep(0.05) + task = subscriber._task + assert task is not None + await subscriber.stop() + + assert task.done() is True + assert subscriber._task is None + assert pubsub.closed is True + await subscriber.stop() + + +async def test_stop_before_start_is_a_noop() -> None: + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([])) + subscriber = ConfigSyncSubscriber(redis_cache=cache, resync_callbacks=()) + + await subscriber.stop() + + +async def test_subscriber_exits_without_callbacks_when_client_lacks_pubsub() -> None: + cache = _FakeRedisCache(_NotRedisClient()) + resyncs: List[str] = [] + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(_recording_callback(resyncs, "resync", asyncio.Event()),), + ) + + subscriber.start() + task = subscriber._task + assert task is not None + await asyncio.wait_for(task, timeout=5) + + assert resyncs == [] + + +class _FakeTableActions: + def __init__(self, calls: List[Tuple[str, str]]) -> None: + self._calls = calls + + async def create(self, **kwargs: object) -> object: + self._calls.append(("write", "create")) + return {"id": "m-1"} + + async def find_many(self, **kwargs: object) -> object: + self._calls.append(("read", "find_many")) + return [] + + +class _AllWritesTableActions: + def __init__(self, calls: List[str]) -> None: + self._calls = calls + + def __getattr__(self, name: str) -> Callable[..., Coroutine[None, None, str]]: + async def action(*args: object, **kwargs: object) -> str: + self._calls.append(name) + return name + + return action + + +def _recording_publish(calls: List[Tuple[str, str]]) -> Callable[[str], Coroutine[None, None, None]]: + async def publish(object_type: str) -> None: + calls.append(("publish", object_type)) + + return publish + + +def test_wrapper_passes_through_unsynced_tables() -> None: + actions = object() + + wrapped = wrap_table_actions_for_config_sync(actions=actions, table_name="litellm_spendlogs") + + assert wrapped is actions + + +async def test_wrapper_publishes_table_name_after_write() -> None: + calls: List[Tuple[str, str]] = [] + wrapped = wrap_table_actions_for_config_sync( + actions=_FakeTableActions(calls), + table_name="litellm_proxymodeltable", + publish=_recording_publish(calls), + ) + + result = await wrapped.create(data={"model_name": "gpt-5.2"}) + + assert result == {"id": "m-1"} + assert calls == [("write", "create"), ("publish", "litellm_proxymodeltable")] + + +async def test_wrapper_does_not_publish_on_reads() -> None: + calls: List[Tuple[str, str]] = [] + wrapped = wrap_table_actions_for_config_sync( + actions=_FakeTableActions(calls), + table_name="litellm_proxymodeltable", + publish=_recording_publish(calls), + ) + + result = await wrapped.find_many(where={}) + + assert result == [] + assert calls == [("read", "find_many")] + + +def test_write_action_names_are_pinned() -> None: + assert _WRITE_ACTION_NAMES == frozenset(_EXPECTED_WRITE_ACTION_NAMES) + + +def test_config_synced_table_membership_is_pinned() -> None: + assert _CONFIG_SYNCED_TABLE_NAMES == _EXPECTED_CONFIG_SYNCED_TABLE_NAMES + + +def test_tool_telemetry_table_writes_pass_through_unwrapped() -> None: + actions = object() + + wrapped = wrap_table_actions_for_config_sync(actions=actions, table_name="litellm_tooltable") + + assert wrapped is actions + + +@pytest.mark.parametrize("action_name", _EXPECTED_WRITE_ACTION_NAMES) +async def test_wrapper_publishes_for_every_write_action(action_name: str) -> None: + write_calls: List[str] = [] + publish_calls: List[Tuple[str, str]] = [] + wrapped = wrap_table_actions_for_config_sync( + actions=_AllWritesTableActions(write_calls), + table_name="litellm_guardrailstable", + publish=_recording_publish(publish_calls), + ) + + result = await getattr(wrapped, action_name)(data={}) + + assert result == action_name + assert write_calls == [action_name] + assert publish_calls == [("publish", "litellm_guardrailstable")] + + +async def test_model_repository_write_publishes_via_live_coordination_cache() -> None: + from litellm.proxy import proxy_server + from litellm.proxy.proxy_server import _set_redis_usage_cache + from litellm.repositories.model_repository import ModelRepository + + client = _RecordingRedisClient() + prisma_client = MagicMock() + prisma_client.db.litellm_proxymodeltable.update = AsyncMock(return_value={"model_id": "m-1"}) + repository = ModelRepository(prisma_client) + table = repository.table + assert isinstance(table, _PublishOnWriteActions) + + previous_cache = proxy_server.redis_usage_cache + _set_redis_usage_cache(_FakeRedisCache(client)) + try: + await table.update(where={"model_id": "m-1"}, data={"model_name": "gpt-5.2"}) + finally: + _set_redis_usage_cache(previous_cache) + + prisma_client.db.litellm_proxymodeltable.update.assert_awaited_once_with( + where={"model_id": "m-1"}, data={"model_name": "gpt-5.2"} + ) + assert len(client.published) == 1 + channel, message = client.published[0] + assert channel == CONFIG_SYNC_CHANNEL + assert json.loads(message) == {"object_type": "litellm_proxymodeltable"} + + +async def test_invalidate_config_param_publishes_param_name() -> None: + from litellm.proxy import proxy_server + from litellm.proxy.proxy_server import _set_redis_usage_cache + from litellm.proxy.utils import invalidate_config_param + + client = _RecordingRedisClient() + previous_cache = proxy_server.redis_usage_cache + _set_redis_usage_cache(_FakeRedisCache(client)) + try: + await invalidate_config_param("environment_variables") + finally: + _set_redis_usage_cache(previous_cache) + + assert len(client.published) == 1 + channel, message = client.published[0] + assert channel == CONFIG_SYNC_CHANNEL + assert json.loads(message) == {"object_type": "environment_variables"} + + +async def test_evict_config_param_does_not_publish() -> None: + from litellm.proxy import proxy_server + from litellm.proxy.proxy_server import _set_redis_usage_cache + from litellm.proxy.utils import evict_config_param + + client = _RecordingRedisClient() + previous_cache = proxy_server.redis_usage_cache + _set_redis_usage_cache(_FakeRedisCache(client)) + try: + await evict_config_param("model_cost_map_reload_config") + finally: + _set_redis_usage_cache(previous_cache) + + assert client.published == [] + + +def _reload_config_prisma_client() -> MagicMock: + config_record = MagicMock() + config_record.param_value = {"interval_hours": 6, "force_reload": True} + prisma_client = MagicMock() + prisma_client.get_generic_data = AsyncMock(return_value=config_record) + prisma_client.db.litellm_config.upsert = AsyncMock(return_value=None) + return prisma_client + + +async def test_model_cost_map_reload_does_not_publish_config_change() -> None: + from litellm.proxy import proxy_server + from litellm.proxy.proxy_server import ProxyConfig, _set_redis_usage_cache + from litellm.proxy.utils import litellm_config_cache + from litellm.utils import _invalidate_model_cost_lowercase_map + + litellm_config_cache.flush_cache() + prisma_client = _reload_config_prisma_client() + client = _RecordingRedisClient() + previous_cache = proxy_server.redis_usage_cache + original_model_cost = litellm.model_cost.copy() + _set_redis_usage_cache(_FakeRedisCache(client)) + try: + with patch("litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map") as mock_get_map: + mock_get_map.return_value = {"gpt-5.2": {"input_cost_per_token": 0.001}} + await ProxyConfig()._check_and_reload_model_cost_map(prisma_client=prisma_client) + finally: + litellm.model_cost = original_model_cost + _invalidate_model_cost_lowercase_map() + _set_redis_usage_cache(previous_cache) + + prisma_client.db.litellm_config.upsert.assert_awaited_once() + assert client.published == [] + + +async def test_anthropic_beta_headers_reload_does_not_publish_config_change() -> None: + from litellm.proxy import proxy_server + from litellm.proxy.proxy_server import ProxyConfig, _set_redis_usage_cache + from litellm.proxy.utils import litellm_config_cache + + litellm_config_cache.flush_cache() + prisma_client = _reload_config_prisma_client() + client = _RecordingRedisClient() + previous_cache = proxy_server.redis_usage_cache + _set_redis_usage_cache(_FakeRedisCache(client)) + try: + with patch("litellm.anthropic_beta_headers_manager.reload_beta_headers_config") as mock_reload: + mock_reload.return_value = {} + await ProxyConfig()._check_and_reload_anthropic_beta_headers(prisma_client=prisma_client) + finally: + _set_redis_usage_cache(previous_cache) + + prisma_client.db.litellm_config.upsert.assert_awaited_once() + assert client.published == [] diff --git a/tests/test_litellm/repositories/test_repositories.py b/tests/test_litellm/repositories/test_repositories.py index c923b722991..994e73d33f5 100644 --- a/tests/test_litellm/repositories/test_repositories.py +++ b/tests/test_litellm/repositories/test_repositories.py @@ -307,6 +307,15 @@ class TestModelRepository: client = MockPrismaClient() return ModelRepository(client) + def test_table_is_wrapped_for_config_sync(self, repo): + from litellm.proxy.common_utils.config_sync_pubsub import ( + _PublishOnWriteActions, + ) + + table = repo.table + assert isinstance(table, _PublishOnWriteActions) + assert table._actions is repo.prisma_client.db.litellm_proxymodeltable + @pytest.mark.asyncio @patch( "litellm.repositories.model_repository.encrypt_value_helper", @@ -1313,6 +1322,15 @@ class TestCredentialsRepository: client = MockPrismaClient() return CredentialsRepository(client) + def test_table_is_wrapped_for_config_sync(self, repo): + from litellm.proxy.common_utils.config_sync_pubsub import ( + _PublishOnWriteActions, + ) + + table = repo.table + assert isinstance(table, _PublishOnWriteActions) + assert table._actions is repo.prisma_client.db.litellm_credentialstable + @pytest.mark.asyncio async def test_create(self, repo): record = await repo.create( @@ -2188,6 +2206,9 @@ class TestConfigRepositoryDeepCopy: class TestPrismaTableRepository: def test_table_property_returns_named_delegate(self): + from litellm.proxy.common_utils.config_sync_pubsub import ( + _PublishOnWriteActions, + ) from litellm.repositories.table_repositories import ( AgentsRepository, PolicyRepository, @@ -2197,9 +2218,11 @@ class TestPrismaTableRepository: agents = AgentsRepository(prisma_client) policy = PolicyRepository(prisma_client) - assert agents.table is prisma_client.db.litellm_agentstable - assert policy.table is prisma_client.db.litellm_policytable - assert agents.table is not policy.table + assert isinstance(agents.table, _PublishOnWriteActions) + assert isinstance(policy.table, _PublishOnWriteActions) + assert agents.table._actions is prisma_client.db.litellm_agentstable + assert policy.table._actions is prisma_client.db.litellm_policytable + assert agents.table._actions is not policy.table._actions def test_table_access_raises_without_db(self): from litellm.repositories.table_repositories import SpendLogsRepository @@ -2208,8 +2231,28 @@ class TestPrismaTableRepository: with pytest.raises(RuntimeError, match="No DB Connected"): _ = repo.table + CONFIG_SYNCED_TABLE_NAMES = frozenset( + { + "litellm_agentstable", + "litellm_cacheconfig", + "litellm_configoverrides", + "litellm_guardrailstable", + "litellm_managedvectorstoreindextable", + "litellm_managedvectorstorestable", + "litellm_mcpservertable", + "litellm_policyattachmenttable", + "litellm_policytable", + "litellm_prompttable", + "litellm_searchtoolstable", + "litellm_ssoconfig", + } + ) + def test_each_repository_binds_its_own_table_name(self): import litellm.repositories.table_repositories as tr + from litellm.proxy.common_utils.config_sync_pubsub import ( + _PublishOnWriteActions, + ) prisma_client = MagicMock() repos = [ @@ -2226,7 +2269,14 @@ class TestPrismaTableRepository: assert name.startswith("litellm_") assert name not in seen, f"duplicate table_name {name}" seen.add(name) - assert repo_cls(prisma_client).table is getattr(prisma_client.db, name) + table = repo_cls(prisma_client).table + raw_actions = getattr(prisma_client.db, name) + if name in self.CONFIG_SYNCED_TABLE_NAMES: + assert isinstance(table, _PublishOnWriteActions), name + assert table._actions is raw_actions + else: + assert table is raw_actions, name + assert self.CONFIG_SYNCED_TABLE_NAMES <= seen def _json_path_equals( From a8018f7500fe5ed801803eada2fbe1c219f30af9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 31 Jul 2026 21:08:32 -0700 Subject: [PATCH 30/50] fix(proxy): throttle pub/sub resyncs and stop publishing startup-only config params Caps fleet-wide reload rate at one resync per 10s per pod so a burst of authenticated writes cannot amplify into continuous cross-pod reloads, and skips publishing config params (environment_variables, router_settings) that no resync callback applies outside proxy startup --- .../proxy/common_utils/config_sync_pubsub.py | 43 +++ litellm/proxy/proxy_server.py | 56 ++-- litellm/proxy/utils.py | 7 +- .../common_utils/test_config_sync_pubsub.py | 302 +++++++++++++++++- 4 files changed, 380 insertions(+), 28 deletions(-) diff --git a/litellm/proxy/common_utils/config_sync_pubsub.py b/litellm/proxy/common_utils/config_sync_pubsub.py index 4292c67c8c2..e4521c397b7 100644 --- a/litellm/proxy/common_utils/config_sync_pubsub.py +++ b/litellm/proxy/common_utils/config_sync_pubsub.py @@ -1,6 +1,7 @@ import asyncio import json import random +import time from collections.abc import Awaitable, Callable from dataclasses import asdict, dataclass from typing import TYPE_CHECKING, Protocol, cast # noqa: TID251 # untyped prisma/redis boundary needs cast @@ -28,6 +29,7 @@ class _ConfigSyncPubSubClient(Protocol): CONFIG_SYNC_CHANNEL = "litellm_proxy.config_change" CONFIG_SYNC_DEBOUNCE_SECONDS = 1.0 CONFIG_SYNC_JITTER_MAX_SECONDS = 5.0 +CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS = 10.0 _POLL_TIMEOUT_SECONDS = 1.0 _BACKOFF_INITIAL_SECONDS = 5.0 _BACKOFF_MAX_SECONDS = 60.0 @@ -55,6 +57,15 @@ _CONFIG_SYNCED_TABLE_NAMES: frozenset[str] = frozenset( } ) +_RESYNC_APPLIED_CONFIG_PARAM_NAMES: frozenset[str] = frozenset( + { + "general_settings", + "litellm_settings", + "model_cost_map_reload_config", + "anthropic_beta_headers_reload_config", + } +) + def coordination_redis_cache() -> "RedisCache | None": from litellm.proxy.proxy_server import redis_usage_cache @@ -113,6 +124,16 @@ async def publish_config_change_for_object_type(object_type: str) -> None: await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type) +async def publish_config_param_change(param_name: str) -> None: + if param_name not in _RESYNC_APPLIED_CONFIG_PARAM_NAMES: + verbose_proxy_logger.debug( + "config sync publish for %s skipped: no resync callback applies this param outside proxy startup", + param_name, + ) + return + await publish_config_change_for_object_type(param_name) + + class _PublishOnWriteActions: __slots__ = ("_actions", "_object_type", "_publish") @@ -156,6 +177,9 @@ class ConfigSyncSubscriber: "_backoff_max_seconds", "_debounce_seconds", "_jitter_max_seconds", + "_last_resync_at", + "_min_resync_interval_seconds", + "_monotonic", "_redis_cache", "_resync_callbacks", "_rng", @@ -169,20 +193,25 @@ class ConfigSyncSubscriber: resync_callbacks: tuple[Callable[[], Awaitable[None]], ...], debounce_seconds: float = CONFIG_SYNC_DEBOUNCE_SECONDS, jitter_max_seconds: float = CONFIG_SYNC_JITTER_MAX_SECONDS, + min_resync_interval_seconds: float = CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS, backoff_initial_seconds: float = _BACKOFF_INITIAL_SECONDS, backoff_max_seconds: float = _BACKOFF_MAX_SECONDS, rng: random.Random | None = None, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + monotonic: Callable[[], float] = time.monotonic, ) -> None: self._redis_cache = redis_cache self._resync_callbacks = resync_callbacks self._debounce_seconds = debounce_seconds self._jitter_max_seconds = jitter_max_seconds + self._min_resync_interval_seconds = min_resync_interval_seconds self._backoff_initial_seconds = backoff_initial_seconds self._backoff_max_seconds = backoff_max_seconds self._rng = rng if rng is not None else random.Random() self._sleep = sleep + self._monotonic = monotonic self._task: asyncio.Task[None] | None = None + self._last_resync_at: float | None = None def start(self) -> None: if self._task is not None: @@ -235,8 +264,22 @@ class ConfigSyncSubscriber: if message is None: continue await self._sleep(self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds)) + await self._wait_for_min_resync_interval() await self._drain_pending(pubsub) await self._run_resync_callbacks() + self._last_resync_at = self._monotonic() + + async def _wait_for_min_resync_interval(self) -> None: + if self._last_resync_at is None: + return + seconds_until_next_resync = self._min_resync_interval_seconds - (self._monotonic() - self._last_resync_at) + if seconds_until_next_resync <= 0: + return + verbose_proxy_logger.debug( + "config sync resync throttled for %.1fs to cap fleet-wide reload rate", + seconds_until_next_resync, + ) + await self._sleep(seconds_until_next_resync) @staticmethod async def _drain_pending(pubsub: _ConfigSyncPubSub) -> None: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6bb0032b38c..7d3e58bfe52 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1153,11 +1153,7 @@ async def proxy_startup_event(app: FastAPI): except Exception as e: verbose_proxy_logger.error(f"Error stopping DB health watchdog task: {e}") - if proxy_config.config_sync_subscriber is not None: - try: - await proxy_config.config_sync_subscriber.stop() - except Exception as e: - verbose_proxy_logger.error(f"Error stopping config sync subscriber: {e}") + await proxy_config.stop_config_sync_subscriber() await proxy_shutdown_event() # type: ignore[reportGeneralTypeIssues] @@ -6213,6 +6209,38 @@ class ProxyConfig: "litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - {}".format(str(e)) ) + def start_config_sync_subscriber( + self, + prisma_client: PrismaClient, + proxy_logging_obj: ProxyLogging, + redis_cache: Optional[RedisCache], + ) -> None: + if redis_cache is None or self.config_sync_subscriber is not None: + return + + async def _resync_config_from_db() -> None: + await self.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) + + async def _resync_credentials_from_db() -> None: + await self.get_credentials(prisma_client=prisma_client) + + subscriber = ConfigSyncSubscriber( + redis_cache=redis_cache, + resync_callbacks=(_resync_config_from_db, _resync_credentials_from_db), + ) + self.config_sync_subscriber = subscriber + subscriber.start() + + async def stop_config_sync_subscriber(self) -> None: + subscriber = self.config_sync_subscriber + if subscriber is None: + return + self.config_sync_subscriber = None + try: + await subscriber.stop() + except Exception as e: + verbose_proxy_logger.error(f"Error stopping config sync subscriber: {e}") + async def _init_non_llm_objects_in_db(self, prisma_client: PrismaClient): """ Use this to read non-llm objects from the db and initialize them @@ -8174,19 +8202,11 @@ class ProxyStartupEvent: ) await proxy_config.get_credentials(prisma_client=prisma_client) - if redis_usage_cache is not None and proxy_config.config_sync_subscriber is None: - - async def _resync_config_from_db() -> None: - await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) - - async def _resync_credentials_from_db() -> None: - await proxy_config.get_credentials(prisma_client=prisma_client) - - proxy_config.config_sync_subscriber = ConfigSyncSubscriber( - redis_cache=redis_usage_cache, - resync_callbacks=(_resync_config_from_db, _resync_credentials_from_db), - ) - proxy_config.config_sync_subscriber.start() + proxy_config.start_config_sync_subscriber( + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + redis_cache=redis_usage_cache, + ) if store_model_in_db is not True: await proxy_config.init_mcp_servers_from_db() diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index fbcca73e779..5d2d29efa12 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -119,10 +119,7 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.route_checks import RouteChecks -from litellm.proxy.common_utils.config_sync_pubsub import ( - coordination_redis_cache, - publish_config_change, -) +from litellm.proxy.common_utils.config_sync_pubsub import publish_config_param_change from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.db.create_views import ( create_missing_views, @@ -2982,7 +2979,7 @@ async def evict_config_param(param_name: str) -> None: async def invalidate_config_param(param_name: str) -> None: """Evict from both cache layers; call after every LiteLLM_Config write.""" await evict_config_param(param_name) - await publish_config_change(redis_cache=coordination_redis_cache(), object_type=param_name) + await publish_config_param_change(param_name) async def prefetch_config_params(prisma_client: Any, param_names: List[str]) -> None: diff --git a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py index 8a8ced8bc41..f50eef4f1cc 100644 --- a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py +++ b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py @@ -11,9 +11,11 @@ import litellm from litellm.proxy.common_utils.config_sync_pubsub import ( CONFIG_SYNC_CHANNEL, CONFIG_SYNC_JITTER_MAX_SECONDS, + CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS, ConfigSyncSubscriber, _CONFIG_SYNCED_TABLE_NAMES, _PublishOnWriteActions, + _RESYNC_APPLIED_CONFIG_PARAM_NAMES, _WRITE_ACTION_NAMES, publish_config_change, wrap_table_actions_for_config_sync, @@ -48,6 +50,17 @@ _EXPECTED_CONFIG_SYNCED_TABLE_NAMES = frozenset( } ) +_EXPECTED_RESYNC_APPLIED_CONFIG_PARAM_NAMES = frozenset( + { + "anthropic_beta_headers_reload_config", + "general_settings", + "litellm_settings", + "model_cost_map_reload_config", + } +) + +_STARTUP_ONLY_CONFIG_PARAM_NAMES = ("environment_variables", "router_settings") + class _RecordingRedisClient(Redis): def __init__(self) -> None: @@ -106,6 +119,31 @@ class _BrokenPubSub(_QueuePubSub): raise ConnectionError("connection lost") +class _CloseFailingBrokenPubSub(_BrokenPubSub): + async def aclose(self) -> None: + raise ConnectionError("close failed") + + +class _EmptyPollsThenMessagePubSub(_QueuePubSub): + def __init__(self, empty_polls: int, initial_messages: Iterable[str] = ()) -> None: + super().__init__(initial_messages=initial_messages) + self.remaining_empty_polls = empty_polls + + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[str]: + if timeout != 0 and self.remaining_empty_polls > 0: + self.remaining_empty_polls -= 1 + return None + return await super().get_message(ignore_subscribe_messages=ignore_subscribe_messages, timeout=timeout) + + +class _FakeClock: + def __init__(self, now: float = 1000.0) -> None: + self.now = now + + def __call__(self) -> float: + return self.now + + class _ScriptedPubSubRedisClient(Redis): def __init__(self, pubsubs: Iterable[_QueuePubSub]) -> None: self._scripted_pubsubs = iter(pubsubs) @@ -286,6 +324,158 @@ def test_default_jitter_window_is_nonzero() -> None: assert CONFIG_SYNC_JITTER_MAX_SECONDS > 0 +def test_default_min_resync_interval_caps_reload_rate() -> None: + assert CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS > CONFIG_SYNC_JITTER_MAX_SECONDS + + +def _throttled_subscriber( + cache: object, + events: List[str], + fired: asyncio.Event, + clock: _FakeClock, + min_resync_interval_seconds: float = 10.0, +) -> ConfigSyncSubscriber: + async def recording_sleep(seconds: float) -> None: + events.append(f"sleep:{seconds}") + await asyncio.sleep(0) + + async def resync() -> None: + events.append("resync") + fired.set() + + return ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(resync,), + debounce_seconds=0.0, + jitter_max_seconds=0.0, + min_resync_interval_seconds=min_resync_interval_seconds, + sleep=recording_sleep, + monotonic=clock, + ) + + +async def test_resync_arriving_inside_min_interval_waits_out_the_remainder() -> None: + pubsub = _QueuePubSub() + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + events: List[str] = [] + fired = asyncio.Event() + clock = _FakeClock() + subscriber = _throttled_subscriber(cache=cache, events=events, fired=fired, clock=clock) + + subscriber.start() + pubsub.queue.put_nowait("change") + await asyncio.wait_for(fired.wait(), timeout=5) + fired.clear() + clock.now += 4.0 + pubsub.queue.put_nowait("change") + await asyncio.wait_for(fired.wait(), timeout=5) + await subscriber.stop() + + assert events == ["sleep:0.0", "resync", "sleep:0.0", "sleep:6.0", "resync"] + + +async def test_resync_after_min_interval_elapsed_is_not_throttled() -> None: + pubsub = _QueuePubSub() + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + events: List[str] = [] + fired = asyncio.Event() + clock = _FakeClock() + subscriber = _throttled_subscriber(cache=cache, events=events, fired=fired, clock=clock) + + subscriber.start() + pubsub.queue.put_nowait("change") + await asyncio.wait_for(fired.wait(), timeout=5) + fired.clear() + clock.now += 30.0 + pubsub.queue.put_nowait("change") + await asyncio.wait_for(fired.wait(), timeout=5) + await subscriber.stop() + + assert events == ["sleep:0.0", "resync", "sleep:0.0", "resync"] + + +async def test_writes_during_the_throttle_wait_collapse_into_the_next_resync() -> None: + pubsub = _QueuePubSub() + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + events: List[str] = [] + fired = asyncio.Event() + clock = _FakeClock() + subscriber = _throttled_subscriber(cache=cache, events=events, fired=fired, clock=clock) + + subscriber.start() + pubsub.queue.put_nowait("change") + await asyncio.wait_for(fired.wait(), timeout=5) + fired.clear() + for _ in range(5): + pubsub.queue.put_nowait("change") + await asyncio.wait_for(fired.wait(), timeout=5) + await asyncio.sleep(0.1) + await subscriber.stop() + + assert events.count("resync") == 2 + assert pubsub.queue.empty() + + +async def test_polls_without_messages_do_not_trigger_resyncs() -> None: + pubsub = _EmptyPollsThenMessagePubSub(empty_polls=3, initial_messages=["change"]) + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + resyncs: List[str] = [] + fired = asyncio.Event() + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(_recording_callback(resyncs, "resync", fired),), + debounce_seconds=0.01, + jitter_max_seconds=0.0, + ) + + subscriber.start() + await asyncio.wait_for(fired.wait(), timeout=5) + await asyncio.sleep(0.1) + await subscriber.stop() + + assert pubsub.remaining_empty_polls == 0 + assert resyncs == ["resync"] + + +async def test_failing_pubsub_close_still_reconnects() -> None: + broken = _CloseFailingBrokenPubSub() + healthy = _QueuePubSub(initial_messages=["change"]) + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([broken, healthy])) + resyncs: List[str] = [] + fired = asyncio.Event() + subscriber = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(_recording_callback(resyncs, "resync", fired),), + debounce_seconds=0.01, + jitter_max_seconds=0.0, + backoff_initial_seconds=0.02, + backoff_max_seconds=0.05, + ) + + subscriber.start() + await asyncio.wait_for(fired.wait(), timeout=5) + await subscriber.stop() + + assert healthy.subscribed_channels == [CONFIG_SYNC_CHANNEL] + assert resyncs == ["resync"] + + +async def test_second_start_does_not_open_a_second_subscription() -> None: + pubsub = _QueuePubSub() + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) + subscriber = ConfigSyncSubscriber(redis_cache=cache, resync_callbacks=(), debounce_seconds=0.01) + + subscriber.start() + task = subscriber._task + subscriber.start() + assert task is not None + assert subscriber._task is task + await asyncio.sleep(0.05) + await subscriber.stop() + + assert pubsub.subscribed_channels == [CONFIG_SYNC_CHANNEL] + + async def test_redis_error_leads_to_backoff_and_resubscribe() -> None: broken = _BrokenPubSub() healthy = _QueuePubSub(initial_messages=[json.dumps({"object_type": "litellm_credentialstable"})]) @@ -328,6 +518,7 @@ async def test_failing_resync_callback_does_not_kill_subscriber() -> None: resync_callbacks=(failing_callback, _recording_callback(resyncs, "resync", fired)), debounce_seconds=0.01, jitter_max_seconds=0.0, + min_resync_interval_seconds=0.0, ) subscriber.start() @@ -510,7 +701,7 @@ async def test_model_repository_write_publishes_via_live_coordination_cache() -> assert json.loads(message) == {"object_type": "litellm_proxymodeltable"} -async def test_invalidate_config_param_publishes_param_name() -> None: +async def _publish_calls_for_invalidated_param(param_name: str) -> List[Tuple[str, str]]: from litellm.proxy import proxy_server from litellm.proxy.proxy_server import _set_redis_usage_cache from litellm.proxy.utils import invalidate_config_param @@ -519,14 +710,30 @@ async def test_invalidate_config_param_publishes_param_name() -> None: previous_cache = proxy_server.redis_usage_cache _set_redis_usage_cache(_FakeRedisCache(client)) try: - await invalidate_config_param("environment_variables") + await invalidate_config_param(param_name) finally: _set_redis_usage_cache(previous_cache) + return client.published - assert len(client.published) == 1 - channel, message = client.published[0] + +async def test_invalidate_config_param_publishes_params_a_resync_applies() -> None: + published = await _publish_calls_for_invalidated_param("general_settings") + + assert len(published) == 1 + channel, message = published[0] assert channel == CONFIG_SYNC_CHANNEL - assert json.loads(message) == {"object_type": "environment_variables"} + assert json.loads(message) == {"object_type": "general_settings"} + + +@pytest.mark.parametrize("param_name", _STARTUP_ONLY_CONFIG_PARAM_NAMES) +async def test_invalidate_config_param_does_not_publish_startup_only_params(param_name: str) -> None: + published = await _publish_calls_for_invalidated_param(param_name) + + assert published == [] + + +def test_resync_applied_config_param_membership_is_pinned() -> None: + assert _RESYNC_APPLIED_CONFIG_PARAM_NAMES == _EXPECTED_RESYNC_APPLIED_CONFIG_PARAM_NAMES async def test_evict_config_param_does_not_publish() -> None: @@ -598,3 +805,88 @@ async def test_anthropic_beta_headers_reload_does_not_publish_config_change() -> prisma_client.db.litellm_config.upsert.assert_awaited_once() assert client.published == [] + + +class _StopFailingSubscriber(ConfigSyncSubscriber): + async def stop(self) -> None: + raise RuntimeError("stop failed") + + +async def test_proxy_config_subscriber_resyncs_deployments_and_credentials() -> None: + from litellm.proxy.proxy_server import ProxyConfig + + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([_QueuePubSub()])) + config = ProxyConfig() + prisma_client = MagicMock() + proxy_logging_obj = MagicMock() + calls: List[Tuple[str, object, object]] = [] + + async def fake_add_deployment(prisma_client: object, proxy_logging_obj: object) -> None: + calls.append(("add_deployment", prisma_client, proxy_logging_obj)) + + async def fake_get_credentials(prisma_client: object) -> None: + calls.append(("get_credentials", prisma_client, None)) + + config.add_deployment = fake_add_deployment + config.get_credentials = fake_get_credentials + config.start_config_sync_subscriber( + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + redis_cache=cache, + ) + subscriber = config.config_sync_subscriber + assert subscriber is not None + for callback in subscriber._resync_callbacks: + await callback() + await config.stop_config_sync_subscriber() + + assert calls == [ + ("add_deployment", prisma_client, proxy_logging_obj), + ("get_credentials", prisma_client, None), + ] + assert config.config_sync_subscriber is None + assert subscriber._task is None + + +async def test_proxy_config_does_not_start_subscriber_without_coordination_redis() -> None: + from litellm.proxy.proxy_server import ProxyConfig + + config = ProxyConfig() + + config.start_config_sync_subscriber( + prisma_client=MagicMock(), + proxy_logging_obj=MagicMock(), + redis_cache=None, + ) + + assert config.config_sync_subscriber is None + + +async def test_proxy_config_keeps_the_first_subscriber_on_repeat_start() -> None: + from litellm.proxy.proxy_server import ProxyConfig + + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([_QueuePubSub()])) + config = ProxyConfig() + + config.start_config_sync_subscriber(prisma_client=MagicMock(), proxy_logging_obj=MagicMock(), redis_cache=cache) + first = config.config_sync_subscriber + config.start_config_sync_subscriber(prisma_client=MagicMock(), proxy_logging_obj=MagicMock(), redis_cache=cache) + second = config.config_sync_subscriber + await config.stop_config_sync_subscriber() + + assert first is not None + assert second is first + + +async def test_proxy_config_shutdown_survives_a_failing_subscriber_stop() -> None: + from litellm.proxy.proxy_server import ProxyConfig + + config = ProxyConfig() + config.config_sync_subscriber = _StopFailingSubscriber( + redis_cache=_FakeRedisCache(_ScriptedPubSubRedisClient([])), + resync_callbacks=(), + ) + + await config.stop_config_sync_subscriber() + + assert config.config_sync_subscriber is None From 2a13bbe1cb502ba472ad41fecac95363b077c68f Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 22:13:57 -0700 Subject: [PATCH 31/50] refactor(proxy): resolve team member lookups in one query and cap the rejection message Resolve the requested member user_ids with a single find_many instead of one lookup per member, so a large member list no longer turns into that many round-trips before the permission check runs. Write the member-add audit entries concurrently rather than one after another, and list at most a few ids in the rejection message instead of echoing the whole request back. Update the team-admin member-add case that covered adding a user_id with no user row, which the endpoint now leaves to proxy admins. --- .../management_endpoints/team_endpoints.py | 49 +++++++++++----- tests/proxy_unit_tests/test_proxy_server.py | 2 +- .../test_team_endpoints.py | 57 ++++++++++++++++--- 3 files changed, 87 insertions(+), 21 deletions(-) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 7b5ce33cb8b..a8c5fcea4df 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -2431,12 +2431,23 @@ async def _resolve_existing_member_user_ids( members: Sequence[Member], prisma_client: PrismaClient, ) -> frozenset[str]: - """Return the caller-supplied user_ids that already have a user row.""" - user_repository = UserRepository(prisma_client) - found = await asyncio.gather( - *(user_repository.find_by_id(member.user_id) for member in members if member.user_id is not None) + """Return the caller-supplied user_ids that already have a user row. + + Resolved with a single query so the number of members in the request does + not translate into that many concurrent connections. + """ + requested_user_ids = frozenset(member.user_id for member in members if member.user_id is not None) + if not requested_user_ids: + return frozenset() + + found = await UserRepository(prisma_client).table.find_many( + where={ # mutable-ok: Prisma query filters are dict-shaped + "user_id": { # mutable-ok: Prisma query filters are dict-shaped + "in": sorted(requested_user_ids) + } + } ) - return frozenset(user.user_id for user in found if user is not None and user.user_id is not None) + return frozenset(user.user_id for user in found or () if user.user_id is not None) def _pre_existing_user_ids( @@ -2459,6 +2470,9 @@ def _pre_existing_user_ids( return existing_user_ids | populated_user_ids +_MAX_REPORTED_UNKNOWN_USER_IDS = 10 + + def _validate_member_user_id_provisioning( members: Sequence[Member], existing_user_ids: frozenset[str], @@ -2481,13 +2495,15 @@ def _validate_member_user_id_provisioning( if not unknown_user_ids: return + listed = ", ".join(unknown_user_ids[:_MAX_REPORTED_UNKNOWN_USER_IDS]) + remaining = len(unknown_user_ids) - _MAX_REPORTED_UNKNOWN_USER_IDS raise HTTPException( status_code=403, detail={ # mutable-ok: HTTPException detail must be a plain mapping to keep this route's {"error": ...} response shape "error": ( - "Only proxy admins can add a user_id that does not exist yet: {}. " + "Only proxy admins can add a user_id that does not exist yet: {}{}. " "Add the member by user_email to invite a new user, or ask a proxy admin " - "to create the user first.".format(", ".join(unknown_user_ids)) + "to create the user first.".format(listed, " and {} more".format(remaining) if remaining > 0 else "") ) }, ) @@ -2515,13 +2531,15 @@ async def _create_team_member_add_audit_logs( user_api_key_dict: UserAPIKeyAuth, litellm_proxy_admin_name: str, ) -> None: - """Record the membership change, and any user row it created, in the audit log.""" + """Record the membership change, and any user row it created, in the audit log. + + The entries are written concurrently so a request adding many members does + not pay for them one after another. + """ from litellm.proxy.management_helpers.audit_logs import create_object_audit_log - for user in updated_users: - if user.user_id is None or user.user_id in existing_user_ids: - continue - await create_object_audit_log( + created_user_entries = tuple( + create_object_audit_log( object_id=user.user_id, action="created", litellm_changed_by=None, @@ -2531,8 +2549,11 @@ async def _create_team_member_add_audit_logs( before_value=None, after_value=safe_dumps(user.model_dump(exclude_none=True)), ) + for user in updated_users + if user.user_id is not None and user.user_id not in existing_user_ids + ) - await create_object_audit_log( + membership_entry = create_object_audit_log( object_id=team_id, action="updated", litellm_changed_by=None, @@ -2543,6 +2564,8 @@ async def _create_team_member_add_audit_logs( after_value=_members_audit_value(after_members), ) + await asyncio.gather(*created_user_entries, membership_entry) + async def _validate_and_populate_member_user_info( member: Member, diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index bedd4dd1838..f64994cb3b1 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -1475,7 +1475,7 @@ async def test_create_team_member_add_team_admin( user_api_key_dict=valid_token, ) except HTTPException as e: - if user_role == "user": + if user_role == "user" or new_member_method == "user_id": assert e.status_code == 403 return else: diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index b1438447e4f..0de2c1ac71d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -10440,20 +10440,18 @@ def test_validate_member_user_id_provisioning_reports_every_unknown_member(): @pytest.mark.asyncio async def test_resolve_existing_member_user_ids_matches_caller_supplied_user_ids(): - """Only caller-supplied user_ids are looked up; unknown ones resolve to nothing.""" + """Caller-supplied user_ids resolve in one query; unknown ones resolve to nothing.""" from litellm.proxy.management_endpoints.team_endpoints import ( _resolve_existing_member_user_ids, ) prisma_client = MagicMock() - - async def find_by_id(user_id): - if user_id == "by-id": - return LiteLLM_UserTable(user_id="by-id", max_budget=None, spend=0.0, user_email=None, models=[]) - return None + find_many = AsyncMock( + return_value=[LiteLLM_UserTable(user_id="by-id", max_budget=None, spend=0.0, user_email=None, models=[])] + ) with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: - repo.return_value.find_by_id = AsyncMock(side_effect=find_by_id) + repo.return_value.table.find_many = find_many resolved = await _resolve_existing_member_user_ids( members=[ @@ -10465,6 +10463,28 @@ async def test_resolve_existing_member_user_ids_matches_caller_supplied_user_ids ) assert resolved == frozenset({"by-id"}) + # one round-trip, and email-only members contribute no id to look up + find_many.assert_awaited_once() + assert find_many.await_args.kwargs["where"] == {"user_id": {"in": ["by-id", "missing"]}} + + +@pytest.mark.asyncio +async def test_resolve_existing_member_user_ids_skips_the_query_when_no_user_ids(): + """An all-email payload must not hit the database at all.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _resolve_existing_member_user_ids, + ) + + with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: + repo.return_value.table.find_many = AsyncMock() + + resolved = await _resolve_existing_member_user_ids( + members=[Member(user_email="a@example.com", role="user")], + prisma_client=MagicMock(), + ) + + assert resolved == frozenset() + repo.return_value.table.find_many.assert_not_awaited() def test_pre_existing_user_ids_counts_ids_filled_in_by_member_resolution(): @@ -10574,3 +10594,26 @@ async def test_team_member_add_audits_a_user_created_from_a_list_payload(monkeyp mock_audit.assert_called_once() assert created_user_id not in mock_audit.call_args.kwargs["existing_user_ids"] + + +def test_validate_member_user_id_provisioning_caps_the_ids_it_echoes_back(): + """A large member list must not echo every id back in the error body.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _MAX_REPORTED_UNKNOWN_USER_IDS, + _validate_member_user_id_provisioning, + ) + + members = [Member(user_id=f"u{i}", role="user") for i in range(500)] + + with pytest.raises(HTTPException) as exc_info: + _validate_member_user_id_provisioning( + members=members, + existing_user_ids=frozenset(), + user_api_key_dict=_provisioning_caller(LitellmUserRoles.INTERNAL_USER), + ) + + detail = str(exc_info.value.detail) + assert "u0" in detail + assert f"u{_MAX_REPORTED_UNKNOWN_USER_IDS}" not in detail + assert f"and {500 - _MAX_REPORTED_UNKNOWN_USER_IDS} more" in detail + assert len(detail) < 1000 From 77e490a69513c44fa2157d41d8a69ed5333b529f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 31 Jul 2026 22:52:00 -0700 Subject: [PATCH 32/50] fix(proxy): publish router_settings changes so peer pods apply them on resync add_deployment already reapplies DB router settings through _update_llm_router, so gating router_settings out of the pub/sub publish set left the push path covering less than the resync actually applies --- .../proxy/common_utils/config_sync_pubsub.py | 1 + .../common_utils/test_config_sync_pubsub.py | 10 +++--- .../proxy/proxy_server/test_proxy_config.py | 36 +++++++++++++++++++ 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/common_utils/config_sync_pubsub.py b/litellm/proxy/common_utils/config_sync_pubsub.py index e4521c397b7..76c3066ae83 100644 --- a/litellm/proxy/common_utils/config_sync_pubsub.py +++ b/litellm/proxy/common_utils/config_sync_pubsub.py @@ -60,6 +60,7 @@ _CONFIG_SYNCED_TABLE_NAMES: frozenset[str] = frozenset( _RESYNC_APPLIED_CONFIG_PARAM_NAMES: frozenset[str] = frozenset( { "general_settings", + "router_settings", "litellm_settings", "model_cost_map_reload_config", "anthropic_beta_headers_reload_config", diff --git a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py index f50eef4f1cc..6872407808c 100644 --- a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py +++ b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py @@ -56,10 +56,11 @@ _EXPECTED_RESYNC_APPLIED_CONFIG_PARAM_NAMES = frozenset( "general_settings", "litellm_settings", "model_cost_map_reload_config", + "router_settings", } ) -_STARTUP_ONLY_CONFIG_PARAM_NAMES = ("environment_variables", "router_settings") +_STARTUP_ONLY_CONFIG_PARAM_NAMES = ("environment_variables",) class _RecordingRedisClient(Redis): @@ -716,13 +717,14 @@ async def _publish_calls_for_invalidated_param(param_name: str) -> List[Tuple[st return client.published -async def test_invalidate_config_param_publishes_params_a_resync_applies() -> None: - published = await _publish_calls_for_invalidated_param("general_settings") +@pytest.mark.parametrize("param_name", sorted(_EXPECTED_RESYNC_APPLIED_CONFIG_PARAM_NAMES)) +async def test_invalidate_config_param_publishes_params_a_resync_applies(param_name: str) -> None: + published = await _publish_calls_for_invalidated_param(param_name) assert len(published) == 1 channel, message = published[0] assert channel == CONFIG_SYNC_CHANNEL - assert json.loads(message) == {"object_type": "general_settings"} + assert json.loads(message) == {"object_type": param_name} @pytest.mark.parametrize("param_name", _STARTUP_ONLY_CONFIG_PARAM_NAMES) diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 8d1d8185e4d..a79bd25b60b 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -2058,6 +2058,42 @@ async def test_ProxyConfig__add_router_settings_from_db_config_none_router_noop( await pc._add_router_settings_from_db_config() # type: ignore[call-arg] +# --------------------------------------------------------------------------- +# ProxyConfig.add_deployment +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_ProxyConfig_add_deployment_applies_db_router_settings(monkeypatch): + from litellm.proxy import proxy_server + + pc = ProxyConfig() + fake_router = MagicMock() + fake_router.get_model_list = MagicMock(return_value=[]) + fake_prisma = MagicMock() + fake_prisma.db.litellm_config.find_first = AsyncMock( + return_value=SimpleNamespace(param_value={"routing_strategy": "latency-based-routing"}) + ) + + async def fake_get_config(*args, **kwargs): + return {} + + monkeypatch.setattr(pc, "get_config", fake_get_config) + monkeypatch.setattr(pc, "_get_models_from_db", AsyncMock(return_value=[])) + monkeypatch.setattr(pc, "_init_non_llm_objects_in_db", AsyncMock()) + monkeypatch.setattr(proxy_server, "prefetch_config_params", AsyncMock()) + monkeypatch.setattr(proxy_server, "get_config_param", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_server, "llm_router", fake_router) + monkeypatch.setattr(proxy_server, "master_key", "sk-master") + monkeypatch.setattr(proxy_server, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "proxy_config", pc) + + await pc.add_deployment(prisma_client=fake_prisma, proxy_logging_obj=MagicMock()) + + fake_router.update_settings.assert_called_once_with(routing_strategy="latency-based-routing") + + # --------------------------------------------------------------------------- # ProxyConfig._add_general_settings_from_db_config # --------------------------------------------------------------------------- From add2a1ce3f6c5c7f2462091d97b70cb7c889bbc0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 1 Aug 2026 03:57:48 -0700 Subject: [PATCH 33/50] chore(typing): clear 2.4k basedpyright errors across 15 Any hotspot files Replace Any-typed seams with real types in the files carrying the highest reportAny/reportExplicitAny density. The dominant source was the repository layer: BaseRepository.table is declared Any, so every repository read poisoned its rows and every downstream call. Typed pass-through accessors under a _PrismaTableActions Protocol pay that crossing once per table, and TypedDicts and Protocols replace the remaining Any-typed request, row, and tool payloads across the team, key, SCIM, spend, MCP, guardrail, video, and websearch surfaces No casts, no type: ignore, no noqa, no new Any annotations, no behavior changes. Whole-tree basedpyright: reportAny 20,840 -> 19,397, reportExplicitAny 7,253 -> 6,518, all rules 151,424 -> 149,066, with no rule increased in any file. Budgets ratcheted: basedpyright -2,358, ruff-strict -300, type-discipline -68 --- basedpyright-code-budget.json | 18 +- .../websearch_interception/handler.py | 107 ++-- .../context_management/editors/compact.py | 134 +++-- litellm/llms/custom_httpx/llm_http_handler.py | 558 +++++++++--------- litellm/llms/openai/videos/transformation.py | 61 +- .../mcp_server/mcp_server_manager.py | 188 ++++-- .../mcp_server/sampling_handler.py | 209 ++++--- .../proxy/guardrails/guardrail_endpoints.py | 96 ++- .../internal_user_endpoints.py | 159 +++-- .../key_management_endpoints.py | 174 ++++-- .../model_management_endpoints.py | 17 +- .../management_endpoints/scim/scim_v2.py | 255 +++++--- .../management_endpoints/team_endpoints.py | 297 +++++++--- .../spend_management_endpoints.py | 368 +++++++++--- .../mcp/litellm_proxy_mcp_handler.py | 72 ++- litellm/videos/main.py | 204 +++---- ruff-strict-budget.json | 20 +- type-discipline-budget.json | 8 +- 18 files changed, 1903 insertions(+), 1042 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 43df27ea2e2..f6dd90077b1 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 31256 + "limit": 29813 }, "reportArgumentType": { "limit": 2645 @@ -24,7 +24,7 @@ "limit": 42 }, "reportExplicitAny": { - "limit": 10208 + "limit": 9473 }, "reportFunctionMemberAccess": { "limit": 11 @@ -54,10 +54,10 @@ "limit": 0 }, "reportMissingParameterType": { - "limit": 5869 + "limit": 5855 }, "reportMissingTypeArgument": { - "limit": 15861 + "limit": 15852 }, "reportMissingTypeStubs": { "limit": 41 @@ -99,19 +99,19 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 45357 + "limit": 45324 }, "reportUnknownLambdaType": { "limit": 113 }, "reportUnknownMemberType": { - "limit": 40477 + "limit": 40452 }, "reportUnknownParameterType": { - "limit": 20338 + "limit": 20309 }, "reportUnknownVariableType": { - "limit": 32047 + "limit": 31978 }, "reportUnnecessaryCast": { "limit": 177 @@ -123,7 +123,7 @@ "limit": 7 }, "reportUnnecessaryIsInstance": { - "limit": 1205 + "limit": 1204 }, "reportUntypedBaseClass": { "limit": 165 diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 21d990e8e60..531caf273f1 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -9,7 +9,8 @@ server-side using litellm router's search tools. import asyncio import math import uuid -from typing import Any, Dict, List, Optional, Tuple, Union, cast +from collections.abc import AsyncIterator, Mapping +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast import litellm from litellm._logging import verbose_logger @@ -29,19 +30,31 @@ from litellm.integrations.websearch_interception.transformation import ( WebSearchTransformation, ) from litellm.llms.base_llm.search.transformation import SearchResponse -from litellm.types.integrations.websearch_interception import ( - WebSearchInterceptionConfig, -) from litellm.types.integrations.custom_logger import ( CHAT_COMPLETION_AGENTIC_SURFACE, RESPONSES_AGENTIC_SURFACE, AgenticLoopPlan, AgenticLoopRequestPatch, ) +from litellm.types.integrations.websearch_interception import ( + WebSearchInterceptionConfig, +) from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import CallTypes, LlmProviders from litellm.utils import ProviderConfigManager +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm.anthropic_messages.transformation import ( + BaseAnthropicMessagesConfig, + ) + from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.llms.anthropic_messages.anthropic_response import ( + AnthropicMessagesResponse, + ) + from litellm.types.utils import ModelResponse + from litellm.utils import CustomStreamWrapper + # Key used to flag, on per-request kwargs, that the originating client sent # an Anthropic-native ``web_search_*`` tool — meaning the final response # should include ``web_search_tool_result`` content blocks so the client @@ -94,8 +107,8 @@ class WebSearchInterceptionLogger(CustomLogger): messages: List[Dict], tools: Optional[List[Dict]], custom_llm_provider: Optional[str], - kwargs: Optional[dict[str, Any]] = None, - ) -> Optional[Dict[str, Any]]: + kwargs: Mapping[str, object] | None = None, + ) -> dict[str, object] | None: """ Short-circuit web-search-only requests by executing the search directly. @@ -188,7 +201,7 @@ class WebSearchInterceptionLogger(CustomLogger): verbose_logger.error(f"WebSearchInterception: Short-circuit search failed: {e}") search_result_text, structured = f"Search failed: {e}", None - content: List[Dict[str, Any]] = [] + content: list[dict[str, object]] = [] if native_tool is not None: tool_use_id = f"srvtoolu_{uuid.uuid4().hex}" tool_name = native_tool.get("name") or "web_search" @@ -210,7 +223,7 @@ class WebSearchInterceptionLogger(CustomLogger): # github_copilot, etc.) see the same payload they always have. content.append({"type": "text", "text": search_result_text}) - response: Dict[str, Any] = { + response: dict[str, object] = { "id": f"msg_{str(uuid.uuid4())}", "type": "message", "role": "assistant", @@ -228,7 +241,9 @@ class WebSearchInterceptionLogger(CustomLogger): ) return response - async def async_pre_call_deployment_hook(self, kwargs: Dict[str, Any], call_type: Optional[Any]) -> Optional[dict]: + async def async_pre_call_deployment_hook( + self, kwargs: Dict[str, Any], call_type: Optional[CallTypes] + ) -> Optional[dict]: """ Pre-call hook to convert native Anthropic web_search tools to regular tools. @@ -297,7 +312,7 @@ class WebSearchInterceptionLogger(CustomLogger): return kwargs - def _convert_responses_tools(self, kwargs: dict[str, Any], tools: list[dict[str, Any]]) -> dict | None: + def _convert_responses_tools(self, kwargs: Mapping[str, object], tools: list[dict[str, object]]) -> dict | None: """Convert Responses API web search tools to the LiteLLM standard function tool.""" if not any(is_web_search_tool_responses(tool) for tool in tools): return None @@ -370,7 +385,7 @@ class WebSearchInterceptionLogger(CustomLogger): return tool.get("name") @classmethod - def _sync_forced_tool_choice(cls, tool_choice: Any, converted_tools: list[dict[str, Any]]) -> Any: + def _sync_forced_tool_choice(cls, tool_choice: Any, converted_tools: list[dict[str, object]]) -> object: """Repoint a forced ``tool_choice`` at ``litellm_web_search`` when it names a web-search tool that was just converted away. @@ -468,7 +483,7 @@ class WebSearchInterceptionLogger(CustomLogger): async def async_should_run_agentic_loop( self, - response: Any, + response: object, model: str, messages: List[Dict], tools: Optional[List[Dict]], @@ -578,7 +593,7 @@ class WebSearchInterceptionLogger(CustomLogger): async def async_should_run_chat_completion_agentic_loop( self, - response: Any, + response: object, model: str, messages: List[Dict], tools: Optional[List[Dict]], @@ -636,7 +651,7 @@ class WebSearchInterceptionLogger(CustomLogger): async def async_should_run_responses_agentic_loop( self, - response: Any, + response: object, model: str, messages: list[dict], tools: list[dict] | None, @@ -687,13 +702,13 @@ class WebSearchInterceptionLogger(CustomLogger): tools: Dict, model: str, messages: List[Dict], - response: Any, - anthropic_messages_provider_config: Any, + response: object, + anthropic_messages_provider_config: "BaseAnthropicMessagesConfig | None", anthropic_messages_optional_request_params: Dict, - logging_obj: Any, + logging_obj: "LiteLLMLoggingObj | None", stream: bool, kwargs: Dict, - ) -> Any: + ) -> "AnthropicMessagesResponse | AsyncIterator[object]": """ Execute agentic loop with WebSearch execution for Anthropic Messages API. @@ -721,10 +736,10 @@ class WebSearchInterceptionLogger(CustomLogger): tools: Dict, model: str, messages: List[Dict], - response: Any, - anthropic_messages_provider_config: Any, + response: object, + anthropic_messages_provider_config: "BaseAnthropicMessagesConfig | None", anthropic_messages_optional_request_params: Dict, - logging_obj: Any, + logging_obj: "LiteLLMLoggingObj | None", stream: bool, kwargs: Dict, ) -> AgenticLoopPlan: @@ -764,7 +779,7 @@ class WebSearchInterceptionLogger(CustomLogger): kwargs=kwargs, ) - metadata: Dict[str, Any] = { + metadata: dict[str, object] = { "tool_type": "websearch", "response_format": "anthropic", } @@ -787,10 +802,10 @@ class WebSearchInterceptionLogger(CustomLogger): async def async_post_agentic_loop_response_hook( self, - response: Any, + response: object, plan: AgenticLoopPlan, kwargs: Dict, - ) -> Any: + ) -> object: """ Inject Anthropic-native ``web_search_tool_result`` blocks into the final response when the originating client used a native @@ -810,9 +825,9 @@ class WebSearchInterceptionLogger(CustomLogger): def _build_native_result_blocks( tool_calls: List[Dict], structured_results: List[Optional[SearchResponse]], - ) -> List[Dict[str, Any]]: + ) -> list[dict[str, object]]: """Build one ``web_search_tool_result`` block per tool_call.""" - blocks: List[Dict[str, Any]] = [] + blocks: list[dict[str, object]] = [] for i, tool_call in enumerate(tool_calls): tool_use_id = tool_call.get("id") or "" structured = structured_results[i] if i < len(structured_results) else None @@ -825,7 +840,7 @@ class WebSearchInterceptionLogger(CustomLogger): return blocks @staticmethod - def _inject_native_blocks(response: Any, native_blocks: List[Dict[str, Any]]) -> Any: + def _inject_native_blocks(response: Any, native_blocks: list[dict[str, object]]) -> Any: """Prepend native blocks to response content, dict or object form.""" if not native_blocks: return response @@ -849,12 +864,12 @@ class WebSearchInterceptionLogger(CustomLogger): tools: Dict, model: str, messages: List[Dict], - response: Any, + response: object, optional_params: Dict, - logging_obj: Any, + logging_obj: "LiteLLMLoggingObj | None", stream: bool, kwargs: Dict, - ) -> Any: + ) -> "ModelResponse | CustomStreamWrapper": """ Execute agentic loop with WebSearch execution for Chat Completions API. @@ -884,9 +899,9 @@ class WebSearchInterceptionLogger(CustomLogger): tools: Dict, model: str, messages: List[Dict], - response: Any, + response: object, optional_params: Dict, - logging_obj: Any, + logging_obj: "LiteLLMLoggingObj | None", stream: bool, kwargs: Dict, ) -> AgenticLoopPlan: @@ -911,9 +926,9 @@ class WebSearchInterceptionLogger(CustomLogger): tools: dict, model: str, messages: list[dict], - response: Any, + response: object, optional_params: dict, - logging_obj: Any, + logging_obj: "LiteLLMLoggingObj | None", stream: bool, kwargs: dict, ) -> AgenticLoopPlan: @@ -1023,7 +1038,7 @@ class WebSearchInterceptionLogger(CustomLogger): return [] @staticmethod - def _extract_search_text(result: Any) -> str: + def _extract_search_text(result: object) -> str: if isinstance(result, Exception): verbose_logger.error(f"WebSearchInterception: Responses search failed with error: {str(result)}") return f"Search failed: {str(result)}" @@ -1091,10 +1106,10 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls: List[Dict], thinking_blocks: List[Dict], anthropic_messages_optional_request_params: Dict, - logging_obj: Any, + logging_obj: "LiteLLMLoggingObj | None", stream: bool, kwargs: Dict, - ) -> Any: + ) -> "AnthropicMessagesResponse | AsyncIterator[object]": """Legacy path: execute search + build patch + run follow-up call.""" request_patch, structured_results = await self._build_anthropic_request_patch( model=model, @@ -1118,7 +1133,7 @@ class WebSearchInterceptionLogger(CustomLogger): if max_tokens is None: max_tokens = cast(int, kwargs.get("max_tokens", 1024)) - response = await anthropic_messages.acreate( + response: AnthropicMessagesResponse | AsyncIterator[object] = await anthropic_messages.acreate( max_tokens=max_tokens, messages=request_patch.messages, model=request_patch.model or model, @@ -1145,7 +1160,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls: List[Dict], thinking_blocks: List[Dict], anthropic_messages_optional_request_params: Dict, - logging_obj: Any, + logging_obj: "LiteLLMLoggingObj | None", kwargs: Dict, ) -> Tuple[AgenticLoopRequestPatch, List[Optional[SearchResponse]]]: """ @@ -1238,7 +1253,7 @@ class WebSearchInterceptionLogger(CustomLogger): return patch, structured_results async def _execute_search( - self, query: str, kwargs: Optional[dict[str, Any]] = None + self, query: str, kwargs: Mapping[str, object] | None = None ) -> Tuple[str, Optional[SearchResponse]]: """ Execute a single web search using router's search tools. @@ -1300,8 +1315,8 @@ class WebSearchInterceptionLogger(CustomLogger): async def _authorize_search_tool( self, - search_tool: dict[str, Any], - kwargs: Optional[dict[str, Any]], + search_tool: Mapping[str, object], + kwargs: Mapping[str, object] | None, ) -> None: search_tool_name = search_tool.get("search_tool_name") if not isinstance(search_tool_name, str) or not search_tool_name: @@ -1343,7 +1358,7 @@ class WebSearchInterceptionLogger(CustomLogger): ) @staticmethod - def _get_user_api_key_auth_from_kwargs(kwargs: Optional[dict[str, Any]]) -> Any: + def _get_user_api_key_auth_from_kwargs(kwargs: Mapping[str, object] | None) -> "UserAPIKeyAuth | None": if not kwargs: return None @@ -1363,7 +1378,7 @@ class WebSearchInterceptionLogger(CustomLogger): return None - def _select_search_tool_from_router(self, llm_router: Any) -> Optional[dict[str, Any]]: + def _select_search_tool_from_router(self, llm_router: object) -> Optional[dict[str, Any]]: if llm_router is None or not hasattr(llm_router, "search_tools"): return None search_tools = list(getattr(llm_router, "search_tools") or []) @@ -1405,11 +1420,11 @@ class WebSearchInterceptionLogger(CustomLogger): messages: List[Dict], tool_calls: List[Dict], optional_params: Dict, - logging_obj: Any, + logging_obj: "LiteLLMLoggingObj | None", stream: bool, kwargs: Dict, response_format: str = "openai", - ) -> Any: + ) -> "ModelResponse | CustomStreamWrapper": """Legacy path: execute search + build patch + run follow-up call.""" request_patch = await self._build_chat_completion_request_patch( model=model, diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py index f18a9f41939..c87014ffc1e 100644 --- a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py @@ -13,7 +13,8 @@ Mirrors Anthropic's native ``compact_20260112`` for non-Anthropic providers: """ import re -from typing import Any, Dict, List, Literal, Optional, Tuple, Union, cast +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union, cast import litellm from litellm._logging import verbose_logger @@ -23,6 +24,18 @@ from litellm.types.llms.anthropic import ( UsageIteration, ) +if TYPE_CHECKING: + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.proxy._types import UserAPIKeyAuth + from litellm.router import Router + from litellm.types.llms.anthropic import ( + AllAnthropicToolsValues, + AnthopicMessagesAssistantMessageParam, + AnthropicMessagesUserMessageParam, + ) + from litellm.types.llms.openai import ChatCompletionToolParam + from litellm.types.utils import ModelResponse + from ..constants import ( COMPACT_DEFAULT_INSTRUCTIONS, COMPACT_DEFAULT_TRIGGER_TOKENS, @@ -98,9 +111,9 @@ def _read_summary_max_tokens_setting() -> int: async def _check_summary_model_access( - user_api_key_auth: Any, + user_api_key_auth: Optional["UserAPIKeyAuth"], summary_model: str, - llm_router: Any, + llm_router: Optional["Router"], ) -> bool: """Return True when every model-allowlist scope on the parent request is satisfied for ``summary_model``. @@ -294,7 +307,7 @@ async def _check_summary_model_access( async def _check_summary_model_budget( - user_api_key_auth: Any, + user_api_key_auth: Optional["UserAPIKeyAuth"], summary_model: str, ) -> bool: """Return True when the caller is within their per-model budget for @@ -357,7 +370,7 @@ async def _check_summary_model_budget( async def _check_summary_model_rate_limit( - user_api_key_auth: Any, + user_api_key_auth: Optional["UserAPIKeyAuth"], summary_model: str, ) -> bool: """Return True when the caller is within their configured RPM/TPM limits @@ -433,7 +446,7 @@ async def _check_summary_model_rate_limit( def _find_latest_compaction_index( - messages: List[Dict[str, Any]], + messages: List[Dict[str, object]], ) -> Tuple[Optional[int], Optional[int]]: """Return (message_index, block_index) of the most recent compaction block. @@ -453,7 +466,7 @@ def _find_latest_compaction_index( def _slice_around_compaction_block( messages: List[Dict[str, Any]], -) -> Tuple[List[Dict[str, Any]], Optional[Dict[str, Any]]]: +) -> Tuple[List[Dict[str, object]], Optional[Dict[str, object]]]: """Apply Anthropic's "drop everything before the compaction block" rule. Returns ``(sliced_messages_with_compaction_block, compaction_block_dict)`` @@ -468,27 +481,26 @@ def _slice_around_compaction_block( original_msg = messages[msg_idx] original_content = original_msg["content"] - compaction_block = cast(Dict[str, Any], original_content[blk_idx]) + compaction_block = cast(Dict[str, object], original_content[blk_idx]) # Per Anthropic's contract everything before the compaction block is # dropped, including earlier blocks within the same assistant message. sliced_content = list(original_content[blk_idx:]) - sliced_first_msg = {**original_msg, "content": sliced_content} - sliced_messages: List[Dict[str, Any]] = [sliced_first_msg] + sliced_messages: List[Dict[str, object]] = [{**original_msg, "content": sliced_content}] sliced_messages.extend(messages[msg_idx + 1 :]) return sliced_messages, compaction_block def _strip_compaction_blocks( - messages: List[Dict[str, Any]], -) -> List[Dict[str, Any]]: + messages: List[Dict[str, object]], +) -> List[Dict[str, object]]: """Drop any ``compaction`` content blocks from messages. Used to build the downstream-bound message list — the adapter has no concept of a compaction block, so it must not see one. """ - cleaned: List[Dict[str, Any]] = [] + cleaned: List[Dict[str, object]] = [] for msg in messages: content = msg.get("content") if not isinstance(content, list): @@ -503,9 +515,9 @@ def _strip_compaction_blocks( def _augment_system_with_summary( - system: Optional[Union[str, List[Dict[str, Any]]]], + system: Optional[Union[str, List[Dict[str, object]]]], summary_text: str, -) -> Union[str, List[Dict[str, Any]]]: +) -> Union[str, List[Dict[str, object]]]: """Prepend a "Previous conversation summary: ..." block to ``system``.""" prefix = f"{COMPACT_SUMMARY_SYSTEM_PREFIX}{summary_text}\n\n" if system is None: @@ -522,7 +534,7 @@ def _augment_system_with_summary( return [{"type": "text", "text": prefix.rstrip()}, *system] -def _resolve_trigger_tokens(edit_spec: Dict[str, Any]) -> Tuple[int, List[str]]: +def _resolve_trigger_tokens(edit_spec: Dict[str, object]) -> Tuple[int, List[str]]: """Validate and resolve ``trigger.value``. Raises ``AnthropicContextManagementError`` if the explicitly-supplied value @@ -556,7 +568,7 @@ def _resolve_trigger_tokens(edit_spec: Dict[str, Any]) -> Tuple[int, List[str]]: return value, warnings -def _build_summary_prompt(edit_spec: Dict[str, Any], tools: Optional[List[Dict[str, Any]]]) -> str: +def _build_summary_prompt(edit_spec: Dict[str, object], tools: Optional[List[Dict[str, object]]]) -> str: custom = edit_spec.get("instructions") if isinstance(custom, str) and custom.strip(): return custom @@ -567,8 +579,8 @@ def _build_summary_prompt(edit_spec: Dict[str, Any], tools: Optional[List[Dict[s def _propagate_metadata( - parent_litellm_metadata: Optional[Dict[str, Any]], -) -> Dict[str, Any]: + parent_litellm_metadata: Optional[Mapping[str, object]], +) -> Dict[str, object]: """Extract the parent request's auth/spend-attribution fields for the summary subcall. The proxy attaches ``user_api_key``, ``user_api_key_team_id`` etc. to @@ -579,7 +591,7 @@ def _propagate_metadata( """ if not parent_litellm_metadata: return {} - propagated: Dict[str, Any] = {} + propagated: Dict[str, object] = {} for key in _PROPAGATED_METADATA_KEYS: if key in parent_litellm_metadata: propagated[key] = parent_litellm_metadata[key] @@ -588,10 +600,10 @@ def _propagate_metadata( def _count_effective_tokens( model: str, - effective_messages: List[Dict[str, Any]], - compaction_block: Optional[Dict[str, Any]], - tools: Optional[List[Dict[str, Any]]], - system: Optional[Union[str, List[Dict[str, Any]]]] = None, + effective_messages: List[Dict[str, object]], + compaction_block: Optional[CompactionBlock], + tools: Optional[List[Dict[str, object]]], + system: Optional[Union[str, List[Dict[str, object]]]] = None, ) -> int: """Token-count the conversation as it will appear downstream. @@ -609,25 +621,32 @@ def _count_effective_tokens( messages_without_compaction = _strip_compaction_blocks(effective_messages) adapter = LiteLLMAnthropicMessagesAdapter() try: - openai_shape = adapter.translate_anthropic_messages_to_openai(messages=cast(Any, messages_without_compaction)) + openai_shape = adapter.translate_anthropic_messages_to_openai( + messages=cast( + "List[Union[AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam]]", + messages_without_compaction, + ) + ) except Exception as e: verbose_logger.debug( "compact_20260112: anthropic→openai translation failed during token " "count, falling back to raw messages: %s", e, ) - openai_shape = cast(Any, messages_without_compaction) + openai_shape = messages_without_compaction # Translate Anthropic-shaped tools (``input_schema``) to OpenAI-shaped # tools (``{"type": "function", "function": {...}}``) so ``token_counter`` # gets a consistent format regardless of which counting path it uses. # An inaccurate tool token count here could cause the polyfill to skip # needed compaction or trigger unnecessary summarization. - openai_tools: Optional[List[Dict[str, Any]]] = None + openai_tools: Optional[List[Dict[str, object]]] = None if tools: try: - translated_tools, _ = adapter.translate_anthropic_tools_to_openai(tools=cast(Any, tools)) - openai_tools = cast(List[Dict[str, Any]], translated_tools) + translated_tools, _ = adapter.translate_anthropic_tools_to_openai( + tools=cast("List[AllAnthropicToolsValues]", tools) + ) + openai_tools = cast(List[Dict[str, object]], translated_tools) except Exception as e: verbose_logger.debug( "compact_20260112: anthropic→openai tools translation failed " @@ -638,8 +657,8 @@ def _count_effective_tokens( total = litellm.token_counter( model=model, - messages=cast(Any, openai_shape), - tools=cast(Any, openai_tools), + messages=cast(List[Dict[str, object]], openai_shape), + tools=cast("Optional[List[ChatCompletionToolParam]]", openai_tools), ) if compaction_block is not None: content = compaction_block.get("content") or "" @@ -652,7 +671,7 @@ def _count_effective_tokens( def _system_to_text( - system: Optional[Union[str, List[Dict[str, Any]]]], + system: Optional[Union[str, List[Dict[str, object]]]], ) -> str: """Flatten an Anthropic-style ``system`` value into a single string for token counting. Returns ``""`` when ``system`` carries no text.""" @@ -670,8 +689,8 @@ def _system_to_text( def _select_last_user_question( - messages: List[Dict[str, Any]], -) -> List[Dict[str, Any]]: + messages: List[Dict[str, object]], +) -> List[Dict[str, object]]: """Pick the most recent ``user`` turn that is a real question. Returns a one-element message list with any ``tool_result`` blocks @@ -735,10 +754,10 @@ def _system_to_openai_message( def _build_summary_messages( - effective_messages: List[Dict[str, Any]], + effective_messages: List[Dict[str, object]], prompt: str, - system: Optional[Union[str, List[Dict[str, Any]]]] = None, -) -> List[Dict[str, Any]]: + system: Optional[Union[str, List[Dict[str, object]]]] = None, +) -> List[Dict[str, object]]: """Build the OpenAI-shape message list for the summary call. The caller's ``system`` prompt is prepended (the default summarization @@ -753,7 +772,10 @@ def _build_summary_messages( stripped = _strip_compaction_blocks(effective_messages) try: openai_messages = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( - messages=cast(Any, stripped) + messages=cast( + "List[Union[AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam]]", + stripped, + ) ) except Exception as e: verbose_logger.warning( @@ -761,9 +783,9 @@ def _build_summary_messages( "building summary call; falling back to raw shape: %s", e, ) - openai_messages = cast(Any, stripped) + openai_messages = stripped - summary_messages: List[Dict[str, Any]] = [] + summary_messages: List[Dict[str, object]] = [] system_message = _system_to_openai_message(system) if system_message is not None: summary_messages.append(system_message) @@ -783,7 +805,7 @@ def _build_summary_messages( return summary_messages -def _is_user_message(msg: Any) -> bool: +def _is_user_message(msg: object) -> bool: return isinstance(msg, dict) and msg.get("role") == "user" @@ -805,12 +827,12 @@ def _append_text_to_content(content: Any, extra_text: str) -> Any: async def _call_summary_model( *, summary_model: str, - summary_messages: List[Dict[str, Any]], - metadata: Dict[str, Any], + summary_messages: List[Dict[str, object]], + metadata: Mapping[str, object], llm_router: Any, allowed_model_region: Optional[str] = None, max_tokens: int = COMPACT_SUMMARY_MAX_TOKENS, -) -> Any: +) -> Union["ModelResponse", "CustomStreamWrapper"]: """Invoke the configured summary model. Prefers ``llm_router.acompletion`` so the model alias resolves against the @@ -877,7 +899,7 @@ def _extract_response_text(response: Any) -> Optional[str]: return None -def _extract_usage(response: Any) -> Tuple[int, int]: +def _extract_usage(response: object) -> Tuple[int, int]: usage = getattr(response, "usage", None) if usage is None: return 0, 0 @@ -889,8 +911,8 @@ def _extract_usage(response: Any) -> Tuple[int, int]: def apply_client_compaction_block_history( *, - messages: List[Dict[str, Any]], - system: Optional[Union[str, List[Dict[str, Any]]]], + messages: List[Dict[str, object]], + system: Optional[Union[str, List[Dict[str, object]]]], ) -> Optional[PolyfillResult]: """Honor client-sent compaction blocks without a ``compact_20260112`` edit. @@ -911,7 +933,7 @@ def apply_client_compaction_block_history( ) prior_summary_text = prior_compaction_block.get("content") or "" - augmented_system: Union[str, List[Dict[str, Any]], None] = system + augmented_system: Union[str, List[Dict[str, object]], None] = system if isinstance(prior_summary_text, str) and prior_summary_text: augmented_system = _augment_system_with_summary(system, prior_summary_text) verbose_logger.info( @@ -936,13 +958,13 @@ def apply_client_compaction_block_history( async def apply_compact_20260112( *, model: str, - messages: List[Dict[str, Any]], - tools: Optional[List[Dict[str, Any]]], - system: Optional[Union[str, List[Dict[str, Any]]]], - edit_spec: Dict[str, Any], - litellm_metadata: Optional[Dict[str, Any]] = None, - llm_router: Any = None, - user_api_key_auth: Any = None, + messages: List[Dict[str, object]], + tools: Optional[List[Dict[str, object]]], + system: Optional[Union[str, List[Dict[str, object]]]], + edit_spec: Dict[str, object], + litellm_metadata: Optional[Mapping[str, object]] = None, + llm_router: Optional["Router"] = None, + user_api_key_auth: Optional["UserAPIKeyAuth"] = None, ) -> PolyfillResult: """Apply ``compact_20260112``; return a ``PolyfillResult``. @@ -971,7 +993,7 @@ async def apply_compact_20260112( # non-Anthropic backends (which would reject them). effective_messages, prior_compaction_block = _slice_around_compaction_block(messages) prior_summary_text = prior_compaction_block.get("content") if prior_compaction_block else None - augmented_system: Union[str, List[Dict[str, Any]], None] = system + augmented_system: Union[str, List[Dict[str, object]], None] = system if isinstance(prior_summary_text, str) and prior_summary_text: augmented_system = _augment_system_with_summary(system, prior_summary_text) verbose_logger.info( diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index d6acbaae434..b64d8e3dc7e 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -13,8 +13,10 @@ from typing import ( Iterator, List, Literal, + Mapping, Optional, Tuple, + TypeVar, Union, cast, get_type_hints, @@ -162,7 +164,11 @@ from .http_handler import get_shared_realtime_ssl_context if TYPE_CHECKING: from aiohttp import ClientSession + from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import ( + FakeAnthropicMessagesStreamIterator, + ) from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import ( AnthropicMessagesStreamingResponse, ) @@ -182,6 +188,8 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_ResponseT = TypeVar("_ResponseT") + def _google_genai_streaming_hidden_params( *, @@ -189,11 +197,11 @@ def _google_genai_streaming_hidden_params( litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, response_headers: httpx.Headers, -) -> Dict[str, Any]: +) -> Dict[str, object]: """Pre-stream metadata for proxy response headers (mirrors CustomStreamWrapper._hidden_params).""" from litellm.litellm_core_utils.core_helpers import process_response_headers - _model_info: Dict[str, Any] = dict(getattr(litellm_params, "model_info", None) or {}) + _model_info: Mapping[str, object] = dict(getattr(litellm_params, "model_info", None) or {}) _raw_id = _model_info.get("id") or logging_obj.get_router_model_id() or "" _model_id = _raw_id if isinstance(_raw_id, str) else str(_raw_id) return { @@ -210,7 +218,7 @@ def _responses_api_optional_request_param_names() -> frozenset[str]: return frozenset(get_type_hints(ResponsesAPIOptionalRequestParams).keys()) -def _custom_logger_callbacks(logging_obj: Any) -> list[Any]: +def _custom_logger_callbacks(logging_obj: LiteLLMLoggingObj) -> list["CustomLogger"]: from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import ( get_custom_logger_compatible_class, @@ -221,7 +229,7 @@ def _custom_logger_callbacks(logging_obj: Any) -> list[Any]: if isinstance(dynamic_success_callbacks, (list, tuple)): callbacks.extend(dynamic_success_callbacks) - custom_loggers: list[Any] = [] + custom_loggers: list[CustomLogger] = [] for cb in callbacks: if isinstance(cb, str): resolved = get_custom_logger_compatible_class(cb) # type: ignore[arg-type] @@ -233,7 +241,7 @@ def _custom_logger_callbacks(logging_obj: Any) -> list[Any]: return custom_loggers -def _has_pre_call_deployment_hook(logging_obj: Any) -> bool: +def _has_pre_call_deployment_hook(logging_obj: LiteLLMLoggingObj) -> bool: from litellm.integrations.custom_logger import CustomLogger base_func = CustomLogger.async_pre_call_deployment_hook @@ -359,7 +367,7 @@ class BaseLLMHTTPHandler: messages: list, optional_params: dict, litellm_params: dict, - encoding: Any, + encoding: object, api_key: Optional[str] = None, client: Optional[AsyncHTTPHandler] = None, json_mode: bool = False, @@ -425,7 +433,7 @@ class BaseLLMHTTPHandler: api_base: Optional[str], custom_llm_provider: str, model_response: ModelResponse, - encoding, + encoding: object, logging_obj: LiteLLMLoggingObj, optional_params: dict, timeout: Union[float, httpx.Timeout], @@ -474,7 +482,7 @@ class BaseLLMHTTPHandler: litellm_params=litellm_params, ) - data = provider_config.transform_request( + data: Dict[str, object] = provider_config.transform_request( model=model, messages=messages, optional_params=optional_params, @@ -651,7 +659,7 @@ class BaseLLMHTTPHandler: fake_stream: bool = False, client: Optional[HTTPHandler] = None, json_mode: bool = False, - ) -> Tuple[Any, dict]: + ) -> Tuple[object, dict]: if client is None or not isinstance(client, HTTPHandler): sync_httpx_client = _get_httpx_client( { @@ -691,7 +699,7 @@ class BaseLLMHTTPHandler: json_mode=json_mode, ) - completion_stream: Any = MockResponseIterator(model_response=model_response, json_mode=json_mode) + completion_stream: object = MockResponseIterator(model_response=model_response, json_mode=json_mode) else: completion_stream = provider_config.get_model_response_iterator( streaming_response=response.iter_lines(), @@ -783,7 +791,7 @@ class BaseLLMHTTPHandler: client: Optional[AsyncHTTPHandler] = None, json_mode: Optional[bool] = None, signed_json_body: Optional[bytes] = None, - ) -> Tuple[Any, httpx.Headers]: + ) -> Tuple[object, httpx.Headers]: """ Helper function for making an async call with stream. @@ -827,7 +835,7 @@ class BaseLLMHTTPHandler: json_mode=json_mode, ) - completion_stream: Any = MockResponseIterator(model_response=model_response, json_mode=json_mode) + completion_stream: object = MockResponseIterator(model_response=model_response, json_mode=json_mode) else: completion_stream = provider_config.get_model_response_iterator( streaming_response=response.aiter_lines(), sync_stream=False @@ -846,10 +854,10 @@ class BaseLLMHTTPHandler: def _add_stream_param_to_request_body( self, - data: dict, + data: Dict[str, object], provider_config: BaseConfig, fake_stream: bool, - ) -> dict: + ) -> Dict[str, object]: """ Some providers like Bedrock invoke do not support the stream parameter in the request body, we only pass `stream` in the request body the provider supports it. """ @@ -1051,7 +1059,7 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]], model_response: RerankResponse, _is_async: bool = False, - headers: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, object]] = None, api_key: Optional[str] = None, api_base: Optional[str] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, @@ -1177,7 +1185,7 @@ class BaseLLMHTTPHandler: logging_obj: LiteLLMLoggingObj, api_key: Optional[str], api_base: Optional[str], - headers: Optional[Dict[str, Any]], + headers: Optional[Dict[str, object]], provider_config: BaseAudioTranscriptionConfig, ) -> Tuple[dict, str, Union[dict, bytes, None], Optional[dict]]: """ @@ -1266,10 +1274,10 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, atranscription: bool = False, - headers: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, object]] = None, provider_config: Optional[BaseAudioTranscriptionConfig] = None, shared_session: Optional["ClientSession"] = None, - ) -> Union[TranscriptionResponse, Coroutine[Any, Any, TranscriptionResponse]]: + ) -> Union[TranscriptionResponse, Coroutine[object, object, TranscriptionResponse]]: if provider_config is None: raise ValueError(f"No provider config found for model: {model} and provider: {custom_llm_provider}") @@ -1351,7 +1359,7 @@ class BaseLLMHTTPHandler: api_base: Optional[str], custom_llm_provider: str, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - headers: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, object]] = None, provider_config: Optional[BaseAudioTranscriptionConfig] = None, shared_session: Optional["ClientSession"] = None, ) -> TranscriptionResponse: @@ -1417,7 +1425,7 @@ class BaseLLMHTTPHandler: logging_obj: LiteLLMLoggingObj, api_key: Optional[str], api_base: Optional[str], - headers: Optional[Dict[str, Any]], + headers: Optional[Dict[str, object]], provider_config: BaseOCRConfig, litellm_params: dict, ) -> Tuple[Dict[str, Any], str, Dict[str, Any], None]: @@ -1483,7 +1491,7 @@ class BaseLLMHTTPHandler: logging_obj: LiteLLMLoggingObj, api_key: Optional[str], api_base: Optional[str], - headers: Optional[Dict[str, Any]], + headers: Optional[Dict[str, object]], provider_config: BaseOCRConfig, litellm_params: dict, ) -> Tuple[Dict[str, Any], str, Dict[str, Any], None]: @@ -1567,10 +1575,10 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, aocr: bool = False, - headers: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, object]] = None, provider_config: Optional[BaseOCRConfig] = None, litellm_params: Optional[dict] = None, - ) -> Union[OCRResponse, Coroutine[Any, Any, OCRResponse]]: + ) -> Union[OCRResponse, Coroutine[object, object, OCRResponse]]: """ Sync OCR handler. """ @@ -1641,7 +1649,7 @@ class BaseLLMHTTPHandler: api_base: Optional[str], custom_llm_provider: str, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - headers: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, object]] = None, provider_config: Optional[BaseOCRConfig] = None, litellm_params: Optional[dict] = None, ) -> OCRResponse: @@ -1703,9 +1711,9 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, asearch: bool = False, - headers: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, object]] = None, provider_config: Optional[BaseSearchConfig] = None, - ) -> Union[SearchResponse, Coroutine[Any, Any, SearchResponse]]: + ) -> Union[SearchResponse, Coroutine[object, object, SearchResponse]]: """ Sync Search handler. """ @@ -1798,7 +1806,7 @@ class BaseLLMHTTPHandler: api_base: Optional[str], custom_llm_provider: str, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - headers: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, object]] = None, provider_config: Optional[BaseSearchConfig] = None, ) -> SearchResponse: """ @@ -1976,7 +1984,7 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, client: Optional[AsyncHTTPHandler] = None, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, api_key: Optional[str] = None, api_base: Optional[str] = None, stream: Optional[bool] = False, @@ -2339,10 +2347,10 @@ class BaseLLMHTTPHandler: api_key: Optional[str] = None, api_base: Optional[str] = None, stream: Optional[bool] = False, - kwargs: Optional[Dict[str, Any]] = None, + kwargs: Optional[Dict[str, object]] = None, ) -> Union[ AnthropicMessagesResponse, - Coroutine[Any, Any, Union[AnthropicMessagesResponse, AsyncIterator]], + Coroutine[object, object, Union[AnthropicMessagesResponse, AsyncIterator]], ]: """ LLM HTTP Handler for Anthropic Messages @@ -2449,18 +2457,18 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Mapping[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, fake_stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, shared_session: Optional["ClientSession"] = None, ) -> Union[ ResponsesAPIResponse, BaseResponsesAPIStreamingIterator, - Coroutine[Any, Any, Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]], + Coroutine[object, object, Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]], ]: """ Handles responses API requests. @@ -2543,7 +2551,7 @@ class BaseLLMHTTPHandler: # Preserve the OpenAI-style request context (not sent to the provider) for streaming # hooks/metadata; the streaming iterator now consumes this to run deployment hooks # with the same info as chat, including litellm_params. - request_context: Dict[str, Any] = {"input": input} + request_context: Dict[str, object] = {"input": input} try: request_context.update(response_api_optional_request_params) except Exception: @@ -2663,12 +2671,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Mapping[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, fake_stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, shared_session: Optional["ClientSession"] = None, ) -> Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]: """ @@ -2720,7 +2728,7 @@ class BaseLLMHTTPHandler: # Preserve the OpenAI-style request context (not sent to the provider) for streaming # hooks/metadata; the streaming iterator now consumes this to run deployment hooks # with the same info as chat, including litellm_params. - request_context: Dict[str, Any] = {"input": input} + request_context: Dict[str, object] = {"input": input} try: request_context.update(response_api_optional_request_params) except Exception: @@ -2847,8 +2855,8 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, @@ -2931,13 +2939,13 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union[DeleteResponseResult, Coroutine[Any, Any, DeleteResponseResult]]: + ) -> Union[DeleteResponseResult, Coroutine[object, object, DeleteResponseResult]]: """ Async version of the responses API handler. Uses async HTTP client to make requests. @@ -3021,13 +3029,13 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union[ResponsesAPIResponse, Coroutine[Any, Any, ResponsesAPIResponse]]: + ) -> Union[ResponsesAPIResponse, Coroutine[object, object, ResponsesAPIResponse]]: """ Get a response by ID Uses GET /v1/responses/{response_id} endpoint in the responses API @@ -3102,8 +3110,8 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -3183,12 +3191,12 @@ class BaseLLMHTTPHandler: include: Optional[List[str]] = None, limit: int = 20, order: Literal["asc", "desc"] = "desc", - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union[Dict, Coroutine[Any, Any, Dict]]: + ) -> Union[Dict, Coroutine[object, object, Dict]]: if _is_async: return self.async_list_responses_input_items( response_id=response_id, @@ -3269,7 +3277,7 @@ class BaseLLMHTTPHandler: include: Optional[List[str]] = None, limit: int = 20, order: Literal["asc", "desc"] = "desc", - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -3375,7 +3383,7 @@ class BaseLLMHTTPHandler: _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> Union[OpenAIFileObject, Coroutine[Any, Any, OpenAIFileObject]]: + ) -> Union[OpenAIFileObject, Coroutine[object, object, OpenAIFileObject]]: """ Creates a file using Gemini's two-step upload process """ @@ -3789,7 +3797,7 @@ class BaseLLMHTTPHandler: client: Optional[Union["HTTPHandler", "AsyncHTTPHandler"]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, model: Optional[str] = None, - ) -> Union["LiteLLMBatch", Coroutine[Any, Any, "LiteLLMBatch"]]: + ) -> Union["LiteLLMBatch", Coroutine[object, object, "LiteLLMBatch"]]: """ Creates a batch using provider-specific batch creation process """ @@ -3901,7 +3909,7 @@ class BaseLLMHTTPHandler: client: Optional[Union["HTTPHandler", "AsyncHTTPHandler"]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, model: Optional[str] = None, - ) -> Union["LiteLLMBatch", Coroutine[Any, Any, "LiteLLMBatch"]]: + ) -> Union["LiteLLMBatch", Coroutine[object, object, "LiteLLMBatch"]]: """ Retrieve a batch using provider-specific configuration. """ @@ -4138,13 +4146,13 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union[ResponsesAPIResponse, Coroutine[Any, Any, ResponsesAPIResponse]]: + ) -> Union[ResponsesAPIResponse, Coroutine[object, object, ResponsesAPIResponse]]: """ Async version of the responses API handler. Uses async HTTP client to make requests. @@ -4218,8 +4226,8 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, @@ -4294,13 +4302,13 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union[ResponsesAPIResponse, Coroutine[Any, Any, ResponsesAPIResponse]]: + ) -> Union[ResponsesAPIResponse, Coroutine[object, object, ResponsesAPIResponse]]: """ Handler for the compact responses API. """ @@ -4393,8 +4401,8 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, @@ -4485,7 +4493,7 @@ class BaseLLMHTTPHandler: _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> Union[OpenAIFileObject, Coroutine[Any, Any, OpenAIFileObject]]: + ) -> Union[OpenAIFileObject, Coroutine[object, object, OpenAIFileObject]]: """ Retrieve file metadata by ID """ @@ -4609,7 +4617,7 @@ class BaseLLMHTTPHandler: _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> Union["FileDeleted", Coroutine[Any, Any, "FileDeleted"]]: + ) -> Union["FileDeleted", Coroutine[object, object, "FileDeleted"]]: """ Delete a file by ID """ @@ -4733,7 +4741,7 @@ class BaseLLMHTTPHandler: _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> Union[List[OpenAIFileObject], Coroutine[Any, Any, List[OpenAIFileObject]]]: + ) -> Union[List[OpenAIFileObject], Coroutine[object, object, List[OpenAIFileObject]]]: """ List all files """ @@ -4857,7 +4865,7 @@ class BaseLLMHTTPHandler: _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> Union["HttpxBinaryResponseContent", Coroutine[Any, Any, "HttpxBinaryResponseContent"]]: + ) -> Union["HttpxBinaryResponseContent", Coroutine[object, object, "HttpxBinaryResponseContent"]]: """ Retrieve file content by ID """ @@ -5008,7 +5016,7 @@ class BaseLLMHTTPHandler: return depth, max(max_loops, 1), fingerprints @staticmethod - def _has_agentic_completion_hook(logging_obj: Any) -> bool: + def _has_agentic_completion_hook(logging_obj: LiteLLMLoggingObj) -> bool: """ True if any registered callback actually overrides ``async_should_run_agentic_loop`` (the gate every agentic hook goes @@ -5039,7 +5047,7 @@ class BaseLLMHTTPHandler: @staticmethod def _check_agentic_loop_safety( - tool_calls: Any, + tool_calls: object, fingerprints: List[str], depth: int, max_loops: int, @@ -5062,7 +5070,7 @@ class BaseLLMHTTPHandler: return fingerprint @staticmethod - def _fingerprint_agentic_tools(tools: Dict) -> str: + def _fingerprint_agentic_tools(tools: object) -> str: try: return json.dumps(tools, sort_keys=True, default=str) except Exception: @@ -5081,8 +5089,8 @@ class BaseLLMHTTPHandler: fingerprints: List[str], fingerprint: str, stream: bool = False, - callback: Optional[Any] = None, - ) -> Any: + callback: Optional["CustomLogger"] = None, + ) -> Union[AnthropicMessagesResponse, AsyncIterator[object]]: from litellm.anthropic_interface import messages as anthropic_messages patch = plan.request_patch or AgenticLoopRequestPatch() @@ -5091,7 +5099,7 @@ class BaseLLMHTTPHandler: full_model_name = model if logging_obj is not None: - agentic_params = logging_obj.model_call_details.get("agentic_loop_params", {}) + agentic_params: Mapping[str, object] = logging_obj.model_call_details.get("agentic_loop_params", {}) full_model_name = cast(str, agentic_params.get("model", model)) optional_params = dict(anthropic_messages_optional_request_params) @@ -5121,7 +5129,7 @@ class BaseLLMHTTPHandler: kwargs_for_followup["max_agentic_loops"] = max_loops kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint] - response = await anthropic_messages.acreate( + response: Union[AnthropicMessagesResponse, AsyncIterator[object]] = await anthropic_messages.acreate( **{ "max_tokens": max_tokens, "messages": patch.messages, @@ -5160,8 +5168,8 @@ class BaseLLMHTTPHandler: max_loops: int, fingerprints: list[str], fingerprint: str, - callback: Any | None = None, - ) -> Any: + callback: Optional["CustomLogger"] = None, + ) -> Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]: patch = plan.request_patch or AgenticLoopRequestPatch() if patch.messages is None: raise ValueError("Agentic loop plan missing patched responses input") @@ -5192,7 +5200,7 @@ class BaseLLMHTTPHandler: kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint] try: - response = await litellm.aresponses( + response: Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator] = await litellm.aresponses( model=patch.model or model, input=patch.messages, **optional_params, @@ -5227,7 +5235,7 @@ class BaseLLMHTTPHandler: @staticmethod async def _run_agentic_loop_cleanup( - callback: Any, + callback: "CustomLogger", plan: AgenticLoopPlan, kwargs: dict, logging_obj: "LiteLLMLoggingObj", @@ -5248,10 +5256,10 @@ class BaseLLMHTTPHandler: self, result: Any, model: str, - responses_api_provider_config: Any, + responses_api_provider_config: BaseResponsesAPIConfig, logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str, - ) -> Any: + ) -> MockResponsesAPIStreamingIterator: """ Wrap a completed responses result as a synthetic stream. @@ -5330,10 +5338,10 @@ class BaseLLMHTTPHandler: def _maybe_wrap_in_fake_stream( self, - response: Any, + response: _ResponseT, logging_obj: Optional["LiteLLMLoggingObj"], api_surface: str, - ) -> Any: + ) -> Union[_ResponseT, "FakeAnthropicMessagesStreamIterator"]: """ If the original request was streaming but converted to non-streaming for WebSearch interception, wrap the dict response in a FakeAnthropicMessagesStreamIterator. @@ -5402,7 +5410,7 @@ class BaseLLMHTTPHandler: continue should_run: bool = False - tool_calls: Any = None + tool_calls: object = None try: # First: Check if agentic loop should run. Wrap in try/except # to shield from buggy user callbacks — a callback crash should @@ -5449,7 +5457,7 @@ class BaseLLMHTTPHandler: callback.__class__.async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan ) if not build_plan_overridden: - agentic_result = await callback.async_run_agentic_loop( + agentic_result: object = await callback.async_run_agentic_loop( tools=tool_calls, model=model, messages=messages, @@ -5571,7 +5579,7 @@ class BaseLLMHTTPHandler: continue should_run: bool = False - tool_calls: Any = None + tool_calls: object = None try: ( should_run, @@ -5828,7 +5836,7 @@ class BaseLLMHTTPHandler: client: Optional[Any] = None, timeout: Optional[float] = None, user_api_key_dict: Optional[Any] = None, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, query_params: Optional[RealtimeQueryParams] = None, ): import websockets @@ -5927,7 +5935,7 @@ class BaseLLMHTTPHandler: timeout: Union[float, httpx.Timeout], provider_config: Optional[Any] = None, model: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, api_version: Optional[str] = None, ) -> httpx.Response: @@ -5960,7 +5968,7 @@ class BaseLLMHTTPHandler: timeout: Union[float, httpx.Timeout], provider_config: Optional[Any] = None, model: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, api_version: Optional[str] = None, ) -> httpx.Response: @@ -5989,7 +5997,7 @@ class BaseLLMHTTPHandler: timeout: Union[float, httpx.Timeout], provider_config: Optional[Any] = None, model: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, api_version: Optional[str] = None, ) -> httpx.Response: @@ -6061,8 +6069,8 @@ class BaseLLMHTTPHandler: timeout: Union[float, httpx.Timeout], provider_config: Optional[Any] = None, model: Optional[str] = None, - session_config: Optional[Dict[str, Any]] = None, - extra_headers: Optional[Dict[str, Any]] = None, + session_config: Optional[Dict[str, object]] = None, + extra_headers: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, api_version: Optional[str] = None, ) -> httpx.Response: @@ -6144,7 +6152,7 @@ class BaseLLMHTTPHandler: api_key: Optional[str] = None, timeout: Optional[float] = None, user_api_key_dict: Optional[Any] = None, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, custom_llm_provider: Optional[str] = None, first_message: Optional[str] = None, **kwargs: Any, @@ -6324,15 +6332,15 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, fake_stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, ) -> Union[ ImageResponse, - Coroutine[Any, Any, ImageResponse], + Coroutine[object, object, ImageResponse], ]: """ @@ -6444,11 +6452,11 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, fake_stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, ) -> ImageResponse: """ Async version of the image edit handler. @@ -6542,16 +6550,16 @@ class BaseLLMHTTPHandler: litellm_params: Dict, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, fake_stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, api_key: Optional[str] = None, ) -> Union[ ImageResponse, - Coroutine[Any, Any, ImageResponse], + Coroutine[object, object, ImageResponse], ]: """ Handles image generation requests. @@ -6669,11 +6677,11 @@ class BaseLLMHTTPHandler: litellm_params: Dict, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, fake_stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, api_key: Optional[str] = None, ) -> ImageResponse: """ @@ -6777,16 +6785,16 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, fake_stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, api_key: Optional[str] = None, ) -> Union[ VideoObject, - Coroutine[Any, Any, VideoObject], + Coroutine[object, object, VideoObject], ]: """ Handles video generation requests. @@ -6901,11 +6909,11 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, fake_stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, api_key: Optional[str] = None, ) -> VideoObject: """ @@ -7001,12 +7009,12 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, api_key: Optional[str] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, variant: Optional[str] = None, - ) -> Union[bytes, Coroutine[Any, Any, bytes]]: + ) -> Union[bytes, Coroutine[object, object, bytes]]: """ Handle video content download requests. """ @@ -7091,7 +7099,7 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, api_key: Optional[str] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, variant: Optional[str] = None, @@ -7169,8 +7177,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, _is_async: bool = False, client=None, @@ -7268,8 +7276,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, client=None, api_key: Optional[str] = None, @@ -7351,7 +7359,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, _is_async: bool = False, client=None, @@ -7435,7 +7443,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, client=None, api_key: Optional[str] = None, @@ -7506,7 +7514,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, _is_async: bool = False, client=None, @@ -7575,7 +7583,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, client=None, api_key: Optional[str] = None, @@ -7634,8 +7642,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, _is_async: bool = False, client=None, @@ -7743,8 +7751,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, client=None, api_key: Optional[str] = None, @@ -7840,8 +7848,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, _is_async: bool = False, client=None, @@ -7929,8 +7937,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, client=None, api_key: Optional[str] = None, @@ -8004,8 +8012,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, _is_async: bool = False, client=None, @@ -8058,8 +8066,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, client=None, api_key: Optional[str] = None, @@ -8139,7 +8147,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, client=None, api_key: Optional[str] = None, @@ -8215,8 +8223,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, _is_async: bool = False, client=None, @@ -8320,8 +8328,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params, logging_obj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[float] = None, client=None, api_key: Optional[str] = None, @@ -8410,11 +8418,11 @@ class BaseLLMHTTPHandler: container_provider_config: "BaseContainerConfig", litellm_params: GenericLiteLLMParams, logging_obj: "LiteLLMLoggingObj", - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - ) -> Union["ContainerObject", Coroutine[Any, Any, "ContainerObject"]]: + ) -> Union["ContainerObject", Coroutine[object, object, "ContainerObject"]]: if _is_async: # Return the async coroutine if called with _is_async=True return self.async_container_create_handler( @@ -8497,7 +8505,7 @@ class BaseLLMHTTPHandler: container_provider_config: "BaseContainerConfig", litellm_params: GenericLiteLLMParams, logging_obj: "LiteLLMLoggingObj", - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> "ContainerObject": @@ -8574,12 +8582,12 @@ class BaseLLMHTTPHandler: after: Optional[str] = None, limit: Optional[int] = None, order: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - ) -> Union["ContainerListResponse", Coroutine[Any, Any, "ContainerListResponse"]]: + ) -> Union["ContainerListResponse", Coroutine[object, object, "ContainerListResponse"]]: if _is_async: # Return the async coroutine if called with _is_async=True return self.async_container_list_handler( @@ -8664,8 +8672,8 @@ class BaseLLMHTTPHandler: after: Optional[str] = None, limit: Optional[int] = None, order: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> "ContainerListResponse": @@ -8739,12 +8747,12 @@ class BaseLLMHTTPHandler: container_provider_config: "BaseContainerConfig", litellm_params: GenericLiteLLMParams, logging_obj: "LiteLLMLoggingObj", - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - ) -> Union["ContainerObject", Coroutine[Any, Any, "ContainerObject"]]: + ) -> Union["ContainerObject", Coroutine[object, object, "ContainerObject"]]: if _is_async: # Return the async coroutine if called with _is_async=True return self.async_container_retrieve_handler( @@ -8827,8 +8835,8 @@ class BaseLLMHTTPHandler: container_provider_config: "BaseContainerConfig", litellm_params: GenericLiteLLMParams, logging_obj: "LiteLLMLoggingObj", - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> "ContainerObject": @@ -8904,12 +8912,12 @@ class BaseLLMHTTPHandler: container_provider_config: "BaseContainerConfig", litellm_params: GenericLiteLLMParams, logging_obj: "LiteLLMLoggingObj", - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - ) -> Union["DeleteContainerResult", Coroutine[Any, Any, "DeleteContainerResult"]]: + ) -> Union["DeleteContainerResult", Coroutine[object, object, "DeleteContainerResult"]]: if _is_async: # Return the async coroutine if called with _is_async=True return self.async_container_delete_handler( @@ -8992,8 +9000,8 @@ class BaseLLMHTTPHandler: container_provider_config: "BaseContainerConfig", litellm_params: GenericLiteLLMParams, logging_obj: "LiteLLMLoggingObj", - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> "DeleteContainerResult": @@ -9072,12 +9080,12 @@ class BaseLLMHTTPHandler: after: Optional[str] = None, limit: Optional[int] = None, order: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - ) -> Union["ContainerFileListResponse", Coroutine[Any, Any, "ContainerFileListResponse"]]: + ) -> Union["ContainerFileListResponse", Coroutine[object, object, "ContainerFileListResponse"]]: if _is_async: return self.async_container_file_list_handler( container_id=container_id, @@ -9164,8 +9172,8 @@ class BaseLLMHTTPHandler: after: Optional[str] = None, limit: Optional[int] = None, order: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> "ContainerFileListResponse": @@ -9241,11 +9249,11 @@ class BaseLLMHTTPHandler: container_provider_config: "BaseContainerConfig", litellm_params: GenericLiteLLMParams, logging_obj: "LiteLLMLoggingObj", - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, - ) -> Union[bytes, Coroutine[Any, Any, bytes]]: + ) -> Union[bytes, Coroutine[object, object, bytes]]: if _is_async: return self.async_container_file_content_handler( container_id=container_id, @@ -9327,7 +9335,7 @@ class BaseLLMHTTPHandler: container_provider_config: "BaseContainerConfig", litellm_params: GenericLiteLLMParams, logging_obj: "LiteLLMLoggingObj", - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Union[float, httpx.Timeout] = 600, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> bytes: @@ -9406,8 +9414,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, @@ -9504,12 +9512,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, - ) -> Union[VectorStoreSearchResponse, Coroutine[Any, Any, VectorStoreSearchResponse]]: + ) -> Union[VectorStoreSearchResponse, Coroutine[object, object, VectorStoreSearchResponse]]: if _is_async: return self.async_vector_store_search_handler( vector_store_id=vector_store_id, @@ -9598,8 +9606,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, @@ -9658,12 +9666,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, - ) -> Union[VectorStoreCreateResponse, Coroutine[Any, Any, VectorStoreCreateResponse]]: + ) -> Union[VectorStoreCreateResponse, Coroutine[object, object, VectorStoreCreateResponse]]: if _is_async: return self.async_vector_store_create_handler( vector_store_create_optional_params=vector_store_create_optional_params, @@ -9728,8 +9736,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> VectorStoreCreateResponse: @@ -9781,12 +9789,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, - ) -> Union[VectorStoreCreateResponse, Coroutine[Any, Any, VectorStoreCreateResponse]]: + ) -> Union[VectorStoreCreateResponse, Coroutine[object, object, VectorStoreCreateResponse]]: if _is_async: return self.async_vector_store_retrieve_handler( vector_store_id=vector_store_id, @@ -9848,8 +9856,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ): @@ -9912,8 +9920,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, @@ -9988,8 +9996,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> VectorStoreCreateResponse: @@ -10054,12 +10062,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, - ) -> Union[VectorStoreCreateResponse, Coroutine[Any, Any, VectorStoreCreateResponse]]: + ) -> Union[VectorStoreCreateResponse, Coroutine[object, object, VectorStoreCreateResponse]]: if _is_async: return self.async_vector_store_update_handler( vector_store_id=vector_store_id, @@ -10131,8 +10139,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ): @@ -10182,8 +10190,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, @@ -10249,8 +10257,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> VectorStoreFileObject: @@ -10314,12 +10322,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, - ) -> Union[VectorStoreFileObject, Coroutine[Any, Any, VectorStoreFileObject]]: + ) -> Union[VectorStoreFileObject, Coroutine[object, object, VectorStoreFileObject]]: if _is_async: return self.async_vector_store_file_create_handler( vector_store_id=vector_store_id, @@ -10391,8 +10399,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> VectorStoreFileListResponse: @@ -10455,12 +10463,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, + extra_query: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, - ) -> Union[VectorStoreFileListResponse, Coroutine[Any, Any, VectorStoreFileListResponse]]: + ) -> Union[VectorStoreFileListResponse, Coroutine[object, object, VectorStoreFileListResponse]]: if _is_async: return self.async_vector_store_file_list_handler( vector_store_id=vector_store_id, @@ -10531,7 +10539,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> VectorStoreFileObject: @@ -10590,11 +10598,11 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, - ) -> Union[VectorStoreFileObject, Coroutine[Any, Any, VectorStoreFileObject]]: + ) -> Union[VectorStoreFileObject, Coroutine[object, object, VectorStoreFileObject]]: if _is_async: return self.async_vector_store_file_retrieve_handler( vector_store_id=vector_store_id, @@ -10660,7 +10668,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> VectorStoreFileContentResponse: @@ -10721,13 +10729,13 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, ) -> Union[ VectorStoreFileContentResponse, - Coroutine[Any, Any, VectorStoreFileContentResponse], + Coroutine[object, object, VectorStoreFileContentResponse], ]: if _is_async: return self.async_vector_store_file_content_handler( @@ -10797,8 +10805,8 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> VectorStoreFileObject: @@ -10863,12 +10871,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, - ) -> Union[VectorStoreFileObject, Coroutine[Any, Any, VectorStoreFileObject]]: + ) -> Union[VectorStoreFileObject, Coroutine[object, object, VectorStoreFileObject]]: if _is_async: return self.async_vector_store_file_update_handler( vector_store_id=vector_store_id, @@ -10941,7 +10949,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> VectorStoreFileDeleteResponse: @@ -11000,13 +11008,13 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, str]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, ) -> Union[ VectorStoreFileDeleteResponse, - Coroutine[Any, Any, VectorStoreFileDeleteResponse], + Coroutine[object, object, VectorStoreFileDeleteResponse], ]: if _is_async: return self.async_vector_store_file_delete_handler( @@ -11077,13 +11085,13 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, _is_async: bool = False, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, system_instruction: Optional[Any] = None, ) -> Any: """ @@ -11209,12 +11217,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[AsyncHTTPHandler] = None, stream: bool = False, - litellm_metadata: Optional[Dict[str, Any]] = None, + litellm_metadata: Optional[Dict[str, object]] = None, system_instruction: Optional[Any] = None, ) -> Any: """ @@ -11328,12 +11336,12 @@ class BaseLLMHTTPHandler: litellm_params: Dict, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, ) -> Union[ "HttpxBinaryResponseContent", - Coroutine[Any, Any, "HttpxBinaryResponseContent"], + Coroutine[object, object, "HttpxBinaryResponseContent"], ]: """ Handles text-to-speech requests. @@ -11443,7 +11451,7 @@ class BaseLLMHTTPHandler: litellm_params: Dict, logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> "HttpxBinaryResponseContent": """ @@ -11575,12 +11583,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["Skill", Coroutine[Any, Any, "Skill"]]: + ) -> Union["Skill", Coroutine[object, object, "Skill"]]: """Create a skill""" if _is_async: return self.async_create_skill_handler( @@ -11641,7 +11649,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -11697,12 +11705,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["ListSkillsResponse", Coroutine[Any, Any, "ListSkillsResponse"]]: + ) -> Union["ListSkillsResponse", Coroutine[object, object, "ListSkillsResponse"]]: """List skills""" if _is_async: return self.async_list_skills_handler( @@ -11756,7 +11764,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -11802,12 +11810,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["Skill", Coroutine[Any, Any, "Skill"]]: + ) -> Union["Skill", Coroutine[object, object, "Skill"]]: """Get a skill""" if _is_async: return self.async_get_skill_handler( @@ -11858,7 +11866,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -11903,12 +11911,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["DeleteSkillResponse", Coroutine[Any, Any, "DeleteSkillResponse"]]: + ) -> Union["DeleteSkillResponse", Coroutine[object, object, "DeleteSkillResponse"]]: """Delete a skill""" if _is_async: return self.async_delete_skill_handler( @@ -11959,7 +11967,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12009,12 +12017,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["Eval", Coroutine[Any, Any, "Eval"]]: + ) -> Union["Eval", Coroutine[object, object, "Eval"]]: """Create an eval""" if _is_async: return self.async_create_eval_handler( @@ -12068,7 +12076,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12115,12 +12123,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["ListEvalsResponse", Coroutine[Any, Any, "ListEvalsResponse"]]: + ) -> Union["ListEvalsResponse", Coroutine[object, object, "ListEvalsResponse"]]: """List evals""" if _is_async: return self.async_list_evals_handler( @@ -12174,7 +12182,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12220,12 +12228,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["Eval", Coroutine[Any, Any, "Eval"]]: + ) -> Union["Eval", Coroutine[object, object, "Eval"]]: """Get an eval""" if _is_async: return self.async_get_eval_handler( @@ -12276,7 +12284,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12322,12 +12330,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["Eval", Coroutine[Any, Any, "Eval"]]: + ) -> Union["Eval", Coroutine[object, object, "Eval"]]: """Update an eval""" if _is_async: return self.async_update_eval_handler( @@ -12381,7 +12389,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12427,12 +12435,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["DeleteEvalResponse", Coroutine[Any, Any, "DeleteEvalResponse"]]: + ) -> Union["DeleteEvalResponse", Coroutine[object, object, "DeleteEvalResponse"]]: """Delete an eval""" if _is_async: return self.async_delete_eval_handler( @@ -12483,7 +12491,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12528,12 +12536,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["CancelEvalResponse", Coroutine[Any, Any, "CancelEvalResponse"]]: + ) -> Union["CancelEvalResponse", Coroutine[object, object, "CancelEvalResponse"]]: """Cancel an eval""" if _is_async: return self.async_cancel_eval_handler( @@ -12584,7 +12592,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12634,12 +12642,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["Run", Coroutine[Any, Any, "Run"]]: + ) -> Union["Run", Coroutine[object, object, "Run"]]: """Create a run""" if _is_async: return self.async_create_run_handler( @@ -12693,7 +12701,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12740,12 +12748,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["ListRunsResponse", Coroutine[Any, Any, "ListRunsResponse"]]: + ) -> Union["ListRunsResponse", Coroutine[object, object, "ListRunsResponse"]]: """List runs""" if _is_async: return self.async_list_runs_handler( @@ -12799,7 +12807,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12845,12 +12853,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["Run", Coroutine[Any, Any, "Run"]]: + ) -> Union["Run", Coroutine[object, object, "Run"]]: """Get a run""" if _is_async: return self.async_get_run_handler( @@ -12901,7 +12909,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -12946,12 +12954,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["CancelRunResponse", Coroutine[Any, Any, "CancelRunResponse"]]: + ) -> Union["CancelRunResponse", Coroutine[object, object, "CancelRunResponse"]]: """Cancel a run""" if _is_async: return self.async_cancel_run_handler( @@ -13002,7 +13010,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, @@ -13047,12 +13055,12 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> Union["RunDeleteResponse", Coroutine[Any, Any, "RunDeleteResponse"]]: + ) -> Union["RunDeleteResponse", Coroutine[object, object, "RunDeleteResponse"]]: """Delete a run""" if _is_async: return self.async_delete_run_handler( @@ -13103,7 +13111,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, shared_session: Optional["ClientSession"] = None, diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index 684601367b6..855bc410cea 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast from urllib.parse import quote import httpx -from httpx._types import RequestFiles +from httpx._types import FileContent, FileTypes, RequestFiles import litellm from litellm.litellm_core_utils.url_utils import encode_url_path_segment @@ -128,7 +128,7 @@ class OpenAIVideoConfig(BaseVideoConfig): # Handle input_reference parameter if provided _input_reference = video_create_optional_request_params.get("input_reference") data_without_files = {k: v for k, v in request_dict.items() if k not in ["input_reference"]} - files_list: List[Tuple[str, Any]] = [] + files_list: List[Tuple[str, FileTypes]] = [] # Handle input_reference parameter if _input_reference is not None: @@ -177,9 +177,7 @@ class OpenAIVideoConfig(BaseVideoConfig): request_data: Optional[Dict] = None, ) -> VideoObject: """Transform the OpenAI video creation response.""" - response_data = raw_response.json() - - video_obj = VideoObject(**response_data) # type: ignore[arg-type] + video_obj = VideoObject.model_validate(raw_response.json()) if custom_llm_provider and video_obj.id: video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, model) @@ -223,7 +221,7 @@ class OpenAIVideoConfig(BaseVideoConfig): url = f"{url}?variant={quote(variant, safe='')}" # No additional data needed for GET content request - data: Dict[str, Any] = {} + data: Dict[str, object] = {} return url, data @@ -274,10 +272,8 @@ class OpenAIVideoConfig(BaseVideoConfig): """ Transform the OpenAI video remix response. """ - response_data = raw_response.json() - # Transform the response data - video_obj = VideoObject(**response_data) # type: ignore[arg-type] + video_obj = VideoObject.model_validate(raw_response.json()) if custom_llm_provider and video_obj.id: video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, None) @@ -392,7 +388,7 @@ class OpenAIVideoConfig(BaseVideoConfig): url = f"{api_base.rstrip('/')}/{encoded_video_id}" # No data needed for DELETE request - data: Dict[str, Any] = {} + data: Dict[str, object] = {} return url, data @@ -404,10 +400,8 @@ class OpenAIVideoConfig(BaseVideoConfig): """ Transform the OpenAI video delete response. """ - response_data = raw_response.json() - # Transform the response data - video_obj = VideoObject(**response_data) # type: ignore[arg-type] # type: ignore[arg-type] + video_obj = VideoObject.model_validate(raw_response.json()) return video_obj @@ -429,7 +423,7 @@ class OpenAIVideoConfig(BaseVideoConfig): url = f"{api_base.rstrip('/')}/{encoded_video_id}" # No additional data needed for GET request - data: Dict[str, Any] = {} + data: Dict[str, object] = {} return url, data @@ -442,9 +436,8 @@ class OpenAIVideoConfig(BaseVideoConfig): """ Transform the OpenAI video retrieve response. """ - response_data = raw_response.json() # Transform the response data - video_obj = VideoObject(**response_data) # type: ignore[arg-type] + video_obj = VideoObject.model_validate(raw_response.json()) if custom_llm_provider and video_obj.id: video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, None) @@ -465,22 +458,22 @@ class OpenAIVideoConfig(BaseVideoConfig): def transform_video_create_character_request( self, name: str, - video: Any, + video: FileContent, api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, ) -> Tuple[str, list]: url = f"{api_base.rstrip('/')}/characters" - files_list: List[Tuple[str, Any]] = [("name", (None, name))] + files_list: List[Tuple[str, FileTypes]] = [("name", (None, name))] self._add_video_to_files(files_list, video, "video") return url, files_list def transform_video_create_character_response( self, raw_response: httpx.Response, - logging_obj: Any, + logging_obj: LiteLLMLoggingObj, ) -> CharacterObject: - return CharacterObject(**raw_response.json()) + return CharacterObject.model_validate(raw_response.json()) def transform_video_get_character_request( self, @@ -497,9 +490,9 @@ class OpenAIVideoConfig(BaseVideoConfig): def transform_video_get_character_response( self, raw_response: httpx.Response, - logging_obj: Any, + logging_obj: LiteLLMLoggingObj, ) -> CharacterObject: - return CharacterObject(**raw_response.json()) + return CharacterObject.model_validate(raw_response.json()) def transform_video_edit_request( self, @@ -508,12 +501,12 @@ class OpenAIVideoConfig(BaseVideoConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - extra_body: Optional[Dict[str, Any]] = None, - prefetched_source_data: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, object]] = None, + prefetched_source_data: Optional[Dict[str, object]] = None, ) -> Tuple[str, Dict]: original_video_id = extract_original_video_id(video_id) url = f"{api_base.rstrip('/')}/edits" - data: Dict[str, Any] = {"prompt": prompt, "video": {"id": original_video_id}} + data: Dict[str, object] = {"prompt": prompt, "video": {"id": original_video_id}} if extra_body: data.update(extra_body) return url, data @@ -521,11 +514,11 @@ class OpenAIVideoConfig(BaseVideoConfig): def transform_video_edit_response( self, raw_response: httpx.Response, - logging_obj: Any, + logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str] = None, request_data: Optional[Dict] = None, ) -> VideoObject: - video_obj = VideoObject(**raw_response.json()) + video_obj = VideoObject.model_validate(raw_response.json()) if custom_llm_provider and video_obj.id: video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, None) return video_obj @@ -538,11 +531,11 @@ class OpenAIVideoConfig(BaseVideoConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - extra_body: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, object]] = None, ) -> Tuple[str, Dict]: original_video_id = extract_original_video_id(video_id) url = f"{api_base.rstrip('/')}/extensions" - data: Dict[str, Any] = { + data: Dict[str, object] = { "prompt": prompt, "seconds": seconds, "video": {"id": original_video_id}, @@ -554,10 +547,10 @@ class OpenAIVideoConfig(BaseVideoConfig): def transform_video_extension_response( self, raw_response: httpx.Response, - logging_obj: Any, + logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str] = None, ) -> VideoObject: - video_obj = VideoObject(**raw_response.json()) + video_obj = VideoObject.model_validate(raw_response.json()) if custom_llm_provider and video_obj.id: video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, None) return video_obj @@ -578,8 +571,8 @@ class OpenAIVideoConfig(BaseVideoConfig): def _add_video_to_files( self, - files_list: List[Tuple[str, Any]], - video: Any, + files_list: List[Tuple[str, FileTypes]], + video: FileContent, field_name: str, ) -> None: """ @@ -592,7 +585,7 @@ class OpenAIVideoConfig(BaseVideoConfig): content_type = self._get_video_content_type(video=video, filename=filename) files_list.append((field_name, (filename, video, content_type))) - def _get_video_content_type(self, video: Any, filename: str) -> str: + def _get_video_content_type(self, video: FileContent, filename: str) -> str: guessed_content_type, _ = mimetypes.guess_type(filename) if guessed_content_type and guessed_content_type.startswith("video/"): return guessed_content_type diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index e3f7352e8ca..0a6d72881be 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -15,8 +15,19 @@ import re import time from collections.abc import Sequence from contextlib import asynccontextmanager -from typing import Any, AsyncIterator, Callable, Literal, Optional, Union, cast -from urllib.parse import urlparse +from typing import ( + TYPE_CHECKING, + Any, + AsyncIterator, + Callable, + Literal, + Optional, + TypeAlias, + TypedDict, + Union, + cast, +) +from urllib.parse import ParseResult, urlparse import anyio import httpx @@ -32,7 +43,7 @@ from mcp.types import ( ResourceTemplate, ) from mcp.types import Tool as MCPTool -from pydantic import AnyUrl +from pydantic import AnyUrl, BaseModel import litellm from litellm._logging import verbose_logger @@ -139,10 +150,15 @@ from litellm.proxy._types import ( from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper from litellm.proxy.common_utils.user_api_key_cache import get_management_object_ttl -from litellm.proxy.utils import ProxyLogging, get_server_root_path +from litellm.proxy.utils import PrismaClient, ProxyLogging, get_server_root_path from litellm.repositories.table_repositories import MCPServerRepository from litellm.types.llms.custom_http import httpxSpecialProvider -from litellm.types.mcp import DEFAULT_SUBJECT_TOKEN_TYPE, MCPAuth, MCPStdioConfig +from litellm.types.mcp import ( + DEFAULT_SUBJECT_TOKEN_TYPE, + MCPAuth, + MCPStdioConfig, + MCPTokenEndpointAuthMethod, +) from litellm.types.mcp_server.mcp_server_manager import ( MCPInfo, MCPOAuthMetadata, @@ -150,6 +166,14 @@ from litellm.types.mcp_server.mcp_server_manager import ( ) from litellm.types.utils import CallTypes +if TYPE_CHECKING: + from mcp.client.session import ClientSession + from mcp.shared.context import RequestContext + from mcp.types import CreateMessageRequestParams + + from litellm.caching.caching import InMemoryCache + from litellm.types.mcp_server.mcp_toolset import MCPToolset + try: from mcp.shared.tool_name_validation import ( SEP_986_URL, @@ -209,6 +233,95 @@ _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES: tuple[MCPAuth, ...] = ( _OAUTH_DISCOVERY_RETRY_BASE_SECONDS = 30.0 _OAUTH_DISCOVERY_RETRY_MAX_SECONDS = 900.0 +_StringList: TypeAlias = list[str] +_StringMap: TypeAlias = dict[str, str] +_ToolParamMap: TypeAlias = dict[str, list[str]] +_EnvVarList: TypeAlias = list[dict[str, object]] +_InMemoryCacheDict: TypeAlias = dict[str, object] +_ToolArguments: TypeAlias = dict[str, object] + + +class MCPServerConfig(TypedDict, total=False): + """Shape of a single ``mcp_servers`` entry in config.yaml, as consumed by + :meth:`MCPServerManager.load_servers_from_config`. Every key is optional: YAML supplies + whatever the admin wrote, and each read applies its own default.""" + + alias: str + description: str + mcp_info: MCPInfo + url: str + spec_path: str + transport: MCPTransportType + auth_type: MCPAuthType + authentication_token: str + auth_value: str + instructions: str + command: str + args: _StringList + env: _StringMap + client_id: str + client_secret: str + oauth2_flow: str + issuer: str + authorization_url: str + token_url: str + registration_url: str + token_endpoint_auth_method: MCPTokenEndpointAuthMethod + scopes: str | Sequence[str] + dcr_bridge: object + extra_headers: _StringList + allowed_tools: _StringList + disallowed_tools: _StringList + allowed_params: _ToolParamMap + access_groups: _StringList + static_headers: _StringMap + env_vars: _EnvVarList + allow_all_keys: bool + available_on_public_internet: bool + delegate_auth_to_upstream: bool + oauth_passthrough: bool + allow_sampling: bool + allow_elicitation: bool + aws_access_key_id: str + aws_secret_access_key: str + aws_session_token: str + aws_region_name: str + aws_service_name: str + aws_role_name: str + aws_session_name: str + token_exchange_endpoint: str + token_exchange_profile: str + audience: str + subject_token_type: str + upstream_resource: str + id_jag_resource_token_endpoint: str + id_jag_resource: str + client_private_key: str + client_private_key_id: str + client_assertion_signing_alg: str + timeout: float + max_concurrent_requests: int + + +class _ProtectedResourceMetadataPayload(TypedDict, total=False): + """The RFC 9728 protected-resource metadata document fields this gateway reads.""" + + authorization_servers: Sequence[object] + scopes_supported: Sequence[str] + scopes: Sequence[str] + + +class _AuthorizationServerMetadataPayload(TypedDict, total=False): + """The RFC 8414 / OpenID Discovery authorization-server metadata fields this gateway reads.""" + + issuer: str + authorization_endpoint: str + token_endpoint: str + registration_endpoint: str + scopes_supported: Sequence[str] + grant_types_supported: Sequence[str] + token_endpoint_auth_methods_supported: Sequence[str] + def _blank_to_none(value: str | None) -> str | None: """Collapse an absent, empty, or whitespace-only string to ``None``. @@ -968,7 +1081,7 @@ def _warn_internal_delegate_pkce_if_applicable(server: MCPServer, *, source: str ) -def _deserialize_json_dict(data: Any) -> Optional[dict[str, str]]: +def _deserialize_json_dict(data: str | _StringMap | None) -> Optional[dict[str, str]]: """ Deserialize optional JSON mappings stored in the database. @@ -1057,7 +1170,7 @@ def _normalize_mcp_server_cost_info(mcp_info: MCPInfo) -> None: mcp_info["mcp_server_cost_info"] = normalized -def _create_sampling_callback(user_api_key_auth: Optional[Any] = None): +def _create_sampling_callback(user_api_key_auth: Optional[UserAPIKeyAuth] = None): """ Create a sampling callback for MCP ClientSession. Returns a callable that handles sampling/createMessage requests from @@ -1066,7 +1179,10 @@ def _create_sampling_callback(user_api_key_auth: Optional[Any] = None): if not MCP_SAMPLING_AVAILABLE: return None - async def _sampling_callback(context, params): + async def _sampling_callback( + context: "RequestContext[ClientSession, object]", + params: "CreateMessageRequestParams", + ): import litellm from litellm.proxy._experimental.mcp_server.sampling_handler import ( handle_sampling_create_message, @@ -1309,8 +1425,9 @@ class MCPServerManager: if state is None: return True failures, attempted_at = state + backoff_multiplier: int = 2 ** max(failures - 1, 0) delay = min( - _OAUTH_DISCOVERY_RETRY_BASE_SECONDS * (2 ** max(failures - 1, 0)), + _OAUTH_DISCOVERY_RETRY_BASE_SECONDS * backoff_multiplier, _OAUTH_DISCOVERY_RETRY_MAX_SECONDS, ) return (time.monotonic() - attempted_at) >= delay @@ -1324,7 +1441,7 @@ class MCPServerManager: self._oauth_discovery_retry_state[server.server_id] = (failures + 1, time.monotonic()) def _remember_upstream_initialize_instructions(self, server: MCPServer, client: MCPClient) -> None: - raw = getattr(client, "_last_initialize_instructions", None) + raw: str | None = getattr(client, "_last_initialize_instructions", None) if raw and str(raw).strip(): self._upstream_initialize_instructions_by_server_id[server.server_id] = str(raw).strip() @@ -1430,9 +1547,10 @@ class MCPServerManager: # Track which aliases have been used to ensure only first occurrence is used used_aliases = set() - for server_name, server_config in mcp_servers_config.items(): + for server_name, raw_server_config in mcp_servers_config.items(): + server_config: MCPServerConfig = raw_server_config validate_mcp_server_name(server_name) - _mcp_info: dict[str, Any] = server_config.get("mcp_info", None) or {} + _mcp_info: MCPInfo = server_config.get("mcp_info", None) or {} # Preserve all custom fields from config while setting defaults for core fields mcp_info: MCPInfo = _mcp_info.copy() # Set default values for core fields if not present @@ -1895,7 +2013,7 @@ class MCPServerManager: mcp_server: LiteLLM_MCPServerTable, *, env_vars_are_encrypted: bool, - ) -> Optional[list[dict[str, Any]]]: + ) -> Optional[_EnvVarList]: env_vars_list = _deserialize_json_list(getattr(mcp_server, "env_vars", None)) if env_vars_are_encrypted: from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 @@ -2279,7 +2397,7 @@ class MCPServerManager: async def _get_active_submitted_mcp_server_ids_for_user( self, user_api_key_auth: UserAPIKeyAuth | None ) -> list[str]: - submitter_user_id = getattr(user_api_key_auth, "user_id", None) if user_api_key_auth else None + submitter_user_id: str | None = getattr(user_api_key_auth, "user_id", None) if user_api_key_auth else None if not submitter_user_id: return [] @@ -2551,10 +2669,10 @@ class MCPServerManager: try: from litellm.proxy.proxy_server import user_api_key_cache - in_mem = getattr(user_api_key_cache, "in_memory_cache", None) + in_mem: InMemoryCache | None = getattr(user_api_key_cache, "in_memory_cache", None) if in_mem is None: return - cache_dict = getattr(in_mem, "cache_dict", {}) + cache_dict: _InMemoryCacheDict = getattr(in_mem, "cache_dict", {}) if toolset_id is None: keys_to_remove = [k for k in cache_dict if k.startswith("toolset_")] else: @@ -2574,9 +2692,9 @@ class MCPServerManager: async def get_toolset_by_name_cached( self, - prisma_client: Any, + prisma_client: PrismaClient, toolset_name: str, - ) -> Optional[Any]: + ) -> "Optional[MCPToolset]": """Return a toolset by name, cached in ``user_api_key_cache`` (Redis-backed ``DualCache`` in production) to avoid a DB hit on every routed request. @@ -2803,7 +2921,7 @@ class MCPServerManager: and report ``unknown`` instead of a misleading ``unhealthy``. """ static_headers = server.static_headers - env_vars = getattr(server, "env_vars", None) + env_vars: _EnvVarList | None = getattr(server, "env_vars", None) if not static_headers or not env_vars: return False _global_values, user_specs = parse_admin_env_vars(env_vars) @@ -2929,7 +3047,7 @@ class MCPServerManager: """ if user_api_key_auth is None: return {} - user_id = getattr(user_api_key_auth, "user_id", None) + user_id: str | None = getattr(user_api_key_auth, "user_id", None) if not user_id: return {} @@ -2976,7 +3094,7 @@ class MCPServerManager: match await provider.resolve_credentials(to_subject(user_api_key_auth, subject_token), spec): case Ok(auth): # NoOpAuth has no header_name and so never conflicts. - header_name = getattr(auth, "header_name", None) + header_name: str | None = getattr(auth, "header_name", None) conflicts = bool( header_name and extra_headers and any(key.lower() == header_name.lower() for key in extra_headers) ) @@ -3546,7 +3664,7 @@ class MCPServerManager: self, server: MCPServer, prompt_name: str, - arguments: Optional[dict[str, Any]] = None, + arguments: Optional[dict[str, str]] = None, mcp_auth_header: Optional[Union[str, dict[str, str]]] = None, extra_headers: Optional[dict[str, str]] = None, raw_headers: Optional[dict[str, str]] = None, @@ -3606,7 +3724,7 @@ class MCPServerManager: and base_port == target_port ) - async def _fetch_oauth_discovery_url(self, url: str, server_url: str) -> Any: + async def _fetch_oauth_discovery_url(self, url: str, server_url: str) -> httpx.Response: client = get_async_httpx_client( llm_provider=httpxSpecialProvider.MCP, params={"timeout": MCP_METADATA_TIMEOUT}, @@ -3807,7 +3925,7 @@ class MCPServerManager: try: response = await self._fetch_oauth_discovery_url(resource_metadata_url, server_url) response.raise_for_status() - data = response.json() + data: _ProtectedResourceMetadataPayload = response.json() except SSRFError as exc: verbose_logger.warning( "MCP OAuth discovery: refusing to fetch resource metadata from %s " @@ -3932,7 +4050,7 @@ class MCPServerManager: try: response = await self._fetch_oauth_discovery_url(url, server_url) response.raise_for_status() - data = response.json() + data: _AuthorizationServerMetadataPayload = response.json() except SSRFError as exc: verbose_logger.warning( "MCP OAuth discovery: refusing to fetch authorization-server " @@ -3993,7 +4111,7 @@ class MCPServerManager: @staticmethod def _build_azure_authorization_server_metadata( - parsed_issuer_url: Any, + parsed_issuer_url: ParseResult, ) -> Optional[MCPOAuthMetadata]: path_parts = [part for part in (parsed_issuer_url.path or "").split("/") if part] if parsed_issuer_url.netloc not in _AZURE_ENTRA_HOSTS or len(path_parts) != 2 or path_parts[1] != "v2.0": @@ -4054,7 +4172,7 @@ class MCPServerManager: "aws_session_name": credentials_dict.get("aws_session_name"), } - def _extract_scopes(self, scopes_value: Any) -> Optional[list[str]]: + def _extract_scopes(self, scopes_value: str | Sequence[object] | None) -> Optional[list[str]]: if isinstance(scopes_value, str): scopes = [s.strip() for s in scopes_value.split() if s.strip()] return scopes or None @@ -4292,7 +4410,7 @@ class MCPServerManager: return match_known_tool_name(tool_name, server, server.allowed_tools or ()) is not None return match_known_tool_name(tool_name, server, server.disallowed_tools or ()) is None - def validate_allowed_params(self, tool_name: str, arguments: dict[str, Any], server: MCPServer) -> None: + def validate_allowed_params(self, tool_name: str, arguments: _ToolArguments, server: MCPServer) -> None: """ Filter arguments to only include allowed parameters for the given tool. @@ -4373,7 +4491,7 @@ class MCPServerManager: self, server: MCPServer, tool_name: str, - arguments: dict[str, Any], + arguments: _ToolArguments, ) -> CallToolResult: """ Call an OpenAPI tool handler directly. @@ -4537,7 +4655,7 @@ class MCPServerManager: def _create_during_hook_task( self, name: str, - arguments: dict[str, Any], + arguments: _ToolArguments, server_name_from_prefix: Optional[str], user_api_key_auth: Optional[UserAPIKeyAuth], proxy_logging_obj: ProxyLogging, @@ -4636,7 +4754,7 @@ class MCPServerManager: self, mcp_server: MCPServer, original_tool_name: str, - arguments: dict[str, Any], + arguments: _ToolArguments, tasks: list, mcp_auth_header: Optional[str], mcp_server_auth_headers: Optional[dict[str, dict[str, str]]], @@ -4990,7 +5108,7 @@ class MCPServerManager: # shadow the resolver, double-resolving and hiding the per-server challenge. return oauth2_headers - user_id = getattr(user_api_key_auth, "user_id", None) + user_id: str | None = getattr(user_api_key_auth, "user_id", None) if not user_id: return oauth2_headers @@ -5091,7 +5209,7 @@ class MCPServerManager: self, server_name: str, name: str, - arguments: dict[str, Any], + arguments: _ToolArguments, user_api_key_auth: Optional[UserAPIKeyAuth] = None, mcp_auth_header: Optional[str] = None, mcp_server_auth_headers: Optional[dict[str, dict[str, str]]] = None, @@ -5330,7 +5448,7 @@ class MCPServerManager: # Pending/rejected servers are excluded at the DB level so we never load them. from litellm.proxy._experimental.mcp_server.db import LiteLLM_MCPServerTable - raw_rows = await MCPServerRepository(prisma_client).table.find_many( + raw_rows: Sequence[BaseModel] = await MCPServerRepository(prisma_client).table.find_many( where={ "OR": [ {"approval_status": None}, @@ -5836,7 +5954,7 @@ class MCPServerManager: @staticmethod def _env_vars_to_models( - env_vars: Optional[list[dict[str, Any]]], + env_vars: Optional[_EnvVarList], ) -> Optional[list[MCPEnvVar]]: if env_vars is None: return None diff --git a/litellm/proxy/_experimental/mcp_server/sampling_handler.py b/litellm/proxy/_experimental/mcp_server/sampling_handler.py index 65630f74e90..5bad530f37b 100644 --- a/litellm/proxy/_experimental/mcp_server/sampling_handler.py +++ b/litellm/proxy/_experimental/mcp_server/sampling_handler.py @@ -10,16 +10,23 @@ MCP Spec Reference: https://modelcontextprotocol.io/specification/2025-11-25/client/sampling """ -from typing import Any, Dict, List, Optional, Union import typing +from collections.abc import Mapping, Sequence +from typing import Any, Dict, List, NamedTuple, Optional, Protocol, Union if typing.TYPE_CHECKING: + from fastapi import Request + from mcp.client.session import ClientSession + from mcp.shared.context import RequestContext + from mcp.types import ContentBlock, SamplingMessageContentBlock + + from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging -from litellm._logging import verbose_logger - from fastapi import HTTPException +from litellm._logging import verbose_logger + # Guard imports that require the mcp package try: from mcp.types import ( @@ -65,7 +72,7 @@ def _resolve_model_from_preferences( import litellm # Build list of available model names from proxy Router or litellm.model_list - available_model_names: list = [] + available_model_names: list[str] = [] try: from litellm.proxy.proxy_server import llm_router @@ -83,7 +90,7 @@ def _resolve_model_from_preferences( available_model_names.append(entry) if model_preferences and model_preferences.hints: for hint in model_preferences.hints: - hint_name = getattr(hint, "name", None) + hint_name: str | None = getattr(hint, "name", None) if not hint_name: continue # Try direct match first @@ -133,7 +140,7 @@ def _resolve_model_from_preferences( ) return available_model_names[0] # Last resort - use LiteLLM default or raise error - default_sampling_model = getattr(litellm, "default_mcp_sampling_model", None) + default_sampling_model: str | None = getattr(litellm, "default_mcp_sampling_model", None) if default_sampling_model: verbose_logger.debug( "MCP sampling model resolution: using litellm.default_mcp_sampling_model='%s'", @@ -153,6 +160,13 @@ def _has_priorities(model_preferences: "ModelPreferences") -> bool: ) +class _ScoredModel(NamedTuple): + name: str + cost: float + max_output: float + output_tps: float + + def _select_model_by_priority( model_names: List[str], model_preferences: "ModelPreferences", @@ -183,12 +197,12 @@ def _select_model_by_priority( """ import litellm as _litellm - cost_weight = getattr(model_preferences, "costPriority", None) or 0.0 - speed_weight = getattr(model_preferences, "speedPriority", None) or 0.0 - intel_weight = getattr(model_preferences, "intelligencePriority", None) or 0.0 + cost_weight: float = getattr(model_preferences, "costPriority", None) or 0.0 + speed_weight: float = getattr(model_preferences, "speedPriority", None) or 0.0 + intel_weight: float = getattr(model_preferences, "intelligencePriority", None) or 0.0 # Gather raw metrics for each model - scored: List[Dict[str, Any]] = [] + scored: list[_ScoredModel] = [] for name in model_names: try: info = _litellm.get_model_info(name) @@ -200,12 +214,12 @@ def _select_model_by_priority( max_output = info.get("max_output_tokens") or info.get("max_tokens") or 0 output_tps = info.get("output_tokens_per_second") or 0.0 scored.append( - { - "name": name, - "cost": total_cost, - "max_output": max_output, - "output_tps": output_tps, - } + _ScoredModel( + name=name, + cost=total_cost, + max_output=max_output, + output_tps=output_tps, + ) ) if not scored: @@ -222,9 +236,9 @@ def _select_model_by_priority( normed = [1.0 - n for n in normed] return normed - costs = [s["cost"] for s in scored] - max_outputs = [float(s["max_output"]) for s in scored] - output_tps_values = [s["output_tps"] for s in scored] + costs = [s.cost for s in scored] + max_outputs = [float(s.max_output) for s in scored] + output_tps_values = [s.output_tps for s in scored] # costPriority: lower cost → higher score (invert) cost_scores = _normalise(costs, invert=True) @@ -243,7 +257,7 @@ def _select_model_by_priority( score = cost_weight * cost_scores[i] + speed_weight * speed_scores[i] + intel_weight * intel_scores[i] verbose_logger.debug( "MCP priority scoring: model=%s cost_score=%.3f speed_score=%.3f intel_score=%.3f → weighted=%.3f", - entry["name"], + entry.name, cost_scores[i], speed_scores[i], intel_scores[i], @@ -251,14 +265,14 @@ def _select_model_by_priority( ) if score > best_score: best_score = score - best_name = entry["name"] + best_name = entry.name return best_name def _convert_mcp_content_to_openai( - content: Any, -) -> Union[str, Dict[str, Any], List[Dict[str, Any]]]: + content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]", +) -> "str | dict[str, object] | list[dict[str, object]]": """ Convert MCP SamplingMessage content to OpenAI message content format. Handles: @@ -283,7 +297,7 @@ def _convert_mcp_content_to_openai( def _convert_single_content( content: Any, -) -> Union[Dict[str, Any], List[Dict[str, Any]]]: +) -> "dict[str, object] | list[dict[str, object]]": """Convert a single MCP content item to OpenAI format. For text/image/audio content, returns a single content-part dict. @@ -339,7 +353,7 @@ def _convert_single_content( # Marked so the message-level converter can emit it as a # separate ``{"role": "tool", ...}`` message. tool_use_id = getattr(content, "toolUseId", "") - nested_content = getattr(content, "content", []) + nested_content: Sequence[ContentBlock] = getattr(content, "content", []) if isinstance(nested_content, list): text_parts = [getattr(c, "text", str(c)) for c in nested_content if getattr(c, "type", None) == "text"] result_text = "\n".join(text_parts) if text_parts else "" @@ -358,7 +372,7 @@ def _convert_single_content( def _convert_mcp_messages_to_openai( messages: List["SamplingMessage"], system_prompt: Optional[str] = None, -) -> List[Dict[str, Any]]: +) -> "Sequence[Mapping[str, object]]": """ Convert MCP SamplingMessage list to OpenAI messages format. MCP messages use: @@ -369,7 +383,7 @@ def _convert_mcp_messages_to_openai( - role: "system" | "user" | "assistant" | "tool" - content: str | list[content_part] """ - openai_messages: List[Dict[str, Any]] = [] + openai_messages: list[Mapping[str, object]] = [] # Add system prompt if provided if system_prompt: openai_messages.append({"role": "system", "content": system_prompt}) @@ -380,7 +394,7 @@ def _convert_mcp_messages_to_openai( if role == "assistant" and _has_tool_use(content): tool_calls = _extract_tool_calls(content) if tool_calls: - openai_msg: Dict[str, Any] = { + openai_msg: dict[str, object] = { "role": "assistant", "tool_calls": tool_calls, } @@ -400,7 +414,7 @@ def _convert_mcp_messages_to_openai( # tool_use / tool_result that slipped past the fast-path checks # above (e.g. unexpected role, single non-list content). converted = _convert_mcp_content_to_openai(content) - converted_parts = ( + converted_parts: Sequence[Mapping[str, object]] = ( converted if isinstance(converted, list) else ([converted] if isinstance(converted, dict) else []) ) @@ -422,7 +436,7 @@ def _convert_mcp_messages_to_openai( # Emit assistant message with tool_calls if any were found if tool_call_markers: - openai_msg_tc: Dict[str, Any] = { + openai_msg_tc: dict[str, object] = { "role": "assistant", "tool_calls": tool_call_markers, } @@ -442,21 +456,25 @@ def _convert_mcp_messages_to_openai( return openai_messages -def _has_tool_use(content: Any) -> bool: +def _has_tool_use(content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]") -> bool: """Check if content contains ToolUseContent.""" if isinstance(content, list): return any(getattr(c, "type", None) == "tool_use" for c in content) - return getattr(content, "type", None) == "tool_use" + content_type: str | None = getattr(content, "type", None) + return content_type == "tool_use" -def _has_tool_result(content: Any) -> bool: +def _has_tool_result(content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]") -> bool: """Check if content contains ToolResultContent.""" if isinstance(content, list): return any(getattr(c, "type", None) == "tool_result" for c in content) - return getattr(content, "type", None) == "tool_result" + content_type: str | None = getattr(content, "type", None) + return content_type == "tool_result" -def _extract_tool_calls(content: Any) -> List[Dict[str, Any]]: +def _extract_tool_calls( + content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]", +) -> "Sequence[Mapping[str, object]]": """Extract OpenAI-format tool_calls from MCP ToolUseContent.""" import json @@ -477,7 +495,9 @@ def _extract_tool_calls(content: Any) -> List[Dict[str, Any]]: return tool_calls -def _extract_text_parts(content: Any) -> Optional[str]: +def _extract_text_parts( + content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]", +) -> Optional[str]: """Extract text parts from mixed content.""" items = content if isinstance(content, list) else [content] texts = [] @@ -487,7 +507,9 @@ def _extract_text_parts(content: Any) -> Optional[str]: return "\n".join(texts) if texts else None -def _extract_tool_results(content: Any) -> List[Dict[str, Any]]: +def _extract_tool_results( + content: "SamplingMessageContentBlock | Sequence[SamplingMessageContentBlock]", +) -> "Sequence[Mapping[str, object]]": """Extract OpenAI-format tool messages from MCP ToolResultContent.""" items = content if isinstance(content, list) else [content] results = [] @@ -495,7 +517,7 @@ def _extract_tool_results(content: Any) -> List[Dict[str, Any]]: if getattr(item, "type", None) == "tool_result": tool_use_id = getattr(item, "toolUseId", "") # Extract text from nested content - nested_content = getattr(item, "content", []) + nested_content: Sequence[ContentBlock] = getattr(item, "content", []) if isinstance(nested_content, list): text_parts = [getattr(c, "text", str(c)) for c in nested_content if getattr(c, "type", None) == "text"] result_text = "\n".join(text_parts) if text_parts else "" @@ -513,7 +535,7 @@ def _extract_tool_results(content: Any) -> List[Dict[str, Any]]: def _convert_mcp_tools_to_openai( tools: Optional[List["Tool"]], -) -> Optional[List[Dict[str, Any]]]: +) -> "Sequence[Mapping[str, object]] | None": """ Convert MCP Tool definitions to OpenAI function calling format. MCP Tool: {name, description, inputSchema} @@ -541,7 +563,7 @@ def _convert_mcp_tools_to_openai( def _convert_mcp_tool_choice_to_openai( tool_choice: Optional["ToolChoice"], -) -> Optional[Union[str, Dict[str, Any]]]: +) -> "str | None": """ Convert MCP ToolChoice to OpenAI tool_choice format. MCP: {mode: "auto"} | {mode: "required"} | {mode: "none"} @@ -559,8 +581,32 @@ def _convert_mcp_tool_choice_to_openai( return "auto" +class _SamplingResponseMessage(Protocol): + @property + def content(self) -> str | None: ... + + @property + def tool_calls(self) -> Sequence[object] | None: ... + + +class _SamplingResponseChoice(Protocol): + @property + def message(self) -> _SamplingResponseMessage: ... + + @property + def finish_reason(self) -> str | None: ... + + +class _SamplingCompletionResponse(Protocol): + @property + def choices(self) -> Sequence[_SamplingResponseChoice]: ... + + @property + def model(self) -> str | None: ... + + def _convert_openai_response_to_mcp_result( - response: Any, + response: _SamplingCompletionResponse, model_name: str, ) -> Union["CreateMessageResult", "CreateMessageResultWithTools", "ErrorData"]: """ @@ -593,12 +639,12 @@ def _convert_openai_response_to_mcp_result( stop_reason = "maxTokens" else: stop_reason = "endTurn" - actual_model = getattr(response, "model", model_name) or model_name + actual_model: str = getattr(response, "model", model_name) or model_name # Check if response has tool calls tool_calls = getattr(message, "tool_calls", None) if tool_calls: # Build ToolUseContent items - content_parts: "List[Any]" = [] + content_parts: list[SamplingMessageContentBlock] = [] # Include text content if present if message.content: content_parts.append(TextContent(type="text", text=message.content)) @@ -636,7 +682,7 @@ def _convert_openai_response_to_mcp_result( ) -async def _check_model_access(model: str, user_api_key_auth: Any) -> Optional["ErrorData"]: +async def _check_model_access(model: str, user_api_key_auth: "UserAPIKeyAuth | None") -> Optional["ErrorData"]: """Enforce model-permission checks for MCP sampling requests. Runs the same authorization checks as ``/chat/completions``: @@ -678,14 +724,14 @@ async def _check_model_access(model: str, user_api_key_auth: Any) -> Optional["E try: import litellm from litellm.proxy.auth.auth_checks import ( + _check_team_member_model_access, can_key_call_model, + can_project_access_model, can_team_access_model, can_user_call_model, - can_project_access_model, - _check_team_member_model_access, + get_project_object, get_team_object, get_user_object, - get_project_object, ) try: @@ -700,16 +746,20 @@ async def _check_model_access(model: str, user_api_key_auth: Any) -> Optional["E llm_router=_llm_router, ) - _team_id = getattr(user_api_key_auth, "team_id", None) - _user_id = getattr(user_api_key_auth, "user_id", None) - _project_id = getattr(user_api_key_auth, "project_id", None) + _team_id: str | None = getattr(user_api_key_auth, "team_id", None) + _user_id: str | None = getattr(user_api_key_auth, "user_id", None) + _project_id: str | None = getattr(user_api_key_auth, "project_id", None) try: from litellm.proxy.proxy_server import ( prisma_client as _prisma_client, - user_api_key_cache as _user_api_key_cache, + ) + from litellm.proxy.proxy_server import ( proxy_logging_obj as _proxy_logging_obj, ) + from litellm.proxy.proxy_server import ( + user_api_key_cache as _user_api_key_cache, + ) except ImportError: _prisma_client = None _user_api_key_cache = None # type: ignore[assignment] @@ -799,7 +849,7 @@ async def _check_model_access(model: str, user_api_key_auth: Any) -> Optional["E async def _run_budget_checks( model: str, - user_api_key_auth: Any, + user_api_key_auth: "UserAPIKeyAuth", raw_headers: Optional[Dict[str, str]] = None, client_ip: Optional[str] = None, ) -> Optional["ErrorData"]: @@ -811,25 +861,33 @@ async def _run_budget_checks( Returns None if all checks pass, or an ErrorData describing the denial. """ try: - from litellm.proxy.auth.auth_checks import common_checks - from litellm.proxy.proxy_server import ( - general_settings, - llm_router as _llm_router, - prisma_client as _prisma_client, - proxy_logging_obj as _proxy_logging_obj, - user_api_key_cache as _user_api_key_cache, - ) + import litellm from litellm.proxy.auth.auth_checks import ( + common_checks, get_team_object, get_user_object, ) - import litellm + from litellm.proxy.proxy_server import ( + general_settings, + ) + from litellm.proxy.proxy_server import ( + llm_router as _llm_router, + ) + from litellm.proxy.proxy_server import ( + prisma_client as _prisma_client, + ) + from litellm.proxy.proxy_server import ( + proxy_logging_obj as _proxy_logging_obj, + ) + from litellm.proxy.proxy_server import ( + user_api_key_cache as _user_api_key_cache, + ) except ImportError as import_err: verbose_logger.warning("MCP sampling: budget check imports unavailable: %s", import_err) return None # Can't enforce budgets without the modules - _team_id = getattr(user_api_key_auth, "team_id", None) - _user_id = getattr(user_api_key_auth, "user_id", None) + _team_id: str | None = getattr(user_api_key_auth, "team_id", None) + _user_id: str | None = getattr(user_api_key_auth, "user_id", None) team_obj = None if _team_id and _prisma_client and _user_api_key_cache: @@ -889,7 +947,7 @@ async def _run_budget_checks( # common_checks runs. _tag_max_budget_check inside common_checks only # inspects request_body; without this pre-merge, header-supplied tags # bypass per-tag budget enforcement (mirroring the regular auth path). - request_body: Dict[str, Any] = {"model": model} + request_body: dict[str, object] = {"model": model} try: from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup @@ -935,7 +993,7 @@ async def _run_budget_checks( def _build_sampling_request( raw_headers: Optional[Dict[str, str]] = None, client_ip: Optional[str] = None, -) -> Any: +) -> "Request": """Build a synthetic FastAPI Request for sampling sub-calls. Converts the original MCP connection's HTTP headers into ASGI @@ -961,7 +1019,7 @@ def _build_sampling_request( from fastapi import Request # --- Build ASGI headers --- - _scope_headers: list = [(b"content-type", b"application/json")] + _scope_headers: list[tuple[bytes, bytes]] = [(b"content-type", b"application/json")] # Hop-by-hop headers that must NOT be forwarded into the # synthetic request (they describe the original HTTP framing, # not the logical request). @@ -1001,8 +1059,8 @@ def _build_sampling_request( try: import litellm.proxy.proxy_server as proxy_server - _proxy_host = getattr(proxy_server, "server_host", None) - _proxy_port = getattr(proxy_server, "server_port", None) + _proxy_host: str | None = getattr(proxy_server, "server_host", None) + _proxy_port: str | int | None = getattr(proxy_server, "server_port", None) if _proxy_host: _server_host = str(_proxy_host) @@ -1016,7 +1074,7 @@ def _build_sampling_request( if client_ip: _client_tuple = (client_ip, 0) - scope: Dict[str, Any] = { + scope: dict[str, object] = { "type": "http", "method": "POST", "path": "/mcp/sampling/createMessage", @@ -1035,7 +1093,7 @@ def _build_sampling_request( async def _build_completion_kwargs( params: "CreateMessageRequestParams", model: str, - user_api_key_auth: Any, + user_api_key_auth: "UserAPIKeyAuth", raw_headers: Optional[Dict[str, str]], client_ip: Optional[str], ) -> Dict[str, Any]: @@ -1078,7 +1136,7 @@ async def _build_completion_kwargs( async def _run_guardrails_and_call_llm( completion_kwargs: Dict[str, Any], - user_api_key_auth: Any, + user_api_key_auth: "UserAPIKeyAuth", ) -> Any: try: from litellm.proxy.proxy_server import proxy_logging_obj as _plo @@ -1111,10 +1169,10 @@ async def _run_guardrails_and_call_llm( async def handle_sampling_create_message( - context: Any, + context: "RequestContext[ClientSession, object]", params: "CreateMessageRequestParams", default_model: Optional[str] = None, - user_api_key_auth: Optional[Any] = None, + user_api_key_auth: "UserAPIKeyAuth | None" = None, raw_headers: Optional[Dict[str, str]] = None, client_ip: Optional[str] = None, ) -> Union["CreateMessageResult", "CreateMessageResultWithTools", "ErrorData"]: @@ -1184,7 +1242,7 @@ async def handle_sampling_create_message( client_ip=client_ip, ) - openai_messages = completion_kwargs["messages"] + openai_messages: Sequence[Mapping[str, object]] = completion_kwargs["messages"] openai_tools = completion_kwargs.get("tools") verbose_logger.debug( "MCP sampling: calling litellm.acompletion with model=%s, num_messages=%d, has_tools=%s", @@ -1193,7 +1251,7 @@ async def handle_sampling_create_message( bool(openai_tools), ) - response = await _run_guardrails_and_call_llm( + response: _SamplingCompletionResponse = await _run_guardrails_and_call_llm( completion_kwargs=completion_kwargs, user_api_key_auth=user_api_key_auth, ) @@ -1214,7 +1272,6 @@ async def handle_sampling_create_message( RateLimitError, ServiceUnavailableError, ) - from litellm.proxy._types import ProxyException if isinstance( diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index da7a72e1cff..a29899a3965 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -6,8 +6,20 @@ import concurrent.futures import inspect import json import os +from collections.abc import Mapping, Sequence from datetime import datetime, timezone -from typing import Any, Dict, List, Literal, Optional, Type, TypeVar, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Dict, + List, + Literal, + Optional, + Type, + TypeVar, + Union, + cast, +) from urllib.parse import urlparse from fastapi import APIRouter, Depends, HTTPException, Request @@ -49,12 +61,44 @@ from litellm.types.guardrails import ( ToolPermissionGuardrailConfigModel, ) +if TYPE_CHECKING: + from types import CodeType + + from prisma.actions import LiteLLM_GuardrailsTableActions + from prisma.models import LiteLLM_GuardrailsTable + + from litellm.proxy.utils import PrismaClient + #### GUARDRAILS ENDPOINTS #### router = APIRouter() GUARDRAIL_REGISTRY = GuardrailRegistry() +def _guardrails_table(prisma_client: "PrismaClient") -> "LiteLLM_GuardrailsTableActions[LiteLLM_GuardrailsTable]": + table: LiteLLM_GuardrailsTableActions[LiteLLM_GuardrailsTable] = GuardrailsRepository(prisma_client).table + return table + + +async def _create_guardrail_row(prisma_client: "PrismaClient", data: Mapping[str, object]) -> "LiteLLM_GuardrailsTable": + row: LiteLLM_GuardrailsTable = await GuardrailsRepository(prisma_client).table.create(data=data) + return row + + +async def _delete_guardrail_row(prisma_client: "PrismaClient", where: Mapping[str, object]) -> None: + await GuardrailsRepository(prisma_client).table.delete(where=where) + + +async def _find_team_guardrail_rows( + prisma_client: "PrismaClient", where: Mapping[str, object] +) -> "Sequence[LiteLLM_GuardrailsTable]": + rows: Sequence[LiteLLM_GuardrailsTable] = await GuardrailsRepository(prisma_client).table.find_many( + where=where, + order={"created_at": "desc"}, + ) + return rows + + def _get_guardrails_list_response( guardrails_config: List[Dict], ) -> ListGuardrailsResponse: @@ -363,7 +407,7 @@ async def create_guardrail( # Configuration error — roll back the DB write so the guardrail isn't orphaned if prisma_client is not None: try: - await GuardrailsRepository(prisma_client).table.delete(where={"guardrail_id": guardrail_id}) + await _delete_guardrail_row(prisma_client, where={"guardrail_id": guardrail_id}) except Exception as rollback_err: verbose_proxy_logger.warning(f"Rollback failed for guardrail '{guardrail_id}': {rollback_err}") raise HTTPException( @@ -571,7 +615,7 @@ class RegisterGuardrailRequest(BaseModel): guardrail_name: str litellm_params: Dict[str, Any] # guardrail, mode, api_base required; api_key, headers, etc. optional - guardrail_info: Optional[Dict[str, Any]] = None + guardrail_info: Optional[Dict[str, object]] = None team_id: Optional[str] = None def get_litellm_params_dict(self) -> Dict[str, Any]: @@ -600,8 +644,8 @@ class GuardrailSubmissionItem(BaseModel): team_guardrail: bool = ( False # True when submitted via team (team_id set); use to distinguish team vs regular guardrails ) - litellm_params: Optional[Dict[str, Any]] = None - guardrail_info: Optional[Dict[str, Any]] = None + litellm_params: Optional[Dict[str, object]] = None + guardrail_info: Optional[Dict[str, object]] = None submitted_by_user_id: Optional[str] = None submitted_by_email: Optional[str] = None submitted_at: Optional[datetime] = None @@ -685,9 +729,7 @@ async def register_guardrail( ) try: - existing = await GuardrailsRepository(prisma_client).table.find_unique( - where={"guardrail_name": request.guardrail_name} - ) + existing = await _guardrails_table(prisma_client).find_unique(where={"guardrail_name": request.guardrail_name}) if existing is not None: raise HTTPException( status_code=400, @@ -708,7 +750,8 @@ async def register_guardrail( guardrail_info_str = safe_dumps(guardrail_info) try: - created = await GuardrailsRepository(prisma_client).table.create( + created = await _create_guardrail_row( + prisma_client, data={ "guardrail_name": request.guardrail_name, "litellm_params": litellm_params_str, @@ -718,7 +761,7 @@ async def register_guardrail( "submitted_at": now, "created_at": now, "updated_at": now, - } + }, ) return RegisterGuardrailResponse( guardrail_id=created.guardrail_id, @@ -731,7 +774,7 @@ async def register_guardrail( raise HTTPException(status_code=500, detail=str(e)) -def _parse_json_field(value: Any) -> Optional[Dict[str, Any]]: +def _parse_json_field(value: object) -> Optional[Dict[str, Any]]: if value is None: return None if isinstance(value, dict): @@ -768,7 +811,7 @@ async def _get_user_team_ids(user_api_key_dict: UserAPIKeyAuth) -> List[str]: return [t for t in user_obj.teams if t] -def _row_to_submission_item(row: Any) -> GuardrailSubmissionItem: +def _row_to_submission_item(row: "LiteLLM_GuardrailsTable") -> GuardrailSubmissionItem: from litellm.litellm_core_utils.litellm_logging import _get_masked_values guardrail_info = _parse_json_field(row.guardrail_info) or {} @@ -835,7 +878,7 @@ async def list_guardrail_submissions( ) try: - where_clause: Dict[str, Any] = {"team_id": {"not": None}} + where_clause: Dict[str, object] = {"team_id": {"not": None}} if visible_team_ids is not None: if not visible_team_ids: # Non-admin with no team memberships: nothing visible. @@ -846,10 +889,7 @@ async def list_guardrail_submissions( where_clause["team_id"] = {"in": visible_team_ids} # Single query: fetch team guardrails visible to the caller - all_team_rows = await GuardrailsRepository(prisma_client).table.find_many( - where=where_clause, - order={"created_at": "desc"}, - ) + all_team_rows = await _find_team_guardrail_rows(prisma_client, where_clause) # Derive summary counts from the full result set total = len(all_team_rows) @@ -909,7 +949,7 @@ async def get_guardrail_submission( is_admin = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN try: - row = await GuardrailsRepository(prisma_client).table.find_unique(where={"guardrail_id": guardrail_id}) + row = await _guardrails_table(prisma_client).find_unique(where={"guardrail_id": guardrail_id}) if row is None: raise HTTPException(status_code=404, detail="Guardrail submission not found") if not is_admin: @@ -946,7 +986,7 @@ async def approve_guardrail_submission( raise HTTPException(status_code=500, detail="Prisma client not initialized") try: - row = await GuardrailsRepository(prisma_client).table.find_unique(where={"guardrail_id": guardrail_id}) + row = await _guardrails_table(prisma_client).find_unique(where={"guardrail_id": guardrail_id}) if row is None: raise HTTPException(status_code=404, detail="Guardrail submission not found") if row.status != "pending_review": @@ -956,7 +996,7 @@ async def approve_guardrail_submission( ) now = datetime.now(timezone.utc) - await GuardrailsRepository(prisma_client).table.update( + await _guardrails_table(prisma_client).update( where={"guardrail_id": guardrail_id}, data={"status": "active", "reviewed_at": now, "updated_at": now}, ) @@ -1026,7 +1066,7 @@ async def reject_guardrail_submission( raise HTTPException(status_code=500, detail="Prisma client not initialized") try: - row = await GuardrailsRepository(prisma_client).table.find_unique(where={"guardrail_id": guardrail_id}) + row = await _guardrails_table(prisma_client).find_unique(where={"guardrail_id": guardrail_id}) if row is None: raise HTTPException(status_code=404, detail="Guardrail submission not found") if row.status != "pending_review": @@ -1036,7 +1076,7 @@ async def reject_guardrail_submission( ) now = datetime.now(timezone.utc) - await GuardrailsRepository(prisma_client).table.update( + await _guardrails_table(prisma_client).update( where={"guardrail_id": guardrail_id}, data={"status": "rejected", "reviewed_at": now, "updated_at": now}, ) @@ -1886,13 +1926,13 @@ class TestCustomCodeGuardrailRequest(BaseModel): custom_code: str """The Python-like code containing the apply_guardrail function.""" - test_input: Dict[str, Any] + test_input: Dict[str, object] """The test input to pass to the guardrail. Should contain 'texts', optionally 'images', 'tools', etc.""" input_type: str = "request" """Whether this is a 'request' or 'response' input type.""" - request_data: Optional[Dict[str, Any]] = None + request_data: Optional[Dict[str, object]] = None """Optional mock request_data (model, user_id, team_id, metadata, etc.).""" @@ -1902,7 +1942,7 @@ class TestCustomCodeGuardrailResponse(BaseModel): success: bool """Whether the test executed successfully (no errors).""" - result: Optional[Dict[str, Any]] = None + result: Optional[Dict[str, object]] = None """The guardrail result: action (allow/block/modify), reason, modified_texts, etc.""" error: Optional[str] = None @@ -2006,7 +2046,7 @@ async def test_custom_code_guardrail( exec_globals = build_sandbox_globals() try: - compiled = compile_sandboxed(request.custom_code) + compiled: CodeType = compile_sandboxed(request.custom_code) exec(compiled, exec_globals) # noqa: S102 except SyntaxError as e: return TestCustomCodeGuardrailResponse( @@ -2030,7 +2070,7 @@ async def test_custom_code_guardrail( error_type="compilation", ) - apply_fn = exec_globals["apply_guardrail"] + apply_fn: object = exec_globals["apply_guardrail"] if not callable(apply_fn): return TestCustomCodeGuardrailResponse( success=False, @@ -2055,7 +2095,7 @@ async def test_custom_code_guardrail( # Step 4: Execute the function with timeout protection - def execute_guardrail(): + def execute_guardrail() -> object: return apply_fn(test_inputs, safe_request_data, request.input_type) try: diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 80d9ee21a44..1a0978c8eec 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -15,9 +15,9 @@ These are members of a Team on LiteLLM import asyncio import json import traceback -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from datetime import datetime, timezone -from typing import Any, Optional, cast +from typing import Any, cast import fastapi from fastapi import APIRouter, Depends, Header, HTTPException, Request, status @@ -28,6 +28,10 @@ from litellm._uuid import uuid from litellm.proxy._types import * from litellm.proxy.auth.auth_checks import get_team_object, get_user_object from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.user_api_key_cache import ( + object_permission_cache_key, + user_object_permission_id_cache_key, +) from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks from litellm.proxy.management_endpoints.common_daily_activity import ( DailySpendRecord, @@ -45,10 +49,6 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( generate_key_helper_fn, prepare_metadata_fields, ) -from litellm.proxy.common_utils.user_api_key_cache import ( - object_permission_cache_key, - user_object_permission_id_cache_key, -) from litellm.proxy.management_helpers.object_permission_utils import ( _set_object_permission, handle_update_object_permission_common, @@ -82,11 +82,74 @@ from litellm.types.proxy.management_endpoints.scim_v2 import ( ) if TYPE_CHECKING: + from prisma import models as prisma_models + from prisma import types as prisma_types + from prisma.actions import ( + LiteLLM_InvitationLinkActions, + LiteLLM_OrganizationMembershipActions, + LiteLLM_TeamMembershipActions, + LiteLLM_TeamTableActions, + LiteLLM_UserTableActions, + LiteLLM_VerificationTokenActions, + ) + + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.proxy_server import PrismaClient + from litellm.proxy.utils import ProxyLogging router = APIRouter() +def _user_table( + prisma_client: "PrismaClient | None", +) -> "LiteLLM_UserTableActions[prisma_models.LiteLLM_UserTable]": + user_table: LiteLLM_UserTableActions[prisma_models.LiteLLM_UserTable] = UserRepository(prisma_client).table + return user_table + + +def _team_table( + prisma_client: "PrismaClient | None", +) -> "LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable]": + team_table: LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable] = TeamRepository(prisma_client).table + return team_table + + +def _verification_token_table( + prisma_client: "PrismaClient | None", +) -> "LiteLLM_VerificationTokenActions[prisma_models.LiteLLM_VerificationToken]": + token_table: LiteLLM_VerificationTokenActions[prisma_models.LiteLLM_VerificationToken] = ( + VerificationTokenRepository(prisma_client).table + ) + return token_table + + +def _organization_membership_table( + prisma_client: "PrismaClient | None", +) -> "LiteLLM_OrganizationMembershipActions[prisma_models.LiteLLM_OrganizationMembership]": + membership_table: LiteLLM_OrganizationMembershipActions[prisma_models.LiteLLM_OrganizationMembership] = ( + OrganizationMembershipRepository(prisma_client).table + ) + return membership_table + + +def _invitation_link_table( + prisma_client: "PrismaClient | None", +) -> "LiteLLM_InvitationLinkActions[prisma_models.LiteLLM_InvitationLink]": + invitation_table: LiteLLM_InvitationLinkActions[prisma_models.LiteLLM_InvitationLink] = InvitationLinkRepository( + prisma_client + ).table + return invitation_table + + +def _team_membership_table( + prisma_client: "PrismaClient | None", +) -> "LiteLLM_TeamMembershipActions[prisma_models.LiteLLM_TeamMembership]": + team_membership_table: LiteLLM_TeamMembershipActions[prisma_models.LiteLLM_TeamMembership] = ( + TeamMembershipRepository(prisma_client).table + ) + return team_membership_table + + def _hash_password_in_dict(data: dict) -> None: """Hash password field in-place if present.""" if "password" in data and data["password"] is not None: @@ -138,7 +201,7 @@ def _update_internal_new_user_params(data_json: dict, data: NewUserRequest) -> d async def _check_duplicate_user_field( field_name: str, field_value: str | None, - prisma_client: Any, + prisma_client: "PrismaClient | None", *, case_insensitive: bool = False, label: str | None = None, @@ -177,7 +240,7 @@ async def _check_duplicate_user_field( ) -async def _check_duplicate_user_email(user_email: str | None, prisma_client: Any) -> None: +async def _check_duplicate_user_email(user_email: str | None, prisma_client: "PrismaClient | None") -> None: """ Helper function to check if a user email already exists in the database. """ @@ -190,7 +253,7 @@ async def _check_duplicate_user_email(user_email: str | None, prisma_client: Any ) -async def _check_duplicate_user_id(user_id: str | None, prisma_client: Any) -> None: +async def _check_duplicate_user_id(user_id: str | None, prisma_client: "PrismaClient | None") -> None: """ Helper function to check if a user id already exists in the database. """ @@ -724,8 +787,8 @@ _SCIM_DIRECTORY_METADATA_KEYS = frozenset( def _redact_scim_enterprise_metadata( - metadata: dict[str, Any] | None, -) -> dict[str, Any] | None: + metadata: dict[str, object] | None, +) -> dict[str, object] | None: """SCIM enterprise attributes, entitlements, and roles are persisted in user metadata so reporting can group on them, but they are directory-only fields that generic user-info endpoints must not surface; SCIM clients read them @@ -845,7 +908,7 @@ async def user_info( async def _check_user_info_v2_access( user_api_key_dict: UserAPIKeyAuth, target_user_id: str, -) -> Optional["LiteLLM_UserTable"]: +) -> "prisma_models.LiteLLM_UserTable | None": """ Check if the caller is allowed to access the target user's info. @@ -867,7 +930,7 @@ async def _check_user_info_v2_access( # Helper: fetch the target user row (reused across branches). object_permission is included so # callers can read the user's MCP/vector-store entitlements without a second round trip. async def _fetch_target_user(): - return await UserRepository(prisma_client).table.find_unique( + return await _user_table(prisma_client).find_unique( where={"user_id": target_user_id}, include={"object_permission": True} ) @@ -882,9 +945,7 @@ async def _check_user_info_v2_access( # Rule 3: Team admins can look up users in their teams if user_api_key_dict.user_id is not None: # Get caller's teams - caller_user = await UserRepository(prisma_client).table.find_unique( - where={"user_id": user_api_key_dict.user_id} - ) + caller_user = await _user_table(prisma_client).find_unique(where={"user_id": user_api_key_dict.user_id}) if caller_user is not None and caller_user.teams: # Fetch the target user ONCE, before the loop target_user = await _fetch_target_user() @@ -892,7 +953,7 @@ async def _check_user_info_v2_access( return None # Get all teams the caller belongs to - teams = await TeamRepository(prisma_client).table.find_many(where={"team_id": {"in": caller_user.teams}}) + teams = await _team_table(prisma_client).find_many(where={"team_id": {"in": caller_user.teams}}) for team in teams: team_obj = LiteLLM_TeamTable.model_validate(team.model_dump()) if _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team_obj): @@ -1160,7 +1221,7 @@ async def _schedule_user_update_audit_log( if prisma_client is None: return try: - updated_user_row = await UserRepository(prisma_client).table.find_first(where={"user_id": response["user_id"]}) + updated_user_row = await _user_table(prisma_client).find_first(where={"user_id": response["user_id"]}) if updated_user_row: user_row_typed = LiteLLM_UserTable.model_validate(updated_user_row.model_dump(exclude_none=True)) asyncio.create_task( @@ -1207,7 +1268,7 @@ def _check_user_update_authz( async def _invalidate_user_spend_counter_if_changed( - non_default_values: dict[str, Any], + non_default_values: Mapping[str, object], ) -> None: """Invalidate the cross-pod spend counter after a direct ``spend`` change. @@ -1295,13 +1356,9 @@ async def _update_single_user_helper( existing_user_row: BaseModel | None = None if user_request.user_id: - existing_user_row = await UserRepository(prisma_client).table.find_first( - where={"user_id": user_request.user_id} - ) + existing_user_row = await _user_table(prisma_client).find_first(where={"user_id": user_request.user_id}) elif user_request.user_email: - existing_user_row = await UserRepository(prisma_client).table.find_first( - where={"user_email": user_request.user_email} - ) + existing_user_row = await _user_table(prisma_client).find_first(where={"user_email": user_request.user_email}) _check_user_update_authz(user_request, user_api_key_dict, existing_user_row) @@ -1690,7 +1747,7 @@ async def bulk_user_update( detail="Only proxy admins can update all users at once.", ) # Optimized path for updating all users directly in database - all_users_in_db = await UserRepository(prisma_client).table.find_many(order={"created_at": "desc"}) + all_users_in_db = await _user_table(prisma_client).find_many(order={"created_at": "desc"}) if not all_users_in_db: raise HTTPException( @@ -1805,9 +1862,9 @@ async def bulk_user_update( async def get_user_key_counts( - prisma_client, + prisma_client: "PrismaClient | None", user_ids: list[str] | None = None, -): +) -> Mapping[str, int]: """ Helper function to get the count of keys for each user using Prisma's count method. @@ -1823,7 +1880,7 @@ async def get_user_key_counts( if not user_ids or len(user_ids) == 0: return {} - result = {} + result: dict[str, int] = {} # Get count for each user_id individually for user_id in user_ids: @@ -1876,9 +1933,9 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str async def _authorize_user_list_request( user_api_key_dict: UserAPIKeyAuth, organization_ids: str | None, - prisma_client: Any, - user_api_key_cache: Any, - proxy_logging_obj: Any, + prisma_client: "PrismaClient | None", + user_api_key_cache: "UserApiKeyCache", + proxy_logging_obj: "ProxyLogging | None", ) -> str | None: """ Authorize the /user/list request and return the (possibly scoped) organization_ids string. @@ -2016,7 +2073,7 @@ async def get_users( skip = (page - 1) * page_size # Build where conditions based on provided parameters - where_conditions: dict[str, Any] = {} + where_conditions: dict[str, object] = {} if role: where_conditions["user_role"] = role @@ -2064,7 +2121,7 @@ async def get_users( _validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None ) - users = await UserRepository(prisma_client).table.find_many( + users: Sequence[prisma_models.LiteLLM_UserTable] | None = await UserRepository(prisma_client).table.find_many( where=where_conditions, skip=skip, take=page_size, @@ -2072,7 +2129,7 @@ async def get_users( ) # Get total count of user rows - total_count = await UserRepository(prisma_client).table.count(where=where_conditions) + total_count: int = await UserRepository(prisma_client).table.count(where=where_conditions) # Get key count for each user if users is not None: @@ -2168,7 +2225,7 @@ async def delete_user( caller_admin_org_ids: set = set() if not caller_is_proxy_admin: caller_memberships = ( - await OrganizationMembershipRepository(prisma_client).table.find_many( + await _organization_membership_table(prisma_client).find_many( where={ "user_id": user_api_key_dict.user_id, "user_role": LitellmUserRoles.ORG_ADMIN.value, @@ -2188,7 +2245,7 @@ async def delete_user( # an N+1 DB call when delete_user is called with a large user_ids list. target_org_ids_by_user: dict[str, set] = {} if not caller_is_proxy_admin: - all_target_memberships = await OrganizationMembershipRepository(prisma_client).table.find_many( + all_target_memberships = await _organization_membership_table(prisma_client).find_many( where={"user_id": {"in": data.user_ids}} ) for m in all_target_memberships: @@ -2276,10 +2333,10 @@ async def delete_user( # End of Audit logging ## DELETE ASSOCIATED KEYS - await VerificationTokenRepository(prisma_client).table.delete_many(where={"user_id": {"in": data.user_ids}}) + await _verification_token_table(prisma_client).delete_many(where={"user_id": {"in": data.user_ids}}) ## DELETE ASSOCIATED INVITATION LINKS - await InvitationLinkRepository(prisma_client).table.delete_many( + await _invitation_link_table(prisma_client).delete_many( where={ "OR": [ {"user_id": {"in": data.user_ids}}, @@ -2290,13 +2347,13 @@ async def delete_user( ) ## DELETE ASSOCIATED ORGANIZATION MEMBERSHIPS - await OrganizationMembershipRepository(prisma_client).table.delete_many(where={"user_id": {"in": data.user_ids}}) + await _organization_membership_table(prisma_client).delete_many(where={"user_id": {"in": data.user_ids}}) ## DELETE ASSOCIATED TEAM MEMBERSHIPS - await TeamMembershipRepository(prisma_client).table.delete_many(where={"user_id": {"in": data.user_ids}}) + await _team_membership_table(prisma_client).delete_many(where={"user_id": {"in": data.user_ids}}) ## DELETE USERS - deleted_users = await UserRepository(prisma_client).table.delete_many(where={"user_id": {"in": data.user_ids}}) + deleted_users = await _user_table(prisma_client).delete_many(where={"user_id": {"in": data.user_ids}}) return deleted_users @@ -2348,9 +2405,9 @@ async def add_internal_user_to_organization( async def _resolve_org_filter_for_user_search( user_api_key_dict: UserAPIKeyAuth, team_id: str | None, - prisma_client: Any, - user_api_key_cache: Any, - proxy_logging_obj: Any, + prisma_client: "PrismaClient | None", + user_api_key_cache: "UserApiKeyCache", + proxy_logging_obj: "ProxyLogging | None", ) -> list[str] | None: """ Return a list of org IDs to filter by, or ``None`` for no filter. @@ -2414,9 +2471,9 @@ async def _resolve_org_filter_for_user_search( async def _resolve_team_org_filter( user_api_key_dict: UserAPIKeyAuth, team_id: str, - prisma_client: Any, - user_api_key_cache: Any, - proxy_logging_obj: Any, + prisma_client: "PrismaClient | None", + user_api_key_cache: "UserApiKeyCache", + proxy_logging_obj: "ProxyLogging | None", ) -> list[str]: """Look up the team and return its org as a filter list, or raise 403.""" from litellm.proxy.management_endpoints.common_utils import _is_user_team_admin @@ -2506,7 +2563,7 @@ async def ui_view_users( skip = (page - 1) * page_size # Build where conditions based on provided parameters - where_conditions: dict[str, Any] = {} + where_conditions: prisma_types.LiteLLM_UserTableWhereInput = {} if user_id: where_conditions["user_id"] = { @@ -2525,7 +2582,7 @@ async def ui_view_users( where_conditions["organization_memberships"] = {"some": {"organization_id": {"in": org_filter_ids}}} # Query users with pagination and filters - users: list[BaseModel] | None = await UserRepository(prisma_client).table.find_many( + users = await _user_table(prisma_client).find_many( where=where_conditions, skip=skip, take=page_size, @@ -2557,7 +2614,7 @@ async def _resolve_user_email_metadata( } if not user_ids: return {} - users = await UserRepository(prisma_client).table.find_many(where={"user_id": {"in": list(user_ids)}}) + users = await _user_table(prisma_client).find_many(where={"user_id": {"in": list(user_ids)}}) return {user.user_id: {"user_email": user.user_email, "user_alias": user.user_alias} for user in users} diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index a94a75fdfa3..b910eee7130 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -18,9 +18,9 @@ import os import re import secrets import traceback -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import datetime, timedelta, timezone -from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, cast +from typing import Any, Callable, Dict, List, Literal, Optional, Protocol, Tuple, TypeVar, cast import fastapi import yaml @@ -37,6 +37,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.models.credentials import CredentialItem from litellm.proxy._experimental.mcp_server.db import ( rotate_mcp_server_credentials_master_key, rotate_mcp_user_credentials_master_key, @@ -101,8 +102,9 @@ from litellm.proxy.utils import ( handle_exception_on_proxy, is_valid_api_key, ) +from litellm.repositories.base_repository import BaseRepository from litellm.repositories.budget_repository import BudgetRepository -from litellm.repositories.config_repository import ConfigRepository +from litellm.repositories.config_repository import ConfigParam, ConfigRepository from litellm.repositories.credentials_repository import CredentialsRepository from litellm.repositories.model_repository import ModelRepository from litellm.repositories.table_repositories import ( @@ -131,6 +133,68 @@ from litellm.types.utils import ( TeamUIKeyGenerationConfig, ) +_PrismaRowT = TypeVar("_PrismaRowT") +_RepositoryModelT = TypeVar("_RepositoryModelT", bound=BaseModel) + + +class _PrismaTableActions(Protocol[_PrismaRowT]): + """Typed view of the Prisma table actions a repository exposes through its untyped ``table``.""" + + async def find_unique( + self, + *, + where: Mapping[str, object], + include: Mapping[str, object] | None = None, + ) -> _PrismaRowT | None: ... + + async def find_first( + self, + *, + where: Mapping[str, object], + include: Mapping[str, object] | None = None, + ) -> _PrismaRowT | None: ... + + async def find_many( + self, + *, + where: Mapping[str, object] | None = None, + include: Mapping[str, object] | None = None, + order: Mapping[str, object] | None = None, + skip: int | None = None, + take: int | None = None, + ) -> list[_PrismaRowT]: ... + + async def count(self, *, where: Mapping[str, object] | None = None) -> int: ... + + async def create_many(self, *, data: Sequence[Mapping[str, object]]) -> int: ... + + async def update( + self, + *, + where: Mapping[str, object], + data: Mapping[str, object], + ) -> _PrismaRowT | None: ... + + +def _prisma_table( + repository: BaseRepository[_RepositoryModelT], +) -> _PrismaTableActions[_RepositoryModelT]: + return repository.table + + +def _deleted_verification_token_table( + prisma_client: PrismaClient, +) -> _PrismaTableActions[LiteLLM_DeletedVerificationToken]: + return DeletedVerificationTokenRepository(prisma_client).table + + +def _credentials_table(prisma_client: PrismaClient) -> _PrismaTableActions[CredentialItem]: + return CredentialsRepository(prisma_client).table + + +def _config_table(prisma_client: PrismaClient) -> _PrismaTableActions[ConfigParam]: + return ConfigRepository(prisma_client).table + async def _check_custom_key_allowed(custom_key_value: Optional[str]) -> None: """Raise 403 if custom API keys are disabled and a custom key was provided.""" @@ -490,7 +554,7 @@ _NON_ADMIN_SAFE_ALLOWED_ROUTES_PRESETS = frozenset({"llm_api_routes", "info_rout def _validate_caller_can_change_key_ownership( data: Optional[BaseModel], - existing_key_row: Any, + existing_key_row: LiteLLM_VerificationToken, user_api_key_dict: UserAPIKeyAuth, ) -> None: """ @@ -670,7 +734,7 @@ async def validate_team_id_used_in_service_account_request( ) # check if team_id exists in the database - team = await TeamRepository(prisma_client).table.find_unique( + team = await _prisma_table(TeamRepository(prisma_client)).find_unique( where={"team_id": team_id}, ) if team is None: @@ -1261,7 +1325,7 @@ async def _check_team_key_limits( # calculate allocated tpm/rpm limit # check if specified tpm/rpm limit is greater than allocated tpm/rpm limit - keys = await VerificationTokenRepository(prisma_client).table.find_many( + keys = await _prisma_table(VerificationTokenRepository(prisma_client)).find_many( where={"team_id": team_table.team_id}, ) # Exclude the key being updated to avoid double-counting its limits. @@ -1405,7 +1469,7 @@ async def _validate_caller_can_assign_key_org( detail="Cannot assign a key to an organization without a user_id on the caller's token", ) - user_row = await UserRepository(prisma_client).table.find_unique( + user_row = await _prisma_table(UserRepository(prisma_client)).find_unique( where={"user_id": user_api_key_dict.user_id}, include={"organization_memberships": True}, ) @@ -1443,7 +1507,7 @@ async def _check_org_key_limits( # get all organization keys # calculate allocated tpm/rpm limit # check if specified tpm/rpm limit is greater than allocated tpm/rpm limit - keys = await VerificationTokenRepository(prisma_client).table.find_many( + keys = await _prisma_table(VerificationTokenRepository(prisma_client)).find_many( where={"organization_id": org_table.organization_id}, ) # Exclude the key being updated to avoid double-counting its limits. @@ -2048,9 +2112,9 @@ async def _get_and_validate_existing_key( if token is not None: hashed_token = _hash_token_if_needed(token=token) - existing_key_row: LiteLLM_VerificationToken | None = await VerificationTokenRepository( - prisma_client - ).table.find_unique(where={"token": hashed_token}) + existing_key_row: LiteLLM_VerificationToken | None = await _prisma_table( + VerificationTokenRepository(prisma_client) + ).find_unique(where={"token": hashed_token}) if existing_key_row is None: raise ProxyException( @@ -2070,7 +2134,7 @@ async def _get_and_validate_existing_key( code=status.HTTP_400_BAD_REQUEST, ) - rows: list[LiteLLM_VerificationToken] = await VerificationTokenRepository(prisma_client).table.find_many( + rows: list[LiteLLM_VerificationToken] = await _prisma_table(VerificationTokenRepository(prisma_client)).find_many( where={"key_alias": key_alias}, take=2 ) @@ -2112,7 +2176,7 @@ async def _process_single_key_update( litellm_changed_by: Optional[str], prisma_client: Optional[PrismaClient], user_api_key_cache: UserApiKeyCache, - proxy_logging_obj: Any, + proxy_logging_obj: ProxyLogging, llm_router: Optional[Router], user_custom_key_update: Optional[Callable] = None, existing_key_row: Optional[LiteLLM_VerificationToken] = None, @@ -2265,9 +2329,9 @@ async def _process_single_key_update( async def _validate_mcp_servers_for_key_update( data: "UpdateKeyRequest", team_obj: Optional["LiteLLM_TeamTableCachedObj"], - existing_key_row: Any, - prisma_client: Any, - user_api_key_cache: Any, + existing_key_row: LiteLLM_VerificationToken, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache, is_proxy_admin: bool, ) -> Optional[ObjectPermissionDict]: """Validate MCP servers in object_permission against the effective team.""" @@ -2302,12 +2366,12 @@ async def _validate_mcp_servers_for_key_update( async def _validate_update_key_data( data: UpdateKeyRequest, - existing_key_row: Any, + existing_key_row: LiteLLM_VerificationToken, user_api_key_dict: UserAPIKeyAuth, - llm_router: Any, + llm_router: Router | None, premium_user: bool, prisma_client: Any, - user_api_key_cache: Any, + user_api_key_cache: UserApiKeyCache, ) -> None: """Validate permissions and constraints for key update.""" # Reject NaN/±inf spend before it can reach the DB / spend counter. @@ -2939,7 +3003,7 @@ def _build_failed_team_key_update( else: error_message = str(exception) - key_info: Optional[Dict[str, Any]] = None + key_info: dict[str, object] | None = None if existing_key_row is not None: if hasattr(existing_key_row, "model_dump"): key_info = existing_key_row.model_dump() @@ -3416,7 +3480,7 @@ async def info_key_fn_v2( # Resolve key_aliases to tokens so we never pass token=None (unbounded query) tokens_to_query = list(data.keys) if data.keys else [] if data.key_aliases: - alias_rows = await VerificationTokenRepository(prisma_client).table.find_many( + alias_rows = await _prisma_table(VerificationTokenRepository(prisma_client)).find_many( where={"key_alias": {"in": data.key_aliases}}, include={"litellm_budget_table": True}, ) @@ -4088,7 +4152,7 @@ def _transform_verification_tokens_to_deleted_records( keys: List[LiteLLM_VerificationToken], user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: Optional[str] = None, -) -> List[Dict[str, Any]]: +) -> list[dict[str, object]]: """Transform verification tokens into deleted token records ready for persistence.""" if not keys: return [] @@ -4141,13 +4205,13 @@ def _transform_verification_tokens_to_deleted_records( async def _save_deleted_verification_token_records( - records: List[Dict[str, Any]], + records: Sequence[Mapping[str, object]], prisma_client: PrismaClient, ) -> None: """Save deleted verification token records to the database.""" if not records: return - await DeletedVerificationTokenRepository(prisma_client).table.create_many(data=records) + await _deleted_verification_token_table(prisma_client).create_many(data=records) async def _persist_deleted_verification_tokens( @@ -4175,7 +4239,7 @@ async def delete_key_aliases( user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: Optional[str] = None, ) -> Tuple[Optional[Dict], List[LiteLLM_VerificationToken]]: - _keys_being_deleted = await VerificationTokenRepository(prisma_client).table.find_many( + _keys_being_deleted = await _prisma_table(VerificationTokenRepository(prisma_client)).find_many( where={"key_alias": {"in": key_aliases}} ) @@ -4212,7 +4276,7 @@ async def _rotate_master_key( from litellm.proxy.proxy_server import proxy_config try: - models: Optional[List] = await ModelRepository(prisma_client).table.find_many() + models: Optional[List] = await _prisma_table(ModelRepository(prisma_client)).find_many() except Exception: models = None # 2. process model table @@ -4242,7 +4306,7 @@ async def _rotate_master_key( ) # 3. process config table try: - config = await ConfigRepository(prisma_client).table.find_many() + config = await _config_table(prisma_client).find_many() except Exception: config = None @@ -4263,7 +4327,7 @@ async def _rotate_master_key( ) if encrypted_env_vars: - await ConfigRepository(prisma_client).table.update( + await _config_table(prisma_client).update( where={"param_name": "environment_variables"}, data={"param_value": prisma.Json(encrypted_env_vars)}, # type: ignore[attr-defined] ) @@ -4307,7 +4371,7 @@ async def _rotate_master_key( # 5. process credentials table try: - credentials = await CredentialsRepository(prisma_client).table.find_many() + credentials = await _credentials_table(prisma_client).find_many() except Exception: credentials = None if credentials: @@ -4330,7 +4394,7 @@ async def _rotate_master_key( _cred_data["credential_info"] = prisma.Json( # type: ignore[attr-defined] _cred_data["credential_info"] ) - await CredentialsRepository(prisma_client).table.update( + await _credentials_table(prisma_client).update( where={"credential_name": cred.credential_name}, data={ **_cred_data, @@ -4772,7 +4836,7 @@ async def regenerate_key_fn( else: hashed_api_key = hash_token(key) - _key_in_db = await VerificationTokenRepository(prisma_client).table.find_unique( + _key_in_db = await _prisma_table(VerificationTokenRepository(prisma_client)).find_unique( where={"token": hashed_api_key}, ) if _key_in_db is None: @@ -4976,7 +5040,7 @@ async def reset_key_spend_fn( else: hashed_api_key = hash_token(key) - _key_in_db = await VerificationTokenRepository(prisma_client).table.find_unique( + _key_in_db = await _prisma_table(VerificationTokenRepository(prisma_client)).find_unique( where={"token": hashed_api_key}, include={"litellm_budget_table": True}, ) @@ -4996,7 +5060,7 @@ async def reset_key_spend_fn( user_api_key_cache=user_api_key_cache, ) - updated_key = await VerificationTokenRepository(prisma_client).table.update( + updated_key = await _prisma_table(VerificationTokenRepository(prisma_client)).update( where={"token": hashed_api_key}, data={"spend": reset_to}, ) @@ -5067,7 +5131,7 @@ async def validate_key_list_check( param="user_id", code=status.HTTP_403_FORBIDDEN, ) - complete_user_info_db_obj: Optional[BaseModel] = await UserRepository(prisma_client).table.find_unique( + complete_user_info_db_obj: Optional[BaseModel] = await _prisma_table(UserRepository(prisma_client)).find_unique( where={"user_id": user_api_key_dict.user_id}, include={"organization_memberships": True}, ) @@ -5421,8 +5485,8 @@ async def list_keys( async def _apply_non_admin_alias_scope( user_api_key_dict: UserAPIKeyAuth, - prisma_client: Any, - query_params: List[Any], + prisma_client: PrismaClient, + query_params: list[object], where_parts: List[str], ) -> None: """Append SQL scope conditions so non-admin users only see aliases for @@ -5435,7 +5499,9 @@ async def _apply_non_admin_alias_scope( # Look up the user's teams from the user table user_teams: List[str] = [] if user_api_key_dict.user_id: - user_row = await UserRepository(prisma_client).table.find_unique(where={"user_id": user_api_key_dict.user_id}) + user_row = await _prisma_table(UserRepository(prisma_client)).find_unique( + where={"user_id": user_api_key_dict.user_id} + ) if user_row is not None: user_teams = getattr(user_row, "teams", []) or [] @@ -5493,7 +5559,7 @@ async def key_aliases( # support column-level SELECT projection on find_many. # # $1 is always UI_SESSION_TOKEN_TEAM_ID (filters out UI session tokens). - query_params: List[Any] = [UI_SESSION_TOKEN_TEAM_ID] + query_params: list[object] = [UI_SESSION_TOKEN_TEAM_ID] where_parts = [ "key_alias IS NOT NULL", "key_alias != ''", @@ -5601,7 +5667,7 @@ def _validate_sort_params(sort_by: Optional[str], sort_order: str) -> Optional[D return order_by -def _build_expires_where_clause(expires_filter: str, now: datetime) -> dict[str, Any]: +def _build_expires_where_clause(expires_filter: str, now: datetime) -> dict[str, object]: if expires_filter == "expired": return {"AND": [{"expires": {"not": None}}, {"expires": {"lt": now}}]} return {"OR": [{"expires": None}, {"expires": {"gte": now}}]} @@ -5848,11 +5914,11 @@ async def _list_key_helper( # Get total count of keys if use_deleted_table: - total_count = await DeletedVerificationTokenRepository(prisma_client).table.count( + total_count = await _deleted_verification_token_table(prisma_client).count( where=where # type: ignore ) else: - total_count = await VerificationTokenRepository(prisma_client).table.count( + total_count = await _prisma_table(VerificationTokenRepository(prisma_client)).count( where=where # type: ignore ) @@ -5931,8 +5997,8 @@ def _get_condition_to_filter_out_ui_session_tokens() -> Dict[str, Any]: async def _check_key_admin_access( user_api_key_dict: UserAPIKeyAuth, - hashed_token: str, - prisma_client: Any, + hashed_token: str | None, + prisma_client: PrismaClient | None, user_api_key_cache: UserApiKeyCache, route: str, ) -> None: @@ -5951,7 +6017,9 @@ async def _check_key_admin_access( return # Look up the target key to find its team - target_key_row = await VerificationTokenRepository(prisma_client).table.find_unique(where={"token": hashed_token}) + target_key_row = await _prisma_table(VerificationTokenRepository(prisma_client)).find_unique( + where={"token": hashed_token} + ) if target_key_row is None: raise HTTPException( status_code=404, @@ -6047,7 +6115,9 @@ async def block_key( ) # Check if the key exists before trying to block it - existing_record = await VerificationTokenRepository(prisma_client).table.find_unique(where={"token": hashed_token}) + existing_record = await _prisma_table(VerificationTokenRepository(prisma_client)).find_unique( + where={"token": hashed_token} + ) if existing_record is None: raise ProxyException( message="Key not found.", @@ -6077,7 +6147,7 @@ async def block_key( ) ) - record = await VerificationTokenRepository(prisma_client).table.update( + record = await _prisma_table(VerificationTokenRepository(prisma_client)).update( where={"token": hashed_token}, data={"blocked": True}, # type: ignore ) @@ -6158,7 +6228,9 @@ async def unblock_key( ) # Check if the key exists before trying to unblock it - existing_record = await VerificationTokenRepository(prisma_client).table.find_unique(where={"token": hashed_token}) + existing_record = await _prisma_table(VerificationTokenRepository(prisma_client)).find_unique( + where={"token": hashed_token} + ) if existing_record is None: raise ProxyException( message="Key not found.", @@ -6188,7 +6260,7 @@ async def unblock_key( ) ) - record = await VerificationTokenRepository(prisma_client).table.update( + record = await _prisma_table(VerificationTokenRepository(prisma_client)).update( where={"token": hashed_token}, data={"blocked": False}, # type: ignore ) @@ -6443,7 +6515,7 @@ def _validate_key_alias_format(key_alias: Optional[str]) -> None: async def _enforce_unique_key_alias( key_alias: Optional[str], - prisma_client: Any, + prisma_client: PrismaClient | None, existing_key_token: Optional[str] = None, ) -> None: """ @@ -6459,12 +6531,12 @@ async def _enforce_unique_key_alias( ProxyException: If key alias already exists on a different key """ if key_alias is not None and prisma_client is not None: - where_clause: dict[str, Any] = {"key_alias": key_alias} + where_clause: dict[str, object] = {"key_alias": key_alias} if existing_key_token: # Exclude the current key from the uniqueness check where_clause["NOT"] = {"token": existing_key_token} - existing_key = await VerificationTokenRepository(prisma_client).table.find_first(where=where_clause) + existing_key = await _prisma_table(VerificationTokenRepository(prisma_client)).find_first(where=where_clause) if existing_key is not None: raise ProxyException( message=f"Key with alias '{key_alias}' already exists. Unique key aliases across all keys are required.", diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 41904e3883b..4885ec42578 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -16,7 +16,7 @@ import json from collections.abc import Mapping, Sequence from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union, cast -from fastapi import APIRouter, Depends, HTTPException, Header, Request, status +from fastapi import APIRouter, Depends, Header, HTTPException, Request, status from pydantic import BaseModel, ConfigDict, Field from litellm._logging import verbose_proxy_logger @@ -39,6 +39,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.management_endpoints.common_utils import _is_user_team_admin from litellm.proxy.management_endpoints.team_endpoints import ( _refresh_cached_team, @@ -49,18 +50,18 @@ from litellm.proxy.management_endpoints.team_endpoints import ( update_team as _legacy_update_team, ) from litellm.proxy.management_helpers.audit_logs import create_object_audit_log -from litellm.proxy.utils import PrismaClient +from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.model_repository import ModelRepository from litellm.repositories.table_repositories import ModelTableRepository from litellm.repositories.team_repository import TeamRepository from litellm.router import Router -from litellm.types.proxy.management_endpoints.model_management_endpoints import ( - UpdateUsefulLinksRequest, -) from litellm.router_utils.auto_router_model_naming import ( STRATEGY_ROUTER_PARAM_FIELDS, validate_strategy_router_model_write, ) +from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateUsefulLinksRequest, +) from litellm.types.router import ( SPECIAL_MODEL_INFO_PARAMS, Deployment, @@ -843,8 +844,8 @@ async def _get_team_public_model_names( async def _remove_unbacked_team_models( model_params: Deployment, prisma_client: PrismaClient, - user_api_key_cache: Any, - proxy_logging_obj: Any, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging, llm_router: Router | None = None, ) -> None: """ @@ -904,7 +905,7 @@ async def _remove_unbacked_team_models( if existing_team_row is None: return - updated_team_row = await prisma_client.db.litellm_teamtable.update( + updated_team_row: LiteLLM_TeamTable = await prisma_client.db.litellm_teamtable.update( where={"team_id": team_id}, data={"models": [model for model in existing_team_row.models if model not in names_to_remove]}, include={"object_permission": True}, # type: ignore diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py index e44299d018d..f1f71bf6458 100644 --- a/litellm/proxy/management_endpoints/scim/scim_v2.py +++ b/litellm/proxy/management_endpoints/scim/scim_v2.py @@ -7,7 +7,18 @@ This is an enterprise feature and requires a premium license. import re from collections.abc import Mapping, Sequence from itertools import chain -from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Set, Tuple +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + List, + NamedTuple, + Optional, + Protocol, + Set, + Tuple, + overload, +) from fastapi import ( APIRouter, @@ -69,13 +80,95 @@ from litellm.repositories.verification_token_repository import ( ) from litellm.types.proxy.management_endpoints.scim_v2 import * +if TYPE_CHECKING: + from prisma.models import LiteLLM_VerificationToken as PrismaVerificationToken + + +class _UserTableClient(Protocol): + async def find_first(self, where: Mapping[str, object]) -> LiteLLM_UserTable | None: ... + + async def find_unique(self, where: Mapping[str, object]) -> LiteLLM_UserTable | None: ... + + async def find_many( + self, + where: Mapping[str, object] | None = None, + skip: int | None = None, + take: int | None = None, + order: Mapping[str, str] | None = None, + ) -> Sequence[LiteLLM_UserTable]: ... + + async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> LiteLLM_UserTable: ... + + async def delete(self, where: Mapping[str, object]) -> LiteLLM_UserTable | None: ... + + async def count(self, where: Mapping[str, object] | None = None) -> int: ... + + +class _TeamTableClient(Protocol): + async def find_unique(self, where: Mapping[str, object]) -> LiteLLM_TeamTable | None: ... + + async def find_many( + self, + where: Mapping[str, object] | None = None, + skip: int | None = None, + take: int | None = None, + order: Mapping[str, str] | None = None, + ) -> Sequence[LiteLLM_TeamTable]: ... + + async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> LiteLLM_TeamTable: ... + + async def delete(self, where: Mapping[str, object]) -> LiteLLM_TeamTable | None: ... + + async def count(self, where: Mapping[str, object] | None = None) -> int: ... + + +class _VerificationTokenTableClient(Protocol): + async def find_many(self, where: Mapping[str, object] | None = None) -> "Sequence[PrismaVerificationToken]": ... + + async def update( + self, where: Mapping[str, object], data: Mapping[str, object] + ) -> "PrismaVerificationToken | None": ... + + +class _UserReferencingTableClient(Protocol): + async def delete_many(self, where: Mapping[str, object]) -> int: ... + + +@overload +def _table(repository: UserRepository) -> _UserTableClient: ... + + +@overload +def _table(repository: TeamRepository) -> _TeamTableClient: ... + + +@overload +def _table(repository: VerificationTokenRepository) -> _VerificationTokenTableClient: ... + + +@overload +def _table( + repository: InvitationLinkRepository | OrganizationMembershipRepository | TeamMembershipRepository, +) -> _UserReferencingTableClient: ... + + +def _table( + repository: UserRepository + | TeamRepository + | VerificationTokenRepository + | InvitationLinkRepository + | OrganizationMembershipRepository + | TeamMembershipRepository, +) -> object: + return repository.table + class UserProvisionerHelpers: """Helper methods for user provisioning operations.""" @staticmethod async def handle_existing_user_by_email( - prisma_client, + prisma_client: PrismaClient, new_user_request: NewUserRequest, admin_group: Optional[str] = None, ) -> Optional[SCIMUser]: @@ -97,7 +190,7 @@ class UserProvisionerHelpers: if not new_user_request.user_email: return None - existing_user = await UserRepository(prisma_client).table.find_first( + existing_user = await _table(UserRepository(prisma_client)).find_first( where={"user_email": new_user_request.user_email} ) @@ -107,7 +200,7 @@ class UserProvisionerHelpers: new_teams = list(dict.fromkeys(new_user_request.teams or [])) if new_user_request.user_id != existing_user.user_id: - await UserRepository(prisma_client).table.update( + await _table(UserRepository(prisma_client)).update( where={"user_id": existing_user.user_id}, data={"user_id": new_user_request.user_id}, ) @@ -119,7 +212,7 @@ class UserProvisionerHelpers: raise_on_error=True, ) - updated_user = await UserRepository(prisma_client).table.update( + updated_user = await _table(UserRepository(prisma_client)).update( where={"user_id": new_user_request.user_id}, data={ "user_email": new_user_request.user_email, @@ -177,11 +270,11 @@ async def _get_prisma_client_or_raise_exception(): return prisma_client -async def _check_user_exists(user_id: str): +async def _check_user_exists(user_id: str) -> LiteLLM_UserTable: """Check if user exists and return user, raise 404 if not found.""" prisma_client = await _get_prisma_client_or_raise_exception() - user = await UserRepository(prisma_client).table.find_unique(where={"user_id": user_id}) + user = await _table(UserRepository(prisma_client)).find_unique(where={"user_id": user_id}) if not user: raise HTTPException(status_code=404, detail={"error": f"User not found with ID: {user_id}"}) @@ -189,11 +282,11 @@ async def _check_user_exists(user_id: str): return user -async def _check_team_exists(team_id: str): +async def _check_team_exists(team_id: str) -> LiteLLM_TeamTable: """Check if team exists and return team, raise 404 if not found.""" prisma_client = await _get_prisma_client_or_raise_exception() - team = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id}) + team = await _table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id}) if not team: raise HTTPException(status_code=404, detail={"error": f"Group not found with ID: {team_id}"}) @@ -236,9 +329,9 @@ def _build_scim_metadata( enterprise: Optional[SCIMEnterpriseUser] = None, entitlements: list[SCIMMultiValuedAttribute] | None = None, roles: list[SCIMMultiValuedAttribute] | None = None, -) -> Dict[str, Any]: +) -> dict[str, object]: """Build metadata dictionary with SCIM data.""" - metadata: Dict[str, Any] = { + metadata: dict[str, object] = { "scim_metadata": LiteLLM_UserScimMetadata( givenName=given_name, familyName=family_name, @@ -338,13 +431,15 @@ def _resolve_scim_user_role( return default_role -async def _scim_groups_from_team_ids(prisma_client: Any, team_ids: list[str]) -> list[SCIMUserGroup]: +async def _scim_groups_from_team_ids(prisma_client: PrismaClient, team_ids: list[str]) -> list[SCIMUserGroup]: """ Build SCIMUserGroup objects from team ids, populating display from each team's alias so admin-group matching by display name works the same way it does on PUT (where SCIM groups carry display names natively). """ - teams = [await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id}) for team_id in team_ids] + teams = [ + await _table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id}) for team_id in team_ids + ] return [ SCIMUserGroup( value=team_id, @@ -354,7 +449,7 @@ async def _scim_groups_from_team_ids(prisma_client: Any, team_ids: list[str]) -> ] -async def _recompute_scim_member_roles(prisma_client: Any, user_ids: Iterable[str]) -> None: +async def _recompute_scim_member_roles(prisma_client: PrismaClient, user_ids: Iterable[str]) -> None: """ Recompute and persist each user's global proxy role from their resulting team membership. No-op unless scim_admin_group is configured, so a SCIM group write @@ -367,7 +462,7 @@ async def _recompute_scim_member_roles(prisma_client: Any, user_ids: Iterable[st default_role = _default_scim_user_role() for user_id in user_ids: - user = await UserRepository(prisma_client).table.find_unique(where={"user_id": user_id}) + user = await _table(UserRepository(prisma_client)).find_unique(where={"user_id": user_id}) if user is None: continue resolved_role = _resolve_scim_user_role( @@ -375,7 +470,7 @@ async def _recompute_scim_member_roles(prisma_client: Any, user_ids: Iterable[st admin_group, default_role, ) - await UserRepository(prisma_client).table.update( + await _table(UserRepository(prisma_client)).update( where={"user_id": user_id}, data={"user_role": resolved_role}, ) @@ -471,7 +566,7 @@ async def _classify_group_member(member: SCIMMember, prisma_client: PrismaClient if member_type == "group": return _SkippedGroupMember(value=value, reason="nested_group") - user = await UserRepository(prisma_client).table.find_unique(where={"user_id": value}) + user = await _table(UserRepository(prisma_client)).find_unique(where={"user_id": value}) if user is not None: return _ResolvedUserMember(user_id=value) @@ -479,7 +574,7 @@ async def _classify_group_member(member: SCIMMember, prisma_client: PrismaClient return _SkippedGroupMember(value=value, reason="non_user_type") if member_type is None: - team = await TeamRepository(prisma_client).table.find_unique(where={"team_id": value}) + team = await _table(TeamRepository(prisma_client)).find_unique(where={"team_id": value}) if team is not None and _team_metadata_has_scim_provenance(team.metadata): return _SkippedGroupMember(value=value, reason="existing_team") @@ -619,7 +714,7 @@ async def _get_team_members_display(member_ids: List[str]) -> List[SCIMMember]: members: List[SCIMMember] = [] for member_id in member_ids: - user = await UserRepository(prisma_client).table.find_unique(where={"user_id": member_id}) + user = await _table(UserRepository(prisma_client)).find_unique(where={"user_id": member_id}) if user: display_name = user.user_email or user.user_id members.append(SCIMMember(value=user.user_id, display=display_name, type="User")) @@ -652,7 +747,7 @@ async def _handle_team_membership_changes( SCIM_BLOCKED_METADATA_KEY = "scim_blocked" -def _key_was_scim_blocked(metadata: Any) -> bool: +def _key_was_scim_blocked(metadata: object) -> bool: """True if a verification token carries the SCIM-block marker in metadata.""" return isinstance(metadata, dict) and metadata.get(SCIM_BLOCKED_METADATA_KEY) is True @@ -676,7 +771,7 @@ async def _set_user_keys_blocked(user_id: str, blocked: bool) -> int: # `blocked` is a nullable column with no default, so existing rows # typically hold NULL; treat NULL as "not blocked" since SQL equality # on NULL would otherwise silently skip them. - candidates = await VerificationTokenRepository(prisma_client).table.find_many( + candidates = await _table(VerificationTokenRepository(prisma_client)).find_many( where={ "user_id": user_id, "OR": [{"blocked": False}, {"blocked": None}], @@ -684,7 +779,7 @@ async def _set_user_keys_blocked(user_id: str, blocked: bool) -> int: ) affected_keys = candidates else: - candidates = await VerificationTokenRepository(prisma_client).table.find_many( + candidates = await _table(VerificationTokenRepository(prisma_client)).find_many( where={"user_id": user_id, "blocked": True}, ) affected_keys = [k for k in candidates if _key_was_scim_blocked(k.metadata)] @@ -693,12 +788,12 @@ async def _set_user_keys_blocked(user_id: str, blocked: bool) -> int: return 0 for key_row in affected_keys: - current_metadata: Dict[str, Any] = dict(key_row.metadata) if isinstance(key_row.metadata, dict) else {} + current_metadata: dict[str, object] = dict(key_row.metadata) if isinstance(key_row.metadata, dict) else {} if blocked: new_metadata = {**current_metadata, SCIM_BLOCKED_METADATA_KEY: True} else: new_metadata = {k: v for k, v in current_metadata.items() if k != SCIM_BLOCKED_METADATA_KEY} - await VerificationTokenRepository(prisma_client).table.update( + await _table(VerificationTokenRepository(prisma_client)).update( where={"token": key_row.token}, data={"blocked": blocked, "metadata": safe_dumps(new_metadata)}, ) @@ -719,14 +814,14 @@ async def _set_user_keys_blocked(user_id: str, blocked: bool) -> int: return len(affected_keys) -async def _delete_rows_referencing_user(prisma_client: Any, *, user_id: str) -> None: +async def _delete_rows_referencing_user(prisma_client: PrismaClient, *, user_id: str) -> None: """Drop rows whose foreign keys reference ``LiteLLM_UserTable.user_id``. Required before deleting the user row itself, otherwise Postgres rejects the user delete with an FK constraint violation (e.g. ``LiteLLM_InvitationLink_user_id_fkey``). """ - await InvitationLinkRepository(prisma_client).table.delete_many( + await _table(InvitationLinkRepository(prisma_client)).delete_many( where={ "OR": [ {"user_id": user_id}, @@ -735,11 +830,11 @@ async def _delete_rows_referencing_user(prisma_client: Any, *, user_id: str) -> ] } ) - await OrganizationMembershipRepository(prisma_client).table.delete_many(where={"user_id": user_id}) - await TeamMembershipRepository(prisma_client).table.delete_many(where={"user_id": user_id}) + await _table(OrganizationMembershipRepository(prisma_client)).delete_many(where={"user_id": user_id}) + await _table(TeamMembershipRepository(prisma_client)).delete_many(where={"user_id": user_id}) -def _scim_active_value(metadata: Optional[Dict[str, Any]]) -> Optional[bool]: +def _scim_active_value(metadata: Optional[Mapping[str, object]]) -> Optional[bool]: """Read the SCIM active flag from a user's metadata dict, if present.""" if not metadata: return None @@ -749,6 +844,12 @@ def _scim_active_value(metadata: Optional[Dict[str, Any]]) -> Optional[bool]: return bool(value) +def _user_scim_active(user: LiteLLM_UserTable) -> Optional[bool]: + """Read the SCIM active flag off a user row's metadata, if present.""" + metadata: dict[str, object] | None = user.metadata + return _scim_active_value(metadata) + + async def _create_user_if_not_exists(user_id: str, created_via: str = "scim_group") -> Optional[NewUserResponse]: """ Helper function to create a user if they don't exist. @@ -820,7 +921,7 @@ async def set_scim_content_type(response: Response): response.headers["Content-Type"] = "application/scim+json" -def _get_resource_types(base_url: str = "/scim/v2") -> list: +def _get_resource_types(base_url: str = "/scim/v2") -> Sequence[SCIMResourceType]: """Return the list of SCIM ResourceType definitions per RFC 7643 Section 6.""" return [ SCIMResourceType( @@ -848,7 +949,7 @@ def _get_resource_types(base_url: str = "/scim/v2") -> list: ] -def _get_schemas() -> list: +def _get_schemas() -> Sequence[SCIMSchema]: """Return the list of SCIM Schema definitions per RFC 7643 Section 7.""" return [ SCIMSchema( @@ -1241,7 +1342,7 @@ async def get_users( try: prisma_client = await _get_prisma_client_or_raise_exception() # Parse filter if provided (basic support) - where_conditions: Dict[str, Any] = {} + where_conditions: dict[str, object] = {} if filter: # Okta locates users by userName before deprovisioning. LiteLLM # exposes SCIM userName from user_email, while older SCIM-created @@ -1258,7 +1359,7 @@ async def get_users( where_conditions["user_email"] = filter_value # Get users from database - users: List[LiteLLM_UserTable] = await UserRepository(prisma_client).table.find_many( + users: Sequence[LiteLLM_UserTable] = await _table(UserRepository(prisma_client)).find_many( where=where_conditions, skip=(startIndex - 1), take=count, @@ -1266,7 +1367,7 @@ async def get_users( ) # Get total count for pagination - total_count = await UserRepository(prisma_client).table.count(where=where_conditions) + total_count = await _table(UserRepository(prisma_client)).count(where=where_conditions) # Convert to SCIM format scim_users: List[SCIMUser] = [] @@ -1330,7 +1431,7 @@ async def create_user( # Check if user already exists if user.userName: - existing_user = await UserRepository(prisma_client).table.find_unique(where={"user_id": user.userName}) + existing_user = await _table(UserRepository(prisma_client)).find_unique(where={"user_id": user.userName}) if existing_user: raise HTTPException( status_code=409, @@ -1406,7 +1507,7 @@ async def update_user( prisma_client = await _get_prisma_client_or_raise_exception() existing_user = await _check_user_exists(user_id) - prev_active = _scim_active_value(existing_user.metadata) + prev_active = _user_scim_active(existing_user) user_data = _extract_scim_user_data(user) @@ -1447,7 +1548,7 @@ async def update_user( user.groups or [], admin_group, _default_scim_user_role() ) - updated_user = await UserRepository(prisma_client).table.update( + updated_user = await _table(UserRepository(prisma_client)).update( where={"user_id": user_id}, data=update_data, ) @@ -1483,19 +1584,20 @@ async def delete_user( existing_user = await _check_user_exists(user_id) # Get teams user belongs to - teams = [] - if existing_user.teams: - for team_id in existing_user.teams: - team = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id}) - if team: - teams.append(team) + found_teams = tuple( + [ + await _table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id}) + for team_id in existing_user.teams or [] + ] + ) + teams = tuple(team for team in found_teams if team) # Remove user from all teams for team in teams: - current_members = team.members or [] + current_members: Sequence[str] = team.members or [] if user_id in current_members: new_members = [m for m in current_members if m != user_id] - await TeamRepository(prisma_client).table.update( + await _table(TeamRepository(prisma_client)).update( where={"team_id": team.team_id}, data={"members": new_members} ) @@ -1511,7 +1613,7 @@ async def delete_user( await _delete_rows_referencing_user(prisma_client, user_id=user_id) # Delete user - await UserRepository(prisma_client).table.delete(where={"user_id": user_id}) + await _table(UserRepository(prisma_client)).delete(where={"user_id": user_id}) return Response(status_code=204) except Exception as e: @@ -1581,7 +1683,7 @@ def _extract_ids_from_path_filter(path: str | None, attribute: str) -> List[str] return [extracted] if extracted else [] -def _handle_displayname_update(op_type: str, value: Any, update_data: Dict[str, Any]) -> None: +def _handle_displayname_update(op_type: str, value: object, update_data: dict[str, object]) -> None: """Handle displayname updates.""" if op_type == "remove": update_data["user_alias"] = None @@ -1589,7 +1691,7 @@ def _handle_displayname_update(op_type: str, value: Any, update_data: Dict[str, update_data["user_alias"] = str(value) -def _handle_externalid_update(op_type: str, value: Any, update_data: Dict[str, Any]) -> None: +def _handle_externalid_update(op_type: str, value: object, update_data: dict[str, object]) -> None: """Handle externalid updates.""" if op_type == "remove": update_data["sso_user_id"] = None @@ -1597,7 +1699,7 @@ def _handle_externalid_update(op_type: str, value: Any, update_data: Dict[str, A update_data["sso_user_id"] = str(value) -def _handle_active_update(op_type: str, value: Any, metadata: Dict[str, Any]) -> None: +def _handle_active_update(op_type: str, value: object, metadata: dict[str, object]) -> None: """Handle active status updates.""" if op_type == "remove": metadata.pop("scim_active", None) @@ -1610,7 +1712,7 @@ def _handle_active_update(op_type: str, value: Any, metadata: Dict[str, Any]) -> metadata["scim_active"] = bool_val -def _handle_name_update(path: str, op_type: str, value: Any, scim_metadata: Dict[str, Any]) -> None: +def _handle_name_update(path: str, op_type: str, value: object, scim_metadata: dict[str, object]) -> None: """Handle name field updates (givenName, familyName).""" if path == "name.givenname": if op_type == "remove": @@ -1624,7 +1726,7 @@ def _handle_name_update(path: str, op_type: str, value: Any, scim_metadata: Dict scim_metadata["familyName"] = str(value) -def _handle_group_operations(op_type: str, value: Any, teams_set: Set[str], path: str | None) -> Set[str] | None: +def _handle_group_operations(op_type: str, value: object, teams_set: Set[str], path: str | None) -> Set[str] | None: """Handle group/team membership operations.""" group_values = _extract_group_values(value) if not group_values and value is None: @@ -1644,7 +1746,7 @@ def _multi_valued_attribute_base(path: str) -> str: return path.split("[", 1)[0].split(".", 1)[0] -def _handle_multi_valued_attribute_update(path: str, op_type: str, value: Any, metadata: dict[str, Any]) -> None: +def _handle_multi_valued_attribute_update(path: str, op_type: str, value: object, metadata: dict[str, object]) -> None: """Handle add/replace/remove for the entitlements and roles multi-valued attributes.""" base = _multi_valued_attribute_base(path) metadata_key = SCIM_MULTI_VALUED_ATTRIBUTE_METADATA_KEYS[base] @@ -1681,7 +1783,7 @@ def _handle_multi_valued_attribute_update(path: str, op_type: str, value: Any, m metadata[metadata_key] = dumped -def _handle_generic_metadata(path: str, op_type: str, value: Any, metadata: Dict[str, Any]) -> None: +def _handle_generic_metadata(path: str, op_type: str, value: object, metadata: dict[str, object]) -> None: """Handle generic metadata operations for unknown paths.""" if op_type == "remove": metadata.pop(path, None) @@ -1692,9 +1794,9 @@ def _handle_generic_metadata(path: str, op_type: str, value: Any, metadata: Dict def _apply_patch_ops( existing_user: LiteLLM_UserTable, patch_ops: SCIMPatchOp, -) -> Tuple[Dict[str, Any], Set[str]]: +) -> Tuple[dict[str, object], Set[str]]: """Apply patch operations and return update data and final team set.""" - update_data: Dict[str, Any] = {} + update_data: dict[str, object] = {} metadata = existing_user.metadata or {} scim_metadata = metadata.get("scim_metadata", {}) @@ -1843,14 +1945,15 @@ async def patch_user( prisma_client = await _get_prisma_client_or_raise_exception() existing_user = await _check_user_exists(user_id) - prev_active = _scim_active_value(existing_user.metadata) + prev_active = _user_scim_active(existing_user) update_data, final_team_set = _apply_patch_ops( existing_user=existing_user, patch_ops=patch_ops, ) - new_active = _scim_active_value(update_data.get("metadata")) + patched_metadata = update_data.get("metadata") + new_active = _scim_active_value(patched_metadata if isinstance(patched_metadata, Mapping) else None) # Handle team membership changes await _handle_team_membership_changes( @@ -1875,7 +1978,7 @@ async def patch_user( update_data["metadata"] = safe_dumps(update_data["metadata"]) - updated_user = await UserRepository(prisma_client).table.update( + updated_user = await _table(UserRepository(prisma_client)).update( where={"user_id": user_id}, data=update_data, ) @@ -1891,6 +1994,12 @@ async def patch_user( raise handle_exception_on_proxy(e) +class _TeamWhereConditions(TypedDict, total=False): + """The team columns SCIM GET /Groups can filter on, as Prisma where-conditions.""" + + team_alias: str + + # Group Endpoints @scim_router.get( "/Groups", @@ -1915,7 +2024,7 @@ async def get_groups( try: prisma_client = await _get_prisma_client_or_raise_exception() # Parse filter if provided (basic support) - where_conditions = {} + where_conditions: _TeamWhereConditions = {} if filter: # Very basic filter support - only handling displayName eq if "displayName eq" in filter: @@ -1923,7 +2032,7 @@ async def get_groups( where_conditions["team_alias"] = team_alias # Get teams from database - teams = await TeamRepository(prisma_client).table.find_many( + teams = await _table(TeamRepository(prisma_client)).find_many( where=where_conditions, skip=(startIndex - 1), take=count, @@ -1931,10 +2040,10 @@ async def get_groups( ) # Get total count for pagination - total_count = await TeamRepository(prisma_client).table.count(where=where_conditions) + total_count = await _table(TeamRepository(prisma_client)).count(where=where_conditions) # Convert to SCIM format - scim_groups = [] + scim_groups: List[SCIMGroup] = [] for team in teams: # Get team members with display names. members_with_roles is the # source of truth; the legacy `members` column is not populated by @@ -2018,7 +2127,7 @@ async def create_group( team_id = group.id or group.externalId or str(uuid.uuid4()) # Check if team already exists - existing_team = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id}) + existing_team = await _table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id}) if existing_team: raise HTTPException( @@ -2091,7 +2200,7 @@ async def update_group( } # Update team in database - updated_team = await TeamRepository(prisma_client).table.update( + updated_team = await _table(TeamRepository(prisma_client)).update( where={"team_id": group_id}, data=update_data, ) @@ -2145,19 +2254,19 @@ async def delete_group( # For each member, remove this team from their teams list for member_id in member_ids: - user = await UserRepository(prisma_client).table.find_unique(where={"user_id": member_id}) + user = await _table(UserRepository(prisma_client)).find_unique(where={"user_id": member_id}) if user: current_teams = user.teams or [] if group_id in current_teams: new_teams = [t for t in current_teams if t != group_id] - await UserRepository(prisma_client).table.update( + await _table(UserRepository(prisma_client)).update( where={"user_id": member_id}, data={"teams": new_teams} ) await _recompute_scim_member_roles(prisma_client, member_ids) # Delete team - await TeamRepository(prisma_client).table.delete(where={"team_id": group_id}) + await _table(TeamRepository(prisma_client)).delete(where={"team_id": group_id}) return Response(status_code=204) @@ -2166,8 +2275,8 @@ async def delete_group( async def _process_group_patch_operations( - patch_ops: SCIMPatchOp, existing_team, prisma_client -) -> Tuple[Dict[str, Any], Set[str], Set[str] | None]: + patch_ops: SCIMPatchOp, existing_team: LiteLLM_TeamTable, prisma_client: PrismaClient +) -> Tuple[dict[str, object], Set[str], Set[str] | None]: """Process patch operations for a group and return update data, final members and, when the request contained a member ``replace`` op, the absolute target roster it declared (``None`` otherwise). @@ -2183,7 +2292,7 @@ async def _process_group_patch_operations( have admitted - the phantom users this endpoint used to create for nested groups - impossible to clean up. """ - update_data: Dict[str, Any] = {} + update_data: dict[str, object] = {} # Create a fresh copy of existing metadata to avoid Prisma issues metadata = {**(existing_team.metadata or {}), SCIM_MANAGED_TEAM_METADATA_KEY: True} @@ -2251,7 +2360,7 @@ async def _process_group_patch_operations( return update_data, final_members, replace_target -async def _apply_group_patch_updates(group_id: str, update_data: Dict[str, Any], prisma_client): +async def _apply_group_patch_updates(group_id: str, update_data: dict[str, object], prisma_client: PrismaClient): """Apply the group's metadata/displayName patch updates to the database. Membership itself is not written here; it is reconciled onto the source of @@ -2330,7 +2439,7 @@ async def patch_group( # Apply the metadata/displayName updates to the database updated_team = await _apply_group_patch_updates(group_id, update_data, prisma_client) - refreshed_team = await TeamRepository(prisma_client).table.find_unique(where={"team_id": group_id}) + refreshed_team = await _table(TeamRepository(prisma_client)).find_unique(where={"team_id": group_id}) refreshed_current = ( set( await _get_team_member_user_ids_from_team(LiteLLM_TeamTable.model_validate(refreshed_team.model_dump())) @@ -2356,7 +2465,7 @@ async def patch_group( ) # Refresh team one more time to get final state after membership changes - final_team = await TeamRepository(prisma_client).table.find_unique(where={"team_id": group_id}) + final_team = await _table(TeamRepository(prisma_client)).find_unique(where={"team_id": group_id}) if final_team: updated_team = final_team diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index fa01f43d049..95c89c24851 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -14,7 +14,19 @@ import json import math import traceback from datetime import datetime, timezone -from typing import Annotated, Any, Dict, List, Mapping, Optional, Tuple, Union, cast +from typing import ( + Annotated, + Dict, + List, + Mapping, + Optional, + Protocol, + Sequence, + Tuple, + TypeVar, + Union, + cast, +) import fastapi from fastapi import APIRouter, Depends, Header, HTTPException, Request, status @@ -30,11 +42,14 @@ from litellm.proxy._types import ( BlockTeamRequest, CommonProxyErrors, DeleteTeamRequest, + LiteLLM_AccessGroupTable, LiteLLM_AuditLogs, + LiteLLM_BudgetTableFull, LiteLLM_DeletedTeamTable, LiteLLM_ManagementEndpoint_MetadataFields, LiteLLM_ManagementEndpoint_MetadataFields_Premium, LiteLLM_ModelTable, + LiteLLM_OrganizationMembershipTable, LiteLLM_OrganizationTable, LiteLLM_OrganizationTableWithMembers, LiteLLM_TeamMembership, @@ -78,6 +93,7 @@ from litellm.proxy.auth.auth_checks import ( from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars from litellm.proxy.common_utils.json_merge_patch import apply_json_merge_patch +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.management_endpoints.common_utils import ( _check_passthrough_routes_caller_permission, _is_user_org_admin_for_team, @@ -106,7 +122,7 @@ from litellm.proxy.management_helpers.utils import ( add_new_member, management_endpoint_wrapper, ) -from litellm.proxy.utils import PrismaClient, handle_exception_on_proxy +from litellm.proxy.utils import PrismaClient, ProxyLogging, handle_exception_on_proxy from litellm.repositories.budget_repository import BudgetRepository from litellm.repositories.organization_repository import OrganizationRepository from litellm.repositories.table_repositories import ( @@ -141,6 +157,127 @@ from litellm.types.proxy.management_endpoints.team_endpoints import ( router = APIRouter() +_DbRecordT = TypeVar("_DbRecordT") + + +class _PrismaTableActions(Protocol[_DbRecordT]): + async def find_unique( + self, + where: Mapping[str, object], + include: Mapping[str, bool] | None = None, + ) -> _DbRecordT | None: ... + + async def find_first( + self, + where: Mapping[str, object] | None = None, + order: Mapping[str, str] | None = None, + ) -> _DbRecordT | None: ... + + async def find_many( + self, + where: Mapping[str, object] | None = None, + include: Mapping[str, bool] | None = None, + order: Mapping[str, str] | None = None, + skip: int | None = None, + take: int | None = None, + cursor: Mapping[str, object] | None = None, + ) -> list[_DbRecordT]: ... + + async def create( + self, + data: Mapping[str, object], + include: Mapping[str, bool] | None = None, + ) -> _DbRecordT: ... + + async def create_many( + self, + data: Sequence[Mapping[str, object]], + skip_duplicates: bool | None = None, + ) -> int: ... + + async def update( + self, + where: Mapping[str, object], + data: Mapping[str, object], + include: Mapping[str, bool] | None = None, + ) -> _DbRecordT: ... + + async def update_many( + self, + where: Mapping[str, object], + data: Mapping[str, object], + ) -> int: ... + + async def upsert( + self, + where: Mapping[str, object], + data: Mapping[str, Mapping[str, object]], + ) -> _DbRecordT: ... + + async def delete_many( + self, + where: Mapping[str, object] | None = None, + ) -> int: ... + + async def count( + self, + where: Mapping[str, object] | None = None, + ) -> int: ... + + +def _team_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_TeamTable]": + team_table: _PrismaTableActions[LiteLLM_TeamTable] = TeamRepository(prisma_client).table + return team_table + + +def _team_membership_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_TeamMembership]": + membership_table: _PrismaTableActions[LiteLLM_TeamMembership] = TeamMembershipRepository(prisma_client).table + return membership_table + + +def _user_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_UserTable]": + user_table: _PrismaTableActions[LiteLLM_UserTable] = UserRepository(prisma_client).table + return user_table + + +def _model_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_ModelTable]": + model_table: _PrismaTableActions[LiteLLM_ModelTable] = ModelTableRepository(prisma_client).table + return model_table + + +def _org_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_OrganizationTable]": + org_table: _PrismaTableActions[LiteLLM_OrganizationTable] = OrganizationRepository(prisma_client).table + return org_table + + +def _org_membership_db( + prisma_client: PrismaClient | None, +) -> "_PrismaTableActions[LiteLLM_OrganizationMembershipTable]": + org_membership_table: _PrismaTableActions[LiteLLM_OrganizationMembershipTable] = OrganizationMembershipRepository( + prisma_client + ).table + return org_membership_table + + +def _budget_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_BudgetTableFull]": + budget_table: _PrismaTableActions[LiteLLM_BudgetTableFull] = BudgetRepository(prisma_client).table + return budget_table + + +def _deleted_team_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_DeletedTeamTable]": + deleted_team_table: _PrismaTableActions[LiteLLM_DeletedTeamTable] = DeletedTeamRepository(prisma_client).table + return deleted_team_table + + +def _access_group_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_AccessGroupTable]": + access_group_table: _PrismaTableActions[LiteLLM_AccessGroupTable] = AccessGroupRepository(prisma_client).table + return access_group_table + + +def _tokens_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_VerificationToken]": + tokens_table: _PrismaTableActions[LiteLLM_VerificationToken] = VerificationTokenRepository(prisma_client).table + return tokens_table + def _sanitize_for_log(value: object) -> str: """Strip CR/LF from user-controlled values to prevent log injection.""" @@ -152,9 +289,9 @@ def _sanitize_for_log(value: object) -> str: async def _refresh_cached_team( - team_row: Any, - user_api_key_cache: Any, - proxy_logging_obj: Any, + team_row: LiteLLM_TeamTable, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging, ) -> None: """ Refresh the in-memory cached team object after a DB write. @@ -396,7 +533,7 @@ class TeamMemberBudgetHandler: @staticmethod async def backfill_team_member_budget_entries( team_id: str, - members_with_roles: List[Union[Member, dict]], + members_with_roles: Sequence[Union[Member, dict[str, object]]], team_member_budget_id: str, prisma_client: PrismaClient, ) -> None: @@ -415,7 +552,7 @@ class TeamMemberBudgetHandler: return # Batch-fetch existing memberships for this team (avoids N+1 queries) - existing_memberships = await TeamMembershipRepository(prisma_client).table.find_many(where={"team_id": team_id}) + existing_memberships = await _team_membership_db(prisma_client).find_many(where={"team_id": team_id}) existing_user_ids = {m.user_id for m in existing_memberships} # Identify members with no existing membership row. @@ -448,7 +585,7 @@ class TeamMemberBudgetHandler: # Heal existing membership rows that predate the team_member_budget # configuration: populate budget_id where it is currently NULL. # Rows with an explicit budget_id (per-member override) are left alone. - updated = await TeamMembershipRepository(prisma_client).table.update_many( + updated = await _team_membership_db(prisma_client).update_many( where={"team_id": team_id, "budget_id": None}, data={"budget_id": team_member_budget_id}, ) @@ -461,7 +598,7 @@ class TeamMemberBudgetHandler: ) -def _get_default_team_param(field: str) -> Any: +def _get_default_team_param(field: str) -> object: """ Returns a default value for the given field from litellm.default_team_params config. Returns None if no default is configured. @@ -504,7 +641,7 @@ async def get_all_team_memberships( # else: # where_obj = {"user_id": str(user_id), "team_id": {"in": team_id}} - team_memberships = await TeamMembershipRepository(prisma_client).table.find_many( + team_memberships = await _team_membership_db(prisma_client).find_many( where=where_obj, include={"litellm_budget_table": True}, ) @@ -766,7 +903,7 @@ async def _check_org_team_limits( # calculate allocated tpm/rpm limit # check if specified tpm/rpm limit is greater than allocated tpm/rpm limit - teams = await TeamRepository(prisma_client).table.find_many( + teams = await _team_db(prisma_client).find_many( where={"organization_id": org_table.organization_id}, ) @@ -791,7 +928,7 @@ async def _check_user_team_limits( data: Union[NewTeamRequest, UpdateTeamRequest], user_api_key_dict: UserAPIKeyAuth, prisma_client: PrismaClient, - user_api_key_cache: Any, + user_api_key_cache: UserApiKeyCache, ) -> None: """ Enforce the caller's personal limits when CREATING a standalone team. @@ -1052,7 +1189,7 @@ async def new_team( ) # Check if license is over limit - total_teams = await TeamRepository(prisma_client).table.count() + total_teams = await _team_db(prisma_client).count() if total_teams and _license_check.is_team_count_over_limit(team_count=total_teams): raise HTTPException( status_code=403, @@ -1154,7 +1291,7 @@ async def new_team( created_by=user_api_key_dict.user_id or litellm_proxy_admin_name, updated_by=user_api_key_dict.user_id or litellm_proxy_admin_name, ) - model_dict = await ModelTableRepository(prisma_client).table.create( + model_dict = await _model_db(prisma_client).create( {**litellm_modeltable.json(exclude_none=True)} # type: ignore ) # type: ignore @@ -1358,11 +1495,11 @@ async def _create_team_update_audit_log( async def _update_model_table( data: UpdateTeamRequest, - model_id: Optional[str], + model_id: Optional[int], prisma_client: PrismaClient, user_api_key_dict: UserAPIKeyAuth, litellm_proxy_admin_name: str, -) -> Optional[str]: +) -> Optional[int]: """ Upsert model table and return the model id """ @@ -1375,11 +1512,11 @@ async def _update_model_table( updated_by=user_api_key_dict.user_id or litellm_proxy_admin_name, ) if model_id is None: - model_dict = await ModelTableRepository(prisma_client).table.create( + model_dict = await _model_db(prisma_client).create( data={**litellm_modeltable.json(exclude_none=True)} # type: ignore ) else: - model_dict = await ModelTableRepository(prisma_client).table.upsert( + model_dict = await _model_db(prisma_client).upsert( where={"id": model_id}, data={ "update": {**litellm_modeltable.json(exclude_none=True)}, # type: ignore @@ -1395,7 +1532,7 @@ async def _update_model_table( async def _auto_add_team_members_to_organization( team: LiteLLM_TeamTable, organization: LiteLLM_OrganizationTableWithMembers, - prisma_client: Any, + prisma_client: PrismaClient, ) -> None: """ When moving a team to an org, ensure all team members are also org members. @@ -1433,11 +1570,11 @@ async def _auto_add_team_members_to_organization( async def fetch_and_validate_organization( organization_id: str, - existing_team_row: Any, + existing_team_row: LiteLLM_TeamTable, llm_router: Optional[Router], - prisma_client: Any, + prisma_client: PrismaClient, user_api_key_dict: Optional[UserAPIKeyAuth] = None, -) -> Any: +) -> LiteLLM_OrganizationTable: """ Fetch and validate an organization for team update operations. @@ -1456,7 +1593,7 @@ async def fetch_and_validate_organization( if llm_router is None: raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value}) - organization_row = await OrganizationRepository(prisma_client).table.find_unique( + organization_row = await _org_db(prisma_client).find_unique( where={"organization_id": organization_id}, include={"litellm_budget_table": True, "members": True, "teams": True}, ) @@ -1758,7 +1895,7 @@ async def update_team( ): # Is the caller org_admin of the destination org? caller_memberships = ( - await OrganizationMembershipRepository(prisma_client).table.find_many( + await _org_membership_db(prisma_client).find_many( where={ "user_id": user_api_key_dict.user_id, "organization_id": data.organization_id, @@ -2005,7 +2142,7 @@ async def patch_team( patch_fields = data.model_dump(exclude_unset=True, exclude={"team_id"}) if "metadata" in patch_fields: - existing_team_row = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id}) + existing_team_row = await _team_db(prisma_client).find_unique(where={"team_id": team_id}) if existing_team_row is None: raise HTTPException( status_code=404, @@ -2487,7 +2624,7 @@ async def _validate_and_populate_member_user_info( # Case 2: Only user_email provided - populate user_id from DB if member.user_email is not None and member.user_id is None: - user_by_email = await UserRepository(prisma_client).table.find_first( + user_by_email = await _user_db(prisma_client).find_first( where={"user_email": {"equals": member.user_email, "mode": "insensitive"}} ) @@ -2516,7 +2653,7 @@ async def _validate_and_populate_member_user_info( # Case 3: Only user_id provided - populate user_email from DB if user exists if member.user_id is not None and member.user_email is None: - user_by_id = await UserRepository(prisma_client).table.find_unique(where={"user_id": member.user_id}) + user_by_id = await _user_db(prisma_client).find_unique(where={"user_id": member.user_id}) if user_by_id is None: # User doesn't exist yet - allow it to pass with user_email as None @@ -2707,7 +2844,7 @@ async def team_member_delete( detail={"error": "Either user_id or user_email needs to be passed in"}, ) - _existing_team_row = await TeamRepository(prisma_client).table.find_unique(where={"team_id": data.team_id}) + _existing_team_row = await _team_db(prisma_client).find_unique(where={"team_id": data.team_id}) if _existing_team_row is None: raise HTTPException( @@ -2745,7 +2882,7 @@ async def team_member_delete( _db_new_team_members: List[dict] = [m.model_dump() for m in new_team_members] - _ = await TeamRepository(prisma_client).table.update( + _ = await _team_db(prisma_client).update( where={ "team_id": data.team_id, }, @@ -2835,7 +2972,7 @@ _MEMBER_BUDGET_PATCH_FIELDS = { } -def _build_member_budget_patch(data: TeamMemberUpdateRequest) -> Dict[str, Any]: +def _build_member_budget_patch(data: TeamMemberUpdateRequest) -> Dict[str, object]: """Map the budget fields the request actually set (merge-patch: a sent value updates, an explicit null clears, an absent field is left untouched) to their budget-table columns.""" @@ -2911,7 +3048,7 @@ async def team_member_update( _validate_budget_duration(data.budget_duration) - _existing_team_row = await TeamRepository(prisma_client).table.find_unique(where={"team_id": data.team_id}) + _existing_team_row = await _team_db(prisma_client).find_unique(where={"team_id": data.team_id}) if _existing_team_row is None: raise HTTPException( @@ -3007,7 +3144,7 @@ async def team_member_update( team_table.members_with_roles = team_members _db_team_members: List[dict] = [m.model_dump() for m in team_members] - await TeamRepository(prisma_client).table.update( + await _team_db(prisma_client).update( where={"team_id": data.team_id}, data={"members_with_roles": json.dumps(_db_team_members)}, # type: ignore ) @@ -3138,7 +3275,7 @@ async def bulk_team_member_add( }, ) # get all users from the database - all_users_in_db = await UserRepository(prisma_client).table.find_many(order={"created_at": "desc"}) + all_users_in_db = await _user_db(prisma_client).find_many(order={"created_at": "desc"}) data.members = [ Member( user_id=user.user_id, @@ -3254,9 +3391,7 @@ async def delete_team( team_rows: List[LiteLLM_TeamTable] = [] for team_id in data.team_ids: try: - team_row_base: Optional[BaseModel] = await TeamRepository(prisma_client).table.find_unique( - where={"team_id": team_id} - ) + team_row_base: Optional[BaseModel] = await _team_db(prisma_client).find_unique(where={"team_id": team_id}) if team_row_base is None: raise Exception except Exception: @@ -3379,7 +3514,7 @@ def _transform_teams_to_deleted_records( teams: List[LiteLLM_TeamTable], user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: Optional[str] = None, -) -> List[Dict[str, Any]]: +) -> List[Dict[str, object]]: """Transform teams into deleted team records ready for persistence.""" if not teams: return [] @@ -3424,13 +3559,13 @@ def _transform_teams_to_deleted_records( async def _save_deleted_team_records( - records: List[Dict[str, Any]], + records: List[Dict[str, object]], prisma_client: PrismaClient, ) -> None: """Save deleted team records to the database.""" if not records: return - await DeletedTeamRepository(prisma_client).table.create_many(data=records) + await _deleted_team_db(prisma_client).create_many(data=records) async def _persist_deleted_team_records( @@ -3506,9 +3641,7 @@ async def _add_team_member_budget_table( team_info_response_object: TeamInfoResponseObjectTeamTable, ) -> TeamInfoResponseObjectTeamTable: try: - team_budget = await BudgetRepository(prisma_client).table.find_unique( - where={"budget_id": team_member_budget_id} - ) + team_budget = await _budget_db(prisma_client).find_unique(where={"budget_id": team_member_budget_id}) team_info_response_object.team_member_budget_table = team_budget except Exception: verbose_proxy_logger.info( @@ -3518,7 +3651,7 @@ async def _add_team_member_budget_table( return team_info_response_object -async def _resolve_team_access_group_resources(_team_info: Any) -> None: +async def _resolve_team_access_group_resources(_team_info: TeamInfoResponseObjectTeamTable) -> None: """Populate access_group_models / mcp_server_ids / agent_ids on the team info response by resolving inherited resources from its access groups.""" if not _team_info.access_group_ids: @@ -3572,7 +3705,7 @@ async def team_info( ) try: - team_info: Optional[BaseModel] = await TeamRepository(prisma_client).table.find_unique( + team_info: Optional[BaseModel] = await _team_db(prisma_client).find_unique( where={"team_id": team_id}, include={"litellm_model_table": True, "object_permission": True}, ) @@ -3819,7 +3952,7 @@ async def block_team( if prisma_client is None: raise Exception("No DB Connected.") - existing_team = await TeamRepository(prisma_client).table.find_unique(where={"team_id": data.team_id}) + existing_team = await _team_db(prisma_client).find_unique(where={"team_id": data.team_id}) if existing_team is None: raise HTTPException( status_code=404, @@ -3832,7 +3965,7 @@ async def block_team( user_api_key_dict=user_api_key_dict, ) - record = await TeamRepository(prisma_client).table.update( + record = await _team_db(prisma_client).update( where={"team_id": data.team_id}, data={"blocked": True}, # type: ignore ) @@ -3868,7 +4001,7 @@ async def unblock_team( if prisma_client is None: raise Exception("No DB Connected.") - existing_team = await TeamRepository(prisma_client).table.find_unique(where={"team_id": data.team_id}) + existing_team = await _team_db(prisma_client).find_unique(where={"team_id": data.team_id}) if existing_team is None: raise HTTPException( status_code=404, @@ -3881,7 +4014,7 @@ async def unblock_team( user_api_key_dict=user_api_key_dict, ) - record = await TeamRepository(prisma_client).table.update( + record = await _team_db(prisma_client).update( where={"team_id": data.team_id}, data={"blocked": False}, # type: ignore ) @@ -3915,7 +4048,7 @@ async def list_available_teams( return [] # filter out teams that the user is already a member of - user_info = await UserRepository(prisma_client).table.find_unique(where={"user_id": user_api_key_dict.user_id}) + user_info = await _user_db(prisma_client).find_unique(where={"user_id": user_api_key_dict.user_id}) if user_info is None: raise HTTPException( status_code=404, @@ -3925,7 +4058,7 @@ async def list_available_teams( available_teams = [team for team in available_teams if team not in user_info_correct_type.teams] - available_teams_db = await TeamRepository(prisma_client).table.find_many(where={"team_id": {"in": available_teams}}) + available_teams_db = await _team_db(prisma_client).find_many(where={"team_id": {"in": available_teams}}) available_teams_correct_type = [LiteLLM_TeamTable.model_validate(team.model_dump()) for team in available_teams_db] @@ -3934,9 +4067,9 @@ async def list_available_teams( async def _get_org_admin_org_ids( user_id: str, - prisma_client: Any, - user_api_key_cache: Any, - proxy_logging_obj: Any, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging, ) -> Optional[List[str]]: """ Return the list of organization IDs where the user is an org admin. @@ -3976,16 +4109,16 @@ async def _build_team_list_where_conditions( search: Optional[str] = None, search_team_id_match: TeamIdSearchMatch = "exact", org_admin_org_ids: Optional[List[str]] = None, - user_api_key_cache: Optional[Any] = None, - proxy_logging_obj: Optional[Any] = None, -) -> Optional[Dict[str, Any]]: + user_api_key_cache: Optional[UserApiKeyCache] = None, + proxy_logging_obj: Optional[ProxyLogging] = None, +) -> Optional[Dict[str, object]]: """ Build where conditions for team list query. Returns None when the query is guaranteed to yield no results (e.g. user has no team memberships), allowing the caller to skip the DB round-trip. """ - where_conditions: Dict[str, Any] = {} + where_conditions: Dict[str, object] = {} if team_id: where_conditions["team_id"] = team_id @@ -4067,7 +4200,7 @@ async def _batch_resolve_access_group_resources( return {} unique_ids = list(set(all_access_group_ids)) - rows = await AccessGroupRepository(_prisma_client).table.find_many( + rows = await _access_group_db(_prisma_client).find_many( where={"access_group_id": {"in": unique_ids}}, ) @@ -4115,8 +4248,8 @@ def _convert_teams_to_response_models( async def _get_keys_count_by_team( - prisma_client: Any, - teams: list, + prisma_client: PrismaClient, + teams: Sequence[LiteLLM_TeamTable], ) -> Dict[str, int]: """Aggregate virtual-key counts per team for the given page of teams. @@ -4140,9 +4273,9 @@ async def _enforce_list_team_v2_access( user_api_key_dict: UserAPIKeyAuth, user_id: Optional[str], organization_id: Optional[str], - prisma_client: Any, - user_api_key_cache: Any, - proxy_logging_obj: Any, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging, ) -> Tuple[Optional[str], Optional[List[str]]]: """Enforce access control for list_team_v2. @@ -4341,23 +4474,23 @@ async def list_team_v2( # Get teams with pagination if use_deleted_table: - teams = await DeletedTeamRepository(prisma_client).table.find_many( + teams = await _deleted_team_db(prisma_client).find_many( where=where_conditions, skip=skip, take=page_size, order=order_by if order_by else {"created_at": "desc"}, # Default sort ) # Get total count for pagination - total_count = await DeletedTeamRepository(prisma_client).table.count(where=where_conditions) + total_count = await _deleted_team_db(prisma_client).count(where=where_conditions) else: - teams = await TeamRepository(prisma_client).table.find_many( + teams = await _team_db(prisma_client).find_many( where=where_conditions, skip=skip, take=page_size, order=order_by if order_by else {"created_at": "desc"}, # Default sort ) # Get total count for pagination - total_count = await TeamRepository(prisma_client).table.count(where=where_conditions) + total_count = await _team_db(prisma_client).count(where=where_conditions) # Calculate total pages total_pages = -(-total_count // page_size) # Ceiling division @@ -4400,9 +4533,9 @@ async def list_team_v2( async def _authorize_and_filter_teams( user_api_key_dict: UserAPIKeyAuth, user_id: Optional[str], - prisma_client: Any, - user_api_key_cache: Any, - proxy_logging_obj: Any, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging, ) -> list: """ Authorize the /team/list request and return filtered teams. @@ -4574,7 +4707,7 @@ async def get_paginated_teams( # Calculate skip for pagination skip = (page - 1) * page_size # Get total count - total_count = await TeamRepository(prisma_client).table.count() + total_count = await _team_db(prisma_client).count() # Get paginated teams teams = await TeamRepository(prisma_client).table.find_many( @@ -4710,7 +4843,7 @@ async def team_model_add( raise HTTPException(status_code=500, detail={"error": "No db connected"}) # Get existing team - team_row = await TeamRepository(prisma_client).table.find_unique(where={"team_id": data.team_id}) + team_row = await _team_db(prisma_client).find_unique(where={"team_id": data.team_id}) if team_row is None: raise HTTPException( @@ -4756,7 +4889,7 @@ async def team_model_add( # the writer and lets Prisma bump updated_at. # `include` mirrors the relations the auth path consumes off the cached # team object so that `_refresh_cached_team` doesn't null them out. - updated_team = await TeamRepository(prisma_client).table.update( + updated_team = await _team_db(prisma_client).update( where={"team_id": data.team_id}, data={"updated_at": datetime.now(timezone.utc)}, include={"object_permission": True}, # type: ignore @@ -4810,7 +4943,7 @@ async def team_model_delete( raise HTTPException(status_code=500, detail={"error": "No db connected"}) # Get existing team - team_row = await TeamRepository(prisma_client).table.find_unique(where={"team_id": data.team_id}) + team_row = await _team_db(prisma_client).find_unique(where={"team_id": data.team_id}) if team_row is None: raise HTTPException( @@ -4973,7 +5106,7 @@ async def update_team_member_permissions( }, ) # Update the team member permissions - updated_team = await TeamRepository(prisma_client).table.update( + updated_team = await _team_db(prisma_client).update( where={"team_id": data.team_id}, data={"team_member_permissions": data.team_member_permissions}, ) @@ -5043,7 +5176,7 @@ async def bulk_update_team_member_permissions( } -async def _compute_and_batch_updates(prisma_client, teams, permissions_to_add: set) -> int: +async def _compute_and_batch_updates(prisma_client, teams: Sequence[LiteLLM_TeamTable], permissions_to_add: set) -> int: """Compute merged permissions and batch-write updates. Returns count of teams updated.""" updates = [] for team in teams: @@ -5065,9 +5198,11 @@ async def _compute_and_batch_updates(prisma_client, teams, permissions_to_add: s return len(updates) -async def _append_permissions_to_specific_teams(prisma_client, team_ids: List[str], permissions_to_add: set) -> int: +async def _append_permissions_to_specific_teams( + prisma_client: PrismaClient, team_ids: List[str], permissions_to_add: set +) -> int: """Fetch specific teams by ID and append permissions.""" - teams = await TeamRepository(prisma_client).table.find_many( + teams = await _team_db(prisma_client).find_many( where={"team_id": {"in": team_ids}}, ) @@ -5082,7 +5217,7 @@ async def _append_permissions_to_specific_teams(prisma_client, team_ids: List[st return await _compute_and_batch_updates(prisma_client, teams, permissions_to_add) -async def _append_permissions_to_all_teams(prisma_client, permissions_to_add: set) -> int: +async def _append_permissions_to_all_teams(prisma_client: PrismaClient, permissions_to_add: set) -> int: """Paginated read + batched write across all teams.""" teams_updated = 0 cursor = None @@ -5228,9 +5363,7 @@ async def get_team_daily_activity( # If user does not have full team view, filter by their API keys if not has_full_team_view: # Get all API keys for this user - user_keys = await VerificationTokenRepository(prisma_client).table.find_many( - where={"user_id": user_api_key_dict.user_id} - ) + user_keys = await _tokens_db(prisma_client).find_many(where={"user_id": user_api_key_dict.user_id}) user_api_keys = [key.token for key in user_keys if key.token] # If user has no API keys, return empty result if not user_api_keys: diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index f7e429ddb31..bc37058689e 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -11,7 +11,10 @@ from typing import ( Literal, Mapping, NamedTuple, + Protocol, Sequence, + TypedDict, + TypeVar, Union, ) @@ -48,6 +51,204 @@ router = APIRouter() SPEND_LOGS_PAGINATION_COUNT_CAP = 10000 +_RowT = TypeVar("_RowT") + + +class _SupportsModelDump(Protocol): + def model_dump(self) -> Mapping[str, object]: ... + + +class _SpendLogOwnershipRow(Protocol): + user: str | None + team_id: str | None + + +class _ActivityRow(TypedDict): + date: str + api_requests: int + total_tokens: int + + +class _ActivityModelRow(TypedDict): + model_group: str + date: str + api_requests: int + total_tokens: int + + +class _DeploymentExceptionsRow(TypedDict): + api_base: str + date: str + num_rate_limit_exceptions: int + + +class _ExceptionsRow(TypedDict): + date: str + num_rate_limit_exceptions: int + + +class _ModelIdSpendRow(TypedDict): + model_id: str + spend: float + + +class _TagNameRow(TypedDict): + individual_request_tag: str + + +class _TeamSpendRow(TypedDict): + team_alias: str | None + total_spend: float + + +class _TagSpendRow(TypedDict): + individual_request_tag: str + total_spend: float + + +class _SpendLogsCountRow(TypedDict): + total_count: int + + +class _PgClassRow(TypedDict): + relname: str + relkind: str + + +class _TotalSpendRow(TypedDict): + total_spend: float + + +class _TeamDailySpendRow(TypedDict): + team_alias: str | None + spend_date: str | None + total_spend: float + + +class _EndUserRow(TypedDict): + end_user: str | None + + +class _DailyTagSpendRow(TypedDict): + individual_request_tag: str + log_count: int + total_spend: float + + +class _SessionCountAggregate(TypedDict): + session_id: int + + +class _SessionCountRow(TypedDict): + session_id: str + _count: _SessionCountAggregate + + +class _SessionSpendRow(TypedDict): + session_id: str + session_total_spend: float + mcp_tool_call_count: int + mcp_tool_call_spend: float + + +async def _query_raw(prisma_client: PrismaClient, sql_query: str, *args: object) -> Sequence[_RowT]: + """Run a raw read query and return its rows as the row type the caller declares.""" + return await prisma_client.db.query_raw(sql_query, *args) + + +async def _query_raw_or_none(prisma_client: PrismaClient, sql_query: str, *args: object) -> Sequence[_RowT] | None: + """``_query_raw`` for the call sites that guard the result against ``None``.""" + return await _query_raw(prisma_client, sql_query, *args) + + +class _SpendLogsTable(Protocol): + """The subset of the Prisma spend-logs table API this module uses.""" + + async def find_many( + self, *, where: Mapping[str, object], order: Mapping[str, str] + ) -> Sequence[_SupportsModelDump]: ... + + async def find_unique( + self, *, where: Mapping[str, object], include: None = None + ) -> _SpendLogOwnershipRow | None: ... + + async def count(self, *, where: Mapping[str, object]) -> int: ... + + async def group_by( + self, *, by: Sequence[str], where: Mapping[str, object], count: Mapping[str, bool] + ) -> Sequence[_SessionCountRow]: ... + + +class _TeamTable(Protocol): + """The subset of the Prisma team table API this module uses.""" + + async def find_unique(self, *, where: Mapping[str, object]) -> _SupportsModelDump | None: ... + + async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_SupportsModelDump]: ... + + async def update_many(self, *, data: Mapping[str, float], where: Mapping[str, object]) -> int: ... + + +class _VerificationTokenTable(Protocol): + """The subset of the Prisma verification token table API this module uses.""" + + async def update_many(self, *, data: Mapping[str, float], where: Mapping[str, object]) -> int: ... + + +def _spend_logs_table(prisma_client: PrismaClient) -> _SpendLogsTable: + return SpendLogsRepository(prisma_client).table + + +def _team_table(prisma_client: PrismaClient) -> _TeamTable: + return TeamRepository(prisma_client).table + + +def _verification_token_table(prisma_client: PrismaClient) -> _VerificationTokenTable: + return VerificationTokenRepository(prisma_client).table + + +async def _find_spend_logs( + prisma_client: PrismaClient, + where: Mapping[str, object], + order: Mapping[str, str], +) -> Sequence[_SupportsModelDump]: + """Read spend log rows as Prisma model instances.""" + return await _spend_logs_table(prisma_client).find_many(where=where, order=order) + + +async def _find_spend_log_row(prisma_client: PrismaClient, request_id: str) -> _SpendLogOwnershipRow | None: + """Read the single spend log row identified by ``request_id``.""" + return await _spend_logs_table(prisma_client).find_unique( + where={"request_id": request_id}, + include=None, + ) + + +async def _count_spend_logs(prisma_client: PrismaClient, where: Mapping[str, object]) -> int: + """Count the spend log rows matching ``where``.""" + return await _spend_logs_table(prisma_client).count(where=where) + + +async def _count_logs_per_session( + prisma_client: PrismaClient, session_ids: Sequence[str | None] +) -> Sequence[_SessionCountRow]: + """Count spend log rows per session for the given session ids.""" + return await _spend_logs_table(prisma_client).group_by( + by=["session_id"], + where={"session_id": {"in": session_ids}}, + count={"session_id": True}, + ) + + +async def _find_team_row(prisma_client: PrismaClient, team_id: str) -> _SupportsModelDump | None: + """Read a single team row as a Prisma model instance.""" + return await _team_table(prisma_client).find_unique(where={"team_id": team_id}) + + +async def _find_team_rows(prisma_client: PrismaClient, team_ids: Sequence[str]) -> Sequence[_SupportsModelDump]: + """Read team rows as Prisma model instances.""" + return await _team_table(prisma_client).find_many(where={"team_id": {"in": team_ids}}) + @router.get( "/spend/keys", @@ -281,7 +482,9 @@ async def get_global_activity_internal_user( AND "user" = $3 GROUP BY date_trunc('day', "startTime") """ - db_response = await prisma_client.db.query_raw(sql_query, start_date, end_date, user_id) + db_response: Sequence[_ActivityRow] | None = await _query_raw_or_none( + prisma_client, sql_query, start_date, end_date, user_id + ) return db_response @@ -345,6 +548,7 @@ async def get_global_activity( "Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" ) + db_response: Sequence[_ActivityRow] | None if ( user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER or user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY @@ -361,7 +565,7 @@ async def get_global_activity( AND "startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') GROUP BY date_trunc('day', "startTime") """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj) + db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj) if db_response is None: return [] @@ -420,7 +624,9 @@ async def get_global_activity_model_internal_user( AND "user" = $3 GROUP BY model_group, date_trunc('day', "startTime") """ - db_response = await prisma_client.db.query_raw(sql_query, start_date, end_date, user_id) + db_response: Sequence[_ActivityModelRow] | None = await _query_raw_or_none( + prisma_client, sql_query, start_date, end_date, user_id + ) return db_response @@ -507,6 +713,7 @@ async def get_global_activity_model( "Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" ) + db_response: Sequence[_ActivityModelRow] | None if ( user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER or user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY @@ -524,7 +731,7 @@ async def get_global_activity_model( AND "startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') GROUP BY model_group, date_trunc('day', "startTime") """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj) + db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj) if db_response is None: return [] @@ -672,7 +879,9 @@ async def get_global_activity_exceptions_per_deployment( ORDER BY date; """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj, model_group) + db_response: Sequence[_DeploymentExceptionsRow] | None = await _query_raw_or_none( + prisma_client, sql_query, start_date_obj, end_date_obj, model_group + ) if db_response is None: return [] @@ -795,7 +1004,9 @@ async def get_global_activity_exceptions( ORDER BY date; """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj, model_group) + db_response: Sequence[_ExceptionsRow] | None = await _query_raw_or_none( + prisma_client, sql_query, start_date_obj, end_date_obj, model_group + ) if db_response is None: return [] @@ -883,6 +1094,7 @@ async def get_global_spend_provider( "Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" ) + db_response: Sequence[_ModelIdSpendRow] | None if ( user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER or user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY @@ -902,7 +1114,7 @@ async def get_global_spend_provider( AND "user" = $3 GROUP BY model_id """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj, user_id) + db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj, user_id) else: sql_query = """ SELECT @@ -914,7 +1126,7 @@ async def get_global_spend_provider( AND length(model_id) > 0 GROUP BY model_id """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj) + db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj) if db_response is None: return [] @@ -1042,6 +1254,7 @@ async def get_global_spend_report( if premium_user is not True: verbose_proxy_logger.debug("accessing /spend/report but not a premium user") raise ValueError("/spend/report endpoint " + CommonProxyErrors.not_premium_user.value) + db_response: Sequence[Mapping[str, object]] | None if api_key is not None: verbose_proxy_logger.debug("Getting /spend for api_key: [set=%s]", api_key is not None) if api_key.startswith("sk-"): @@ -1082,7 +1295,7 @@ async def get_global_spend_report( ORDER BY total_cost DESC; """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj, api_key) + db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj, api_key) if db_response is None: return [] @@ -1125,7 +1338,9 @@ async def get_global_spend_report( ORDER BY total_cost DESC; """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj, internal_user_id) + db_response = await _query_raw_or_none( + prisma_client, sql_query, start_date_obj, end_date_obj, internal_user_id + ) if db_response is None: return [] @@ -1190,7 +1405,7 @@ async def get_global_spend_report( group_by_day; """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj) + db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj) if db_response is None: return [] @@ -1231,7 +1446,7 @@ async def get_global_spend_report( ORDER BY total_cost DESC; """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj) + db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj) if db_response is None: return [] @@ -1268,7 +1483,7 @@ async def global_get_all_tag_names(): FROM "LiteLLM_SpendLogs"; """ - db_response = await prisma_client.db.query_raw(sql_query) + db_response: Sequence[_TagNameRow] | None = await _query_raw_or_none(prisma_client, sql_query) if db_response is None: return [] @@ -1415,7 +1630,9 @@ async def _get_spend_report_for_time_range( ORDER BY total_spend DESC; """ - response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj) + response: Sequence[_TeamSpendRow] | None = await _query_raw_or_none( + prisma_client, sql_query, start_date_obj, end_date_obj + ) # get spend per tag for today sql_query = """ @@ -1429,7 +1646,9 @@ async def _get_spend_report_for_time_range( ORDER BY total_spend DESC; """ - spend_per_tag = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj) + spend_per_tag: Sequence[_TagSpendRow] | None = await _query_raw_or_none( + prisma_client, sql_query, start_date_obj, end_date_obj + ) return response, spend_per_tag except Exception as e: @@ -1894,7 +2113,7 @@ async def ui_view_spend_logs( # (messages, response, proxy_server_request can be hundreds of KB per row). # These are only needed in the detail endpoint /spend/logs/ui/{request_id}. sql_conditions: List[str] = [] - sql_params: List[Any] = [] + sql_params: list[object] = [] p = 1 # parameter index counter # Date range. Wrap the param side with `AT TIME ZONE 'UTC'` so comparison @@ -2002,7 +2221,9 @@ async def ui_view_spend_logs( LIMIT ${p} ) AS bounded_matches """ - count_rows = await prisma_client.db.query_raw(count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1) + count_rows: Sequence[_SpendLogsCountRow] | None = await _query_raw_or_none( + prisma_client, count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1 + ) raw_total = int(count_rows[0]["total_count"]) if count_rows else 0 total_is_capped = raw_total > SPEND_LOGS_PAGINATION_COUNT_CAP total_records = SPEND_LOGS_PAGINATION_COUNT_CAP if total_is_capped else raw_total @@ -2067,7 +2288,7 @@ def _spend_log_field_has_content(value: Union[str, list, dict] | None) -> bool: return True -def _hydrate_spend_log_metadata(rows: Sequence[Any]) -> None: +def _hydrate_spend_log_metadata(rows: Sequence[Mapping[str, object]]) -> None: """Re-hydrate the JSONB ``metadata`` column returned by ``query_raw`` as a string. The Prisma serialiser bypasses the model-layer JSON hydration we get on the ORM @@ -2227,7 +2448,9 @@ async def ui_view_request_response_for_request_id( WHERE request_id = $1 LIMIT 1 """ - db_result = await prisma_client.db.query_raw(sql_query, request_id) + db_result: Sequence[Mapping[str, object]] | None = await _query_raw_or_none( + prisma_client, sql_query, request_id + ) if db_result and len(db_result) > 0: resolved = await _resolve_request_response_payload(db_result[0], cold_storage_handler=ColdStorageHandler()) return resolved._asdict() @@ -2359,11 +2582,10 @@ async def view_spend_logs( # Check if user wants unsummarized data if not summarize: # Return filtered individual log entries (similar to UI endpoint) - data = await SpendLogsRepository(prisma_client).table.find_many( - where=filter_query, # type: ignore - order={ - "startTime": "desc", - }, + data = await _find_spend_logs( + prisma_client, + where=filter_query, + order={"startTime": "desc"}, ) return data @@ -2421,7 +2643,7 @@ async def view_spend_logs( return response else: - scoped_filter: Dict[str, Any] = {} + scoped_filter: dict[str, str] = {} if api_key is not None and isinstance(api_key, str): if api_key.startswith("sk-"): hashed_token = prisma_client.hash_token(token=api_key) @@ -2437,8 +2659,9 @@ async def view_spend_logs( spend_logs = await prisma_client.get_data(table_name="spend", query_type="find_all") return spend_logs - data = await SpendLogsRepository(prisma_client).table.find_many( - where=scoped_filter, # type: ignore + data = await _find_spend_logs( + prisma_client, + where=scoped_filter, order={"startTime": "desc"}, ) return data @@ -2489,8 +2712,8 @@ async def global_spend_reset(): code=status.HTTP_401_UNAUTHORIZED, ) - await VerificationTokenRepository(prisma_client).table.update_many(data={"spend": 0.0}, where={}) - await TeamRepository(prisma_client).table.update_many(data={"spend": 0.0}, where={}) + await _verification_token_table(prisma_client).update_many(data={"spend": 0.0}, where={}) + await _team_table(prisma_client).update_many(data={"spend": 0.0}, where={}) return { "message": "Spend for all API Keys and Teams reset successfully", @@ -2533,7 +2756,7 @@ async def global_spend_refresh(): WHERE relname = 'MonthlyGlobalSpend'; """ try: - resp = await prisma_client.db.query_raw(sql_query) + resp: Sequence[_PgClassRow] = await _query_raw(prisma_client, sql_query) return resp[0]["relkind"] == "m" except Exception: @@ -2562,7 +2785,7 @@ async def global_spend_refresh(): }, ) await new_client.db.connect() - await new_client.db.query_raw(sql_query) + await _query_raw(new_client, sql_query) verbose_proxy_logger.info("MonthlyGlobalSpend view refreshed") return { "message": "MonthlyGlobalSpend view refreshed", @@ -2601,13 +2824,13 @@ async def global_spend_for_internal_user( ORDER BY "date"; """ - response = await prisma_client.db.query_raw(sql_query, api_key, user_id) + response: Sequence[Mapping[str, object]] = await _query_raw(prisma_client, sql_query, api_key, user_id) return response sql_query = """SELECT * FROM "MonthlyGlobalSpendPerUserPerKey" WHERE "user" = $1 ORDER BY "date";""" - response = await prisma_client.db.query_raw(sql_query, user_id) + response = await _query_raw(prisma_client, sql_query, user_id) return response except Exception as e: @@ -2652,6 +2875,7 @@ async def global_spend_logs( code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) + response: Sequence[Mapping[str, object]] if ( user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER or user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY @@ -2669,7 +2893,7 @@ async def global_spend_logs( if api_key is None: sql_query = """SELECT * FROM "MonthlyGlobalSpend" ORDER BY "date";""" - response = await prisma_client.db.query_raw(query=sql_query) + response = await _query_raw(prisma_client, sql_query) return response else: @@ -2679,7 +2903,7 @@ async def global_spend_logs( ORDER BY "date"; """ - response = await prisma_client.db.query_raw(sql_query, api_key) + response = await _query_raw(prisma_client, sql_query, api_key) return response @@ -2726,7 +2950,7 @@ async def global_spend(): if prisma_client is None: raise HTTPException(status_code=500, detail={"error": "No db connected"}) sql_query = """SELECT SUM(spend) as total_spend FROM "MonthlyGlobalSpend";""" - response = await prisma_client.db.query_raw(query=sql_query) + response: Sequence[_TotalSpendRow] | None = await _query_raw_or_none(prisma_client, sql_query) if response is not None: if isinstance(response, list) and len(response) > 0: total_spend = response[0].get("total_spend", 0.0) @@ -2791,7 +3015,7 @@ async def global_spend_key_internal_user(user_api_key_dict: UserAPIKeyAuth, limi """ - response = await prisma_client.db.query_raw(sql_query, user_id, limit) + response: Sequence[Mapping[str, object]] = await _query_raw(prisma_client, sql_query, user_id, limit) return response @@ -2816,6 +3040,7 @@ async def global_spend_keys( """ from litellm.proxy.proxy_server import prisma_client + response: Sequence[Mapping[str, object]] if ( user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER or user_api_key_dict.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY @@ -2828,14 +3053,14 @@ async def global_spend_keys( sql_query = """SELECT * FROM "Last30dKeysBySpend";""" if limit is None: - response = await prisma_client.db.query_raw(sql_query) + response = await _query_raw(prisma_client, sql_query) return response try: limit = int(limit) if limit < 1: raise ValueError("Limit must be greater than 0") sql_query = """SELECT * FROM "Last30dKeysBySpend" LIMIT $1 ;""" - response = await prisma_client.db.query_raw(sql_query, limit) + response = await _query_raw(prisma_client, sql_query, limit) except ValueError as e: raise HTTPException(status_code=422, detail={"error": f"Invalid limit: {limit}, error: {e}"}) from e @@ -2875,7 +3100,7 @@ async def global_spend_per_team(): ORDER BY spend_date; """ - response = await prisma_client.db.query_raw(query=sql_query) + response: Sequence[_TeamDailySpendRow] = await _query_raw(prisma_client, sql_query) # transform the response for the Admin UI spend_by_date = {} @@ -2952,7 +3177,7 @@ async def global_view_all_end_users(): SELECT DISTINCT end_user FROM "LiteLLM_SpendLogs" """ - db_response = await prisma_client.db.query_raw(query=sql_query) + db_response: Sequence[_EndUserRow] | None = await _query_raw_or_none(prisma_client, sql_query) if db_response is None: return [] @@ -3009,7 +3234,9 @@ GROUP BY end_user ORDER BY total_spend DESC LIMIT 100 """ - response = await prisma_client.db.query_raw(sql_query, startTime, endTime, selected_api_key) + response: Sequence[Mapping[str, object]] = await _query_raw( + prisma_client, sql_query, startTime, endTime, selected_api_key + ) return response @@ -3040,7 +3267,7 @@ async def global_spend_models_internal_user(user_api_key_dict: UserAPIKeyAuth, l LIMIT $2; """ - response = await prisma_client.db.query_raw(sql_query, user_id, limit) + response: Sequence[Mapping[str, object]] = await _query_raw(prisma_client, sql_query, user_id, limit) return response @@ -3077,7 +3304,7 @@ async def global_spend_models( sql_query = """SELECT * FROM "Last30dModelsBySpend" LIMIT $1 ;""" - response = await prisma_client.db.query_raw(sql_query, int(limit)) + response: Sequence[Mapping[str, object]] = await _query_raw(prisma_client, sql_query, int(limit)) return response @@ -3169,14 +3396,17 @@ async def provider_budgets() -> ProviderBudgetResponse: async def get_spend_by_tags(prisma_client: PrismaClient, start_date=None, end_date=None): - response = await prisma_client.db.query_raw(""" + response: Sequence[Mapping[str, object]] = await _query_raw( + prisma_client, + """ SELECT jsonb_array_elements_text(request_tags) AS individual_request_tag, COUNT(*) AS log_count, SUM(spend) AS total_spend FROM "LiteLLM_SpendLogs" GROUP BY individual_request_tag; - """) + """, + ) return response @@ -3203,7 +3433,7 @@ async def ui_get_spend_by_tags( if prisma_client is None: raise HTTPException(status_code=500, detail={"error": "No db connected"}) - response = None + response: Sequence[_DailyTagSpendRow] | None = None if tags_list is None or (isinstance(tags_list, list) and "all-tags" in tags_list): # Get spend for all tags sql_query = """ @@ -3216,7 +3446,8 @@ async def ui_get_spend_by_tags( WHERE spend_date >= $1::date AND spend_date <= $2::date ORDER BY total_spend DESC; """ - response = await prisma_client.db.query_raw( + response = await _query_raw( + prisma_client, sql_query, start_date, end_date, @@ -3234,7 +3465,8 @@ async def ui_get_spend_by_tags( GROUP BY individual_request_tag ORDER BY total_spend DESC; """ - response = await prisma_client.db.query_raw( + response = await _query_raw( + prisma_client, sql_query, start_date, end_date, @@ -3353,7 +3585,7 @@ async def ui_view_session_spend_logs( skip = (page - 1) * page_size # Get total count for pagination metadata - total_records = await SpendLogsRepository(prisma_client).table.count(where=where_conditions) + total_records = await _count_spend_logs(prisma_client, where_conditions) # Query with raw SQL to exclude heavy columns (messages, response, proxy_server_request) sql_query = f""" @@ -3370,7 +3602,9 @@ async def ui_view_session_spend_logs( ORDER BY "startTime" DESC LIMIT $2 OFFSET $3 """ - result = await prisma_client.db.query_raw(sql_query, session_id, page_size, skip, *scope_params) + result: Sequence[Mapping[str, object]] = await _query_raw( + prisma_client, sql_query, session_id, page_size, skip, *scope_params + ) _hydrate_spend_log_metadata(result) total_pages = (total_records + page_size - 1) // page_size @@ -3434,7 +3668,7 @@ async def _build_ui_spend_logs_response( """ count_map: dict[str, int] = {} if enrich_session_counts: - session_ids = list( + session_ids: Sequence[str | None] = list( { (row.get("session_id") if isinstance(row, dict) else getattr(row, "session_id", None)) for row in data @@ -3446,11 +3680,7 @@ async def _build_ui_spend_logs_response( # is bounded by page_size (typically 25-50 distinct session IDs). # If performance degrades at scale, consider short-lived caching or # folding the count into the main query via a window function. - counts = await SpendLogsRepository(prisma_client).table.group_by( - by=["session_id"], - where={"session_id": {"in": session_ids}}, - count={"session_id": True}, - ) + counts = await _count_logs_per_session(prisma_client, session_ids) count_map = {r["session_id"]: r["_count"]["session_id"] for r in counts if r.get("session_id")} session_spend_map: dict[str, dict[str, Union[int, float]]] = {} @@ -3461,14 +3691,15 @@ async def _build_ui_spend_logs_response( # Collect api_keys already present in the authorized page rows so the # aggregate is scoped to the same ownership as the main query — prevents # cross-tenant disclosure via a colliding session_id. - authorized_api_keys = list( + authorized_api_keys: Sequence[str | None] = list( { (row.get("api_key") if isinstance(row, dict) else getattr(row, "api_key", None)) for row in data if (row.get("api_key") if isinstance(row, dict) else getattr(row, "api_key", None)) } ) - rows = await prisma_client.db.query_raw( + rows: Sequence[_SessionSpendRow] = await _query_raw( + prisma_client, """ SELECT session_id, COALESCE(SUM(spend), 0)::double precision AS session_total_spend, @@ -3531,7 +3762,7 @@ async def _build_ui_spend_logs_response( } -def _build_status_filter_condition(status_filter: str | None) -> Dict[str, Any]: +def _build_status_filter_condition(status_filter: str | None) -> Mapping[str, object]: """ Helper function to build the status filter condition for database queries. @@ -3539,7 +3770,7 @@ def _build_status_filter_condition(status_filter: str | None) -> Dict[str, Any]: status_filter (Optional[str]): The status to filter by. Can be "success" or "failure". Returns: - Dict[str, Any]: A dictionary containing the status filter condition. + Mapping[str, object]: A mapping containing the status filter condition. """ if status_filter is None: return {} @@ -3568,7 +3799,7 @@ def _is_admin_view_safe(user_api_key_dict: UserAPIKeyAuth) -> bool: async def _can_team_member_view_log( - prisma_client, + prisma_client: PrismaClient, user_api_key_dict: UserAPIKeyAuth, team_id: str | None, ) -> bool: @@ -3584,7 +3815,7 @@ async def _can_team_member_view_log( if team_id is None: return False - team_row = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id}) + team_row = await _find_team_row(prisma_client, team_id) if team_row is None: return False team_obj = LiteLLM_TeamTable.model_validate(team_row.model_dump()) @@ -3614,7 +3845,7 @@ def _can_user_view_spend_log(user_api_key_dict: UserAPIKeyAuth) -> bool: async def _assert_user_can_view_request_id( - prisma_client, + prisma_client: PrismaClient, user_api_key_dict: UserAPIKeyAuth, request_id: str, ) -> None: @@ -3624,10 +3855,7 @@ async def _assert_user_can_view_request_id( permitted teams (admin or ``/spend/logs`` permission). Raises HTTP 403 if not. """ - row = await SpendLogsRepository(prisma_client).table.find_unique( - where={"request_id": request_id}, - include=None, - ) + row = await _find_spend_log_row(prisma_client, request_id) if row is None: return @@ -3650,7 +3878,7 @@ async def _assert_user_can_view_request_id( async def _get_permitted_team_ids_for_spend_logs( - prisma_client, + prisma_client: PrismaClient, user_api_key_dict: UserAPIKeyAuth, ) -> List[str]: """ @@ -3675,7 +3903,7 @@ async def _get_permitted_team_ids_for_spend_logs( if user_obj is None or not user_obj.teams: return [] - team_rows = await TeamRepository(prisma_client).table.find_many(where={"team_id": {"in": user_obj.teams}}) + team_rows = await _find_team_rows(prisma_client, user_obj.teams) permitted: List[str] = [] for team_row in team_rows: diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 9d30e40cd54..02c4712828d 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -1,5 +1,6 @@ import re import traceback +from collections.abc import Mapping, Sequence from datetime import datetime from typing import ( TYPE_CHECKING, @@ -22,7 +23,11 @@ from litellm.proxy._experimental.mcp_server.utils import ( ) from litellm.responses.main import aresponses from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator -from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.llms.openai import ( + ResponseInputParam, + ResponsesAPIResponse, + ResponsesAPIStreamingResponse, +) from litellm.types.utils import ( CallTypes, Choices, @@ -32,8 +37,10 @@ from litellm.types.utils import ( from litellm.utils import Rules, function_setup if TYPE_CHECKING: + from mcp.types import CallToolResult from mcp.types import Tool as MCPTool + from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging else: MCPTool = Any @@ -94,7 +101,7 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def _parse_mcp_tools( - tools: Optional[Iterable[ToolParam]], + tools: Iterable[Mapping[str, object]] | None, ) -> Tuple[List[ToolParam], List[Any]]: """ Parse tools and separate MCP tools with litellm_proxy from other tools. @@ -134,8 +141,8 @@ class LiteLLM_Proxy_MCP_Handler: async def _apply_toolset_permissions( resolved_toolset_ids: List[str], resolved_mcp_servers: List[str], - user_api_key_auth: Any, - ) -> Any: + user_api_key_auth: "UserAPIKeyAuth", + ) -> "UserAPIKeyAuth": """Apply resolved toolset permissions to user_api_key_auth and return updated auth.""" from litellm.proxy._types import LiteLLM_ObjectPermissionTable @@ -174,8 +181,8 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod async def _get_mcp_tools_from_manager( - user_api_key_auth: Any, - mcp_tools_with_litellm_proxy: Optional[Iterable[ToolParam]], + user_api_key_auth: "UserAPIKeyAuth | None", + mcp_tools_with_litellm_proxy: Iterable[Mapping[str, object]] | None, litellm_trace_id: Optional[str] = None, mcp_auth_header: Optional[str] = None, mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, @@ -330,7 +337,7 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def _filter_mcp_tools_by_allowed_tools( - mcp_tools: List[MCPTool], mcp_tools_with_litellm_proxy: List[ToolParam] + mcp_tools: List[MCPTool], mcp_tools_with_litellm_proxy: Sequence[Mapping[str, object]] ) -> List[MCPTool]: """Filter MCP tools based on allowed_tools parameter from the original tool configs.""" # Collect all allowed tool names from all MCP tool configs @@ -368,8 +375,8 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod async def _process_mcp_tools_to_openai_format( - user_api_key_auth: Any, - mcp_tools_with_litellm_proxy: List[ToolParam], + user_api_key_auth: "UserAPIKeyAuth | None", + mcp_tools_with_litellm_proxy: Sequence[Mapping[str, object]], litellm_trace_id: Optional[str] = None, request_tags: Optional[list[str]] = None, ) -> tuple[List[Any], dict[str, str]]: @@ -402,12 +409,12 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod async def _process_mcp_tools_without_openai_transform( user_api_key_auth: Any, - mcp_tools_with_litellm_proxy: List[ToolParam], + mcp_tools_with_litellm_proxy: Sequence[Mapping[str, object]], litellm_trace_id: Optional[str] = None, mcp_auth_header: Optional[str] = None, mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, request_tags: Optional[list[str]] = None, - ) -> tuple[List[Any], dict[str, str]]: + ) -> tuple[List[MCPTool], dict[str, str]]: """ Process MCP tools through filtering and deduplication pipeline without OpenAI transformation. This is useful for cases where we need the original MCP tool objects (e.g., for events). @@ -453,7 +460,7 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def _transform_mcp_tools_to_openai( - mcp_tools: List[Any], + mcp_tools: Sequence[MCPTool], target_format: Literal["responses", "chat"] = "responses", ) -> List[Any]: """Transform MCP tools to OpenAI-compatible format.""" @@ -464,7 +471,6 @@ class LiteLLM_Proxy_MCP_Handler: openai_tools: List[Any] = [] for mcp_tool in mcp_tools: - openai_tool: Any if target_format == "chat": openai_tool = transform_mcp_tool_to_openai_tool(mcp_tool) else: @@ -475,7 +481,7 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def _should_auto_execute_tools( - mcp_tools_with_litellm_proxy: Union[List[Dict[str, Any]], List[ToolParam]], + mcp_tools_with_litellm_proxy: Sequence[Mapping[str, object]], ) -> bool: """Check if we should auto-execute tool calls. @@ -514,9 +520,9 @@ class LiteLLM_Proxy_MCP_Handler: return tool_calls @staticmethod - def _extract_tool_calls_from_chat_response(response: ModelResponse) -> List[Any]: + def _extract_tool_calls_from_chat_response(response: ModelResponse) -> list[object]: """Extract tool calls from a chat completion response.""" - tool_calls: List[Any] = [] + tool_calls: list[object] = [] try: for choice in response.choices: @@ -583,7 +589,7 @@ class LiteLLM_Proxy_MCP_Handler: return tool_arguments or {} @staticmethod - def _parse_mcp_result(result: Any) -> str: + def _parse_mcp_result(result: "CallToolResult") -> str: """Parse MCP tool call result and extract meaningful content.""" if not result or not hasattr(result, "content") or not result.content: return "Tool executed successfully" @@ -626,7 +632,7 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod async def _execute_tool_calls( tool_server_map: dict[str, str], - tool_calls: List[Any], + tool_calls: Sequence[object], user_api_key_auth: Any, mcp_auth_header: Optional[str] = None, mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, @@ -908,7 +914,7 @@ class LiteLLM_Proxy_MCP_Handler: def _create_follow_up_messages_for_chat( original_messages: List[Any], response: ModelResponse, - tool_results: List[Dict[str, Any]], + tool_results: Sequence[Mapping[str, object]], ) -> List[Any]: """Create follow-up chat messages that include tool execution results.""" from copy import deepcopy @@ -952,8 +958,8 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def _create_follow_up_input( response: ResponsesAPIResponse, - tool_results: List[Dict[str, Any]], - original_input: Any = None, + tool_results: Sequence[Mapping[str, object]], + original_input: str | ResponseInputParam | None = None, ) -> List[Any]: """Create follow-up input with tool results in proper format.""" follow_up_input: List[Any] = [] @@ -1049,7 +1055,7 @@ class LiteLLM_Proxy_MCP_Handler: *, proxy_logging_obj: Optional["ProxyLogging"], user_api_key_auth: Any, - request_data: Dict[str, Any], + request_data: dict[str, object], error: Exception, ) -> None: """Log MCP tool failures via proxy logging hooks.""" @@ -1071,11 +1077,11 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def _create_mcp_streaming_response( - input: Union[str, Any], + input: str | ResponseInputParam, model: str, - all_tools: Optional[List[Any]], - mcp_tools_with_litellm_proxy: List[Any], - mcp_discovery_events: List[Any], + all_tools: Sequence[object] | None, + mcp_tools_with_litellm_proxy: list[Mapping[str, object]], + mcp_discovery_events: list[ResponsesAPIStreamingResponse], call_params: Dict[str, Any], previous_response_id: Optional[str], tool_server_map: dict[str, str], @@ -1116,9 +1122,9 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def _build_request_params( - input: Union[str, Any], + input: str | ResponseInputParam, model: str, - all_tools: Optional[List[Any]], + all_tools: Sequence[object] | None, call_params: Dict[str, Any], previous_response_id: Optional[str], **kwargs, @@ -1149,7 +1155,9 @@ class LiteLLM_Proxy_MCP_Handler: return request_params @staticmethod - def _create_tool_execution_events(tool_calls: List[Any], tool_results: List[Dict[str, Any]]) -> List[Any]: + def _create_tool_execution_events( + tool_calls: Sequence[object], tool_results: List[Dict[str, Any]] + ) -> list[ResponsesAPIStreamingResponse]: """ Create MCP tool execution events for streaming. @@ -1163,7 +1171,7 @@ class LiteLLM_Proxy_MCP_Handler: from litellm._uuid import uuid from litellm.responses.mcp.mcp_streaming_iterator import create_mcp_call_events - tool_execution_events: List[Any] = [] + tool_execution_events: list[ResponsesAPIStreamingResponse] = [] # Create events for each tool execution for tool_result in tool_results: @@ -1233,8 +1241,8 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def _add_mcp_output_elements_to_response( response: ResponsesAPIResponse, - mcp_tools_fetched: List[Any], - tool_results: List[Dict[str, Any]], + mcp_tools_fetched: Sequence[object], + tool_results: Sequence[Mapping[str, object]], ) -> ResponsesAPIResponse: """Add custom output elements to the final response for MCP tool execution.""" # Import the required classes for creating output items diff --git a/litellm/videos/main.py b/litellm/videos/main.py index 6d81fec36b3..cc3a15f9cd2 100644 --- a/litellm/videos/main.py +++ b/litellm/videos/main.py @@ -2,7 +2,7 @@ import asyncio import contextvars import json from functools import partial -from typing import Any, Coroutine, Dict, List, Literal, Optional, Union, overload +from typing import Coroutine, Dict, List, Literal, Optional, Union, overload import litellm from litellm.constants import DEFAULT_VIDEO_ENDPOINT_MODEL @@ -40,9 +40,9 @@ async def avideo_generation( custom_llm_provider=None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> VideoObject: """ @@ -126,13 +126,13 @@ def video_generation( user: Optional[str] = None, timeout: int = 600, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, *, avideo_generation: Literal[True], - **kwargs: Any, -) -> Coroutine[Any, Any, VideoObject]: + **kwargs: object, +) -> Coroutine[object, object, VideoObject]: ... @@ -146,12 +146,12 @@ def video_generation( user: Optional[str] = None, timeout: int = 600, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, *, avideo_generation: Literal[False] = False, - **kwargs: Any, + **kwargs: object, ) -> VideoObject: ... @@ -170,13 +170,13 @@ def video_generation( custom_llm_provider=None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> Union[ VideoObject, - Coroutine[Any, Any, VideoObject], + Coroutine[object, object, VideoObject], ]: """ Maps the https://api.openai.com/v1/videos endpoint. @@ -277,13 +277,13 @@ def video_content( variant: Optional[str] = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> Union[ bytes, - Coroutine[Any, Any, bytes], + Coroutine[object, object, bytes], ]: """ Download video content from OpenAI's video API. @@ -390,9 +390,9 @@ async def avideo_content( variant: Optional[str] = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> bytes: """ @@ -461,9 +461,9 @@ async def avideo_remix( custom_llm_provider=None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> VideoObject: """ @@ -528,13 +528,13 @@ def video_remix( prompt: str, timeout: int = 600, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, *, avideo_remix: Literal[True], - **kwargs: Any, -) -> Coroutine[Any, Any, VideoObject]: + **kwargs: object, +) -> Coroutine[object, object, VideoObject]: ... @@ -544,12 +544,12 @@ def video_remix( prompt: str, timeout: int = 600, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, *, avideo_remix: Literal[False] = False, - **kwargs: Any, + **kwargs: object, ) -> VideoObject: ... @@ -564,13 +564,13 @@ def video_remix( custom_llm_provider=None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> Union[ VideoObject, - Coroutine[Any, Any, VideoObject], + Coroutine[object, object, VideoObject], ]: """ Maps the https://api.openai.com/v1/videos/{video_id}/remix endpoint. @@ -668,9 +668,9 @@ async def avideo_list( custom_llm_provider=None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> List[VideoObject]: """ @@ -744,13 +744,13 @@ def video_list( order: Optional[str] = None, timeout: int = 600, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, *, avideo_list: Literal[True], - **kwargs: Any, -) -> Coroutine[Any, Any, List[VideoObject]]: + **kwargs: object, +) -> Coroutine[object, object, List[VideoObject]]: ... @@ -761,12 +761,12 @@ def video_list( order: Optional[str] = None, timeout: int = 600, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, *, avideo_list: Literal[False] = False, - **kwargs: Any, + **kwargs: object, ) -> List[VideoObject]: ... @@ -782,13 +782,13 @@ def video_list( custom_llm_provider=None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> Union[ List[VideoObject], - Coroutine[Any, Any, List[VideoObject]], + Coroutine[object, object, List[VideoObject]], ]: """ Maps the https://api.openai.com/v1/videos endpoint. @@ -882,9 +882,9 @@ async def avideo_status( custom_llm_provider=None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> VideoObject: """ @@ -947,13 +947,13 @@ def video_status( video_id: str, timeout: int = 600, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, *, avideo_status: Literal[True], - **kwargs: Any, -) -> Coroutine[Any, Any, VideoObject]: + **kwargs: object, +) -> Coroutine[object, object, VideoObject]: ... # Overload for when avideo_status=False (returns VideoObject) @@ -962,12 +962,12 @@ def video_status( video_id: str, timeout: int = 600, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, *, avideo_status: Literal[False] = False, - **kwargs: Any, + **kwargs: object, ) -> VideoObject: ... @@ -981,13 +981,13 @@ def video_status( custom_llm_provider=None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> Union[ VideoObject, - Coroutine[Any, Any, VideoObject], + Coroutine[object, object, VideoObject], ]: """ Retrieve video status from OpenAI's video API. @@ -1097,12 +1097,12 @@ def video_status( @client async def avideo_create_character( name: str, - video: Any, + video: FileTypes, timeout=600, custom_llm_provider=None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> CharacterObject: """ @@ -1152,14 +1152,14 @@ async def avideo_create_character( @client def video_create_character( name: str, - video: Any, + video: FileTypes, timeout=600, custom_llm_provider=None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, -) -> Union[CharacterObject, Coroutine[Any, Any, CharacterObject]]: +) -> Union[CharacterObject, Coroutine[object, object, CharacterObject]]: """ Create a character from an uploaded video file. Maps to POST /v1/videos/characters @@ -1230,9 +1230,9 @@ async def avideo_get_character( character_id: str, timeout=600, custom_llm_provider=None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> CharacterObject: """ @@ -1280,11 +1280,11 @@ def video_get_character( character_id: str, timeout=600, custom_llm_provider=None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, -) -> Union[CharacterObject, Coroutine[Any, Any, CharacterObject]]: +) -> Union[CharacterObject, Coroutine[object, object, CharacterObject]]: """ Retrieve a character by ID. Maps to GET /v1/videos/characters/{character_id} @@ -1355,9 +1355,9 @@ async def avideo_edit( prompt: str, timeout=600, custom_llm_provider=None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> VideoObject: """ @@ -1407,11 +1407,11 @@ def video_edit( prompt: str, timeout=600, custom_llm_provider=None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, -) -> Union[VideoObject, Coroutine[Any, Any, VideoObject]]: +) -> Union[VideoObject, Coroutine[object, object, VideoObject]]: """ Create a video edit job. Maps to POST /v1/videos/edits @@ -1486,9 +1486,9 @@ async def avideo_extension( seconds: str, timeout=600, custom_llm_provider=None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, ) -> VideoObject: """ @@ -1540,11 +1540,11 @@ def video_extension( seconds: str, timeout=600, custom_llm_provider=None, - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, object]] = None, + extra_query: Optional[Dict[str, object]] = None, + extra_body: Optional[Dict[str, object]] = None, **kwargs, -) -> Union[VideoObject, Coroutine[Any, Any, VideoObject]]: +) -> Union[VideoObject, Coroutine[object, object, VideoObject]]: """ Create a video extension. Maps to POST /v1/videos/extensions diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 6fb3ed748b6..b8650eea7aa 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -1,6 +1,6 @@ { "ANN001": { - "limit": 3118 + "limit": 3104 }, "ANN002": { "limit": 69 @@ -9,10 +9,10 @@ "limit": 831 }, "ANN201": { - "limit": 2138 + "limit": 2137 }, "ANN202": { - "limit": 944 + "limit": 941 }, "ANN204": { "limit": 724 @@ -24,7 +24,7 @@ "limit": 130 }, "ANN401": { - "limit": 2009 + "limit": 1851 }, "ASYNC230": { "limit": 14 @@ -123,7 +123,7 @@ "limit": 52 }, "I001": { - "limit": 270 + "limit": 261 }, "LOG015": { "limit": 8 @@ -222,7 +222,7 @@ "limit": 38 }, "RET504": { - "limit": 716 + "limit": 702 }, "RUF010": { "limit": 874 @@ -306,7 +306,7 @@ "limit": 9 }, "TID251": { - "limit": 2652 + "limit": 2649 }, "TRY002": { "limit": 547 @@ -324,7 +324,7 @@ "limit": 879 }, "UP006": { - "limit": 12135 + "limit": 12050 }, "UP007": { "limit": 2526 @@ -360,9 +360,9 @@ "limit": 4 }, "UP037": { - "limit": 105 + "limit": 104 }, "UP045": { - "limit": 17805 + "limit": 17793 } } diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 05499e83c42..ff037a2872e 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 23250 + "limit": 23191 }, "LIT002": { - "limit": 27277 + "limit": 27276 }, "LIT003": { "limit": 292 @@ -15,7 +15,7 @@ "limit": 0 }, "LIT006": { - "limit": 1108 + "limit": 1106 }, "LIT007": { "limit": 0 @@ -24,6 +24,6 @@ "limit": 1004 }, "LIT009": { - "limit": 2473 + "limit": 2467 } } From 48b3d1889395bb63c0a3a458167ed65073538b5d Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 1 Aug 2026 09:28:02 -0700 Subject: [PATCH 34/50] feat(ui): note in the add-member modal that search covers existing users only Both fields select from a server-side search over existing accounts, so a typed-in address or id never becomes a value. Say so up front rather than letting the form look like it accepts a new user and fail on submit. Applies to the organization member modal too, which shares this component. --- .../common_components/user_search_modal.test.tsx | 10 ++++++++++ .../components/common_components/user_search_modal.tsx | 10 +++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx b/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx index 72b0e10e5d6..634c78d28d0 100644 --- a/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx +++ b/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx @@ -65,4 +65,14 @@ describe("UserSearchModal", () => { expect(userFilterUICall).not.toHaveBeenCalled(); }); + + it("tells the user that only existing accounts can be selected", () => { + renderModal(); + + const notice = screen.getByRole("alert"); + expect(notice).toHaveTextContent(/users that already exist/i); + expect(notice).toHaveTextContent(/ask a proxy admin to create their account first/i); + // info, not warning: a warning here would read as an error state on an empty form + expect(notice.className).toMatch(/ant-alert-info/); + }); }); diff --git a/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx b/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx index fafafd8e5d5..9c1d64a1f86 100644 --- a/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx +++ b/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx @@ -1,5 +1,5 @@ import { useState } from "react"; -import { Modal, Form, Button, Select, Tooltip } from "antd"; +import { Modal, Form, Button, Select, Tooltip, Alert } from "antd"; import { UserAddOutlined } from "@ant-design/icons"; import { useDebouncedCallback } from "@tanstack/react-pacer/debouncer"; import { userFilterUICall } from "@/components/networking"; @@ -140,6 +140,14 @@ const UserSearchModal: React.FC = ({ role: defaultRole, }} > + +