From 28c1e431968d61ea7caf1d82e351aca765ec83a1 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 14 Aug 2026 00:01:16 -0700 Subject: [PATCH 01/24] feat(ui): standardize the Teams page header --- .../_components/AccessGroupsPage.tsx | 4 +- .../budgets/_components/budget_panel.tsx | 4 +- .../projects/_components/ProjectsPage.tsx | 4 +- .../src/components/Teams.test.tsx | 37 ++++++--- ui/litellm-dashboard/src/components/Teams.tsx | 47 +++++------ .../VirtualKeysPage/VirtualKeysTable.tsx | 4 +- .../shared/LegacyPageHeader.test.tsx | 33 ++++++++ .../components/shared/LegacyPageHeader.tsx | 25 ++++++ .../src/components/shared/PageHeader.test.tsx | 80 +++++++++++++++---- .../src/components/shared/PageHeader.tsx | 63 +++++++++++---- 10 files changed, 224 insertions(+), 77 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/shared/LegacyPageHeader.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/LegacyPageHeader.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx index f37acb3d85a..aeff249fdd3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx @@ -3,7 +3,7 @@ import { useDeleteAccessGroup } from "@/app/(dashboard)/hooks/accessGroups/useDe import { Plus, SearchIcon, X } from "lucide-react"; import { useMemo, useState } from "react"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; -import { PageHeader } from "@/components/shared/PageHeader"; +import { LegacyPageHeader } from "@/components/shared/LegacyPageHeader"; import { Button } from "@/components/ui/button"; import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; import { AccessGroupDetail } from "./AccessGroupsDetailsPage"; @@ -61,7 +61,7 @@ export function AccessGroupsPage() { return (
- = ({ accessToken }) => { return (
- } title="Budgets" subtitle="Spend, TPM and RPM limits you can assign to customers." diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectsPage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectsPage.tsx index 91ad9f847f5..dc18a05edca 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectsPage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectsPage.tsx @@ -3,7 +3,7 @@ import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import { Plus, SearchIcon, X } from "lucide-react"; import { parseAsString, useQueryState } from "nuqs"; import { useMemo, useState } from "react"; -import { PageHeader } from "@/components/shared/PageHeader"; +import { LegacyPageHeader } from "@/components/shared/LegacyPageHeader"; import { Button } from "@/components/ui/button"; import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; import { CreateProjectModal } from "./ProjectModals/CreateProjectModal"; @@ -56,7 +56,7 @@ export function ProjectsPage() { return (
- { expect(onUrlUpdate.mock.calls.at(-1)![0].searchParams.has("team")).toBe(false); await waitFor(() => expect(screen.queryByTestId("team-info-view")).not.toBeInTheDocument()); }); + + it("should preserve the legacy inset for the team detail view", async () => { + renderWithQueryClient(, { + searchParams: "?team=team-from-url", + }); + + await waitFor(() => expect(mockTeamInfoView).toHaveBeenCalled()); + expect(screen.getByRole("main")).toHaveClass("px-12", "py-6"); + }); }); describe("Teams - Create Team CTA is grouped with the tabs on the left", () => { @@ -521,22 +530,28 @@ describe("Teams - Create Team CTA is grouped with the tabs on the left", () => { mockUseOrganizations.mockReturnValue({ data: [] }); }); - it("renders the Create Team button inside the tab bar, ahead of the tabs", () => { - const { container } = renderWithQueryClient(); + it("should render the Create Team button inside the tab bar, ahead of the tabs", () => { + renderWithQueryClient(); - const createButton = screen.getByTestId("create-team-button"); - const tabNav = container.querySelector(".ant-tabs-nav"); + const tabNav = screen.getByRole("tablist"); + const createButton = within(tabNav).getByTestId("create-team-button"); + const firstTab = within(tabNav).getByRole("tab", { name: "Your Teams" }); + const tabs = tabNav.closest(".ant-tabs"); - // The CTA lives in the tab bar's left slot, not the standalone page header. - expect(tabNav).not.toBeNull(); - expect(tabNav!.contains(createButton)).toBe(true); - - // It reads as the left end of the cluster: it precedes the first tab in DOM order. - const firstTab = screen.getByRole("tab", { name: "Your Teams" }); + expect(screen.getByRole("main")).toHaveClass("p-8"); + expect(within(tabNav).getByRole("separator")).toBeInTheDocument(); expect(createButton.compareDocumentPosition(firstTab) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + expect(tabs).toHaveClass( + "[&>.ant-tabs-nav]:!mb-6", + "[&>.ant-tabs-nav]:before:!border-b-0", + "[&_.ant-tabs-ink-bar]:!h-0.5", + "[&_.ant-tabs-tab]:!py-[7px]", + "[&_.ant-tabs-tab+_.ant-tabs-tab]:!ml-[22px]", + "[&_.ant-tabs-tab-active]:font-semibold", + ); }); - it("omits the Create Team CTA for a role that cannot manage teams", () => { + it("should omit the Create Team CTA for a role that cannot manage teams", () => { renderWithQueryClient(); expect(screen.queryByTestId("create-team-button")).not.toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/components/Teams.tsx b/ui/litellm-dashboard/src/components/Teams.tsx index becbe0e2b48..5b79b067412 100644 --- a/ui/litellm-dashboard/src/components/Teams.tsx +++ b/ui/litellm-dashboard/src/components/Teams.tsx @@ -6,7 +6,7 @@ import TeamSSOSettings from "@/components/TeamSSOSettings"; import { isProxyAdminRole } from "@/utils/roles"; import { InfoCircleOutlined } from "@ant-design/icons"; import { Accordion, AccordionBody, AccordionHeader, TextInput } from "@tremor/react"; -import { Button, Form, Input, Layout, Modal, Select, Switch, Tabs, theme, Tooltip, Typography } from "antd"; +import { Button, Form, Input, Layout, Modal, Select, Switch, Tabs, Tooltip, Typography } from "antd"; import { Plus, Users } from "lucide-react"; import React, { useEffect, useState } from "react"; import { useQuery, useQueryClient } from "@tanstack/react-query"; @@ -403,7 +403,6 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser return false; }; - const { token } = theme.useToken(); const { Text } = Typography; const { Content } = Layout; @@ -474,7 +473,7 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser ]; return ( - + {selectedTeamId ? ( = ({ accessToken, userID, userRole, premiumUser premiumUser={premiumUser} /> ) : ( - <> -
- } - title="Teams" - subtitle="Manage teams, members, and their access to models and budgets" + } + title="Teams" + subtitle="Manage teams, members, and their access to models and budgets" + primaryAction={ + canCreateOrManageTeams(userRole, userID, organizations) ? ( + setIsTeamModalVisible(true)} data-testid="create-team-button"> + + Create Team + + ) : undefined + } + tabs={({ leadingControls }) => ( + -
- - - setIsTeamModalVisible(true)} data-testid="create-team-button"> - - Create Team - -
-
- ) : undefined, - }} - /> - + )} + /> )} {canCreateOrManageTeams(userRole, userID, organizations) && ( diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx index fa0360c0dda..b278b4f675d 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx @@ -12,7 +12,7 @@ import { DataTableToolbar, } from "@/components/shared/DataTable"; import { SearchSelect } from "@/components/shared/SearchSelect"; -import { PageHeader } from "@/components/shared/PageHeader"; +import { LegacyPageHeader } from "@/components/shared/LegacyPageHeader"; import { Input } from "@/components/ui/input"; import { useDebouncedValue } from "@tanstack/react-pacer/debouncer"; import { ColumnFiltersState, OnChangeFn, PaginationState, SortingState } from "@tanstack/react-table"; @@ -172,7 +172,7 @@ export function VirtualKeysTable({ headerActions }: VirtualKeysTableProps) { return (
- } title="Virtual Keys" subtitle="Every key that authenticates requests to the gateway." diff --git a/ui/litellm-dashboard/src/components/shared/LegacyPageHeader.test.tsx b/ui/litellm-dashboard/src/components/shared/LegacyPageHeader.test.tsx new file mode 100644 index 00000000000..a0081c1c7f1 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/LegacyPageHeader.test.tsx @@ -0,0 +1,33 @@ +import { renderWithProviders, screen } from "@/../tests/test-utils"; +import { describe, expect, it } from "vitest"; + +import { LegacyPageHeader } from "./LegacyPageHeader"; + +describe("LegacyPageHeader", () => { + it("should render the title as a heading", () => { + renderWithProviders(); + + expect(screen.getByRole("heading", { name: "Virtual Keys" })).toBeInTheDocument(); + }); + + it("should render the optional identity and actions", () => { + renderWithProviders( + Key icon} + actions={} + />, + ); + + expect(screen.getByText("Every key that authenticates requests")).toBeInTheDocument(); + expect(screen.getByText("Key icon")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Create New Key" })).toBeInTheDocument(); + }); + + it("should omit optional actions when none are provided", () => { + renderWithProviders(); + + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/LegacyPageHeader.tsx b/ui/litellm-dashboard/src/components/shared/LegacyPageHeader.tsx new file mode 100644 index 00000000000..43979ad00b4 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/LegacyPageHeader.tsx @@ -0,0 +1,25 @@ +"use client"; + +import * as React from "react"; + +interface LegacyPageHeaderProps { + title: React.ReactNode; + subtitle?: React.ReactNode; + icon?: React.ReactNode; + actions?: React.ReactNode; +} + +export function LegacyPageHeader({ title, subtitle, icon, actions }: LegacyPageHeaderProps) { + return ( +
+
+ {icon != null && {icon}} +
+

{title}

+ {subtitle != null &&

{subtitle}

} +
+
+ {actions != null &&
{actions}
} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/shared/PageHeader.test.tsx b/ui/litellm-dashboard/src/components/shared/PageHeader.test.tsx index f7a313271da..3741542abad 100644 --- a/ui/litellm-dashboard/src/components/shared/PageHeader.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/PageHeader.test.tsx @@ -1,31 +1,77 @@ -import { render, screen } from "@testing-library/react"; +import { renderWithProviders, screen, within } from "@/../tests/test-utils"; import { describe, expect, it } from "vitest"; import { PageHeader } from "./PageHeader"; +const identity = { + icon: Teams icon, + title: "Teams", + subtitle: "Manage teams, members, and their access to models and budgets", +}; + describe("PageHeader", () => { - it("renders the title as a heading", () => { - render(); - expect(screen.getByRole("heading", { name: "Virtual Keys" })).toBeInTheDocument(); + it("should render the page identity", () => { + renderWithProviders(); + + expect(screen.getByRole("heading", { name: "Teams" })).toBeInTheDocument(); + expect(screen.getByText("Teams icon").parentElement).toHaveAttribute("aria-hidden", "true"); + expect(screen.getByText(identity.subtitle)).toBeInTheDocument(); }); - it("renders the subtitle, icon, and actions when provided", () => { - render( + it("should apply the standard title and subtext typography", () => { + renderWithProviders(); + + const icon = screen.getByText("Teams icon").parentElement; + expect(screen.getByRole("heading", { name: "Teams" })).toHaveClass("text-2xl", "font-semibold", "tracking-tight"); + expect(screen.getByText(identity.subtitle)).toHaveClass("mt-1.5", "text-sm", "text-muted-foreground"); + expect(icon).toHaveClass("size-5", "[&_svg]:size-5", "[&_svg]:stroke-[1.75]"); + expect(icon?.parentElement).toHaveClass("gap-2.5"); + }); + + it("should render the primary action, divider, tabs, and utilities in the standard control row", () => { + renderWithProviders( } - actions={} + {...identity} + primaryAction={} + tabs={ +
+ +
+ } + utilities={} />, ); - expect(screen.getByText("Every key that authenticates requests")).toBeInTheDocument(); - expect(screen.getByTestId("icon")).toBeInTheDocument(); - expect(screen.getByRole("button", { name: "Create New Key" })).toBeInTheDocument(); + + const controls = screen.getByRole("group", { name: "Page controls" }); + expect(controls).toHaveClass("mt-5", "h-9"); + expect(within(controls).getByRole("separator")).toHaveClass("mx-4", "h-6"); + expect(controls).toHaveTextContent("Create TeamYour TeamsRefresh"); }); - it("omits the optional slots when not provided", () => { - render(); - expect(screen.queryByRole("button")).not.toBeInTheDocument(); - expect(document.querySelector("p")).toBeNull(); + it("should omit the divider when tabs are absent", () => { + renderWithProviders(Create Team} />); + + expect(screen.queryByRole("separator")).not.toBeInTheDocument(); + }); + + it("should provide standard controls to an embedded tab shell", () => { + renderWithProviders( + Create Team} + tabs={({ leadingControls, utilities }) => ( +
+ {leadingControls} + + {utilities} +
+ )} + utilities={} + />, + ); + + const tabs = screen.getByRole("tablist"); + expect(within(tabs).getByRole("separator")).toBeInTheDocument(); + expect(tabs).toHaveTextContent("Create TeamYour TeamsRefresh"); }); }); diff --git a/ui/litellm-dashboard/src/components/shared/PageHeader.tsx b/ui/litellm-dashboard/src/components/shared/PageHeader.tsx index e314e8e8bc2..81092821efc 100644 --- a/ui/litellm-dashboard/src/components/shared/PageHeader.tsx +++ b/ui/litellm-dashboard/src/components/shared/PageHeader.tsx @@ -2,24 +2,57 @@ import * as React from "react"; -interface PageHeaderProps { - title: React.ReactNode; - subtitle?: React.ReactNode; - icon?: React.ReactNode; - actions?: React.ReactNode; +import { ToolbarSeparator } from "./ToolbarSeparator"; + +interface EmbeddedTabsSlots { + leadingControls: React.ReactNode; + utilities: React.ReactNode; } -export function PageHeader({ title, subtitle, icon, actions }: PageHeaderProps) { - return ( -
-
- {icon != null && {icon}} -
-

{title}

- {subtitle != null &&

{subtitle}

} -
+interface PageHeaderProps { + title: React.ReactNode; + subtitle: React.ReactNode; + icon: React.ReactNode; + primaryAction?: React.ReactNode; + tabs?: React.ReactNode | ((slots: EmbeddedTabsSlots) => React.ReactNode); + utilities?: React.ReactNode; +} + +export function PageHeader({ title, subtitle, icon, primaryAction, tabs, utilities }: PageHeaderProps) { + const leadingControls = + primaryAction == null ? null : ( +
+ {primaryAction} + {tabs != null && }
- {actions != null &&
{actions}
} + ); + const utilityControls = utilities == null ? null :
{utilities}
; + const hasControlRow = primaryAction != null || tabs != null || utilities != null; + + return ( +
+
+ +

{title}

+
+

{subtitle}

+ + {typeof tabs === "function" ? ( +
{tabs({ leadingControls, utilities: utilityControls })}
+ ) : ( + hasControlRow && ( +
+ {leadingControls} + {tabs} + {utilityControls != null &&
{utilityControls}
} +
+ ) + )}
); } From 20a3a16c2f6ba7a7d5755cf56740d5180973ede0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:25:43 -0700 Subject: [PATCH 02/24] fix(proxy): populate deployment fields on failed-request spend logs from the standard logging payload --- .../spend_tracking/spend_tracking_utils.py | 30 ++++++-- .../test_spend_tracking_utils.py | 77 +++++++++++++++++++ 2 files changed, 102 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 3146d8bccfb..0b56f0d8246 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -216,6 +216,15 @@ def _extract_usage_for_ocr_call(response_obj: Any, response_obj_dict: dict) -> d return {} +def _sl_attribution_fallback( + standard_logging_payload: StandardLoggingPayload | None, + field: Literal["model_id", "model_group", "api_base", "custom_llm_provider"], +) -> str: + if standard_logging_payload is None: + return "" + return standard_logging_payload.get(field) or "" + + def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogsPayload: if kwargs is None: kwargs = {} @@ -288,8 +297,15 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs ): # use 'tags' from standard logging payload instead request_tags = safe_dumps(standard_logging_payload["request_tags"]) - _model_id: Final = metadata.get("model_info", {}).get("id", "") - _model_group: Final = metadata.get("model_group", "") + _model_id: Final = metadata.get("model_info", {}).get("id", "") or _sl_attribution_fallback( + standard_logging_payload, "model_id" + ) + _model_group: Final = metadata.get("model_group", "") or _sl_attribution_fallback( + standard_logging_payload, "model_group" + ) + _api_base: Final = litellm_params.get("api_base", "") or _sl_attribution_fallback( + standard_logging_payload, "api_base" + ) # Extract overhead from hidden_params if available litellm_overhead_time_ms = None @@ -389,7 +405,11 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs # Extract agent_id for A2A requests (set directly on model_call_details) agent_id: Final[str | None] = kwargs.get("agent_id") or metadata.get("agent_id") - custom_llm_provider: Final = kwargs.get("custom_llm_provider") + custom_llm_provider: Final = ( + kwargs.get("custom_llm_provider") + or _sl_attribution_fallback(standard_logging_payload, "custom_llm_provider") + or None + ) raw_model: Final = cast(str, kwargs.get("model") or "") model_name: Final = reconstruct_model_name(raw_model, custom_llm_provider, metadata or {}) @@ -414,13 +434,13 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs completion_tokens=usage.get("completion_tokens", standard_logging_completion_tokens), request_tags=request_tags, end_user=end_user_id or "", - api_base=litellm_params.get("api_base", ""), + api_base=_api_base, model_group=_model_group, model_id=_model_id, mcp_namespaced_tool_name=mcp_namespaced_tool_name, agent_id=agent_id, requester_ip_address=clean_metadata.get("requester_ip_address", None), - custom_llm_provider=kwargs.get("custom_llm_provider", ""), + custom_llm_provider=custom_llm_provider or "", messages=_get_messages_for_spend_logs_payload( standard_logging_payload=standard_logging_payload, metadata=metadata ), diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index e5add059260..9710dc44e99 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -3164,3 +3164,80 @@ def test_batch_cost_row_id_is_stable_across_repeated_accounting(): ] assert ids[0] == ids[1] == "batch_same_batch_cost" + + +def _make_failed_request_standard_logging_payload() -> StandardLoggingPayload: + base: Final = _make_standard_logging_payload_with_usage_object(usage_object={}) + return cast( + StandardLoggingPayload, + { + **base, + "status": "failure", + "call_type": "aresponses", + "model_id": "mid-123", + "model_group": "group-x", + "api_base": "https://api.openai.com/v1/responses", + "custom_llm_provider": "openai", + }, + ) + + +def test_get_logging_payload_failed_request_falls_back_to_standard_logging_payload(): + """Failed-request kwargs from the proxy failure hook carry no deployment info + (LIT-5795), so the attribution columns must come from the failure-time + standard_logging_object.""" + payload = get_logging_payload( + kwargs={ + "model": "group-x", + "litellm_params": {"metadata": {"user_api_key": "test-key", "status": "failure"}}, + "standard_logging_object": _make_failed_request_standard_logging_payload(), + }, + response_obj={}, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + assert payload["model_id"] == "mid-123" + assert payload["model_group"] == "group-x" + assert payload["api_base"] == "https://api.openai.com/v1/responses" + assert payload["custom_llm_provider"] == "openai" + + +def test_get_logging_payload_request_kwargs_win_over_standard_logging_payload(): + payload = get_logging_payload( + kwargs={ + "model": "group-y", + "custom_llm_provider": "anthropic", + "litellm_params": { + "api_base": "https://kwargs.example.com", + "metadata": { + "user_api_key": "test-key", + "model_group": "kwargs-group", + "model_info": {"id": "kwargs-mid"}, + }, + }, + "standard_logging_object": _make_failed_request_standard_logging_payload(), + }, + response_obj={}, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + assert payload["model_id"] == "kwargs-mid" + assert payload["model_group"] == "kwargs-group" + assert payload["api_base"] == "https://kwargs.example.com" + assert payload["custom_llm_provider"] == "anthropic" + + +def test_get_logging_payload_failed_request_without_standard_logging_payload_leaves_fields_empty(): + payload = get_logging_payload( + kwargs={ + "model": "group-x", + "litellm_params": {"metadata": {"user_api_key": "test-key", "status": "failure"}}, + }, + response_obj={}, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + assert payload["model_id"] == "" + assert payload["model_group"] == "" + assert payload["api_base"] == "" + assert payload["custom_llm_provider"] == "" From b39a339b7d9e67efcd8fddde8c289c59410d9160 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:21:06 -0700 Subject: [PATCH 03/24] fix(vertex_ai): apply regional endpoint uplift to cost tracking --- basedpyright-code-budget.json | 8 +- ci_cd/generate_model_prices_schema.py | 5 + litellm/cost_calculator.py | 15 ++ litellm/litellm_core_utils/litellm_logging.py | 28 +++- .../litellm_core_utils/llm_cost_calc/utils.py | 48 +++++++ litellm/llms/vertex_ai/cost_calculator.py | 19 ++- litellm/llms/vertex_ai/vertex_llm_base.py | 17 ++- ...odel_prices_and_context_window_backup.json | 25 ++++ litellm/proxy/spend_tracking/savings.py | 8 +- litellm/types/utils.py | 17 ++- litellm/utils.py | 1 + model_prices_and_context_window.json | 25 ++++ model_prices_and_context_window.schema.json | 5 + .../llm_cost_calc/test_llm_cost_calc_utils.py | 132 ++++++++++++++++++ .../test_litellm_logging.py | 77 ++++++++++ .../proxy/spend_tracking/test_savings.py | 38 +++++ tests/test_litellm/test_cost_calculator.py | 69 +++++++++ type-discipline-budget.json | 2 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 + 19 files changed, 516 insertions(+), 27 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 1ce71c5bd2c..32c77146b42 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5663 }, "reportMissingTypeArgument": { - "limit": 15557 + "limit": 15556 }, "reportMissingTypeStubs": { "limit": 40 @@ -105,13 +105,13 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 39043 + "limit": 39030 }, "reportUnknownParameterType": { - "limit": 19887 + "limit": 19886 }, "reportUnknownVariableType": { - "limit": 30574 + "limit": 30573 }, "reportUnnecessaryCast": { "limit": 117 diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 153fbc0fdc2..252e3675329 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -145,6 +145,11 @@ NUMBER_KEYS: dict[str, JsonSchema] = { "minimum": 1, "description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).", }, + "regional_endpoint_uplift_multiplier": { + "type": "number", + "minimum": 1, + "description": "Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%).", + }, } COST_DESCRIPTIONS: dict[str, str] = { diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 7d7380665d3..8f7cd09d364 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -327,6 +327,8 @@ def cost_per_token( service_tier: str | None = None, # for OpenAI service tier pricing ### DATA RESIDENCY ### data_residency: str | None = None, # for OpenAI regional-processing uplift (e.g. "eu", "us") + ### VERTEX LOCATION ### + vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") response: Any | None = None, ### REQUEST MODEL ### request_model: str | None = None, # original request model for router detection @@ -587,6 +589,7 @@ def cost_per_token( prompt_characters=prompt_characters, completion_characters=completion_characters, usage=usage_block, + vertex_location=vertex_location, ) elif cost_router == "cost_per_token": return google_cost_per_token( @@ -594,6 +597,7 @@ def cost_per_token( custom_llm_provider=custom_llm_provider, usage=usage_block, service_tier=service_tier, + vertex_location=vertex_location, ) elif custom_llm_provider == "anthropic": return anthropic_cost_per_token(model=model, usage=usage_block, service_tier=service_tier) @@ -1071,6 +1075,7 @@ def _store_cost_breakdown_in_logging_obj( reasoning_cost: float | None = None, service_tier: str | None = None, data_residency: str | None = None, + vertex_location: str | None = None, ) -> None: """ Helper function to store cost breakdown in the logging object. @@ -1090,6 +1095,7 @@ def _store_cost_breakdown_in_logging_obj( margin_total_amount: Total margin added in USD service_tier: Tier the costs above were priced on, already resolved data_residency: Region uplift the costs above were priced on, already resolved + vertex_location: Vertex AI location the costs above were priced on, already resolved """ if litellm_logging_obj is None: return @@ -1113,6 +1119,7 @@ def _store_cost_breakdown_in_logging_obj( reasoning_cost=reasoning_cost, service_tier=service_tier, data_residency=data_residency, + vertex_location=vertex_location, ) except Exception as breakdown_error: @@ -1149,6 +1156,8 @@ def completion_cost( service_tier: str | None = None, # for OpenAI service tier pricing ### DATA RESIDENCY ### data_residency: str | None = None, # for OpenAI regional-processing uplift (e.g. "eu", "us") + ### VERTEX LOCATION ### + vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") ) -> float: """ Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm. @@ -1577,6 +1586,7 @@ def completion_cost( rerank_billed_units=rerank_billed_units, service_tier=service_tier, data_residency=data_residency, + vertex_location=vertex_location, response=completion_response, request_model=request_model_for_cost, ) @@ -1664,6 +1674,7 @@ def completion_cost( usage=cost_per_token_usage_object, service_tier=service_tier, data_residency=data_residency, + vertex_location=vertex_location, ) _reasoning_cost = _token_type_breakdown.reasoning_cost _cache_read_cost = _token_type_breakdown.cache_read_cost @@ -1686,6 +1697,7 @@ def completion_cost( reasoning_cost=_reasoning_cost, service_tier=service_tier, data_residency=data_residency, + vertex_location=vertex_location, ) return _final_cost @@ -1765,6 +1777,8 @@ def response_cost_calculator( service_tier: str | None = None, # for OpenAI service tier pricing ### DATA RESIDENCY ### data_residency: str | None = None, # for OpenAI regional-processing uplift (e.g. "eu", "us") + ### VERTEX LOCATION ### + vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") ) -> float: """ Returns @@ -1797,6 +1811,7 @@ def response_cost_calculator( litellm_logging_obj=litellm_logging_obj, service_tier=service_tier, data_residency=data_residency, + vertex_location=vertex_location, ) return response_cost except Exception as e: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 946110abf9e..e97abcf0af6 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -13,7 +13,7 @@ import traceback from collections.abc import Callable, Mapping, Sequence from datetime import datetime as dt_object from functools import lru_cache -from types import TracebackType +from types import MappingProxyType, TracebackType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast from httpx import Response @@ -372,6 +372,24 @@ def _published_pricing(deployment_model: str | None) -> ModelInfo | None: return None +def _resolve_vertex_location_for_cost( + custom_llm_provider: str | None, + litellm_params: Mapping[str, object] | None, + model: str, +) -> str | None: + """ + The Vertex AI location a request was served from, resolved the same way + dispatch resolves it, so regional deployments price with the + regional-endpoint uplift. None for non-Vertex providers. + """ + if custom_llm_provider is None or not custom_llm_provider.startswith("vertex_ai"): + return None + from litellm.llms.vertex_ai.vertex_llm_base import VertexBase + + configured_location: Final = VertexBase.safe_get_vertex_ai_location(litellm_params or MappingProxyType({})) + return VertexBase.get_vertex_region(configured_location, model) + + class Logging(LiteLLMLoggingBaseClass): global \ supabaseClient, \ @@ -1432,6 +1450,7 @@ class Logging(LiteLLMLoggingBaseClass): reasoning_cost: float | None = None, service_tier: str | None = None, data_residency: str | None = None, + vertex_location: str | None = None, ) -> None: """ Helper method to store cost breakdown in the logging object. @@ -1450,6 +1469,7 @@ class Logging(LiteLLMLoggingBaseClass): margin_total_amount: Total margin added in USD service_tier: Tier the costs above were priced on, already resolved data_residency: Region uplift the costs above were priced on, already resolved + vertex_location: Vertex AI location the costs above were priced on, already resolved """ self.cost_breakdown = CostBreakdown( @@ -1459,6 +1479,7 @@ class Logging(LiteLLMLoggingBaseClass): tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar, service_tier=service_tier, data_residency=data_residency, + vertex_location=vertex_location, ) if cache_read_cost is not None and cache_read_cost > 0: self.cost_breakdown["cache_read_cost"] = cache_read_cost @@ -1574,6 +1595,11 @@ class Logging(LiteLLMLoggingBaseClass): if hasattr(self, "litellm_params") and self.litellm_params else None ), + "vertex_location": _resolve_vertex_location_for_cost( + custom_llm_provider=self.model_call_details.get("custom_llm_provider", None), + litellm_params=(self.litellm_params if hasattr(self, "litellm_params") else None), + model=litellm_model_name or self.model, + ), } except Exception as e: # error creating kwargs for cost calculation debug_info = StandardLoggingModelCostFailureDebugInformation( diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index f73c4942a1c..dec35d16ea0 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -757,6 +757,33 @@ def _get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | return 1.0 +def get_vertex_regional_endpoint_uplift(model_info: ModelInfo, vertex_location: str | None) -> float: + """ + Resolve the per-model uplift multiplier for Vertex AI non-global (regional and + multi-region) endpoints. + + Google prices every non-global endpoint at a flat premium over the global + endpoint (e.g. 1.10 = +10%) on all token types for the models that carry + regional pricing. The multiplier is stored on the model entry as + ``regional_endpoint_uplift_multiplier``. + + Returns 1.0 (no uplift) when ``vertex_location`` is ``None`` or ``"global"``, + or when the model has no multiplier configured. + """ + if vertex_location is None or vertex_location.lower() == "global": + return 1.0 + multiplier: Final = model_info.get("regional_endpoint_uplift_multiplier") + if multiplier is None: + return 1.0 + try: + return float(cast(float, multiplier)) + except (TypeError, ValueError): + verbose_logger.exception( + "Invalid regional_endpoint_uplift_multiplier for model; defaulting to 1.0", + ) + return 1.0 + + def get_provider_specific_geo_multiplier(model_info: ModelInfo, usage: Usage) -> float: """ Resolve the provider-specific regional pricing multiplier for the geo the @@ -798,6 +825,7 @@ def generic_cost_per_token( service_tier: str | None = None, data_residency: str | None = None, model_info: ModelInfo | None = None, + vertex_location: str | None = None, ) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -809,6 +837,9 @@ def generic_cost_per_token( - usage: LiteLLM Usage block, containing anthropic caching information - data_residency: optional OpenAI data-residency region (e.g. "eu", "us"), used to apply the per-model regional-processing uplift multiplier. + - vertex_location: optional Vertex AI location the request was served from + (e.g. "us-east5", "global"), used to apply the per-model + regional-endpoint uplift multiplier when non-global. Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd @@ -968,6 +999,14 @@ def generic_cost_per_token( prompt_cost *= uplift completion_cost *= uplift + ## VERTEX REGIONAL-ENDPOINT UPLIFT + # Applied as a flat multiplier across all token costs for the request + # when the Vertex AI endpoint serving it is non-global. + vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) + if vertex_uplift != 1.0: + prompt_cost *= vertex_uplift + completion_cost *= vertex_uplift + return prompt_cost, completion_cost @@ -988,6 +1027,7 @@ def get_token_type_cost_breakdown( usage: Usage, service_tier: str | None = None, data_residency: str | None = None, + vertex_location: str | None = None, ) -> TokenTypeCostBreakdown: """ Provider-agnostic cost of reasoning and cache tokens, derived from the usage @@ -1069,6 +1109,14 @@ def get_token_type_cost_breakdown( cache_read_cost *= uplift cache_creation_cost *= uplift + # Same flat uplift for Vertex AI non-global endpoints, keeping per-type + # costs reconciled with the totals for regional Vertex deployments. + vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) + if vertex_uplift != 1.0: + reasoning_cost *= vertex_uplift + cache_read_cost *= vertex_uplift + cache_creation_cost *= vertex_uplift + # Mirror the provider-specific geo uplift (e.g. Anthropic us: 1.1) the totals # apply, so cache and reasoning line items stay reconciled with them. geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=model_info, usage=usage) diff --git a/litellm/llms/vertex_ai/cost_calculator.py b/litellm/llms/vertex_ai/cost_calculator.py index 86a5bb207ec..a9f5d77350c 100644 --- a/litellm/llms/vertex_ai/cost_calculator.py +++ b/litellm/llms/vertex_ai/cost_calculator.py @@ -7,6 +7,7 @@ from litellm import verbose_logger from litellm.litellm_core_utils.llm_cost_calc.utils import ( _is_above_128k, generic_cost_per_token, + get_vertex_regional_endpoint_uplift, ) from litellm.types.utils import ModelInfo, Usage @@ -63,6 +64,7 @@ def cost_per_character( usage: Usage, prompt_characters: float | None = None, completion_characters: float | None = None, + vertex_location: str | None = None, ) -> tuple[float, float]: """ Calculates the cost per character for a given VertexAI model, input messages, and response object. @@ -72,6 +74,8 @@ def cost_per_character( - custom_llm_provider: str, "vertex_ai-*" - prompt_characters: float, the number of input characters - completion_characters: float, the number of output characters + - vertex_location: the Vertex AI location serving the request; non-global + locations apply the model's regional-endpoint uplift multiplier Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd @@ -79,8 +83,6 @@ def cost_per_character( Raises: Exception if model requires >128k pricing, but model cost not mapped """ - model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) - ## GET MODEL INFO model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) @@ -162,7 +164,10 @@ def cost_per_character( usage=usage, ) - return prompt_cost, completion_cost + # Applied once here; the cost_per_token fallbacks above are called without + # vertex_location so the uplift can never compound. + vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) + return prompt_cost * vertex_uplift, completion_cost * vertex_uplift def _handle_128k_pricing( @@ -196,6 +201,7 @@ def cost_per_token( custom_llm_provider: str, usage: Usage, service_tier: str | None = None, + vertex_location: str | None = None, ) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -207,6 +213,8 @@ def cost_per_token( - completion_tokens: float, the number of output tokens - service_tier: optional tier derived from Gemini trafficType ("priority" for ON_DEMAND_PRIORITY, "flex" for FLEX/batch). + - vertex_location: the Vertex AI location serving the request; non-global + locations apply the model's regional-endpoint uplift multiplier Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd @@ -222,14 +230,17 @@ def cost_per_token( input_cost_per_token_above_128k_tokens: Final = model_info.get("input_cost_per_token_above_128k_tokens") output_cost_per_token_above_128k_tokens: Final = model_info.get("output_cost_per_token_above_128k_tokens") if input_cost_per_token_above_128k_tokens is not None or output_cost_per_token_above_128k_tokens is not None: - return _handle_128k_pricing( + prompt_cost_128k, completion_cost_128k = _handle_128k_pricing( model_info=model_info, usage=usage, ) + vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) + return prompt_cost_128k * vertex_uplift, completion_cost_128k * vertex_uplift return generic_cost_per_token( model=model, custom_llm_provider=custom_llm_provider, usage=usage, service_tier=service_tier, + vertex_location=vertex_location, ) diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 445e34966a9..0b3c003a60a 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -8,6 +8,7 @@ import asyncio import json import os import threading +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final, Literal from urllib.parse import urlparse @@ -68,7 +69,8 @@ class VertexBase: # re-acquire it without deadlocking the current thread. self._sync_refresh_lock = threading.RLock() - def get_vertex_region(self, vertex_region: str | None, model: str) -> str: + @staticmethod + def get_vertex_region(vertex_region: str | None, model: str) -> str: import litellm # Try to get supported_regions directly from model_cost @@ -1191,7 +1193,7 @@ class VertexBase: ) @staticmethod - def safe_get_vertex_ai_location(litellm_params: dict) -> str | None: + def safe_get_vertex_ai_location(litellm_params: Mapping[str, object]) -> str | None: """ Safely get Vertex AI location without mutating the litellm_params dict. @@ -1204,10 +1206,7 @@ class VertexBase: Returns: Vertex AI location/region or None """ - return ( - litellm_params.get("vertex_location") - or litellm_params.get("vertex_ai_location") - or litellm.vertex_location - or get_secret_str("VERTEXAI_LOCATION") - or get_secret_str("VERTEX_LOCATION") - ) + for configured in (litellm_params.get("vertex_location"), litellm_params.get("vertex_ai_location")): + if isinstance(configured, str) and configured: + return configured + return litellm.vertex_location or get_secret_str("VERTEXAI_LOCATION") or get_secret_str("VERTEX_LOCATION") diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 07f9027313b..52f1763a5ba 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -19624,6 +19624,7 @@ "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -19679,6 +19680,7 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -19733,6 +19735,7 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -38685,6 +38688,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5", "supports_assistant_prefill": true, "supports_function_calling": true, @@ -38708,6 +38712,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5", "supports_assistant_prefill": true, "supports_function_calling": true, @@ -38923,6 +38928,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38951,6 +38957,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38970,6 +38977,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39000,6 +39008,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6@default": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39030,6 +39039,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-7": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39061,6 +39071,7 @@ "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-opus-4-7@default": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39092,6 +39103,7 @@ "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-fable-5": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -39123,6 +39135,7 @@ "supports_max_reasoning_effort": true }, "vertex_ai/claude-fable-5@default": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -39154,6 +39167,7 @@ "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-5": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39186,6 +39200,7 @@ "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-opus-5@default": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39218,6 +39233,7 @@ "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-opus-4-8": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39250,6 +39266,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-8@default": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39298,6 +39315,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -39310,6 +39328,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-5": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, @@ -39342,6 +39361,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-6": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -39388,6 +39408,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -39794,6 +39815,7 @@ "output_cost_per_token_batches": 7.5e-07, "output_cost_per_token_flex": 7.5e-07, "output_cost_per_token_priority": 2.7e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", "supported_endpoints": [ "/v1/chat/completions", @@ -39849,6 +39871,7 @@ "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_flex": 1.25e-06, "output_cost_per_token_priority": 4.5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", "supported_endpoints": [ "/v1/chat/completions", @@ -47014,6 +47037,7 @@ } }, "vertex_ai/claude-sonnet-5@default": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, @@ -47046,6 +47070,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-6@default": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 448723ab3bc..997180efdde 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -130,6 +130,7 @@ class PricingBasis(NamedTuple): service_tier: str | None = None data_residency: str | None = None + vertex_location: str | None = None _STANDARD_RATES: Final = PricingBasis() @@ -141,8 +142,8 @@ def _pricing_basis(cost_breakdown: Mapping[str, object] | None) -> PricingBasis: Rows written before this field shipped carry neither key, and there is no backfill: they price at standard rates, which is what they already did. - Both values survive a JSON round trip on the way here, so neither is guaranteed to be - a string. `generic_cost_per_token` calls `.lower()` on both without a type check, and + These values survive a JSON round trip on the way here, so none is guaranteed to be + a string. `generic_cost_per_token` calls `.lower()` on them without a type check, and the resulting `AttributeError` would be swallowed into a silent zero by the caller's `except`, so anything that is not a string is dropped here instead. """ @@ -150,9 +151,11 @@ def _pricing_basis(cost_breakdown: Mapping[str, object] | None) -> PricingBasis: return _STANDARD_RATES service_tier: Final = cost_breakdown.get("service_tier") data_residency: Final = cost_breakdown.get("data_residency") + vertex_location: Final = cost_breakdown.get("vertex_location") return PricingBasis( service_tier=service_tier if isinstance(service_tier, str) else None, data_residency=data_residency if isinstance(data_residency, str) else None, + vertex_location=vertex_location if isinstance(vertex_location, str) else None, ) @@ -193,6 +196,7 @@ def _cost_of_usage( service_tier=basis.service_tier, data_residency=basis.data_residency, model_info=model_info, + vertex_location=basis.vertex_location, ) except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings verbose_proxy_logger.debug( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 07005d7f9ad..5f629eb129f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -248,6 +248,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): regional_processing_uplift_multiplier_us: ( float | None ) # OpenAI US data-residency uplift multiplier applied to all token costs (e.g. 1.10 = +10%) + regional_endpoint_uplift_multiplier: ReadOnly[ + float | None + ] # Vertex AI non-global (regional) endpoint uplift multiplier applied to all token costs (e.g. 1.10 = +10%) output_cost_per_character: float | None # only for vertex ai models output_cost_per_audio_token: float | None output_cost_per_token_above_128k_tokens: float | None # only for vertex ai models @@ -3113,16 +3116,17 @@ class CostBreakdown(TypedDict, total=False): """ Detailed cost breakdown for a request. - ``service_tier`` and ``data_residency`` record the pricing basis the cost was - computed on, not what the caller asked for. A consumer that has to price a - counterfactual against this request (what another model would have charged for - it) needs the same basis to compare like with like, and re-deriving it from the - request is not possible after the fact: the tier the biller used comes from - ``optional_params``, which no log record carries. + ``service_tier``, ``data_residency``, and ``vertex_location`` record the pricing + basis the cost was computed on, not what the caller asked for. A consumer that has + to price a counterfactual against this request (what another model would have + charged for it) needs the same basis to compare like with like, and re-deriving it + from the request is not possible after the fact: the tier the biller used comes + from ``optional_params``, which no log record carries. """ service_tier: str | None data_residency: str | None + vertex_location: ReadOnly[str | None] input_cost: float # Cost of raw (non-cached) input tokens only cache_read_cost: float # Cost of cache-read tokens (discounted rate) cache_creation_cost: float # Cost of cache-write tokens (premium rate) @@ -3388,6 +3392,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): annotation_cost_per_page: float | None = None regional_processing_uplift_multiplier_eu: float | None = None regional_processing_uplift_multiplier_us: float | None = None + regional_endpoint_uplift_multiplier: float | None = None @classmethod def strip_custom_pricing_fields(cls, model_info: dict[str, Any]) -> dict[str, Any]: diff --git a/litellm/utils.py b/litellm/utils.py index a7b70c4129a..8be2c98fc68 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5662,6 +5662,7 @@ def _get_model_info_helper( regional_processing_uplift_multiplier_us=_model_info.get( "regional_processing_uplift_multiplier_us", None ), + regional_endpoint_uplift_multiplier=_model_info.get("regional_endpoint_uplift_multiplier", None), output_cost_per_audio_token=_model_info.get("output_cost_per_audio_token", None), output_cost_per_character=_model_info.get("output_cost_per_character", None), output_cost_per_reasoning_token=_model_info.get("output_cost_per_reasoning_token", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 07f9027313b..52f1763a5ba 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -19624,6 +19624,7 @@ "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -19679,6 +19680,7 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -19733,6 +19735,7 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -38685,6 +38688,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5", "supports_assistant_prefill": true, "supports_function_calling": true, @@ -38708,6 +38712,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5", "supports_assistant_prefill": true, "supports_function_calling": true, @@ -38923,6 +38928,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38951,6 +38957,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38970,6 +38977,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39000,6 +39008,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6@default": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39030,6 +39039,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-7": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39061,6 +39071,7 @@ "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-opus-4-7@default": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39092,6 +39103,7 @@ "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-fable-5": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -39123,6 +39135,7 @@ "supports_max_reasoning_effort": true }, "vertex_ai/claude-fable-5@default": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -39154,6 +39167,7 @@ "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-5": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39186,6 +39200,7 @@ "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-opus-5@default": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39218,6 +39233,7 @@ "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-opus-4-8": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39250,6 +39266,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-8@default": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39298,6 +39315,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -39310,6 +39328,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-5": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, @@ -39342,6 +39361,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-6": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -39388,6 +39408,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -39794,6 +39815,7 @@ "output_cost_per_token_batches": 7.5e-07, "output_cost_per_token_flex": 7.5e-07, "output_cost_per_token_priority": 2.7e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", "supported_endpoints": [ "/v1/chat/completions", @@ -39849,6 +39871,7 @@ "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_flex": 1.25e-06, "output_cost_per_token_priority": 4.5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", "supported_endpoints": [ "/v1/chat/completions", @@ -47014,6 +47037,7 @@ } }, "vertex_ai/claude-sonnet-5@default": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, @@ -47046,6 +47070,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-6@default": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index cd02fde595f..82854a3b717 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -514,6 +514,11 @@ "type": "object", "description": "Provider-internal routing hints (e.g. bedrock_invocation_schema)." }, + "regional_endpoint_uplift_multiplier": { + "type": "number", + "minimum": 1, + "description": "Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%)." + }, "regional_processing_uplift_multiplier_eu": { "type": "number", "minimum": 1, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 1826f56d667..06be96fefdf 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2324,6 +2324,87 @@ def test_data_residency_composes_with_service_tier(_local_model_cost_map): assert priority_eu_total == pytest.approx(priority_base_total * 1.10, rel=1e-9) +@pytest.mark.parametrize("model", ["gemini-3.5-flash", "claude-haiku-4-5@20251001"]) +@pytest.mark.parametrize("vertex_location", ["us-central1", "us-east5", "europe-west1", "asia-southeast1"]) +def test_vertex_regional_location_applies_uplift(vertex_location, model, _local_model_cost_map): + """Google bills every non-global Vertex endpoint at 1.1x the global rate for GA + Gemini 3+ and regional-pricing Claude models, so a request served from a regional + location must cost 1.1x what the same usage costs on the global endpoint.""" + from litellm.types.utils import Usage + + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + base = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="vertex_ai") + regional = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider="vertex_ai", + vertex_location=vertex_location, + ) + + base_total = base[0] + base[1] + regional_total = regional[0] + regional[1] + + assert base_total > 0 + assert regional_total == pytest.approx(base_total * 1.10, rel=1e-9) + assert regional[0] == pytest.approx(base[0] * 1.10, rel=1e-9) + assert regional[1] == pytest.approx(base[1] * 1.10, rel=1e-9) + + +@pytest.mark.parametrize("vertex_location", [None, "global", "GLOBAL"]) +def test_vertex_global_or_absent_location_no_uplift(vertex_location, _local_model_cost_map): + """The global endpoint prices at the base rate, whatever the casing, and an + unresolved location must never uplift.""" + from litellm.types.utils import Usage + + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + base = generic_cost_per_token( + model="claude-haiku-4-5@20251001", usage=usage, custom_llm_provider="vertex_ai" + ) + located = generic_cost_per_token( + model="claude-haiku-4-5@20251001", + usage=usage, + custom_llm_provider="vertex_ai", + vertex_location=vertex_location, + ) + + assert base == located + + +@pytest.mark.parametrize("model", ["claude-opus-4-1", "gemini-2.0-flash-001"]) +def test_vertex_location_no_uplift_for_uniformly_priced_model(model, _local_model_cost_map): + """Models Google prices uniformly across endpoints (Gemini 2.x, Claude Opus 4.1 + and older) carry no multiplier and must not move with the location.""" + from litellm.types.utils import Usage + + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + base = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="vertex_ai") + regional = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider="vertex_ai", + vertex_location="us-east5", + ) + + assert base == regional, f"{model} should not have a regional-endpoint uplift" + + +def test_vertex_uplift_invalid_multiplier_defaults_to_one(): + """A malformed multiplier in the cost map degrades to base pricing, never raises.""" + from litellm.litellm_core_utils.llm_cost_calc.utils import ( + get_vertex_regional_endpoint_uplift, + ) + + assert ( + get_vertex_regional_endpoint_uplift( + {"regional_endpoint_uplift_multiplier": "not-a-number"}, "us-east5" + ) + == 1.0 + ) + + def test_priority_service_tier_above_threshold_uses_priority_tier_rates_for_cached_tokens( _local_model_cost_map, ): @@ -2877,6 +2958,57 @@ def test_token_type_cost_breakdown_applies_regional_uplift(): assert text_input_cost + eu.cache_read_cost == pytest.approx(prompt_cost) +def test_token_type_cost_breakdown_applies_vertex_regional_uplift(): + """ + Non-global Vertex endpoints apply a flat 1.1x uplift to every token cost. The + per-type breakdown must apply the same uplift via vertex_location so it stays + reconciled with the uplifted input_cost/output_cost totals, instead of being + logged at the global rate. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "claude-haiku-4-5@20251001" + custom_llm_provider = "vertex_ai" + usage = Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=400, text_tokens=600 + ), + ) + + model_info = litellm.get_model_info( + model=model, custom_llm_provider=custom_llm_provider + ) + uplift = model_info["regional_endpoint_uplift_multiplier"] + assert uplift > 1.0 + + base = get_token_type_cost_breakdown( + model=model, custom_llm_provider=custom_llm_provider, usage=usage + ) + regional = get_token_type_cost_breakdown( + model=model, + custom_llm_provider=custom_llm_provider, + usage=usage, + vertex_location="us-east5", + ) + + assert base.cache_read_cost > 0 + assert regional.cache_read_cost == pytest.approx(base.cache_read_cost * uplift) + + # The uplifted breakdown must still reconcile with the uplifted totals. + prompt_cost, _completion_cost = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider=custom_llm_provider, + vertex_location="us-east5", + ) + text_input_cost = 600 * model_info["input_cost_per_token"] * uplift + assert text_input_cost + regional.cache_read_cost == pytest.approx(prompt_cost) + + def test_token_type_cost_breakdown_applies_anthropic_geo_multiplier(monkeypatch): """ Anthropic's regional (geo) uplift lives in provider_specific_entry and is diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 54016470f8b..01e5ec26b1e 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -4958,3 +4958,80 @@ def test_pre_call_redacts_and_masks_raw_request(logging_obj): raw_api_base = logging_obj.model_call_details["raw_request_typed_dict"]["raw_request_api_base"] assert _GEMINI_KEY not in raw_api_base assert "key=*****" in raw_api_base + + +def test_resolve_vertex_location_for_cost(): + """Vertex requests resolve the serving location the way dispatch does; other providers get None.""" + from litellm.litellm_core_utils.litellm_logging import ( + _resolve_vertex_location_for_cost, + ) + + assert _resolve_vertex_location_for_cost("openai", {"vertex_location": "us-east5"}, "gpt-4o") is None + assert _resolve_vertex_location_for_cost(None, {}, "gemini-3.5-flash") is None + assert ( + _resolve_vertex_location_for_cost("vertex_ai", {"vertex_location": "us-east5"}, "gemini-3.5-flash") + == "us-east5" + ) + assert ( + _resolve_vertex_location_for_cost("vertex_ai", {"vertex_location": "global"}, "gemini-3.5-flash") == "global" + ) + assert ( + _resolve_vertex_location_for_cost( + "vertex_ai_beta", {"vertex_ai_location": "europe-west1"}, "claude-haiku-4-5@20251001" + ) + == "europe-west1" + ) + + +def test_resolve_vertex_location_for_cost_default_region(monkeypatch): + """With no location configured anywhere, resolution lands on the dispatch default us-central1.""" + from litellm.litellm_core_utils.litellm_logging import ( + _resolve_vertex_location_for_cost, + ) + + monkeypatch.delenv("VERTEXAI_LOCATION", raising=False) + monkeypatch.delenv("VERTEX_LOCATION", raising=False) + monkeypatch.setattr(litellm, "vertex_location", None) + + assert _resolve_vertex_location_for_cost("vertex_ai", {}, "gemini-3.5-flash") == "us-central1" + assert _resolve_vertex_location_for_cost("vertex_ai", None, "gemini-3.5-flash") == "us-central1" + + +def test_set_cost_breakdown_stores_vertex_location(): + """vertex_location is recorded in the pricing basis, None for non-vertex requests.""" + from datetime import datetime + + logging_obj = LitellmLogging( + model="vertex_ai/claude-haiku-4-5@20251001", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="vertex-location-set", + function_id="f", + ) + logging_obj.set_cost_breakdown( + input_cost=0.001, + output_cost=0.002, + total_cost=0.003, + cost_for_built_in_tools_cost_usd_dollar=0.0, + vertex_location="us-east5", + ) + assert logging_obj.cost_breakdown["vertex_location"] == "us-east5" + + no_location = LitellmLogging( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="vertex-location-absent", + function_id="f", + ) + no_location.set_cost_breakdown( + input_cost=0.001, + output_cost=0.002, + total_cost=0.003, + cost_for_built_in_tools_cost_usd_dollar=0.0, + ) + assert no_location.cost_breakdown.get("vertex_location") is None diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 1435547c434..9006288bdae 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -841,6 +841,44 @@ def test_the_baseline_is_priced_on_the_basis_the_request_was_billed_at(basis, ex assert reported == pytest.approx(expected_multiplier * baseline - served) +def test_the_baseline_is_priced_on_the_vertex_location_the_request_was_billed_at(monkeypatch): + """A request served from a regional Vertex endpoint was billed with the + regional-endpoint uplift, so the counterfactual single-model operator would + have paid it too. The served model carries no uplift field, so only the + baseline moves with the recorded location.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + gemini = litellm.get_model_info("gemini-3.5-flash", "vertex_ai") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + assert gemini.get("regional_endpoint_uplift_multiplier") == 1.1 + assert haiku.get("regional_endpoint_uplift_multiplier") is None, "served model must not move with the basis" + + usage = _usage(fresh=20_000, cached=0, written=0, out=1_000) + served = 20_000 * haiku["input_cost_per_token"] + 1_000 * haiku["output_cost_per_token"] + baseline = 20_000 * gemini["input_cost_per_token"] + 1_000 * gemini["output_cost_per_token"] + + regional = compute_autorouter_savings( + baseline_model="vertex_ai/gemini-3.5-flash", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + conversation_continuing=False, + cost_breakdown=_breakdown(served, vertex_location="us-east5"), + ) + global_endpoint = compute_autorouter_savings( + baseline_model="vertex_ai/gemini-3.5-flash", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + conversation_continuing=False, + cost_breakdown=_breakdown(served, vertex_location="global"), + ) + + assert regional == pytest.approx(1.1 * baseline - served) + assert global_endpoint == pytest.approx(baseline - served) + + def test_a_baseline_recorded_on_the_decision_turns_the_driver_on(): """An operator who configures nothing still sees the driver work.""" result = compute_savings_spend( diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 75c90d793fe..6deaf5479e0 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1742,6 +1742,75 @@ def test_azure_ai_cache_cost_calculation(): ), f"Output cost mismatch: got {output_cost}, expected {expected_output_cost}" +def test_vertex_regional_deployment_costs_uplift_over_global(monkeypatch): + """ + Regression for https://github.com/BerriAI/litellm/issues/34393: two Vertex + deployments differing only in vertex_location must not price identically. + Google bills non-global endpoints at 1.1x for regional-pricing models, so the + regional request costs 1.1x the global one for the exact same usage, through + both vertex cost routes (Claude via cost_per_token, Gemini via + cost_per_character's token fallback). + """ + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + usage = Usage(prompt_tokens=15, completion_tokens=5, total_tokens=20) + for model in ("claude-haiku-4-5@20251001", "gemini-3.5-flash"): + global_prompt, global_completion = cost_per_token( + model=model, + custom_llm_provider="vertex_ai", + usage_object=usage, + vertex_location="global", + ) + regional_prompt, regional_completion = cost_per_token( + model=model, + custom_llm_provider="vertex_ai", + usage_object=usage, + vertex_location="us-east5", + ) + global_total = global_prompt + global_completion + regional_total = regional_prompt + regional_completion + assert global_total > 0 + assert regional_total == pytest.approx(global_total * 1.10, rel=1e-9), ( + f"{model}: regional Vertex request must cost 1.1x the global one" + ) + + +def test_vertex_uplift_composes_with_above_128k_pricing(monkeypatch): + """The regional-endpoint uplift multiplies whatever rate the request priced at, + including the above-128k dynamic rates, so a synthetic model carrying both keys + prices regional above-128k usage at 1.1x the above-128k rate.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.model_cost["vertex_ai/fake-regional-128k-model"] = { + "litellm_provider": "vertex_ai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "input_cost_per_token_above_128k_tokens": 2e-06, + "output_cost_per_token_above_128k_tokens": 4e-06, + "regional_endpoint_uplift_multiplier": 1.1, + } + + usage = Usage(prompt_tokens=200_000, completion_tokens=10, total_tokens=200_010) + global_prompt, global_completion = cost_per_token( + model="fake-regional-128k-model", + custom_llm_provider="vertex_ai", + usage_object=usage, + vertex_location="global", + ) + regional_prompt, regional_completion = cost_per_token( + model="fake-regional-128k-model", + custom_llm_provider="vertex_ai", + usage_object=usage, + vertex_location="europe-west1", + ) + + assert global_prompt == pytest.approx(200_000 * 2e-06, rel=1e-9) + assert regional_prompt == pytest.approx(global_prompt * 1.10, rel=1e-9) + assert regional_completion == pytest.approx(global_completion * 1.10, rel=1e-9) + + def test_cost_discount_vertex_ai(): """ Test that cost discount is applied correctly for Vertex AI provider diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 6f77a621a9e..56ffbe9fde8 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 22809 + "limit": 22808 }, "LIT002": { "limit": 26878 diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 5794c60e97b..3cc7397f5bb 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27559,6 +27559,8 @@ export interface components { quality_router_default_model?: string | null; /** Region Name */ region_name?: string | null; + /** Regional Endpoint Uplift Multiplier */ + regional_endpoint_uplift_multiplier?: number | null; /** Regional Processing Uplift Multiplier Eu */ regional_processing_uplift_multiplier_eu?: number | null; /** Regional Processing Uplift Multiplier Us */ @@ -36689,6 +36691,8 @@ export interface components { quality_router_default_model?: string | null; /** Region Name */ region_name?: string | null; + /** Regional Endpoint Uplift Multiplier */ + regional_endpoint_uplift_multiplier?: number | null; /** Regional Processing Uplift Multiplier Eu */ regional_processing_uplift_multiplier_eu?: number | null; /** Regional Processing Uplift Multiplier Us */ From b477d0967a4bf1f4a869c70ddf5a740f1762ae86 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:34:38 -0700 Subject: [PATCH 04/24] fix(proxy): lift standard_logging_object onto request_data before the logging object is popped --- litellm/proxy/utils.py | 52 ++++++++++--------- tests/test_litellm/proxy/test_proxy_utils.py | 53 ++++++++++++++++++++ 2 files changed, 82 insertions(+), 23 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index a743526e975..58ac7882cb8 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -517,6 +517,34 @@ def _failure_usage_to_lift( return estimated_usage, 0.0 +_EMPTY_LIFT: Final = MappingProxyType({}) + + +def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str, object]: + """Failure-path callbacks run after ``litellm_logging_obj`` is popped from + request_data (it is not serialisable), so the caller merges these fields + onto request_data first: the first-handoff instant for preprocessing + latency, recovered or estimated usage for token counts, and the standard + logging object for deployment attribution on failed-request spend logs.""" + _logging_obj: Final = request_data.get("litellm_logging_obj") + if _logging_obj is None: + return _EMPTY_LIFT + _model_call_details: Final = getattr(_logging_obj, "model_call_details", {}) + _first_handoff: Final = _model_call_details.get("first_api_call_start_time") + _usage_to_lift: Final = _failure_usage_to_lift( + model_call_details=_model_call_details, + request_body=request_data, + dispatched=_first_handoff is not None, + ) + _entries: Final = ( + ("first_api_call_start_time", _first_handoff), + ("combined_usage_object", None if _usage_to_lift is None else _usage_to_lift[0]), + ("response_cost", None if _usage_to_lift is None else _usage_to_lift[1]), + ("standard_logging_object", _model_call_details.get("standard_logging_object")), + ) + return MappingProxyType({key: value for key, value in _entries if value is not None}) + + @dataclass(frozen=True) class _CallbackCapabilities: """Cached per-hook capability flags derived from ``litellm.callbacks``. @@ -2294,29 +2322,7 @@ class ProxyLogging: original_exception=original_exception, ) - # Lift the first-handoff instant onto request_data (top-level - # internal key, not metadata) so failure-path callbacks can still - # compute preprocessing latency after the logging object is popped. - _logging_obj: Final = request_data.get("litellm_logging_obj") - if _logging_obj is not None: - _model_call_details: Final = getattr(_logging_obj, "model_call_details", {}) - _first_handoff: Final = _model_call_details.get("first_api_call_start_time") - if _first_handoff is not None: - request_data["first_api_call_start_time"] = _first_handoff - - # Lift recovered partial-stream usage, or an estimated input-side - # usage for a dispatched failure, onto request_data so the - # failure-path spend callbacks (which run after the logging object - # is popped) record real token counts instead of zero. - _usage_to_lift: Final = _failure_usage_to_lift( - model_call_details=_model_call_details, - request_body=request_data, - dispatched=_first_handoff is not None, - ) - if _usage_to_lift is not None: - _lifted_usage, _lifted_cost = _usage_to_lift - request_data["combined_usage_object"] = _lifted_usage - request_data["response_cost"] = _lifted_cost + request_data.update(_failure_fields_to_lift(request_data)) # Remove before callbacks iterate — not serialisable request_data.pop("litellm_logging_obj", None) diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index d6cf0e30139..c80130b44da 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -478,6 +478,59 @@ class TestPostCallFailureHookLiftsRecoveredPartialSpend: assert "response_cost" not in request_data +class TestPostCallFailureHookLiftsStandardLoggingObject: + """Failure callbacks read standard_logging_object from request_data, but + post_call_failure_hook pops litellm_logging_obj before they run. The hook + must lift the logging obj's standard_logging_object onto request_data so + failed-request spend logs keep deployment attribution (LIT-5795). + """ + + async def _run(self, request_data): + from unittest.mock import AsyncMock, patch + + from litellm.proxy._types import UserAPIKeyAuth + + proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging_obj.alert_types = [] + with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()): + await proxy_logging_obj.post_call_failure_hook( + request_data=request_data, + original_exception=Exception("boom"), + user_api_key_dict=UserAPIKeyAuth(), + ) + + @pytest.mark.asyncio + async def test_lifts_standard_logging_object(self): + sl_object = {"model_id": "mid-123", "model_group": "group-x"} + logging_obj = MagicMock() + logging_obj.model_call_details = {"standard_logging_object": sl_object} + request_data = {"litellm_logging_obj": logging_obj, "metadata": {}} + await self._run(request_data) + assert request_data["standard_logging_object"] is sl_object + assert "litellm_logging_obj" not in request_data + + @pytest.mark.asyncio + async def test_logging_obj_value_overwrites_preexisting_key(self): + authoritative = {"model_id": "from-logging-obj"} + logging_obj = MagicMock() + logging_obj.model_call_details = {"standard_logging_object": authoritative} + request_data = { + "litellm_logging_obj": logging_obj, + "standard_logging_object": {"model_id": "client-injected"}, + "metadata": {}, + } + await self._run(request_data) + assert request_data["standard_logging_object"] is authoritative + + @pytest.mark.asyncio + async def test_no_standard_logging_object_is_noop(self): + logging_obj = MagicMock() + logging_obj.model_call_details = {} + request_data = {"litellm_logging_obj": logging_obj, "metadata": {}} + await self._run(request_data) + assert "standard_logging_object" not in request_data + + class TestPostCallFailureHookEstimatesDispatchedInputTokens: """A non-stream request that failed after dispatch (timeout, provider error) consumed provider-billed input tokens but recovered no usage. From 8494a4deee8bcc693de97e11ac23a0e6d062374b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:58:12 -0700 Subject: [PATCH 05/24] fix(vertex_ai): read the served location from optional_params when pricing proxy calls --- litellm/litellm_core_utils/litellm_logging.py | 14 ++- litellm/llms/vertex_ai/vertex_llm_base.py | 21 +++- .../test_litellm_logging.py | 108 +++++++++++++++--- 3 files changed, 119 insertions(+), 24 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e97abcf0af6..91a312b4f45 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -375,18 +375,29 @@ def _published_pricing(deployment_model: str | None) -> ModelInfo | None: def _resolve_vertex_location_for_cost( custom_llm_provider: str | None, litellm_params: Mapping[str, object] | None, + optional_params: Mapping[str, object] | None, model: str, ) -> str | None: """ The Vertex AI location a request was served from, resolved the same way dispatch resolves it, so regional deployments price with the regional-endpoint uplift. None for non-Vertex providers. + + Chat dispatch reads the location from request kwargs, which reach this + logging object through optional_params: on the proxy the logging object is + created before the router picks a deployment, so the deployment's location + never lands in litellm_params. """ if custom_llm_provider is None or not custom_llm_provider.startswith("vertex_ai"): return None from litellm.llms.vertex_ai.vertex_llm_base import VertexBase - configured_location: Final = VertexBase.safe_get_vertex_ai_location(litellm_params or MappingProxyType({})) + empty: Final[Mapping[str, object]] = MappingProxyType({}) + configured_location: Final = ( + VertexBase.explicit_vertex_ai_location(optional_params or empty) + or VertexBase.explicit_vertex_ai_location(litellm_params or empty) + or VertexBase.safe_get_vertex_ai_location(empty) + ) return VertexBase.get_vertex_region(configured_location, model) @@ -1598,6 +1609,7 @@ class Logging(LiteLLMLoggingBaseClass): "vertex_location": _resolve_vertex_location_for_cost( custom_llm_provider=self.model_call_details.get("custom_llm_provider", None), litellm_params=(self.litellm_params if hasattr(self, "litellm_params") else None), + optional_params=self.optional_params, model=litellm_model_name or self.model, ), } diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 0b3c003a60a..75098515deb 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -1192,6 +1192,17 @@ class VertexBase: or get_secret_str("VERTEXAI_CREDENTIALS") ) + @staticmethod + def explicit_vertex_ai_location(params: Mapping[str, object]) -> str | None: + """ + The location explicitly configured in the given params, without any + module-level or environment fallback. None when not configured. + """ + for configured in (params.get("vertex_location"), params.get("vertex_ai_location")): + if isinstance(configured, str) and configured: + return configured + return None + @staticmethod def safe_get_vertex_ai_location(litellm_params: Mapping[str, object]) -> str | None: """ @@ -1206,7 +1217,9 @@ class VertexBase: Returns: Vertex AI location/region or None """ - for configured in (litellm_params.get("vertex_location"), litellm_params.get("vertex_ai_location")): - if isinstance(configured, str) and configured: - return configured - return litellm.vertex_location or get_secret_str("VERTEXAI_LOCATION") or get_secret_str("VERTEX_LOCATION") + return ( + VertexBase.explicit_vertex_ai_location(litellm_params) + or litellm.vertex_location + or get_secret_str("VERTEXAI_LOCATION") + or get_secret_str("VERTEX_LOCATION") + ) diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 01e5ec26b1e..c2d73ea467d 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -4960,41 +4960,111 @@ def test_pre_call_redacts_and_masks_raw_request(logging_obj): assert "key=*****" in raw_api_base -def test_resolve_vertex_location_for_cost(): - """Vertex requests resolve the serving location the way dispatch does; other providers get None.""" +def _resolve(custom_llm_provider, litellm_params, optional_params, model): from litellm.litellm_core_utils.litellm_logging import ( _resolve_vertex_location_for_cost, ) - assert _resolve_vertex_location_for_cost("openai", {"vertex_location": "us-east5"}, "gpt-4o") is None - assert _resolve_vertex_location_for_cost(None, {}, "gemini-3.5-flash") is None - assert ( - _resolve_vertex_location_for_cost("vertex_ai", {"vertex_location": "us-east5"}, "gemini-3.5-flash") - == "us-east5" + return _resolve_vertex_location_for_cost( + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + optional_params=optional_params, + model=model, ) + + +def test_resolve_vertex_location_for_cost(): + """Vertex requests resolve the serving location the way dispatch does; other providers get None.""" + assert _resolve("openai", {"vertex_location": "us-east5"}, None, "gpt-4o") is None + assert _resolve(None, {}, None, "gemini-3.5-flash") is None + assert _resolve("vertex_ai", {"vertex_location": "us-east5"}, None, "gemini-3.5-flash") == "us-east5" + assert _resolve("vertex_ai", {"vertex_location": "global"}, None, "gemini-3.5-flash") == "global" assert ( - _resolve_vertex_location_for_cost("vertex_ai", {"vertex_location": "global"}, "gemini-3.5-flash") == "global" - ) - assert ( - _resolve_vertex_location_for_cost( - "vertex_ai_beta", {"vertex_ai_location": "europe-west1"}, "claude-haiku-4-5@20251001" - ) + _resolve("vertex_ai_beta", {"vertex_ai_location": "europe-west1"}, None, "claude-haiku-4-5@20251001") == "europe-west1" ) +def test_resolve_vertex_location_for_cost_reads_optional_params(monkeypatch): + """ + On the proxy the logging object predates deployment selection, so the deployment's + configured location only reaches it through optional_params. A configured global + location must beat the environment fallback, or every proxy call gets the regional uplift. + """ + monkeypatch.setenv("VERTEXAI_LOCATION", "us-east5") + monkeypatch.setattr(litellm, "vertex_location", None) + + assert _resolve("vertex_ai", {}, {"vertex_location": "global"}, "gemini-3.5-flash") == "global" + assert _resolve("vertex_ai", None, {"vertex_location": "europe-west1"}, "gemini-3.5-flash") == "europe-west1" + assert ( + _resolve( + "vertex_ai", + {"vertex_location": "us-east5"}, + {"vertex_location": "global"}, + "gemini-3.5-flash", + ) + == "global" + ) + assert _resolve("vertex_ai", {"vertex_location": "global"}, {}, "gemini-3.5-flash") == "global" + assert _resolve("vertex_ai", {}, {}, "gemini-3.5-flash") == "us-east5" + + def test_resolve_vertex_location_for_cost_default_region(monkeypatch): """With no location configured anywhere, resolution lands on the dispatch default us-central1.""" - from litellm.litellm_core_utils.litellm_logging import ( - _resolve_vertex_location_for_cost, - ) - monkeypatch.delenv("VERTEXAI_LOCATION", raising=False) monkeypatch.delenv("VERTEX_LOCATION", raising=False) monkeypatch.setattr(litellm, "vertex_location", None) - assert _resolve_vertex_location_for_cost("vertex_ai", {}, "gemini-3.5-flash") == "us-central1" - assert _resolve_vertex_location_for_cost("vertex_ai", None, "gemini-3.5-flash") == "us-central1" + assert _resolve("vertex_ai", {}, None, "gemini-3.5-flash") == "us-central1" + assert _resolve("vertex_ai", None, None, "gemini-3.5-flash") == "us-central1" + + +def test_response_cost_calculator_prices_proxy_vertex_calls_on_the_configured_location(monkeypatch): + """ + Proxy-shaped logging objects (created before the router picks a deployment) carry the + deployment's vertex_location only in optional_params. A global deployment must price at + base rates even when the environment points at a regional location, and a regional one + must price with the uplift. + """ + from datetime import datetime + + from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map + + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url="")) + monkeypatch.setenv("VERTEXAI_LOCATION", "us-east5") + monkeypatch.setattr(litellm, "vertex_location", None) + + def cost_at(location): + logging_obj = LitellmLogging( + model="gemini-3.5-flash", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id=f"vertex-loc-{location}", + function_id="f", + ) + logging_obj.update_environment_variables( + model="gemini-3.5-flash", + user="", + optional_params={"vertex_location": location}, + litellm_params={"api_base": ""}, + custom_llm_provider="vertex_ai", + ) + response = ModelResponse( + id="resp-1", + model="gemini-3.5-flash", + choices=[{"message": {"role": "assistant", "content": "hello"}, "index": 0, "finish_reason": "stop"}], + usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + ) + return logging_obj._response_cost_calculator(result=response) + + info = litellm.model_cost["vertex_ai/gemini-3.5-flash"] + expected_global = 10 * info["input_cost_per_token"] + 5 * info["output_cost_per_token"] + + assert cost_at("global") == pytest.approx(expected_global) + assert cost_at("us-east5") == pytest.approx(info["regional_endpoint_uplift_multiplier"] * expected_global) def test_set_cost_breakdown_stores_vertex_location(): From dce207add43a6ece53a5a0a12458dd7edf03ed3e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:19:26 -0700 Subject: [PATCH 06/24] fix(proxy): strip client standard_logging_object and zero-fill unknown recovered cost on the failure path Auth and pass-through failures reach post_call_failure_hook with the raw request body unstripped, so a client-supplied standard_logging_object could feed the new attribution fallback when the logging object carries none. Pop the key before the lift so only the logging object may supply it. Also coalesce a None recovered cost to 0.0 so the lift always overwrites any client-supplied response_cost, matching the merge base's clobber semantics. --- litellm/proxy/utils.py | 5 ++- tests/test_litellm/proxy/test_proxy_utils.py | 34 ++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 58ac7882cb8..acafc600b80 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -539,7 +539,7 @@ def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str, _entries: Final = ( ("first_api_call_start_time", _first_handoff), ("combined_usage_object", None if _usage_to_lift is None else _usage_to_lift[0]), - ("response_cost", None if _usage_to_lift is None else _usage_to_lift[1]), + ("response_cost", None if _usage_to_lift is None else (_usage_to_lift[1] or 0.0)), ("standard_logging_object", _model_call_details.get("standard_logging_object")), ) return MappingProxyType({key: value for key, value in _entries if value is not None}) @@ -2322,6 +2322,9 @@ class ProxyLogging: original_exception=original_exception, ) + # Auth and pass-through failures reach this hook with the raw request + # body unstripped, so only the logging object may supply this key. + request_data.pop("standard_logging_object", None) request_data.update(_failure_fields_to_lift(request_data)) # Remove before callbacks iterate — not serialisable diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index c80130b44da..71af44d6682 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -468,6 +468,23 @@ class TestPostCallFailureHookLiftsRecoveredPartialSpend: assert request_data["response_cost"] == 3.5e-05 assert "litellm_logging_obj" not in request_data + @pytest.mark.asyncio + async def test_recovered_usage_without_cost_clobbers_client_cost_with_zero(self): + from litellm.types.utils import Usage + + recovered_usage = Usage(prompt_tokens=30, completion_tokens=1, total_tokens=31) + logging_obj = MagicMock() + logging_obj.model_call_details = {"combined_usage_object": recovered_usage} + request_data = { + "litellm_logging_obj": logging_obj, + "response_cost": 999.0, + "metadata": {}, + } + await self._run(request_data) + + assert request_data["combined_usage_object"] is recovered_usage + assert request_data["response_cost"] == 0.0 + @pytest.mark.asyncio async def test_no_recovered_usage_is_noop(self): logging_obj = MagicMock() @@ -522,6 +539,23 @@ class TestPostCallFailureHookLiftsStandardLoggingObject: await self._run(request_data) assert request_data["standard_logging_object"] is authoritative + @pytest.mark.asyncio + async def test_client_supplied_key_is_stripped_when_logging_obj_supplies_none(self): + spoofed = {"model_id": "client-injected"} + request_data = {"standard_logging_object": spoofed, "metadata": {}} + await self._run(request_data) + assert "standard_logging_object" not in request_data + + logging_obj = MagicMock() + logging_obj.model_call_details = {} + request_data_with_obj = { + "litellm_logging_obj": logging_obj, + "standard_logging_object": spoofed, + "metadata": {}, + } + await self._run(request_data_with_obj) + assert "standard_logging_object" not in request_data_with_obj + @pytest.mark.asyncio async def test_no_standard_logging_object_is_noop(self): logging_obj = MagicMock() From 7b6f537855bf684a6b118ba1591c34475647b79a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:26:34 -0700 Subject: [PATCH 07/24] fix(anthropic_messages): price native /v1/messages vertex calls on the deployment location The proxy pre-creates the logging object before the router picks a deployment, and the native /v1/messages handler never copied the deployment's vertex_location into the logging params it updates, so cost resolution fell back to the environment or the default region and priced every call on this surface with the regional endpoint uplift. Copy the explicitly configured location from the request's litellm params, the same source dispatch builds the request URL from, and register the new regional_endpoint_uplift_multiplier field in the cost map schema test. --- litellm/llms/custom_httpx/llm_http_handler.py | 11 ++- .../custom_httpx/test_llm_http_handler.py | 75 +++++++++++++++++++ tests/test_litellm/test_utils.py | 1 + 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 3ee803646a9..e03942100b0 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -5,7 +5,7 @@ import ssl from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence from contextlib import asynccontextmanager from functools import lru_cache -from types import ModuleType +from types import MappingProxyType, ModuleType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, TypeVar, Union, cast, get_type_hints from urllib.parse import parse_qs, urlencode, urlparse, urlunparse @@ -2069,6 +2069,14 @@ class BaseLLMHTTPHandler: if anthropic_messages_provider_config.should_filter_anthropic_beta_headers(): headers = update_headers_with_filtered_beta(headers=headers, provider=custom_llm_provider) + from litellm.llms.vertex_ai.vertex_llm_base import VertexBase + + explicit_vertex_location: Final = VertexBase.explicit_vertex_ai_location(MappingProxyType(dict(litellm_params))) + vertex_location_params: Final = ( + MappingProxyType({"vertex_location": explicit_vertex_location}) + if explicit_vertex_location + else MappingProxyType({}) + ) logging_obj.update_from_kwargs( kwargs=kwargs, model=model, @@ -2077,6 +2085,7 @@ class BaseLLMHTTPHandler: "preset_cache_key": None, "stream_response": {}, "model_info": kwargs.get("model_info"), + **vertex_location_params, **anthropic_messages_optional_request_params, }, custom_llm_provider=custom_llm_provider, diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index 69f2312f203..9e9242137e6 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -2226,3 +2226,78 @@ def test_direct_vector_store_search_debug_log_omits_stored_credentials(caplog, i logged = "\n".join(record.getMessage() for record in caplog.records) assert "sup3r-s3cret-valkey-pw" not in logged assert "sk-embedding-s3cret" not in logged + + +@pytest.mark.asyncio +async def test_async_anthropic_messages_handler_carries_deployment_vertex_location_for_pricing(monkeypatch): + """ + The proxy pre-creates the logging object before the router picks a deployment, so the + native /v1/messages path must copy the deployment's vertex_location into the logging + params it updates; otherwise cost resolution falls back to the environment and every + call on this surface prices with the regional uplift (#34393). + """ + import contextlib + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import ( + Logging, + _resolve_vertex_location_for_cost, + ) + + monkeypatch.setenv("VERTEXAI_LOCATION", "us-east5") + monkeypatch.setattr(litellm, "vertex_location", None) + + handler = BaseLLMHTTPHandler() + + async def logging_obj_after_handler(generic_params): + logging_obj = Logging( + model="vertex_ai/claude-haiku-4-5@20251001", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="vertex-messages-location", + function_id="f", + ) + logging_obj.update_environment_variables( + model="vertex_ai/claude-haiku-4-5@20251001", + user="", + optional_params={}, + litellm_params={"api_base": ""}, + custom_llm_provider="vertex_ai", + ) + mock_config = Mock() + mock_config.validate_anthropic_messages_environment = Mock( + return_value=({"authorization": "Bearer t"}, "https://us-east5-aiplatform.googleapis.com") + ) + mock_config.transform_anthropic_messages_request = Mock( + return_value={"model": "claude-haiku-4-5@20251001", "messages": []} + ) + with contextlib.suppress(Exception): + await handler.async_anthropic_messages_handler( + model="claude-haiku-4-5@20251001", + messages=[{"role": "user", "content": "hi"}], + anthropic_messages_provider_config=mock_config, + anthropic_messages_optional_request_params={"max_tokens": 10}, + custom_llm_provider="vertex_ai", + litellm_params=generic_params, + logging_obj=logging_obj, + client=AsyncMock(), + kwargs={}, + ) + return logging_obj + + global_deployment = await logging_obj_after_handler(GenericLiteLLMParams(vertex_location="global")) + assert global_deployment.litellm_params["vertex_location"] == "global" + assert ( + _resolve_vertex_location_for_cost( + custom_llm_provider="vertex_ai", + litellm_params=global_deployment.litellm_params, + optional_params=global_deployment.optional_params, + model="claude-haiku-4-5@20251001", + ) + == "global" + ) + + unconfigured_deployment = await logging_obj_after_handler(GenericLiteLLMParams()) + assert "vertex_location" not in unconfigured_deployment.litellm_params diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 1d9477432ee..60453931595 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -832,6 +832,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "output_cost_per_token_above_200k_tokens_priority": {"type": "number"}, "output_cost_per_token_above_272k_tokens_priority": {"type": "number"}, "output_cost_per_token_above_272k_tokens_flex": {"type": "number"}, + "regional_endpoint_uplift_multiplier": {"type": "number"}, "regional_processing_uplift_multiplier_eu": {"type": "number"}, "regional_processing_uplift_multiplier_us": {"type": "number"}, "input_cost_per_pixel": {"type": "number"}, From dc70c144d7d60bbbe09562fa3006feb2bc1c613b Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 19 Aug 2026 23:35:54 +0000 Subject: [PATCH 08/24] chore(codeowners): require @mateo-berri approval for the model prices jsons Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 51d489459d9..118e5491939 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,5 @@ /ui/ @yuneng-jiang @ryan-crabbe-berri /litellm/proxy/_experimental/out/ @yuneng-jiang @ryan-crabbe-berri /ui/litellm-dashboard/src/lib/http/schema.d.ts +/model_prices_and_context_window.json @mateo-berri +/litellm/model_prices_and_context_window_backup.json @mateo-berri From c549cddada10190a3a593c6c92850fceaa10e009 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:44:52 -0700 Subject: [PATCH 09/24] fix(vertex_ai): price passthrough calls on the URL's serving location --- .../litellm_core_utils/llm_cost_calc/utils.py | 5 - litellm/llms/vertex_ai/cost_calculator.py | 2 - .../vertex_passthrough_logging_handler.py | 11 +++ .../test_llm_pass_through_endpoints.py | 92 +++++++++++++++++++ tests/test_litellm/test_cost_calculator.py | 26 ++++-- 5 files changed, 119 insertions(+), 17 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index dec35d16ea0..0793fe20b21 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -999,9 +999,6 @@ def generic_cost_per_token( prompt_cost *= uplift completion_cost *= uplift - ## VERTEX REGIONAL-ENDPOINT UPLIFT - # Applied as a flat multiplier across all token costs for the request - # when the Vertex AI endpoint serving it is non-global. vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) if vertex_uplift != 1.0: prompt_cost *= vertex_uplift @@ -1109,8 +1106,6 @@ def get_token_type_cost_breakdown( cache_read_cost *= uplift cache_creation_cost *= uplift - # Same flat uplift for Vertex AI non-global endpoints, keeping per-type - # costs reconciled with the totals for regional Vertex deployments. vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) if vertex_uplift != 1.0: reasoning_cost *= vertex_uplift diff --git a/litellm/llms/vertex_ai/cost_calculator.py b/litellm/llms/vertex_ai/cost_calculator.py index a9f5d77350c..23cb1e5b580 100644 --- a/litellm/llms/vertex_ai/cost_calculator.py +++ b/litellm/llms/vertex_ai/cost_calculator.py @@ -164,8 +164,6 @@ def cost_per_character( usage=usage, ) - # Applied once here; the cost_per_token fallbacks above are called without - # vertex_location so the uplift can never compound. vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) return prompt_cost * vertex_uplift, completion_cost * vertex_uplift diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index 621b3ff9c83..f8e521410bf 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -10,6 +10,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import VERTEX_BATCH_PREDICTION_JOBS_ROUTE from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.vertex_ai.common_utils import get_vertex_location_from_url from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( ModelResponseIterator as VertexModelResponseIterator, ) @@ -60,6 +61,7 @@ class VertexPassthroughLoggingHandler: request_body: dict | None = None, **kwargs, ) -> PassThroughEndpointLoggingTypedDict: + vertex_location: Final = get_vertex_location_from_url(url_route) if "predictLongRunning" in url_route: model = VertexPassthroughLoggingHandler.extract_model_from_url(url_route) @@ -82,6 +84,7 @@ class VertexPassthroughLoggingHandler: model=model, custom_llm_provider="vertex_ai", call_type="create_video", + vertex_location=vertex_location, ) # Set response_cost in _hidden_params to prevent recalculation @@ -123,6 +126,7 @@ class VertexPassthroughLoggingHandler: end_time=end_time, logging_obj=logging_obj, custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url(url_route), + vertex_location=vertex_location, ) return { @@ -190,6 +194,7 @@ class VertexPassthroughLoggingHandler: end_time=end_time, logging_obj=logging_obj, custom_llm_provider="vertex_ai", + vertex_location=vertex_location, ) return { @@ -206,6 +211,7 @@ class VertexPassthroughLoggingHandler: model="vertex_ai/search_api", custom_llm_provider="vertex_ai", call_type="vector_store_search", + vertex_location=vertex_location, ) standard_pass_through_response_object: Final[StandardPassThroughResponseObject] = { @@ -302,6 +308,7 @@ class VertexPassthroughLoggingHandler: completion_response=litellm_prediction_response, model=model, custom_llm_provider="vertex_ai", + vertex_location=get_vertex_location_from_url(url_route), ) kwargs["response_cost"] = response_cost @@ -381,6 +388,7 @@ class VertexPassthroughLoggingHandler: completion_response=litellm_embedding_response, model=model, custom_llm_provider=custom_llm_provider, + vertex_location=get_vertex_location_from_url(url_route), ) kwargs["response_cost"] = response_cost @@ -438,6 +446,7 @@ class VertexPassthroughLoggingHandler: end_time=end_time, logging_obj=litellm_logging_obj, custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url(url_route), + vertex_location=get_vertex_location_from_url(url_route), ) return { @@ -591,6 +600,7 @@ class VertexPassthroughLoggingHandler: end_time: datetime, logging_obj: LiteLLMLoggingObj, custom_llm_provider: str, + vertex_location: str | None, ) -> dict: """ Create the standard logging object for Vertex passthrough generateContent (streaming and non-streaming) @@ -601,6 +611,7 @@ class VertexPassthroughLoggingHandler: completion_response=litellm_model_response, model=model, custom_llm_provider="vertex_ai", + vertex_location=vertex_location, ) kwargs["response_cost"] = response_cost diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index b56a8da7c66..8da6d0fe81d 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -1109,6 +1109,98 @@ class TestVertexAIPassThroughHandler: assert result["kwargs"].get("model") == "gemini-embedding-2-preview" mock_completion_cost.assert_called_once() + @pytest.mark.parametrize("streaming", [False, True]) + def test_vertex_passthrough_handler_prices_regional_endpoint_with_uplift(self, monkeypatch, streaming): + """ + Passthrough cost is computed inside the handler and stored as response_cost before the + logging cost resolver runs, so the handler itself must read the serving location out of + the passthrough URL; otherwise regional Vertex passthrough traffic bills at the global + rate (#34393). + """ + import datetime + + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import ( + VertexPassthroughLoggingHandler, + ) + + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.get_model_cost_map(url=""), + "vertex_ai/gemini-fake-regional": { + "litellm_provider": "vertex_ai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "regional_endpoint_uplift_multiplier": 1.1, + }, + }, + ) + + response_body: Final = { + "candidates": [ + { + "content": {"parts": [{"text": "hello"}], "role": "model"}, + "finishReason": "STOP", + } + ], + "usageMetadata": { + "promptTokenCount": 10, + "candidatesTokenCount": 20, + "totalTokenCount": 30, + }, + } + + def cost_for(location: str) -> float: + url_route: Final = ( + f"https://{location}-aiplatform.googleapis.com/v1/projects/p/locations/{location}" + "/publishers/google/models/gemini-fake-regional:" + f"{'streamGenerateContent' if streaming else 'generateContent'}" + ) + mock_logging_obj: Final = Mock() + mock_logging_obj.litellm_call_id = "call-id" + mock_logging_obj.model_call_details = {} + mock_logging_obj.optional_params = {} + start_time: Final = datetime.datetime.now() + end_time: Final = datetime.datetime.now() + if streaming: + result = VertexPassthroughLoggingHandler._handle_logging_vertex_collected_chunks( + litellm_logging_obj=mock_logging_obj, + passthrough_success_handler_obj=Mock(), + url_route=url_route, + request_body={}, + endpoint_type="vertex_ai", + start_time=start_time, + all_chunks=[json.dumps(response_body)], + model=None, + end_time=end_time, + ) + else: + mock_httpx_response: Final = Mock() + mock_httpx_response.json.return_value = response_body + mock_httpx_response.headers = {} + mock_httpx_response.status_code = 200 + result = VertexPassthroughLoggingHandler.vertex_passthrough_handler( + httpx_response=mock_httpx_response, + logging_obj=mock_logging_obj, + url_route=url_route, + result="test-result", + start_time=start_time, + end_time=end_time, + cache_hit=False, + ) + return result["kwargs"]["response_cost"] + + global_cost: Final = cost_for("global") + regional_cost: Final = cost_for("us-east5") + + assert global_cost == pytest.approx(10 * 1e-06 + 20 * 2e-06, rel=1e-9) + assert regional_cost == pytest.approx(global_cost * 1.10, rel=1e-9), ( + "regional Vertex passthrough traffic must bill at 1.1x the global rate" + ) + class TestVertexAIDiscoveryPassThroughHandler: """ diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 6deaf5479e0..98938dee62e 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1781,16 +1781,22 @@ def test_vertex_uplift_composes_with_above_128k_pricing(monkeypatch): including the above-128k dynamic rates, so a synthetic model carrying both keys prices regional above-128k usage at 1.1x the above-128k rate.""" monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - litellm.model_cost["vertex_ai/fake-regional-128k-model"] = { - "litellm_provider": "vertex_ai", - "mode": "chat", - "input_cost_per_token": 1e-06, - "output_cost_per_token": 2e-06, - "input_cost_per_token_above_128k_tokens": 2e-06, - "output_cost_per_token_above_128k_tokens": 4e-06, - "regional_endpoint_uplift_multiplier": 1.1, - } + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.get_model_cost_map(url=""), + "vertex_ai/fake-regional-128k-model": { + "litellm_provider": "vertex_ai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "input_cost_per_token_above_128k_tokens": 2e-06, + "output_cost_per_token_above_128k_tokens": 4e-06, + "regional_endpoint_uplift_multiplier": 1.1, + }, + }, + ) usage = Usage(prompt_tokens=200_000, completion_tokens=10, total_tokens=200_010) global_prompt, global_completion = cost_per_token( From 919bf1a09785c020dd859605cb1074bfa6c42383 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:53:02 -0700 Subject: [PATCH 10/24] fix(proxy): strip client standard_logging_object before the failure logging handler --- litellm/proxy/utils.py | 8 +++-- tests/test_litellm/proxy/test_proxy_utils.py | 35 ++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index acafc600b80..19dcd64c58d 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -2309,6 +2309,11 @@ class ProxyLogging: ) ) + # Auth and pass-through failure bodies are unstripped client input, and + # the logging handler below flattens body keys into model_call_details, + # so drop the key before it can masquerade as the built payload. + request_data.pop("standard_logging_object", None) + ### LOGGING ### if self._is_proxy_only_llm_api_error( original_exception=original_exception, @@ -2322,9 +2327,6 @@ class ProxyLogging: original_exception=original_exception, ) - # Auth and pass-through failures reach this hook with the raw request - # body unstripped, so only the logging object may supply this key. - request_data.pop("standard_logging_object", None) request_data.update(_failure_fields_to_lift(request_data)) # Remove before callbacks iterate — not serialisable diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 71af44d6682..07877514b69 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -556,6 +556,41 @@ class TestPostCallFailureHookLiftsStandardLoggingObject: await self._run(request_data_with_obj) assert "standard_logging_object" not in request_data_with_obj + @pytest.mark.asyncio + async def test_pass_through_failure_never_relifts_client_supplied_key(self): + from datetime import datetime + from unittest.mock import AsyncMock, patch + + from fastapi import HTTPException + + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy._types import UserAPIKeyAuth + + logging_obj = Logging( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id="test-call-id", + function_id="test-function-id", + ) + request_data = { + "litellm_logging_obj": logging_obj, + "standard_logging_object": {"model_id": "client-injected"}, + "metadata": {}, + } + proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging_obj.alert_types = [] + with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()): + await proxy_logging_obj.post_call_failure_hook( + request_data=request_data, + original_exception=HTTPException(status_code=401, detail="unauthorized"), + user_api_key_dict=UserAPIKeyAuth(request_route="/v1/chat/completions"), + ) + assert "standard_logging_object" not in request_data + assert "standard_logging_object" not in logging_obj.model_call_details + @pytest.mark.asyncio async def test_no_standard_logging_object_is_noop(self): logging_obj = MagicMock() From f22eeb2ce05427083318c04a91e7281a56268249 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Wed, 19 Aug 2026 17:00:26 -0700 Subject: [PATCH 11/24] fix(proxy): initialize the secret manager before resolving os.environ config references (#37544) `ProxyConfig.get_config()` walked the parsed config and replaced every `os.environ/` string with `get_secret(value)` before anything initialized the secret manager, so a key held only by the manager resolved to `None` and that `None` was written back into the config. The later fallback in `load_config` could not recover it, because the key now existed with a `None` value. Hoist the initialization into `get_config()`, ahead of the resolution pass, so every entrypoint gets it: the CLI already did this itself, but the microservice entrypoints (`gateway/main.py`, `backend/main.py`) uvicorn the app directly and bypass the CLI. `load_config`'s own call is now redundant and is dropped, so startup builds the manager once instead of building one and discarding it. `get_config()` also runs on management-endpoint request paths, so this returns early once a manager exists rather than rebuilding the client per request. Also warn when a reference the manager would have been asked for resolves to `None`. The reporter had no log line at all to work from. `get_secret` only reaches the manager when reads are enabled and the name is in `hosted_keys`, so `secret_manager_would_be_consulted` mirrors that gate and keeps the warning off env-only references, which are expected rather than an error. --- litellm/proxy/proxy_server.py | 62 +++++- litellm/proxy/read_model_list.py | 3 +- litellm/secret_managers/main.py | 28 ++- .../proxy/proxy_server/test_proxy_config.py | 208 ++++++++++++++++++ .../test_secret_managers_main.py | 69 +++++- 5 files changed, 352 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5adee476f33..0f39ef2e18e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -639,6 +639,7 @@ from litellm.secret_managers.main import ( get_secret_bool, get_secret_str, normalize_nonempty_secret_str, + secret_manager_would_be_consulted, str_to_bool, ) from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingArgs @@ -4380,9 +4381,55 @@ class ProxyConfig: item = self._check_for_os_environ_vars(config=item, depth=depth + 1, max_depth=max_depth) # if the value is a string and starts with "os.environ/" - then it's an environment variable elif isinstance(value, str) and value.startswith("os.environ/"): - config[key] = get_secret(value) + resolved = get_secret(value) + if resolved is None and secret_manager_would_be_consulted(value): + verbose_proxy_logger.warning("%s is absent from the configured secret manager", value) + config[key] = resolved return config + def _initialize_secret_manager_from_raw_config( + self, config: Mapping[str, object], config_file_path: str | None + ) -> None: + """ + Bring the secret manager up before `os.environ/` references are resolved. + + `_check_for_os_environ_vars` writes whatever it resolves back into the config, so a key + held only by the secret manager would otherwise become a permanent `None` that the later + fallbacks in `load_config` can no longer recover from. + + `get_config` also runs on management-endpoint request paths, so this returns early once a + manager exists rather than rebuilding the client on every request. + + The manager's own settings can only come from real environment variables, so they are + resolved against a throwaway copy and the config is left untouched for the main pass. + """ + if litellm.secret_manager_client is not None: + return + + general_settings: Final = config.get("general_settings") + if not isinstance(general_settings, dict): + return + + raw_system: Final = general_settings.get("key_management_system") + key_management_system: Final = ( + get_secret(raw_system) + if isinstance(raw_system, str) and raw_system.startswith("os.environ/") + else raw_system + ) + if not isinstance(key_management_system, str): + return + + raw_settings: Final = general_settings.get("key_management_settings") + if isinstance(raw_settings, dict): + litellm._key_management_settings = KeyManagementSettings( + **self._check_for_os_environ_vars(config=copy.deepcopy(raw_settings)) + ) + + self.initialize_secret_manager( + key_management_system=key_management_system, + config_file_path=config_file_path, + ) + def _get_team_config(self, team_id: str, all_teams_config: list[dict]) -> dict: team_config: dict = {} for team in all_teams_config: @@ -4553,6 +4600,8 @@ class ProxyConfig: printed_yaml: Final = copy.deepcopy(config) printed_yaml.pop("environment_variables", None) + self._initialize_secret_manager_from_raw_config(config=config, config_file_path=config_file_path) + config = self._check_for_os_environ_vars(config=config) self.update_config_state(config=config) @@ -5123,17 +5172,14 @@ class ProxyConfig: key: general_settings[key] for key in SPEND_LOG_CLEANUP_BOUND_SETTINGS if key in general_settings } - ### LOAD KEY MANAGEMENT SETTINGS FIRST (needed for custom secret manager) ### + ### LOAD KEY MANAGEMENT SETTINGS ### + # The secret manager itself is brought up by get_config(), which runs before the + # `os.environ/` references in this config were resolved. Re-reading the settings here + # picks up any of them that were themselves secret-manager backed. key_management_settings: Final = general_settings.get("key_management_settings", None) if key_management_settings is not None: litellm._key_management_settings = KeyManagementSettings(**key_management_settings) - ### LOAD SECRET MANAGER ### - key_management_system: Final = general_settings.get("key_management_system", None) - self.initialize_secret_manager( - key_management_system=key_management_system, - config_file_path=config_file_path, - ) ### [DEPRECATED] LOAD FROM GOOGLE KMS ### old way of loading from google kms use_google_kms: Final = general_settings.get("use_google_kms", False) load_google_kms(use_google_kms=use_google_kms) diff --git a/litellm/proxy/read_model_list.py b/litellm/proxy/read_model_list.py index cdd6680aa40..a1830e7f2bc 100644 --- a/litellm/proxy/read_model_list.py +++ b/litellm/proxy/read_model_list.py @@ -9,7 +9,8 @@ effects. Instead we reuse ``ProxyConfig.get_config`` — the actual config reader — so the gateway inherits the same heavy lifting the proxy does: ``include:`` merging, ``os.environ/`` + secret-manager resolution, and DB-stored models (when a DB is -configured). It has no proxy-setup side effects. Returns the resolved +configured). Its only proxy-setup side effect is bringing up the configured +secret manager, which is what makes that resolution work. Returns the resolved ``model_list``; the Rust side deserializes each entry into its ``Deployment``. """ diff --git a/litellm/secret_managers/main.py b/litellm/secret_managers/main.py index d1e4b3bb2ce..e89fbbdab65 100644 --- a/litellm/secret_managers/main.py +++ b/litellm/secret_managers/main.py @@ -365,6 +365,22 @@ def get_secret( raise e +def secret_manager_would_be_consulted(secret_name: str) -> bool: + """ + Returns True if a `get_secret` read for `secret_name` would actually reach the hosted manager. + + Mirrors the gating `get_secret` applies below: the manager has to be up and readable, and + `hosted_keys`, when set, is an allowlist of the names it is consulted for. Callers use this to + tell "the manager does not have this key" apart from "the manager was never asked". + """ + if not _should_read_secret_from_secret_manager(): + return False + key_management_settings: Final = litellm._key_management_settings + if key_management_settings is None or key_management_settings.hosted_keys is None: + return True + return secret_name.removeprefix("os.environ/") in key_management_settings.hosted_keys + + def _should_read_secret_from_secret_manager() -> bool: """ Returns True if the secret manager should be used to read the secret, False otherwise @@ -373,11 +389,7 @@ def _should_read_secret_from_secret_manager() -> bool: - If the `_key_management_settings` access mode is "read_only" or "read_and_write", return True - Otherwise, return False """ - if litellm.secret_manager_client is not None: - if litellm._key_management_settings is not None: - if ( - litellm._key_management_settings.access_mode == "read_only" - or litellm._key_management_settings.access_mode == "read_and_write" - ): - return True - return False + key_management_settings: Final = litellm._key_management_settings + if litellm.secret_manager_client is None or key_management_settings is None: + return False + return key_management_settings.access_mode in ("read_only", "read_and_write") diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 37a03617a65..58465772b3b 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -769,6 +769,214 @@ async def test_ProxyConfig_get_config_missing_file_raises(monkeypatch): await pc.get_config(config_file_path="/no/such/path.yaml") +# --------------------------------------------------------------------------- +# ProxyConfig._initialize_secret_manager_from_raw_config +# --------------------------------------------------------------------------- + +VAULT_SECRET_MANAGER_MODULE = ''' +import os + +from litellm.integrations.custom_secret_manager import CustomSecretManager + +VAULT = {"LITELLM_MASTER_KEY": "master-from-vault", "MY_PROVIDER_KEY": "provider-from-vault"} + + +class VaultSecretManager(CustomSecretManager): + def __init__(self): + super().__init__() + # The loader re-executes this module on every construction, so an in-module counter + # would reset. Append to a file instead, to count constructions across the whole load. + with open(os.environ["VAULT_CONSTRUCTION_LOG"], "a") as f: + f.write("constructed\\n") + + def sync_read_secret(self, secret_name, optional_params=None, timeout=None, **kwargs): + return VAULT.get(secret_name) + + async def async_read_secret(self, secret_name, optional_params=None, timeout=None, **kwargs): + return VAULT.get(secret_name) +''' + +VAULT_BACKED_CONFIG = """ +model_list: + - model_name: my-model + litellm_params: + model: openai/gpt-4o-mini + api_key: os.environ/MY_PROVIDER_KEY + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + key_management_system: custom + key_management_settings: + custom_secret_manager: vault_secret_manager.VaultSecretManager + hosted_keys: + - LITELLM_MASTER_KEY + - MY_PROVIDER_KEY +""" + + +def _write_vault_backed_config(tmp_path, monkeypatch, config_yaml: str) -> str: + """Write a config whose secrets live only in a custom secret manager, never in the env.""" + (tmp_path / "vault_secret_manager.py").write_text(VAULT_SECRET_MANAGER_MODULE) + config_file = tmp_path / "c.yaml" + config_file.write_text(config_yaml) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + monkeypatch.delenv("LITELLM_MASTER_KEY", raising=False) + monkeypatch.delenv("MY_PROVIDER_KEY", raising=False) + monkeypatch.setenv("VAULT_CONSTRUCTION_LOG", str(tmp_path / "constructions.log")) + monkeypatch.setattr(litellm, "secret_manager_client", None) + return str(config_file) + + +def _construction_count(tmp_path) -> int: + log = tmp_path / "constructions.log" + return len(log.read_text().splitlines()) if log.exists() else 0 + + +@pytest.mark.asyncio +async def test_ProxyConfig_get_config_resolves_keys_held_only_by_the_secret_manager(tmp_path, monkeypatch): + """Regression for GH #35239. + + get_config() used to resolve every ``os.environ/`` reference and write the result + back into the config before the secret manager was initialized, so any key that lived + only in the manager became a permanent ``None``. + """ + config_file_path = _write_vault_backed_config(tmp_path, monkeypatch, VAULT_BACKED_CONFIG) + + cfg = await ProxyConfig().get_config(config_file_path=config_file_path) + + assert { + "master_key": cfg["general_settings"]["master_key"], + "api_key": cfg["model_list"][0]["litellm_params"]["api_key"], + "hosted_keys": litellm._key_management_settings.hosted_keys, + } == { + "master_key": "master-from-vault", + "api_key": "provider-from-vault", + "hosted_keys": ["LITELLM_MASTER_KEY", "MY_PROVIDER_KEY"], + } + + +@pytest.mark.asyncio +async def test_ProxyConfig_load_config_builds_the_secret_manager_exactly_once(tmp_path, monkeypatch): + """The full startup path must not build the manager, then throw it away and build another. + + A discarded client costs a Vault/CyberArk re-auth and leaks a gRPC channel on Google KMS. + """ + config_file_path = _write_vault_backed_config(tmp_path, monkeypatch, VAULT_BACKED_CONFIG) + + _router, _model_list, general_settings = await ProxyConfig().load_config( + router=None, config_file_path=config_file_path + ) + + assert { + "constructions": _construction_count(tmp_path), + "master_key": general_settings["master_key"], + } == {"constructions": 1, "master_key": "master-from-vault"} + + +@pytest.mark.asyncio +async def test_ProxyConfig_get_config_reuses_an_already_initialized_secret_manager(tmp_path, monkeypatch): + """get_config() also runs on management-endpoint request paths. + + Rebuilding the client on every call would re-execute the custom manager module, drop the + Vault/CyberArk token caches, and leak a gRPC channel per request on Google KMS. + """ + config_file_path = _write_vault_backed_config(tmp_path, monkeypatch, VAULT_BACKED_CONFIG) + + await ProxyConfig().get_config(config_file_path=config_file_path) + first_client = litellm.secret_manager_client + second = await ProxyConfig().get_config(config_file_path=config_file_path) + + assert { + "client_reused": litellm.secret_manager_client is first_client, + "master_key": second["general_settings"]["master_key"], + } == {"client_reused": True, "master_key": "master-from-vault"} + + +@pytest.mark.asyncio +async def test_ProxyConfig_get_config_without_key_management_system_leaves_secret_manager_unset( + tmp_path, monkeypatch +): + """No ``key_management_system`` means no manager, an unresolvable reference stays None, and + nothing is warned about: with no manager there is nothing to have been absent from.""" + config_yaml = VAULT_BACKED_CONFIG.replace(" key_management_system: custom\n", "") + config_file_path = _write_vault_backed_config(tmp_path, monkeypatch, config_yaml) + warn = MagicMock() + monkeypatch.setattr("litellm.proxy.proxy_server.verbose_proxy_logger.warning", warn) + + cfg = await ProxyConfig().get_config(config_file_path=config_file_path) + + assert { + "master_key": cfg["general_settings"]["master_key"], + "api_key": cfg["model_list"][0]["litellm_params"]["api_key"], + "client": litellm.secret_manager_client, + "warned_about": [call.args[1] for call in warn.call_args_list], + } == {"master_key": None, "api_key": None, "client": None, "warned_about": []} + + +@pytest.mark.asyncio +async def test_ProxyConfig_get_config_warns_when_a_reference_is_missing_from_the_secret_manager( + tmp_path, monkeypatch +): + """A reference the manager cannot resolve is logged, instead of silently becoming None.""" + config_yaml = VAULT_BACKED_CONFIG.replace("MY_PROVIDER_KEY", "NOT_IN_VAULT") + config_file_path = _write_vault_backed_config(tmp_path, monkeypatch, config_yaml) + warn = MagicMock() + monkeypatch.setattr("litellm.proxy.proxy_server.verbose_proxy_logger.warning", warn) + + cfg = await ProxyConfig().get_config(config_file_path=config_file_path) + + assert { + "api_key": cfg["model_list"][0]["litellm_params"]["api_key"], + "warned_about": [call.args[1] for call in warn.call_args_list], + } == {"api_key": None, "warned_about": ["os.environ/NOT_IN_VAULT"]} + + +@pytest.mark.asyncio +async def test_ProxyConfig_get_config_does_not_warn_for_a_name_outside_hosted_keys(tmp_path, monkeypatch): + """``hosted_keys`` is an allowlist, so a name outside it is never looked up in the manager. + + Warning about it would claim a lookup that never happened, on every optional env-only + reference, on every config reload. + """ + config_yaml = VAULT_BACKED_CONFIG.replace("api_key: os.environ/MY_PROVIDER_KEY", "api_key: os.environ/ENV_ONLY") + config_file_path = _write_vault_backed_config(tmp_path, monkeypatch, config_yaml) + warn = MagicMock() + monkeypatch.setattr("litellm.proxy.proxy_server.verbose_proxy_logger.warning", warn) + + cfg = await ProxyConfig().get_config(config_file_path=config_file_path) + + assert { + "api_key": cfg["model_list"][0]["litellm_params"]["api_key"], + "client_is_up": litellm.secret_manager_client is not None, + "warned_about": [call.args[1] for call in warn.call_args_list], + } == {"api_key": None, "client_is_up": True, "warned_about": []} + + +@pytest.mark.asyncio +async def test_ProxyConfig_get_config_does_not_warn_under_write_only_access_mode(tmp_path, monkeypatch): + """``write_only`` means reads never reach the manager, so an absent name is not its fault. + + That mode exists so the manager can store virtual keys while config secrets stay in the + environment, which makes env-only references the expected state rather than an error. + """ + config_yaml = VAULT_BACKED_CONFIG.replace( + " key_management_settings:\n", " key_management_settings:\n access_mode: write_only\n" + ) + config_file_path = _write_vault_backed_config(tmp_path, monkeypatch, config_yaml) + warn = MagicMock() + monkeypatch.setattr("litellm.proxy.proxy_server.verbose_proxy_logger.warning", warn) + + cfg = await ProxyConfig().get_config(config_file_path=config_file_path) + + assert { + "master_key": cfg["general_settings"]["master_key"], + "client_is_up": litellm.secret_manager_client is not None, + "warned_about": [call.args[1] for call in warn.call_args_list], + } == {"master_key": None, "client_is_up": True, "warned_about": []} + + # --------------------------------------------------------------------------- # ProxyConfig.update_config_state / get_config_state # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/secret_managers/test_secret_managers_main.py b/tests/test_litellm/secret_managers/test_secret_managers_main.py index 3631f640136..acc91b691d4 100644 --- a/tests/test_litellm/secret_managers/test_secret_managers_main.py +++ b/tests/test_litellm/secret_managers/test_secret_managers_main.py @@ -7,7 +7,14 @@ from unittest.mock import Mock, patch import pytest -from litellm.secret_managers.main import get_secret, normalize_nonempty_secret_str +import litellm +from litellm.integrations.custom_secret_manager import CustomSecretManager +from litellm.secret_managers.main import ( + get_secret, + normalize_nonempty_secret_str, + secret_manager_would_be_consulted, +) +from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem # Set up logging for debugging logging.basicConfig(level=logging.DEBUG) @@ -364,3 +371,63 @@ def test_unsupported_oidc_provider(): ) def test_normalize_nonempty_secret_str(raw, expected): assert normalize_nonempty_secret_str(raw) == expected + + +class _SpySecretManager(CustomSecretManager): + """Records every name the manager is actually asked for.""" + + def __init__(self, asked): + self.asked = asked + + def sync_read_secret(self, secret_name, optional_params=None, timeout=None, **kwargs): + self.asked.append(secret_name) + return "a-value" + + async def async_read_secret(self, secret_name, optional_params=None, timeout=None, **kwargs): + self.asked.append(secret_name) + return "a-value" + + +@pytest.mark.parametrize( + ("access_mode", "hosted_keys", "secret_name", "expected"), + [ + ("read_only", None, "ANY_NAME", True), + ("read_only", ["ALLOWED"], "ALLOWED", True), + ("read_only", ["ALLOWED"], "NOT_ALLOWED", False), + ("read_and_write", ["ALLOWED"], "ALLOWED", True), + ("write_only", None, "ANY_NAME", False), + ("write_only", ["ALLOWED"], "ALLOWED", False), + ], +) +def test_secret_manager_would_be_consulted_matches_get_secret( + monkeypatch, access_mode, hosted_keys, secret_name, expected +): + """The predicate must agree with what get_secret actually does, not with a reading of it. + + Callers use it to tell "the manager does not have this key" apart from "the manager was + never asked", so a predicate that drifts from get_secret's gating makes them state a + lookup that never happened. + """ + asked = [] + monkeypatch.setattr(litellm, "secret_manager_client", _SpySecretManager(asked)) + monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM) + monkeypatch.setattr( + litellm, + "_key_management_settings", + KeyManagementSettings(access_mode=access_mode, hosted_keys=hosted_keys), + ) + monkeypatch.delenv(secret_name, raising=False) + + predicted = secret_manager_would_be_consulted(f"os.environ/{secret_name}") + get_secret(f"os.environ/{secret_name}") + + assert {"predicted": predicted, "actually_consulted": bool(asked)} == { + "predicted": expected, + "actually_consulted": expected, + } + + +def test_secret_manager_would_be_consulted_is_false_without_a_client(monkeypatch): + monkeypatch.setattr(litellm, "secret_manager_client", None) + + assert secret_manager_would_be_consulted("os.environ/ANY_NAME") is False From 629d7683f2be588ab69af2e6832e310b3c38b33e Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 19 Aug 2026 17:04:14 -0700 Subject: [PATCH 12/24] refactor(ui): swap @ant-design/icons for lucide-react (#37553) The dashboard drew its icons from two libraries at once: lucide-react, which shadcn/ui ships with, and @ant-design/icons, left over from antd. This moves the last 39 files onto lucide and drops the dependency, so the icon set matches the component library everywhere. antd icons sized themselves from the inherited font-size and rendered as role="img" with an aria-label, neither of which a lucide svg does, so the swap carries explicit size classes and gives the two icon-only plugin buttons real accessible names. --- ui/litellm-dashboard/eslint.config.mjs | 4 ++ ui/litellm-dashboard/package-lock.json | 1 - ui/litellm-dashboard/package.json | 1 - .../agents/_components/add_agent_form.tsx | 16 ++--- .../_components/GuardrailConfig.tsx | 18 ++---- .../guardrail_garden_card.test.tsx | 4 -- .../_components/guardrail_info.test.tsx | 2 +- .../guardrails/_components/guardrail_info.tsx | 7 +-- .../_components/AwsSigV4Fields.tsx | 4 +- .../_components/CreateMCPServer.tsx | 10 +-- .../_components/DcrBridgeToggle.tsx | 4 +- .../_components/EnvVarsSection.tsx | 12 ++-- .../_components/IdJagFormFields.tsx | 4 +- .../_components/MCPPermissionManagement.tsx | 23 ++++--- .../_components/OAuthFormFields.tsx | 4 +- .../_components/OpenAPIFormSection.tsx | 4 +- .../_components/OpenApiByokFields.tsx | 12 ++-- .../_components/StdioConfiguration.tsx | 4 +- .../TokenEndpointAuthMethodField.tsx | 4 +- .../_components/TokenExchangeFormFields.tsx | 4 +- .../_components/mcp_server_edit.tsx | 23 ++++--- .../LoggingSettings/LoggingSettings.tsx | 5 +- .../PluginSettings.integration.test.tsx | 6 +- .../PluginSettings/PluginSettings.tsx | 20 +++--- .../VirtualKeysPage/keyTableColumns.tsx | 4 +- .../add_model/ClassificationMethodConfig.tsx | 8 +-- .../add_model/ComplexityRouterConfig.tsx | 8 +-- .../add_model/EscalationKeywords.tsx | 4 +- .../components/add_model/KeywordTierRules.tsx | 8 +-- .../add_model/SemanticKeywordMatching.tsx | 4 +- .../add_model/advanced_settings.tsx | 7 +-- .../add_model/provider_specific_fields.tsx | 4 +- .../claude_code_plugins/skill_detail.tsx | 10 +-- .../common_components/AccessGroupSelector.tsx | 6 +- .../check_openapi_schema.tsx | 4 +- .../common_components/user_search_modal.tsx | 5 +- .../mcp_tools/ByokCredentialModal.tsx | 28 +++------ .../src/components/model_info_view.tsx | 5 +- .../organisms/RegenerateKeyModal.tsx | 7 +-- .../organisms/create_key_button.tsx | 61 +++++++++---------- .../src/components/routing_groups/index.tsx | 8 +-- .../shared/advanced_date_picker.test.tsx | 6 +- .../shared/advanced_date_picker.tsx | 6 +- .../src/components/team/LoggingSettings.tsx | 7 +-- .../src/components/team/TeamInfo.tsx | 17 +++--- 45 files changed, 195 insertions(+), 218 deletions(-) diff --git a/ui/litellm-dashboard/eslint.config.mjs b/ui/litellm-dashboard/eslint.config.mjs index 35603123736..3cbb93f9a48 100644 --- a/ui/litellm-dashboard/eslint.config.mjs +++ b/ui/litellm-dashboard/eslint.config.mjs @@ -62,6 +62,10 @@ const eslintConfig = [ message: "antd is being phased out; build new UI with shadcn/ui primitives instead of adding antd imports.", }, + { + group: ["@ant-design/icons", "@ant-design/icons/*"], + message: "@ant-design/icons is gone from the dashboard; use lucide-react instead.", + }, ], }, ], diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 7069450a75c..154a19da7f3 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -9,7 +9,6 @@ "version": "0.1.0", "dependencies": { "@ant-design/cssinjs": "1.24.0", - "@ant-design/icons": "5.6.1", "@anthropic-ai/sdk": "0.92.0", "@base-ui/react": "^1.6.0", "@headlessui/tailwindcss": "0.2.2", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 962cabcba7b..3f9c29d2afe 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -25,7 +25,6 @@ }, "dependencies": { "@ant-design/cssinjs": "1.24.0", - "@ant-design/icons": "5.6.1", "@anthropic-ai/sdk": "0.92.0", "@base-ui/react": "^1.6.0", "@headlessui/tailwindcss": "0.2.2", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx index 22471ee66f9..09737c3151b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx @@ -3,7 +3,7 @@ import { Select, Steps, Tag } from "antd"; import { FormProvider, useForm, useWatch } from "react-hook-form"; import { toast } from "@/lib/toast"; import { Logo } from "@/components/molecules/logo/Logo"; -import { CheckCircleFilled, KeyOutlined, RobotOutlined, AppstoreOutlined } from "@ant-design/icons"; +import { Bot, CircleCheck, Key, LayoutGrid } from "lucide-react"; import CreatedKeyDisplay from "@/components/shared/CreatedKeyDisplay"; import { Button } from "@/components/ui/button"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; @@ -707,7 +707,7 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok }`} onClick={() => handleAgentTypeChange(CUSTOM_AGENT_TYPE)} > - +
Custom / Other @@ -846,7 +846,8 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok
{/* Agent name chip */}
- } color="purple" className="px-3 py-1 text-sm"> + + {agentName}
@@ -884,7 +885,7 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok
- + Create a new key for this agent

A dedicated key scoped to this agent.

@@ -920,7 +921,7 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok
- + Assign an existing key

Re-assign a key you already have to this agent.

@@ -958,10 +959,11 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok const renderReadyStep = () => (
- +

Agent Created!

- } color="purple" className="px-3 py-1 text-sm"> + + {createdAgentName}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx index 619ccd8d974..271de78c272 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx @@ -1,10 +1,4 @@ -import { - CheckCircleOutlined, - CodeOutlined, - PlayCircleOutlined, - RollbackOutlined, - SaveOutlined, -} from "@ant-design/icons"; +import { CircleCheck, CirclePlay, Code, Save, Undo2 } from "lucide-react"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; @@ -100,11 +94,11 @@ export function GuardrailConfig({ guardrailName, guardrailType, provider }: Guar
@@ -214,7 +208,7 @@ export function GuardrailConfig({ guardrailName, guardrailType, provider }: Guar

- + Custom Code Override

Replace the built-in guardrail with custom evaluation code

@@ -247,13 +241,13 @@ export function GuardrailConfig({ guardrailName, guardrailType, provider }: Guar
{rerunStatus === "success" && ( - 7/10 would now pass with new config + 7/10 would now pass with new config )} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_card.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_card.test.tsx index d81eaa5a4fc..25f63ca2902 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_card.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_card.test.tsx @@ -4,10 +4,6 @@ import userEvent from "@testing-library/user-event"; import GuardrailCard from "./guardrail_garden_card"; import type { GuardrailCardInfo } from "./guardrail_garden_data"; -vi.mock("@ant-design/icons", () => ({ - CheckCircleFilled: ({ style, ...props }: any) => , -})); - const baseCard: GuardrailCardInfo = { id: "test-guard", name: "Test Guardrail", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx index b6ee130d50a..4f1ac3e7d7a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx @@ -153,7 +153,7 @@ describe("Guardrail Info", () => { expect(getByText("Guardrail Settings")).toBeInTheDocument(); }); - await userEvent.hover(within(container).getByRole("img", { name: "info-circle" })); + await userEvent.hover(within(container).getByRole("img", { name: "Config guardrail details" })); expect(await findByText("Guardrail is defined in the config file and cannot be edited.")).toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx index f89e8a3248c..8f3ec6185fd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx @@ -5,9 +5,8 @@ import { updateGuardrailCall, } from "@/components/networking"; import { copyToClipboard as utilCopyToClipboard } from "@/utils/dataUtils"; -import { EyeInvisibleOutlined, InfoCircleOutlined, StopOutlined } from "@ant-design/icons"; -import { ArrowLeft, CheckIcon, Code, CopyIcon } from "lucide-react"; +import { ArrowLeft, Ban, CheckIcon, Code, CopyIcon, EyeOff, Info } from "lucide-react"; import { Badge } from "@/components/ui/badge"; import { Card } from "@/components/ui/card"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; @@ -607,7 +606,7 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, value === "MASK" ? "text-blue-600" : "text-red-600" }`} > - {value === "MASK" ? : } + {value === "MASK" ? : } {String(value)}

@@ -667,7 +666,7 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose,

Guardrail Settings

{isConfigGuardrail && ( - + )} {!isEditing && diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx index 09016bda266..8137e551028 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx @@ -1,6 +1,6 @@ import React from "react"; import { Input, Tooltip } from "antd"; -import { InfoCircleOutlined } from "@ant-design/icons"; +import { Info } from "lucide-react"; import { MountedFormField } from "@/components/common_components/MountedFormField"; import { antdRequired } from "@/components/common_components/antdFormRules"; @@ -12,7 +12,7 @@ const FieldLabel: React.FC<{ label: string; tooltip: string }> = ({ label, toolt {label} - + ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx index 5233081fc39..f5acd116f33 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx @@ -1,7 +1,7 @@ import React, { useState } from "react"; import { Tooltip, Select, Input as AntdInput, InputNumber, Collapse } from "antd"; import { FormProvider, useForm, useWatch } from "react-hook-form"; -import { InfoCircleOutlined } from "@ant-design/icons"; +import { Info } from "lucide-react"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; @@ -688,7 +688,7 @@ const CreateMCPServer: React.FC = ({ MCP Server Name - + } @@ -709,7 +709,7 @@ const CreateMCPServer: React.FC = ({ Alias - + } @@ -827,7 +827,7 @@ const CreateMCPServer: React.FC = ({ Max Concurrent Requests (optional) - + } @@ -910,7 +910,7 @@ const CreateMCPServer: React.FC = ({ Authentication Value - + } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx index 35b23f9873c..a18c4592838 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx @@ -1,6 +1,6 @@ import React from "react"; import { Switch, Tooltip } from "antd"; -import { InfoCircleOutlined } from "@ant-design/icons"; +import { Info } from "lucide-react"; import { MountedFormField } from "@/components/common_components/MountedFormField"; import { isClientForwardedTokenMode } from "@/components/mcp_tools/types"; @@ -29,7 +29,7 @@ export default function DcrBridgeToggle({ Gateway-hosted sign-in (DCR bridge) - + } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx index 23a55d2dc1a..01f68d2a531 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx @@ -1,7 +1,7 @@ import React from "react"; import { Input, Select, Tooltip, Typography } from "antd"; import { Button } from "@/components/ui/button"; -import { InfoCircleOutlined, MinusCircleOutlined, PlusOutlined } from "@ant-design/icons"; +import { CircleMinus, Info, Plus } from "lucide-react"; import { useFieldArray, useFormContext, useWatch } from "react-hook-form"; import { @@ -55,7 +55,7 @@ const EnvVarsSection: React.FC = () => { } > - +
@@ -100,15 +100,15 @@ const EnvVarsSection: React.FC = () => { {(control) => = ({ {label}{" "} - + ) : ( diff --git a/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx b/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx index f4c88f09f2c..2977a9e4941 100644 --- a/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx +++ b/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx @@ -1,7 +1,6 @@ import { useRef, useState } from "react"; -import { Info } from "lucide-react"; +import { Info, UserPlus } from "lucide-react"; import { Alert, AlertTitle } from "@/components/shared/Alert"; -import { UserAddOutlined } from "@ant-design/icons"; import { useDebouncedCallback } from "@tanstack/react-pacer/debouncer"; import { useForm } from "react-hook-form"; import { userFilterUICall } from "@/components/networking"; @@ -266,7 +265,7 @@ const UserSearchModal: React.FC = ({
diff --git a/ui/litellm-dashboard/src/components/mcp_tools/ByokCredentialModal.tsx b/ui/litellm-dashboard/src/components/mcp_tools/ByokCredentialModal.tsx index 0e86492731c..f21af04d5d5 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/ByokCredentialModal.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/ByokCredentialModal.tsx @@ -5,15 +5,7 @@ import { Input, Switch } from "antd"; import { toast } from "@/lib/toast"; import { fetchClient } from "@/lib/http/api"; import { ApiError } from "@/lib/http/client"; -import { - KeyOutlined, - LockOutlined, - CheckOutlined, - ArrowRightOutlined, - ArrowLeftOutlined, - CloseOutlined, - LinkOutlined, -} from "@ant-design/icons"; +import { ArrowLeft, ArrowRight, Check, Key, Link2, Lock, X } from "lucide-react"; import { MCPServer } from "./types"; import { Dialog, DialogContent } from "@/components/ui/dialog"; @@ -84,7 +76,7 @@ export const ByokCredentialModal: React.FC = ({ server onClick={() => setStep(1)} className="flex items-center gap-1 text-gray-500 hover:text-gray-800 text-sm" > - Back + Back ) : (
@@ -94,7 +86,7 @@ export const ByokCredentialModal: React.FC = ({ server
@@ -105,7 +97,7 @@ export const ByokCredentialModal: React.FC = ({ server
L
- +
{firstLetter}
@@ -148,7 +140,7 @@ export const ByokCredentialModal: React.FC = ({ server
    {server.byok_description.map((item, i) => (
  • - + {item}
  • ))} @@ -160,7 +152,7 @@ export const ByokCredentialModal: React.FC = ({ server onClick={() => setStep(2)} className="w-full bg-gray-900 hover:bg-gray-700 text-white font-medium py-3 px-6 rounded-xl flex items-center justify-center gap-2 transition-colors" > - Continue to Authentication + Continue to Authentication
)} diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index 58d7a7bca6e..407144d1a52 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -2,7 +2,6 @@ import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap" import { useModelHub, useModelsInfo } from "@/app/(dashboard)/hooks/models/useModels"; import { useQueryClient } from "@tanstack/react-query"; import { transformModelData } from "@/app/(dashboard)/models-and-endpoints/utils/modelDataTransformer"; -import { InfoCircleOutlined } from "@ant-design/icons"; import { KeyIcon, RefreshIcon, TrashIcon } from "@heroicons/react/outline"; import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; @@ -10,7 +9,7 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { SimpleTooltip } from "@/components/ui/tooltip"; import { applyPtuModelInfo } from "../utils/ptuModelInfo"; import { usePtuCostAttributionEnabled } from "@/app/(dashboard)/hooks/uiSettings/usePtuCostAttributionEnabled"; -import { ArrowLeft, CheckIcon, CopyIcon } from "lucide-react"; +import { ArrowLeft, CheckIcon, CopyIcon, Info } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; import { copyToClipboard as utilCopyToClipboard } from "../utils/dataUtils"; import { stripMaskedSecrets } from "../utils/maskedSecretUtils"; @@ -757,7 +756,7 @@ export default function ModelInfoView({ ) ) : ( - + )}
diff --git a/ui/litellm-dashboard/src/components/organisms/RegenerateKeyModal.tsx b/ui/litellm-dashboard/src/components/organisms/RegenerateKeyModal.tsx index 5158d8612ab..81d3dcc61a2 100644 --- a/ui/litellm-dashboard/src/components/organisms/RegenerateKeyModal.tsx +++ b/ui/litellm-dashboard/src/components/organisms/RegenerateKeyModal.tsx @@ -1,9 +1,8 @@ import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { CheckOutlined, CopyOutlined, SyncOutlined } from "@ant-design/icons"; import { Alert, AlertTitle } from "@/components/shared/Alert"; import { Button } from "@/components/ui/button"; import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; -import { CircleHelp, TriangleAlert } from "lucide-react"; +import { Check, CircleHelp, Copy, RefreshCw, TriangleAlert } from "lucide-react"; import React, { useEffect, useMemo, useState } from "react"; import { useWatch } from "react-hook-form"; import { CopyToClipboard } from "react-copy-to-clipboard"; @@ -264,7 +263,7 @@ export function RegenerateKeyModal({ selectedToken, visible, onClose, onKeyUpdat @@ -275,7 +274,7 @@ export function RegenerateKeyModal({ selectedToken, visible, onClose, onKeyUpdat Cancel diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index eab5abb9936..168b7f6cf25 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -7,14 +7,13 @@ import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings" import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import useCan from "@/app/(dashboard)/hooks/useCan"; import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { InfoCircleOutlined } from "@ant-design/icons"; import { useQueryClient } from "@tanstack/react-query"; import { Button } from "@/components/ui/button"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import { Input } from "@/components/ui/input"; import { Field, FieldLabel } from "@/components/shared/form/field"; import { Input as AntdInput, Radio, Select, Switch, Tag, Tooltip, Typography } from "antd"; -import { ChevronDown } from "lucide-react"; +import { ChevronDown, Info } from "lucide-react"; import { useDebouncedCallback } from "@tanstack/react-pacer/debouncer"; import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants"; import React, { useEffect, useMemo, useRef, useState } from "react"; @@ -647,7 +646,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Owned By{" "} - + @@ -667,7 +666,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp User ID{" "} - + } @@ -740,7 +739,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Organization{" "} - + } @@ -763,7 +762,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Team{" "} - + } @@ -790,7 +789,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Project{" "} - + } @@ -836,7 +835,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp : "Unique identifier for this service account" } > - + } @@ -856,7 +855,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Models{" "} - + } @@ -909,7 +908,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Key Type{" "} - + } @@ -972,7 +971,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Max Budget (USD){" "} - + } @@ -999,7 +998,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Reset Budget{" "} - + } @@ -1021,7 +1020,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Budget Windows{" "} - + @@ -1032,7 +1031,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Budget Fallbacks{" "} - + @@ -1049,7 +1048,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Tokens per minute Limit (TPM){" "} - + } @@ -1090,7 +1089,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Requests per minute Limit (RPM){" "} - + } @@ -1130,7 +1129,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Per-Tag Rate Limits{" "} - + @@ -1142,7 +1141,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Throttle on budget exceeded{" "} - + } @@ -1164,7 +1163,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Enable Prompt Caching{" "} - + } @@ -1191,7 +1190,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp rel="noopener noreferrer" onClick={(e) => e.stopPropagation()} // Prevent accordion from collapsing when clicking link > - + @@ -1231,7 +1230,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp rel="noopener noreferrer" onClick={(e) => e.stopPropagation()} // Prevent accordion from collapsing when clicking link > - + @@ -1267,7 +1266,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp rel="noopener noreferrer" onClick={(e) => e.stopPropagation()} // Prevent accordion from collapsing when clicking link > - + @@ -1309,7 +1308,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp rel="noopener noreferrer" onClick={(e) => e.stopPropagation()} // Prevent accordion from collapsing when clicking link > - + @@ -1344,7 +1343,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Access Groups{" "} - + } @@ -1371,7 +1370,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp rel="noopener noreferrer" onClick={(e) => e.stopPropagation()} // Prevent accordion from collapsing when clicking link > - + @@ -1404,7 +1403,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Allowed Vector Stores{" "} - + } @@ -1426,7 +1425,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Metadata{" "} - + } @@ -1447,7 +1446,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Tags{" "} - + } @@ -1478,7 +1477,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Allowed MCP Servers{" "} - + } @@ -1521,7 +1520,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Allowed Agents{" "} - + } @@ -1688,7 +1687,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp } > - +
diff --git a/ui/litellm-dashboard/src/components/routing_groups/index.tsx b/ui/litellm-dashboard/src/components/routing_groups/index.tsx index 560523972ca..9f94fbf6c98 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/index.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/index.tsx @@ -3,7 +3,7 @@ import React, { useMemo, useState } from "react"; import { Card, Flex, Input, Space, Typography } from "antd"; import { Button } from "@/components/ui/button"; -import { PlusOutlined, ReloadOutlined, SearchOutlined } from "@ant-design/icons"; +import { Plus, RefreshCw, Search } from "lucide-react"; import { useRoutingGroups, useSaveRoutingGroups } from "@/app/(dashboard)/hooks/routingGroups/useRoutingGroups"; import { useRouterFields } from "@/app/(dashboard)/hooks/router/useRouterFields"; import { useModelHub } from "@/app/(dashboard)/hooks/models/useModels"; @@ -107,7 +107,7 @@ const RoutingGroups: React.FC = () => { } + prefix={} placeholder="Search groups..." value={searchQuery} onChange={(e) => setSearchQuery(e.target.value)} @@ -120,11 +120,11 @@ const RoutingGroups: React.FC = () => { disabled={isFetching && !isLoading} aria-busy={isFetching && !isLoading} > - + Refresh diff --git a/ui/litellm-dashboard/src/components/shared/advanced_date_picker.test.tsx b/ui/litellm-dashboard/src/components/shared/advanced_date_picker.test.tsx index c907aeae035..bb4e0865f3f 100644 --- a/ui/litellm-dashboard/src/components/shared/advanced_date_picker.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/advanced_date_picker.test.tsx @@ -49,10 +49,8 @@ describe("AdvancedDatePicker", () => { }); it("should display formatted date range", () => { - render(); - // The component displays date range in the format "D MMM, HH:mm - D MMM, HH:mm" - // Just check that the clock icon is present - expect(screen.getByLabelText("clock-circle")).toBeInTheDocument(); + const { container } = render(); + expect(getTrigger(container)).toHaveTextContent(/\d{1,2} \w{3}, \d{2}:\d{2} - \d{1,2} \w{3}, \d{2}:\d{2}/); }); it("should open dropdown when clicked", () => { diff --git a/ui/litellm-dashboard/src/components/shared/advanced_date_picker.tsx b/ui/litellm-dashboard/src/components/shared/advanced_date_picker.tsx index b9a4d28539b..0013f702362 100644 --- a/ui/litellm-dashboard/src/components/shared/advanced_date_picker.tsx +++ b/ui/litellm-dashboard/src/components/shared/advanced_date_picker.tsx @@ -1,4 +1,4 @@ -import { CalendarOutlined, ClockCircleOutlined } from "@ant-design/icons"; +import { Calendar, Clock } from "lucide-react"; import { Button } from "@/components/ui/button"; import { cn } from "@/lib/cva.config"; import type { DateRangePickerValue } from "./date_picker_types"; @@ -291,7 +291,7 @@ const AdvancedDatePicker: React.FC = ({ > - + {formatDisplayRange(value.from, value.to)} = ({
- + Start and end dates
diff --git a/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx b/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx index e9ec3ce9439..b7f2dfaf398 100644 --- a/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx +++ b/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx @@ -3,14 +3,13 @@ import React from "react"; import { SimpleTooltip } from "@/components/ui/tooltip"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Separator } from "@/components/ui/separator"; -import { InfoCircleOutlined } from "@ant-design/icons"; import { Button } from "@/components/ui/button"; import { Badge } from "@/components/ui/badge"; import { Card } from "@/components/ui/card"; import { Input } from "@/components/ui/input"; import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; import { CogIcon, BanIcon } from "@heroicons/react/outline"; -import { Eye, EyeOff, Plus, Trash2 } from "lucide-react"; +import { Eye, EyeOff, Info, Plus, Trash2 } from "lucide-react"; import { callbackInfo, callback_map, mapDisplayToInternalNames } from "../callback_info_helpers"; import { Logo } from "@/components/molecules/logo/Logo"; import NumericalInput from "../shared/numerical_input"; @@ -198,7 +197,7 @@ const LoggingSettings: React.FC = ({ Disabled Callbacks - +
@@ -244,7 +243,7 @@ const LoggingSettings: React.FC = ({ Logging Integrations - +
)} @@ -1809,7 +1808,7 @@ const TeamInfoView: React.FC = ({

Team Member Settings{" "} - +

Max Budget: {info.team_member_budget_table?.max_budget || "No Limit"}
@@ -1970,7 +1969,7 @@ const TeamInfoView: React.FC = ({ Team Member Budget (USD){" "} - + ), @@ -1985,7 +1984,7 @@ const TeamInfoView: React.FC = ({ Budget Reset Period{" "} - + ), @@ -1997,7 +1996,7 @@ const TeamInfoView: React.FC = ({ Team Member TPM Limit{" "} - + ), @@ -2012,7 +2011,7 @@ const TeamInfoView: React.FC = ({ Team Member RPM Limit{" "} - + ), @@ -2027,7 +2026,7 @@ const TeamInfoView: React.FC = ({ Allowed Models{" "} - + ), From 1140366beeadf659ed4214e7004af63490e6e4dd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:24:44 -0700 Subject: [PATCH 13/24] fix(vertex_ai): resolve passthrough serving location in the logging cost recompute --- .../vertex_passthrough_logging_handler.py | 7 ++- .../test_llm_pass_through_endpoints.py | 56 +++++++++++++------ 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index f8e521410bf..ddcca1d372b 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -62,6 +62,8 @@ class VertexPassthroughLoggingHandler: **kwargs, ) -> PassThroughEndpointLoggingTypedDict: vertex_location: Final = get_vertex_location_from_url(url_route) + if vertex_location is not None: + logging_obj.optional_params["vertex_location"] = vertex_location if "predictLongRunning" in url_route: model = VertexPassthroughLoggingHandler.extract_model_from_url(url_route) @@ -421,6 +423,9 @@ class VertexPassthroughLoggingHandler: - Logs in litellm callbacks """ kwargs: dict[str, Any] = {} + vertex_location: Final = get_vertex_location_from_url(url_route) + if vertex_location is not None: + litellm_logging_obj.optional_params["vertex_location"] = vertex_location model = model or VertexPassthroughLoggingHandler.extract_model_from_url(url_route) complete_streaming_response: Final = VertexPassthroughLoggingHandler._build_complete_streaming_response( all_chunks=all_chunks, @@ -446,7 +451,7 @@ class VertexPassthroughLoggingHandler: end_time=end_time, logging_obj=litellm_logging_obj, custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url(url_route), - vertex_location=get_vertex_location_from_url(url_route), + vertex_location=vertex_location, ) return { diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 8da6d0fe81d..a9454854948 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -719,6 +719,7 @@ class TestVertexAIPassThroughHandler: # Create mock logging object mock_logging_obj = Mock(spec=LiteLLMLoggingObj) + mock_logging_obj.optional_params = {} mock_logging_obj.litellm_call_id = "test-call-id-123" mock_logging_obj.model_call_details = {} @@ -895,6 +896,7 @@ class TestVertexAIPassThroughHandler: # Create mock logging object mock_logging_obj = Mock(spec=LiteLLMLoggingObj) + mock_logging_obj.optional_params = {} mock_logging_obj.litellm_call_id = "test-call-id-123" mock_logging_obj.model_call_details = {} @@ -965,6 +967,7 @@ class TestVertexAIPassThroughHandler: mock_httpx_response.status_code = 200 mock_logging_obj = Mock(spec=LiteLLMLoggingObj) + mock_logging_obj.optional_params = {} mock_logging_obj.litellm_call_id = "test-call-id-embed" mock_logging_obj.model_call_details = {} @@ -1023,6 +1026,7 @@ class TestVertexAIPassThroughHandler: mock_httpx_response.status_code = 200 mock_logging_obj = Mock(spec=LiteLLMLoggingObj) + mock_logging_obj.optional_params = {} mock_logging_obj.litellm_call_id = "test-call-id-batch" mock_logging_obj.model_call_details = {} @@ -1079,6 +1083,7 @@ class TestVertexAIPassThroughHandler: mock_httpx_response.status_code = 200 mock_logging_obj = Mock(spec=LiteLLMLoggingObj) + mock_logging_obj.optional_params = {} mock_logging_obj.litellm_call_id = "test-call-id-gemini-studio" mock_logging_obj.model_call_details = {} @@ -1112,13 +1117,14 @@ class TestVertexAIPassThroughHandler: @pytest.mark.parametrize("streaming", [False, True]) def test_vertex_passthrough_handler_prices_regional_endpoint_with_uplift(self, monkeypatch, streaming): """ - Passthrough cost is computed inside the handler and stored as response_cost before the - logging cost resolver runs, so the handler itself must read the serving location out of - the passthrough URL; otherwise regional Vertex passthrough traffic bills at the global - rate (#34393). + Both cost computations for a passthrough call must price on the URL's serving location: + the handler-computed cost, and the async success recompute, which re-resolves the + location from the logging object and previously fell through empty optional_params to + the us-central1 default, billing the regional uplift on global traffic too (#34393). """ import datetime + from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import ( VertexPassthroughLoggingHandler, ) @@ -1153,21 +1159,33 @@ class TestVertexAIPassThroughHandler: }, } - def cost_for(location: str) -> float: + def costs_for(location: str) -> tuple[float, float]: url_route: Final = ( f"https://{location}-aiplatform.googleapis.com/v1/projects/p/locations/{location}" "/publishers/google/models/gemini-fake-regional:" f"{'streamGenerateContent' if streaming else 'generateContent'}" ) - mock_logging_obj: Final = Mock() - mock_logging_obj.litellm_call_id = "call-id" - mock_logging_obj.model_call_details = {} - mock_logging_obj.optional_params = {} start_time: Final = datetime.datetime.now() end_time: Final = datetime.datetime.now() + logging_obj: Final = Logging( + model="gemini-fake-regional", + messages=[{"role": "user", "content": "hi"}], + stream=streaming, + call_type="pass_through_endpoint", + start_time=start_time, + litellm_call_id="call-id", + function_id="fn-id", + ) + logging_obj.update_environment_variables( + model="gemini-fake-regional", + user="unknown", + optional_params={}, + litellm_params={}, + call_type="pass_through_endpoint", + ) if streaming: result = VertexPassthroughLoggingHandler._handle_logging_vertex_collected_chunks( - litellm_logging_obj=mock_logging_obj, + litellm_logging_obj=logging_obj, passthrough_success_handler_obj=Mock(), url_route=url_route, request_body={}, @@ -1184,22 +1202,28 @@ class TestVertexAIPassThroughHandler: mock_httpx_response.status_code = 200 result = VertexPassthroughLoggingHandler.vertex_passthrough_handler( httpx_response=mock_httpx_response, - logging_obj=mock_logging_obj, + logging_obj=logging_obj, url_route=url_route, result="test-result", start_time=start_time, end_time=end_time, cache_hit=False, ) - return result["kwargs"]["response_cost"] + recomputed: Final = logging_obj._response_cost_calculator(result=result["result"]) + return result["kwargs"]["response_cost"], recomputed - global_cost: Final = cost_for("global") - regional_cost: Final = cost_for("us-east5") + global_handler_cost, global_recomputed_cost = costs_for("global") + regional_handler_cost, regional_recomputed_cost = costs_for("us-east5") - assert global_cost == pytest.approx(10 * 1e-06 + 20 * 2e-06, rel=1e-9) - assert regional_cost == pytest.approx(global_cost * 1.10, rel=1e-9), ( + plain_cost: Final = 10 * 1e-06 + 20 * 2e-06 + assert global_handler_cost == pytest.approx(plain_cost, rel=1e-9) + assert regional_handler_cost == pytest.approx(plain_cost * 1.10, rel=1e-9), ( "regional Vertex passthrough traffic must bill at 1.1x the global rate" ) + assert global_recomputed_cost == pytest.approx(plain_cost, rel=1e-9), ( + "the logging recompute must not price global passthrough traffic as regional" + ) + assert regional_recomputed_cost == pytest.approx(plain_cost * 1.10, rel=1e-9) class TestVertexAIDiscoveryPassThroughHandler: From fc8a6b2a8d8537c3a564fa863a71f001855b5901 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 19 Aug 2026 17:25:09 -0700 Subject: [PATCH 14/24] refactor(ui): migrate shared primitives and common components off antd (#37521) * refactor(ui): migrate shared primitives and common components off antd Adds the success variant to the shared Alert plus success, warning and info variants to Badge, introduces UtcDateTimeInput to replace antd's DatePicker, and converts the common components and key/team helpers onto the shadcn primitives. * fix(ui): keep MultiSelect and budget input faithful to their antd behaviour Restore the clear-all control MultiSelect lost, split comma-separated custom entries into one value per token, and stop rounding the budget input on every keystroke so a fractional amount survives typing. * test(ui): drive the access group picker through the migrated MultiSelect AccessGroupSelector no longer renders an antd Select, so the placeholder is an input label rather than a text node and the popup inerts the page until it closes. --- ui/litellm-dashboard/eslint-suppressions.json | 58 +------------- .../components/UsagePageView.test.tsx | 4 +- .../common_components/AccessGroupSelector.tsx | 55 ++++--------- .../RateLimitTypeFormItem.test.tsx | 34 ++------ .../check_openapi_schema.tsx | 50 ++++++++---- .../user_search_modal.test.tsx | 2 +- .../BudgetFallbacksEditor.tsx | 46 ++++------- .../key_team_helpers/BudgetWindowsEditor.tsx | 53 ++++++++---- .../create_key_button.integration.test.tsx | 9 +-- .../src/components/shared/Alert.test.tsx | 80 +++++++++++++++++++ .../src/components/shared/Alert.tsx | 11 ++- .../components/shared/MultiSelect.test.tsx | 25 ++++++ .../src/components/shared/MultiSelect.tsx | 23 +++++- .../shared/PaginatedSearchSelect.tsx | 4 +- .../src/components/shared/SearchSelect.tsx | 7 ++ .../src/components/ui/badge.tsx | 4 +- .../src/contexts/AntdGlobalProvider.tsx | 7 +- 17 files changed, 266 insertions(+), 206 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/shared/Alert.test.tsx diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 31e8f81f3b3..d625cca0c60 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -542,11 +542,6 @@ "count": 1 } }, - "src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx": { "no-restricted-imports": { "count": 1 @@ -647,9 +642,6 @@ "src/app/(dashboard)/mcp-servers/_components/UserEnvVarsModal.tsx": { "no-nested-ternary": { "count": 2 - }, - "no-restricted-imports": { - "count": 1 } }, "src/app/(dashboard)/mcp-servers/_components/index.tsx": { @@ -661,9 +653,6 @@ "local/filename-pascal-case": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/static-components": { "count": 4 } @@ -1475,11 +1464,6 @@ "count": 1 } }, - "src/components/Settings/AdminSettings/LoggingSettings/LoggingSettings.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings.tsx": { "no-nested-ternary": { "count": 1 @@ -1506,11 +1490,6 @@ "count": 1 } }, - "src/components/Settings/AdminSettings/SSOSettings/RoleMappings.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/Settings/AdminSettings/UISettings/PageVisibilitySettings.tsx": { "react-hooks/set-state-in-render": { "count": 1 @@ -1590,9 +1569,6 @@ "src/components/VirtualKeysPage/keyTableColumns.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 } }, "src/components/activity_metrics.tsx": { @@ -1603,11 +1579,6 @@ "count": 1 } }, - "src/components/add_model/AdaptiveRoutingConfig.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/add_model/AddModelForm.test.tsx": { "no-restricted-imports": { "count": 1 @@ -1620,26 +1591,6 @@ "no-nested-ternary": { "count": 1 }, - "no-restricted-imports": { - "count": 2 - } - }, - "src/components/add_model/ClassificationMethodConfig.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/add_model/ComplexityRouterConfig.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/add_model/EscalationKeywords.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/add_model/KeywordTierRules.tsx": { "no-restricted-imports": { "count": 1 } @@ -1649,11 +1600,6 @@ "count": 1 } }, - "src/components/add_model/SemanticKeywordMatching.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/add_model/add_auto_router_tab.tsx": { "local/filename-pascal-case": { "count": 1 @@ -1672,7 +1618,7 @@ "count": 1 }, "no-restricted-imports": { - "count": 3 + "count": 1 } }, "src/components/add_model/auto_router_connection_test.tsx": { @@ -2078,7 +2024,7 @@ }, "src/components/model_add/CredentialModal.tsx": { "no-restricted-imports": { - "count": 2 + "count": 1 } }, "src/components/model_add/reuse_credentials.tsx": { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx index a6f46ee8242..118371aa9ac 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx @@ -971,8 +971,8 @@ describe("UsagePage", () => { expect(mockUserDailyActivityCall).toHaveBeenCalled(); }); - // Should still render the data from the paginated fallback - expect(screen.getByText("1,500")).toBeInTheDocument(); + // Should still render the data from the paginated fallback, which lands a render after the call + expect(await screen.findByText("1,500")).toBeInTheDocument(); }); it("should stop showing the previous range's paginated pages while a new range is in flight", async () => { diff --git a/ui/litellm-dashboard/src/components/common_components/AccessGroupSelector.tsx b/ui/litellm-dashboard/src/components/common_components/AccessGroupSelector.tsx index 96f21f30385..e522e7c7e62 100644 --- a/ui/litellm-dashboard/src/components/common_components/AccessGroupSelector.tsx +++ b/ui/litellm-dashboard/src/components/common_components/AccessGroupSelector.tsx @@ -1,6 +1,7 @@ import React from "react"; -import { Select, Skeleton } from "antd"; import { Users } from "lucide-react"; +import { Skeleton } from "@/components/ui/skeleton"; +import { MultiSelect, type MultiSelectOption } from "@/components/shared/MultiSelect"; import { useAccessGroups, AccessGroupResponse } from "@/app/(dashboard)/hooks/accessGroups/useAccessGroups"; export interface AccessGroupSelectorProps { @@ -12,8 +13,6 @@ export interface AccessGroupSelectorProps { className?: string; showLabel?: boolean; labelText?: string; - /** Allow clearing the selection */ - allowClear?: boolean; } /** @@ -32,7 +31,6 @@ const AccessGroupSelector: React.FC = ({ className, showLabel = false, labelText = "Access Group", - allowClear = true, }) => { const { data: accessGroups, isLoading, isError } = useAccessGroups(); @@ -45,22 +43,16 @@ const AccessGroupSelector: React.FC = ({ {labelText}

)} - +
); } // ── Build options ──────────────────────────────────────────────────────── - const options = (accessGroups ?? []).map((group: AccessGroupResponse) => ({ - label: ( - - {group.access_group_name}{" "} - ({group.access_group_id}) - - ), + const options: MultiSelectOption[] = (accessGroups ?? []).map((group: AccessGroupResponse) => ({ + label: group.access_group_name, value: group.access_group_id, - selectedLabel: group.access_group_name, - searchText: `${group.access_group_name} ${group.access_group_id}`, + description: group.access_group_id, })); // ── Render ─────────────────────────────────────────────────────────────── @@ -71,30 +63,17 @@ const AccessGroupSelector: React.FC = ({ {labelText}

)} - - {property.enum.map((value) => ( - - {value} - - ))} + ); } if (type === "number" || type === "integer") { return ( - control.onChange(toSchemaNumber(event.target.value, type === "integer"))} + className="w-full" /> ); } diff --git a/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx b/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx index cf4461a2b7a..e320a90f01a 100644 --- a/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx +++ b/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx @@ -74,7 +74,7 @@ describe("UserSearchModal", () => { expect(notice).toHaveTextContent(/users that already exist/i); expect(notice).toHaveTextContent(/ask a proxy admin to create their account first/i); // info, not warning: a warning here would read as an error state on an empty form - expect(notice.className).toMatch(/text-info/); + expect(notice).toHaveAttribute("data-variant", "info"); }); }); diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/BudgetFallbacksEditor.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/BudgetFallbacksEditor.tsx index 1d327b87346..4e1c17a4c68 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/BudgetFallbacksEditor.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/BudgetFallbacksEditor.tsx @@ -1,5 +1,5 @@ -import { SimpleTooltip } from "@/components/ui/tooltip"; -import { Select } from "antd"; +import { MultiSelect } from "@/components/shared/MultiSelect"; +import { SearchSelect } from "@/components/shared/SearchSelect"; import { Button } from "@/components/ui/button"; import { ArrowDown, Plus, X } from "lucide-react"; import React, { useState } from "react"; @@ -94,18 +94,15 @@ export function BudgetFallbacksEditor({ value, onChange, availableModels }: Budg
- updateEntry(entry.id, { fallbackModels: values })} - disabled={!entry.primaryModel} - showSearch - filterOption={(input, option) => (option?.label ?? "").toLowerCase().includes(input.toLowerCase())} + ({ label: m, value: m }))} - getPopupContainer={(trigger) => trigger.parentElement || document.body} - maxTagCount="responsive" - maxTagPlaceholder={(omittedValues) => ( - v).join(", ")}> - +{omittedValues.length} more - - )} + value={entry.fallbackModels} + onValueChange={(values) => updateEntry(entry.id, { fallbackModels: values })} + placeholder={entry.primaryModel ? "Select fallback models" : "Select a primary model first"} + emptyText="No models found" + disabled={!entry.primaryModel} + className="w-full" /> {entry.fallbackModels.length > 1 && (
diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/BudgetWindowsEditor.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/BudgetWindowsEditor.tsx index c9a422dcb85..43861d43c0d 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/BudgetWindowsEditor.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/BudgetWindowsEditor.tsx @@ -1,5 +1,6 @@ -import { InputNumber, Select } from "antd"; import { Button } from "@/components/ui/button"; +import { InputGroup, InputGroupAddon, InputGroupInput, InputGroupText } from "@/components/ui/input-group"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import React from "react"; export interface BudgetWindowEntry { @@ -41,21 +42,43 @@ export function BudgetWindowsEditor({ value, onChange }: BudgetWindowsEditorProp
+ + + $ + + { + const typed = event.target.valueAsNumber; + updateWindow(idx, "max_budget", Number.isNaN(typed) ? null : typed); + }} + onBlur={(event) => { + const typed = event.target.valueAsNumber; + if (!Number.isNaN(typed)) { + updateWindow(idx, "max_budget", Number(typed.toFixed(2))); + } + }} + placeholder="Max spend ($)" + /> + + )} - MCP Logo - + MCP Logo + {isAdmin ? "Add New MCP Server" : "Submit MCP Server for Review"}
+
@@ -687,9 +651,9 @@ const CreateMCPServer: React.FC = ({ label={ MCP Server Name - + - + } name="server_name" @@ -708,9 +672,9 @@ const CreateMCPServer: React.FC = ({ label={ Alias - + - + } name="alias" @@ -765,19 +729,20 @@ const CreateMCPServer: React.FC = ({ > {(control) => ( )} @@ -796,7 +761,7 @@ const CreateMCPServer: React.FC = ({ }} > {(control) => ( - = ({ label={ Max Concurrent Requests (optional) - + - + } name="max_concurrent_requests" > {(control) => ( - )} {/* Authentication - show for HTTP, SSE, and OpenAPI */} {transportType !== "stdio" && transportType !== "" && ( - Authentication, - children: ( - <> - - {(control) => ( - - )} - + + + Authentication settings + + + + + {(control) => ( + + )} + - + - + + {shouldShowAuthValueField && ( + + Authentication Value + + + + + } + name={["credentials", "auth_value"]} + rules={{ + validate: { + notWhitespace: notOnlyWhitespace("Authentication value cannot be empty whitespace"), + }, + }} + > + {(control) => ( + + )} + + )} - {shouldShowAuthValueField && ( - - Authentication Value - - - - - } - name={["credentials", "auth_value"]} - rules={{ - validate: { - notWhitespace: notOnlyWhitespace( - "Authentication value cannot be empty whitespace", - ), - }, - }} - > - {(control) => ( - - )} - - )} + {isOAuthAuthType && ( + + )} - {isOAuthAuthType && ( - - )} + {isTokenExchangeAuthType && } - {isTokenExchangeAuthType && } - - {isIdJagAuthType && } - - ), - }, - ]} - /> + {isIdJagAuthType && } + + )} {transportType !== "stdio" && transportType !== "" && isAwsSigV4AuthType && } @@ -974,9 +918,6 @@ const CreateMCPServer: React.FC = ({ availableAccessGroups={availableAccessGroups} mcpServer={null} mountedAuthType={authSectionMounted ? watchedAuthType : undefined} - searchValue={searchValue} - setSearchValue={setSearchValue} - getAccessGroupOptions={getAccessGroupOptions} />
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx index a18c4592838..acf65992ce1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx @@ -1,7 +1,8 @@ -import React from "react"; -import { Switch, Tooltip } from "antd"; import { Info } from "lucide-react"; +import React from "react"; +import { SimpleTooltip } from "@/components/ui/tooltip"; +import { Switch } from "@/components/ui/switch"; import { MountedFormField } from "@/components/common_components/MountedFormField"; import { isClientForwardedTokenMode } from "@/components/mcp_tools/types"; import { switchControl } from "./mcpFieldRules"; @@ -28,9 +29,9 @@ export default function DcrBridgeToggle({ label={ Gateway-hosted sign-in (DCR bridge) - + - + } name="dcr_bridge" diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx index 01f68d2a531..f6f724b666d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx @@ -1,7 +1,10 @@ -import React from "react"; -import { Input, Select, Tooltip, Typography } from "antd"; -import { Button } from "@/components/ui/button"; import { CircleMinus, Info, Plus } from "lucide-react"; +import React from "react"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { SimpleTooltip } from "@/components/ui/tooltip"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { InputGroup, InputGroupAddon, InputGroupInput } from "@/components/ui/input-group"; import { useFieldArray, useFormContext, useWatch } from "react-hook-form"; import { @@ -10,11 +13,9 @@ import { type MountedFormValues, } from "@/components/common_components/MountedFormField"; import { antdRequired } from "@/components/common_components/antdFormRules"; -import { matchesPattern, selectControl, textControl } from "./mcpFieldRules"; +import { matchesPattern, selectControl, selectTriggerControl, textControl } from "./mcpFieldRules"; import { listControl } from "./mcpFormStore"; -const { Text } = Typography; - const SCOPE_OPTIONS = [ { value: "global", label: "Instance" }, { value: "user", label: "Per-user" }, @@ -40,11 +41,9 @@ const EnvVarsSection: React.FC = () => { return (
- - Variables - - Variables + Define variables you can interpolate in Static Headers or Authentication using{" "} {"${VAR_NAME}"}.
@@ -56,14 +55,14 @@ const EnvVarsSection: React.FC = () => { } > -
+
- + Reference these in Static Headers or Authentication as {"${VAR_NAME}"}. For example:{" "} {"${DB_PROTOCOL}://${CORP_USERNAME}:${CORP_PASSWORD}@${DB_HOSTNAME}"} - +
{fields.length > 0 && ( @@ -97,7 +96,20 @@ const EnvVarsSection: React.FC = () => {
- {(control) => (control)} items={SCOPE_OPTIONS}> + + + + + {SCOPE_OPTIONS.map((option) => ( + + {option.label} + + ))} + + + )}
= ({ index }) => { return ( {(control) => ( - + + + Hint - - } - placeholder="e.g. Your DB username" - styles={{ input: { color: "#9ca3af" } }} - /> + + + + )} ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx index 534f7bdf99b..d4760a65e6c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx @@ -1,10 +1,14 @@ -import React from "react"; -import { Input, Select, Tooltip } from "antd"; import { Info } from "lucide-react"; +import React from "react"; +import { SimpleTooltip } from "@/components/ui/tooltip"; import { MountedFormField } from "@/components/common_components/MountedFormField"; import { antdRequired } from "@/components/common_components/antdFormRules"; -import { requiredUnlessSiblingSet, selectControl, textControl } from "./mcpFieldRules"; +import { MultiSelect } from "@/components/shared/MultiSelect"; +import { PasswordInput } from "@/components/shared/PasswordInput"; +import { Input } from "@/components/ui/input"; +import { Textarea } from "@/components/ui/textarea"; +import { requiredUnlessSiblingSet, tagsControl, textControl } from "./mcpFieldRules"; interface IdJagFormFieldsProps { isEditing?: boolean; @@ -15,9 +19,9 @@ const fieldClassName = "rounded-lg border-gray-300 focus:border-blue-500 focus:r const FieldLabel: React.FC<{ label: string; tooltip: string }> = ({ label, tooltip }) => ( {label} - + - + ); @@ -75,10 +79,10 @@ const IdJagFormFields: React.FC = ({ isEditing = false }) rules={requiredWhenCreating("Client ID is required for ID-JAG")} > {(control) => ( - )} @@ -105,10 +109,10 @@ const IdJagFormFields: React.FC = ({ isEditing = false }) } > {(control) => ( - )} @@ -122,7 +126,7 @@ const IdJagFormFields: React.FC = ({ isEditing = false }) name={PRIVATE_KEY_PATH} > {(control) => ( - = ({ isEditing = false }) label={} name={["credentials", "scopes"]} > - {(control) => ( - )} @@ -64,20 +80,14 @@ const StaticHeadersFieldArray: React.FC = () => { rules={{ validate: { required: antdRequired("Header value is required") } }} > {(valueControl) => ( - + )} remove(index)} className="size-4 text-gray-500 hover:text-red-500 cursor-pointer" /> - +
))}
- } - key="permissions" - className="border-0" - forceRender - > + + + + + + Permission Management / Access Control + + + Configure access permissions and security settings (Optional) + + + + +
Allow All LiteLLM Keys - + - +

Enable if this server should be "public" to all keys.

- {(control) => } + {(control) => }
@@ -211,16 +217,16 @@ const MCPPermissionManagement: React.FC = ({
Internal network only - + - +

Turn on to restrict access to callers within your internal network only.

- {(control) => } + {(control) => }
@@ -229,9 +235,9 @@ const MCPPermissionManagement: React.FC = ({
Delegate auth to upstream (PKCE passthrough) - + - +

Bypass LiteLLM auth so clients authenticate directly with the upstream OAuth MCP server. @@ -242,7 +248,9 @@ const MCPPermissionManagement: React.FC = ({ defaultValue={mcpServer?.delegate_auth_to_upstream ?? false} className="mb-0" > - {(control) => } + {(control) => ( + + )}

)} @@ -252,9 +260,9 @@ const MCPPermissionManagement: React.FC = ({
OAuth pass-through - + - +

Forward upstream OAuth discovery and 401 challenges so clients negotiate OAuth directly with the @@ -266,7 +274,7 @@ const MCPPermissionManagement: React.FC = ({ defaultValue={mcpServer?.oauth_passthrough ?? false} className="mb-0" > - {(control) => } + {(control) => }

)} @@ -287,27 +295,20 @@ const MCPPermissionManagement: React.FC = ({ label={ MCP Access Groups - + - + } name="mcp_access_groups" className="mb-4" > {(control) => ( - 0 ? `Currently: ${mcpServer.extra_headers.join(", ")}` : "Enter header names (e.g., Authorization, X-Custom-Header)" } className="rounded-lg" - size="large" - tokenSeparators={[","]} - allowClear /> )} @@ -349,16 +346,16 @@ const MCPPermissionManagement: React.FC = ({ Static Headers - + - +
- - + + ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx index 549a2d71842..e88c78271d3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx @@ -1,13 +1,24 @@ -import React from "react"; -import { Input as AntdInput, InputNumber, Select, Tooltip } from "antd"; import { Info } from "lucide-react"; +import React from "react"; +import { MultiSelect } from "@/components/shared/MultiSelect"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { SimpleTooltip } from "@/components/ui/tooltip"; +import { PasswordInput } from "@/components/shared/PasswordInput"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; +import { Textarea } from "@/components/ui/textarea"; import { OAUTH_FLOW } from "@/components/mcp_tools/types"; import { MountedFormField } from "@/components/common_components/MountedFormField"; import { antdRequired } from "@/components/common_components/antdFormRules"; import TokenEndpointAuthMethodField from "./TokenEndpointAuthMethodField"; -import { numberControl, parsesAsJson, selectControl, textControl } from "./mcpFieldRules"; +import { + numberControl, + parsesAsJson, + selectControl, + selectTriggerControl, + tagsControl, + textControl, +} from "./mcpFieldRules"; interface OAuthFlowStatus { startOAuthFlow: () => void; @@ -27,6 +38,11 @@ interface OAuthFormFieldsProps { const fieldClassName = "rounded-lg border-gray-300 focus:border-blue-500 focus:ring-blue-500"; +const OAUTH_FLOW_ITEMS = [ + { value: OAUTH_FLOW.M2M, label: "Machine-to-Machine (M2M)" }, + { value: OAUTH_FLOW.INTERACTIVE, label: "Interactive (PKCE)" }, +]; + const UPSTREAM_RESOURCE_TOOLTIP = "RFC 8707 resource indicator sent to the authorization server so it mints a token audienced for this MCP server. " + "Leave blank to send nothing, which is the default and what most providers expect. Use 'auto' to send this server's " + @@ -37,9 +53,9 @@ const UPSTREAM_RESOURCE_TOOLTIP = const FieldLabel: React.FC<{ label: string; tooltip: string }> = ({ label, tooltip }) => ( {label} - + - + ); @@ -78,19 +94,24 @@ const OAuthFormFields: React.FC = ({ {...(initialFlowType ? { defaultValue: initialFlowType } : {})} > {(control) => ( - (control)} items={OAUTH_FLOW_ITEMS}> + + + + + +
+ Machine-to-Machine (M2M) + server-to-server, no user interaction +
+
+ +
+ Interactive (PKCE) + browser-based user authorization +
+
+
)} @@ -104,10 +125,10 @@ const OAuthFormFields: React.FC = ({ rules={requiredWhenCreating("Client ID is required for M2M OAuth")} > {(control) => ( - )} @@ -120,10 +141,10 @@ const OAuthFormFields: React.FC = ({ rules={requiredWhenCreating("Client Secret is required for M2M OAuth")} > {(control) => ( - )} @@ -151,16 +172,7 @@ const OAuthFormFields: React.FC = ({ } name={["credentials", "scopes"]} > - {(control) => ( - - )} + {(control) => } = ({ rules={{ validate: { json: parsesAsJson("Must be valid JSON") } }} > {(control) => ( - = ({ name="token_storage_ttl_seconds" > {(control) => ( - + )} {oauthFlow && ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenAPIFormSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenAPIFormSection.tsx index 9d4dced9759..8907595d92a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenAPIFormSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenAPIFormSection.tsx @@ -1,6 +1,7 @@ -import React, { useState } from "react"; -import { Input, Tooltip } from "antd"; import { Info } from "lucide-react"; +import React, { useState } from "react"; +import { SimpleTooltip } from "@/components/ui/tooltip"; +import { Input } from "@/components/ui/input"; import { AUTH_TYPE, OAUTH_FLOW } from "@/components/mcp_tools/types"; import { MountedFormField } from "@/components/common_components/MountedFormField"; import { antdRequired } from "@/components/common_components/antdFormRules"; @@ -68,9 +69,9 @@ const OpenAPIFormSection: React.FC = ({ label={ OpenAPI Spec URL - + - + } name="spec_path" diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx index f46040ff831..982998c0aae 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx @@ -1,10 +1,13 @@ -import React from "react"; -import { Input, Select, Switch, Tooltip } from "antd"; import { Info } from "lucide-react"; +import React from "react"; +import { MultiSelect } from "@/components/shared/MultiSelect"; +import { SimpleTooltip } from "@/components/ui/tooltip"; import { useWatch } from "react-hook-form"; +import { Input } from "@/components/ui/input"; +import { Switch } from "@/components/ui/switch"; import { MountedFormField } from "@/components/common_components/MountedFormField"; -import { selectControl, switchControl, textControl } from "./mcpFieldRules"; +import { switchControl, tagsControl, textControl } from "./mcpFieldRules"; const AUTH_HEADER_FORMATS: Readonly> = { bearer_token: "Authorization: Bearer {key}", @@ -25,9 +28,9 @@ const OpenApiByokFields: React.FC = () => { label={ BYOK (Bring Your Own Key) - + - + } name="is_byok" @@ -61,20 +64,18 @@ const OpenApiByokFields: React.FC = () => { label={ Access Description - + - + } name="byok_description" > {(control) => ( - - )} + {(control) => { + const placeholder = isEditing + ? "Leave blank to keep existing (default Client Secret Post)" + : "Default (Client Secret Post)"; + return ( + + ); + }} ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenExchangeFormFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenExchangeFormFields.tsx index 7a9f8e70e4b..5aaa013aba4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenExchangeFormFields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenExchangeFormFields.tsx @@ -1,11 +1,15 @@ -import React from "react"; -import { Input, Select, Tooltip } from "antd"; import { Info } from "lucide-react"; +import React from "react"; +import { MultiSelect } from "@/components/shared/MultiSelect"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { SimpleTooltip } from "@/components/ui/tooltip"; import { useWatch } from "react-hook-form"; import { MountedFormField } from "@/components/common_components/MountedFormField"; import { antdRequired } from "@/components/common_components/antdFormRules"; -import { selectControl, textControl } from "./mcpFieldRules"; +import { PasswordInput } from "@/components/shared/PasswordInput"; +import { Input } from "@/components/ui/input"; +import { selectControl, selectTriggerControl, tagsControl, textControl } from "./mcpFieldRules"; interface TokenExchangeFormFieldsProps { isEditing?: boolean; @@ -13,12 +17,17 @@ interface TokenExchangeFormFieldsProps { const fieldClassName = "rounded-lg border-gray-300 focus:border-blue-500 focus:ring-blue-500"; +const TOKEN_EXCHANGE_PROFILE_ITEMS = [ + { value: "rfc8693", label: "RFC 8693 (standard)" }, + { value: "entra_obo", label: "Microsoft Entra OBO" }, +]; + const FieldLabel: React.FC<{ label: string; tooltip: string }> = ({ label, tooltip }) => ( {label} - + - + ); @@ -41,13 +50,17 @@ const TokenExchangeFormFields: React.FC = ({ isEdi {...(isEditing ? {} : { defaultValue: "rfc8693" })} > {(control) => ( - (control)} items={TOKEN_EXCHANGE_PROFILE_ITEMS}> + + + + + {TOKEN_EXCHANGE_PROFILE_ITEMS.map((item) => ( + + {item.label} + + ))} + )} @@ -80,10 +93,10 @@ const TokenExchangeFormFields: React.FC = ({ isEdi rules={requiredWhenCreating("Client ID is required for token exchange")} > {(control) => ( - )} @@ -99,10 +112,10 @@ const TokenExchangeFormFields: React.FC = ({ isEdi rules={requiredWhenCreating("Client Secret is required for token exchange")} > {(control) => ( - )} @@ -164,13 +177,10 @@ const TokenExchangeFormFields: React.FC = ({ isEdi } > {(control) => ( - (control)} - onChange={(value: string) => { - control.onChange(value); - handleTransportChange(value); - }} + items={TRANSPORT_ITEMS} + value={(control.value as string | undefined) ?? null} + onValueChange={handleTransportSelected(control.onChange)} > - Streamable HTTP (Recommended) - Server-Sent Events (SSE) - Standard Input/Output (stdio) - OpenAPI Spec + + + + + {TRANSPORT_ITEMS.map((item) => ( + + {item.label} + + ))} + )} @@ -873,9 +863,9 @@ const MCPServerEdit: React.FC = ({ label={ OpenAPI Spec URL - + - + } name="spec_path" @@ -896,21 +886,20 @@ const MCPServerEdit: React.FC = ({ label={ Max Concurrent Requests (optional) - + - + } name="max_concurrent_requests" > {(control) => ( - )} @@ -925,20 +914,17 @@ const MCPServerEdit: React.FC = ({ rules={{ validate: { required: antdRequired("Authentication is required") } }} > {(control) => ( - (control)} items={AUTH_TYPE_ITEMS}> + + + + + {AUTH_TYPE_ITEMS.map((item) => ( + + {item.label} + + ))} + )} @@ -984,11 +970,8 @@ const MCPServerEdit: React.FC = ({ {(control) => ( - onChange(Array.from(e.target.selectedOptions, (o) => (o as HTMLOptionElement).value))} - > - {children} - - ); - Select.displayName = "MockSelect"; - Select.Option = ({ value, disabled, label }: any) => ( - - ); - Select.Option.displayName = "MockSelectOption"; - return { ...actual, Select }; -}); - import { useMCPAccessGroups } from "@/app/(dashboard)/hooks/mcpServers/useMCPAccessGroups"; import { useMCPServers } from "@/app/(dashboard)/hooks/mcpServers/useMCPServers"; import { useMCPToolsets } from "@/app/(dashboard)/hooks/mcpServers/useMCPToolsets"; @@ -45,6 +23,15 @@ const mockUseMCPServers = vi.mocked(useMCPServers); const mockUseMCPAccessGroups = vi.mocked(useMCPAccessGroups); const mockUseMCPToolsets = vi.mocked(useMCPToolsets); +// The dropdown mounts its list asynchronously, so opening means waiting for the options too. +const openSelector = async (user: ReturnType): Promise => { + await user.click(screen.getByRole("combobox")); + await screen.findAllByRole("option"); +}; + +const optionByLabel = (label: string): HTMLElement | undefined => + screen.queryAllByRole("option").find((option) => option.textContent?.startsWith(label)); + const setupMcpMocks = () => { mockUseMCPServers.mockReturnValue({ data: [{ server_id: "srv-1", server_name: "Server One" }], @@ -60,19 +47,20 @@ describe("MCPServerSelector no-mcp-servers option", () => { setupMcpMocks(); }); - const optionByValue = (value: string) => - Array.from(screen.getByTestId("mcp-select").querySelectorAll("option")).find( - (o) => (o as HTMLOptionElement).value === value, - ) as HTMLOptionElement | undefined; - - it("hides the No MCP Servers option by default", () => { + it("hides the No MCP Servers option by default", async () => { + const user = userEvent.setup(); renderWithProviders( , ); - expect(optionByValue(NO_MCP_SERVERS_SENTINEL)).toBeUndefined(); + + await openSelector(user); + + expect(optionByLabel("Server One")).toBeDefined(); + expect(optionByLabel("No MCP Servers")).toBeUndefined(); }); it("emits an exclusive sentinel when No MCP Servers is selected", async () => { + const user = userEvent.setup(); const onChange = vi.fn(); renderWithProviders( { value={{ servers: ["srv-1"], accessGroups: [] }} />, ); - expect(optionByValue(NO_MCP_SERVERS_SENTINEL)).toBeDefined(); - await userEvent.selectOptions(screen.getByTestId("mcp-select"), [NO_MCP_SERVERS_SENTINEL]); + await openSelector(user); + await user.click(optionByLabel("No MCP Servers")!); expect(onChange).toHaveBeenCalledWith({ servers: [NO_MCP_SERVERS_SENTINEL], accessGroups: [], toolsets: [] }); }); - it("disables real server options while the sentinel is selected", () => { + it("disables real server options while the sentinel is selected", async () => { + const user = userEvent.setup(); renderWithProviders( { value={{ servers: [NO_MCP_SERVERS_SENTINEL], accessGroups: [] }} />, ); - expect(optionByValue("srv-1")?.disabled).toBe(true); - expect(optionByValue(NO_MCP_SERVERS_SENTINEL)?.disabled).toBe(false); + + await openSelector(user); + + expect(optionByLabel("Server One")).toHaveAttribute("aria-disabled", "true"); + expect(optionByLabel("No MCP Servers")).not.toHaveAttribute("aria-disabled", "true"); }); }); @@ -109,19 +101,20 @@ describe("MCPServerSelector all-proxy-mcpservers option", () => { setupMcpMocks(); }); - const optionByValue = (value: string) => - Array.from(screen.getByTestId("mcp-select").querySelectorAll("option")).find( - (o) => (o as HTMLOptionElement).value === value, - ) as HTMLOptionElement | undefined; - - it("hides the All Proxy MCP Servers option by default", () => { + it("hides the All Proxy MCP Servers option by default", async () => { + const user = userEvent.setup(); renderWithProviders( , ); - expect(optionByValue(ALL_PROXY_MCP_SERVERS_SENTINEL)).toBeUndefined(); + + await openSelector(user); + + expect(optionByLabel("Server One")).toBeDefined(); + expect(optionByLabel("All Proxy MCP Servers")).toBeUndefined(); }); it("emits an exclusive sentinel when All Proxy MCP Servers is selected", async () => { + const user = userEvent.setup(); const onChange = vi.fn(); renderWithProviders( { value={{ servers: ["srv-1"], accessGroups: [] }} />, ); - expect(optionByValue(ALL_PROXY_MCP_SERVERS_SENTINEL)).toBeDefined(); - await userEvent.selectOptions(screen.getByTestId("mcp-select"), [ALL_PROXY_MCP_SERVERS_SENTINEL]); + await openSelector(user); + await user.click(optionByLabel("All Proxy MCP Servers")!); expect(onChange).toHaveBeenCalledWith({ servers: [ALL_PROXY_MCP_SERVERS_SENTINEL], @@ -142,7 +135,8 @@ describe("MCPServerSelector all-proxy-mcpservers option", () => { }); }); - it("disables real server options while the sentinel is selected", () => { + it("disables real server options while the sentinel is selected", async () => { + const user = userEvent.setup(); renderWithProviders( { value={{ servers: [ALL_PROXY_MCP_SERVERS_SENTINEL], accessGroups: [] }} />, ); - expect(optionByValue("srv-1")?.disabled).toBe(true); - expect(optionByValue(ALL_PROXY_MCP_SERVERS_SENTINEL)?.disabled).toBe(false); + + await openSelector(user); + + expect(optionByLabel("Server One")).toHaveAttribute("aria-disabled", "true"); + expect(optionByLabel("All Proxy MCP Servers")).not.toHaveAttribute("aria-disabled", "true"); }); - it("renders the friendly option, not the raw literal, when the sentinel is already stored but the flag is off", () => { + it("renders the friendly label, not the raw literal, when the sentinel is already stored but the flag is off", async () => { + const user = userEvent.setup(); renderWithProviders( { value={{ servers: [ALL_PROXY_MCP_SERVERS_SENTINEL], accessGroups: [] }} />, ); - const option = optionByValue(ALL_PROXY_MCP_SERVERS_SENTINEL); - expect(option).toBeDefined(); - expect(option?.textContent).toContain("All Proxy MCP Servers"); - expect(optionByValue("srv-1")?.disabled).toBe(true); + + expect(screen.getByLabelText("All Proxy MCP Servers")).toBeInTheDocument(); + expect(screen.queryByText(ALL_PROXY_MCP_SERVERS_SENTINEL)).not.toBeInTheDocument(); + + await openSelector(user); + + expect(optionByLabel("Server One")).toHaveAttribute("aria-disabled", "true"); }); }); diff --git a/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx b/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx index f31f60d2084..969db57462f 100644 --- a/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx +++ b/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx @@ -1,7 +1,7 @@ import { useMCPAccessGroups } from "@/app/(dashboard)/hooks/mcpServers/useMCPAccessGroups"; import { useMCPServers } from "@/app/(dashboard)/hooks/mcpServers/useMCPServers"; import { useMCPToolsets } from "@/app/(dashboard)/hooks/mcpServers/useMCPToolsets"; -import { Select } from "antd"; +import { MultiSelect, type MultiSelectOption } from "@/components/shared/MultiSelect"; import React from "react"; import { ALL_PROXY_MCP_SERVERS_SENTINEL, NO_MCP_SERVERS_SENTINEL } from "@/components/mcp_tools/constants"; @@ -42,39 +42,25 @@ const MCPServerSelector: React.FC = ({ const accessGroupSet = new Set(accessGroups); - // Combine options: access groups (green) + servers (blue) + toolsets (purple) + // Combine options: access groups + servers + toolsets const options = [ ...accessGroups.map((group) => ({ label: group, value: group, - type: "accessGroup" as const, - searchText: `${group} Access Group`, + description: "Access Group", })), ...mcpServers.map((server) => ({ label: `${server.server_name || server.server_id} (${server.server_id})`, value: server.server_id, - type: "server" as const, - searchText: `${server.server_name || server.server_id} ${server.server_id} MCP Server`, + description: "MCP Server", })), ...toolsets.map((toolset) => ({ label: toolset.toolset_name, value: `${TOOLSET_PREFIX}${toolset.toolset_id}`, - type: "toolset" as const, - searchText: `${toolset.toolset_name} ${toolset.toolset_id} Toolset`, + description: "Toolset", })), ]; - const colorByType: Record = { - accessGroup: "#52c41a", - server: "#1890ff", - toolset: "#722ed1", - }; - const labelByType: Record = { - accessGroup: "Access Group", - server: "MCP Server", - toolset: "Toolset", - }; - // Flatten value for Select — prefix toolset IDs const selectedValues = [ ...(value?.servers || []), @@ -105,76 +91,31 @@ const MCPServerSelector: React.FC = ({ onChange({ servers, accessGroups: accessGroupsSelected, toolsets: toolsetsSelected }); }; + const selectOptions: MultiSelectOption[] = [ + ...(allowAllProxyMcpServers || hasAllProxyMcpServersSelected + ? [{ label: "All Proxy MCP Servers", value: ALL_PROXY_MCP_SERVERS_SENTINEL }] + : []), + ...(allowNoMcpServers + ? [{ label: "No MCP Servers", value: NO_MCP_SERVERS_SENTINEL, description: "Block all" }] + : []), + ...options.map((opt) => ({ + ...opt, + disabled: hasNoMcpServersSelected || hasAllProxyMcpServersSelected, + })), + ]; + return (
- + className={`w-full ${className ?? ""}`} + />
); }; diff --git a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx index 2e8402f52eb..f0b211aecb3 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx @@ -1,10 +1,10 @@ import React, { forwardRef, useImperativeHandle, useMemo } from "react"; -import { Select } from "antd"; import { CircleHelp } from "lucide-react"; import { useForm, type Resolver } from "react-hook-form"; import { FieldGroup } from "@/components/shared/form/field"; import { FormField } from "@/components/shared/form/FormField"; import { Input } from "@/components/ui/input"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Textarea } from "@/components/ui/textarea"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { MCPTool, InputSchema, InputSchemaProperty } from "./types"; @@ -13,6 +13,11 @@ type ToolFormValues = Record; const STRING_SCHEMA_MESSAGES: Readonly> = { input: "Please enter input for this tool" }; +const BOOLEAN_ITEMS = [ + { value: true, label: "True" }, + { value: false, label: "False" }, +]; + const isBlank = (value: unknown): boolean => value === undefined || value === null || value === ""; const jsonErrorFor = (prop: InputSchemaProperty, value: unknown): string | null => { @@ -325,33 +330,43 @@ const MCPToolArgumentsForm = forwardRef { if (prop.type === "string" && prop.enum) { return ( - + + + + + {!required && Select {key}} + {prop.enum.map((v) => ( + + {v} + + ))} + + ); } if (prop.type === "boolean") { return ( - + + + + + {!required && Select {key}} + True + False + + ); } if (prop.type === "number" || prop.type === "integer") { diff --git a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx index 8d0d6dd0b89..734c158ea42 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx @@ -46,6 +46,20 @@ export const AUTH_TYPE = { OAUTH_DELEGATE: "oauth_delegate", }; +export const AUTH_TYPE_ITEMS = [ + { value: AUTH_TYPE.NONE, label: "None" }, + { value: AUTH_TYPE.API_KEY, label: "API Key" }, + { value: AUTH_TYPE.BEARER_TOKEN, label: "Bearer Token" }, + { value: AUTH_TYPE.TOKEN, label: "Token" }, + { value: AUTH_TYPE.BASIC, label: "Basic Auth" }, + { value: AUTH_TYPE.OAUTH2, label: "OAuth" }, + { value: AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE, label: "OAuth Token Exchange (OBO)" }, + { value: AUTH_TYPE.OAUTH2_ID_JAG, label: "ID-JAG (Okta Cross App Access)" }, + { value: AUTH_TYPE.AWS_SIGV4, label: "AWS SigV4 (Bedrock AgentCore MCPs)" }, + { value: AUTH_TYPE.TRUE_PASSTHROUGH, label: "True Passthrough (no LiteLLM auth)" }, + { value: AUTH_TYPE.OAUTH_DELEGATE, label: "OAuth Delegate (client-supplied upstream token)" }, +]; + // The two client-forwarded token modes: the caller supplies the upstream Authorization (forwarded // verbatim for true_passthrough, alongside LiteLLM admission for oauth_delegate). The dashboard holds // their token in sessionStorage instead of persisting it, and the browser-authorize temp payload keeps @@ -250,6 +264,13 @@ export const TRANSPORT = { OPENAPI: "openapi", }; +export const TRANSPORT_ITEMS = [ + { value: TRANSPORT.HTTP, label: "Streamable HTTP (Recommended)" }, + { value: TRANSPORT.SSE, label: "Server-Sent Events (SSE)" }, + { value: TRANSPORT.STDIO, label: "Standard Input/Output (stdio)" }, + { value: TRANSPORT.OPENAPI, label: "OpenAPI Spec" }, +]; + export const handleTransport = (transport?: string | null, specPath?: string | null): string => { if (transport === null || transport === undefined) { return TRANSPORT.SSE; diff --git a/ui/litellm-dashboard/src/components/shared/MultiSelect.test.tsx b/ui/litellm-dashboard/src/components/shared/MultiSelect.test.tsx index 04921974810..cc4084a952e 100644 --- a/ui/litellm-dashboard/src/components/shared/MultiSelect.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/MultiSelect.test.tsx @@ -102,6 +102,24 @@ describe("MultiSelect", () => { expect(onValueChange).toHaveBeenCalledWith(["udp", "kafka", "terraform"]); }); + it("leaves an already selected value that contains a comma alone when a later entry is added", async () => { + const onValueChange = vi.fn(); + render( + , + ); + + await userEvent.type(screen.getByRole("combobox"), "--verbose"); + await userEvent.click(await screen.findByText('Create "--verbose"')); + + expect(onValueChange).toHaveBeenCalledWith(["--filter=a,b", "--verbose"]); + }); + it("clears every selection at once", async () => { const onValueChange = vi.fn(); render( diff --git a/ui/litellm-dashboard/src/components/shared/MultiSelect.tsx b/ui/litellm-dashboard/src/components/shared/MultiSelect.tsx index 31ca4e099f0..3a153ff2a5f 100644 --- a/ui/litellm-dashboard/src/components/shared/MultiSelect.tsx +++ b/ui/litellm-dashboard/src/components/shared/MultiSelect.tsx @@ -89,7 +89,7 @@ export function MultiSelect({ const handleValueChange = (selected: MultiSelectOption[]) => { const next = allowCustomValues - ? selected.flatMap((option) => splitOnCommas(option.value)) + ? selected.flatMap((option) => (value.includes(option.value) ? [option.value] : splitOnCommas(option.value))) : selected.map((option) => option.value); onValueChange(Array.from(new Set(next))); setQuery(""); From 0ab172575703387896d998c108bd32e6ca4bf57f Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 19 Aug 2026 18:19:09 -0700 Subject: [PATCH 17/24] refactor(ui): migrate the model and router settings pages off antd (#37523) * refactor(ui): migrate the model and router settings pages off antd Converts the add model flow, credential panels, model settings and router settings onto the shadcn primitives, moves the mapping table onto the shared DataTable, and drops the dead uploadProps prop chain that only existed to carry antd's UploadProps type. * fix(ui): split comma-separated custom technical keywords into one term each --- ui/litellm-dashboard/eslint-suppressions.json | 131 ------- .../panels/AddModelPanel.integration.test.tsx | 14 +- .../panels/AddModelPanel.tsx | 4 - .../panels/LlmCredentialsPanel.tsx | 12 +- .../vertexCredentialsUpload.test.ts | 47 --- .../vertexCredentialsUpload.ts | 37 -- .../src/components/ModelInfoEditForm.tsx | 27 +- .../Fallbacks/EditFallbacks.test.tsx | 17 +- .../Fallbacks/FallbackGroupConfig.tsx | 80 ++--- .../Fallbacks/FallbackSelectionForm.test.tsx | 18 +- .../Fallbacks/FallbackSelectionForm.tsx | 82 +++-- .../add_model/AdaptiveRoutingConfig.tsx | 110 +++--- .../add_model/AddModelForm.test.tsx | 40 ++- .../src/components/add_model/AddModelForm.tsx | 5 +- .../add_model/ClassificationMethodConfig.tsx | 294 ++++++++-------- .../add_model/ComplexityRouterConfig.test.tsx | 60 ++-- .../add_model/ComplexityRouterConfig.tsx | 319 +++++++++--------- .../add_model/EscalationKeywords.tsx | 26 +- .../components/add_model/KeywordTierRules.tsx | 138 ++++---- .../add_model/SemanticKeywordMatching.tsx | 50 +-- .../add_model/add_auto_router_tab.test.tsx | 95 +++--- .../add_model/add_auto_router_tab.tsx | 96 +++--- .../add_model/advanced_settings.tsx | 146 ++++---- .../conditional_public_model_name.tsx | 31 +- .../add_model/litellm_model_name.tsx | 36 +- .../provider_specific_fields.test.tsx | 65 +++- .../add_model/provider_specific_fields.tsx | 150 ++++---- .../edit_auto_router_modal.test.tsx | 22 +- .../model_add/CredentialModal.test.tsx | 14 +- .../components/model_add/CredentialModal.tsx | 5 +- .../model_add/CredentialsPanel.test.tsx | 5 +- .../components/model_add/CredentialsPanel.tsx | 10 +- .../ModelSettingsModal.test.tsx | 4 +- .../ModelSettingsModal/ModelSettingsModal.tsx | 30 +- .../src/components/model_info_view.test.tsx | 23 +- .../RouterSettingsForm.test.tsx | 26 +- .../RoutingStrategySelector.test.tsx | 53 ++- .../RoutingStrategySelector.tsx | 34 +- .../components/router_settings/index.test.tsx | 52 +-- .../components/shared/SearchSelect.test.tsx | 7 + .../shared/form/UtcDateTimeInput.tsx | 53 +++ .../src/components/ui/combobox.tsx | 3 +- 42 files changed, 1153 insertions(+), 1318 deletions(-) delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/vertexCredentialsUpload.test.ts delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/vertexCredentialsUpload.ts create mode 100644 ui/litellm-dashboard/src/components/shared/form/UtcDateTimeInput.tsx diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index cafdb2fa7dc..2d68ea2aa68 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -57,9 +57,6 @@ "local/no-complex-jsx-arrow": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/immutability": { "count": 1 } @@ -199,11 +196,6 @@ "count": 1 } }, - "src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx": { "no-nested-ternary": { "count": 3 @@ -550,11 +542,6 @@ "count": 1 } }, - "src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.test.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx": { "no-nested-ternary": { "count": 1 @@ -570,11 +557,6 @@ "count": 1 } }, - "src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.test.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/mcp-servers/_components/ToolTestPanel.tsx": { "react-hooks/set-state-in-effect": { "count": 1 @@ -782,11 +764,6 @@ "count": 1 } }, - "src/app/(dashboard)/playground/components/compareUI/components/ModelSelector.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/playground/components/complianceUI/ComplianceUI.tsx": { "local/no-complex-jsx-arrow": { "count": 2 @@ -987,9 +964,6 @@ "local/filename-pascal-case": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/immutability": { "count": 1 } @@ -1257,11 +1231,6 @@ "count": 1 } }, - "src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx": { - "no-restricted-imports": { - "count": 2 - } - }, "src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx": { "no-nested-ternary": { "count": 2 @@ -1400,9 +1369,6 @@ "no-nested-ternary": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 1 } @@ -1437,18 +1403,7 @@ "count": 1 } }, - "src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx": { - "local/no-complex-jsx-arrow": { - "count": 1 - }, - "no-restricted-imports": { - "count": 1 - } - }, "src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx": { - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 1 } @@ -1488,11 +1443,6 @@ "count": 1 } }, - "src/components/UsagePage/components/EntityUsage/TopKeyView.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/UsagePage/utils/value_formatters.tsx": { "local/filename-pascal-case": { "count": 1 @@ -1511,11 +1461,6 @@ "count": 1 } }, - "src/components/add_model/AddModelForm.test.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/add_model/AddModelForm.tsx": { "local/no-complex-jsx-arrow": { "count": 1 @@ -1548,9 +1493,6 @@ "src/components/add_model/advanced_settings.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 } }, "src/components/add_model/auto_router_connection_test.tsx": { @@ -1570,9 +1512,6 @@ "local/no-complex-jsx-arrow": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 2 } @@ -1593,9 +1532,6 @@ }, "no-nested-ternary": { "count": 1 - }, - "no-restricted-imports": { - "count": 2 } }, "src/components/add_model/model_connection_test.tsx": { @@ -1613,9 +1549,6 @@ "no-nested-ternary": { "count": 3 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/immutability": { "count": 3 } @@ -1625,15 +1558,7 @@ "count": 1 } }, - "src/components/agent_management/AgentSelector.test.tsx": { - "react/display-name": { - "count": 1 - } - }, "src/components/agent_management/AgentSelector.tsx": { - "no-restricted-imports": { - "count": 1 - }, "prefer-const": { "count": 1 } @@ -1724,11 +1649,6 @@ "count": 1 } }, - "src/components/common_components/AccessGroupSelector.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/common_components/DeleteResourceModal.tsx": { "react-hooks/set-state-in-effect": { "count": 1 @@ -1739,16 +1659,6 @@ "count": 1 } }, - "src/components/common_components/MetadataKeyValueFields.test.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/common_components/MetadataKeyValueFields.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/common_components/ModelAliasManager.tsx": { "react-hooks/set-state-in-effect": { "count": 1 @@ -1759,11 +1669,6 @@ "count": 1 } }, - "src/components/common_components/RateLimitTypeFormItem.test.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/common_components/budget_duration_dropdown.tsx": { "local/filename-pascal-case": { "count": 1 @@ -1772,9 +1677,6 @@ "src/components/common_components/check_openapi_schema.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "no-restricted-imports": { - "count": 2 } }, "src/components/common_components/fetch_teams.tsx": { @@ -1845,16 +1747,6 @@ "count": 1 } }, - "src/components/key_team_helpers/BudgetFallbacksEditor.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/key_team_helpers/BudgetWindowsEditor.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/key_team_helpers/fetch_available_models_team_key.tsx": { "local/filename-pascal-case": { "count": 1 @@ -1953,11 +1845,6 @@ "count": 1 } }, - "src/components/model_dashboard/ModelSettingsModal/ModelSettingsModal.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/model_filters.tsx": { "local/filename-pascal-case": { "count": 1 @@ -2111,11 +1998,6 @@ "count": 1 } }, - "src/components/router_settings/RoutingStrategySelector.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/router_settings/index.tsx": { "local/filename-pascal-case": { "count": 1 @@ -2264,11 +2146,6 @@ "count": 1 } }, - "src/components/team/LoggingSettings.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/team/TeamInfo.tsx": { "max-lines": { "count": 1 @@ -2276,9 +2153,6 @@ "no-nested-ternary": { "count": 3 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 1 } @@ -2614,11 +2488,6 @@ "count": 2 } }, - "src/contexts/AntdGlobalProvider.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/contexts/AuthContext.tsx": { "react-hooks/set-state-in-effect": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx index b762f006261..19e1e3aa8bd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx @@ -219,7 +219,7 @@ describe("AddModelPanel submit payload contract", () => { const { user, openAdvanced, fillRequired, submit } = await setup(); await fillRequired(); await openAdvanced(); - await user.click(screen.getByLabelText("Custom Pricing")); + await user.click(screen.getByRole("switch", { name: "Custom Pricing" })); await user.type(await screen.findByLabelText("Input Cost (per 1M tokens)"), "3"); await user.type(screen.getByLabelText("Output Cost (per 1M tokens)"), "9"); await submit(); @@ -241,7 +241,7 @@ describe("AddModelPanel submit payload contract", () => { const { user, openAdvanced, fillRequired, submit } = await setup(); await fillRequired(); await openAdvanced(); - await user.click(screen.getByLabelText("Cache Control Injection Points")); + await user.click(screen.getByRole("switch", { name: "Cache Control Injection Points" })); await screen.findByText("Add Injection Point"); await submit(); @@ -260,7 +260,7 @@ describe("AddModelPanel submit payload contract", () => { const { user, openAdvanced, fillRequired, submit } = await setup(); await fillRequired(); await openAdvanced(); - await user.click(screen.getByLabelText("Cache Control Injection Points")); + await user.click(screen.getByRole("switch", { name: "Cache Control Injection Points" })); await screen.findByText("Add Injection Point"); await user.click(screen.getByText("Select a role")); await user.click(await screen.findByText("System")); @@ -385,7 +385,7 @@ describe("AddModelPanel behaviours the removed Advanced Settings form instance n const { user, openAdvanced, fillRequired, submit } = await setup(); await fillRequired(); await openAdvanced(); - await user.click(screen.getByLabelText("Use in pass through routes")); + await user.click(screen.getByRole("switch", { name: "Use in pass through routes" })); expect(screen.getByLabelText("LiteLLM Params")).toHaveValue(""); await submit(); @@ -401,11 +401,11 @@ describe("AddModelPanel behaviours the removed Advanced Settings form instance n const { user, openAdvanced, fillRequired, submit } = await setup(); await fillRequired(); await openAdvanced(); - await user.click(screen.getByLabelText("Custom Pricing")); + await user.click(screen.getByRole("switch", { name: "Custom Pricing" })); await user.type(await screen.findByLabelText("Input Cost (per 1M tokens)"), "3"); - await user.click(screen.getByLabelText("Custom Pricing")); + await user.click(screen.getByRole("switch", { name: "Custom Pricing" })); await waitFor(() => expect(screen.queryByLabelText("Input Cost (per 1M tokens)")).not.toBeInTheDocument()); - await user.click(screen.getByLabelText("Custom Pricing")); + await user.click(screen.getByRole("switch", { name: "Custom Pricing" })); expect(await screen.findByLabelText("Input Cost (per 1M tokens)")).toHaveValue("3"); await submit(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx index 59d4f95c038..1443c065d9b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx @@ -15,7 +15,6 @@ import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap" import { useCredentials } from "@/app/(dashboard)/hooks/credentials/useCredentials"; import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { vertexCredentialsUploadProps } from "@/app/(dashboard)/models-and-endpoints/vertexCredentialsUpload"; const INITIAL_VALUES: MountedFormValues = { litellm_credential_name: null }; @@ -60,9 +59,6 @@ export default function AddModelPanel() { providerModels={providerModels} setProviderModelsFn={(provider) => setProviderModels(getProviderModels(provider, modelCostMapData))} getPlaceholder={getPlaceholder} - uploadProps={vertexCredentialsUploadProps({ - setFieldsValue: (values) => form.setValue("vertex_credentials", values.vertex_credentials), - })} showAdvancedSettings={showAdvancedSettings} setShowAdvancedSettings={setShowAdvancedSettings} teams={teams ?? null} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx index 7251da7c3c3..c71ed177416 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx @@ -1,17 +1,7 @@ "use client"; -import { useForm } from "react-hook-form"; import CredentialsPanel from "@/components/model_add/CredentialsPanel"; -import type { MountedFormValues } from "@/components/common_components/MountedFormField"; -import { vertexCredentialsUploadProps } from "@/app/(dashboard)/models-and-endpoints/vertexCredentialsUpload"; export default function LlmCredentialsPanel() { - const form = useForm(); - return ( - form.setValue("vertex_credentials", values.vertex_credentials), - })} - /> - ); + return ; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/vertexCredentialsUpload.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/vertexCredentialsUpload.test.ts deleted file mode 100644 index e02e4974353..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/vertexCredentialsUpload.test.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { waitFor } from "@testing-library/react"; -import { beforeEach, describe, expect, it, vi } from "vitest"; - -import { toast } from "@/lib/toast"; - -import { vertexCredentialsUploadProps } from "./vertexCredentialsUpload"; - -const makeForm = () => ({ setFieldsValue: vi.fn() }); - -describe("vertexCredentialsUploadProps", () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it("reads a JSON credential file into the vertex_credentials field without uploading it", async () => { - const form = makeForm(); - const props = vertexCredentialsUploadProps(form as never); - const file = new File(['{"project_id":"example"}'], "vertex.json", { type: "application/json" }); - - expect(props.beforeUpload?.(file as never, [file] as never)).toBe(false); - - await waitFor(() => { - expect(form.setFieldsValue).toHaveBeenCalledWith({ vertex_credentials: '{"project_id":"example"}' }); - }); - }); - - it("ignores non-JSON files", async () => { - const form = makeForm(); - const props = vertexCredentialsUploadProps(form as never); - const file = new File(["not json"], "vertex.txt", { type: "text/plain" }); - - expect(props.beforeUpload?.(file as never, [file] as never)).toBe(false); - - await new Promise((resolve) => setTimeout(resolve, 0)); - expect(form.setFieldsValue).not.toHaveBeenCalled(); - }); - - it("reports completed and failed upload states", () => { - const props = vertexCredentialsUploadProps(makeForm() as never); - - props.onChange?.({ file: { name: "vertex.json", status: "done" } } as never); - props.onChange?.({ file: { name: "vertex.json", status: "error" } } as never); - - expect(toast.success).toHaveBeenCalledWith("vertex.json file uploaded successfully"); - expect(toast.fromError).toHaveBeenCalledWith("vertex.json file upload failed."); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/vertexCredentialsUpload.ts b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/vertexCredentialsUpload.ts deleted file mode 100644 index f658ec88c1c..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/vertexCredentialsUpload.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { ComponentProps } from "react"; - -import { toast } from "@/lib/toast"; -import type CredentialsPanel from "@/components/model_add/CredentialsPanel"; - -interface VertexCredentialsForm { - setFieldsValue: (values: { vertex_credentials: string }) => void; -} - -type UploadProps = ComponentProps["uploadProps"]; - -export function vertexCredentialsUploadProps(form: VertexCredentialsForm): UploadProps { - return { - name: "file", - accept: ".json", - pastable: false, - beforeUpload: (file) => { - if (file.type === "application/json") { - const reader = new FileReader(); - reader.onload = (event) => { - if (event.target) { - form.setFieldsValue({ vertex_credentials: event.target.result as string }); - } - }; - reader.readAsText(file); - } - return false; - }, - onChange(info) { - if (info.file.status === "done") { - toast.success(`${info.file.name} file uploaded successfully`); - } else if (info.file.status === "error") { - toast.fromError(`${info.file.name} file upload failed.`); - } - }, - }; -} diff --git a/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx b/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx index fff732fe270..d56e65237eb 100644 --- a/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx +++ b/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx @@ -1,8 +1,6 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; -// eslint-disable-next-line no-restricted-imports -- the dashboard has no shadcn date-time picker; the PTU window fields need one -import { DatePicker } from "antd"; import { CircleHelp } from "lucide-react"; import type { Dayjs } from "dayjs"; import * as React from "react"; @@ -11,6 +9,7 @@ import { z } from "zod/v4"; import { TagsInput } from "@/app/(dashboard)/guardrails/_components/content_filter/TagsInput"; import { FormField } from "@/components/shared/form/FormField"; +import { UtcDateTimeInput } from "@/components/shared/form/UtcDateTimeInput"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; @@ -292,9 +291,16 @@ const Display: React.FC<{ children: React.ReactNode }> = ({ children }) => (
{children}
); -const FieldLabel: React.FC<{ children: React.ReactNode }> = ({ children }) => ( -

{children}

-); +const FIELD_LABEL_CLASS = "text-sm font-medium text-foreground"; + +const FieldLabel: React.FC<{ htmlFor?: string; children: React.ReactNode }> = ({ htmlFor, children }) => + htmlFor === undefined ? ( +

{children}

+ ) : ( + + ); const Hint: React.FC<{ text: string }> = ({ text }) => ( @@ -463,13 +469,14 @@ const ModelInfoEditForm: React.FC = ({ {ptuCostAttributionEnabled && PTU_EDIT_FIELDS.map((ptuField) => (
- {ptuField.label} + {ptuField.label} {isEditing ? ( {({ value, onChange, ...control }) => ptuField.input === "number" ? ( = ({ min={ptuField.isCount ? 1 : 0} /> ) : ( - ) diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/EditFallbacks.test.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/EditFallbacks.test.tsx index 225c308af91..c0db38eb187 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/EditFallbacks.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/EditFallbacks.test.tsx @@ -1,5 +1,5 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { render, screen, waitFor } from "@testing-library/react"; +import { render, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import EditFallbacks, { Fallbacks } from "./EditFallbacks"; @@ -49,10 +49,9 @@ describe("EditFallbacks", () => { it("prefills the existing fallback chain for the primary model", async () => { setup(); - await waitFor(() => { - expect(screen.getByText("gpt-3.5-turbo")).toBeInTheDocument(); - expect(screen.getByText("claude-3-opus")).toBeInTheDocument(); - }); + const chain = await screen.findByRole("list", { name: "Fallback chain" }); + expect(within(chain).getByText("gpt-3.5-turbo")).toBeInTheDocument(); + expect(within(chain).getByText("claude-3-opus")).toBeInTheDocument(); }); it("removes a fallback model and saves only the edited entry", async () => { @@ -61,8 +60,8 @@ describe("EditFallbacks", () => { const onClose = vi.fn(); setup({ onChange, onClose }); - await screen.findByText("gpt-3.5-turbo"); - await user.click(screen.getByTestId("remove-fallback-gpt-3.5-turbo")); + const chain = await screen.findByRole("list", { name: "Fallback chain" }); + await user.click(within(chain).getByRole("button", { name: "Remove gpt-3.5-turbo" })); await user.click(screen.getByRole("button", { name: /save changes/i })); @@ -77,8 +76,8 @@ describe("EditFallbacks", () => { const onChange = vi.fn().mockResolvedValue(undefined); setup({ fallbackEntry: { "gpt-4": ["gpt-3.5-turbo"] }, onChange }); - await screen.findByText("gpt-3.5-turbo"); - await user.click(screen.getByTestId("remove-fallback-gpt-3.5-turbo")); + const chain = await screen.findByRole("list", { name: "Fallback chain" }); + await user.click(within(chain).getByRole("button", { name: "Remove gpt-3.5-turbo" })); const saveButton = screen.getByRole("button", { name: /save changes/i }); expect(saveButton).toBeDisabled(); diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx index e3a36f4fbf2..129f509dcd9 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx @@ -3,10 +3,10 @@ * Handles primary model selection and fallback chain configuration */ -import { SimpleTooltip } from "@/components/ui/tooltip"; -import { Select } from "antd"; +import { MultiSelect } from "@/components/shared/MultiSelect"; +import { SearchSelect } from "@/components/shared/SearchSelect"; import { AlertCircle, ArrowDown, X } from "lucide-react"; -import React from "react"; +import React, { useId } from "react"; export interface FallbackGroup { id: string; @@ -64,25 +64,24 @@ export function FallbackGroupConfig({ }; const canAddMoreFallbacks = group.fallbackModels.length < maxFallbacks; + const primaryModelInputId = useId(); return (
{/* Primary Model Section */}
-
) : ( - group.fallbackModels.map((modelValue, index) => { - return ( -
+ {group.fallbackModels.map((modelValue, index) => ( +
  • @@ -182,15 +154,15 @@ export function FallbackGroupConfig({ -
  • - ); - }) + + ))} + )}
    diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.test.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.test.tsx index ac9b4d98aed..738040149f0 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.test.tsx @@ -1,4 +1,4 @@ -import { render, screen } from "@testing-library/react"; +import { render, screen, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { FallbackSelectionForm } from "./FallbackSelectionForm"; @@ -69,7 +69,7 @@ describe("FallbackSelectionForm", () => { , ); - const addTabButton = screen.getByRole("button", { name: /add tab/i }); + const addTabButton = screen.getByRole("button", { name: /add fallback group/i }); await user.click(addTabButton); expect(mockOnGroupsChange).toHaveBeenCalledTimes(1); @@ -98,7 +98,7 @@ describe("FallbackSelectionForm", () => { maxGroups={5} />, ); - expect(screen.queryByRole("button", { name: /add tab/i })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /add fallback group/i })).not.toBeInTheDocument(); }); it("should show add tab button when below maxGroups with custom maxGroups", () => { @@ -111,7 +111,7 @@ describe("FallbackSelectionForm", () => { maxGroups={3} />, ); - expect(screen.getByRole("button", { name: /add tab/i })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /add fallback group/i })).toBeInTheDocument(); }); it("should call onGroupsChange when a group is removed", async () => { @@ -124,7 +124,8 @@ describe("FallbackSelectionForm", () => { , ); - const removeButtons = screen.getAllByRole("tab", { name: "remove" }); + const removeButtons = screen.getAllByRole("button", { name: /^remove /i }); + expect(removeButtons).toHaveLength(2); await user.click(removeButtons[0]); expect(mockOnGroupsChange).toHaveBeenCalledTimes(1); @@ -139,7 +140,7 @@ describe("FallbackSelectionForm", () => { render( , ); - expect(screen.getByText("Select primary model")).toBeInTheDocument(); + expect(screen.getByRole("combobox", { name: /primary model/i })).toHaveValue(""); expect(screen.getByText("Primary Model")).toBeInTheDocument(); }); @@ -150,7 +151,8 @@ describe("FallbackSelectionForm", () => { ); expect(screen.getByRole("tab", { name: "gpt-4" })).toBeInTheDocument(); expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); - expect(screen.getByText("gpt-3.5-turbo")).toBeInTheDocument(); + const chain = screen.getByRole("list", { name: "Fallback chain" }); + expect(within(chain).getByText("gpt-3.5-turbo")).toBeInTheDocument(); }); it("should not add group when add button clicked at maxGroups", () => { @@ -168,7 +170,7 @@ describe("FallbackSelectionForm", () => { />, ); - expect(screen.queryByRole("button", { name: /add tab/i })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /add fallback group/i })).not.toBeInTheDocument(); expect(mockOnGroupsChange).not.toHaveBeenCalled(); }); }); diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx index 10bbcd9ba3b..2639171fbe8 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx @@ -5,8 +5,8 @@ */ import { Button } from "@/components/ui/button"; -import { Tabs } from "antd"; -import { Plus } from "lucide-react"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { Plus, X } from "lucide-react"; import React, { useEffect, useState } from "react"; import { toast } from "@/lib/toast"; import { FallbackGroup, FallbackGroupConfig } from "./FallbackGroupConfig"; @@ -76,23 +76,8 @@ export function FallbackSelectionForm({ onGroupsChange(newGroups); }; - // Generate tab items - const items = groups.map((group, index) => { - const label = group.primaryModel ? group.primaryModel : `Group ${index + 1}`; - return { - key: group.id, - label: label, - closable: groups.length > 1, // Only allow closing if there's more than 1 group - children: ( - - ), - }; - }); + const groupLabel = (group: FallbackGroup, index: number) => + group.primaryModel ? group.primaryModel : `Group ${index + 1}`; if (groups.length === 0) { return ( @@ -107,22 +92,47 @@ export function FallbackSelectionForm({ } return ( - { - if (action === "add") handleAddGroup(); - else if (action === "remove" && groups.length > 1) { - handleRemoveGroup(targetKey as string); - } - }} - items={items} - className="fallback-tabs" - tabBarStyle={{ - marginBottom: 0, - }} - hideAdd={groups.length >= maxGroups} - /> + +
    + + {groups.map((group, index) => ( +
    + 1 ? "pr-9" : "pr-4"}`} + > + {groupLabel(group, index)} + + {groups.length > 1 && ( + + )} +
    + ))} +
    + {groups.length < maxGroups && ( + + )} +
    + {groups.map((group) => ( + + + + ))} +
    ); } diff --git a/ui/litellm-dashboard/src/components/add_model/AdaptiveRoutingConfig.tsx b/ui/litellm-dashboard/src/components/add_model/AdaptiveRoutingConfig.tsx index 720b6f88db3..137ef297d56 100644 --- a/ui/litellm-dashboard/src/components/add_model/AdaptiveRoutingConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AdaptiveRoutingConfig.tsx @@ -1,4 +1,9 @@ -import { Card, InputNumber, Radio, Slider, Space, Switch, Typography } from "antd"; +import { Card, CardContent } from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"; +import { Slider } from "@/components/ui/slider"; +import { Switch } from "@/components/ui/switch"; import React from "react"; import { AdaptiveEligible, @@ -7,8 +12,6 @@ import { DEFAULT_TIER_DISTANCE_PENALTY, } from "./ComplexityRouterConfig"; -const { Text } = Typography; - interface AdaptiveRoutingConfigProps { value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; @@ -45,84 +48,91 @@ const AdaptiveRoutingConfig: React.FC = ({ value, on return ( <> -
    - - Enable adaptive bandit selection -
    - + + When disabled, each request always uses the model assigned to its classified tier. - + - - How Adaptive Routing Works - - - It learns from how each conversation actually goes: does the user have to rephrase or correct the model, does - it get stuck repeating itself, does it run out of tool calls, does the user seem satisfied. Combined with - cost, this live feedback shifts future routing toward the models that are actually working well, and improves - as more conversations come in. Until there's enough feedback, it defaults to the classified tier's - model. - + + How Adaptive Routing Works + + It learns from how each conversation actually goes: does the user have to rephrase or correct the model, + does it get stuck repeating itself, does it run out of tool calls, does the user seem satisfied. Combined + with cost, this live feedback shifts future routing toward the models that are actually working well, and + improves as more conversations come in. Until there's enough feedback, it defaults to the classified + tier's model. + + {value.adaptive && (
    - + Quality vs. Cost ({Math.round(adaptiveWeights.quality * 100)}% quality /{" "} {Math.round(adaptiveWeights.cost * 100)}% cost) - + `${v}% quality / ${100 - (v ?? 0)}% cost` }} + value={[Math.round(adaptiveWeights.quality * 100)]} + onValueChange={(next) => handleQualityWeightChange(Array.isArray(next) ? next[0] : next)} /> - + Higher quality weight favors more capable (pricier) models; higher cost weight favors cheaper models when the bandit has feedback to act on. Recommended: 30% quality / 70% cost split. - +
    - - Eligible Model Pool - - Eligible Model Pool + handleAdaptiveEligibleChange(e.target.value)} + onValueChange={(eligible: unknown) => handleAdaptiveEligibleChange(eligible as AdaptiveEligible)} className="w-full" > - - - All tiers (soft floor){" "} - — router can pick across tiers, depending on the best fit for the prompt - - - Classified tier only{" "} - — router can only pick models within tier - - - +
    + + +
    +
    {adaptiveEligible === "all" && (
    - - Tier Distance Penalty - - Tier Distance Penalty + + handleTierDistancePenaltyChange(event.target.value === "" ? null : event.target.valueAsNumber) + } min={0} step={0.1} - style={{ width: "100%" }} + className="w-full" /> - + Score penalty applied per tier-step away from the classified tier. - +
    )}
    diff --git a/ui/litellm-dashboard/src/components/add_model/AddModelForm.test.tsx b/ui/litellm-dashboard/src/components/add_model/AddModelForm.test.tsx index 26bd9af94a8..340fd811e97 100644 --- a/ui/litellm-dashboard/src/components/add_model/AddModelForm.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AddModelForm.test.tsx @@ -1,6 +1,5 @@ import { renderHook, screen, waitFor, renderWithProviders } from "../../../tests/test-utils"; import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event"; -import type { UploadProps } from "antd/es/upload"; import { describe, expect, it, vi } from "vitest"; import type { Team } from "../key_team_helpers/key_list"; import type { CredentialItem } from "../networking"; @@ -157,11 +156,6 @@ const createTestProps = (userRole = "proxy_admin", userId = "user-1", isTeamAdmi }, ]; - const uploadProps: UploadProps = { - beforeUpload: () => false, - showUploadList: false, - }; - return { form, registry, @@ -176,7 +170,6 @@ const createTestProps = (userRole = "proxy_admin", userId = "user-1", isTeamAdmi showAdvancedSettings: false, teams, credentials, - uploadProps, userRole, userId, }; @@ -318,6 +311,35 @@ describe("AddModelForm", () => { expect(await screen.findByRole("button", { name: "Add Model" })).toBeInTheDocument(); }); + describe("the enterprise gate on the Team-BYOK switch", () => { + const renderForm = async (premiumUser: boolean) => { + const mockUseAuthorized = vi.mocked(await import("@/app/(dashboard)/hooks/useAuthorized")); + mockUseAuthorized.default.mockReturnValue(mockAuthorizedUser("proxy_admin", "user-1", premiumUser)); + renderWithProviders(); + return screen.findByRole("switch", { name: "Team-BYOK Model" }); + }; + + it("explains the gate on hover even though the switch it sits on is disabled", async () => { + const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never }); + const teamOnlySwitch = await renderForm(false); + expect(teamOnlySwitch).toHaveAttribute("aria-disabled", "true"); + + await user.hover(teamOnlySwitch); + + expect(await screen.findByText(/enterprise-only feature/)).toBeInTheDocument(); + }); + + it("says nothing on hover once the user is premium", async () => { + const user = userEvent.setup(); + const teamOnlySwitch = await renderForm(true); + expect(teamOnlySwitch).not.toHaveAttribute("aria-disabled", "true"); + + await user.hover(teamOnlySwitch); + + expect(screen.queryByText(/enterprise-only feature/)).not.toBeInTheDocument(); + }); + }); + describe("cache control bindings reach the parent form store", () => { const renderWithForm = async () => { const mockUseAuthorized = vi.mocked(await import("@/app/(dashboard)/hooks/useAuthorized")); @@ -331,11 +353,11 @@ describe("AddModelForm", () => { user, openCacheControl: async () => { await user.click(await screen.findByText("Advanced Settings")); - await user.click(screen.getByLabelText("Cache Control Injection Points")); + await user.click(screen.getByRole("switch", { name: "Cache Control Injection Points" })); await screen.findByText("Add Injection Point"); }, closeCacheControl: async () => { - await user.click(screen.getByLabelText("Cache Control Injection Points")); + await user.click(screen.getByRole("switch", { name: "Cache Control Injection Points" })); await waitFor(() => expect(screen.queryByText("Add Injection Point")).not.toBeInTheDocument()); }, mountedValues: async (): Promise> => props.mountedValues(), diff --git a/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx b/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx index 65efd6aed51..b6dddf43588 100644 --- a/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx @@ -9,7 +9,6 @@ import { Select as AntdSelect, Card, Col, Row, Tooltip, Typography } from "antd" import { Info } from "lucide-react"; import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { Button } from "@/components/ui/button"; -import type { UploadProps } from "antd/es/upload"; import React, { useEffect, useMemo, useState } from "react"; import { FormProvider, useWatch, type UseFormReturn } from "react-hook-form"; import TeamDropdown from "../common_components/team_dropdown"; @@ -44,7 +43,6 @@ interface AddModelFormProps { providerModels: string[]; setProviderModelsFn: (provider: Providers) => void; getPlaceholder: (provider: Providers) => string; - uploadProps: UploadProps; showAdvancedSettings: boolean; setShowAdvancedSettings: (show: boolean) => void; teams: Team[] | null; @@ -71,7 +69,6 @@ const AddModelForm: React.FC = ({ providerModels, setProviderModelsFn, getPlaceholder, - uploadProps, showAdvancedSettings, setShowAdvancedSettings, teams, @@ -311,7 +308,7 @@ const AddModelForm: React.FC = ({ OR
    - + )}
    diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index 019df7d4fa9..3c384aed1b3 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -1,6 +1,13 @@ import { Info } from "lucide-react"; import { SimpleTooltip } from "@/components/ui/tooltip"; -import { Select as AntdSelect, Card, InputNumber, Radio, Space, Switch, Typography } from "antd"; +import { MultiSelect } from "@/components/shared/MultiSelect"; +import { SearchSelect } from "@/components/shared/SearchSelect"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Card, CardContent } from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"; +import { Switch } from "@/components/ui/switch"; import React from "react"; import ClassifierPromptEditor from "./ClassifierPromptEditor"; import HeuristicScoringConfig from "./HeuristicScoringConfig"; @@ -21,8 +28,6 @@ import { effectiveTierLabel, } from "./ComplexityRouterConfig"; -const { Text } = Typography; - const DEFAULT_SCORING_EXPLANATION = "The router scores each request across 7 dimensions: token count, code presence, reasoning markers, technical " + "terms, simple indicators, multi-step patterns, and question complexity. The weighted score determines the tier:"; @@ -87,36 +92,35 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> = return ( - - How Classification Works - - - {scoringExplanation(value)} - - {ranges && ( -
      -
    • - {effectiveTierLabel("SIMPLE", value.tier_labels)}: Score < {ranges.simpleMedium} -
    • -
    • - {effectiveTierLabel("MEDIUM", value.tier_labels)}: Score {ranges.simpleMedium} -{" "} - {ranges.mediumComplex} -
    • -
    • - {effectiveTierLabel("COMPLEX", value.tier_labels)}: Score {ranges.mediumComplex} -{" "} - {ranges.complexReasoning} -
    • -
    • - {effectiveTierLabel("REASONING", value.tier_labels)}: Score > {ranges.complexReasoning}{" "} - (or 2+ reasoning markers with a score of at least {ranges.reasoningOverrideFloor}) -
    • -
    - )} - {!ranges && isError && ( - - The tier score ranges could not be loaded from the proxy. - - )} + + How Classification Works + {scoringExplanation(value)} + {ranges && ( +
      +
    • + {effectiveTierLabel("SIMPLE", value.tier_labels)}: Score < {ranges.simpleMedium} +
    • +
    • + {effectiveTierLabel("MEDIUM", value.tier_labels)}: Score {ranges.simpleMedium} -{" "} + {ranges.mediumComplex} +
    • +
    • + {effectiveTierLabel("COMPLEX", value.tier_labels)}: Score {ranges.mediumComplex} -{" "} + {ranges.complexReasoning} +
    • +
    • + {effectiveTierLabel("REASONING", value.tier_labels)}: Score >{" "} + {ranges.complexReasoning} (or 2+ reasoning markers with a score of at least{" "} + {ranges.reasoningOverrideFloor}) +
    • +
    + )} + {!ranges && isError && ( + + The tier score ranges could not be loaded from the proxy. + + )} +
    ); }; @@ -247,61 +251,64 @@ const ClassificationMethodConfig: React.FC = ({ return ( <> - handleClassifierTypeChange(e.target.value)} + onValueChange={(classifierType: unknown) => handleClassifierTypeChange(classifierType as ClassifierType)} className="w-full" > - - - Heuristic{" "} - (default) — rule-based scoring, no API calls, <1ms latency - - - LLM Classifier{" "} - — use a model to decide the tier (e.g. a small/fast model) - - - +
    + + +
    + {value.classifier_type === "llm" && (
    - - Classifier Model - - Classifier Model + - {classifierModelMissing && ( - - A classifier model is required - - )} + {classifierModelMissing && A classifier model is required}
    - - Timeout (ms) - - Timeout (ms) + + handleClassifierTimeoutChange(event.target.value === "" ? null : event.target.valueAsNumber) + } min={1} - style={{ width: "100%" }} + className="w-full" /> - + How long the classifier call has before it fails and the fallback below takes over. - +
    - Classification Rubric + Classification Rubric @@ -310,28 +317,37 @@ const ClassificationMethodConfig: React.FC = ({ content={usesCustomPrompt ? "Your custom prompt replaces the built-in rubric entirely" : undefined} className="w-full" > - ({ + - + {usesCustomPrompt ? "Not in use: the custom prompt below is the classifier's entire rubric." : CLASSIFICATION_RUBRIC_DESCRIPTIONS[classificationRubric].description} - +
    - - Classifier Prompt - + Classifier Prompt = ({ />
    - - If the classifier fails - - If the classifier fails + handleClassifierFallbackChange(e.target.value)} + onValueChange={(fallback: unknown) => handleClassifierFallbackChange(fallback as ClassifierFallback)} > - - - Score with the heuristic{" "} - — right when the classifier grades complexity too - - +
    + + +
    +
    + Applies when the classifier call errors, times out, or returns an unparseable response. - +
    - - Context Window Size - - Context Window Size + + handleClassifierContextWindowSizeChange(event.target.value === "" ? null : event.target.valueAsNumber) + } min={0} - style={{ width: "100%" }} + className="w-full" /> - + Number of prior user turns (tool output and harness reminders excluded) sent to the classifier as context, so a referring follow-up like "now do the same for the streaming path" is classified against what it refers to. Set to 0 to send only the current message. - +
    - - Context Per-Turn Character Limit - - Context Per-Turn Character Limit + + handleClassifierContextPerTurnCharsChange(event.target.value === "" ? null : event.target.valueAsNumber) + } min={1} - style={{ width: "100%" }} + className="w-full" /> - - Prior turns longer than this are truncated. - + Prior turns longer than this are truncated.
    - Include Assistant Turns + Include Assistant Turns
    - + Let the classifier read the assistant's replies, so difficulty the model stated rather than the user stays visible: a plan the assistant calls complex, approved with "yes", is classified on the work being approved. Context Window Size then counts the last N turns across both roles rather than the last N user turns. - +
    )} @@ -429,25 +449,29 @@ const ClassificationMethodConfig: React.FC = ({ {value.classifier_type === "heuristic" && (
    - Custom Technical Keywords + Custom Technical Keywords
    - + Optional: Add terms to the built-in list to improve classification accuracy on the technical dimension. (e.g., udp, kafka, terraform). - - + ({ label: keyword, value: keyword }))} value={customTechnicalKeywords ?? []} - onChange={(keywords: string[]) => onCustomTechnicalKeywordsChange?.(keywords)} - placeholder="Type a keyword and press Enter, or paste a comma-separated list" - tokenSeparators={[","]} - open={false} - suffixIcon={null} - style={{ width: "100%" }} - allowClear + onValueChange={(keywords: string[]) => + onCustomTechnicalKeywordsChange?.( + Array.from( + new Set(keywords.flatMap((keyword) => keyword.split(",").map((part) => part.trim())).filter(Boolean)), + ), + ) + } + placeholder="Type a keyword and press Enter" + emptyText="Type to add a keyword" + allowCustomValues + className="w-full" />
    )} diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index d0fda9962e5..5f5ae703b0e 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -280,11 +280,28 @@ describe("ComplexityRouterConfig", () => { ); fireEvent.click(screen.getByText("Advanced: Classification Method")); const keywordsSection = screen.getByText("Custom Technical Keywords").closest("div")?.parentElement as HTMLElement; - const input = within(keywordsSection).getByRole("combobox"); - fireEvent.change(input, { target: { value: "udp," } }); + await user.type(within(keywordsSection).getByRole("combobox"), "udp"); + await user.click(await screen.findByText('Create "udp"')); expect(onCustomTechnicalKeywordsChange).toHaveBeenCalledWith(["udp"]); }); + it("splits a comma-separated keyword entry into one keyword per token", async () => { + const user = userEvent.setup(); + const onCustomTechnicalKeywordsChange = vi.fn(); + renderWithProviders( + , + ); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + const keywordsSection = screen.getByText("Custom Technical Keywords").closest("div")?.parentElement as HTMLElement; + await user.type(within(keywordsSection).getByRole("combobox"), "udp, kafka ,terraform"); + await user.click(await screen.findByText('Create "udp, kafka ,terraform"')); + expect(onCustomTechnicalKeywordsChange).toHaveBeenCalledWith(["udp", "kafka", "terraform"]); + }); + it("should render an empty state when no keyword tier rules exist", () => { renderWithProviders(); fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); @@ -314,9 +331,7 @@ describe("ComplexityRouterConfig", () => { expect(newRules[0]).toMatchObject({ keywords: [], tier: "COMPLEX" }); }); - // The dropdown is closed, so antd has nothing for Enter to select and the word would only land - // on blur. Submitting used to provide that blur; it no longer can while the row reads as empty. - it("commits a typed keyword on Enter, with the dropdown closed", async () => { + it("commits a typed keyword to the rule it was typed into", async () => { const user = userEvent.setup(); const onKeywordTierRulesChange = vi.fn(); renderWithProviders( @@ -329,7 +344,8 @@ describe("ComplexityRouterConfig", () => { fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); const field = screen.getByText("Keywords 1").closest("div") as HTMLElement; - await user.type(within(field).getByRole("combobox"), "invoice{enter}"); + await user.type(within(field).getByRole("combobox"), "invoice"); + await user.click(await screen.findByText('Create "invoice"')); expect(onKeywordTierRulesChange).toHaveBeenCalledWith([{ id: "rule-1", keywords: ["invoice"], tier: "COMPLEX" }]); }); @@ -482,7 +498,7 @@ describe("ComplexityRouterConfig classifier fallback", () => { }; renderWithProviders(); fireEvent.click(screen.getByText("Advanced: Classification Method")); - expect(screen.getByRole("radio", { name: /Route to the default model/ })).toBeDisabled(); + expect(screen.getByRole("radio", { name: /Route to the default model/ })).toHaveAttribute("aria-disabled", "true"); }); it("hides the fallback choice for the heuristic classifier, which has nothing to fall back from", () => { @@ -584,8 +600,8 @@ describe("ComplexityRouterConfig classifier rubric", () => { it("records the chat preset the operator picks", async () => { const onChange = openClassificationPanel(llmValue); - fireEvent.mouseDown(screen.getByRole("combobox", { name: "Classification Rubric" })); - await userEvent.click(await screen.findByTitle("Chat")); + await userEvent.click(screen.getByRole("combobox", { name: "Classification Rubric" })); + await userEvent.click(await screen.findByRole("option", { name: "Chat" })); expect(onChange).toHaveBeenCalledWith( expect.objectContaining({ classifier_llm_config: expect.objectContaining({ classification_rubric: "chat" }) }), ); @@ -678,7 +694,7 @@ describe("ComplexityRouterConfig tier labels", () => { />, ); fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); - expect(screen.getByTitle("Deep")).toBeInTheDocument(); + expect(screen.getByRole("combobox", { name: "Route keyword rule 1 to tier" })).toHaveTextContent("Deep"); }); }); @@ -716,7 +732,7 @@ describe("ComplexityRouterConfig default model", () => { it("shows what the tiers currently imply, so an untouched router still names its default", () => { renderWithProviders(); - expect(screen.getByText("Derived from tiers: gpt-3.5-turbo")).toBeInTheDocument(); + expect(getDefaultModelSelect()).toHaveAttribute("placeholder", "Derived from tiers: gpt-3.5-turbo"); }); it("asks for a model rather than naming a derived one when no tier holds one", () => { @@ -725,7 +741,7 @@ describe("ComplexityRouterConfig default model", () => { tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }, }; renderWithProviders(); - expect(screen.getByText("Add a model to the Simple or Medium tier")).toBeInTheDocument(); + expect(getDefaultModelSelect()).toHaveAttribute("placeholder", "Add a model to the Simple or Medium tier"); }); it("records a pinned model", async () => { @@ -734,7 +750,7 @@ describe("ComplexityRouterConfig default model", () => { renderWithProviders(); await user.click(getDefaultModelSelect()); - await user.click((await screen.findAllByTitle("claude-3-opus")).slice(-1)[0]); + await user.click(await screen.findByRole("option", { name: "claude-3-opus" })); expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ default_model: "claude-3-opus" })); }); @@ -745,8 +761,7 @@ describe("ComplexityRouterConfig default model", () => { const pinned: ComplexityRouterConfigValue = { ...defaultValue, default_model: "claude-3-opus" }; renderWithProviders(); - // eslint-disable-next-line local/no-antd-class-selectors -- antd marks the clear affordance aria-hidden, so no accessible query reaches it - await user.click(document.querySelector(".ant-select-clear") as HTMLElement); + await user.click(screen.getByRole("button", { name: "Clear" })); expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ default_model: undefined })); }); @@ -754,10 +769,7 @@ describe("ComplexityRouterConfig default model", () => { it("shows a pinned model as the selection instead of the tier-derived one", () => { const pinned: ComplexityRouterConfigValue = { ...defaultValue, default_model: "claude-3-opus" }; renderWithProviders(); - expect( - // eslint-disable-next-line local/no-antd-class-selectors -- the tier selects show the same model as a tag, so the assertion has to scope to this select's root, which antd exposes only as a class - within(getDefaultModelSelect().closest(".ant-select") as HTMLElement).getByTitle("claude-3-opus"), - ).toBeInTheDocument(); + expect(getDefaultModelSelect()).toHaveValue("claude-3-opus"); }); it("unlocks the default model fallback on a pin alone, with no tier to derive from", () => { @@ -770,7 +782,7 @@ describe("ComplexityRouterConfig default model", () => { }; renderWithProviders(); fireEvent.click(screen.getByText("Advanced: Classification Method")); - expect(screen.getByRole("radio", { name: /Route to the default model/ })).toBeEnabled(); + expect(screen.getByRole("radio", { name: /Route to the default model/ })).not.toHaveAttribute("aria-disabled"); }); it("names the resolved default on the fallback option, so the destination is not a guess", () => { @@ -827,9 +839,9 @@ describe("plan-mode override", () => { />, ); openPanel(); - fireEvent.mouseDown(await screen.findByRole("combobox", { name: "Plan-mode minimum tier" })); - expect(await screen.findByTitle("Medium")).toBeInTheDocument(); - expect(screen.queryByTitle("Reasoning")).not.toBeInTheDocument(); + await userEvent.click(await screen.findByRole("combobox", { name: "Plan-mode minimum tier" })); + expect(await screen.findByRole("option", { name: "Medium" })).toBeInTheDocument(); + expect(screen.queryByRole("option", { name: "Reasoning" })).not.toBeInTheDocument(); }); it("disables the toggle until some tier has models", async () => { @@ -840,6 +852,6 @@ describe("plan-mode override", () => { />, ); openPanel(); - expect(await screen.findByRole("switch", { name: switchName })).toBeDisabled(); + expect(await screen.findByRole("switch", { name: switchName })).toHaveAttribute("aria-disabled", "true"); }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index ad5260c2c9c..ab6b1d401ce 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -1,6 +1,13 @@ -import { Info } from "lucide-react"; import { SimpleTooltip } from "@/components/ui/tooltip"; -import { Select as AntdSelect, Card, Collapse, Divider, Input, Space, Switch, Typography } from "antd"; +import { MultiSelect } from "@/components/shared/MultiSelect"; +import { SearchSelect } from "@/components/shared/SearchSelect"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { ChevronRight, Info, X } from "lucide-react"; +import { Switch } from "@/components/ui/switch"; +import { Card, CardContent } from "@/components/ui/card"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; +import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; +import { Separator } from "@/components/ui/separator"; import React from "react"; import { ModelGroup } from "@/components/llm_calls/fetch_models"; import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig"; @@ -13,8 +20,6 @@ import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from export type { DimensionWeights, TierBoundaries, TokenThresholds }; -const { Text } = Typography; - export const DEFAULT_CLASSIFIER_TIMEOUT_MS = 3000; export const DEFAULT_TIER_DISTANCE_PENALTY = 0.5; export const DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE = 3; @@ -218,6 +223,9 @@ const ComplexityRouterConfig: React.FC = ({ showValidationErrors = false, }) => { const planModeTiers = planModeEligibleTiers(value.tiers); + const planModeTierOptions = tierOptions(value.tier_labels).filter((option) => + (planModeTiers as string[]).includes(option.value), + ); const derivedDefaultModel = resolveComplexityDefaultModel(value.tiers); const defaultModel = resolveComplexityDefaultModel(value.tiers, value.default_model); @@ -251,128 +259,119 @@ const ComplexityRouterConfig: React.FC = ({ return (
    - - - Complexity Tier Configuration - +
    +

    Complexity Tier Configuration

    - +
    - + The complexity router automatically classifies requests by complexity using rule-based scoring (no API calls, <1ms latency). Configure which model(s) handle each tier. - + - + Rename a tier to use your own vocabulary in the dashboard and your spend logs. Renaming doesn't change how requests are classified, and callers never see these names. {value.classifier_type === "llm" && " Your classifier model reads these names, so clearer ones can sharpen its choices."} - + - {TIER_KEYS.map((tier, index) => { - const tierInfo = TIER_DESCRIPTIONS[tier]; - const label = effectiveTierLabel(tier, value.tier_labels); - const tierMissing = showValidationErrors && value.tiers[tier].length === 0; - return ( -
    - {index > 0 && } -
    -
    - - {label} Tier - - - - - - Tier {index + 1} of {TIER_KEYS.length} · {tier} - + + {TIER_KEYS.map((tier, index) => { + const tierInfo = TIER_DESCRIPTIONS[tier]; + const label = effectiveTierLabel(tier, value.tier_labels); + const tierMissing = showValidationErrors && value.tiers[tier].length === 0; + return ( +
    + {index > 0 && } +
    +
    + {label} Tier + + + + + Tier {index + 1} of {TIER_KEYS.length} · {tier} + +
    + Examples: {tierInfo.examples} + + handleTierLabelChange(tier, event.target.value)} + placeholder={`Display name (default: ${tierInfo.label})`} + aria-label={`Display name for the ${tierInfo.label} tier`} + /> + {value.tier_labels?.[tier] && ( + + handleTierLabelChange(tier, "")} + > + + + + )} + + handleTierChange(tier, models)} + placeholder={`Select model(s) for ${label.toLowerCase()} queries`} + emptyText="No models found" + className={tierMissing ? "w-full border-destructive" : "w-full"} + /> + {value.tiers[tier].length > 1 && ( + + Multiple models selected — the router randomly picks among them per request (or Thompson-samples + within the pool when adaptive routing is on). + + )} + {tierMissing && The {label} tier is required}
    - - Examples: {tierInfo.examples} - - handleTierLabelChange(tier, event.target.value)} - placeholder={`Display name (default: ${tierInfo.label})`} - aria-label={`Display name for the ${tierInfo.label} tier`} - style={{ marginBottom: 8 }} - allowClear - /> - handleTierChange(tier, models)} - placeholder={`Select model(s) for ${label.toLowerCase()} queries`} - showSearch - style={{ width: "100%" }} - options={modelOptions} - status={tierMissing ? "error" : undefined} - /> - {value.tiers[tier].length > 1 && ( - - Multiple models selected — the router randomly picks among them per request (or Thompson-samples - within the pool when adaptive routing is on). - - )} - {tierMissing && ( - - The {label} tier is required - - )}
    -
    - ); - })} - + ); + })} + -
    -
    - - Default Model - - - - +
    +
    + Default Model + + + +
    + + + Used when the tier the request lands in has no model, and when the classifier fails with "Route to + the default model" selected. +
    - - - Used when the tier the request lands in has no model, and when the classifier fails with "Route to the - default model" selected. - -
    + - + - + {[ { key: "classifier", - label: ( - - Advanced: Classification Method - - ), + label: Advanced: Classification Method, children: ( = ({ }, { key: "adaptive", - label: ( - - Advanced: Adaptive Routing - - ), + label: Advanced: Adaptive Routing, children: , }, { key: "affinity", - label: ( - - Advanced: Affinity - - ), + label: Advanced: Affinity, children: ( <>
    onChange({ ...value, deployment_affinity: deploymentAffinity })} + onCheckedChange={(deploymentAffinity) => + onChange({ ...value, deployment_affinity: deploymentAffinity }) + } aria-label="Pin a session to one deployment per model group" /> - Pin a session to one deployment per model group + Pin a session to one deployment per model group
    - + Keeps a session on the same deployment within a group, so provider prompt caches stay warm. Turn off to load-balance every turn. - +
    onChange({ ...value, session_affinity: sessionAffinity })} + onCheckedChange={(sessionAffinity) => onChange({ ...value, session_affinity: sessionAffinity })} aria-label="Pin a session to its first model" /> - Pin a session to its first model + Pin a session to its first model
    - + Keeps a session on its first turn's model instead of re-classifying each turn. Also pins the deployment. - + ), }, { key: "plan-mode", - label: ( - - Advanced: Plan-Mode Override - - ), + label: Advanced: Plan-Mode Override, children: ( <>
    + onCheckedChange={(enabled) => onChange({ ...value, plan_mode_min_tier: enabled ? planModeTiers.at(-1) : undefined }) } aria-label="Route plan-mode requests to a minimum tier" /> - Route plan-mode requests to a minimum tier + Route plan-mode requests to a minimum tier
    - + Requests from coding agents in plan mode (Claude Code, GitHub Copilot) route to at least this tier. The classifier still wins when it picks higher, and the override only lasts while plan mode is active. {planModeTiers.length === 0 && " Add models to a tier to enable this."} - + {value.plan_mode_min_tier !== undefined && (
    - - (planModeTiers as string[]).includes(option.value), - )} - onChange={(tier: string) => onChange({ ...value, plan_mode_min_tier: tier })} - /> + onValueChange={(tier: string | null) => tier && onChange({ ...value, plan_mode_min_tier: tier })} + > + + + + + {planModeTierOptions.map((option) => ( + + {option.label} + + ))} + +
    )} @@ -473,23 +469,22 @@ const ComplexityRouterConfig: React.FC = ({ }, { key: "response", - label: ( - - Advanced: Response Format - - ), + label: Advanced: Response Format, children: ( <>
    onChange({ ...value, return_raw_model_name: returnRawModelName })} + onCheckedChange={(returnRawModelName) => + onChange({ ...value, return_raw_model_name: returnRawModelName }) + } + aria-label="Return raw model name" /> - Return raw model name + Return raw model name
    - + Return the resolved underlying model name in responses instead of the autorouter alias. - + ), }, @@ -497,11 +492,7 @@ const ComplexityRouterConfig: React.FC = ({ ? [ { key: "escalation", - label: ( - - Advanced: Escalation Keywords - - ), + label: Advanced: Escalation Keywords, children: , }, ] @@ -510,11 +501,7 @@ const ComplexityRouterConfig: React.FC = ({ ? [ { key: "keyword-semantic", - label: ( - - Advanced: Keyword/Semantic Matching - - ), + label: Advanced: Keyword/Semantic Matching, children: ( <> {onKeywordTierRulesChange && ( @@ -524,9 +511,7 @@ const ComplexityRouterConfig: React.FC = ({ tierLabels={value.tier_labels} /> )} - {onKeywordTierRulesChange && onSemanticMatchingEnabledChange && ( - - )} + {onKeywordTierRulesChange && onSemanticMatchingEnabledChange && } {onSemanticMatchingEnabledChange && ( = ({ }, ] : []), - ]} - /> + ].map(({ key, label, children }) => ( + + + + {label} + + {children} + + ))} +
    ); }; diff --git a/ui/litellm-dashboard/src/components/add_model/EscalationKeywords.tsx b/ui/litellm-dashboard/src/components/add_model/EscalationKeywords.tsx index a7c1b7462ea..b1bb25deb13 100644 --- a/ui/litellm-dashboard/src/components/add_model/EscalationKeywords.tsx +++ b/ui/litellm-dashboard/src/components/add_model/EscalationKeywords.tsx @@ -1,10 +1,8 @@ import { Info } from "lucide-react"; import { SimpleTooltip } from "@/components/ui/tooltip"; -import { Select as AntdSelect, Typography } from "antd"; +import { MultiSelect } from "@/components/shared/MultiSelect"; import React from "react"; -const { Text } = Typography; - export const DEFAULT_ESCALATION_KEYWORDS = ["LITELLM ESCALATE"]; interface EscalationKeywordsProps { @@ -16,28 +14,24 @@ const EscalationKeywords: React.FC = ({ keywords, onCha return (
    - - Escalation Keywords - +

    Escalation Keywords

    - + Optional: when a user message contains one of these phrases, the request is bumped one tier higher than it would otherwise route to. Matching is case-sensitive, so "LITELLM ESCALATE" only fires on the exact, shouted form. Leave empty to disable. - - + ({ label: keyword, value: keyword }))} value={keywords} - onChange={onChange} + onValueChange={onChange} placeholder="e.g., LITELLM ESCALATE" - tokenSeparators={[","]} - open={false} - suffixIcon={null} - style={{ width: "100%" }} - allowClear + emptyText="Type to add a phrase" + allowCustomValues + className="w-full" />
    ); diff --git a/ui/litellm-dashboard/src/components/add_model/KeywordTierRules.tsx b/ui/litellm-dashboard/src/components/add_model/KeywordTierRules.tsx index 08d00606987..f7edd26f0d8 100644 --- a/ui/litellm-dashboard/src/components/add_model/KeywordTierRules.tsx +++ b/ui/litellm-dashboard/src/components/add_model/KeywordTierRules.tsx @@ -1,14 +1,14 @@ -import { Info, Plus, Trash2 } from "lucide-react"; +import { Inbox, Info, Plus, Trash2 } from "lucide-react"; import { SimpleTooltip } from "@/components/ui/tooltip"; -import { Card, Empty, Select as AntdSelect, Typography } from "antd"; +import { MultiSelect } from "@/components/shared/MultiSelect"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Card, CardContent } from "@/components/ui/card"; import { Button } from "@/components/ui/button"; import React from "react"; import { emptyKeywordTierRuleIndexes } from "./complexity_router_keywords"; import { tierOptions } from "./complexity_router_tiers"; -const { Text } = Typography; - export type ComplexityTier = "SIMPLE" | "MEDIUM" | "COMPLEX" | "REASONING"; export interface KeywordTierRule { @@ -28,29 +28,9 @@ interface KeywordTierRulesProps { // there is no failed attempt left to surface it. const KeywordTierRules: React.FC = ({ rules, onChange, tierLabels }) => { const emptyRuleIndexes = new Set(emptyKeywordTierRuleIndexes(rules)); - const [drafts, setDrafts] = React.useState>({}); - - const setDraft = (id: string, text: string) => setDrafts((current) => ({ ...current, [id]: text })); - - // The dropdown is kept closed, which leaves antd nothing for Enter to select, so a typed keyword - // would only become a tag on blur. Submitting used to supply that blur; the button is disabled - // while the row reads as empty, so Enter has to commit the word itself or the row cannot be filled. - const commitDraft = (rule: KeywordTierRule) => { - const keyword = (drafts[rule.id] ?? "").trim(); - if (!keyword) return; - updateRule(rule.id, { keywords: [...rule.keywords, keyword] }); - setDraft(rule.id, ""); - }; - - const commitDraftOnEnter = (rule: KeywordTierRule) => (event: React.KeyboardEvent) => { - if (event.key !== "Enter") return; - event.preventDefault(); - commitDraft(rule); - }; const replaceKeywords = (rule: KeywordTierRule) => (keywords: string[]) => { updateRule(rule.id, { keywords }); - setDraft(rule.id, ""); }; const addRule = () => { @@ -69,9 +49,7 @@ const KeywordTierRules: React.FC = ({ rules, onChange, ti
    - - Keyword Tier Overrides - +

    Keyword Tier Overrides

    @@ -81,67 +59,71 @@ const KeywordTierRules: React.FC = ({ rules, onChange, ti Add keyword rule
    - + Optional: route requests containing specific keywords directly to a tier, e.g. route "invoice, refund, billing" to the medium tier. - + {rules.length === 0 ? ( - + +
    +
    +
    ) : (
    {rules.map((rule, index) => ( - -
    -
    - - Keywords {index + 1} - - setDraft(rule.id, text)} - onInputKeyDown={commitDraftOnEnter(rule)} - onBlur={() => commitDraft(rule)} - placeholder="e.g., invoice, refund, billing" - tokenSeparators={[","]} - open={false} - suffixIcon={null} - style={{ width: "100%" }} - allowClear - status={emptyRuleIndexes.has(index) ? "error" : undefined} - /> - {emptyRuleIndexes.has(index) && ( - - At least one keyword is required - - )} + + +
    +
    + Keywords {index + 1} + ({ label: keyword, value: keyword }))} + value={rule.keywords} + onValueChange={replaceKeywords(rule)} + placeholder="e.g., invoice, refund, billing" + emptyText="Type to add a keyword" + allowCustomValues + className={emptyRuleIndexes.has(index) ? "w-full border-destructive" : "w-full"} + /> + {emptyRuleIndexes.has(index) && ( + At least one keyword is required + )} +
    +
    + Route to tier + +
    +
    -
    - - Route to tier - - updateRule(rule.id, { tier })} - options={tierOptions(tierLabels)} - style={{ width: "100%" }} - /> -
    - -
    + ))}
    diff --git a/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx b/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx index 7f8a71b1b8b..9da0bf394e5 100644 --- a/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx +++ b/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx @@ -1,11 +1,11 @@ import { Info } from "lucide-react"; import { SimpleTooltip } from "@/components/ui/tooltip"; -import { InputNumber, Select as AntdSelect, Switch, Typography } from "antd"; +import { SearchSelect } from "@/components/shared/SearchSelect"; +import { Input } from "@/components/ui/input"; +import { Switch } from "@/components/ui/switch"; import React from "react"; import { ModelGroup } from "@/components/llm_calls/fetch_models"; -const { Text } = Typography; - const DEFAULT_MATCH_THRESHOLD = 0.5; interface SemanticKeywordMatchingProps { @@ -41,49 +41,49 @@ const SemanticKeywordMatching: React.FC = ({
    - Semantic keyword matching + Semantic keyword matching
    - + Uses same keyword-tier pairs as above and overrides direct keyword matching. Adds latency based on embedding model network request. - +
    - +
    {enabled && (
    - Embedding model - Embedding model + - {embeddingModelMissing && ( - - An embedding model is required - - )} + {embeddingModelMissing && An embedding model is required}
    - Minimum match score - Minimum match score + onMatchThresholdChange(value ?? DEFAULT_MATCH_THRESHOLD)} + onChange={(event) => + onMatchThresholdChange(event.target.value === "" ? DEFAULT_MATCH_THRESHOLD : event.target.valueAsNumber) + } min={0} max={1} step={0.05} - style={{ width: "100%" }} + className="w-full" /> - Match only at or above this similarity score. + Match only at or above this similarity score.
    )} diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index e3a7482b02d..e45422dee08 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -27,7 +27,7 @@ const ALL_FAMILY_MODELS: ModelGroup[] = [ const ANTHROPIC_ONLY_MODEL = ANTHROPIC_TIERS.COMPLEX[0]; const openTemplateDropdown = (): void => { - fireEvent.mouseDown(within(screen.getByTestId("template-selector")).getByRole("combobox")); + fireEvent.click(screen.getByTestId("template-selector")); }; // Detailed Configuration is collapsed by default, so any test reaching into it (a tier select, an @@ -36,15 +36,32 @@ const expandDetailedConfiguration = (): void => { fireEvent.click(screen.getByTestId("detailed-configuration-toggle")); }; -const visibleOptions = (): HTMLElement[] => - // eslint-disable-next-line local/no-antd-class-selectors -- antd puts role="option" only on a hidden mirror list of raw values; the visible options carry no role, no aria-disabled, and only a tooltip in title - Array.from(document.querySelectorAll(".ant-select-item-option")); +const visibleOptions = (): HTMLElement[] => screen.queryAllByRole("option"); const optionByLabel = (label: string): HTMLElement | undefined => visibleOptions().find((el) => el.textContent?.startsWith(label)); -// eslint-disable-next-line local/no-antd-class-selectors -- antd signals option disabled state only through this class -const isOptionDisabled = (option: HTMLElement): boolean => option.classList.contains("ant-select-item-option-disabled"); +const isOptionDisabled = (option: HTMLElement): boolean => option.getAttribute("aria-disabled") === "true"; + +const selectTemplate = async (label: string): Promise => { + await userEvent.click(optionByLabel(label)!); +}; + +// Opens the dropdown only when it is closed, since openTemplateDropdown toggles: waiting on a +// second preset in the same test would otherwise close the list out from under the poll. +const waitForPresetEnabled = async (label: string) => { + if (visibleOptions().length === 0) openTemplateDropdown(); + await waitFor(() => { + expect(isOptionDisabled(optionByLabel(label)!)).toBe(false); + }); +}; + +// The keyword field is a combobox that offers whatever is typed as a "Create ..." entry, so a +// keyword only lands on the rule once that entry is picked. +const addKeyword = async (user: ReturnType, field: HTMLElement, keyword: string) => { + await user.type(within(field).getByRole("combobox"), keyword); + await user.click(await screen.findByText(`Create "${keyword}"`)); +}; const { mockFetchAvailableModels, mockFetchAllModelDeployments } = vi.hoisted(() => ({ mockFetchAvailableModels: vi.fn(), @@ -202,10 +219,7 @@ describe("AddAutoRouterTab", () => { await user.click(screen.getByRole("button", { name: /add keyword rule/i })); expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled(); - await user.type( - within(screen.getByText("Keywords 1").closest("div") as HTMLElement).getByRole("combobox"), - "invoice{enter}", - ); + await addKeyword(user, screen.getByText("Keywords 1").closest("div") as HTMLElement, "invoice"); expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled(); expect(screen.queryByText("At least one keyword is required")).not.toBeInTheDocument(); @@ -221,10 +235,7 @@ describe("AddAutoRouterTab", () => { expandDetailedConfiguration(); await user.click(screen.getByText("Advanced: Keyword/Semantic Matching")); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); - await user.type( - within(screen.getByText("Keywords 1").closest("div") as HTMLElement).getByRole("combobox"), - "invoice{enter}", - ); + await addKeyword(user, screen.getByText("Keywords 1").closest("div") as HTMLElement, "invoice"); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); expect(await screen.findAllByText("At least one keyword is required")).toHaveLength(1); @@ -242,7 +253,7 @@ describe("AddAutoRouterTab", () => { await user.click(screen.getByText("Advanced: Keyword/Semantic Matching")); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); const keywordsField = screen.getByText("Keywords 1").closest("div") as HTMLElement; - await user.type(within(keywordsField).getByRole("combobox"), "invoice{enter}"); + await addKeyword(user, keywordsField, "invoice"); await user.click(screen.getByRole("button", { name: /add auto router/i })); await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); @@ -407,7 +418,7 @@ describe("AddAutoRouterTab", () => { await user.click(screen.getByText("Advanced: Keyword/Semantic Matching")); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); const keywordsField = screen.getByText("Keywords 1").closest("div") as HTMLElement; - await user.type(within(keywordsField).getByRole("combobox"), "invoice{enter}"); + await addKeyword(user, keywordsField, "invoice"); await user.click(screen.getByTestId("auto-router-test-routing-btn")); await user.type(await screen.findByTestId("auto-router-routing-test-prompt"), "reconcile this invoice"); @@ -452,17 +463,6 @@ describe("AddAutoRouterTab", () => { }); describe("template presets", () => { - // Opens the dropdown once, then waits out the useQuery load: an open antd Select re-renders its - // already-mounted options in place as state changes, so polling only re-reads the DOM here. - // Re-firing the open/close mousedown on every poll (calling openTemplateDropdown inside the - // waitFor callback) fights the dropdown's own open/close animation and hangs the test. - const waitForPresetEnabled = async (label: string) => { - openTemplateDropdown(); - await waitFor(() => { - expect(isOptionDisabled(optionByLabel(label)!)).toBe(false); - }); - }; - it("disables every preset while the model list is loading", async () => { let resolveModels: (models: ModelGroup[]) => void = () => {}; mockFetchAvailableModels.mockImplementation( @@ -537,7 +537,7 @@ describe("AddAutoRouterTab", () => { renderWithProviders(); await waitForPresetEnabled("Anthropic Family"); - fireEvent.click(optionByLabel("Anthropic Family")!); + await selectTemplate("Anthropic Family"); expect(screen.queryByText("Advanced: Keyword/Semantic Matching")).not.toBeInTheDocument(); expect( @@ -548,11 +548,11 @@ describe("AddAutoRouterTab", () => { ).toBeInTheDocument(); }); - it("expands detailed configuration when Custom Configuration is chosen", () => { + it("expands detailed configuration when Custom Configuration is chosen", async () => { renderWithProviders(); openTemplateDropdown(); - fireEvent.click(optionByLabel("Custom Configuration")!); + await selectTemplate("Custom Configuration"); expect(screen.getByText("Advanced: Keyword/Semantic Matching")).toBeInTheDocument(); }); @@ -561,7 +561,7 @@ describe("AddAutoRouterTab", () => { mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); renderWithProviders(); await waitForPresetEnabled("Anthropic Family"); - fireEvent.click(optionByLabel("Anthropic Family")!); + await selectTemplate("Anthropic Family"); expect(screen.queryByText("Advanced: Keyword/Semantic Matching")).not.toBeInTheDocument(); fireEvent.click(screen.getByTestId("detailed-configuration-toggle")); @@ -578,7 +578,7 @@ describe("AddAutoRouterTab", () => { renderWithProviders(); await waitForPresetEnabled("Anthropic Family"); - fireEvent.click(optionByLabel("Anthropic Family")!); + await selectTemplate("Anthropic Family"); await user.type(screen.getByPlaceholderText(/smart_router/i), "anthropic-router"); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -600,7 +600,7 @@ describe("AddAutoRouterTab", () => { const { container } = renderWithProviders(); await waitForPresetEnabled("Anthropic Family"); - fireEvent.click(optionByLabel("Anthropic Family")!); + await selectTemplate("Anthropic Family"); fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "stale-model-router" } }); expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled(); @@ -622,26 +622,16 @@ describe("AddAutoRouterTab", () => { describe("default model pin", () => { const PINNED_MODEL = "pinned-default-model"; - const waitForPresetEnabled = async (label: string) => { - openTemplateDropdown(); - await waitFor(() => { - expect(isOptionDisabled(optionByLabel(label)!)).toBe(false); - }); - }; - const applyPresetAndPin = async (user: ReturnType) => { await waitForPresetEnabled("Anthropic Family"); - fireEvent.click(optionByLabel("Anthropic Family")!); + await selectTemplate("Anthropic Family"); // Applying a preset collapses Detailed Configuration, so the default model row is behind it. expandDetailedConfiguration(); const defaultModel = screen.getByRole("combobox", { name: "Default model" }); await user.click(defaultModel); - // antd virtualizes the option list and jsdom gives every row zero height, so options past - // the first window never render. Typing filters the list down to the pin instead of relying - // on its index, which adding a preset to the bundled JSON shifts. await user.type(defaultModel, PINNED_MODEL); - await user.click((await screen.findAllByTitle(PINNED_MODEL)).slice(-1)[0]); + await user.click(await screen.findByRole("option", { name: PINNED_MODEL })); }; beforeEach(() => { @@ -692,19 +682,12 @@ describe("AddAutoRouterTab", () => { mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); }); - const waitForPresetEnabled = async (label: string) => { - openTemplateDropdown(); - await waitFor(() => { - expect(isOptionDisabled(optionByLabel(label)!)).toBe(false); - }); - }; - it("omits plan_mode_min_tier from the payload when never touched", async () => { const user = userEvent.setup(); renderWithProviders(); await waitForPresetEnabled("Anthropic Family"); - fireEvent.click(optionByLabel("Anthropic Family")!); + await selectTemplate("Anthropic Family"); await user.type(screen.getByPlaceholderText(/smart_router/i), "no-plan-router"); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -719,7 +702,7 @@ describe("AddAutoRouterTab", () => { renderWithProviders(); await waitForPresetEnabled("Anthropic Family"); - fireEvent.click(optionByLabel("Anthropic Family")!); + await selectTemplate("Anthropic Family"); expandDetailedConfiguration(); await user.click(screen.getByText("Advanced: Plan-Mode Override")); await user.click(await screen.findByRole("switch", { name: "Route plan-mode requests to a minimum tier" })); @@ -773,7 +756,7 @@ describe("AddAutoRouterTab", () => { await waitFor(() => { expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); }); - fireEvent.click(optionByLabel("Anthropic Family")!); + await selectTemplate("Anthropic Family"); expect(screen.getByText("Advanced: Keyword/Semantic Matching")).toBeInTheDocument(); @@ -862,7 +845,7 @@ describe("AddAutoRouterTab", () => { await waitFor(() => { expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); }); - fireEvent.click(optionByLabel("Anthropic Family")!); + await selectTemplate("Anthropic Family"); await user.type(screen.getByPlaceholderText(/smart_router/i), "wildcard-router"); await user.click(screen.getByRole("button", { name: /add auto router/i })); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index a071b92fcd5..64d1519f915 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -1,13 +1,14 @@ import React, { useEffect, useState } from "react"; import { useQuery } from "@tanstack/react-query"; import { useWatch } from "react-hook-form"; -import { Card, Select as AntdSelect } from "antd"; +import { Card } from "antd"; import { ChevronDown, ChevronRight, CircleHelp } from "lucide-react"; import { z } from "zod/v4"; import { FieldGroup } from "@/components/shared/form/field"; import { FormField } from "@/components/shared/form/FormField"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; import { useZodForm } from "@/lib/forms/useZodForm"; @@ -289,6 +290,14 @@ const AddAutoRouterTab: React.FC = ({ [presetAvailability], ); + const templateItems = React.useMemo( + () => [ + ...sortedPresetOptions.map(({ preset }) => ({ value: preset.key, label: preset.label })), + { value: "custom", label: "Custom Configuration" }, + ], + [sortedPresetOptions], + ); + const applyPrefill = (prefill: PresetPrefill) => { setComplexityRouterConfig(prefill.complexityRouterConfig); setCustomTechnicalKeywords(prefill.customTechnicalKeywords); @@ -486,49 +495,52 @@ const AddAutoRouterTab: React.FC = ({
    - handlePresetChange(presetKey ?? undefined)} > - {sortedPresetOptions.map(({ preset, availability: presetState }) => { - const disabledHint = presetDisabledHint(presetState); - const isDisabled = disabledHint !== null; - const hintClass = isPresetHintAlarming(presetState) - ? "text-red-500 dark:text-red-400" - : "text-muted-foreground"; - const matchedHint = - presetState.kind === "available" && presetState.viaDeployments ? "Matches your deployments" : null; + + + + + {sortedPresetOptions.map(({ preset, availability: presetState }) => { + const disabledHint = presetDisabledHint(presetState); + const hintClass = isPresetHintAlarming(presetState) + ? "text-red-500 dark:text-red-400" + : "text-muted-foreground"; + const matchedHint = + presetState.kind === "available" && presetState.viaDeployments + ? "Matches your deployments" + : null; - return ( - -
    -
    {preset.label}
    -
    {preset.description}
    - {disabledHint &&
    {disabledHint}
    } - {matchedHint && ( -
    {matchedHint}
    - )} -
    -
    - ); - })} - -
    -
    Custom Configuration
    -
    Define your auto router from scratch
    -
    -
    -
    + return ( + +
    +
    {preset.label}
    +
    {preset.description}
    + {disabledHint &&
    {disabledHint}
    } + {matchedHint && ( +
    {matchedHint}
    + )} +
    +
    + ); + })} + +
    +
    Custom Configuration
    +
    Define your auto router from scratch
    +
    +
    + + {modelsUnverifiable && (
    Could not load available models.{" "} diff --git a/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx b/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx index 2d35a772a63..5bc0dcd143d 100644 --- a/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx +++ b/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx @@ -1,14 +1,18 @@ import React from "react"; -import { Switch, Select, Tooltip, DatePicker } from "antd"; +import { MultiSelect } from "@/components/shared/MultiSelect"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Switch } from "@/components/ui/switch"; +import { SimpleTooltip } from "@/components/ui/tooltip"; +import type { Dayjs } from "dayjs"; import { ChevronDown, Info } from "lucide-react"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import { Input } from "@/components/ui/input"; -import { Row, Col, Typography } from "antd"; -import TextArea from "antd/es/input/TextArea"; +import { Textarea } from "@/components/ui/textarea"; import { Team } from "../key_team_helpers/key_list"; import { antdRules } from "../common_components/antdFormRules"; import { labelWithHint } from "@/components/shared/form/LabelWithHint"; import { MountedFormField } from "../common_components/MountedFormField"; +import { UtcDateTimeInput } from "@/components/shared/form/UtcDateTimeInput"; import CacheControlInjectionPoints, { CACHE_CONTROL_LABEL, CACHE_CONTROL_TOOLTIP, @@ -30,7 +34,6 @@ import { PTU_END_FIELD, } from "../../utils/ptuValidation"; import { usePtuCostAttributionEnabled } from "@/app/(dashboard)/hooks/uiSettings/usePtuCostAttributionEnabled"; -const { Link } = Typography; interface AdvancedSettingsProps { showAdvancedSettings: boolean; @@ -51,6 +54,11 @@ const USAGE_COST_FIELDS = [ const REVALIDATED_WHEN_PTU_COUNT_CHANGES = [PTU_RATE_FIELD, PTU_START_FIELD, ...USAGE_COST_FIELDS]; +const PRICING_MODEL_ITEMS = [ + { value: "per_token", label: "Per Million Tokens" }, + { value: "per_second", label: "Per Second" }, +] as const; + const validateNumber = (_: unknown, value: unknown) => { if (!value) { return Promise.resolve(); @@ -79,6 +87,14 @@ const AdvancedSettings: React.FC = ({ const [showCacheControl, setShowCacheControl] = React.useState(false); const ptuCostAttributionEnabled = usePtuCostAttributionEnabled(); + const handlePricingModelChange = + (onChange: (value: string) => void) => + (value: "per_token" | "per_second" | null): void => { + if (value === null) return; + onChange(value); + setPricingModel(value); + }; + return ( <> @@ -93,11 +109,10 @@ const AdvancedSettings: React.FC = ({ { + onCheckedChange={(checked) => { control.onChange(checked); setCustomPricing(checked); }} - className="bg-gray-600" /> )} @@ -107,7 +122,7 @@ const AdvancedSettings: React.FC = ({ label={ Attached Knowledge Bases (RAG){" "} - + = ({ > - + } className="mt-4" @@ -137,7 +152,7 @@ const AdvancedSettings: React.FC = ({ label={ Guardrails{" "} - + = ({ > - + } className="mt-4" help="Select existing guardrails. Go to 'Guardrails' tab to create new guardrails." > {(control) => ( - ({ value: tag.name, label: tag.name, - title: tag.description || tag.name, + description: tag.description || undefined, }))} + allowCustomValues /> )} @@ -249,11 +262,9 @@ const AdvancedSettings: React.FC = ({ className="mb-4" > {(control) => ( - @@ -273,11 +284,9 @@ const AdvancedSettings: React.FC = ({ className="mb-4" > {(control) => ( - @@ -291,19 +300,21 @@ const AdvancedSettings: React.FC = ({ {(control) => ( )} @@ -401,20 +412,20 @@ const AdvancedSettings: React.FC = ({ "Use in pass through routes", Allow using these credentials in pass through routes.{" "} - + Learn more - + , )} className="mb-4 mt-4" > {(control) => ( - + )} @@ -427,11 +438,10 @@ const AdvancedSettings: React.FC = ({ { + onCheckedChange={(checked) => { control.onChange(checked); setShowCacheControl(checked); }} - className="bg-gray-600" /> )} @@ -456,7 +466,7 @@ const AdvancedSettings: React.FC = ({ rules={{ validate: antdRules({ validator: formItemValidateJSON }) }} > {(control) => ( -