From ebf6167d8acf48499e294ecf3a7642b4112913eb Mon Sep 17 00:00:00 2001 From: Scott Wilson Date: Mon, 3 Aug 2026 16:50:58 -0400 Subject: [PATCH 01/26] fix(anthropic): stop emitting empty thinking blocks on the Responses adapter OpenAI emits a reasoning output item on every reasoning turn, but only emits reasoning_summary_text deltas when a summary was requested and actually produced. The Anthropic /v1/messages Responses stream adapter opened the thinking content block eagerly on response.output_item.added, so a summary-less reasoning item surfaced as {"type": "thinking", "thinking": ""}. Clients persist that in their session transcript and replay it on the next turn; an Anthropic model then rejects the request with "each thinking block must contain thinking", which is what users hit when a resumed session falls back to the default Anthropic model. Open the thinking block on the first non-empty summary delta instead, and only emit content_block_stop for items that actually have an open block. --- .../responses_adapters/streaming_iterator.py | 83 +++++++------------ ...t_responses_adapters_streaming_iterator.py | 79 +++++++++++++++++- 2 files changed, 106 insertions(+), 56 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index f12dd979338..c588e791cd9 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -68,6 +68,19 @@ class AnthropicResponsesStreamWrapper: self._current_block_index += 1 return self._current_block_index + def _open_block(self, item_id: str | None, content_block: dict[str, Any]) -> int: + block_idx = self._next_block_index() + if item_id: + self._item_id_to_block_index[item_id] = block_idx + self._chunk_queue.append( + { + "type": "content_block_start", + "index": block_idx, + "content_block": content_block, + } + ) + return block_idx + def _process_event(self, event: Any) -> None: """Convert one Responses API event into zero or more Anthropic chunks queued for emission.""" event_type = getattr(event, "type", None) @@ -93,47 +106,22 @@ class AnthropicResponsesStreamWrapper: item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) if item_type == "message": - block_idx = self._next_block_index() - if item_id: - self._item_id_to_block_index[item_id] = block_idx - self._chunk_queue.append( - { - "type": "content_block_start", - "index": block_idx, - "content_block": {"type": "text", "text": ""}, - } - ) + self._open_block(item_id, {"type": "text", "text": ""}) elif item_type == "function_call": call_id: Final = ( getattr(item, "call_id", None) or (item.get("call_id") if isinstance(item, dict) else None) or "" ) name = getattr(item, "name", None) or (item.get("name") if isinstance(item, dict) else None) or "" - block_idx = self._next_block_index() if item_id: - self._item_id_to_block_index[item_id] = block_idx self._pending_tool_ids[item_id] = call_id - self._chunk_queue.append( + self._open_block( + item_id, { - "type": "content_block_start", - "index": block_idx, - "content_block": { - "type": "tool_use", - "id": call_id, - "name": name, - "input": {}, - }, - } - ) - elif item_type == "reasoning": - block_idx = self._next_block_index() - if item_id: - self._item_id_to_block_index[item_id] = block_idx - self._chunk_queue.append( - { - "type": "content_block_start", - "index": block_idx, - "content_block": {"type": "thinking", "thinking": ""}, - } + "type": "tool_use", + "id": call_id, + "name": name, + "input": {}, + }, ) return @@ -146,16 +134,7 @@ class AnthropicResponsesStreamWrapper: # Some providers (e.g. LMStudio) skip response.output_item.added, # so no text block is open yet; synthesize content_block_start # instead of emitting a delta with index -1 - block_idx = self._next_block_index() - if item_id: - self._item_id_to_block_index[item_id] = block_idx - self._chunk_queue.append( - { - "type": "content_block_start", - "index": block_idx, - "content_block": {"type": "text", "text": ""}, - } - ) + block_idx = self._open_block(item_id, {"type": "text", "text": ""}) self._chunk_queue.append( { "type": "content_block_delta", @@ -169,11 +148,11 @@ class AnthropicResponsesStreamWrapper: if event_type == "response.reasoning_summary_text.delta": item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") - block_idx = ( - self._item_id_to_block_index.get(item_id, self._current_block_index) - if item_id - else self._current_block_index - ) + block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index + if block_idx < 0: + if not delta: + return + block_idx = self._open_block(item_id, {"type": "thinking", "thinking": ""}) self._chunk_queue.append( { "type": "content_block_delta", @@ -207,11 +186,9 @@ class AnthropicResponsesStreamWrapper: item_id = ( getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) if item else None ) - block_idx = ( - self._item_id_to_block_index.get(item_id, self._current_block_index) - if item_id - else self._current_block_index - ) + block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index + if block_idx < 0: + return self._chunk_queue.append( { "type": "content_block_stop", diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index 73b58e71009..b1ae865fde1 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -76,6 +76,78 @@ class TestProcessEventResponseCreatedGuard: assert len(message_starts) == 1 +class TestReasoningItemWithoutSummaryText: + """Regression: a reasoning item whose summary never produces text must not + surface as a thinking content block. + + OpenAI emits ``response.output_item.added`` with ``type: "reasoning"`` on + every reasoning turn, but only emits + ``response.reasoning_summary_text.delta`` when a summary was requested and + the model actually produced one. Eagerly opening the block on + ``output_item.added`` left ``{"type": "thinking", "thinking": ""}`` in the + assistant turn, which clients persist in their session transcript. Replaying + that transcript against an Anthropic model (what ``claude --resume`` does + once the resumed session falls back to the default Anthropic model) fails + with:: + + 400 invalid_request_error - messages.2.content.0.thinking: + each thinking block must contain thinking + + So the thinking block is opened on the first non-empty summary delta. + """ + + @staticmethod + def _gpt_turn(reasoning_summary_deltas: list) -> list: + return [ + {"type": "response.created"}, + {"type": "response.output_item.added", "item": {"type": "reasoning", "id": "rs_1"}}, + *( + {"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", "delta": delta} + for delta in reasoning_summary_deltas + ), + {"type": "response.output_item.done", "item": {"type": "reasoning", "id": "rs_1"}}, + {"type": "response.output_item.added", "item": {"type": "message", "id": "msg_1"}}, + {"type": "response.output_text.delta", "item_id": "msg_1", "delta": "Hello"}, + {"type": "response.output_item.done", "item": {"type": "message", "id": "msg_1"}}, + ] + + def test_reasoning_without_summary_emits_no_thinking_block(self): + chunks = _drain_async(self._gpt_turn(reasoning_summary_deltas=[])) + + assert not [ + c for c in chunks if c["type"] == "content_block_start" and c["content_block"]["type"] == "thinking" + ] + assert [(c["type"], c.get("index")) for c in chunks[1:]] == [ + ("content_block_start", 0), + ("content_block_delta", 0), + ("content_block_stop", 0), + ] + assert chunks[1]["content_block"] == {"type": "text", "text": ""} + + def test_reasoning_with_only_empty_summary_deltas_emits_no_thinking_block(self): + chunks = _drain_async(self._gpt_turn(reasoning_summary_deltas=["", ""])) + + assert not [c for c in chunks if c["type"] == "content_block_delta" and c["delta"]["type"] == "thinking_delta"] + assert not [ + c for c in chunks if c["type"] == "content_block_start" and c["content_block"]["type"] == "thinking" + ] + + def test_reasoning_with_summary_text_still_emits_a_thinking_block(self): + chunks = _drain_async(self._gpt_turn(reasoning_summary_deltas=["Weigh", "ing options"])) + + assert [(c["type"], c.get("index")) for c in chunks[1:]] == [ + ("content_block_start", 0), + ("content_block_delta", 0), + ("content_block_delta", 0), + ("content_block_stop", 0), + ("content_block_start", 1), + ("content_block_delta", 1), + ("content_block_stop", 1), + ] + assert chunks[1]["content_block"] == {"type": "thinking", "thinking": ""} + assert "".join(c["delta"]["thinking"] for c in chunks[2:4]) == "Weighing options" + + class TestProcessEventTextDeltaWithoutOutputItemAdded: """Streams that skip response.output_item.added (e.g. LMStudio) must still open a text block before any delta and never emit index -1.""" @@ -110,12 +182,13 @@ class TestProcessEventTextDeltaWithoutOutputItemAdded: "type": "response.output_item.added", "item": {"type": "reasoning", "id": "rs_1"}, }, + {"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", "delta": "hm"}, {"type": "response.output_text.delta", "item_id": "m1", "delta": "Hi"}, ] ) - assert chunks[1]["type"] == "content_block_start" - assert chunks[1]["content_block"] == {"type": "text", "text": ""} - assert [c["index"] for c in chunks[1:]] == [1, 1] + assert chunks[2]["type"] == "content_block_start" + assert chunks[2]["content_block"] == {"type": "text", "text": ""} + assert [c["index"] for c in chunks[2:]] == [1, 1] def test_process_event_registered_item_id_does_not_synthesize_start(self): chunks = _process_all( From ee0c0cc7e8f5834a2b64bba442e0a10723f0d254 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 7 Aug 2026 22:18:59 -0700 Subject: [PATCH 02/26] feat(proxy): let USE_V2_MIGRATION_RESOLVER select the v2 migration resolver --use_v2_migration_resolver had no env var, and the Helm migrations job runs prisma_migration.py, which calls run_server with a fixed argv. There was no seam to pass the flag, so a Helm install could not reach the v2 resolver at all. Reading it from the environment makes the existing flag configurable from a deployment. --- litellm/proxy/proxy_cli.py | 1 + tests/test_litellm/proxy/test_proxy_cli.py | 58 ++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index ab159e84b6a..6a0b3c6bfb2 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -916,6 +916,7 @@ class ProxyInitializationHelpers: "path that can cause schema thrashing during rolling deploys where two " "LiteLLM versions contend for the same DB. Default is the v1 resolver." ), + envvar="USE_V2_MIGRATION_RESOLVER", ) @click.option( "--reload", diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 2ccaa0df440..20d17b5a510 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -1925,6 +1925,64 @@ class TestRunServerDbSetup: assert exc_info.value.code == 1 mock_setup_database.assert_not_called() + @patch("subprocess.run") + @patch("atexit.register") + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") + @patch("litellm.proxy.db.check_migration.check_prisma_schema_diff") + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema") + def test_v2_migration_resolver_opts_in_via_env_var( + self, + mock_should_update_schema, + mock_check_schema_diff, + mock_setup_database, + mock_atexit_register, + mock_subprocess_run, + ): + """USE_V2_MIGRATION_RESOLVER must select the v2 resolver. + + The Helm migrations Job runs `python litellm/proxy/prisma_migration.py`, + which calls run_server with a fixed argv, so a deployment has no way to + pass --use_v2_migration_resolver and an env var is the only route in. + """ + from litellm.proxy.proxy_cli import run_server + + mock_subprocess_run.return_value = MagicMock(returncode=0) + mock_should_update_schema.return_value = True + mock_setup_database.return_value = True + + mock_proxy_module = MagicMock( + app=MagicMock(), + ProxyConfig=MagicMock(), + KeyManagementSettings=MagicMock(), + save_worker_config=MagicMock(), + ) + + clean_env = { + k: v + for k, v in os.environ.items() + if k not in ("DATABASE_URL", "DIRECT_URL") + } + clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" + clean_env["USE_V2_MIGRATION_RESOLVER"] = "true" + + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.dict( + "sys.modules", + { + "proxy_server": mock_proxy_module, + "litellm.proxy.proxy_server": mock_proxy_module, + }, + ), + ): + run_server.main( + ["--local", "--skip_server_startup"], standalone_mode=False + ) + + mock_setup_database.assert_called_once_with( + use_migrate=True, use_v2_resolver=True + ) + # --- Module-level helpers for worker startup hook tests --- From c019ce53e3daa6cde544aef6b1d468d719311a25 Mon Sep 17 00:00:00 2001 From: Daniel Meismer Date: Thu, 13 Aug 2026 11:15:45 -0400 Subject: [PATCH 03/26] feat(ui): add user ID request log filter Co-Authored-By: Codex --- .../view_logs/RequestLogsFilters.test.tsx | 82 ++++++++++++++++++- .../view_logs/RequestLogsFilters.tsx | 51 +++++++++++- .../components/view_logs/RequestLogsPanel.tsx | 3 +- .../components/view_logs/RequestLogsTable.tsx | 12 ++- .../components/view_logs/log_filter_logic.tsx | 1 + 5 files changed, 143 insertions(+), 6 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx index 82b179b2654..0d50effabb4 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx @@ -14,6 +14,10 @@ vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({ useInfiniteModelInfo: vi.fn(), })); +vi.mock("@/app/(dashboard)/hooks/users/useUsers", () => ({ + useInfiniteUsers: vi.fn(), +})); + vi.mock("@/app/(dashboard)/hooks/spendLogs/useSpendLogEndUsers", () => ({ useInfiniteSpendLogEndUsers: vi.fn(), })); @@ -21,6 +25,7 @@ vi.mock("@/app/(dashboard)/hooks/spendLogs/useSpendLogEndUsers", () => ({ import { useInfiniteSpendLogEndUsers } from "@/app/(dashboard)/hooks/spendLogs/useSpendLogEndUsers"; import { useInfiniteKeyAliases } from "@/app/(dashboard)/hooks/keys/useKeyAliases"; import { useInfiniteModelInfo } from "@/app/(dashboard)/hooks/models/useModels"; +import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers"; const emptyInfiniteQuery = { data: { pages: [], pageParams: [] }, @@ -32,10 +37,16 @@ const emptyInfiniteQuery = { const LOGS_WINDOW = { start_date: "2026-07-23 00:00:00", end_date: "2026-07-24 00:00:00" }; -function renderFilters(filters: Record = {}) { +function renderFilters(filters: Record = {}, showUserIdFilter = true) { const set = vi.fn(); renderWithProviders( - filters[id]} set={set} teams={[]} logsWindow={LOGS_WINDOW} />, + filters[id]} + set={set} + teams={[]} + logsWindow={LOGS_WINDOW} + showUserIdFilter={showUserIdFilter} + />, ); return { set }; } @@ -50,6 +61,9 @@ describe("RequestLogsFilters", () => { vi.mocked(useInfiniteModelInfo).mockReturnValue( emptyInfiniteQuery as unknown as ReturnType, ); + vi.mocked(useInfiniteUsers).mockReturnValue( + emptyInfiniteQuery as unknown as ReturnType, + ); vi.mocked(useInfiniteSpendLogEndUsers).mockReturnValue( emptyInfiniteQuery as unknown as ReturnType, ); @@ -62,6 +76,7 @@ describe("RequestLogsFilters", () => { "Team ID", "Status", "Key Alias", + "User ID", "End User", "Error Code", "Error Message", @@ -74,6 +89,59 @@ describe("RequestLogsFilters", () => { } }); + it("places User ID between Key Alias and End User", async () => { + renderFilters(); + + const labels = ["Key Alias", "User ID", "End User"].map((label) => screen.getByText(label)); + expect(labels[0].compareDocumentPosition(labels[1]) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + expect(labels[1].compareDocumentPosition(labels[2]) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + }); + + it("selects a user by display name while storing the user ID filter", async () => { + vi.mocked(useInfiniteUsers).mockReturnValue({ + ...emptyInfiniteQuery, + data: { + pages: [ + { + users: [{ user_id: "user-1", user_alias: "Alice", user_email: "alice@example.com" }], + page: 1, + page_size: 50, + total: 1, + total_pages: 1, + }, + ], + pageParams: [1], + }, + } as unknown as ReturnType); + const user = userEvent.setup(); + const { set } = renderFilters(); + + await user.click(await screen.findByPlaceholderText("Search an internal user")); + expect(await screen.findByText("Alice")).toBeInTheDocument(); + expect(screen.getByText("alice@example.com | User ID: user-1")).toBeInTheDocument(); + await user.click(screen.getByText("Alice")); + + expect(set).toHaveBeenCalledWith(LOG_FILTER_IDS.USER_ID, "user-1"); + }); + + it("pushes the User ID picker query to the paginated user lookup", async () => { + const user = userEvent.setup(); + renderFilters(); + + const input = await screen.findByPlaceholderText("Search an internal user"); + await user.click(input); + await user.type(input, "alice@example.com"); + + await waitFor(() => expect(useInfiniteUsers).toHaveBeenCalledWith(50, "alice@example.com")); + }); + + it("does not show or query the User ID filter for non-admin request logs", () => { + renderFilters({}, false); + + expect(screen.queryByText("User ID")).not.toBeInTheDocument(); + expect(useInfiniteUsers).not.toHaveBeenCalled(); + }); + it("scopes the Key Alias lookup to the selected team", async () => { renderFilters({ [LOG_FILTER_IDS.TEAM_ID]: "team-42" }); @@ -164,7 +232,15 @@ describe("RequestLogsFilters", () => { it("scopes the End User lookup to the window the logs table is showing", async () => { const otherWindow = { start_date: "2026-01-01 00:00:00", end_date: "2026-01-02 00:00:00" }; - renderWithProviders( undefined} set={vi.fn()} teams={[]} logsWindow={otherWindow} />); + renderWithProviders( + undefined} + set={vi.fn()} + teams={[]} + logsWindow={otherWindow} + showUserIdFilter + />, + ); await waitFor(() => expect(useInfiniteSpendLogEndUsers).toHaveBeenCalledWith(otherWindow, 50, undefined)); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.tsx index 2005b868cd6..47e0bad6f62 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.tsx @@ -5,6 +5,7 @@ import { useMemo, useState } from "react"; import { useInfiniteSpendLogEndUsers } from "@/app/(dashboard)/hooks/spendLogs/useSpendLogEndUsers"; import { useInfiniteKeyAliases } from "@/app/(dashboard)/hooks/keys/useKeyAliases"; import { useInfiniteModelInfo } from "@/app/(dashboard)/hooks/models/useModels"; +import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers"; import { DataTableFilterField } from "@/components/shared/DataTable"; import { PaginatedSearchSelect } from "@/components/shared/PaginatedSearchSelect"; import { SearchSelect, type SearchSelectOption } from "@/components/shared/SearchSelect"; @@ -144,6 +145,46 @@ function ModelFilterField({ value, onChange }: { value: string; onChange: (value ); } +function UserIdFilterField({ value, onChange }: { value: string; onChange: (value: string | undefined) => void }) { + const [search, setSearch] = useState(""); + const { data, fetchNextPage, hasNextPage, isFetchingNextPage, isLoading } = useInfiniteUsers( + PAGE_SIZE, + emptyToUndefined(search), + ); + + const options = useMemo(() => { + const seen = new Set(); + return (data?.pages ?? []).flatMap((page) => + page.users.flatMap((user) => { + if (!user.user_id || seen.has(user.user_id)) return []; + seen.add(user.user_id); + const label = user.user_alias || user.user_email || user.user_id; + const email = user.user_email && user.user_email !== label ? user.user_email : ""; + const sublabel = + user.user_id === label ? email : [email, `User ID: ${user.user_id}`].filter(Boolean).join(" | "); + return [{ label, value: user.user_id, sublabel }]; + }), + ); + }, [data]); + + return ( + + onChange(emptyToUndefined(next))} + onSearchChange={setSearch} + onLoadMore={() => void fetchNextPage()} + hasNextPage={hasNextPage} + isLoading={isLoading} + isFetchingNextPage={isFetchingNextPage} + placeholder="Search an internal user" + emptyText="No users found" + /> + + ); +} + function EndUserFilterField({ value, onChange, @@ -243,9 +284,10 @@ interface RequestLogsFiltersProps { set: (columnId: string, value: unknown) => void; teams: Team[]; logsWindow: LogsWindow; + showUserIdFilter: boolean; } -export function RequestLogsFilters({ get, set, teams, logsWindow }: RequestLogsFiltersProps) { +export function RequestLogsFilters({ get, set, teams, logsWindow, showUserIdFilter }: RequestLogsFiltersProps) { const valueOf = (id: string): string => asString(get(id)); const setter = (id: string) => (next: string | undefined) => set(id, next); @@ -279,6 +321,13 @@ export function RequestLogsFilters({ get, set, teams, logsWindow }: RequestLogsF teamId={valueOf(LOG_FILTER_IDS.TEAM_ID)} /> + {showUserIdFilter && ( + + )} + void; teams: Team[]; logsWindow: LogsWindow; + showUserIdFilter: boolean; toolbarChildren?: ReactNode; } @@ -69,6 +70,7 @@ export function RequestLogsTable({ onSessionClick, teams, logsWindow, + showUserIdFilter, toolbarChildren, }: RequestLogsTableProps) { const [filtersOpen, setFiltersOpen] = useState(false); @@ -122,7 +124,15 @@ export function RequestLogsTable({ title="Filters" description="Narrow down request logs" > - {({ get, set }) => } + {({ get, set }) => ( + + )} )} diff --git a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx index e1089c6a16c..78ecb52c184 100644 --- a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx @@ -36,6 +36,7 @@ export const LOG_FILTER_LABELS: Record = { [LOG_FILTER_IDS.TEAM_ID]: "Team ID", [LOG_FILTER_IDS.STATUS]: "Status", [LOG_FILTER_IDS.KEY_ALIAS]: "Key Alias", + [LOG_FILTER_IDS.USER_ID]: "User ID", [LOG_FILTER_IDS.END_USER]: "End User", [LOG_FILTER_IDS.ERROR_CODE]: "Error Code", [LOG_FILTER_IDS.ERROR_MESSAGE]: "Error Message", From 151bdbb2a9d6cfb3ccf758a98b04cfa02779a959 Mon Sep 17 00:00:00 2001 From: Daniel Meismer Date: Thu, 13 Aug 2026 11:21:48 -0400 Subject: [PATCH 04/26] test(ui): cover user filter pagination Co-Authored-By: Codex --- .../view_logs/RequestLogsFilters.test.tsx | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx index 0d50effabb4..fd53949c87a 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx @@ -135,6 +135,38 @@ describe("RequestLogsFilters", () => { await waitFor(() => expect(useInfiniteUsers).toHaveBeenCalledWith(50, "alice@example.com")); }); + it("loads the next page when the User ID list is scrolled near the end", async () => { + const fetchNextPage = vi.fn(); + vi.mocked(useInfiniteUsers).mockReturnValue({ + ...emptyInfiniteQuery, + fetchNextPage, + hasNextPage: true, + data: { + pages: [ + { + users: [{ user_id: "user-1", user_alias: "Alice", user_email: "alice@example.com" }], + page: 1, + page_size: 50, + total: 51, + total_pages: 2, + }, + ], + pageParams: [1], + }, + } as unknown as ReturnType); + const user = userEvent.setup(); + renderFilters(); + + await user.click(await screen.findByPlaceholderText("Search an internal user")); + const list = await screen.findByTestId("paginated-search-select-list"); + Object.defineProperty(list, "scrollTop", { value: 90, configurable: true }); + Object.defineProperty(list, "clientHeight", { value: 10, configurable: true }); + Object.defineProperty(list, "scrollHeight", { value: 100, configurable: true }); + list.dispatchEvent(new Event("scroll", { bubbles: true })); + + await waitFor(() => expect(fetchNextPage).toHaveBeenCalled()); + }); + it("does not show or query the User ID filter for non-admin request logs", () => { renderFilters({}, false); From 297fe272ecce15832565a6cc27c13763160944bb Mon Sep 17 00:00:00 2001 From: Daniel Meismer Date: Thu, 13 Aug 2026 11:50:43 -0400 Subject: [PATCH 05/26] feat: scope request log user filter Add a bounded spend-log user facet for the Request Logs picker and intersect explicit user filters with the caller's own and permitted-team scope. Co-Authored-By: Codex --- litellm/proxy/_types.py | 6 +- .../management_v1/spend_logs.py | 222 +++++++++++------- .../spend_management_endpoints.py | 17 +- .../management_v1/test_spend_logs.py | 83 +++++-- .../test_spend_management_endpoints.py | 78 +++++- .../hooks/spendLogs/useSpendLogUsers.test.ts | 40 ++++ .../hooks/spendLogs/useSpendLogUsers.ts | 21 ++ .../view_logs/RequestLogsFilters.test.tsx | 71 ++---- .../view_logs/RequestLogsFilters.tsx | 41 ++-- .../components/view_logs/RequestLogsPanel.tsx | 5 - .../components/view_logs/RequestLogsTable.tsx | 12 +- .../view_logs/log_filter_logic.test.tsx | 12 +- .../components/view_logs/log_filter_logic.tsx | 5 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 60 +++++ 14 files changed, 470 insertions(+), 203 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/spendLogs/useSpendLogUsers.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/spendLogs/useSpendLogUsers.ts diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index bb330d00756..d628d956e73 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -679,6 +679,7 @@ class LiteLLMRoutes(enum.Enum): # permitted teams exactly like /spend/logs/ui — it belongs to the same # access tier, not to customer management. "/management/v1/spend_logs/end_users", + "/management/v1/spend_logs/users", "/cost/estimate", ] @@ -871,12 +872,13 @@ class LiteLLMRoutes(enum.Enum): # PROXY_ADMIN_VIEW_ONLY — the route gate must match). "/customer/list", "/customer/info", - # UI Logs page detail drawer (single + session) and the end-user filter - # facet. The list endpoint `/spend/logs/ui` is covered via + # UI Logs page detail drawer (single + session) and the filter facets. + # The list endpoint `/spend/logs/ui` is covered via # spend_tracking_routes below. "/spend/logs/ui/{logId}", "/spend/logs/session/ui", "/management/v1/spend_logs/end_users", + "/management/v1/spend_logs/users", # Settings / observability read endpoints exposed in admin-only # sidebar groups (Logging & Alerts, Admin Settings, Budgets, # Invitations). diff --git a/litellm/proxy/management_endpoints/management_v1/spend_logs.py b/litellm/proxy/management_endpoints/management_v1/spend_logs.py index 96e60fcfdfc..5fee8eaede3 100644 --- a/litellm/proxy/management_endpoints/management_v1/spend_logs.py +++ b/litellm/proxy/management_endpoints/management_v1/spend_logs.py @@ -1,7 +1,7 @@ """`/management/v1/spend_logs` facets.""" from datetime import datetime, timezone -from typing import Annotated, Any, Final +from typing import Annotated, Any, Final, Literal from fastapi import APIRouter, Depends, Query, Request @@ -35,7 +35,7 @@ def _as_utc(value: datetime) -> datetime: return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc) -async def _end_user_scope_clause( +async def _spend_log_scope_clause( user_api_key_dict: UserAPIKeyAuth, prisma_client: PrismaClient, next_param_index: int, @@ -43,8 +43,8 @@ async def _end_user_scope_clause( """SQL predicate restricting the facet to spend logs this caller may read. Returns ``(None, ())`` for a proxy admin. Mirrors the scoping ``/spend/logs/ui`` - applies, so the dropdown can never offer an end user whose rows the caller - could not open. + applies, so a dropdown can never offer a value from a row the caller could + not open. """ from litellm.proxy.spend_tracking.spend_management_endpoints import ( _get_permitted_team_ids_for_spend_logs, @@ -77,6 +77,98 @@ async def _end_user_scope_clause( return f"({' OR '.join(clauses)})", params +async def _list_spend_log_facet( + request: Request, + user_api_key_dict: UserAPIKeyAuth, + start_time: datetime, + end_time: datetime, + q: str | None, + page: int, + page_size: int, + column: Literal["end_user", "user"], +) -> FacetListResponse: + try: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}database-not-connected", + title="Database not connected", + status=503, + detail=CommonProxyErrors.db_not_connected_error.value, + ) + ) + + column_sql: Final = "end_user" if column == "end_user" else '"user"' + window_params: Final[tuple[Any, ...]] = (_as_utc(start_time), _as_utc(end_time)) + search_params: Final[tuple[Any, ...]] = (f"%{escape_like(q)}%",) if q else () + search_clause: Final = (f"{column_sql} ILIKE ${len(window_params) + 1} ESCAPE '\\'",) if q else () + + scope_clause, scope_params = await _spend_log_scope_clause( + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + next_param_index=len(window_params) + len(search_params) + 1, + ) + + where_parts: Final = ( + ( + "\"startTime\" >= ($1::timestamptz AT TIME ZONE 'UTC')", + "\"startTime\" <= ($2::timestamptz AT TIME ZONE 'UTC')", + f"{column_sql} IS NOT NULL", + f"{column_sql} != ''", + ) + + search_clause + + ((scope_clause,) if scope_clause is not None else ()) + ) + + # The inner LIMIT walks the startTime index newest first and bounds the + # rows DISTINCT can inspect. request_id makes the cut-off deterministic, + # and page_size + 1 reveals has_more without a COUNT(*). + params: Final = ( + window_params + + search_params + + scope_params + + (SPEND_LOGS_FACET_SCAN_CAP, page_size + 1, (page - 1) * page_size) + ) + scan_idx: Final = len(params) - 2 + facet_sql: Final = ( + f"SELECT DISTINCT {column_sql} FROM (" + f" SELECT {column_sql}" + f' FROM "LiteLLM_SpendLogs"' + f" WHERE {' AND '.join(where_parts)}" + f' ORDER BY "startTime" DESC, request_id DESC' + f" LIMIT ${scan_idx}" + f") recent" + f" ORDER BY {column_sql} ASC" + f" LIMIT ${scan_idx + 1} OFFSET ${scan_idx + 2}" + ) + rows: Final = await prisma_client.db.query_raw(facet_sql, *params) + values: Final[list[str]] = [row[column] for row in rows if row.get(column)] + has_more: Final = len(values) > page_size + + return FacetListResponse( + data=values[:page_size], + meta=PageMeta(page=page, page_size=page_size, has_more=has_more), + links=build_page_links(request=request, page=page, has_more=has_more), + ) + except ManagementProblem: + raise + except Exception as e: + verbose_proxy_logger.exception( + "litellm.proxy.management_endpoints.management_v1.spend_logs._list_spend_log_facet(): Exception occured - %s", + e, + ) + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}internal-server-error", + title="Internal server error", + status=500, + detail=f"Failed to list spend log {column.replace('_', ' ')}s.", + ) + ) + + @router.get( "/spend_logs/end_users", tags=["Budget & Spend Tracking"], @@ -116,85 +208,47 @@ async def list_spend_log_end_users( --header 'Authorization: Bearer sk-1234' ``` """ - try: - from litellm.proxy.proxy_server import prisma_client + return await _list_spend_log_facet( + request=request, + user_api_key_dict=user_api_key_dict, + start_time=start_time, + end_time=end_time, + q=q, + page=page, + page_size=page_size, + column="end_user", + ) - if prisma_client is None: - raise ManagementProblem( - ProblemDetail( - type=f"{PROBLEM_TYPE_BASE}database-not-connected", - title="Database not connected", - status=503, - detail=CommonProxyErrors.db_not_connected_error.value, - ) - ) - window_params: Final[tuple[Any, ...]] = (_as_utc(start_time), _as_utc(end_time)) - search_params: Final[tuple[Any, ...]] = (f"%{escape_like(q)}%",) if q else () - search_clause: Final = (f"end_user ILIKE ${len(window_params) + 1} ESCAPE '\\'",) if q else () - - scope_clause, scope_params = await _end_user_scope_clause( - user_api_key_dict=user_api_key_dict, - prisma_client=prisma_client, - next_param_index=len(window_params) + len(search_params) + 1, - ) - - where_parts: Final = ( - ( - "\"startTime\" >= ($1::timestamptz AT TIME ZONE 'UTC')", - "\"startTime\" <= ($2::timestamptz AT TIME ZONE 'UTC')", - "end_user IS NOT NULL", - "end_user != ''", - ) - + search_clause - + ((scope_clause,) if scope_clause is not None else ()) - ) - - # The inner LIMIT is the safety bound: it walks the startTime index newest - # first and stops, so DISTINCT never runs over an unbounded row set. - # request_id breaks startTime ties so the cut-off row is deterministic and - # successive OFFSET pages agree on the set they are paging through. - # page_size + 1: one row beyond the page reveals has_more without a COUNT(*). - params: Final = ( - window_params - + search_params - + scope_params - + (SPEND_LOGS_FACET_SCAN_CAP, page_size + 1, (page - 1) * page_size) - ) - scan_idx: Final = len(params) - 2 - facet_sql: Final = ( - f"SELECT DISTINCT end_user FROM (" - f" SELECT end_user" - f' FROM "LiteLLM_SpendLogs"' - f" WHERE {' AND '.join(where_parts)}" - f' ORDER BY "startTime" DESC, request_id DESC' - f" LIMIT ${scan_idx}" - f") recent" - f" ORDER BY end_user ASC" - f" LIMIT ${scan_idx + 1} OFFSET ${scan_idx + 2}" - ) - rows: Final = await prisma_client.db.query_raw(facet_sql, *params) - end_users: Final[list[str]] = [row["end_user"] for row in rows if row.get("end_user")] - has_more: Final = len(end_users) > page_size - - return FacetListResponse( - data=end_users[:page_size], - meta=PageMeta(page=page, page_size=page_size, has_more=has_more), - links=build_page_links(request=request, page=page, has_more=has_more), - ) - - except ManagementProblem: - raise - except Exception as e: - verbose_proxy_logger.exception( - "litellm.proxy.management_endpoints.management_v1.spend_logs.list_spend_log_end_users(): Exception occured - %s", - e, - ) - raise ManagementProblem( - ProblemDetail( - type=f"{PROBLEM_TYPE_BASE}internal-server-error", - title="Internal server error", - status=500, - detail="Failed to list spend log end users.", - ) - ) +@router.get( + "/spend_logs/users", + tags=["Budget & Spend Tracking"], + dependencies=[Depends(user_api_key_auth), Depends(reject_unknown_query_params)], + response_model=FacetListResponse, +) +async def list_spend_log_users( + request: Request, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + start_time: Annotated[ + datetime, + Query(alias="filter[startTime][gte]", description="Window start (UTC when no offset is given)"), + ], + end_time: Annotated[ + datetime, + Query(alias="filter[startTime][lte]", description="Window end (UTC when no offset is given)"), + ], + q: Annotated[str | None, Query(description="Case-insensitive partial match on the internal user id")] = None, + page: Annotated[int, Query(ge=1, description="Page number")] = 1, + page_size: Annotated[int, Query(ge=1, le=100, description="Page size")] = 50, +) -> FacetListResponse: + """The distinct internal users appearing in spend logs the caller can read.""" + return await _list_spend_log_facet( + request=request, + user_api_key_dict=user_api_key_dict, + start_time=start_time, + end_time=end_time, + q=q, + page=page, + page_size=page_size, + column="user", + ) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 8fb5570965b..99d870f5ad4 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -2427,6 +2427,7 @@ async def ui_view_spend_logs( request_id=request_id, ) permitted_team_ids: list[str] | None = None + scope_to_caller_user = False if not is_request_id_lookup and not is_admin_view: if team_id is not None: can_view_team: Final = await _can_team_member_view_log( @@ -2440,7 +2441,6 @@ async def ui_view_spend_logs( detail={"error": f"Not authorized to view team spend for team_id={team_id}"}, ) where_conditions["team_id"] = team_id - where_conditions.pop("user", None) else: if _can_user_view_spend_log(user_api_key_dict=user_api_key_dict): try: @@ -2451,13 +2451,20 @@ async def ui_view_spend_logs( except Exception: permitted_team_ids = [] if permitted_team_ids: - where_conditions.pop("user", None) + if user_id is None: + where_conditions.pop("user", None) where_conditions["OR"] = [ {"user": user_api_key_dict.user_id}, {"team_id": {"in": permitted_team_ids}}, ] else: - where_conditions["user"] = user_api_key_dict.user_id + if user_id is None: + where_conditions["user"] = user_api_key_dict.user_id + else: + where_conditions["AND"] = where_conditions.get("AND", []) + [ + {"user": user_api_key_dict.user_id} + ] + scope_to_caller_user = True where_conditions.pop("team_id", None) # Calculate skip value for pagination skip: Final = (page - 1) * page_size @@ -2508,6 +2515,10 @@ async def ui_view_spend_logs( sql_params.append(permitted_team_ids) p += 2 sql_conditions.append(or_clause) + elif scope_to_caller_user: + sql_conditions.append(f'"user" = ${p}') + sql_params.append(user_api_key_dict.user_id) + p += 1 if session_id is not None and isinstance(session_id, str): like_escaped_session_id: Final = session_id.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") diff --git a/tests/test_litellm/proxy/management_endpoints/management_v1/test_spend_logs.py b/tests/test_litellm/proxy/management_endpoints/management_v1/test_spend_logs.py index 79f13a6f703..35fcd3b6cd7 100644 --- a/tests/test_litellm/proxy/management_endpoints/management_v1/test_spend_logs.py +++ b/tests/test_litellm/proxy/management_endpoints/management_v1/test_spend_logs.py @@ -1,5 +1,4 @@ from datetime import datetime, timezone -from typing import List from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -45,6 +44,7 @@ app.include_router(router) client = TestClient(app) END_USERS_PATH = f"{MANAGEMENT_V1_PREFIX}/spend_logs/end_users" +USERS_PATH = f"{MANAGEMENT_V1_PREFIX}/spend_logs/users" WINDOW = "filter[startTime][gte]=2026-07-23T00:00:00Z&filter[startTime][lte]=2026-07-24T00:00:00Z" @@ -65,7 +65,7 @@ def as_proxy_admin(): app.dependency_overrides.clear() -def _mock_rows(mock_prisma_client, end_users: List[str]) -> AsyncMock: +def _mock_rows(mock_prisma_client, end_users: list[str]) -> AsyncMock: query_raw = AsyncMock(return_value=[{"end_user": eu} for eu in end_users]) mock_prisma_client.db.query_raw = query_raw return query_raw @@ -82,6 +82,11 @@ def _get(query: str = WINDOW): return client.get(f"{END_USERS_PATH}{suffix}", headers={"Authorization": "Bearer k"}) +def _get_users(query: str = WINDOW): + suffix = f"?{query}" if query else "" + return client.get(f"{USERS_PATH}{suffix}", headers={"Authorization": "Bearer k"}) + + def test_returns_the_control_plane_envelope(mock_prisma_client, as_proxy_admin): """`{data, meta, links}` is the contract; a bare list or a legacy `aliases` key is not.""" _mock_rows(mock_prisma_client, ["a", "b"]) @@ -213,7 +218,7 @@ def test_requires_a_time_window(mock_prisma_client, as_proxy_admin, query): def test_rejects_a_malformed_window_as_a_problem_document(mock_prisma_client, as_proxy_admin): _mock_rows(mock_prisma_client, []) - response = _get(f"filter[startTime][gte]=yesterday&filter[startTime][lte]=2026-07-24T00:00:00Z") + response = _get("filter[startTime][gte]=yesterday&filter[startTime][lte]=2026-07-24T00:00:00Z") assert response.status_code == 400 assert response.headers["content-type"].startswith("application/problem+json") @@ -400,6 +405,49 @@ def test_q_placeholder_precedes_the_scan_limit_and_offset(mock_prisma_client, as assert query_raw.call_args.args[5:] == (11, 0) +def test_user_facet_reads_internal_users_from_spend_logs(mock_prisma_client, as_proxy_admin): + query_raw = AsyncMock(return_value=[{"user": "alice@example.com"}, {"user": "user-42"}]) + mock_prisma_client.db.query_raw = query_raw + + response = _get_users() + + assert response.status_code == 200 + assert response.json()["data"] == ["alice@example.com", "user-42"] + sql = query_raw.call_args.args[0] + assert 'SELECT DISTINCT "user"' in sql + assert '"user" IS NOT NULL' in sql + assert "end_user IS NOT NULL" not in sql + + +def test_user_facet_uses_the_same_team_scope_as_request_logs(mock_prisma_client): + query_raw = AsyncMock(return_value=[{"user": "member@example.com"}]) + mock_prisma_client.db.query_raw = query_raw + original = _as_role(LitellmUserRoles.INTERNAL_USER, user_id="team-admin-1") + try: + with patch( + "litellm.proxy.spend_tracking.spend_management_endpoints._get_permitted_team_ids_for_spend_logs", + new=AsyncMock(return_value=["team-a"]), + ): + response = _get_users() + finally: + app.dependency_overrides = original + + assert response.status_code == 200 + assert '("user" = $3 OR team_id = ANY($4::text[]))' in query_raw.call_args.args[0] + assert query_raw.call_args.args[3] == "team-admin-1" + assert query_raw.call_args.args[4] == ["team-a"] + + +def test_user_facet_searches_the_internal_user_value(mock_prisma_client, as_proxy_admin): + query_raw = AsyncMock(return_value=[]) + mock_prisma_client.db.query_raw = query_raw + + _get_users(f"{WINDOW}&q=alice%40example.com") + + assert '"user" ILIKE $3 ESCAPE' in query_raw.call_args.args[0] + assert query_raw.call_args.args[3] == "%alice@example.com%" + + @pytest.mark.parametrize( "role", [ @@ -416,18 +464,19 @@ def test_is_reachable_by_every_role_that_can_open_the_logs_page(role): """ from litellm.proxy.auth.route_checks import RouteChecks - for allowed in ( - LiteLLMRoutes.internal_user_routes.value, - LiteLLMRoutes.internal_user_view_only_routes.value, - ): - assert ("/spend/logs/ui" in allowed) == (END_USERS_PATH in allowed) + for facet_path in (END_USERS_PATH, USERS_PATH): + for allowed in ( + LiteLLMRoutes.internal_user_routes.value, + LiteLLMRoutes.internal_user_view_only_routes.value, + ): + assert ("/spend/logs/ui" in allowed) == (facet_path in allowed) - if role in (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY): - allowed_routes = ( - LiteLLMRoutes.internal_user_routes.value - if role == LitellmUserRoles.INTERNAL_USER - else LiteLLMRoutes.internal_user_view_only_routes.value - ) - assert RouteChecks.check_route_access(route=END_USERS_PATH, allowed_routes=allowed_routes) - else: - assert END_USERS_PATH in LiteLLMRoutes.admin_viewer_routes.value + if role in (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY): + allowed_routes = ( + LiteLLMRoutes.internal_user_routes.value + if role == LitellmUserRoles.INTERNAL_USER + else LiteLLMRoutes.internal_user_view_only_routes.value + ) + assert RouteChecks.check_route_access(route=facet_path, allowed_routes=allowed_routes) + else: + assert facet_path in LiteLLMRoutes.admin_viewer_routes.value diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 057193a69db..81512cd8e66 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -1321,7 +1321,7 @@ async def test_ui_view_spend_logs_internal_user_scoped_without_user_id( @pytest.mark.asyncio -async def test_ui_view_spend_logs_team_admin_can_view_team_spend(client, monkeypatch): +async def test_ui_view_spend_logs_team_admin_can_filter_team_spend_by_user(client, monkeypatch): """ Team admins should be able to view team-wide spend when team_id is provided. """ @@ -1346,11 +1346,23 @@ async def test_ui_view_spend_logs_team_admin_can_view_team_spend(client, monkeyp "startTime": datetime.datetime.now(timezone.utc).isoformat(), "model": "gpt-4", }, + { + "id": "log3", + "request_id": "req3", + "api_key": "sk-test-key", + "user": "member3", + "team_id": "team_admin_team", + "spend": 0.15, + "startTime": datetime.datetime.now(timezone.utc).isoformat(), + "model": "gpt-4", + }, ] def filter_by_team(where): - if "team_id" in where and where["team_id"] == "team_admin_team": + if where.get("team_id") == "team_admin_team" and where.get("user") == "member1": return [mock_spend_logs[0]] + if where.get("team_id") == "team_admin_team": + return [mock_spend_logs[0], mock_spend_logs[2]] return mock_spend_logs class TeamTable: @@ -1383,6 +1395,7 @@ async def test_ui_view_spend_logs_team_admin_can_view_team_spend(client, monkeyp "/spend/logs/ui", params={ "team_id": "team_admin_team", + "user_id": "member1", "start_date": start_date, "end_date": end_date, }, @@ -1398,6 +1411,66 @@ async def test_ui_view_spend_logs_team_admin_can_view_team_spend(client, monkeyp app.dependency_overrides.pop(ps.user_api_key_auth, None) +@pytest.mark.asyncio +async def test_ui_view_spend_logs_user_filter_intersects_permitted_team_scope(client, monkeypatch): + member_log = { + "id": "log1", + "request_id": "req1", + "api_key": "sk-test-key", + "user": "member@example.com", + "team_id": "team-9", + "spend": 0.05, + "startTime": datetime.datetime.now(timezone.utc).isoformat(), + "model": "gpt-4", + } + other_team_log = { + **member_log, + "id": "log2", + "request_id": "req2", + "team_id": "team-outside-scope", + } + seen_where = [] + + def filter_by_user_and_scope(where): + seen_where.append(where) + if where.get("user") == "member@example.com" and {"multi_team": True} in where.get("OR", []): + return [member_log] + return [member_log, other_team_log] + + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + make_ui_spend_logs_mock_prisma([member_log, other_team_log], filter_by_user_and_scope), + ) + monkeypatch.setattr( + "litellm.proxy.spend_tracking.spend_management_endpoints._get_permitted_team_ids_for_spend_logs", + AsyncMock(return_value=["team-9"]), + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="team-admin" + ) + + try: + start_date, end_date = _default_date_range() + response = client.get( + "/spend/logs/ui", + params={ + "user_id": "member@example.com", + "start_date": start_date, + "end_date": end_date, + }, + headers={"Authorization": "Bearer sk-test"}, + ) + + assert response.status_code == 200 + assert [row["request_id"] for row in response.json()["data"]] == ["req1"] + assert any( + where.get("user") == "member@example.com" and {"multi_team": True} in where.get("OR", []) + for where in seen_where + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + @pytest.mark.asyncio async def test_ui_view_spend_logs_pagination(client, monkeypatch): mock_spend_logs = [ @@ -1578,6 +1651,7 @@ async def test_ui_view_session_spend_logs_pagination(client, monkeypatch): assert data["total_pages"] == 2 assert len(data["data"]) == 1 assert data["data"][0]["request_id"] == "req1" + assert data["data"][0]["user"] == "member1" finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/spendLogs/useSpendLogUsers.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/spendLogs/useSpendLogUsers.test.ts new file mode 100644 index 00000000000..5a79bf74ce3 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/spendLogs/useSpendLogUsers.test.ts @@ -0,0 +1,40 @@ +import { renderHook } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const useInfiniteQuery = vi.fn(); +vi.mock("@/lib/http/api", () => ({ $api: { useInfiniteQuery: (...args: unknown[]) => useInfiniteQuery(...args) } })); + +const mockUseAuthorized = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => mockUseAuthorized(), +})); + +import { useInfiniteSpendLogUsers } from "./useSpendLogUsers"; + +const WINDOW = { start_date: "2026-07-23 00:00:00", end_date: "2026-07-24 00:00:00" }; + +describe("useInfiniteSpendLogUsers", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockUseAuthorized.mockReturnValue({ accessToken: "test-token" }); + }); + + it("calls the scoped spend-log user facet with the visible window", () => { + renderHook(() => useInfiniteSpendLogUsers(WINDOW, 25, "alice")); + + const expectedQuery = { + "filter[startTime][gte]": "2026-07-23 00:00:00", + "filter[startTime][lte]": "2026-07-24 00:00:00", + page_size: 25, + q: "alice", + }; + expect(useInfiniteQuery.mock.calls[0][1]).toBe("/management/v1/spend_logs/users"); + expect(useInfiniteQuery.mock.calls[0][2].params.query).toEqual(expectedQuery); + }); + + it("omits q when the search box is empty", () => { + renderHook(() => useInfiniteSpendLogUsers(WINDOW, 50, "")); + + expect(useInfiniteQuery.mock.calls[0][2].params.query).not.toHaveProperty("q"); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/spendLogs/useSpendLogUsers.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/spendLogs/useSpendLogUsers.ts new file mode 100644 index 00000000000..3a82c9e9d91 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/spendLogs/useSpendLogUsers.ts @@ -0,0 +1,21 @@ +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { $api } from "@/lib/http/api"; + +import { nextPageFromLinks, type SpendLogsWindow } from "./useSpendLogEndUsers"; + +export const useInfiniteSpendLogUsers = (window: SpendLogsWindow, pageSize: number = 50, q?: string) => { + const { accessToken } = useAuthorized(); + const query = { + "filter[startTime][gte]": window.start_date, + "filter[startTime][lte]": window.end_date, + page_size: pageSize, + ...(q !== undefined && q !== "" ? { q } : {}), + }; + const options = { + pageParamName: "page", + initialPageParam: 1, + getNextPageParam: nextPageFromLinks, + enabled: Boolean(accessToken), + }; + return $api.useInfiniteQuery("get", "/management/v1/spend_logs/users", { params: { query } }, options); +}; diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx index fd53949c87a..1c94e6418a0 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx @@ -14,8 +14,8 @@ vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({ useInfiniteModelInfo: vi.fn(), })); -vi.mock("@/app/(dashboard)/hooks/users/useUsers", () => ({ - useInfiniteUsers: vi.fn(), +vi.mock("@/app/(dashboard)/hooks/spendLogs/useSpendLogUsers", () => ({ + useInfiniteSpendLogUsers: vi.fn(), })); vi.mock("@/app/(dashboard)/hooks/spendLogs/useSpendLogEndUsers", () => ({ @@ -23,9 +23,9 @@ vi.mock("@/app/(dashboard)/hooks/spendLogs/useSpendLogEndUsers", () => ({ })); import { useInfiniteSpendLogEndUsers } from "@/app/(dashboard)/hooks/spendLogs/useSpendLogEndUsers"; +import { useInfiniteSpendLogUsers } from "@/app/(dashboard)/hooks/spendLogs/useSpendLogUsers"; import { useInfiniteKeyAliases } from "@/app/(dashboard)/hooks/keys/useKeyAliases"; import { useInfiniteModelInfo } from "@/app/(dashboard)/hooks/models/useModels"; -import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers"; const emptyInfiniteQuery = { data: { pages: [], pageParams: [] }, @@ -37,16 +37,10 @@ const emptyInfiniteQuery = { const LOGS_WINDOW = { start_date: "2026-07-23 00:00:00", end_date: "2026-07-24 00:00:00" }; -function renderFilters(filters: Record = {}, showUserIdFilter = true) { +function renderFilters(filters: Record = {}) { const set = vi.fn(); renderWithProviders( - filters[id]} - set={set} - teams={[]} - logsWindow={LOGS_WINDOW} - showUserIdFilter={showUserIdFilter} - />, + filters[id]} set={set} teams={[]} logsWindow={LOGS_WINDOW} />, ); return { set }; } @@ -61,8 +55,8 @@ describe("RequestLogsFilters", () => { vi.mocked(useInfiniteModelInfo).mockReturnValue( emptyInfiniteQuery as unknown as ReturnType, ); - vi.mocked(useInfiniteUsers).mockReturnValue( - emptyInfiniteQuery as unknown as ReturnType, + vi.mocked(useInfiniteSpendLogUsers).mockReturnValue( + emptyInfiniteQuery as unknown as ReturnType, ); vi.mocked(useInfiniteSpendLogEndUsers).mockReturnValue( emptyInfiniteQuery as unknown as ReturnType, @@ -97,31 +91,27 @@ describe("RequestLogsFilters", () => { expect(labels[1].compareDocumentPosition(labels[2]) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); }); - it("selects a user by display name while storing the user ID filter", async () => { - vi.mocked(useInfiniteUsers).mockReturnValue({ + it("selects an internal user value from the caller's visible spend logs", async () => { + vi.mocked(useInfiniteSpendLogUsers).mockReturnValue({ ...emptyInfiniteQuery, data: { pages: [ { - users: [{ user_id: "user-1", user_alias: "Alice", user_email: "alice@example.com" }], - page: 1, - page_size: 50, - total: 1, - total_pages: 1, + data: ["alice@example.com"], + meta: { page: 1, page_size: 50, has_more: false }, + links: { self: "", next: null }, }, ], pageParams: [1], }, - } as unknown as ReturnType); + } as unknown as ReturnType); const user = userEvent.setup(); const { set } = renderFilters(); await user.click(await screen.findByPlaceholderText("Search an internal user")); - expect(await screen.findByText("Alice")).toBeInTheDocument(); - expect(screen.getByText("alice@example.com | User ID: user-1")).toBeInTheDocument(); - await user.click(screen.getByText("Alice")); + await user.click(await screen.findByText("alice@example.com")); - expect(set).toHaveBeenCalledWith(LOG_FILTER_IDS.USER_ID, "user-1"); + expect(set).toHaveBeenCalledWith(LOG_FILTER_IDS.USER_ID, "alice@example.com"); }); it("pushes the User ID picker query to the paginated user lookup", async () => { @@ -132,28 +122,26 @@ describe("RequestLogsFilters", () => { await user.click(input); await user.type(input, "alice@example.com"); - await waitFor(() => expect(useInfiniteUsers).toHaveBeenCalledWith(50, "alice@example.com")); + await waitFor(() => expect(useInfiniteSpendLogUsers).toHaveBeenCalledWith(LOGS_WINDOW, 50, "alice@example.com")); }); it("loads the next page when the User ID list is scrolled near the end", async () => { const fetchNextPage = vi.fn(); - vi.mocked(useInfiniteUsers).mockReturnValue({ + vi.mocked(useInfiniteSpendLogUsers).mockReturnValue({ ...emptyInfiniteQuery, fetchNextPage, hasNextPage: true, data: { pages: [ { - users: [{ user_id: "user-1", user_alias: "Alice", user_email: "alice@example.com" }], - page: 1, - page_size: 50, - total: 51, - total_pages: 2, + data: ["alice@example.com"], + meta: { page: 1, page_size: 50, has_more: true }, + links: { self: "", next: "?page=2" }, }, ], pageParams: [1], }, - } as unknown as ReturnType); + } as unknown as ReturnType); const user = userEvent.setup(); renderFilters(); @@ -167,13 +155,6 @@ describe("RequestLogsFilters", () => { await waitFor(() => expect(fetchNextPage).toHaveBeenCalled()); }); - it("does not show or query the User ID filter for non-admin request logs", () => { - renderFilters({}, false); - - expect(screen.queryByText("User ID")).not.toBeInTheDocument(); - expect(useInfiniteUsers).not.toHaveBeenCalled(); - }); - it("scopes the Key Alias lookup to the selected team", async () => { renderFilters({ [LOG_FILTER_IDS.TEAM_ID]: "team-42" }); @@ -264,15 +245,7 @@ describe("RequestLogsFilters", () => { it("scopes the End User lookup to the window the logs table is showing", async () => { const otherWindow = { start_date: "2026-01-01 00:00:00", end_date: "2026-01-02 00:00:00" }; - renderWithProviders( - undefined} - set={vi.fn()} - teams={[]} - logsWindow={otherWindow} - showUserIdFilter - />, - ); + renderWithProviders( undefined} set={vi.fn()} teams={[]} logsWindow={otherWindow} />); await waitFor(() => expect(useInfiniteSpendLogEndUsers).toHaveBeenCalledWith(otherWindow, 50, undefined)); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.tsx index 47e0bad6f62..017260230dd 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.tsx @@ -3,9 +3,9 @@ import { useMemo, useState } from "react"; import { useInfiniteSpendLogEndUsers } from "@/app/(dashboard)/hooks/spendLogs/useSpendLogEndUsers"; +import { useInfiniteSpendLogUsers } from "@/app/(dashboard)/hooks/spendLogs/useSpendLogUsers"; import { useInfiniteKeyAliases } from "@/app/(dashboard)/hooks/keys/useKeyAliases"; import { useInfiniteModelInfo } from "@/app/(dashboard)/hooks/models/useModels"; -import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers"; import { DataTableFilterField } from "@/components/shared/DataTable"; import { PaginatedSearchSelect } from "@/components/shared/PaginatedSearchSelect"; import { SearchSelect, type SearchSelectOption } from "@/components/shared/SearchSelect"; @@ -145,9 +145,18 @@ function ModelFilterField({ value, onChange }: { value: string; onChange: (value ); } -function UserIdFilterField({ value, onChange }: { value: string; onChange: (value: string | undefined) => void }) { +function UserIdFilterField({ + value, + onChange, + logsWindow, +}: { + value: string; + onChange: (value: string | undefined) => void; + logsWindow: LogsWindow; +}) { const [search, setSearch] = useState(""); - const { data, fetchNextPage, hasNextPage, isFetchingNextPage, isLoading } = useInfiniteUsers( + const { data, fetchNextPage, hasNextPage, isFetchingNextPage, isLoading } = useInfiniteSpendLogUsers( + logsWindow, PAGE_SIZE, emptyToUndefined(search), ); @@ -155,14 +164,10 @@ function UserIdFilterField({ value, onChange }: { value: string; onChange: (valu const options = useMemo(() => { const seen = new Set(); return (data?.pages ?? []).flatMap((page) => - page.users.flatMap((user) => { - if (!user.user_id || seen.has(user.user_id)) return []; - seen.add(user.user_id); - const label = user.user_alias || user.user_email || user.user_id; - const email = user.user_email && user.user_email !== label ? user.user_email : ""; - const sublabel = - user.user_id === label ? email : [email, `User ID: ${user.user_id}`].filter(Boolean).join(" | "); - return [{ label, value: user.user_id, sublabel }]; + page.data.flatMap((userId) => { + if (!userId || seen.has(userId)) return []; + seen.add(userId); + return [{ label: userId, value: userId }]; }), ); }, [data]); @@ -284,10 +289,9 @@ interface RequestLogsFiltersProps { set: (columnId: string, value: unknown) => void; teams: Team[]; logsWindow: LogsWindow; - showUserIdFilter: boolean; } -export function RequestLogsFilters({ get, set, teams, logsWindow, showUserIdFilter }: RequestLogsFiltersProps) { +export function RequestLogsFilters({ get, set, teams, logsWindow }: RequestLogsFiltersProps) { const valueOf = (id: string): string => asString(get(id)); const setter = (id: string) => (next: string | undefined) => set(id, next); @@ -321,12 +325,11 @@ export function RequestLogsFilters({ get, set, teams, logsWindow, showUserIdFilt teamId={valueOf(LOG_FILTER_IDS.TEAM_ID)} /> - {showUserIdFilter && ( - - )} + void; teams: Team[]; logsWindow: LogsWindow; - showUserIdFilter: boolean; toolbarChildren?: ReactNode; } @@ -70,7 +69,6 @@ export function RequestLogsTable({ onSessionClick, teams, logsWindow, - showUserIdFilter, toolbarChildren, }: RequestLogsTableProps) { const [filtersOpen, setFiltersOpen] = useState(false); @@ -124,15 +122,7 @@ export function RequestLogsTable({ title="Filters" description="Narrow down request logs" > - {({ get, set }) => ( - - )} + {({ get, set }) => } )} diff --git a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.test.tsx b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.test.tsx index 17d26dc00f3..26c5bda1593 100644 --- a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.test.tsx @@ -45,7 +45,6 @@ const defaultProps = { userRole: "Admin" as string | null, userID: "user-1" as string | null, columnFilters: [] as ColumnFiltersState, - filterByCurrentUser: false, activeTab: "request logs", isLiveTail: false, startTime: "2025-01-01T00:00:00", @@ -181,17 +180,16 @@ describe("useLogFilterLogic", () => { }); }); - describe("filterByCurrentUser", () => { - it("scopes to the current user when no explicit user filter is set", async () => { - renderFilterHook({ filterByCurrentUser: true }); + describe("user scope", () => { + it("leaves an empty user filter for the backend to authorize", async () => { + renderFilterHook(); await waitFor(() => expect(uiSpendLogsCall).toHaveBeenCalled()); - expect(lastCallParams()?.params).toMatchObject({ user_id: "user-1" }); + expect(lastCallParams()?.params?.user_id).toBeUndefined(); }); - it("lets an explicit user filter win over the current-user scope", async () => { + it("sends an explicit user filter for the backend to intersect with authorization", async () => { renderFilterHook({ - filterByCurrentUser: true, columnFilters: [{ id: LOG_FILTER_IDS.USER_ID, value: "someone-else" }], }); diff --git a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx index 78ecb52c184..474f51e93b3 100644 --- a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx @@ -99,7 +99,6 @@ export function useLogFilterLogic({ userRole, userID, columnFilters, - filterByCurrentUser, activeTab, isLiveTail, startTime, @@ -113,7 +112,6 @@ export function useLogFilterLogic({ userRole: string | null; userID: string | null; columnFilters: ColumnFiltersState; - filterByCurrentUser: boolean | null; activeTab: string; isLiveTail: boolean; startTime: string; @@ -137,7 +135,6 @@ export function useLogFilterLogic({ endTime, isCustomDate, columnFilters, - filterByCurrentUser ? userID : null, sortBy, sortOrder, ], @@ -167,7 +164,7 @@ export function useLogFilterLogic({ team_id: getFilterValue(columnFilters, LOG_FILTER_IDS.TEAM_ID), request_id: getFilterValue(columnFilters, LOG_FILTER_IDS.REQUEST_ID), session_id: getFilterValue(columnFilters, LOG_FILTER_IDS.SESSION_ID), - user_id: userIdFilter ?? (filterByCurrentUser ? userID ?? undefined : undefined), + user_id: userIdFilter, end_user: getFilterValue(columnFilters, LOG_FILTER_IDS.END_USER), status_filter: getFilterValue(columnFilters, LOG_FILTER_IDS.STATUS), model_id: getFilterValue(columnFilters, LOG_FILTER_IDS.MODEL_ID), diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 1e46ae9c577..75e222f7472 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -7529,6 +7529,26 @@ export interface paths { patch?: never; trace?: never; }; + "/management/v1/spend_logs/users": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * List Spend Log Users + * @description The distinct internal users appearing in spend logs the caller can read. + */ + get: operations["list_spend_log_users_management_v1_spend_logs_users_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/mcp-rest/test/connection": { parameters: { query?: never; @@ -45400,6 +45420,46 @@ export interface operations { }; }; }; + list_spend_log_users_management_v1_spend_logs_users_get: { + parameters: { + query: { + /** @description Window start (UTC when no offset is given) */ + "filter[startTime][gte]": string; + /** @description Window end (UTC when no offset is given) */ + "filter[startTime][lte]": string; + /** @description Case-insensitive partial match on the internal user id */ + q?: string | null; + /** @description Page number */ + page?: number; + /** @description Page size */ + page_size?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["FacetListResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; test_connection_mcp_rest_test_connection_post: { parameters: { query?: never; From fac2b6b56b4020b85c423bac11f5b78367b8833e Mon Sep 17 00:00:00 2001 From: Daniel Meismer Date: Thu, 13 Aug 2026 12:12:05 -0400 Subject: [PATCH 06/26] refactor: derive request log scope immutably Resolve the authorized own-user and permitted-team predicates once and add regression coverage for explicit-user intersection, unfiltered team scope, and team lookup failure fallback. Co-Authored-By: Codex --- .../spend_management_endpoints.py | 78 +++++++---- .../test_spend_management_endpoints.py | 125 +++++++++++++++++- 2 files changed, 174 insertions(+), 29 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 99d870f5ad4..ed2ecd8325a 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -2426,8 +2426,23 @@ async def ui_view_spend_logs( user_api_key_dict=user_api_key_dict, request_id=request_id, ) - permitted_team_ids: list[str] | None = None - scope_to_caller_user = False + user_scope_applies: Final = ( + not is_request_id_lookup + and not is_admin_view + and team_id is None + and _can_user_view_spend_log(user_api_key_dict=user_api_key_dict) + ) + permitted_team_ids: Final = ( + await _get_permitted_team_ids_for_spend_logs_or_empty( + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + ) + if user_scope_applies + else () + ) + explicit_user_requires_caller_scope: Final = ( + user_scope_applies and not permitted_team_ids and user_id is not None + ) if not is_request_id_lookup and not is_admin_view: if team_id is not None: can_view_team: Final = await _can_team_member_view_log( @@ -2441,31 +2456,22 @@ async def ui_view_spend_logs( detail={"error": f"Not authorized to view team spend for team_id={team_id}"}, ) where_conditions["team_id"] = team_id - else: - if _can_user_view_spend_log(user_api_key_dict=user_api_key_dict): - try: - permitted_team_ids = await _get_permitted_team_ids_for_spend_logs( - prisma_client=prisma_client, - user_api_key_dict=user_api_key_dict, - ) - except Exception: - permitted_team_ids = [] - if permitted_team_ids: - if user_id is None: - where_conditions.pop("user", None) - where_conditions["OR"] = [ - {"user": user_api_key_dict.user_id}, - {"team_id": {"in": permitted_team_ids}}, - ] + elif user_scope_applies: + if permitted_team_ids: + if user_id is None: + where_conditions.pop("user", None) + where_conditions["OR"] = [ + {"user": user_api_key_dict.user_id}, + {"team_id": {"in": permitted_team_ids}}, + ] + else: + if user_id is None: + where_conditions["user"] = user_api_key_dict.user_id else: - if user_id is None: - where_conditions["user"] = user_api_key_dict.user_id - else: - where_conditions["AND"] = where_conditions.get("AND", []) + [ - {"user": user_api_key_dict.user_id} - ] - scope_to_caller_user = True - where_conditions.pop("team_id", None) + where_conditions["AND"] = where_conditions.get("AND", []) + [ + {"user": user_api_key_dict.user_id} + ] + where_conditions.pop("team_id", None) # Calculate skip value for pagination skip: Final = (page - 1) * page_size @@ -2509,13 +2515,13 @@ async def ui_view_spend_logs( p += 1 # Multi-team OR filter: (user = $X OR team_id = ANY($Y)) - if permitted_team_ids is not None and len(permitted_team_ids) > 0: + if permitted_team_ids: or_clause: Final = f'("user" = ${p} OR team_id = ANY(${p + 1}::text[]))' sql_params.append(user_api_key_dict.user_id) sql_params.append(permitted_team_ids) p += 2 sql_conditions.append(or_clause) - elif scope_to_caller_user: + elif explicit_user_requires_caller_scope: sql_conditions.append(f'"user" = ${p}') sql_params.append(user_api_key_dict.user_id) p += 1 @@ -4283,3 +4289,19 @@ async def _get_permitted_team_ids_for_spend_logs( ): permitted.append(team_obj.team_id) return permitted + + +async def _get_permitted_team_ids_for_spend_logs_or_empty( + prisma_client: PrismaClient, + user_api_key_dict: UserAPIKeyAuth, +) -> tuple[str, ...]: + """Resolve permitted teams once, falling back to the caller's own-user scope.""" + try: + return tuple( + await _get_permitted_team_ids_for_spend_logs( + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + ) + ) + except Exception: + return () diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 81512cd8e66..87bbb1c2f80 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -150,7 +150,7 @@ def _reconstruct_ui_where_from_sql(sql_query, params): return where -def make_ui_spend_logs_mock_prisma(mock_spend_logs, filter_fn, team_lookup_fn=None): +def make_ui_spend_logs_mock_prisma(mock_spend_logs, filter_fn, team_lookup_fn=None, query_observer=None): """ Create a MockPrismaClient for /spend/logs/ui endpoint tests. @@ -177,6 +177,8 @@ def make_ui_spend_logs_mock_prisma(mock_spend_logs, filter_fn, team_lookup_fn=No return [{col: value, "_count": {col: n}} for value, n in tallied.items()] async def query_raw(self, sql_query, *params): + if query_observer is not None: + query_observer(sql_query, params) if "mcp_tool_call_count" in sql_query: return [] filtered = filter_fn(_reconstruct_ui_where_from_sql(sql_query, params)) @@ -1320,6 +1322,127 @@ async def test_ui_view_spend_logs_internal_user_scoped_without_user_id( app.dependency_overrides.pop(ps.user_api_key_auth, None) +@pytest.mark.asyncio +async def test_ui_view_spend_logs_explicit_user_filter_cannot_escape_own_scope(client, monkeypatch): + caller_log = { + "id": "log1", + "request_id": "req1", + "api_key": "sk-test-key", + "user": "caller@example.com", + "team_id": None, + "spend": 0.05, + "startTime": datetime.datetime.now(timezone.utc).isoformat(), + "model": "gpt-4", + } + observed_queries = [] + + def observe_query(sql_query, params): + if 'FROM "LiteLLM_SpendLogs"' in sql_query: + observed_queries.append((sql_query, params)) + + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + make_ui_spend_logs_mock_prisma([caller_log], lambda _where: [], query_observer=observe_query), + ) + monkeypatch.setattr( + "litellm.proxy.spend_tracking.spend_management_endpoints._get_permitted_team_ids_for_spend_logs", + AsyncMock(return_value=[]), + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="caller@example.com" + ) + + try: + start_date, end_date = _default_date_range() + response = client.get( + "/spend/logs/ui", + params={ + "user_id": "someone-else@example.com", + "start_date": start_date, + "end_date": end_date, + }, + headers={"Authorization": "Bearer sk-test"}, + ) + + assert response.status_code == 200 + assert response.json()["data"] == [] + page_sql, page_params = next((sql, params) for sql, params in observed_queries if "SELECT\n" in sql) + assert page_sql.count('"user" = $') == 2 + assert page_params[2:4] == ("someone-else@example.com", "caller@example.com") + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_ui_view_spend_logs_without_user_filter_includes_permitted_team_scope(client, monkeypatch): + caller_log = { + "id": "log1", + "request_id": "req1", + "api_key": "sk-test-key", + "user": "team-admin@example.com", + "team_id": None, + "spend": 0.05, + "startTime": datetime.datetime.now(timezone.utc).isoformat(), + "model": "gpt-4", + } + member_log = {**caller_log, "id": "log2", "request_id": "req2", "user": "member@example.com", "team_id": "team-9"} + outside_log = { + **caller_log, + "id": "log3", + "request_id": "req3", + "user": "outside@example.com", + "team_id": "outside-team", + } + + def filter_by_scope(where): + if {"multi_team": True} in where.get("OR", []) and "user" not in where: + return [caller_log, member_log] + return [caller_log, member_log, outside_log] + + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + make_ui_spend_logs_mock_prisma([caller_log, member_log, outside_log], filter_by_scope), + ) + monkeypatch.setattr( + "litellm.proxy.spend_tracking.spend_management_endpoints._get_permitted_team_ids_for_spend_logs", + AsyncMock(return_value=["team-9"]), + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="team-admin@example.com" + ) + + try: + start_date, end_date = _default_date_range() + response = client.get( + "/spend/logs/ui", + params={"start_date": start_date, "end_date": end_date}, + headers={"Authorization": "Bearer sk-test"}, + ) + + assert response.status_code == 200 + assert [row["request_id"] for row in response.json()["data"]] == ["req1", "req2"] + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_permitted_team_scope_falls_back_to_own_user_when_lookup_fails(monkeypatch): + monkeypatch.setattr( + "litellm.proxy.spend_tracking.spend_management_endpoints._get_permitted_team_ids_for_spend_logs", + AsyncMock(side_effect=RuntimeError("database unavailable")), + ) + + permitted_team_ids = await spend_management_endpoints._get_permitted_team_ids_for_spend_logs_or_empty( + prisma_client=MagicMock(), + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="caller@example.com", + ), + ) + + assert permitted_team_ids == () + + @pytest.mark.asyncio async def test_ui_view_spend_logs_team_admin_can_filter_team_spend_by_user(client, monkeypatch): """ From 19eae00d71f85405eec104d15d502a6b5bd9a68b Mon Sep 17 00:00:00 2001 From: Daniel Meismer Date: Thu, 13 Aug 2026 12:16:36 -0400 Subject: [PATCH 07/26] fix(ui): make per-user usage filter searchable Reuse the Global Usage user search and pagination behavior in the Per User report, including empty-result handling. Co-Authored-By: Codex --- .../components/EntityUsage/EntityUsage.tsx | 10 ++++- .../components/UsagePageView.test.tsx | 26 +++++++++++- .../_components/components/UsagePageView.tsx | 41 +++++++++++-------- .../UsageExportHeader.test.tsx | 25 +++++++++++ .../EntityUsageExport/UsageExportHeader.tsx | 22 ++++++++-- .../src/components/EntityUsageExport/index.ts | 1 + 6 files changed, 100 insertions(+), 25 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx index 956060fc244..6f63b56c372 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx @@ -37,7 +37,7 @@ import { Alert, Button, Tooltip } from "antd"; import React, { type ReactNode, useMemo, useState } from "react"; import TeamMultiSelect from "@/components/common_components/team_multi_select"; import { ActivityMetrics, processActivityData } from "@/components/activity_metrics"; -import { UsageExportHeader } from "@/components/EntityUsageExport"; +import { UsageExportHeader, type UsageFilterSelectProps } from "@/components/EntityUsageExport"; import type { EntityType } from "@/components/EntityUsageExport/types"; import { agentDailyActivityCall, @@ -97,6 +97,7 @@ interface EntityUsageProps { entityList: EntityList[] | null; premiumUser: boolean; dateValue: DateRangePickerValue; + filterSelectProps?: UsageFilterSelectProps; } const ENTITY_FETCH_FNS: Record Promise> = { @@ -120,6 +121,7 @@ const EntityUsage: React.FC = ({ entityList, userRole, dateValue, + filterSelectProps, }) => { const { teams } = useTeams(); const [selectedTags, setSelectedTags] = useState([]); @@ -678,13 +680,17 @@ const EntityUsage: React.FC = ({ dateValue={dateValue} entityType={entityType} spendData={spendData} - showFilters={entityType !== "team" && entityList !== null && entityList.length > 0} + showFilters={ + entityType !== "team" && + (filterSelectProps?.showSearch === true || (entityList !== null && entityList.length > 0)) + } filterLabel={getFilterLabel(entityType)} filterPlaceholder={getFilterPlaceholder(entityType)} selectedFilters={selectedTags} onFiltersChange={setSelectedTags} filterOptions={getAllTags() || undefined} filterMode={entityType === "user" ? "single" : "multiple"} + filterSelectProps={filterSelectProps} teams={teams || []} /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx index 9085cf961a9..0c6874d0786 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx @@ -46,7 +46,18 @@ vi.mock("@/components/UsagePage/components/EntityUsage/TopKeyView", () => ({ })); vi.mock("./EntityUsage/EntityUsage", () => ({ - default: () =>
Entity Usage
, + default: ({ + entityType, + filterSelectProps, + }: { + entityType?: string; + filterSelectProps?: { showSearch?: boolean }; + }) => ( +
+ Entity Usage + {entityType === "user" && filterSelectProps?.showSearch && Searchable user filter} +
+ ), EntityList: [], })); @@ -76,6 +87,7 @@ vi.mock("./UsageViewSelect/UsageViewSelect", async () => { React.createElement("option", { value: "customer" }, "Customer Usage"), tagOption, React.createElement("option", { value: "agent" }, "Agent Usage"), + React.createElement("option", { value: "user" }, "User Usage"), React.createElement("option", { value: "user-agent-activity" }, "User Agent Activity"), ); }; @@ -924,6 +936,18 @@ describe("UsagePage", () => { expect(mockUseInfiniteUsers).toHaveBeenCalledWith(50, undefined); }); + it("should reuse the searchable user filter in the user usage view", async () => { + renderWithProviders(); + + await waitFor(() => { + expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalled(); + }); + + fireEvent.change(screen.getByTestId("usage-view-select"), { target: { value: "user" } }); + + expect(await screen.findByText("Searchable user filter")).toBeInTheDocument(); + }); + it("should deduplicate users across pages", async () => { mockUseInfiniteUsers.mockReturnValue({ data: { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx index 494df313ac0..9cd499dd7d2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx @@ -38,7 +38,7 @@ import { formatNumberWithCommas } from "@/utils/dataUtils"; import { all_admin_roles, internalUserRoles } from "@/utils/roles"; import { ActivityMetrics, processActivityData } from "@/components/activity_metrics"; import CloudZeroExportModal from "@/components/cloudzero_export_modal"; -import EntityUsageExportModal from "@/components/EntityUsageExport"; +import EntityUsageExportModal, { type UsageFilterSelectProps } from "@/components/EntityUsageExport"; import { Team } from "@/components/key_team_helpers/key_list"; import { gatewayDailyActivityCall, @@ -161,6 +161,26 @@ const UsagePage: React.FC = ({ teams, organizations }) => { } }; + const userFilterSelectProps: UsageFilterSelectProps = { + showSearch: true, + filterOption: false, + onSearch: handleUserSearchChange, + searchValue: userSearchInput, + onPopupScroll: handleUserPopupScroll, + loading: isLoadingUsers, + notFoundContent: isLoadingUsers ? : "No users found", + popupRender: (menu) => ( + <> + {menu} + {isFetchingNextUsersPage && ( +
+ +
+ )} + + ), + }; + // For admins: null means global view (all users), a string means filter by that user // For non-admins: always set to their own user ID const [selectedUserId, setSelectedUserId] = useState(isAdmin ? null : userID || null); @@ -565,29 +585,13 @@ const UsagePage: React.FC = ({ teams, organizations }) => {
Filter by user Date: Thu, 13 Aug 2026 12:27:55 -0400 Subject: [PATCH 08/26] test: remove unrelated session log assertion Drop a stray assertion against a field that is not present in the session pagination fixture. Co-Authored-By: Codex --- .../proxy/spend_tracking/test_spend_management_endpoints.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 87bbb1c2f80..7052e050806 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -1774,7 +1774,6 @@ async def test_ui_view_session_spend_logs_pagination(client, monkeypatch): assert data["total_pages"] == 2 assert len(data["data"]) == 1 assert data["data"][0]["request_id"] == "req1" - assert data["data"][0]["user"] == "member1" finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) From 5edf8e71ce13ce77f1f906a840615d3cb7f069ce Mon Sep 17 00:00:00 2001 From: Daniel Meismer Date: Thu, 13 Aug 2026 13:00:08 -0400 Subject: [PATCH 09/26] chore: rerun CI Generated with AI Co-Authored-By: Codex From d794b613479bc28c095eb7c449d6860d3829c72f Mon Sep 17 00:00:00 2001 From: Daniel Meismer Date: Thu, 13 Aug 2026 13:09:47 -0400 Subject: [PATCH 10/26] chore(ui): bump nanoid to 3.3.18 Update the transitive lockfile entry to the first patched 3.x release so OSV no longer reports GHSA-2v37-7h3g-55p8. Generated with AI Co-Authored-By: Codex --- ui/litellm-dashboard/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 515a992bc85..b36b07631e3 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -10319,9 +10319,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.17", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.17.tgz", - "integrity": "sha512-xQLf0A3HOMlgHq0n247/LRuAOYmB7dXJ/DvAxGvsSBij45XtBSmQycu+F8ODbHwns/XyFZagyL1+J0Offw1E0g==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "funding": [ { "type": "github", From 80f49024a3ce7f1bb5c98a3eab2435b614ee348b Mon Sep 17 00:00:00 2001 From: Daniel Meismer Date: Thu, 13 Aug 2026 13:14:17 -0400 Subject: [PATCH 11/26] chore(ui): bump nanoid to 3.3.18 Update the transitive lockfile entry to the first patched 3.x release so OSV no longer reports GHSA-2v37-7h3g-55p8. Generated with AI Co-Authored-By: Codex --- ui/litellm-dashboard/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 515a992bc85..b36b07631e3 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -10319,9 +10319,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.17", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.17.tgz", - "integrity": "sha512-xQLf0A3HOMlgHq0n247/LRuAOYmB7dXJ/DvAxGvsSBij45XtBSmQycu+F8ODbHwns/XyFZagyL1+J0Offw1E0g==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "funding": [ { "type": "github", From 4c49d03732dc107dc52ea7f469e848ec556fec85 Mon Sep 17 00:00:00 2001 From: Scott Wilson Date: Fri, 14 Aug 2026 14:56:06 -0400 Subject: [PATCH 12/26] fix(anthropic): preserve optional Responses tool properties Translating Anthropic tools left the outbound function-tool `strict` unset, which the Responses API does not read as non-strict. OpenAI's function-calling docs say strict mode requires every field in `properties` to be marked required, and with `strict` omitted the schema gets normalized to satisfy that instead of being rejected. What users see is a tool whose `required` lists every property, so models fill optional Anthropic tool arguments with empty values. Send `strict` explicitly so an unset value stays non-strict and an explicit `strict: true` still reaches the provider On the Chat Completions adapter, `strict` was also missing from `mapped_tool_params`, so a tool-level `strict` was merged into the OpenAI function `parameters` schema (mutating the caller's `input_schema` along the way) instead of being set on the function. Map it to `function.strict` and leave it unset when the caller omits it, since Chat Completions already defaults to non-strict --- .../adapters/transformation.py | 3 + .../responses_adapters/transformation.py | 8 ++- litellm/types/llms/anthropic.py | 3 +- ...al_pass_through_adapters_transformation.py | 47 ++++++++++++++++ .../test_responses_adapters_transformation.py | 55 +++++++++++++++++++ 5 files changed, 114 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 51f2b661421..ea0eebe0511 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -741,6 +741,7 @@ class LiteLLMAnthropicMessagesAdapter: "input_schema", "description", "cache_control", + "strict", "type", ] @@ -770,6 +771,8 @@ class LiteLLMAnthropicMessagesAdapter: function_chunk["parameters"] = tool["input_schema"] if "description" in tool: function_chunk["description"] = tool["description"] + if "strict" in tool: + function_chunk["strict"] = bool(tool["strict"]) for k, v in tool.items(): if k not in mapped_tool_params: # pass additional computer kwargs diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index bf3f6153e7c..03e66388c91 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -231,7 +231,13 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search": result.append({"type": "web_search_preview"}) continue - func_tool: dict[str, Any] = {"type": "function", "name": tool_name} + # Responses turns strict mode on when `strict` is omitted, silently rewriting + # `required` to every property. Anthropic tools are non-strict unless asked. + func_tool: dict[str, Any] = { + "type": "function", + "name": tool_name, + "strict": bool(tool_dict.get("strict")), + } if "description" in tool_dict: func_tool["description"] = tool_dict["description"] if "input_schema" in tool_dict: diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 69d291eebd0..17ba78b0190 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -3,7 +3,7 @@ from enum import Enum from typing import Any, Final, Literal, TypeAlias from pydantic import BaseModel, ConfigDict -from typing_extensions import NotRequired, Required, TypedDict +from typing_extensions import NotRequired, ReadOnly, Required, TypedDict from .openai import ( ChatCompletionCachedContent, @@ -48,6 +48,7 @@ class AnthropicMessagesTool(TypedDict, total=False): name: Required[str] description: str input_schema: AnthropicInputSchema | None + strict: ReadOnly[bool] type: Literal["custom"] cache_control: dict | ChatCompletionCachedContent | None defer_loading: bool diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index fe6adade6a8..0c30d8a8322 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -3508,3 +3508,50 @@ def test_translate_anthropic_tools_to_openai_preserves_parameters_type(): params = new_tools[0]["function"]["parameters"] assert params["type"] == "object" assert new_tools[0]["type"] == "function" + + +def test_translate_anthropic_tools_to_openai_maps_strict_onto_function_not_parameters(): + """A tool-level `strict` lands on the OpenAI function, leaving the caller's `input_schema` untouched.""" + adapter = LiteLLMAnthropicMessagesAdapter() + input_schema = { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, + } + tools = [{"type": "custom", "name": "get_weather", "strict": True, "input_schema": input_schema}] + + new_tools, _ = adapter.translate_anthropic_tools_to_openai(tools=tools) + + function = new_tools[0]["function"] + assert function["strict"] is True + assert "strict" not in function["parameters"] + assert input_schema == { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, + } + + +def test_translate_anthropic_tools_to_openai_omits_unset_strict(): + """Chat Completions already defaults to non-strict, so an unset `strict` stays unset.""" + adapter = LiteLLMAnthropicMessagesAdapter() + tools = [ + { + "type": "custom", + "name": "search", + "input_schema": { + "type": "object", + "properties": {"query": {"type": "string"}, "cursor": {"type": "string"}}, + "required": ["query"], + }, + } + ] + + new_tools, _ = adapter.translate_anthropic_tools_to_openai(tools=tools) + + function = new_tools[0]["function"] + assert "strict" not in function + assert "strict" not in function["parameters"] + assert function["parameters"]["required"] == ["query"] diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index a736ca684aa..90733dc9134 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -605,6 +605,7 @@ class TestTranslateToolsToResponsesAPI: { "type": "function", "name": "get_weather", + "strict": False, "description": "Get current weather for a city.", "parameters": { "type": "object", @@ -614,6 +615,60 @@ class TestTranslateToolsToResponsesAPI: } ] + def test_tool_with_optional_properties_stays_non_strict(self): + """Regression: an unset Anthropic `strict` must not become the Responses strict default, + which would rewrite `required` to include every optional property.""" + tools = [ + { + "name": "search", + "input_schema": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "cursor": {"type": "string"}, + }, + "required": ["query"], + "additionalProperties": False, + }, + } + ] + + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + + assert result[0]["strict"] is False + assert result[0]["parameters"]["required"] == ["query"] + + def test_tool_forwards_explicit_strict_true(self): + """An explicit Anthropic `strict: True` still reaches Responses as True.""" + tools = [ + { + "name": "search", + "strict": True, + "input_schema": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + "additionalProperties": False, + }, + } + ] + + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + + assert result == [ + { + "type": "function", + "name": "search", + "strict": True, + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + "additionalProperties": False, + }, + } + ] + def test_tool_without_description(self): """Tool without a description omits the description key.""" tools = [{"name": "ping", "input_schema": {"type": "object", "properties": {}}}] From ae3e19a83f421b78b61b7415cdaf4d4058859b96 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 14 Aug 2026 14:03:49 -0700 Subject: [PATCH 13/26] fix(helm): bound the migrations Job so a blocked migration cannot stall the release Both charts run schema migrations from a Job that is a pre-install and pre-upgrade hook, and neither set activeDeadlineSeconds. A migration that blocks on the database therefore never fails: backoffLimit is not reached because the pod never terminates, so the Job stays active indefinitely and the release waits on the hook forever. `helm upgrade` and any GitOps controller driving it stop reconciling the whole chart until someone deletes the Job by hand, which means unrelated changes to the gateway, the backend and the UI silently stop shipping. Give the field a 1800s default, guarded by `with` so setting it to null restores the old unbounded behaviour. A migration that has exhausted its retries is not going to succeed on the next one, so failing is strictly better than hanging: a failed sync is visible and retryable, a hung one is neither. Chart.yaml is deliberately untouched. Recent template-only changes to litellm-helm did not bump it either. --- .../templates/migrations-job.yaml | 3 ++ .../tests/migrations-job_tests.yaml | 28 +++++++++++++++++++ helm/litellm-helm/values.yaml | 7 +++++ helm/litellm/templates/migrations-job.yaml | 3 ++ helm/litellm/tests/migration_job_tests.yaml | 21 ++++++++++++++ helm/litellm/values.yaml | 9 ++++++ 6 files changed, 71 insertions(+) diff --git a/helm/litellm-helm/templates/migrations-job.yaml b/helm/litellm-helm/templates/migrations-job.yaml index f8a660e23f8..5a873cbb965 100644 --- a/helm/litellm-helm/templates/migrations-job.yaml +++ b/helm/litellm-helm/templates/migrations-job.yaml @@ -119,4 +119,7 @@ spec: {{- end }} ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }} backoffLimit: {{ .Values.migrationJob.backoffLimit }} + {{- with .Values.migrationJob.activeDeadlineSeconds }} + activeDeadlineSeconds: {{ . }} + {{- end }} {{- end }} diff --git a/helm/litellm-helm/tests/migrations-job_tests.yaml b/helm/litellm-helm/tests/migrations-job_tests.yaml index cb962118a25..e327a3ec201 100644 --- a/helm/litellm-helm/tests/migrations-job_tests.yaml +++ b/helm/litellm-helm/tests/migrations-job_tests.yaml @@ -314,3 +314,31 @@ tests: operator: Equal value: litellm-e2e effect: NoSchedule + + - it: bounds the Job with a deadline by default, so a blocked migration cannot stall the release forever + set: + migrationJob: + enabled: true + asserts: + - equal: + path: spec.activeDeadlineSeconds + value: 1800 + + - it: honours an operator-supplied deadline + set: + migrationJob: + enabled: true + activeDeadlineSeconds: 600 + asserts: + - equal: + path: spec.activeDeadlineSeconds + value: 600 + + - it: omits the deadline entirely when it is nulled out, restoring the unbounded behaviour + set: + migrationJob: + enabled: true + activeDeadlineSeconds: null + asserts: + - notExists: + path: spec.activeDeadlineSeconds diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index df2b55723fe..628ca038339 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -427,6 +427,13 @@ migrationJob: enabled: true # Enable or disable the schema migration Job retries: 3 # Number of retries for the Job in case of failure backoffLimit: 4 # Backoff limit for Job restarts + # Wall-clock budget for the whole Job, shared across every `backoffLimit` + # retry rather than granted per attempt. Without it a migration that blocks + # on the database never fails, and when the Helm hook is enabled the release + # waits on it forever: `helm upgrade` and any GitOps controller driving it + # stop reconciling the whole chart until someone deletes the Job by hand. + # Set to null to opt out and restore the unbounded behaviour. + activeDeadlineSeconds: 1800 disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0. # Optional service account for the migration job. # Only used when migrationJob.hooks.helm.enabled=true and serviceAccount.create=true. diff --git a/helm/litellm/templates/migrations-job.yaml b/helm/litellm/templates/migrations-job.yaml index 2debe8a1e10..9cd8397f794 100644 --- a/helm/litellm/templates/migrations-job.yaml +++ b/helm/litellm/templates/migrations-job.yaml @@ -21,6 +21,9 @@ metadata: spec: backoffLimit: {{ .Values.migrationJob.backoffLimit }} ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }} + {{- with .Values.migrationJob.activeDeadlineSeconds }} + activeDeadlineSeconds: {{ . }} + {{- end }} template: metadata: {{- /* The Job's selector is generated by the controller rather than diff --git a/helm/litellm/tests/migration_job_tests.yaml b/helm/litellm/tests/migration_job_tests.yaml index 12e525c5a8c..c3f3083ece5 100644 --- a/helm/litellm/tests/migration_job_tests.yaml +++ b/helm/litellm/tests/migration_job_tests.yaml @@ -167,3 +167,24 @@ tests: - equal: path: spec.template.metadata.labels['app.kubernetes.io/component'] value: batch-migrations + + - it: bounds the Job with a deadline by default, so a blocked migration cannot stall the release forever + asserts: + - equal: + path: spec.activeDeadlineSeconds + value: 1800 + + - it: honours an operator-supplied deadline + set: + migrationJob.activeDeadlineSeconds: 600 + asserts: + - equal: + path: spec.activeDeadlineSeconds + value: 600 + + - it: omits the deadline entirely when it is nulled out, restoring the unbounded behaviour + set: + migrationJob.activeDeadlineSeconds: null + asserts: + - notExists: + path: spec.activeDeadlineSeconds diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index cd377667602..ea6e47a8e62 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -56,6 +56,15 @@ migrationJob: enabled: true backoffLimit: 4 ttlSecondsAfterFinished: 120 + # Wall-clock budget for the whole Job, shared across every `backoffLimit` + # retry rather than granted per attempt. Without it a migration that blocks + # on the database never fails, and because this is a pre-upgrade hook the + # release waits on it forever: `helm upgrade` and any GitOps controller + # driving it stop reconciling the whole chart until someone deletes the Job + # by hand. A migration that has exhausted its retries is not going to + # succeed on the next one, so failing is strictly better than hanging. + # Set to null to opt out and restore the unbounded behaviour. + activeDeadlineSeconds: 1800 resources: {} # ServiceAccount for the Job pod only. # From 9858d021eef07fefb955d7dc9d4c8e1595afb495 Mon Sep 17 00:00:00 2001 From: Scott Wilson Date: Fri, 14 Aug 2026 15:13:24 -0400 Subject: [PATCH 14/26] fix(guardrails): record MCP tool guardrail evaluations and blocks in usage monitor MCP tool calls run their guardrails against a throwaway LLM-shaped dict built by `ProxyLogging._convert_mcp_to_llm_format`, not against the dict the tool call is logged from. `@log_guardrail_information` therefore appended `standard_logging_guardrail_information` to that throwaway dict's metadata bucket, where `get_standard_logging_object_payload` never saw it, so the Guardrails Monitor reported zero evaluations and zero blocks for all MCP traffic. Thread the request's `litellm_logging_obj` into `pre_call_tool_check` and `_create_during_hook_task` and bridge the guardrail records onto it: - Seed `data["litellm_logging_obj"]`, which unified guardrails read and pass into `apply_guardrail`. - Call `_sync_guardrail_info_to_logging_obj` in a `finally`, which is what native guardrails need and what makes the block path work: a blocked call raises straight out of `pre_call_tool_check`, so the record has to be attached before the exception leaves the frame. Only the guardrail evaluation records are copied. The synthetic request's messages and tool arguments are deliberately left behind -- they can carry end-user data and nothing in the monitor needs them. In `call_mcp_tool`, flush the failure handlers before `post_call_failure_hook` so the `status="failure"` standard logging object exists when `_ProxyDBLogger.async_post_call_failure_hook` writes the spend-log row the monitor's "Total Blocked" counts. Both handlers gate on `should_run_logging("sync_failure")` / `("async_failure")` and then mark it, so the `@client` wrapper's own post-raise logging is a no-op and nothing is double-counted -- the same pattern `_fire_mcp_tool_call_logging` already uses for `isError=True`. Threaded through every MCP tool entry point: the managed-server path, the local-OpenAPI registry path, the legacy registry fallback, and the Responses API's `_execute_tool_calls`. --- .../mcp_server/mcp_server_manager.py | 89 +++++- .../proxy/_experimental/mcp_server/server.py | 17 + .../mcp/litellm_proxy_mcp_handler.py | 1 + .../mcp_server/test_mcp_block_recording.py | 126 ++++++++ .../test_mcp_guardrail_usage_monitor.py | 301 ++++++++++++++++++ .../mcp/test_litellm_proxy_mcp_handler.py | 36 +++ 6 files changed, 560 insertions(+), 10 deletions(-) create mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_block_recording.py create mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_guardrail_usage_monitor.py diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index a1adda2bc95..0ad372064cd 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -13,7 +13,7 @@ import json import os import re import time -from collections.abc import AsyncIterator, Callable, Sequence +from collections.abc import AsyncIterator, Callable, Mapping, Sequence from contextlib import asynccontextmanager from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypedDict, cast from urllib.parse import ParseResult, urlparse @@ -46,6 +46,9 @@ from litellm.constants import ( ) from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.experimental_mcp_client.client import MCPClient, MCPSigV4Auth +from litellm.integrations.custom_guardrail import ( + _sync_guardrail_info_to_logging_obj, # pyright: ignore[reportPrivateUsage] - the same bridge @log_guardrail_information uses; reimplementing it here would fork the metadata-key logic +) from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( @@ -162,6 +165,7 @@ if TYPE_CHECKING: from mcp.types import CreateMessageRequestParams from litellm.caching.caching import InMemoryCache + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.mcp_server.mcp_toolset import MCPToolset try: @@ -1233,6 +1237,35 @@ def _create_elicitation_callback(): return _elicitation_callback +def _record_mcp_guardrail_evaluations( + synthetic_llm_data: dict[str, Any], # mutable-ok: `_sync_guardrail_info_to_logging_obj` takes a concrete dict + litellm_logging_obj: "LiteLLMLoggingObj | None", +) -> None: + """Bridge guardrail decision records off an MCP synthetic request onto the request's logger. + + MCP guardrails run against a throwaway LLM-shaped dict from + ``ProxyLogging._convert_mcp_to_llm_format``, so ``@log_guardrail_information`` + files ``standard_logging_guardrail_information`` in that dict's metadata bucket, + which ``get_standard_logging_object_payload`` never reads. Native (non-unified) + guardrails receive no ``logging_obj`` kwarg, so the decorator cannot bridge on + their behalf; this calls the same helper it would have. + + Only the decision records move. The synthetic request's messages and tool + arguments stay behind: they can carry end-user data, and the monitor needs none + of it. + """ + if litellm_logging_obj is None: + return + + try: + _sync_guardrail_info_to_logging_obj(synthetic_llm_data, litellm_logging_obj) + except Exception as e: # noqa: BLE001 # callers run this from a `finally` on the block path + # The breadth is the point. Narrowing to the knowable AttributeError/TypeError + # would let an unexpected type escape that ``finally`` and replace the guardrail's + # block with a bookkeeping error. + verbose_logger.warning("Failed to record MCP guardrail evaluation for logging: %s", e) + + class MCPServerManager: _STDIO_ENV_TEMPLATE_PATTERN = re.compile(r"^\$\{(X-[^}]+)\}$") @@ -4543,6 +4576,7 @@ class MCPServerManager: proxy_logging_obj: ProxyLogging | None, server: MCPServer, raw_headers: dict[str, str] | None = None, + litellm_logging_obj: "LiteLLMLoggingObj | None" = None, ) -> dict[str, Any]: """ Run pre-call checks and guardrail hooks for an MCP tool call. @@ -4552,6 +4586,10 @@ class MCPServerManager: present. An absent logger must never be able to turn an authorization decision into a no-op. + ``litellm_logging_obj`` is the request's logger, and it is what lands a + ``pre_mcp_call`` evaluation (or a block) on the spend-log row the Guardrails + Monitor counts. It stays optional so callers that do no logging are unchanged. + Returns a dict that may contain: - "arguments": hook-modified tool arguments (only if changed) - "extra_headers": headers injected by pre_mcp_call guardrail hooks @@ -4610,8 +4648,13 @@ class MCPServerManager: # Create MCP request object for processing mcp_request_obj: Final = proxy_logging_obj._create_mcp_request_object_from_kwargs(pre_hook_kwargs) - # Convert to LLM format for existing guardrail compatibility + # Convert to LLM format for existing guardrail compatibility. + # Unified guardrails read the seeded logger off the request dict and pass it + # into ``apply_guardrail``, so ``@log_guardrail_information`` bridges their + # evaluations itself; the ``finally`` below covers native guardrails, which + # never receive it. Same seeding the pass-through routes do. synthetic_llm_data: Final = proxy_logging_obj._convert_mcp_to_llm_format(mcp_request_obj, pre_hook_kwargs) + synthetic_llm_data["litellm_logging_obj"] = litellm_logging_obj try: # Use standard pre_call_hook @@ -4636,6 +4679,12 @@ class MCPServerManager: # Re-raise guardrail exceptions to properly fail the MCP call verbose_logger.error("Guardrail blocked MCP tool call pre call: %s", e) raise e + finally: + # ``finally`` rather than after the ``try``: a block raises straight out of + # here, and the failure spend-log row that "Total Blocked" counts is built + # from this logger further up the stack, so the record has to be attached + # before the exception leaves this frame. + _record_mcp_guardrail_evaluations(synthetic_llm_data, litellm_logging_obj) return hook_result @@ -4647,8 +4696,14 @@ class MCPServerManager: user_api_key_auth: UserAPIKeyAuth | None, proxy_logging_obj: ProxyLogging, start_time: datetime.datetime, + litellm_logging_obj: "LiteLLMLoggingObj | None" = None, ): - """Create and return a during hook task for MCP tool calls.""" + """Create and return a during hook task for MCP tool calls. + + ``litellm_logging_obj`` is the request's logger; see ``pre_call_tool_check``. + The task is awaited before the tool call's success logging runs, so a + ``during_mcp_call`` evaluation recorded on it is serialized with that call. + """ from litellm.types.llms.base import HiddenParams from litellm.types.mcp import MCPDuringCallRequestObject @@ -4667,15 +4722,23 @@ class MCPServerManager: "user_api_key_auth": user_api_key_auth, } + # Seeded for the same reason as in ``pre_call_tool_check``. synthetic_llm_data: Final = proxy_logging_obj._convert_mcp_to_llm_format(request_obj, during_hook_kwargs) + synthetic_llm_data["litellm_logging_obj"] = litellm_logging_obj - return asyncio.create_task( - proxy_logging_obj.during_call_hook( - user_api_key_dict=user_api_key_auth, - data=synthetic_llm_data, - call_type=CallTypes.call_mcp_tool.value, - ) - ) + # Wrapped so the bridge runs inside the task: the caller only holds the task and + # gathers it later, so there is no other point that still sees a block here. + async def _run_during_call_hook() -> Mapping[str, Any] | None: + try: + return await proxy_logging_obj.during_call_hook( + user_api_key_dict=user_api_key_auth, + data=synthetic_llm_data, + call_type=CallTypes.call_mcp_tool.value, + ) + finally: + _record_mcp_guardrail_evaluations(synthetic_llm_data, litellm_logging_obj) + + return asyncio.create_task(_run_during_call_hook()) def _get_call_semaphore(self, mcp_server: MCPServer) -> asyncio.Semaphore | None: limit: Final = mcp_server.max_concurrent_requests @@ -5204,6 +5267,7 @@ class MCPServerManager: oauth2_headers: dict[str, str] | None = None, raw_headers: dict[str, str] | None = None, host_progress_callback: Callable | None = None, + litellm_logging_obj: "LiteLLMLoggingObj | None" = None, ) -> CallToolResult: """ Call a tool with the given name and arguments @@ -5216,6 +5280,9 @@ class MCPServerManager: mcp_auth_header: MCP auth header (deprecated) mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value} proxy_logging_obj: Optional ProxyLogging object for hook integration + litellm_logging_obj: Optional request logger the guardrail hooks record + their evaluations onto, so MCP guardrail activity reaches the + Guardrails Monitor. See ``pre_call_tool_check`` Returns: @@ -5246,6 +5313,7 @@ class MCPServerManager: proxy_logging_obj=proxy_logging_obj, server=mcp_server, raw_headers=raw_headers, + litellm_logging_obj=litellm_logging_obj, ) if "arguments" in hook_result: arguments = hook_result["arguments"] @@ -5260,6 +5328,7 @@ class MCPServerManager: user_api_key_auth=user_api_key_auth, proxy_logging_obj=proxy_logging_obj, start_time=start_time, + litellm_logging_obj=litellm_logging_obj, ) tasks.append(during_hook_task) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index f237529b319..17457c3362f 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -2824,6 +2824,7 @@ if MCP_AVAILABLE: proxy_logging_obj=proxy_logging_obj, server=mcp_server, raw_headers=raw_headers, + litellm_logging_obj=litellm_logging_obj, ) # `pre_call_tool_check` may return guardrail-modified # arguments; honor them on the local path too. @@ -2962,6 +2963,7 @@ if MCP_AVAILABLE: proxy_logging_obj=proxy_logging_obj, server=prefix_server, raw_headers=raw_headers, + litellm_logging_obj=litellm_logging_obj, ) if "arguments" in hook_result: arguments = hook_result["arguments"] # pyright: ignore[reportAny] # hook returns untyped args @@ -3149,6 +3151,20 @@ if MCP_AVAILABLE: traceback_str: Final = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG) from litellm.proxy.proxy_server import proxy_logging_obj + # Ordering is load-bearing. ``_ProxyDBLogger.async_post_call_failure_hook``, + # reached below, writes the failure spend-log row from this logger's + # ``standard_logging_object``, which only exists once the failure handlers + # have run. Flush them first or the row lands with + # ``guardrail_information=None`` and a guardrail block is never counted. + # + # Not double-logged: both handlers gate on ``should_run_logging`` and then + # mark it, so the ``@client`` wrapper's own post-raise logging no-ops on this + # logger, same as ``_fire_mcp_tool_call_logging`` does for ``isError=True``. + if litellm_logging_obj is not None: + end_time: Final = datetime.now() # noqa: DTZ005 # naive to match `start_time`, which it is subtracted from + litellm_logging_obj.failure_handler(e, traceback_str, start_time, end_time) + await litellm_logging_obj.async_failure_handler(e, traceback_str, start_time, end_time) + if proxy_logging_obj and user_api_key_auth: await proxy_logging_obj.post_call_failure_hook( request_data=kwargs, @@ -3326,6 +3342,7 @@ if MCP_AVAILABLE: raw_headers=raw_headers, proxy_logging_obj=proxy_logging_obj, host_progress_callback=host_progress_callback, + litellm_logging_obj=litellm_logging_obj, ) verbose_logger.debug("CALL TOOL RESULT: %s", call_tool_result) return call_tool_result diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 56818717c09..0321034dffe 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -798,6 +798,7 @@ class LiteLLM_Proxy_MCP_Handler: oauth2_headers=oauth2_headers, raw_headers=raw_headers, proxy_logging_obj=proxy_logging_obj, + litellm_logging_obj=litellm_logging_obj, ) if proxy_logging_obj: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_block_recording.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_block_recording.py new file mode 100644 index 00000000000..64d926bc5e3 --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_block_recording.py @@ -0,0 +1,126 @@ +"""Tests for guardrail-block recording in +``litellm.proxy._experimental.mcp_server.server.call_mcp_tool``. + +A pre-call MCP guardrail block *raises* into ``call_mcp_tool``'s +``except Exception``. The failure spend-log row that the Guardrails Monitor's +"Total Blocked" counts is written by ``_ProxyDBLogger.async_post_call_failure_hook`` +(reached via ``proxy_logging_obj.post_call_failure_hook``), which reads +``standard_logging_object`` off the request's logging obj -- and that only exists +once ``failure_handler`` / ``async_failure_handler`` have run. So the failure +handlers must run *before* ``post_call_failure_hook``, otherwise the row persists +with ``guardrail_information=None`` and the block is never counted. These tests +pin that ordering. + +``call_mcp_tool`` is wrapped by ``@client`` (``litellm.utils.client``), which uses +``functools.wraps`` and therefore exposes the raw undecorated coroutine as +``__wrapped__``. The tests drive ``__wrapped__`` directly so the except-block +ordering is observed in isolation, without the wrapper's own post-raise logging +firing. Note that this means they do not exercise the wrapper's dedup path; that +dedup rests on ``should_run_logging("sync_failure")`` / ``("async_failure")``, +which has its own coverage in the logging tests. + +``proxy_logging_obj`` is imported lazily inside the except block via +``from litellm.proxy.proxy_server import proxy_logging_obj``; the real +``proxy_server`` module is heavy, so a fake module is injected into ``sys.modules`` +to satisfy that lazy import without loading it. +""" + +import contextlib +import sys +import types +from unittest import mock + +import pytest +from fastapi import HTTPException + +from litellm.proxy._experimental.mcp_server import server + + +class _RecordingLoggingObj: + """Stands in for ``LiteLLMLoggingObj``, recording the failure flush the fix + makes so the test can assert it happens before ``post_call_failure_hook``.""" + + def __init__(self, order: list) -> None: + self._order = order + self.failure_calls = 0 + self.async_failure_calls = 0 + + def failure_handler(self, *_args, **_kwargs) -> None: + self.failure_calls += 1 + self._order.append("failure_handler") + + async def async_failure_handler(self, *_args, **_kwargs) -> None: + self.async_failure_calls += 1 + self._order.append("async_failure_handler") + + +async def _call_block(logging_obj, order: list, *, user_api_key_auth=mock.sentinel.auth): + """Drive ``call_mcp_tool`` into its except path via ``arguments=None``, which + raises ``HTTPException(400)`` before any server-manager call, and return once it + re-raises.""" + + async def _record_post_call_failure_hook(**_kwargs) -> None: + order.append("post_call_failure_hook") + + proxy_logging_obj = mock.MagicMock() + proxy_logging_obj.post_call_failure_hook.side_effect = _record_post_call_failure_hook + + fake_proxy_server = types.ModuleType("litellm.proxy.proxy_server") + fake_proxy_server.proxy_logging_obj = proxy_logging_obj # pyright: ignore[reportAttributeAccessIssue] + + with mock.patch.dict(sys.modules, {"litellm.proxy.proxy_server": fake_proxy_server}): + with contextlib.suppress(HTTPException): + await server.call_mcp_tool.__wrapped__( + name="t", + arguments=None, + user_api_key_auth=user_api_key_auth, + litellm_logging_obj=logging_obj, + ) + + +@pytest.mark.asyncio +async def test_block_flushes_failure_before_post_call_failure_hook(): + order: list = [] + await _call_block(_RecordingLoggingObj(order), order) + + assert order == ["failure_handler", "async_failure_handler", "post_call_failure_hook"], order + + +@pytest.mark.asyncio +async def test_block_flushes_each_handler_exactly_once(): + """Each handler runs once, so the block yields exactly one counted row rather + than double-counting on the shared logging obj.""" + order: list = [] + obj = _RecordingLoggingObj(order) + await _call_block(obj, order) + + assert (obj.failure_calls, obj.async_failure_calls) == (1, 1) + + +@pytest.mark.asyncio +async def test_block_flushes_failure_for_anonymous_calls(): + """With no ``user_api_key_auth`` the failure handlers still run, so OTel and the + other failure sinks see the block. + + ``post_call_failure_hook`` stays gated on auth, matching the pre-existing + contract: SpendLogs rows are attributable billing/audit records and the + downstream DB logger dereferences authenticated key, budget, and route data. + Counting anonymous MCP blocks needs a counter that does not live in SpendLogs, + which is a separate design change, not part of this fix. + """ + order: list = [] + obj = _RecordingLoggingObj(order) + await _call_block(obj, order, user_api_key_auth=None) + + assert order == ["failure_handler", "async_failure_handler"], order + + +@pytest.mark.asyncio +async def test_absent_logging_obj_still_calls_hook_and_skips_flush(): + """Without a logging obj the flush is skipped (no crash) but + ``post_call_failure_hook`` still fires. Byte-equivalent to stock behavior for + that branch; its value is as a mutation-killer for the ``is not None`` guard.""" + order: list = [] + await _call_block(None, order) + + assert order == ["post_call_failure_hook"], order diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_guardrail_usage_monitor.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_guardrail_usage_monitor.py new file mode 100644 index 00000000000..24e6d2de10d --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_guardrail_usage_monitor.py @@ -0,0 +1,301 @@ +"""Tests for MCP guardrail evaluations reaching the Guardrails Monitor. + +MCP tool calls run their guardrails against a throwaway LLM-shaped dict built by +``ProxyLogging._convert_mcp_to_llm_format``, not against the dict the tool call is +logged from. ``@log_guardrail_information`` therefore appends +``standard_logging_guardrail_information`` to that throwaway dict's metadata +bucket, where ``get_standard_logging_object_payload`` never sees it, so the +Guardrails Monitor reported zero evaluations and zero blocks for MCP traffic. + +``pre_call_tool_check`` and ``_create_during_hook_task`` now take the request's +``litellm_logging_obj`` and bridge those records onto it. These tests pin both the +seeding (which unified guardrails consume off ``data["litellm_logging_obj"]``) and +the bridge (which native guardrails depend on), including on the block path. +""" + +import asyncio +import datetime +from typing import Any +from unittest import mock + +import pytest + +from litellm.exceptions import GuardrailRaisedException +from litellm.proxy._experimental.mcp_server import mcp_server_manager as MOD + + +class _FakeLoggingObj: + """Minimal stand-in for ``LiteLLMLoggingObj``. + + ``_sync_guardrail_info_to_logging_obj`` reads exactly these two attributes, + and the spend-log payload is built from ``litellm_params["metadata"]``, so a + real ``Logging`` instance would add setup cost without adding coverage. + """ + + def __init__(self) -> None: + self.litellm_params: dict[str, Any] = {"metadata": {}} + self.model_call_details: dict[str, Any] = {"litellm_params": self.litellm_params} + + @property + def recorded_guardrails(self) -> list: + return self.litellm_params["metadata"].get("standard_logging_guardrail_information", []) + + +def _bare_manager() -> MOD.MCPServerManager: + """An ``MCPServerManager`` without running ``__init__``. + + The authorization/validation helpers on the path are stubbed out so the test + reaches the guardrail hooks; they have their own coverage elsewhere. + """ + mgr = MOD.MCPServerManager.__new__(MOD.MCPServerManager) + mgr.check_allowed_or_banned_tools = lambda name, server: True + mgr.validate_allowed_params = lambda tool_name, arguments, server: None + + async def _ok(*_args, **_kwargs) -> None: + return None + + mgr.check_tool_permission_for_key_team = _ok + return mgr + + +def _fake_proxy_logging(capture: dict, *, guardrail_effect=None): + """A ``proxy_logging_obj`` double whose hooks capture the data they receive. + + ``guardrail_effect`` stands in for a guardrail: it is handed the synthetic + request dict so it can append a guardrail record (and optionally raise, the + way a blocking guardrail does). + """ + plo = mock.MagicMock() + plo._create_mcp_request_object_from_kwargs.return_value = mock.MagicMock() + # Mirror the real conversion's metadata bucket so a test can prove it survives. + plo._convert_mcp_to_llm_format.side_effect = lambda *_a, **_k: { + "metadata": {"headers": {"x-forwarded-for": "1.2.3.4"}} + } + + async def _hook(*, user_api_key_dict, data, call_type) -> None: + del user_api_key_dict # captured shape is what matters, not the auth double + capture["data"] = data + capture["call_type"] = call_type + if guardrail_effect is not None: + guardrail_effect(data) + + plo.pre_call_hook.side_effect = _hook + plo.during_call_hook.side_effect = _hook + return plo + + +def _record_guardrail(status: str = "success"): + """Write a guardrail record the way ``@log_guardrail_information`` does.""" + + def _effect(data: dict) -> None: + data.setdefault("metadata", {}).setdefault("standard_logging_guardrail_information", []).append( + {"guardrail_name": "test-guardrail", "guardrail_status": status} + ) + + return _effect + + +def _blocking_guardrail(): + record = _record_guardrail(status="guardrail_intervened") + + def _effect(data: dict) -> None: + record(data) + raise GuardrailRaisedException(guardrail_name="test-guardrail", message="blocked") + + return _effect + + +async def _run_pre_call(mgr, plo, logging_obj) -> dict: + return await mgr.pre_call_tool_check( + name="t", + arguments={}, + server_name="s", + user_api_key_auth=None, + proxy_logging_obj=plo, + server=mock.MagicMock(), + raw_headers={}, + litellm_logging_obj=logging_obj, + ) + + +@pytest.mark.asyncio +async def test_pre_call_seeds_request_logging_obj_for_unified_guardrails(): + """Unified guardrails read ``data["litellm_logging_obj"]`` and pass it into + ``apply_guardrail``, whose ``@log_guardrail_information`` wrapper bridges the + evaluation onto that logger itself. Drop the seed and that path records + nothing.""" + capture: dict = {} + logging_obj = _FakeLoggingObj() + await _run_pre_call(_bare_manager(), _fake_proxy_logging(capture), logging_obj) + + assert capture["data"]["litellm_logging_obj"] is logging_obj + + +@pytest.mark.asyncio +async def test_pre_call_keeps_synthetic_request_headers_metadata(): + """The seed must not clobber the metadata bucket ``_convert_mcp_to_llm_format`` + builds: guardrails such as ``MCPJWTSigner`` read ``metadata["headers"]`` off + it.""" + capture: dict = {} + await _run_pre_call(_bare_manager(), _fake_proxy_logging(capture), _FakeLoggingObj()) + + assert capture["data"]["metadata"]["headers"] == {"x-forwarded-for": "1.2.3.4"} + + +@pytest.mark.asyncio +async def test_pre_call_bridges_allowed_evaluation_onto_request_logger(): + """An allowed ``pre_mcp_call`` evaluation must land on the request logger, which + is what the monitor's "Total Evaluations" counts.""" + capture: dict = {} + logging_obj = _FakeLoggingObj() + plo = _fake_proxy_logging(capture, guardrail_effect=_record_guardrail()) + + await _run_pre_call(_bare_manager(), plo, logging_obj) + + assert logging_obj.recorded_guardrails == [{"guardrail_name": "test-guardrail", "guardrail_status": "success"}] + + +@pytest.mark.asyncio +async def test_pre_call_bridges_blocked_evaluation_before_reraising(): + """A block raises straight out of ``pre_call_tool_check``, and the failure + spend-log row that "Total Blocked" counts is built from this logger further up + the stack. So the record has to be attached before the exception leaves the + frame -- hence the bridge lives in a ``finally``.""" + capture: dict = {} + logging_obj = _FakeLoggingObj() + plo = _fake_proxy_logging(capture, guardrail_effect=_blocking_guardrail()) + + with pytest.raises(GuardrailRaisedException): + await _run_pre_call(_bare_manager(), plo, logging_obj) + + assert logging_obj.recorded_guardrails == [ + {"guardrail_name": "test-guardrail", "guardrail_status": "guardrail_intervened"} + ] + + +@pytest.mark.asyncio +async def test_pre_call_without_logging_obj_is_unchanged(): + """Callers that thread no logger are unaffected: the seed is an explicit + ``None`` (which every consumer reads via ``.get``) and nothing is bridged. + Guards against the bridge assuming a logger exists.""" + capture: dict = {} + plo = _fake_proxy_logging(capture, guardrail_effect=_record_guardrail()) + mgr = _bare_manager() + + result = await mgr.pre_call_tool_check( + name="t", + arguments={}, + server_name="s", + user_api_key_auth=None, + proxy_logging_obj=plo, + server=mock.MagicMock(), + raw_headers={}, + ) + + assert result == {} + assert capture["data"]["litellm_logging_obj"] is None + + +@pytest.mark.asyncio +async def test_during_hook_seeds_and_bridges_onto_request_logger(): + """``during_mcp_call`` evaluations need the same treatment. The task is awaited + before the tool call's success logging runs, so the record is serialized with + that call.""" + capture: dict = {} + logging_obj = _FakeLoggingObj() + plo = _fake_proxy_logging(capture, guardrail_effect=_record_guardrail()) + + await _bare_manager()._create_during_hook_task( + name="t", + arguments={}, + server_name_from_prefix="s", + user_api_key_auth=None, + proxy_logging_obj=plo, + start_time=datetime.datetime(2026, 7, 14), + litellm_logging_obj=logging_obj, + ) + + assert capture["data"]["litellm_logging_obj"] is logging_obj + assert logging_obj.recorded_guardrails == [{"guardrail_name": "test-guardrail", "guardrail_status": "success"}] + + +@pytest.mark.asyncio +async def test_during_hook_bridges_even_when_hook_raises(): + """A during-call guardrail block must still be recorded before the task's + exception propagates to the ``asyncio.gather`` in ``call_tool``.""" + capture: dict = {} + logging_obj = _FakeLoggingObj() + plo = _fake_proxy_logging(capture, guardrail_effect=_blocking_guardrail()) + + task = _bare_manager()._create_during_hook_task( + name="t", + arguments={}, + server_name_from_prefix="s", + user_api_key_auth=None, + proxy_logging_obj=plo, + start_time=datetime.datetime(2026, 7, 14), + litellm_logging_obj=logging_obj, + ) + with pytest.raises(GuardrailRaisedException): + await task + + assert logging_obj.recorded_guardrails == [ + {"guardrail_name": "test-guardrail", "guardrail_status": "guardrail_intervened"} + ] + + +@pytest.mark.asyncio +async def test_bridge_failure_does_not_mask_a_guardrail_block(): + """Recording is best-effort bookkeeping. If the bridge itself raises, the guardrail's + block must still be what the caller sees, not a bookkeeping error. + + The bridge is forced to fail by making the logger's ``model_call_details`` raise, and + the swallow is asserted (not just the surviving exception type) so the test cannot go + vacuous if a refactor stops the bridge from touching that attribute. + """ + capture: dict = {} + plo = _fake_proxy_logging(capture, guardrail_effect=_blocking_guardrail()) + + broken_logging_obj = mock.MagicMock() + type(broken_logging_obj).model_call_details = mock.PropertyMock(side_effect=RuntimeError("boom")) + + with mock.patch.object(MOD.verbose_logger, "warning") as warn: + with pytest.raises(GuardrailRaisedException): + await _run_pre_call(_bare_manager(), plo, broken_logging_obj) + + assert warn.call_count == 1, "the bridge did not actually fail, so this test proves nothing" + assert "boom" in str(warn.call_args) + + +@pytest.mark.asyncio +async def test_call_tool_threads_logging_obj_into_both_hooks(): + """``call_tool`` is the single entry point every MCP dispatch route funnels + through, so it must hand the logger to both guardrail hook sites.""" + mgr = _bare_manager() + logging_obj = _FakeLoggingObj() + seen: dict = {} + + async def _fake_pre_call_tool_check(**kwargs): + seen["pre_call"] = kwargs.get("litellm_logging_obj") + return {} + + def _fake_during_hook_task(**kwargs): + seen["during_call"] = kwargs.get("litellm_logging_obj") + return asyncio.get_running_loop().create_future() + + mgr.pre_call_tool_check = _fake_pre_call_tool_check + mgr._create_during_hook_task = _fake_during_hook_task + mgr._resolve_mcp_server_for_tool_call = lambda server_name, name: mock.MagicMock(spec_path=None) + mgr._resolve_oauth2_headers_for_tool_call = mock.AsyncMock(return_value=None) + mgr._call_regular_mcp_tool = mock.AsyncMock(return_value=mock.MagicMock()) + + with mock.patch.object(MOD, "_resolve_byok_mcp_auth_header", mock.AsyncMock(return_value=None)): + await mgr.call_tool( + server_name="s", + name="t", + arguments={}, + proxy_logging_obj=mock.MagicMock(), + litellm_logging_obj=logging_obj, + ) + + assert seen == {"pre_call": logging_obj, "during_call": logging_obj} diff --git a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py index 4981caa10c3..418c716b1a1 100644 --- a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -450,6 +450,42 @@ async def test_execute_tool_calls_passes_litellm_call_id_and_trace_id_to_functio assert captured.get("litellm_trace_id") == "tid" +@pytest.mark.asyncio +async def test_execute_tool_calls_threads_logging_obj_into_call_tool(monkeypatch): + """The Responses-API MCP path must hand the request's litellm_logging_obj to + global_mcp_server_manager.call_tool, otherwise pre_call_tool_check / + _create_during_hook_task get None and no guardrail evaluation is bridged onto + the request logger, so MCP tool calls made through the Responses API report zero + guardrail evaluations in the monitor. Drop the litellm_logging_obj kwarg on the + call_tool invocation and this fails.""" + _setup_proxy_logging(monkeypatch) + call_tool_mock = _setup_mcp_call_environment(monkeypatch) + + sentinel_logging_obj = MagicMock() + sentinel_logging_obj.async_post_mcp_tool_call_hook = AsyncMock() + sentinel_logging_obj.async_success_handler = AsyncMock() + + handler_module = importlib.import_module("litellm.responses.mcp.litellm_proxy_mcp_handler") + monkeypatch.setattr( + handler_module, + "function_setup", + lambda *_args, **_kwargs: (sentinel_logging_obj, None), + ) + + tool_name = "deepwiki-read_wiki_structure" + tool_calls = [{"id": "call-1", "function": {"name": tool_name, "arguments": "{}"}}] + + await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map={tool_name: "deepwiki"}, + tool_calls=tool_calls, + user_api_key_auth=None, + ) + + assert call_tool_mock.await_count == 1 + assert call_tool_mock.await_args is not None + assert call_tool_mock.await_args.kwargs["litellm_logging_obj"] is sentinel_logging_obj + + @pytest.mark.asyncio async def test_get_mcp_tools_from_manager_enables_list_tools_logging(monkeypatch): """ From a62798de63873f146efde812918fb24d30ed0621 Mon Sep 17 00:00:00 2001 From: Scott Wilson Date: Fri, 14 Aug 2026 17:56:44 -0400 Subject: [PATCH 15/26] test(anthropic): type the Responses tool fixtures instead of suppressing The two new `translate_tools_to_responses_api` calls carried `# type: ignore[arg-type]`, which CLAUDE.md bans as LIT009: pyrightconfig.json sets enableTypeIgnoreComments to false, so the comment silently does nothing and the reportArgumentType error stands. Annotating the fixtures as list[AllAnthropicToolsValues] makes both calls check clean with no suppression at all. --- .../test_responses_adapters_transformation.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 90733dc9134..8b34163e075 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -21,7 +21,10 @@ from litellm.constants import ( from litellm.llms.anthropic.experimental_pass_through.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) -from litellm.types.llms.anthropic import AnthropicMessagesRequest +from litellm.types.llms.anthropic import ( + AllAnthropicToolsValues, + AnthropicMessagesRequest, +) from litellm.types.llms.openai import ResponseAPIUsage @@ -618,7 +621,7 @@ class TestTranslateToolsToResponsesAPI: def test_tool_with_optional_properties_stays_non_strict(self): """Regression: an unset Anthropic `strict` must not become the Responses strict default, which would rewrite `required` to include every optional property.""" - tools = [ + tools: List[AllAnthropicToolsValues] = [ { "name": "search", "input_schema": { @@ -633,14 +636,14 @@ class TestTranslateToolsToResponsesAPI: } ] - result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + result = _ADAPTER.translate_tools_to_responses_api(tools) assert result[0]["strict"] is False assert result[0]["parameters"]["required"] == ["query"] def test_tool_forwards_explicit_strict_true(self): """An explicit Anthropic `strict: True` still reaches Responses as True.""" - tools = [ + tools: List[AllAnthropicToolsValues] = [ { "name": "search", "strict": True, @@ -653,7 +656,7 @@ class TestTranslateToolsToResponsesAPI: } ] - result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + result = _ADAPTER.translate_tools_to_responses_api(tools) assert result == [ { From ff3da21aab14dc935ef4087dfbef3ebcf5204c94 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 15 Aug 2026 15:39:50 -0700 Subject: [PATCH 16/26] refactor(ui): decouple bulk invite from the invite user button The bulk invite button was rendered from inside CreateUserButton, so the two actions were locked together and only the bulk one had been migrated, leaving the users page with an antd primary button sitting next to a shadcn one. Move BulkCreateUsersButton up to the users page toolbar so each action stands on its own, and migrate CreateUserButton's buttons to the shared shadcn Button so both triggers render identically. The teams prop only ever fed the bulk button, so it goes away from CreateUserButton and its other call site. --- .../users/_components/view_users.test.tsx | 13 +++++++ .../users/_components/view_users.tsx | 12 +++---- .../src/components/CreateUserButton.test.tsx | 13 ++++--- .../src/components/CreateUserButton.tsx | 35 ++++++------------- .../organisms/create_key_button.tsx | 1 - 5 files changed, 37 insertions(+), 37 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.test.tsx index 2632f40adbe..1a3d517df3e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.test.tsx @@ -148,12 +148,25 @@ describe("ViewUserDashboard", () => { expect(screen.getByRole("textbox", { name: "Default setting" })).toHaveValue("unsaved change"); }); + it("renders invite and bulk invite as toolbar actions alongside the other admin controls", async () => { + renderDashboard(); + + const inviteButton = await screen.findByRole("button", { name: /\+ invite user/i }); + const bulkInviteButton = screen.getByRole("button", { name: /\+ bulk invite users/i }); + const toolbar = screen.getByTestId("toggle-user-selection").parentElement; + + expect(inviteButton.parentElement).toBe(toolbar); + expect(bulkInviteButton.parentElement).toBe(toolbar); + }); + it("shows the users table without admin controls for non-proxy admins", async () => { renderDashboard({ userRole: "Internal User" }); expect(await screen.findByText("test@example.com")).toBeInTheDocument(); expect(screen.queryByRole("tab")).not.toBeInTheDocument(); expect(screen.queryByTestId("toggle-user-selection")).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /\+ invite user/i })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /\+ bulk invite users/i })).not.toBeInTheDocument(); }); it("keeps actions unavailable while the user list is loading", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.tsx index 1de01e88866..df2345b0af4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.tsx @@ -2,6 +2,7 @@ import { parseAsString, useQueryState } from "nuqs"; import React, { useCallback, useEffect, useMemo, useState } from "react"; import BulkEditUserModal from "./BulkEditUsers"; +import BulkCreateUsersButton from "@/components/bulk_create_users_button"; import { CreateUserButton } from "@/components/CreateUserButton"; import { Button } from "@/components/ui/button"; import { Skeleton } from "@/components/ui/skeleton"; @@ -365,12 +366,11 @@ const ViewUserDashboard: React.FC = ({ {!userListQuery.isLoading && userID && accessToken && ( <> {isProxyAdmin && ( - + + )} + + {isProxyAdmin && ( + )} {isProxyAdmin && ( diff --git a/ui/litellm-dashboard/src/components/CreateUserButton.test.tsx b/ui/litellm-dashboard/src/components/CreateUserButton.test.tsx index 246c01e9a9a..de1c3508dac 100644 --- a/ui/litellm-dashboard/src/components/CreateUserButton.test.tsx +++ b/ui/litellm-dashboard/src/components/CreateUserButton.test.tsx @@ -20,10 +20,6 @@ vi.mock("./networking", () => ({ getProxyBaseUrl: vi.fn().mockReturnValue("http://localhost"), })); -vi.mock("./bulk_create_users_button", () => ({ - default: () =>
Bulk Create Users
, -})); - vi.mock("@/app/(dashboard)/hooks/organizations/useOrganizations", () => ({ useOrganizations: vi.fn().mockReturnValue({ data: [], isLoading: false }), })); @@ -42,7 +38,6 @@ const createQueryClient = () => const defaultProps = { userID: "123", accessToken: "token", - teams: [], possibleUIRoles: null as Record> | null, }; @@ -75,6 +70,14 @@ describe("CreateUserButton", () => { }); }); + it("should not render the bulk invite button", async () => { + renderWithProviders(); + await waitFor(() => { + expect(screen.getByRole("button", { name: /\+ invite user/i })).toBeInTheDocument(); + }); + expect(screen.queryByRole("button", { name: /bulk invite users/i })).not.toBeInTheDocument(); + }); + it("should open the invite modal when invite user button is clicked", async () => { const user = userEvent.setup(); renderWithProviders(); diff --git a/ui/litellm-dashboard/src/components/CreateUserButton.tsx b/ui/litellm-dashboard/src/components/CreateUserButton.tsx index 8de9f010fa3..8a9146d756f 100644 --- a/ui/litellm-dashboard/src/components/CreateUserButton.tsx +++ b/ui/litellm-dashboard/src/components/CreateUserButton.tsx @@ -1,22 +1,11 @@ -import { InfoCircleOutlined, UserAddOutlined } from "@ant-design/icons"; +import { InfoCircleOutlined } from "@ant-design/icons"; import { useQueryClient } from "@tanstack/react-query"; import { useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrganizations"; +import { Button } from "@/components/ui/button"; import { Accordion, AccordionBody, AccordionHeader, SelectItem, TextInput } from "@tremor/react"; -import { - Alert, - Button, - Checkbox, - Form, - Input, - Modal, - Select, - Select as Select2, - Space, - Tooltip, - Typography, -} from "antd"; +import { Alert, Checkbox, Form, Input, Modal, Select, Select as Select2, Space, Tooltip, Typography } from "antd"; +import { UserPlus } from "lucide-react"; import React, { useEffect, useState } from "react"; -import BulkCreateUsers from "./bulk_create_users_button"; import TeamDropdown from "./common_components/team_dropdown"; import { getModelDisplayName } from "./key_team_helpers/fetch_available_models_team_key"; import NotificationsManager from "./molecules/notifications_manager"; @@ -46,7 +35,6 @@ const generateUUID = (): string => { interface CreateuserProps { userID: string; accessToken: string; - teams: any[] | null; possibleUIRoles: null | Record>; onUserCreated?: (userId: string) => void; isEmbedded?: boolean; @@ -63,7 +51,6 @@ interface UISettings { export const CreateUserButton: React.FC = ({ userID, accessToken, - teams, possibleUIRoles, onUserCreated, isEmbedded = false, @@ -79,8 +66,6 @@ export const CreateUserButton: React.FC = ({ const [baseUrl, setBaseUrl] = useState(null); const { data: organizations = [] } = useOrganizations(); - // Derive teams from the user's organizations, falling back to the teams prop - useEffect(() => { const fetchData = async () => { try { @@ -237,7 +222,7 @@ export const CreateUserButton: React.FC = ({
- +
); @@ -245,11 +230,10 @@ export const CreateUserButton: React.FC = ({ // Original return for standalone mode return ( -
- - = ({
-
@@ -391,6 +376,6 @@ export const CreateUserButton: React.FC = ({ invitationLinkData={invitationLinkData} /> )} -
+ ); }; diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index c747cfe36fe..a593b2b9f95 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -1718,7 +1718,6 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Date: Mon, 17 Aug 2026 11:55:06 -0700 Subject: [PATCH 17/26] fix(proxy): return 400 naming the missing required param on POST /v1/batches --- litellm/proxy/batches_endpoints/endpoints.py | 3 ++ litellm/proxy/route_llm_request.py | 31 ++++++++++------- .../proxy/batches_endpoints/test_endpoints.py | 30 ++++++++++++++++ .../proxy/test_route_llm_request.py | 34 ++++++++++++++++--- 4 files changed, 82 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index 6952c0c6f89..623900322be 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -40,6 +40,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( update_batch_in_database, validate_managed_id_requirement, ) +from litellm.proxy.route_llm_request import raise_if_required_body_param_missing from litellm.proxy.utils import handle_exception_on_proxy, is_known_model from litellm.repositories.table_repositories import ManagedFileRepository from litellm.types.llms.openai import LiteLLMBatchCreateRequest @@ -140,6 +141,8 @@ async def create_batch( ) data["metadata"] = sanitize_openai_provider_metadata(data.get("metadata")) + raise_if_required_body_param_missing(route_type="acreate_batch", data=data) + ## check if model is a loadbalanced model router_model: str | None = None is_router_model = False diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index b347360a939..c85325b6fa9 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -6,7 +6,7 @@ import httpx from fastapi import HTTPException, status import litellm -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.router_utils.common_utils import _is_proxy_admin_request # Client-supplied params that make the router or the call path fabricate a @@ -141,6 +141,7 @@ ROUTE_ENDPOINT_MAPPING: Final = { "aget_run": "/evals/{eval_id}/runs/{run_id}", "acancel_run": "/evals/{eval_id}/runs/{run_id}/cancel", "adelete_run": "/evals/{eval_id}/runs/{run_id}", + "acreate_batch": "/batches", } @@ -152,27 +153,33 @@ class ProxyModelNotFoundError(HTTPException): super().__init__(status_code=status.HTTP_400_BAD_REQUEST, detail=detail) -REQUIRED_BODY_PARAM_BY_ROUTE: Final[Mapping[str, str]] = { - "acompletion": "messages", - "aembedding": "input", +REQUIRED_BODY_PARAMS_BY_ROUTE: Final[Mapping[str, tuple[str, ...]]] = { + "acompletion": ("messages",), + "aembedding": ("input",), + "acreate_batch": ("input_file_id", "endpoint", "completion_window"), } -class ProxyMissingRequiredParamError(HTTPException): +class ProxyMissingRequiredParamError(ProxyException): def __init__(self, route: str, param: str): - detail: Final = {"error": f"{route}: Missing required parameter: '{param}'."} - super().__init__(status_code=status.HTTP_400_BAD_REQUEST, detail=detail) - self.type = "invalid_request_error" - self.param = param + super().__init__( + message=f"{route}: Missing required parameter: '{param}'.", + type="invalid_request_error", + param=param, + code=status.HTTP_400_BAD_REQUEST, + ) def raise_if_required_body_param_missing(route_type: str, data: Mapping[str, object]) -> None: - required_param: Final = REQUIRED_BODY_PARAM_BY_ROUTE.get(route_type) - if required_param is None or data.get(required_param) is not None: + missing_param: Final = next( + (param for param in REQUIRED_BODY_PARAMS_BY_ROUTE.get(route_type, ()) if data.get(param) is None), + None, + ) + if missing_param is None: return raise ProxyMissingRequiredParamError( route=ROUTE_ENDPOINT_MAPPING.get(route_type, route_type), - param=required_param, + param=missing_param, ) diff --git a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py index 824654dcf66..bef111d62bc 100644 --- a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py @@ -758,6 +758,36 @@ async def test_create__model_encoded_beats_loadbalancing(harness): harness.creds_resolver.assert_called_once_with(model_id="azure/gpt-4o") +# =========================================================================== # +# Missing required body params - 400 naming the field, never a 500 TypeError +# from acreate_batch() (https://github.com/BerriAI/litellm/issues/37146). +# =========================================================================== # + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body, missing_param", + [ + ({"endpoint": "/v1/chat/completions", "completion_window": "24h"}, "input_file_id"), + ({"input_file_id": "file-abc", "completion_window": "24h"}, "endpoint"), + ({"input_file_id": "file-abc", "endpoint": "/v1/chat/completions"}, "completion_window"), + ({}, "input_file_id"), + ], +) +async def test_create__missing_required_param_is_400(harness, body, missing_param): + set_body(harness, body) + + with pytest.raises(ProxyException) as exc_info: + await call_create(harness) + + assert exc_info.value.code == "400" + assert exc_info.value.type == "invalid_request_error" + assert exc_info.value.param == missing_param + assert exc_info.value.message == f"/batches: Missing required parameter: '{missing_param}'." + harness.litellm_acreate.assert_not_called() + harness.router_acreate.assert_not_called() + + # =========================================================================== # # Team-level batch expiry enforcement (independent of routing). # =========================================================================== # diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index 08e26125bd3..fc3b14592cd 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -1042,9 +1042,10 @@ async def test_route_request_override_enable_tag_filtering_beats_body_value(): [ ("acompletion", "messages", "/chat/completions"), ("aembedding", "input", "/embeddings"), + ("acreate_batch", "input_file_id", "/batches"), ], ) -@pytest.mark.parametrize("data_extra", [{}, {"messages": None, "input": None}]) +@pytest.mark.parametrize("data_extra", [{}, {"messages": None, "input": None, "input_file_id": None}]) def test_raise_if_required_body_param_missing_rejects_missing_param(route_type, param, route, data_extra): from litellm.proxy.route_llm_request import ( ProxyMissingRequiredParamError, @@ -1054,10 +1055,31 @@ def test_raise_if_required_body_param_missing_rejects_missing_param(route_type, with pytest.raises(ProxyMissingRequiredParamError) as exc_info: raise_if_required_body_param_missing(route_type=route_type, data={"model": "gpt-4o", **data_extra}) - assert exc_info.value.status_code == 400 + assert exc_info.value.code == "400" assert exc_info.value.param == param assert exc_info.value.type == "invalid_request_error" - assert exc_info.value.detail == {"error": f"{route}: Missing required parameter: '{param}'."} + assert exc_info.value.message == f"{route}: Missing required parameter: '{param}'." + + +@pytest.mark.parametrize( + "data, param", + [ + ({"endpoint": "/v1/chat/completions", "completion_window": "24h"}, "input_file_id"), + ({"input_file_id": "file-abc", "completion_window": "24h"}, "endpoint"), + ({"input_file_id": "file-abc", "endpoint": "/v1/chat/completions"}, "completion_window"), + ({}, "input_file_id"), + ], +) +def test_raise_if_required_body_param_missing_names_first_missing_batch_param(data, param): + from litellm.proxy.route_llm_request import ( + ProxyMissingRequiredParamError, + raise_if_required_body_param_missing, + ) + + with pytest.raises(ProxyMissingRequiredParamError) as exc_info: + raise_if_required_body_param_missing(route_type="acreate_batch", data=data) + + assert exc_info.value.param == param @pytest.mark.parametrize( @@ -1069,6 +1091,10 @@ def test_raise_if_required_body_param_missing_rejects_missing_param(route_type, ("aembedding", {"model": "text-embedding-3-small", "input": "hi"}), ("arerank", {"model": "rerank-model"}), ("aimage_generation", {"model": "dall-e-3"}), + ( + "acreate_batch", + {"input_file_id": "file-abc", "endpoint": "/v1/chat/completions", "completion_window": "24h"}, + ), ], ) def test_raise_if_required_body_param_missing_allows_valid_requests(route_type, data): @@ -1088,7 +1114,7 @@ async def test_route_request_rejects_chat_completion_without_messages(): with pytest.raises(ProxyMissingRequiredParamError) as exc_info: await route_request({"model": "gpt-4o"}, llm_router, None, "acompletion") - assert exc_info.value.status_code == 400 + assert exc_info.value.code == "400" assert exc_info.value.param == "messages" llm_router.acompletion.assert_not_called() From d5b91b94d38a9ae8be959c0a44fb1a9a1d48ba1b Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 17 Aug 2026 11:59:26 -0700 Subject: [PATCH 18/26] test(e2e): replay a real tool-search assistant turn back to Bedrock Invoke (#36856) The tool_search x bedrock_invoke cell only ever probed the first turn, so nothing in the suite has sent a server_tool_use block back to a provider. Every turn of a real Claude Code session after the first carries the server_tool_use and tool_search_tool_result blocks the previous turn produced, and that path was uncovered. Adds probe_tool_search_multiturn, which takes the real assistant turn back, answers any client-side tool_use with the id the model actually emitted, and replays the whole thing as history with the tools still declared. The assertion refuses to go green unless both server-tool blocks made it into the replayed history, so a first turn truncated at max_tokens reads as a failure instead of a vacuous pass. The replay assertion's red paths never run in a green cell, so they get markerless harness tests of their own alongside the existing _builder_unit_tests tree. No production code. --- .../claude_code/_probe_unit_tests/__init__.py | 0 .../_probe_unit_tests/test_http_probe.py | 106 +++++++++++++ tests/e2e/claude_code/http_probe.py | 144 +++++++++++++++++- .../tool_search/test_bedrock_invoke.py | 39 ++++- .../llm_claude_code_compat.yaml | 4 +- tests/e2e/coverage_registry/schema.py | 1 + tests/e2e/models.py | 43 +++++- 7 files changed, 327 insertions(+), 10 deletions(-) create mode 100644 tests/e2e/claude_code/_probe_unit_tests/__init__.py create mode 100644 tests/e2e/claude_code/_probe_unit_tests/test_http_probe.py diff --git a/tests/e2e/claude_code/_probe_unit_tests/__init__.py b/tests/e2e/claude_code/_probe_unit_tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/_probe_unit_tests/test_http_probe.py b/tests/e2e/claude_code/_probe_unit_tests/test_http_probe.py new file mode 100644 index 00000000000..6989868ba57 --- /dev/null +++ b/tests/e2e/claude_code/_probe_unit_tests/test_http_probe.py @@ -0,0 +1,106 @@ +"""Unit tests for the tool-search replay assertion in `http_probe`. + +Markerless harness tests: they exercise probe plumbing over hand-built +`Result` values, not a product feature, so they run without a proxy and carry +no `e2e` marker. + +The red paths are what these are for. A live cell only ever executes the green +one, so a broken diagnostic in the failure branch would sit undetected until +the day the provider actually rejects the history, which is the day the +diagnostic has to be right. +""" + +from __future__ import annotations + +from e2e_http import Result, Success, UnknownApiError +from models import ( + AnthropicContentBlock, + AnthropicMessagesResponse, + AnthropicToolResultTurn, + ChatMessage, +) + +from claude_code.http_probe import ( + ToolSearchReplay, + _replay_history, + assert_tool_search_replay_shape, +) + +_REJECTED: Result[AnthropicMessagesResponse] = UnknownApiError( + status_code=400, + body="server_tool_use blocks are not supported", +) +_ACCEPTED: Result[AnthropicMessagesResponse] = Success( + status_code=200, + data=AnthropicMessagesResponse(content=[AnthropicContentBlock(type="text", text="done")]), +) + + +def _replay(block_types: tuple[str, ...], second_turn: Result[AnthropicMessagesResponse]) -> ToolSearchReplay: + answer = AnthropicMessagesResponse( + content=[AnthropicContentBlock(type=block_type, id="srvtoolu_01") for block_type in block_types] + ) + return ToolSearchReplay( + first_turn=Success(status_code=200, data=answer), + history=_replay_history(answer), + second_turn=second_turn, + ) + + +def test_accepts_a_replayed_server_tool_pair() -> None: + replay = _replay(("text", "server_tool_use", "tool_search_tool_result"), _ACCEPTED) + assert assert_tool_search_replay_shape(replay) is None + + +def test_reports_the_status_when_the_replayed_history_is_rejected() -> None: + replay = _replay(("server_tool_use", "tool_search_tool_result"), _REJECTED) + error = assert_tool_search_replay_shape(replay) + assert error is not None + assert "status 400" in error + assert "server_tool_use" in error + + +def test_a_turn_truncated_before_the_result_block_is_not_a_pass() -> None: + replay = _replay(("server_tool_use",), _ACCEPTED) + error = assert_tool_search_replay_shape(replay) + assert error is not None + assert "tool_search_tool_result" in error + + +def test_a_history_with_no_server_tool_block_is_not_a_pass() -> None: + replay = _replay(("text",), _ACCEPTED) + error = assert_tool_search_replay_shape(replay) + assert error is not None + assert "server_tool_use" in error + + +def test_a_failed_first_turn_is_reported_as_the_first_turn() -> None: + replay = ToolSearchReplay(first_turn=_REJECTED, history=(), second_turn=None) + error = assert_tool_search_replay_shape(replay) + assert error is not None + assert error.startswith("first turn: ") + + +def test_a_pending_tool_use_is_answered_with_the_id_the_model_returned() -> None: + answer = AnthropicMessagesResponse( + content=[ + AnthropicContentBlock(type="server_tool_use", id="srvtoolu_01"), + AnthropicContentBlock(type="tool_search_tool_result", id=None), + AnthropicContentBlock(type="tool_use", id="toolu_99"), + ] + ) + last_turn = _replay_history(answer)[-1] + assert isinstance(last_turn, AnthropicToolResultTurn) + assert [block.tool_use_id for block in last_turn.content] == ["toolu_99"] + + +def test_a_turn_with_no_pending_tool_use_gets_a_plain_follow_up() -> None: + answer = AnthropicMessagesResponse( + content=[ + AnthropicContentBlock(type="server_tool_use", id="srvtoolu_01"), + AnthropicContentBlock(type="tool_search_tool_result"), + ] + ) + last_turn = _replay_history(answer)[-1] + assert isinstance(last_turn, ChatMessage) + assert last_turn.role == "user" diff --git a/tests/e2e/claude_code/http_probe.py b/tests/e2e/claude_code/http_probe.py index c77020acd6e..8aba54576c4 100644 --- a/tests/e2e/claude_code/http_probe.py +++ b/tests/e2e/claude_code/http_probe.py @@ -28,6 +28,7 @@ the upstream, or LiteLLM 500 on a transformation bug). from __future__ import annotations +from dataclasses import dataclass from typing import TYPE_CHECKING from pydantic import BaseModel @@ -42,10 +43,14 @@ from e2e_http import ( ValidationError, ) from models import ( + AnthropicAssistantTurn, AnthropicCustomTool, + AnthropicMessage, AnthropicMessagesBody, AnthropicMessagesResponse, AnthropicTool, + AnthropicToolResultBlock, + AnthropicToolResultTurn, AnthropicToolSearchTool, ChatMessage, CountTokensBody, @@ -132,6 +137,7 @@ def probe_tool_search( client: ProxyClient, api_key: str, model: str, + max_tokens: int = 64, rate_limiter: RateLimiter | None = None, ) -> Result[AnthropicMessagesResponse]: """POST to `/v1/messages` with a `tool_search_tool_regex_20251119` tool @@ -155,13 +161,115 @@ def probe_tool_search( api_key, AnthropicMessagesBody( model=model, - max_tokens=64, + max_tokens=max_tokens, messages=[ChatMessage(role="user", content=_TOOL_SEARCH_PROMPT)], tools=list(_TOOL_SEARCH_TOOLS), ), ) +_TOOL_SEARCH_FOLLOW_UP = "Thanks. Now reply with the word 'done'." +_TOOL_RESULT_STUB = "3" +# A `server_tool_use` block and the `tool_search_tool_result` answering it are +# one indivisible pair: replaying the request without its result is malformed +# Anthropic and 400s on any provider. 64 output tokens is not enough room for +# both, so the turn we replay is generated with a budget that fits the whole +# discovery round trip. +_REPLAY_SOURCE_MAX_TOKENS = 1024 +_REPLAYED_SERVER_BLOCKS = frozenset({"server_tool_use", "tool_search_tool_result"}) + + +@dataclass(frozen=True, slots=True) +class ToolSearchReplay: + """Both turns of the multi-turn probe plus the history the second turn + carried, so a failing cell can report which turn broke and what was on the + wire when it did.""" + + first_turn: Result[AnthropicMessagesResponse] + history: tuple[AnthropicMessage, ...] + second_turn: Result[AnthropicMessagesResponse] | None + + +def _replayed_server_block_types(history: tuple[AnthropicMessage, ...]) -> frozenset[str]: + return frozenset( + block.type + for turn in history + if isinstance(turn, AnthropicAssistantTurn) + for block in turn.content + if block.type in _REPLAYED_SERVER_BLOCKS + ) + + +def _replay_history(answer: AnthropicMessagesResponse) -> tuple[AnthropicMessage, ...]: + """Turn a real first-turn answer into a well-formed two-turn history. + + Every client-side `tool_use` the model emitted gets a `tool_result` keyed on + the id the model actually returned; a turn with none gets a plain follow-up + instead. An unanswered `tool_use`, or a `tool_result` pointing at an invented + id, is malformed Anthropic and 400s on any provider, which would make this + probe measure our own request rather than the provider's handling of the + replayed server-tool blocks.""" + blocks = tuple(answer.content or ()) + pending = tuple(block.id for block in blocks if block.type == "tool_use" and block.id is not None) + reply: AnthropicMessage = ( + AnthropicToolResultTurn( + content=[ + AnthropicToolResultBlock(tool_use_id=tool_use_id, content=_TOOL_RESULT_STUB) + for tool_use_id in pending + ] + ) + if pending + else ChatMessage(role="user", content=_TOOL_SEARCH_FOLLOW_UP) + ) + return ( + ChatMessage(role="user", content=_TOOL_SEARCH_PROMPT), + AnthropicAssistantTurn(content=list(blocks)), + reply, + ) + + +def probe_tool_search_multiturn( + *, + client: ProxyClient, + api_key: str, + model: str, + rate_limiter: RateLimiter | None = None, +) -> ToolSearchReplay: + """Run `probe_tool_search`, then send the real assistant turn back as + history with the same tools still declared. + + The first turn only proves the proxy attaches the tool-search beta header on + the way out. Nothing proves the provider accepts the `server_tool_use` and + `tool_search_tool_result` blocks it produced when they come back in + `messages`, which is every turn of a real Claude Code session after the + first.""" + first_turn = probe_tool_search( + client=client, + api_key=api_key, + model=model, + max_tokens=_REPLAY_SOURCE_MAX_TOKENS, + rate_limiter=rate_limiter, + ) + if not isinstance(first_turn, Success): + return ToolSearchReplay(first_turn=first_turn, history=(), second_turn=None) + + history = _replay_history(first_turn.data) + _acquire(model, rate_limiter) + return ToolSearchReplay( + first_turn=first_turn, + history=history, + second_turn=client.messages( + api_key, + AnthropicMessagesBody( + model=model, + max_tokens=64, + messages=list(history), + tools=list(_TOOL_SEARCH_TOOLS), + ), + ), + ) + + def _failure_diagnostic[R: BaseModel](result: Result[R], route: str) -> str: """Map a non-success `Result` to a one-line diagnostic. The `status 429` wording is load-bearing: the compat conftest classifies a rate-limited cell @@ -207,6 +315,40 @@ def assert_tool_search_shape(result: Result[AnthropicMessagesResponse]) -> str | return _failure_diagnostic(result, "/v1/messages") +def assert_tool_search_replay_shape(replay: ToolSearchReplay) -> str | None: + """Return None on success, else describe the first violation. + + Acceptance criteria: + + 1. The first turn succeeded, on the same terms as `assert_tool_search_shape`. + 2. That turn produced a complete `server_tool_use` / `tool_search_tool_result` + pair to replay. Without both the second turn carries either an ordinary + text history or a half-finished tool call, and the cell would report on + our own request rather than on the provider's handling of server-tool + blocks in history. + 3. The provider accepted the history containing those blocks. + """ + first_error = assert_tool_search_shape(replay.first_turn) + if first_error is not None: + return f"first turn: {first_error}" + + replayed = _replayed_server_block_types(replay.history) + missing = _REPLAYED_SERVER_BLOCKS - replayed + if missing: + return ( + f"first turn returned no {' or '.join(sorted(missing))} block to replay, so the history " + "proves nothing about server-tool handling; a turn truncated at max_tokens looks like this" + ) + + if replay.second_turn is None: + return "second turn was never sent" + + second_error = assert_tool_search_shape(replay.second_turn) + if second_error is not None: + return f"history replaying {sorted(replayed)} rejected: {second_error}" + return None + + def assert_count_tokens_shape(result: Result[CountTokensResponse]) -> str | None: """Return None on success, or an error string describing the first violation. diff --git a/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py index c4735c78f0c..5b4c50e9dc5 100644 --- a/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py @@ -1,12 +1,17 @@ """tool_search x Bedrock (Invoke). -HTTP-probe row. Sends a single `/v1/messages` request whose `tools` -array includes a `tool_search_tool_regex_20251119` discovery tool, and +HTTP-probe row. Sends a `/v1/messages` request whose `tools` array +includes a `tool_search_tool_regex_20251119` discovery tool, and asserts the proxy round-trips it to the upstream without a 400. This verifies LiteLLM's tool-search beta-header translation (`advanced-tool-use-2025-11-20` for Anthropic-shape providers, `tool-search-tool-2025-10-19` for Vertex/Bedrock) survives end-to-end. +A second probe then replays that turn's answer as history, which is +what every turn of a real session after the first looks like: the +first turn only exercises the outbound header, and the blocks the +model sends back have to be accepted on the way in too. + The (feature, provider) for this cell is inferred from the file path by `tests/e2e/claude_code/conftest.py`: @@ -47,8 +52,10 @@ import pytest from claude_code._env import require_proxy_client from claude_code.http_probe import ( + assert_tool_search_replay_shape, assert_tool_search_shape, probe_tool_search, + probe_tool_search_multiturn, ) @@ -80,3 +87,31 @@ def test_tool_search_bedrock_invoke(compat_result): if failures: pytest.fail("; ".join(failures), pytrace=False) + + +@pytest.mark.covers("llm.messages.bedrock_invoke.tool_search_history.nonstream.works") +def test_tool_search_history_bedrock_invoke(compat_result): + """Send the tool-search request, take the real assistant turn back, and + replay it as history with the tools still declared. + + Every turn of a real Claude Code session after the first carries the + `server_tool_use` and `tool_search_tool_result` blocks the previous turn + produced. The single-turn probe above never sends them, so it cannot see a + provider or a transformation that accepts tool_search on the way out and + rejects the blocks it gets back.""" + client, api_key = require_proxy_client(compat_result) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + replay = probe_tool_search_multiturn(client=client, api_key=api_key, model=model) + shape_error = assert_tool_search_replay_shape(replay) + if shape_error is not None: + error = f"[{model}] tool_search history replay failed: {shape_error}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/coverage_registry/llm_claude_code_compat.yaml b/tests/e2e/coverage_registry/llm_claude_code_compat.yaml index c2c17a6e764..d78f07564aa 100644 --- a/tests/e2e/coverage_registry/llm_claude_code_compat.yaml +++ b/tests/e2e/coverage_registry/llm_claude_code_compat.yaml @@ -8,7 +8,8 @@ # route : anthropic | azure_foundry | bedrock_converse | bedrock_invoke | vertex # capability : basic | tool_use | vision | thinking | prompt_cache_5m | prompt_cache_1h # | structured_output | pdf_input | long_context_1m -# | thinking_with_tool_use | tool_search | count_tokens | web_search +# | thinking_with_tool_use | tool_search | tool_search_history | count_tokens +# | web_search # streaming : stream | nonstream # ---- basic / non-streaming ---- @@ -94,6 +95,7 @@ - {id: llm.messages.bedrock_converse.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "tool_search discovery tool over Bedrock Converse"} - {id: llm.messages.bedrock_invoke.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "tool_search discovery tool over Bedrock Invoke"} - {id: llm.messages.vertex.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "tool_search discovery tool over Vertex AI"} +- {id: llm.messages.bedrock_invoke.tool_search_history.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: tool_search_history, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "A real server_tool_use / tool_search_tool_result pair replayed as history over Bedrock Invoke"} # ---- count_tokens ---- - {id: llm.messages.anthropic.count_tokens.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: count_tokens, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "/v1/messages/count_tokens over Anthropic direct"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index 76844c039f1..a5c723f8965 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -76,6 +76,7 @@ LlmCapability = Literal[ "thinking", "thinking_with_tool_use", "tool_search", + "tool_search_history", "tool_use", "vision", "web_search", diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 9ba191d7f0e..734e63a94e6 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -384,9 +384,45 @@ class AnthropicCustomTool(BaseModel): type AnthropicTool = AnthropicToolSearchTool | AnthropicWebSearchTool | AnthropicCustomTool +class AnthropicContentBlock(BaseModel): + """One block of a `content` array. Only the fields a test reads are + declared; `extra="allow"` keeps the rest (a `server_tool_use` block's + `input`, a `tool_search_tool_result` block's nested `content`) so an + assistant turn read off the wire can be replayed into history verbatim + instead of being silently flattened to its text.""" + + model_config = ConfigDict(extra="allow") + type: str | None = None + text: str | None = None + id: str | None = None + + +class AnthropicToolResultBlock(BaseModel): + """The user-turn answer to a client-side `tool_use`. `tool_use_id` must be + the id the model actually emitted; an invented one is rejected by + Anthropic's own schema validator, which Bedrock inherits.""" + + type: Literal["tool_result"] = "tool_result" + tool_use_id: str + content: str + + +class AnthropicAssistantTurn(BaseModel): + role: Literal["assistant"] = "assistant" + content: list[AnthropicContentBlock] + + +class AnthropicToolResultTurn(BaseModel): + role: Literal["user"] = "user" + content: list[AnthropicToolResultBlock] + + +type AnthropicMessage = ChatMessage | AnthropicAssistantTurn | AnthropicToolResultTurn + + class AnthropicMessagesBody(BaseModel): model: str - messages: list[ChatMessage] + messages: list[AnthropicMessage] max_tokens: int stream: bool | None = None tools: list[AnthropicTool] | None = None @@ -401,11 +437,6 @@ class CountTokensBody(BaseModel): messages: list[ChatMessage] -class AnthropicContentBlock(BaseModel): - type: str | None = None - text: str | None = None - - class AnthropicMessagesResponse(BaseModel): """A /v1/messages answer. `content` is the Anthropic-native passthrough shape; `choices` is the OpenAI-normalized shape LiteLLM emits for some From a9bb09905d2f14fd50d62ff0badf7e64c3c87fef Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:00:33 -0700 Subject: [PATCH 19/26] test(batches): drop redundant section banner --- .../test_litellm/proxy/batches_endpoints/test_endpoints.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py index bef111d62bc..b720ea16968 100644 --- a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py @@ -758,12 +758,6 @@ async def test_create__model_encoded_beats_loadbalancing(harness): harness.creds_resolver.assert_called_once_with(model_id="azure/gpt-4o") -# =========================================================================== # -# Missing required body params - 400 naming the field, never a 500 TypeError -# from acreate_batch() (https://github.com/BerriAI/litellm/issues/37146). -# =========================================================================== # - - @pytest.mark.asyncio @pytest.mark.parametrize( "body, missing_param", From 4ef7c3f41c8d8f55f486a55906789da27d3b1039 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:02:14 -0700 Subject: [PATCH 20/26] fix(ci): bump sqlparse to 0.6.0 to resolve osv-scan CVEs --- uv.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uv.lock b/uv.lock index 1cc924e50fe..63e8e31223c 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-12T18:09:57.187615Z" +exclude-newer = "2026-08-14T18:59:56.524034Z" exclude-newer-span = "P3D" [manifest] @@ -9075,11 +9075,11 @@ wheels = [ [[package]] name = "sqlparse" -version = "0.5.5" +version = "0.6.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/90/76/437d71068094df0726366574cf3432a4ed754217b436eb7429415cf2d480/sqlparse-0.5.5.tar.gz", hash = "sha256:e20d4a9b0b8585fdf63b10d30066c7c94c5d7a7ec47c889a2d83a3caa93ff28e", size = 120815, upload-time = "2025-12-19T07:17:45.073Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/d3/3f06a1006f2261d1342aefb3c71eed02f5d4ca5bdbecd86ebc12ad38306e/sqlparse-0.6.0.tar.gz", hash = "sha256:113c35c75365ab9cc9c7231d68c6428fb11c085fc8e9eb1ad659b7ddbf6cd2b9", size = 178477, upload-time = "2026-08-13T19:16:06.396Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/49/4b/359f28a903c13438ef59ebeee215fb25da53066db67b305c125f1c6d2a25/sqlparse-0.5.5-py3-none-any.whl", hash = "sha256:12a08b3bf3eec877c519589833aed092e2444e68240a3577e8e26148acc7b1ba", size = 46138, upload-time = "2025-12-19T07:17:46.573Z" }, + { url = "https://files.pythonhosted.org/packages/d9/50/f00935da0ec7cbf325f8dc4f772ae46fbc7b672dd62876e73f0a94adda57/sqlparse-0.6.0-py3-none-any.whl", hash = "sha256:b861c0288ce2fa56209a9a6412d2e066ac664b3873b89c26c9d8415e8e32996f", size = 50070, upload-time = "2026-08-13T19:16:04.062Z" }, ] [[package]] From 57e1bf41c6e546440681bfe7e423876b9945decb Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:08:56 -0700 Subject: [PATCH 21/26] fix(proxy): return 400 for non-object metadata and litellm_metadata instead of silent drop or 500 --- basedpyright-code-budget.json | 2 +- litellm/proxy/litellm_pre_call_utils.py | 63 +++++++++++------- ruff-strict-budget.json | 2 +- .../proxy/test_litellm_pre_call_utils.py | 65 ++++++++++++++++++- type-discipline-budget.json | 2 +- 5 files changed, 107 insertions(+), 27 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index c09369b9b2c..ba131ecac4c 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 22344 + "limit": 22343 }, "reportArgumentType": { "limit": 2578 diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index c4a350fb285..1b1e3c70110 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -38,6 +38,8 @@ from litellm.proxy._types import ( CommonProxyErrors, LitellmDataForBackendLLMCall, LitellmUserRoles, + ProxyErrorTypes, + ProxyException, SpecialHeaders, TeamCallbackMetadata, UserAPIKeyAuth, @@ -348,6 +350,36 @@ def reject_url_valued_destination(field: str, value: str) -> None: ) +_METADATA_JSON_TYPE_NAMES: Final[Mapping[type, str]] = MappingProxyType( + {bool: "a boolean", int: "an integer", float: "a number", str: "a string", list: "an array"} +) + + +def _invalid_metadata_type_error(field: str, value: object) -> ProxyException: + received_type: Final = _METADATA_JSON_TYPE_NAMES.get(type(value), f"a {type(value).__name__}") + return ProxyException( + message=f"Invalid type for '{field}': expected an object, but got {received_type} instead.", + type=ProxyErrorTypes.bad_request_error, + param=field, + code=400, + ) + + +def _normalized_metadata_object(field: str, value: object) -> Mapping[str, Any]: + """Return ``value`` as a metadata object or raise a 400 like OpenAI does. + + A JSON string that parses to an object is accepted because multipart/form-data + and ``extra_body`` callers can only send metadata as a string. The caller pops + the raw value from the request body before validating so the failure-logging + hooks that inspect the body afterwards don't crash on it and mask the 400 as a 500. + """ + if isinstance(value, dict): + return value + if isinstance(value, str) and isinstance((parsed := safe_json_loads(value)), dict): + return parsed + raise _invalid_metadata_type_error(field=field, value=value) + + def _strip_untrusted_request_header_controls( headers: Any, *, @@ -1572,6 +1604,10 @@ async def add_litellm_data_to_request( continue data.pop(_internal_key, None) _reject_url_valued_destinations(data) + for _metadata_field in ("metadata", "litellm_metadata"): + if (_raw_metadata := data.get(_metadata_field)) is not None: + data.pop(_metadata_field) + data[_metadata_field] = _normalized_metadata_object(_metadata_field, _raw_metadata) # Strip spoofable auth metadata from user-supplied metadata dict _user_metadata = data.get("metadata") if isinstance(_user_metadata, dict): @@ -1711,29 +1747,10 @@ async def add_litellm_data_to_request( verbose_proxy_logger.debug("receiving data: %s", data) - # Parse metadata if it's a string (e.g., from multipart/form-data) - if "metadata" in data and data["metadata"] is not None: - if isinstance(data["metadata"], str): - data["metadata"] = safe_json_loads(data["metadata"]) - if not isinstance(data["metadata"], dict): - verbose_proxy_logger.warning( - "Failed to parse 'metadata' as JSON dict. Received value: %s", data["metadata"] - ) - # requester_metadata is snapshotted AFTER the strip below so - # downstream consumers (e.g. PANW guardrail reading user_ip / - # profile_id) don't see attacker-injected admin slots preserved in - # the deepcopy. - - # Parse litellm_metadata if it's a string (e.g., from multipart/form-data or extra_body) - if "litellm_metadata" in data and data["litellm_metadata"] is not None: - if isinstance(data["litellm_metadata"], str): - parsed_litellm_metadata: Final = safe_json_loads(data["litellm_metadata"]) - if not isinstance(parsed_litellm_metadata, dict): - verbose_proxy_logger.warning( - "Failed to parse 'litellm_metadata' as JSON dict. Received value: %s", data["litellm_metadata"] - ) - else: - data["litellm_metadata"] = parsed_litellm_metadata + # requester_metadata is snapshotted AFTER the strip below so + # downstream consumers (e.g. PANW guardrail reading user_ip / + # profile_id) don't see attacker-injected admin slots preserved in + # the deepcopy. # Strip internal pipeline state and admin-injection slots from user input. # Runs AFTER the string-to-dict parse above so JSON-string metadata (sent diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 22d3119d457..15396a95632 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -201,7 +201,7 @@ "limit": 58 }, "SIM102": { - "limit": 319 + "limit": 317 }, "SIM103": { "limit": 119 diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index e31058f402e..f6b801d3122 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -11,7 +11,7 @@ from pydantic import ValidationError as PydanticValidationError from starlette.datastructures import Headers import litellm -from litellm.proxy._types import AddTeamCallback, TeamCallbackMetadata, UserAPIKeyAuth +from litellm.proxy._types import AddTeamCallback, ProxyException, TeamCallbackMetadata, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import ( KeyAndTeamLoggingSettings, LiteLLMProxyRequestSetup, @@ -417,6 +417,69 @@ async def test_add_litellm_data_to_request_string_metadata_does_not_crash(): assert updated["metadata"].get("generation_name") == "test" +def _batches_request_mock() -> MagicMock: + request_mock = MagicMock(spec=Request) + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/batches" + request_mock.url.path = "/v1/batches" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + return request_mock + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "field,value,received_type", + [ + ("metadata", "abc", "a string"), + ("litellm_metadata", "abc", "a string"), + ("metadata", 42, "an integer"), + ("litellm_metadata", [1, 2], "an array"), + ("metadata", True, "a boolean"), + ], +) +async def test_add_litellm_data_to_request_rejects_non_object_metadata(field, value, received_type): + """Regression for https://github.com/BerriAI/litellm/issues/37147: a + non-object metadata was silently dropped with a 200, and a non-object + litellm_metadata crashed later with a 500 ('str' object has no attribute + 'update'). Both must be a 400 naming the field, like OpenAI returns.""" + data = {"input_file_id": "file-abc", "endpoint": "/v1/chat/completions", field: value} + + with pytest.raises(ProxyException) as exc_info: + await add_litellm_data_to_request( + data=data, + request=_batches_request_mock(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert exc_info.value.code == "400" + assert exc_info.value.param == field + assert exc_info.value.message == f"Invalid type for '{field}': expected an object, but got {received_type} instead." + assert field not in data + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_parses_json_object_string_litellm_metadata(): + data = {"input_file_id": "file-abc", "litellm_metadata": json.dumps({"cost_centre": "research"})} + + updated = await add_litellm_data_to_request( + data=data, + request=_batches_request_mock(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated["litellm_metadata"]["cost_centre"] == "research" + + @pytest.mark.asyncio async def test_add_litellm_data_to_request_proxy_server_request_body_is_post_strip(): """Regression: proxy_server_request['body'] used to be snapshotted before diff --git a/type-discipline-budget.json b/type-discipline-budget.json index ca848190a32..6d70a6aa5f4 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -30,7 +30,7 @@ "limit": 16713 }, "LIT011": { - "limit": 5591 + "limit": 5590 }, "LIT012": { "limit": 4519 From c894697a2a88dd85c874c36f8710f384ff7d93a4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:21:12 -0700 Subject: [PATCH 22/26] fix(proxy): keep ProxyException status codes on /v1/messages instead of wrapping into 500 --- .../proxy/anthropic_endpoints/endpoints.py | 3 ++ .../anthropic_endpoints/test_endpoints.py | 39 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index f75899b91dc..a48ef0f08bb 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -193,6 +193,9 @@ async def anthropic_response( ) verbose_proxy_logger.exception("litellm.proxy.proxy_server.anthropic_response(): Exception occured - %s", e) + if isinstance(e, ProxyException): + raise + # Extract model_id from request metadata (same as success path) litellm_metadata: Final = data.get("litellm_metadata", {}) or {} model_info: Final = litellm_metadata.get("model_info", {}) or {} diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py index 69d90a8b59b..0a427df0cb7 100644 --- a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py @@ -125,6 +125,45 @@ class TestBlockedResponseUsage: mock_logging.post_call_failure_hook.assert_awaited_once() +class TestProxyExceptionPassthrough: + @pytest.mark.asyncio + async def test_anthropic_response_reraises_proxy_exception_unwrapped(self): + """A 400 ProxyException from request validation must surface as-is, + not be re-wrapped into a code-500 ProxyException.""" + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy._types import ProxyErrorTypes, ProxyException + + exc = ProxyException( + message="Invalid type for 'metadata': expected an object, but got a string instead.", + type=ProxyErrorTypes.bad_request_error, + param="metadata", + code=400, + ) + + with ( + patch.object(ep, "_read_request_body", new=AsyncMock(return_value={})), + patch.object( + ep.ProxyBaseLLMRequestProcessing, + "base_process_llm_request", + new=AsyncMock(side_effect=exc), + ), + patch.object(proxy_server, "proxy_logging_obj") as mock_logging, + ): + mock_logging.post_call_failure_hook = AsyncMock() + with pytest.raises(ProxyException) as exc_info: + await ep.anthropic_response( + fastapi_response=MagicMock(), + request=MagicMock(), + user_api_key_dict=MagicMock(), + ) + + assert exc_info.value is exc + assert exc_info.value.code == "400" + assert exc_info.value.param == "metadata" + mock_logging.post_call_failure_hook.assert_awaited_once() + + class TestEventLoggingBatchEndpoint: """Test the stubbed event logging batch endpoint""" From f3b0cdca433e8683d4af9944fb008fa9666c90c5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:32:30 -0700 Subject: [PATCH 23/26] fix(proxy): keep ProxyException status codes on /v1/moderations instead of wrapping into 500 --- litellm/proxy/proxy_server.py | 2 ++ tests/test_litellm/proxy/test_proxy_server.py | 34 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d76128a76e5..9692d4449d4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10373,6 +10373,8 @@ async def moderations( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) verbose_proxy_logger.exception("litellm.proxy.proxy_server.moderations(): Exception occured - %s", e) + if isinstance(e, ProxyException): + raise if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e)), diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index ba207242e29..5545ee92e84 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -11064,3 +11064,37 @@ async def test_ptu_rollup_job_not_registered_without_opt_in(monkeypatch): assert scheduler.get_job(PTU_ROLLUP_JOB_ID) is None assert len(scheduler.get_jobs()) > 0 + + +@pytest.mark.asyncio +async def test_moderations_reraises_proxy_exception_unwrapped(): + """A 400 ProxyException from request validation must surface as-is, + not be re-wrapped into a code-500 ProxyException.""" + from litellm.proxy._types import ProxyErrorTypes, ProxyException + + exc = ProxyException( + message="Invalid type for 'metadata': expected an object, but got a string instead.", + type=ProxyErrorTypes.bad_request_error, + param="metadata", + code=400, + ) + + request = MagicMock() + request.body = AsyncMock(return_value=b'{"input": "hi", "metadata": "abc"}') + + with ( + patch.object(proxy_server_module, "add_litellm_data_to_request", new=AsyncMock(side_effect=exc)), + patch.object(proxy_server_module, "proxy_logging_obj") as mock_logging, + ): + mock_logging.post_call_failure_hook = AsyncMock() + with pytest.raises(ProxyException) as exc_info: + await proxy_server_module.moderations( + request=request, + fastapi_response=MagicMock(), + user_api_key_dict=MagicMock(), + ) + + assert exc_info.value is exc + assert exc_info.value.code == "400" + assert exc_info.value.param == "metadata" + mock_logging.post_call_failure_hook.assert_awaited_once() From a21de07b3faea70032d9f14fc59348fd322b1426 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:44:46 -0700 Subject: [PATCH 24/26] fix(proxy): drop every invalid metadata field before raising so failure hooks never see them --- litellm/proxy/litellm_pre_call_utils.py | 11 ++++++---- .../proxy/test_litellm_pre_call_utils.py | 22 +++++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 1b1e3c70110..6172f3a9158 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1604,10 +1604,13 @@ async def add_litellm_data_to_request( continue data.pop(_internal_key, None) _reject_url_valued_destinations(data) - for _metadata_field in ("metadata", "litellm_metadata"): - if (_raw_metadata := data.get(_metadata_field)) is not None: - data.pop(_metadata_field) - data[_metadata_field] = _normalized_metadata_object(_metadata_field, _raw_metadata) + _raw_metadata_by_field: Final = { + _metadata_field: data.pop(_metadata_field) + for _metadata_field in ("metadata", "litellm_metadata") + if data.get(_metadata_field) is not None + } + for _metadata_field, _raw_metadata in _raw_metadata_by_field.items(): + data[_metadata_field] = _normalized_metadata_object(_metadata_field, _raw_metadata) # Strip spoofable auth metadata from user-supplied metadata dict _user_metadata = data.get("metadata") if isinstance(_user_metadata, dict): diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index f6b801d3122..d3b1e089489 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -464,6 +464,28 @@ async def test_add_litellm_data_to_request_rejects_non_object_metadata(field, va assert field not in data +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_removes_every_invalid_metadata_field_before_raising(): + """When both fields are invalid, the raise for the first must not leave the + second invalid value in data, or failure-logging hooks that inspect the body + can crash on it and mask the 400 as a 500.""" + data = {"input_file_id": "file-abc", "metadata": "abc", "litellm_metadata": "xyz"} + + with pytest.raises(ProxyException) as exc_info: + await add_litellm_data_to_request( + data=data, + request=_batches_request_mock(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert exc_info.value.param == "metadata" + assert "metadata" not in data + assert "litellm_metadata" not in data + + @pytest.mark.asyncio async def test_add_litellm_data_to_request_parses_json_object_string_litellm_metadata(): data = {"input_file_id": "file-abc", "litellm_metadata": json.dumps({"cost_centre": "research"})} From d0a815d2e1578e6a475b2c38d3db8de85258e9df Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 17 Aug 2026 13:03:05 -0700 Subject: [PATCH 25/26] fix(ui): stop pairing key spend with the team budget when a key has no budget (#37196) The key overview card and the Virtual Keys spend meter fell back to the parent team's max_budget as the denominator while the numerator stayed the key's own spend, so a $0.50 key on a $1,200 team read as "$0.50 of $1,200 (Team)" and drew a meter against a limit that governs the whole team's aggregate spend, not this key. Both surfaces now show Unlimited for a budgetless key and, when the parent team or organization does carry a budget, a hover hint listing those inherited caps so the reader knows what still gates the key --- ui/litellm-dashboard/eslint-suppressions.json | 3 - .../VirtualKeysPage/keyTableColumns.tsx | 8 +- .../shared/InheritedBudgetHint.test.tsx | 60 +++++++++++++++ .../components/shared/InheritedBudgetHint.tsx | 67 +++++++++++++++++ .../table_cells/spend_budget_cell.test.tsx | 23 +++++- .../shared/table_cells/spend_budget_cell.tsx | 14 ++-- .../key_info_view.budget_display.test.tsx | 74 ++++++++++++++++--- .../components/templates/key_info_view.tsx | 15 ++-- 8 files changed, 229 insertions(+), 35 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/shared/InheritedBudgetHint.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/InheritedBudgetHint.tsx diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 4a6d7006893..b1d9cf28503 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -2947,9 +2947,6 @@ "max-lines": { "count": 1 }, - "no-nested-ternary": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 1 } diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx index fdbc07ee020..48eb4dd08d5 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx @@ -5,6 +5,7 @@ import { ColumnDef } from "@tanstack/react-table"; import { Popover, Typography } from "antd"; import { DataTableMultiSortHeader, DataTableSortHeader, type DataTableSortField } from "@/components/shared/DataTable"; +import { inheritedBudgetGates } from "@/components/shared/InheritedBudgetHint"; import { Skeleton } from "@/components/ui/skeleton"; import { DateCell, @@ -304,13 +305,14 @@ export const getKeyTableColumns = ({ size: 180, enableSorting: true, cell: ({ row }) => { - const teamId = row.original.team_id; - const team = allTeams.find((t) => t.team_id === teamId); + const team = allTeams.find((t) => t.team_id === row.original.team_id); + const orgId = row.original.organization_id || row.original.org_id || team?.organization_id; + const organization = organizations.find((o) => o.organization_id === orgId); return ( ); }, diff --git a/ui/litellm-dashboard/src/components/shared/InheritedBudgetHint.test.tsx b/ui/litellm-dashboard/src/components/shared/InheritedBudgetHint.test.tsx new file mode 100644 index 00000000000..657e859aef9 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/InheritedBudgetHint.test.tsx @@ -0,0 +1,60 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it } from "vitest"; + +import { InheritedBudgetHint, inheritedBudgetGates } from "./InheritedBudgetHint"; + +const team = { team_id: "team-1", team_alias: "Platform", max_budget: 1200, budget_duration: "30d" }; +const organization = { + organization_id: "org-1", + organization_alias: "Acme", + litellm_budget_table: { max_budget: 5000, budget_duration: null }, +}; + +describe("inheritedBudgetGates", () => { + it("returns team then org gates when both have budgets", () => { + expect(inheritedBudgetGates(team, organization)).toEqual([ + { scope: "Team", alias: "Platform", maxBudget: 1200, budgetDuration: "30d" }, + { scope: "Organization", alias: "Acme", maxBudget: 5000, budgetDuration: null }, + ]); + }); + + it("skips a team or org whose max_budget is null", () => { + expect(inheritedBudgetGates({ ...team, max_budget: null }, organization)).toEqual([ + { scope: "Organization", alias: "Acme", maxBudget: 5000, budgetDuration: null }, + ]); + expect(inheritedBudgetGates(team, { ...organization, litellm_budget_table: { max_budget: null } })).toEqual([ + { scope: "Team", alias: "Platform", maxBudget: 1200, budgetDuration: "30d" }, + ]); + }); + + it("returns nothing when team and org are missing or budgetless", () => { + expect(inheritedBudgetGates(null, undefined)).toEqual([]); + expect( + inheritedBudgetGates({ ...team, max_budget: null }, { ...organization, litellm_budget_table: null }), + ).toEqual([]); + }); + + it("falls back to ids when aliases are empty", () => { + expect( + inheritedBudgetGates({ ...team, team_alias: "" }, { ...organization, organization_alias: "" }).map( + (g) => g.alias, + ), + ).toEqual(["team-1", "org-1"]); + }); +}); + +describe("InheritedBudgetHint", () => { + it("renders nothing without gates", () => { + const { container } = render(); + expect(container).toBeEmptyDOMElement(); + }); + + it("shows each gate with its budget and duration on hover", async () => { + render(); + await userEvent.setup().hover(screen.getByLabelText("question-circle")); + expect(screen.getByTestId("inherited-budget-hint")).toHaveTextContent("Team Platform: $1,200.00 / 30d"); + expect(screen.getByTestId("inherited-budget-hint")).toHaveTextContent("Organization Acme: $5,000.00"); + expect(screen.getByTestId("inherited-budget-hint")).not.toHaveTextContent("Organization Acme: $5,000.00 /"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/InheritedBudgetHint.tsx b/ui/litellm-dashboard/src/components/shared/InheritedBudgetHint.tsx new file mode 100644 index 00000000000..6be1dad564f --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/InheritedBudgetHint.tsx @@ -0,0 +1,67 @@ +"use client"; + +import { Tooltip } from "@/components/atoms/Tooltip"; +import type { Team } from "@/components/key_team_helpers/key_list"; +import type { Organization } from "@/components/networking"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; + +export interface InheritedBudgetGate { + scope: "Team" | "Organization"; + alias: string; + maxBudget: number; + budgetDuration: string | null; +} + +type TeamBudgetSource = Pick; +type OrganizationBudgetSource = Pick; + +const teamGate = (team: TeamBudgetSource | null | undefined): InheritedBudgetGate | null => + team && team.max_budget != null + ? { + scope: "Team", + alias: team.team_alias || team.team_id, + maxBudget: team.max_budget, + budgetDuration: team.budget_duration ?? null, + } + : null; + +const organizationGate = (organization: OrganizationBudgetSource | null | undefined): InheritedBudgetGate | null => { + const budgetTable: { max_budget?: number | null; budget_duration?: string | null } | null | undefined = + organization?.litellm_budget_table; + return organization && budgetTable?.max_budget != null + ? { + scope: "Organization", + alias: organization.organization_alias || organization.organization_id, + maxBudget: budgetTable.max_budget, + budgetDuration: budgetTable.budget_duration ?? null, + } + : null; +}; + +export const inheritedBudgetGates = ( + team: TeamBudgetSource | null | undefined, + organization: OrganizationBudgetSource | null | undefined, +): readonly InheritedBudgetGate[] => [teamGate(team), organizationGate(organization)].filter((gate) => gate !== null); + +const formatGate = (gate: InheritedBudgetGate): string => + `${gate.scope} ${gate.alias}: $${formatNumberWithCommas(gate.maxBudget, 2)}${gate.budgetDuration ? ` / ${gate.budgetDuration}` : ""}`; + +interface InheritedBudgetHintProps { + gates: readonly InheritedBudgetGate[]; +} + +export function InheritedBudgetHint({ gates }: InheritedBudgetHintProps) { + if (gates.length === 0) return null; + return ( + + This key has no budget of its own, but its spend still counts toward: + {gates.map((gate) => ( + {formatGate(gate)} + ))} +
+ } + /> + ); +} diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/spend_budget_cell.test.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/spend_budget_cell.test.tsx index d4af8428d69..0bc14849040 100644 --- a/ui/litellm-dashboard/src/components/shared/table_cells/spend_budget_cell.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/table_cells/spend_budget_cell.test.tsx @@ -53,9 +53,24 @@ describe("SpendBudgetCell", () => { expect(indicator(container)?.className).toContain("bg-destructive"); }); - it("falls back to the team budget and labels it", () => { - render(); - expect(screen.getByText("of $200 (Team)")).toBeInTheDocument(); - expect(screen.getByRole("meter")).toHaveAttribute("aria-valuemax", "200"); + it("never meters key spend against an inherited team/org budget", () => { + const gates = [{ scope: "Team" as const, alias: "Team A", maxBudget: 200, budgetDuration: "30d" }]; + render(); + expect(screen.getByText("· Unlimited")).toBeInTheDocument(); + expect(screen.queryByText(/\(Team\)/)).not.toBeInTheDocument(); + expect(screen.queryByRole("meter")).not.toBeInTheDocument(); + expect(screen.getByLabelText("question-circle")).toBeInTheDocument(); + }); + + it("shows no inherited-budget hint when there is nothing to inherit", () => { + render(); + expect(screen.queryByLabelText("question-circle")).not.toBeInTheDocument(); + }); + + it("shows no inherited-budget hint when the key has its own budget", () => { + const gates = [{ scope: "Team" as const, alias: "Team A", maxBudget: 200, budgetDuration: null }]; + render(); + expect(screen.getByText("of $50")).toBeInTheDocument(); + expect(screen.queryByLabelText("question-circle")).not.toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/spend_budget_cell.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/spend_budget_cell.tsx index 60b42615967..943b9aa766c 100644 --- a/ui/litellm-dashboard/src/components/shared/table_cells/spend_budget_cell.tsx +++ b/ui/litellm-dashboard/src/components/shared/table_cells/spend_budget_cell.tsx @@ -1,12 +1,13 @@ "use client"; +import { InheritedBudgetHint, type InheritedBudgetGate } from "@/components/shared/InheritedBudgetHint"; import { Meter, MeterIndicator, MeterTrack } from "@/components/ui/meter"; import { formatNumberWithCommas, getSpendString } from "@/utils/dataUtils"; interface SpendBudgetCellProps { spend: number | null | undefined; maxBudget: number | null | undefined; - teamMaxBudget?: number | null; + inheritedGates?: readonly InheritedBudgetGate[]; spendDecimals?: number; budgetDecimals?: number; } @@ -20,27 +21,24 @@ const meterTone = (pct: number): "default" | "warning" | "over" => { export function SpendBudgetCell({ spend, maxBudget, - teamMaxBudget, + inheritedGates = [], spendDecimals = 4, budgetDecimals = 0, }: SpendBudgetCellProps) { const spendValue = typeof spend === "number" && !Number.isNaN(spend) ? spend : 0; - const budget = maxBudget ?? teamMaxBudget ?? null; - const isTeamBudget = maxBudget == null && teamMaxBudget != null; + const budget = maxBudget ?? null; const hasBudget = typeof budget === "number" && budget > 0; const pct = hasBudget ? (spendValue / budget) * 100 : 0; const spendText = spendValue > 0 ? getSpendString(spendValue, spendDecimals) : "$0.00"; - const budgetLabel = - budget === null - ? "· Unlimited" - : `of $${formatNumberWithCommas(budget, budgetDecimals)}${isTeamBudget ? " (Team)" : ""}`; + const budgetLabel = budget === null ? "· Unlimited" : `of $${formatNumberWithCommas(budget, budgetDecimals)}`; return (
{spendText}{" "} {budgetLabel} + {budget === null && }
{hasBudget && ( ({ useRouter: () => ({ push: vi.fn() }) })); -vi.mock("@/app/(dashboard)/hooks/organizations/useOrganizations", () => ({ - useOrganizations: () => ({ data: [] }), -})); - vi.mock("./key_edit_view", () => ({ KeyEditView: () =>
, })); vi.mock("@/app/(dashboard)/hooks/useTeams", () => ({ default: vi.fn() })); +vi.mock("@/app/(dashboard)/hooks/organizations/useOrganizations", () => ({ useOrganizations: vi.fn() })); vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: vi.fn() })); vi.mock("@/app/(dashboard)/hooks/projects/useProjects", () => ({ useProjects: vi.fn().mockReturnValue({ data: [], isLoading: false }), @@ -130,10 +130,34 @@ const makeTeam = (overrides: Partial): Team => ({ ...overrides, }); +const makeOrganization = (overrides: Partial): Organization => + ({ + organization_id: "org-1", + organization_alias: "Acme Org", + budget_id: "budget-1", + metadata: {}, + models: [], + spend: 0, + model_spend: {}, + created_at: "2026-01-01T00:00:00Z", + created_by: "admin", + updated_at: "2026-01-01T00:00:00Z", + updated_by: "admin", + litellm_budget_table: { max_budget: null, budget_duration: null }, + teams: null, + users: null, + members: null, + ...overrides, + }) as Organization; + +const mockOrganizations = (organizations: Organization[]) => + vi.mocked(useOrganizations).mockReturnValue({ data: organizations } as ReturnType); + describe("KeyInfoView overview budget display (LIT-2845)", () => { beforeEach(() => { vi.mocked(useTeams).mockReturnValue({ teams: [], setTeams: vi.fn() }); vi.mocked(useAuthorized).mockReturnValue(baseAuthorized); + mockOrganizations([]); }); it("renders a sub-dollar max_budget ($0.10) with 2-decimal precision in the overview Spend card", async () => { @@ -188,7 +212,7 @@ describe("KeyInfoView overview budget display (LIT-2845)", () => { }); }); - it("renders team budget with alias and duration when key has no own budget but team has one", async () => { + it("never pairs key spend with the team budget: shows Unlimited plus an inherited-budget hint", async () => { vi.mocked(useTeams).mockReturnValue({ teams: [makeTeam({ team_id: "team-123", team_alias: "Test Budget", max_budget: 1200, budget_duration: "30d" })], setTeams: vi.fn(), @@ -203,15 +227,20 @@ describe("KeyInfoView overview budget display (LIT-2845)", () => { />, ); await waitFor(() => { - expect(screen.getByText(/of \$1,200\.00 \(Team: Test Budget \/ 30d\)/)).toBeInTheDocument(); + expect(screen.getByText(/of Unlimited/)).toBeInTheDocument(); }); + expect(screen.queryByText(/of \$1,200\.00/)).not.toBeInTheDocument(); + expect(screen.queryByText(/\(Team: Test Budget/)).not.toBeInTheDocument(); + await userEvent.setup().hover(screen.getByLabelText("question-circle")); + expect(screen.getByTestId("inherited-budget-hint")).toHaveTextContent("Team Test Budget: $1,200.00 / 30d"); }); - it("renders team budget without duration when team has no budget_duration", async () => { + it("lists the organization budget in the hint when the team's org has one", async () => { vi.mocked(useTeams).mockReturnValue({ - teams: [makeTeam({ team_id: "team-456", team_alias: "No Duration Team", max_budget: 500 })], + teams: [makeTeam({ team_id: "team-456", team_alias: "Org Team", organization_id: "org-1" })], setTeams: vi.fn(), }); + mockOrganizations([makeOrganization({ litellm_budget_table: { max_budget: 5000, budget_duration: null } })]); renderWithProviders( { />, ); await waitFor(() => { - expect(screen.getByText(/of \$500\.00 \(Team: No Duration Team\)/)).toBeInTheDocument(); + expect(screen.getByText(/of Unlimited/)).toBeInTheDocument(); }); + await userEvent.setup().hover(screen.getByLabelText("question-circle")); + expect(screen.getByTestId("inherited-budget-hint")).toHaveTextContent("Organization Acme Org: $5,000.00"); + expect(screen.getByTestId("inherited-budget-hint")).not.toHaveTextContent("Team Org Team"); }); - it("renders 'Unlimited' when key has no budget and team also has no budget", async () => { + it("renders 'Unlimited' with no hint when neither key, team, nor org has a budget", async () => { vi.mocked(useTeams).mockReturnValue({ teams: [makeTeam({ team_id: "team-789", team_alias: "Free Team" })], setTeams: vi.fn(), @@ -243,6 +275,27 @@ describe("KeyInfoView overview budget display (LIT-2845)", () => { await waitFor(() => { expect(screen.getByText(/of Unlimited/)).toBeInTheDocument(); }); + expect(screen.queryByLabelText("question-circle")).not.toBeInTheDocument(); + }); + + it("shows no hint when the key has its own budget even if the team has one", async () => { + vi.mocked(useTeams).mockReturnValue({ + teams: [makeTeam({ team_id: "team-123", team_alias: "Test Budget", max_budget: 1200 })], + setTeams: vi.fn(), + }); + renderWithProviders( + {}} + keyId={"test-key-id"} + onKeyDataUpdate={() => {}} + teams={[]} + />, + ); + await waitFor(() => { + expect(screen.getByText(/of \$25\.00/)).toBeInTheDocument(); + }); + expect(screen.queryByLabelText("question-circle")).not.toBeInTheDocument(); }); }); @@ -250,6 +303,7 @@ describe("KeyInfoView budget reset visibility", () => { beforeEach(() => { vi.mocked(useTeams).mockReturnValue({ teams: [], setTeams: vi.fn() }); vi.mocked(useAuthorized).mockReturnValue(baseAuthorized); + mockOrganizations([]); }); const KEY_WITH_RESET = { diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index 3ee11325a65..99981edb1c8 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -36,6 +36,7 @@ import { extractMcpEntitlement } from "../mcp_server_management/mcpEntitlement"; import ObjectPermissionsView from "../object_permissions_view"; import { RegenerateKeyModal } from "../organisms/RegenerateKeyModal"; import { parseErrorMessage } from "../shared/errorUtils"; +import { InheritedBudgetHint, inheritedBudgetGates } from "../shared/InheritedBudgetHint"; import { KeyEditView } from "./key_edit_view"; interface KeyInfoViewProps { @@ -460,12 +461,9 @@ export default function KeyInfoView({ const orgId = currentKeyData.organization_id || currentKeyData.org_id || parentTeam?.organization_id || ""; const parentOrg = orgId ? organizations?.find((org) => org.organization_id === orgId) : null; - const budgetDisplay = - currentKeyData.max_budget !== null - ? `$${formatNumberWithCommas(currentKeyData.max_budget, 2)}` - : parentTeam?.max_budget != null - ? `$${formatNumberWithCommas(parentTeam.max_budget, 2)} (Team: ${parentTeam.team_alias || parentTeam.team_id}${parentTeam.budget_duration ? ` / ${parentTeam.budget_duration}` : ""})` - : "Unlimited"; + const hasOwnBudget = currentKeyData.max_budget !== null; + const budgetDisplay = hasOwnBudget ? `$${formatNumberWithCommas(currentKeyData.max_budget, 2)}` : "Unlimited"; + const inheritedGates = hasOwnBudget ? [] : inheritedBudgetGates(parentTeam, parentOrg); return (
@@ -616,7 +614,10 @@ export default function KeyInfoView({

Spend

${formatNumberWithCommas(currentKeyData.spend, 4)}

-

of {budgetDisplay}

+

+ of {budgetDisplay} + +

{currentKeyData.budget_reset_at && (

Resets {formatTimestamp(currentKeyData.budget_reset_at)}

)} From 5295da055cd75061e8e265381a888289023fb51f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:10:21 -0700 Subject: [PATCH 26/26] refactor(anthropic): take Mapping in _open_block to satisfy LIT001 --- .../responses_adapters/streaming_iterator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index c588e791cd9..e2ad9c9c6d3 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -3,7 +3,7 @@ import json import traceback from collections import deque -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Mapping from typing import Any, Final from litellm import verbose_logger @@ -68,7 +68,7 @@ class AnthropicResponsesStreamWrapper: self._current_block_index += 1 return self._current_block_index - def _open_block(self, item_id: str | None, content_block: dict[str, Any]) -> int: + def _open_block(self, item_id: str | None, content_block: Mapping[str, Any]) -> int: block_idx = self._next_block_index() if item_id: self._item_id_to_block_index[item_id] = block_idx