From 1f3d75a67e2e6e4d5ae0d5f5b082d8bef7cc5b74 Mon Sep 17 00:00:00 2001 From: Dominic Feliton <37809476+dominicfeliton@users.noreply.github.com> Date: Tue, 13 Jan 2026 14:19:38 -0800 Subject: [PATCH 1/6] Add QueryClient to model hub --- ui/litellm-dashboard/src/app/model_hub_table/page.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/app/model_hub_table/page.tsx b/ui/litellm-dashboard/src/app/model_hub_table/page.tsx index 1df7019ad25..dc5ae01935e 100644 --- a/ui/litellm-dashboard/src/app/model_hub_table/page.tsx +++ b/ui/litellm-dashboard/src/app/model_hub_table/page.tsx @@ -2,6 +2,9 @@ import React, { useEffect, useState } from "react"; import { useSearchParams } from "next/navigation"; import ModelHubTable from "@/components/AIHub/ModelHubTable"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; + +const queryClient = new QueryClient(); export default function PublicModelHubTable() { const searchParams = useSearchParams()!; @@ -19,5 +22,9 @@ export default function PublicModelHubTable() { * populate navbar * */ - return ; + return ( + + + + ); } From 9c2f380a99491cf05d77148dc466930a1c4c0be9 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 13 Jan 2026 17:01:51 -0800 Subject: [PATCH 2/6] only show own internal user usage --- .../common_daily_activity.py | 9 +- .../management_endpoints/team_endpoints.py | 34 +- .../test_team_endpoints.py | 363 ++++++++++++++++++ 3 files changed, 401 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index f52abf86b97..c52491efc7c 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -343,7 +343,7 @@ def _build_where_conditions( start_date: str, end_date: str, model: Optional[str], - api_key: Optional[str], + api_key: Optional[Union[str, List[str]]], exclude_entity_ids: Optional[List[str]] = None, ) -> Dict[str, Any]: """Build prisma where clause for daily activity queries.""" @@ -357,7 +357,10 @@ def _build_where_conditions( if model: where_conditions["model"] = model if api_key: - where_conditions["api_key"] = api_key + if isinstance(api_key, list): + where_conditions["api_key"] = {"in": api_key} + else: + where_conditions["api_key"] = api_key if entity_id is not None: if isinstance(entity_id, list): @@ -445,7 +448,7 @@ async def get_daily_activity( start_date: Optional[str], end_date: Optional[str], model: Optional[str], - api_key: Optional[str], + api_key: Optional[Union[str, List[str]]], page: int, page_size: int, exclude_entity_ids: Optional[List[str]] = None, diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 78caa86db7b..d1549b51167 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -3601,7 +3601,7 @@ async def get_team_daily_activity( }, ) - ## Fetch team aliases + ## Fetch team aliases and check team admin status where_condition = {} if team_ids_list: where_condition["team_id"] = {"in": list(team_ids_list)} @@ -3612,6 +3612,36 @@ async def get_team_daily_activity( t.team_id: {"team_alias": t.team_alias} for t in team_aliases } + # Check if user is team admin for any requested teams + # If not, filter by user's API keys + user_api_keys: Optional[List[str]] = None + if not _user_has_admin_view(user_api_key_dict) and team_ids_list and team_aliases: + # Check if user is team admin for any of the teams + is_team_admin_for_any = False + for team_alias in team_aliases: + team_obj = LiteLLM_TeamTable(**team_alias.model_dump()) + if _is_user_team_admin( + user_api_key_dict=user_api_key_dict, team_obj=team_obj + ): + is_team_admin_for_any = True + break + + # If user is not a team admin for any team, filter by their API keys + if not is_team_admin_for_any: + # Get all API keys for this user + user_keys = await prisma_client.db.litellm_verificationtoken.find_many( + where={"user_id": user_api_key_dict.user_id} + ) + user_api_keys = [key.token for key in user_keys if key.token] + # If user has no API keys, return empty result + if not user_api_keys: + user_api_keys = [""] # Use empty string to ensure no matches + + # If api_key parameter is provided, use it; otherwise use user_api_keys if set + final_api_key_filter: Optional[Union[str, List[str]]] = api_key + if final_api_key_filter is None and user_api_keys is not None: + final_api_key_filter = user_api_keys + return await get_daily_activity( prisma_client=prisma_client, table_name="litellm_dailyteamspend", @@ -3622,7 +3652,7 @@ async def get_team_daily_activity( start_date=start_date, end_date=end_date, model=model, - api_key=api_key, + api_key=final_api_key_filter, page=page, page_size=page_size, ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index e296066b998..bbff7448e13 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -20,6 +20,7 @@ from litellm.proxy._types import ( LiteLLM_OrganizationTable, LiteLLM_OrganizationTableWithMembers, LiteLLM_TeamTable, + LiteLLM_UserTable, LitellmUserRoles, Member, ProxyErrorTypes, @@ -4476,6 +4477,187 @@ async def test_new_team_with_router_settings(mock_db_client, mock_admin_auth): assert deserialized_settings == router_settings_data +@pytest.mark.asyncio +async def test_get_team_daily_activity_non_admin_filters_by_user_api_keys( + mock_db_client, +): + """ + Test that non-team-admin users only see their own spend (filtered by their API keys) + when calling /team/daily/activity endpoint. + """ + from litellm.proxy.management_endpoints.team_endpoints import ( + get_team_daily_activity, + ) + + # Create a non-admin user + user_id = "test_user_123" + team_id = "test_team_456" + user_api_key_dict = UserAPIKeyAuth( + user_id=user_id, user_role=LitellmUserRoles.INTERNAL_USER + ) + + # Mock user info + mock_user_info = LiteLLM_UserTable( + user_id=user_id, + teams=[team_id], + max_budget=1000.0, + spend=0.0, + user_email="test@example.com", + user_role="internal_user", + ) + + # Mock team with user as non-admin member + mock_team_member = Member(user_id=user_id, role="user") + mock_team = MagicMock(spec=LiteLLM_TeamTable) + mock_team.team_id = team_id + mock_team.team_alias = "Test Team" + mock_team.members_with_roles = [mock_team_member] + mock_team.model_dump.return_value = { + "team_id": team_id, + "team_alias": "Test Team", + "members_with_roles": [{"user_id": user_id, "role": "user"}], + } + + # Mock user's API keys + user_api_key_1 = MagicMock() + user_api_key_1.token = "user_key_1" + user_api_key_2 = MagicMock() + user_api_key_2.token = "user_key_2" + + # Setup mocks + mock_db_client.db.litellm_teamtable.find_many = AsyncMock( + return_value=[mock_team] + ) + mock_db_client.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[user_api_key_1, user_api_key_2] + ) + + # Mock get_user_object + with patch( + "litellm.proxy.management_endpoints.team_endpoints.get_user_object", + new_callable=AsyncMock, + ) as mock_get_user_object: + mock_get_user_object.return_value = mock_user_info + + # Mock get_daily_activity to capture the api_key parameter + with patch( + "litellm.proxy.management_endpoints.team_endpoints.get_daily_activity", + new_callable=AsyncMock, + ) as mock_get_daily_activity: + mock_get_daily_activity.return_value = MagicMock() + + # Call the endpoint + await get_team_daily_activity( + team_ids=team_id, + start_date="2024-01-01", + end_date="2024-01-02", + model=None, + api_key=None, + page=1, + page_size=10, + exclude_team_ids=None, + user_api_key_dict=user_api_key_dict, + ) + + # Verify get_daily_activity was called with user's API keys as filter + mock_get_daily_activity.assert_called_once() + call_kwargs = mock_get_daily_activity.call_args[1] + assert call_kwargs["api_key"] == ["user_key_1", "user_key_2"] + assert call_kwargs["entity_id"] == [team_id] + + # Verify user's API keys were fetched + mock_db_client.db.litellm_verificationtoken.find_many.assert_called_once() + api_key_call_kwargs = ( + mock_db_client.db.litellm_verificationtoken.find_many.call_args[1] + ) + assert api_key_call_kwargs["where"] == {"user_id": user_id} + + +@pytest.mark.asyncio +async def test_get_team_daily_activity_team_admin_sees_all_spend(mock_db_client): + """ + Test that team admin users see all team spend (no API key filtering) + when calling /team/daily/activity endpoint. + """ + from litellm.proxy.management_endpoints.team_endpoints import ( + get_team_daily_activity, + ) + + # Create a team admin user + user_id = "test_admin_123" + team_id = "test_team_456" + user_api_key_dict = UserAPIKeyAuth( + user_id=user_id, user_role=LitellmUserRoles.INTERNAL_USER + ) + + # Mock user info + mock_user_info = LiteLLM_UserTable( + user_id=user_id, + teams=[team_id], + max_budget=1000.0, + spend=0.0, + user_email="admin@example.com", + user_role="internal_user", + ) + + # Mock team with user as admin member + mock_team_member = Member(user_id=user_id, role="admin") + mock_team = MagicMock(spec=LiteLLM_TeamTable) + mock_team.team_id = team_id + mock_team.team_alias = "Test Team" + mock_team.members_with_roles = [mock_team_member] + mock_team.model_dump.return_value = { + "team_id": team_id, + "team_alias": "Test Team", + "members_with_roles": [{"user_id": user_id, "role": "admin"}], + } + + # Setup mocks + mock_db_client.db.litellm_teamtable.find_many = AsyncMock( + return_value=[mock_team] + ) + + # Mock get_user_object + with patch( + "litellm.proxy.management_endpoints.team_endpoints.get_user_object", + new_callable=AsyncMock, + ) as mock_get_user_object: + mock_get_user_object.return_value = mock_user_info + + # Mock get_daily_activity to capture the api_key parameter + with patch( + "litellm.proxy.management_endpoints.team_endpoints.get_daily_activity", + new_callable=AsyncMock, + ) as mock_get_daily_activity: + mock_get_daily_activity.return_value = MagicMock() + + # Call the endpoint + await get_team_daily_activity( + team_ids=team_id, + start_date="2024-01-01", + end_date="2024-01-02", + model=None, + api_key=None, + page=1, + page_size=10, + exclude_team_ids=None, + user_api_key_dict=user_api_key_dict, + ) + + # Verify get_daily_activity was called WITHOUT API key filtering + mock_get_daily_activity.assert_called_once() + call_kwargs = mock_get_daily_activity.call_args[1] + assert call_kwargs["api_key"] is None + assert call_kwargs["entity_id"] == [team_id] + + # Verify user's API keys were NOT fetched (since they're admin) + if hasattr( + mock_db_client.db.litellm_verificationtoken, "find_many" + ) and mock_db_client.db.litellm_verificationtoken.find_many.called: + # If it was called, that's unexpected for admin users + assert False, "API keys should not be fetched for team admin users" + + @pytest.mark.asyncio async def test_update_team_with_router_settings(mock_db_client, mock_admin_auth): """ @@ -4552,3 +4734,184 @@ async def test_update_team_with_router_settings(mock_db_client, mock_admin_auth) # Verify router_settings can be deserialized and matches input deserialized_settings = json.loads(team_data["router_settings"]) assert deserialized_settings == router_settings_data + + +@pytest.mark.asyncio +async def test_get_team_daily_activity_non_admin_filters_by_user_api_keys( + mock_db_client, +): + """ + Test that non-team-admin users only see their own spend (filtered by their API keys) + when calling /team/daily/activity endpoint. + """ + from litellm.proxy.management_endpoints.team_endpoints import ( + get_team_daily_activity, + ) + + # Create a non-admin user + user_id = "test_user_123" + team_id = "test_team_456" + user_api_key_dict = UserAPIKeyAuth( + user_id=user_id, user_role=LitellmUserRoles.INTERNAL_USER + ) + + # Mock user info + mock_user_info = LiteLLM_UserTable( + user_id=user_id, + teams=[team_id], + max_budget=1000.0, + spend=0.0, + user_email="test@example.com", + user_role="internal_user", + ) + + # Mock team with user as non-admin member + mock_team_member = Member(user_id=user_id, role="user") + mock_team = MagicMock(spec=LiteLLM_TeamTable) + mock_team.team_id = team_id + mock_team.team_alias = "Test Team" + mock_team.members_with_roles = [mock_team_member] + mock_team.model_dump.return_value = { + "team_id": team_id, + "team_alias": "Test Team", + "members_with_roles": [{"user_id": user_id, "role": "user"}], + } + + # Mock user's API keys + user_api_key_1 = MagicMock() + user_api_key_1.token = "user_key_1" + user_api_key_2 = MagicMock() + user_api_key_2.token = "user_key_2" + + # Setup mocks + mock_db_client.db.litellm_teamtable.find_many = AsyncMock( + return_value=[mock_team] + ) + mock_db_client.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[user_api_key_1, user_api_key_2] + ) + + # Mock get_user_object + with patch( + "litellm.proxy.management_endpoints.team_endpoints.get_user_object", + new_callable=AsyncMock, + ) as mock_get_user_object: + mock_get_user_object.return_value = mock_user_info + + # Mock get_daily_activity to capture the api_key parameter + with patch( + "litellm.proxy.management_endpoints.team_endpoints.get_daily_activity", + new_callable=AsyncMock, + ) as mock_get_daily_activity: + mock_get_daily_activity.return_value = MagicMock() + + # Call the endpoint + await get_team_daily_activity( + team_ids=team_id, + start_date="2024-01-01", + end_date="2024-01-02", + model=None, + api_key=None, + page=1, + page_size=10, + exclude_team_ids=None, + user_api_key_dict=user_api_key_dict, + ) + + # Verify get_daily_activity was called with user's API keys as filter + mock_get_daily_activity.assert_called_once() + call_kwargs = mock_get_daily_activity.call_args[1] + assert call_kwargs["api_key"] == ["user_key_1", "user_key_2"] + assert call_kwargs["entity_id"] == [team_id] + + # Verify user's API keys were fetched + mock_db_client.db.litellm_verificationtoken.find_many.assert_called_once() + api_key_call_kwargs = ( + mock_db_client.db.litellm_verificationtoken.find_many.call_args[1] + ) + assert api_key_call_kwargs["where"] == {"user_id": user_id} + + +@pytest.mark.asyncio +async def test_get_team_daily_activity_team_admin_sees_all_spend(mock_db_client): + """ + Test that team admin users see all team spend (no API key filtering) + when calling /team/daily/activity endpoint. + """ + from litellm.proxy.management_endpoints.team_endpoints import ( + get_team_daily_activity, + ) + + # Create a team admin user + user_id = "test_admin_123" + team_id = "test_team_456" + user_api_key_dict = UserAPIKeyAuth( + user_id=user_id, user_role=LitellmUserRoles.INTERNAL_USER + ) + + # Mock user info + mock_user_info = LiteLLM_UserTable( + user_id=user_id, + teams=[team_id], + max_budget=1000.0, + spend=0.0, + user_email="admin@example.com", + user_role="internal_user", + ) + + # Mock team with user as admin member + mock_team_member = Member(user_id=user_id, role="admin") + mock_team = MagicMock(spec=LiteLLM_TeamTable) + mock_team.team_id = team_id + mock_team.team_alias = "Test Team" + mock_team.members_with_roles = [mock_team_member] + mock_team.model_dump.return_value = { + "team_id": team_id, + "team_alias": "Test Team", + "members_with_roles": [{"user_id": user_id, "role": "admin"}], + } + + # Setup mocks + mock_db_client.db.litellm_teamtable.find_many = AsyncMock( + return_value=[mock_team] + ) + + # Mock get_user_object + with patch( + "litellm.proxy.management_endpoints.team_endpoints.get_user_object", + new_callable=AsyncMock, + ) as mock_get_user_object: + mock_get_user_object.return_value = mock_user_info + + # Mock get_daily_activity to capture the api_key parameter + with patch( + "litellm.proxy.management_endpoints.team_endpoints.get_daily_activity", + new_callable=AsyncMock, + ) as mock_get_daily_activity: + mock_get_daily_activity.return_value = MagicMock() + + # Call the endpoint + await get_team_daily_activity( + team_ids=team_id, + start_date="2024-01-01", + end_date="2024-01-02", + model=None, + api_key=None, + page=1, + page_size=10, + exclude_team_ids=None, + user_api_key_dict=user_api_key_dict, + ) + + # Verify get_daily_activity was called WITHOUT API key filtering + mock_get_daily_activity.assert_called_once() + call_kwargs = mock_get_daily_activity.call_args[1] + assert call_kwargs["api_key"] is None + assert call_kwargs["entity_id"] == [team_id] + + # Verify user's API keys were NOT fetched (since they're admin) + if hasattr( + mock_db_client.db.litellm_verificationtoken, "find_many" + ) and mock_db_client.db.litellm_verificationtoken.find_many.called: + # If it was called, that's unexpected for admin users + assert False, "API keys should not be fetched for team admin users" From f34371375fd01a7e118df4c5680e8dc06c03b469 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 13 Jan 2026 20:37:20 -0800 Subject: [PATCH 3/6] Anthrpoic QOL --- .../conditional_public_model_name.test.tsx | 28 +++++++++++++++++++ .../conditional_public_model_name.tsx | 25 ++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.test.tsx diff --git a/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.test.tsx b/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.test.tsx new file mode 100644 index 00000000000..81633bd7a8a --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.test.tsx @@ -0,0 +1,28 @@ +import { render, screen } from "@testing-library/react"; +import { Form } from "antd"; +import { describe, expect, it } from "vitest"; +import ConditionalPublicModelName from "./conditional_public_model_name"; + +describe("ConditionalPublicModelName", () => { + it("should render", () => { + render( +
+ + , + ); + + expect(screen.getByText("Model Mappings")).toBeInTheDocument(); + expect(screen.getByText("Public Model Name")).toBeInTheDocument(); + expect(screen.getByText("LiteLLM Model Name")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.tsx b/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.tsx index b21a91dbe3c..0a77c252133 100644 --- a/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.tsx +++ b/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.tsx @@ -129,8 +129,31 @@ const ConditionalPublicModelName: React.FC = () => { { + const newValue = e.target.value; const newMappings = [...form.getFieldValue("model_mappings")]; - newMappings[index].public_name = e.target.value; + + // Check conditions for Anthropic -1m suffix handling + const isAnthropic = selectedProvider === Providers.Anthropic; + const endsWith1m = newValue.endsWith("-1m"); + const litellmParams = form.getFieldValue("litellm_extra_params"); + const isLitellmParamsEmpty = !litellmParams || litellmParams.trim() === ""; + + let finalPublicName = newValue; + + if (isAnthropic && endsWith1m && isLitellmParamsEmpty) { + // Set litellm params with extra_headers + const litellmParamsValue = JSON.stringify( + { extra_headers: { "anthropic-beta": "context-1m-2025-08-07" } }, + null, + 2, + ); + form.setFieldValue("litellm_extra_params", litellmParamsValue); + + // Remove -1m suffix from public_name + finalPublicName = newValue.slice(0, -3); // Remove "-1m" (3 characters) + } + + newMappings[index].public_name = finalPublicName; form.setFieldValue("model_mappings", newMappings); }} /> From 62eee47618cc6549a484fc4096ab79ffa1a07b64 Mon Sep 17 00:00:00 2001 From: Peter Dave Hello <3691490+PeterDaveHello@users.noreply.github.com> Date: Thu, 15 Jan 2026 03:49:40 +0800 Subject: [PATCH 4/6] Add support for OpenAI's gpt-5.2-codex (#19101) Reference: - https://openai.com/index/introducing-gpt-5-2-codex/ - https://platform.openai.com/docs/models/gpt-5.2-codex --- ...odel_prices_and_context_window_backup.json | 52 +++++++++++++++++++ model_prices_and_context_window.json | 52 +++++++++++++++++++ 2 files changed, 104 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 16d624cfa58..a130aefa5de 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18099,6 +18099,39 @@ "supports_tool_choice": true, "supports_vision": true }, + "gpt-5.2-codex": { + "cache_read_input_token_cost": 1.75e-07, + "cache_read_input_token_cost_priority": 3.5e-07, + "input_cost_per_token": 1.75e-06, + "input_cost_per_token_priority": 3.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 1.4e-05, + "output_cost_per_token_priority": 2.8e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, "gpt-5-mini": { "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, @@ -23266,6 +23299,25 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "openrouter/openai/gpt-5.2-codex": { + "cache_read_input_token_cost": 1.75e-07, + "input_cost_per_token": 1.75e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.4e-05, + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_reasoning": true, + "supports_tool_choice": true + }, "openrouter/openai/gpt-5": { "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 16d624cfa58..a130aefa5de 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18099,6 +18099,39 @@ "supports_tool_choice": true, "supports_vision": true }, + "gpt-5.2-codex": { + "cache_read_input_token_cost": 1.75e-07, + "cache_read_input_token_cost_priority": 3.5e-07, + "input_cost_per_token": 1.75e-06, + "input_cost_per_token_priority": 3.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 1.4e-05, + "output_cost_per_token_priority": 2.8e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, "gpt-5-mini": { "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, @@ -23266,6 +23299,25 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "openrouter/openai/gpt-5.2-codex": { + "cache_read_input_token_cost": 1.75e-07, + "input_cost_per_token": 1.75e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.4e-05, + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_reasoning": true, + "supports_tool_choice": true + }, "openrouter/openai/gpt-5": { "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, From 747829dadb82a1c55d664cc36f5c574edce4d307 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 14 Jan 2026 12:02:27 -0800 Subject: [PATCH 5/6] [Fix] Claude Code + Bedrock Converse Usage - ensure budget tokens are passed to converse api correctly (#19107) * test_bedrock_converse_budget_tokens_preserved * test_openai_model_with_thinking_converts_to_reasoning_effort * fix translate_anthropic_thinking_to_reasoning_effort * test_bedrock_converse_budget_tokens_preserved * test_anthropic_messages_bedrock_converse_with_thinking --- .../adapters/transformation.py | 83 +++++++++-- litellm/proxy/proxy_config.yaml | 10 +- .../test_bedrock_anthropic_messages_test.py | 32 ++++ ...erimental_pass_through_messages_handler.py | 138 +++++++++++++++++- 4 files changed, 244 insertions(+), 19 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 06092755b17..cb2110aee9a 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -17,7 +17,6 @@ from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingCho from litellm.litellm_core_utils.prompt_templates.common_utils import ( parse_tool_call_arguments, ) - from litellm.types.llms.anthropic import ( AllAnthropicToolsValues, AnthopicMessagesAssistantMessageParam, @@ -210,7 +209,7 @@ class LiteLLMAnthropicMessagesAdapter: # Convert Anthropic image format to OpenAI format source = content.get("source", {}) openai_image_url = ( - self._translate_anthropic_image_to_openai(source) + self._translate_anthropic_image_to_openai(cast(dict, source)) ) if openai_image_url: @@ -240,7 +239,7 @@ class LiteLLMAnthropicMessagesAdapter: # Combine all content items into a single tool message # to avoid creating multiple tool_result blocks with the same ID # (each tool_use must have exactly one tool_result) - content_items = content.get("content", []) + content_items = list(content.get("content", [])) # For single-item content, maintain backward compatibility with string/url format if len(content_items) == 1: @@ -266,7 +265,7 @@ class LiteLLMAnthropicMessagesAdapter: source = c.get("source", {}) openai_image_url = ( self._translate_anthropic_image_to_openai( - source + cast(dict, source) ) or "" ) @@ -306,7 +305,7 @@ class LiteLLMAnthropicMessagesAdapter: source = c.get("source", {}) openai_image_url = ( self._translate_anthropic_image_to_openai( - source + cast(dict, source) ) or "" ) @@ -363,7 +362,7 @@ class LiteLLMAnthropicMessagesAdapter: } signature = ( self._extract_signature_from_tool_use_content( - content + cast(Dict[str, Any], content) ) ) @@ -424,14 +423,21 @@ class LiteLLMAnthropicMessagesAdapter: return new_messages - def translate_anthropic_thinking_to_openai( - self, thinking: Dict[str, Any] + @staticmethod + def translate_anthropic_thinking_to_reasoning_effort( + thinking: Dict[str, Any] ) -> Optional[str]: """ Translate Anthropic's thinking parameter to OpenAI's reasoning_effort. Anthropic thinking format: {'type': 'enabled'|'disabled', 'budget_tokens': int} OpenAI reasoning_effort: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'default' + + Mapping: + - budget_tokens >= 10000 -> 'high' + - budget_tokens >= 5000 -> 'medium' + - budget_tokens >= 2000 -> 'low' + - budget_tokens < 2000 -> 'minimal' """ if not isinstance(thinking, dict): return None @@ -453,6 +459,53 @@ class LiteLLMAnthropicMessagesAdapter: return None + @staticmethod + def is_anthropic_claude_model(model: str) -> bool: + """ + Check if the model is an Anthropic Claude model that supports the thinking parameter. + + Returns True for: + - anthropic/* models + - bedrock/*anthropic* models (including converse) + - vertex_ai/*claude* models + """ + model_lower = model.lower() + return ( + "anthropic" in model_lower + or "claude" in model_lower + ) + + @staticmethod + def translate_thinking_for_model( + thinking: Dict[str, Any], + model: str, + ) -> Dict[str, Any]: + """ + Translate Anthropic thinking parameter based on the target model. + + For Claude/Anthropic models: returns {'thinking': } + - Preserves exact budget_tokens value + + For non-Claude models: returns {'reasoning_effort': } + - Converts thinking to reasoning_effort to avoid UnsupportedParamsError + + Args: + thinking: Anthropic thinking dict with 'type' and 'budget_tokens' + model: The target model name + + Returns: + Dict with either 'thinking' or 'reasoning_effort' key + """ + if LiteLLMAnthropicMessagesAdapter.is_anthropic_claude_model(model): + return {"thinking": thinking} + else: + reasoning_effort = LiteLLMAnthropicMessagesAdapter.translate_anthropic_thinking_to_reasoning_effort( + thinking + ) + if reasoning_effort: + return {"reasoning_effort": reasoning_effort} + return {} + def translate_anthropic_tool_choice_to_openai( self, tool_choice: AnthropicMessagesToolChoice ) -> ChatCompletionToolChoiceValues: @@ -566,11 +619,15 @@ class LiteLLMAnthropicMessagesAdapter: if "thinking" in anthropic_message_request: thinking = anthropic_message_request["thinking"] if thinking: - reasoning_effort = self.translate_anthropic_thinking_to_openai( - thinking=cast(Dict[str, Any], thinking) - ) - if reasoning_effort: - new_kwargs["reasoning_effort"] = reasoning_effort + model = new_kwargs.get("model", "") + if self.is_anthropic_claude_model(model): + new_kwargs["thinking"] = thinking # type: ignore + else: + reasoning_effort = self.translate_anthropic_thinking_to_reasoning_effort( + cast(Dict[str, Any], thinking) + ) + if reasoning_effort: + new_kwargs["reasoning_effort"] = reasoning_effort translatable_params = self.translatable_anthropic_params() for k, v in anthropic_message_request.items(): diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 576ba24aac2..54a923e3bbb 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,10 +1,10 @@ model_list: - - model_name: anthropic/* + - model_name: us.anthropic.claude-sonnet-4-20250514-v1:0 litellm_params: - model: anthropic/* - - model_name: openai/* - litellm_params: - model: openai/* + model: bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0 + model_info: + litellm_provider: bedrock_converse + mode: chat general_settings: store_prompts_in_spend_logs: true \ No newline at end of file diff --git a/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py b/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py index 41edc8572cd..155af6b6a95 100644 --- a/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py +++ b/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py @@ -66,3 +66,35 @@ async def test_anthropic_messages_litellm_router_bedrock(): INSTANCE_BASE_ANTHROPIC_MESSAGES_TEST._validate_response(response) +@pytest.mark.asyncio +async def test_anthropic_messages_bedrock_converse_with_thinking(): + """ + Test that bedrock/converse model works with thinking parameter. + Validates the request body from issue where budget_tokens was being lost. + """ + router = Router( + model_list=[ + { + "model_name": "bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", + "litellm_params": { + "model": "bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", + }, + }, + ] + ) + + messages = [{"role": "user", "content": "What is 2+2?"}] + + response = await router.aanthropic_messages( + messages=messages, + model="bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", + max_tokens=1026, + thinking={ + "type": "enabled", + "budget_tokens": 1025 + }, + ) + print("bedrock response: ", response) + + # Verify response + INSTANCE_BASE_ANTHROPIC_MESSAGES_TEST._validate_response(response) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 653f9e8e31e..66d62aae1ec 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -1,3 +1,4 @@ +import json import os import sys @@ -6,8 +7,10 @@ from fastapi.testclient import TestClient sys.path.insert(0, os.path.abspath("../../../../..")) -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch +from litellm.anthropic_interface import messages +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.utils import Delta, ModelResponse, StreamingChoices @@ -87,3 +90,136 @@ def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provide assert call_kwargs["custom_llm_provider"] == "my-custom-llm" assert call_kwargs["model"] == "my-custom-llm/my-custom-model" assert call_kwargs["api_key"] == "test-api-key" + + +@pytest.mark.asyncio +async def test_bedrock_converse_budget_tokens_preserved(): + """ + Test that budget_tokens value in thinking parameter is correctly passed to Bedrock Converse API + when using messages.acreate with bedrock/converse model. + + The bug was that the messages -> completion adapter was converting thinking to reasoning_effort + and losing the original budget_tokens value, causing it to use the default (128) instead. + """ + client = AsyncHTTPHandler() + + with patch.object(client, "post") as mock_post: + mock_response = AsyncMock() + mock_response.status_code = 200 + mock_response.headers = {} + mock_response.text = "mock response" + mock_response.json.return_value = { + "output": { + "message": { + "role": "assistant", + "content": [{"text": "4"}] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 10, + "outputTokens": 5, + "totalTokens": 15 + } + } + mock_post.return_value = mock_response + + try: + await messages.acreate( + client=client, + max_tokens=1024, + messages=[{"role": "user", "content": "What is 2+2?"}], + model="bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", + thinking={ + "budget_tokens": 1024, + "type": "enabled" + }, + ) + except Exception: + pass # Expected due to mock response format + + mock_post.assert_called_once() + + call_kwargs = mock_post.call_args.kwargs + json_data = call_kwargs.get("json") or json.loads(call_kwargs.get("data", "{}")) + print("Request json: ", json.dumps(json_data, indent=4, default=str)) + + additional_fields = json_data.get("additionalModelRequestFields", {}) + thinking_config = additional_fields.get("thinking", {}) + + assert "thinking" in additional_fields, "thinking parameter should be in additionalModelRequestFields" + assert thinking_config.get("type") == "enabled", "thinking.type should be 'enabled'" + assert thinking_config.get("budget_tokens") == 1024, f"thinking.budget_tokens should be 1024, but got {thinking_config.get('budget_tokens')}" + + +def test_openai_model_with_thinking_converts_to_reasoning_effort(): + """ + Test that when using a non-Anthropic model (like OpenAI gpt-5.2) with thinking parameter, + the thinking is converted to reasoning_effort and NOT passed as thinking. + + This ensures we don't regress on issue #16052 where non-Anthropic models would fail + with UnsupportedParamsError when thinking was passed directly. + """ + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + anthropic_messages_handler, + ) + + with patch("litellm.completion", return_value="test-response") as mock_completion: + try: + anthropic_messages_handler( + max_tokens=1024, + messages=[{"role": "user", "content": "What is 2+2?"}], + model="openai/gpt-5.2", + api_key="test-api-key", + thinking={ + "type": "enabled", + "budget_tokens": 1024 + }, + ) + except Exception as e: + print(f"Error: {e}") + + mock_completion.assert_called_once() + + call_kwargs = mock_completion.call_args.kwargs + + # Verify reasoning_effort is set (converted from thinking) + assert "reasoning_effort" in call_kwargs, "reasoning_effort should be passed to completion" + assert call_kwargs["reasoning_effort"] == "minimal", f"reasoning_effort should be 'minimal' for budget_tokens=1024, got {call_kwargs.get('reasoning_effort')}" + + # Verify thinking is NOT passed (non-Claude model) + assert "thinking" not in call_kwargs, "thinking should NOT be passed for non-Claude models" + + +class TestThinkingParameterTransformation: + """Core tests for thinking parameter transformation logic.""" + + def test_claude_model_preserves_thinking_with_budget_tokens(self): + """Test that Claude models get thinking parameter passed through with exact budget_tokens.""" + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) + + thinking = {"type": "enabled", "budget_tokens": 5000} + result = LiteLLMAnthropicMessagesAdapter.translate_thinking_for_model( + thinking=thinking, + model="bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", + ) + + assert result == {"thinking": thinking} + assert result["thinking"]["budget_tokens"] == 5000 + + def test_non_claude_model_converts_thinking_to_reasoning_effort(self): + """Test that non-Claude models convert thinking to reasoning_effort.""" + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) + + thinking = {"type": "enabled", "budget_tokens": 1024} + result = LiteLLMAnthropicMessagesAdapter.translate_thinking_for_model( + thinking=thinking, + model="openai/gpt-5.2", + ) + + assert result == {"reasoning_effort": "minimal"} + assert "thinking" not in result From b352d0d4fd76cc5ece5daadb6871a21cd2efe7b8 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Wed, 14 Jan 2026 12:40:27 -0800 Subject: [PATCH 6/6] [Perf] Remove premature model.dump call on the hot path (#19109) --- litellm/litellm_core_utils/litellm_logging.py | 184 +++++++++++++++--- 1 file changed, 160 insertions(+), 24 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 7d036615f59..619c5d1cf00 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4456,7 +4456,7 @@ class StandardLoggingPayloadSetup: @staticmethod def get_usage_from_response_obj( - response_obj: Optional[dict], combined_usage_object: Optional[Usage] = None + response_obj: Optional[Union[dict, BaseModel]], combined_usage_object: Optional[Usage] = None ) -> Usage: ## BASE CASE ## if combined_usage_object is not None: @@ -4468,27 +4468,32 @@ class StandardLoggingPayloadSetup: total_tokens=0, ) - usage = response_obj.get("usage", None) or {} - if usage is None or ( - not isinstance(usage, dict) and not isinstance(usage, Usage) - ): + usage = _safe_extract_usage_from_obj(response_obj) + + if usage is None: return Usage( prompt_tokens=0, completion_tokens=0, total_tokens=0, ) - elif isinstance(usage, Usage): + + if isinstance(usage, Usage): return usage - elif isinstance(usage, dict): - if ResponseAPILoggingUtils._is_response_api_usage(usage): - return ( - ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( - usage - ) - ) - return Usage(**usage) - - raise ValueError(f"usage is required, got={usage} of type {type(usage)}") + + transformed_usage = _try_transform_response_api_usage(usage) + if transformed_usage is not None: + return transformed_usage + + if isinstance(usage, dict): + created_usage = _try_create_usage_from_dict(usage) + if created_usage is not None: + return created_usage + + return Usage( + prompt_tokens=0, + completion_tokens=0, + total_tokens=0, + ) @staticmethod def get_model_cost_information( @@ -4529,13 +4534,18 @@ class StandardLoggingPayloadSetup: @staticmethod def get_final_response_obj( - response_obj: dict, init_response_obj: Union[Any, BaseModel, dict], kwargs: dict + response_obj: Union[dict, BaseModel], init_response_obj: Union[Any, BaseModel, dict], kwargs: dict ) -> Optional[Union[dict, str, list]]: """ Get final response object after redacting the message input/output from logging """ if response_obj: - final_response_obj: Optional[Union[dict, str, list]] = response_obj + if isinstance(response_obj, BaseModel): + final_response_obj: Optional[Union[dict, str, list]] = _safe_model_dump( + response_obj, default={} + ) + else: + final_response_obj = response_obj elif isinstance(init_response_obj, list) or isinstance(init_response_obj, str): final_response_obj = init_response_obj else: @@ -4549,7 +4559,7 @@ class StandardLoggingPayloadSetup: if modified_final_response_obj is not None and isinstance( modified_final_response_obj, BaseModel ): - final_response_obj = modified_final_response_obj.model_dump() + final_response_obj = _safe_model_dump(modified_final_response_obj, default={}) else: final_response_obj = modified_final_response_obj @@ -4820,6 +4830,125 @@ class StandardLoggingPayloadSetup: return request_tags +def _safe_model_dump( + obj: BaseModel, default: Optional[Union[dict, str, list]] = None +) -> Union[dict, str, list]: + """ + Safely call model_dump() on a BaseModel with fallback strategies. + + Args: + obj: BaseModel instance to dump + default: Default value to return if all strategies fail + + Returns: + Dict representation of the BaseModel, or fallback value + """ + if default is None: + default = {} + + try: + return obj.model_dump() + except (AttributeError, TypeError) as e: + verbose_logger.debug( + f"Error calling model_dump() on BaseModel: {e}, type: {type(obj)}" + ) + try: + if hasattr(obj, "__dict__"): + return obj.__dict__ + else: + return str(obj) + except Exception: + return default + + +def _safe_get_attribute( + obj: Union[dict, BaseModel, Any], attr_name: str, default: Any = None +) -> Any: + """ + Safely get an attribute from a dict or BaseModel object. + + Args: + obj: Object to get attribute from (dict, BaseModel, or any object) + attr_name: Name of the attribute to get + default: Default value to return if attribute doesn't exist + + Returns: + Attribute value or default + """ + try: + if isinstance(obj, dict): + return obj.get(attr_name, default) + else: + return getattr(obj, attr_name, default) + except (AttributeError, TypeError) as e: + verbose_logger.debug( + f"Error getting attribute '{attr_name}' from object: {e}, type: {type(obj)}" + ) + return default + + +def _safe_extract_usage_from_obj( + response_obj: Union[dict, BaseModel, Any] +) -> Optional[Union[dict, Usage, Any]]: + """ + Safely extract usage from response_obj (dict or BaseModel). + + Args: + response_obj: Response object (dict, BaseModel, or any object) + + Returns: + Usage object, dict, or None + """ + return _safe_get_attribute(response_obj, "usage", None) + + +def _try_transform_response_api_usage(usage: Any) -> Optional[Usage]: + """ + Try to transform ResponseAPIUsage to Usage object. + + Args: + usage: Usage object (dict, ResponseAPIUsage, or other) + + Returns: + Transformed Usage object, or None if transformation fails + """ + try: + if ResponseAPILoggingUtils._is_response_api_usage(usage): + return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage) + except (AttributeError, TypeError, KeyError) as e: + verbose_logger.debug( + f"Error checking/transforming ResponseAPIUsage: {e}, type: {type(usage)}" + ) + return None + + +def _try_create_usage_from_dict(usage: dict) -> Optional[Usage]: + """ + Try to create Usage object from dict. + + Args: + usage: Dict containing usage information + + Returns: + Usage object, or None if creation fails + """ + try: + return Usage(**usage) + except (TypeError, ValueError) as e: + # Avoid logging full dict contents, which may include sensitive data + try: + usage_keys = list(usage.keys()) + except Exception: + usage_keys = None + verbose_logger.debug( + "Error creating Usage from dict: %s, usage keys: %s, usage type: %s", + e, + usage_keys, + type(usage), + ) + return None + + def _get_status_fields( status: StandardLoggingPayloadStatus, guardrail_information: Optional[List[dict]], @@ -4869,17 +4998,21 @@ def _get_status_fields( def _extract_response_obj_and_hidden_params( init_response_obj: Union[Any, BaseModel, dict], original_exception: Optional[Exception], -) -> Tuple[dict, Optional[dict]]: +) -> Tuple[Union[dict, BaseModel], Optional[dict]]: + """Extract response_obj and hidden_params from init_response_obj.""" hidden_params: Optional[dict] = None if init_response_obj is None: - response_obj = {} + response_obj: Union[dict, BaseModel] = {} elif isinstance(init_response_obj, BaseModel): - response_obj = init_response_obj.model_dump() - hidden_params = getattr(init_response_obj, "_hidden_params", None) + response_obj = init_response_obj + hidden_params = _safe_get_attribute(init_response_obj, "_hidden_params", None) elif isinstance(init_response_obj, dict): response_obj = init_response_obj else: + verbose_logger.debug( + f"Unknown init_response_obj type: {type(init_response_obj)}, defaulting to empty dict" + ) response_obj = {} if original_exception is not None and hidden_params is None: @@ -4942,7 +5075,10 @@ def get_standard_logging_object_payload( ), ) - id = response_obj.get("id", kwargs.get("litellm_call_id")) + # Preserve falsy values (0, "", False) if they exist in response_obj + id = _safe_get_attribute(response_obj, "id", None) + if id is None: + id = kwargs.get("litellm_call_id") _model_id = metadata.get("model_info", {}).get("id", "") _model_group = metadata.get("model_group", "")