diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 3ad5caade1b..de433b4aa05 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1349,6 +1349,17 @@ def redact_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]: } +def sanitized_user_api_key_auth(user_api_key_dict: UserAPIKeyAuth) -> UserAPIKeyAuth: + return user_api_key_dict.model_copy( + update={ + "metadata": strip_callback_config(user_api_key_dict.metadata), + "team_metadata": strip_callback_config(user_api_key_dict.team_metadata), + "project_metadata": strip_callback_config(user_api_key_dict.project_metadata), + "organization_metadata": strip_callback_config(user_api_key_dict.organization_metadata), + } + ) + + class LiteLLMProxyRequestSetup: @staticmethod def _get_timeout_from_request(headers: dict) -> float | None: @@ -1824,14 +1835,7 @@ class LiteLLMProxyRequestSetup: user_api_key_dict.matched_model_access_groups ) # UserAPIKeyAuth object for MCP server access control - data[_metadata_variable_name]["user_api_key_auth"] = user_api_key_dict.model_copy( - update={ - "metadata": strip_callback_config(user_api_key_dict.metadata), - "team_metadata": strip_callback_config(user_api_key_dict.team_metadata), - "project_metadata": strip_callback_config(user_api_key_dict.project_metadata), - "organization_metadata": strip_callback_config(user_api_key_dict.organization_metadata), - } - ) + data[_metadata_variable_name]["user_api_key_auth"] = sanitized_user_api_key_auth(user_api_key_dict) return data @staticmethod diff --git a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py index 610aed3b6d4..af03b25a0b8 100644 --- a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py +++ b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py @@ -479,16 +479,29 @@ async def _acompletion( messages: list[Mapping[str, object]], # mutable-ok: callers pass the request message list unchanged tools: list[_ToolDef] | None = None, # mutable-ok: forwarded verbatim to acompletion stream: bool = False, - team_id: str | None = None, - model_aliases: Mapping[str, str] | None = None, + user_api_key_dict: "UserAPIKeyAuth | None" = None, ) -> ModelResponse | CustomStreamWrapper: + from litellm.proxy.auth.auth_checks import key_model_aliases_for_auth_check + from litellm.proxy.litellm_pre_call_utils import sanitized_user_api_key_auth from litellm.proxy.proxy_server import llm_router from litellm.types.llms.openai import AllMessageValues completion_messages: Final = cast( # cast-ok: acompletion accepts the same Mapping messages it already receives list[AllMessageValues], messages ) + team_id: Final = user_api_key_dict.team_id if user_api_key_dict is not None else None + model_aliases: Final = ( + key_model_aliases_for_auth_check(user_api_key_dict) if user_api_key_dict is not None else None + ) resolved_model: Final = model_aliases.get(model, model) if model_aliases is not None else model + router_metadata: Final = ( + { + "user_api_key_team_id": team_id, + "user_api_key_auth": sanitized_user_api_key_auth(user_api_key_dict), + } + if user_api_key_dict is not None + else {"user_api_key_team_id": team_id} + ) team_model_name: Final = ( llm_router.map_team_model(resolved_model, team_id) if llm_router is not None and team_id is not None else None ) @@ -508,7 +521,7 @@ async def _acompletion( stream=True, temperature=USAGE_AI_TEMPERATURE, drop_params=True, - metadata={"user_api_key_team_id": team_id}, + metadata=router_metadata, ) return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs model=router_model, @@ -516,7 +529,7 @@ async def _acompletion( tools=tools, temperature=USAGE_AI_TEMPERATURE, drop_params=True, - metadata={"user_api_key_team_id": team_id}, + metadata=router_metadata, ) return await litellm.acompletion( model=resolved_model, @@ -668,8 +681,7 @@ async def _process_tool_call( async def _stream_final_response( model: str, chat_messages: list[Mapping[str, object]], - team_id: str | None, - model_aliases: Mapping[str, str] | None, + user_api_key_dict: "UserAPIKeyAuth | None", ) -> AsyncIterator[str]: """Stream the final LLM response after tool results are appended.""" yield _sse({"type": "status", "message": "Analyzing results..."}) @@ -678,8 +690,7 @@ async def _stream_final_response( model=model, messages=chat_messages, stream=True, - team_id=team_id, - model_aliases=model_aliases, + user_api_key_dict=user_api_key_dict, ) async for chunk in response: delta = chunk.choices[0].delta.content @@ -692,8 +703,7 @@ async def stream_usage_ai_chat( model: str | None = None, user_id: str | None = None, is_admin: bool = False, - team_id: str | None = None, - model_aliases: Mapping[str, str] | None = None, + user_api_key_dict: "UserAPIKeyAuth | None" = None, ) -> AsyncGenerator[str, None]: """Stream SSE events: status → tool_call → chunk → done.""" resolved_model: Final = _resolve_model(model) @@ -710,8 +720,7 @@ async def stream_usage_ai_chat( model=resolved_model, messages=chat_messages, tools=tools, - team_id=team_id, - model_aliases=model_aliases, + user_api_key_dict=user_api_key_dict, ) choice: Final = response.choices[0] @@ -725,7 +734,7 @@ async def stream_usage_ai_chat( for tc in choice.message.tool_calls: async for event in _process_tool_call(tc, chat_messages, user_id, is_admin): yield event - async for event in _stream_final_response(resolved_model, chat_messages, team_id, model_aliases): + async for event in _stream_final_response(resolved_model, chat_messages, user_api_key_dict): yield event yield _sse({"type": "done"}) diff --git a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py b/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py index 8c67a7881ad..021ad3fb938 100644 --- a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py +++ b/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py @@ -46,7 +46,6 @@ async def usage_ai_chat( AI chat about usage data. Streams SSE events with the AI response. The AI agent has access to tools that query aggregated daily activity data. """ - from litellm.proxy.auth.auth_checks import key_model_aliases_for_auth_check from litellm.proxy.management_endpoints.common_utils import ( require_caller_user_id_for_non_admin, user_api_key_has_admin_view, @@ -72,8 +71,7 @@ async def usage_ai_chat( model=data.model, user_id=user_id, is_admin=is_admin, - team_id=user_api_key_dict.team_id, - model_aliases=key_model_aliases_for_auth_check(user_api_key_dict), + user_api_key_dict=user_api_key_dict, ), ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds, ping_chunk=SSE_COMMENT_PING, diff --git a/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py b/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py index 0675592d07b..bc032acb1a2 100644 --- a/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py +++ b/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py @@ -1155,6 +1155,7 @@ class TestUsageAiChatModelAccess: class TestUsageAiChatTeamModelRouting: @pytest.mark.asyncio async def test_team_public_model_name_routes_through_router(self): + from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( _acompletion, ) @@ -1188,7 +1189,7 @@ class TestUsageAiChatTeamModelRouting: result: Final = await _acompletion( model="my-team-model", messages=[{"role": "user", "content": "hi"}], - team_id="team-a", + user_api_key_dict=UserAPIKeyAuth(team_id="team-a"), ) assert result is expected_response @@ -1200,6 +1201,7 @@ class TestUsageAiChatTeamModelRouting: @pytest.mark.asyncio async def test_team_public_model_name_without_team_id_uses_litellm(self): + from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( _acompletion, ) @@ -1233,6 +1235,7 @@ class TestUsageAiChatTeamModelRouting: result: Final = await _acompletion( model="my-team-model", messages=[{"role": "user", "content": "hi"}], + user_api_key_dict=UserAPIKeyAuth(), ) assert result is expected_response @@ -1241,6 +1244,7 @@ class TestUsageAiChatTeamModelRouting: @pytest.mark.asyncio async def test_key_alias_resolves_to_router_model(self): + from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( _acompletion, ) @@ -1270,7 +1274,7 @@ class TestUsageAiChatTeamModelRouting: result: Final = await _acompletion( model="my-alias", messages=[{"role": "user", "content": "hi"}], - model_aliases={"my-alias": "gpt-4o-mini"}, + user_api_key_dict=UserAPIKeyAuth(aliases={"my-alias": "gpt-4o-mini"}), ) assert result is expected_response @@ -1278,6 +1282,54 @@ class TestUsageAiChatTeamModelRouting: assert router.acompletion.await_args.kwargs["model"] == "gpt-4o-mini" mock_litellm_acompletion.assert_not_called() + @pytest.mark.asyncio + async def test_router_calls_receive_sanitized_caller_auth_for_access_group_filtering(self): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( + _acompletion, + ) + from litellm.router import Router + + router: Final = Router( + model_list=[ + { + "model_name": "shared-model", + "litellm_params": {"model": "gpt-4o", "api_key": "sk-dummy"}, + "model_info": {"access_groups": ["AG-A"]}, + }, + { + "model_name": "shared-model", + "litellm_params": {"model": "gpt-4o-mini", "api_key": "sk-dummy"}, + "model_info": {"access_groups": ["AG-B"]}, + }, + ] + ) + expected_response: Final = MagicMock() + router.acompletion = AsyncMock(return_value=expected_response) + mock_litellm_acompletion = AsyncMock(return_value=MagicMock()) + + with ( + patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam + "litellm.proxy.proxy_server.llm_router", router + ), + patch( # test-quality-ok: _acompletion calls the module-level litellm.acompletion directly; no injection seam + "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion", + new=mock_litellm_acompletion, + ), + ): + result: Final = await _acompletion( + model="shared-model", + messages=[{"role": "user", "content": "hi"}], + user_api_key_dict=UserAPIKeyAuth(models=["AG-A"], api_key="sk-test"), + ) + + assert result is expected_response + router.acompletion.assert_awaited_once() + forwarded_auth: Final = router.acompletion.await_args.kwargs["metadata"]["user_api_key_auth"] + assert forwarded_auth.models == ["AG-A"] + assert router.acompletion.await_args.kwargs["metadata"]["user_api_key_team_id"] is None + mock_litellm_acompletion.assert_not_called() + class TestUsageAiChatEndpointAccessCheck: @pytest.mark.asyncio