diff --git a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py index 3bbff623085..610aed3b6d4 100644 --- a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py +++ b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py @@ -479,6 +479,8 @@ async def _acompletion( messages: list[Mapping[str, object]], # mutable-ok: callers pass the request message list unchanged tools: list[_ToolDef] | None = None, # mutable-ok: forwarded verbatim to acompletion stream: bool = False, + team_id: str | None = None, + model_aliases: Mapping[str, str] | None = None, ) -> ModelResponse | CustomStreamWrapper: from litellm.proxy.proxy_server import llm_router from litellm.types.llms.openai import AllMessageValues @@ -486,25 +488,38 @@ async def _acompletion( completion_messages: Final = cast( # cast-ok: acompletion accepts the same Mapping messages it already receives list[AllMessageValues], messages ) - if llm_router is not None and _router_knows_model(llm_router, model): + resolved_model: Final = model_aliases.get(model, model) if model_aliases is not None else model + team_model_name: Final = ( + llm_router.map_team_model(resolved_model, team_id) if llm_router is not None and team_id is not None else None + ) + router_model: Final = ( + team_model_name + if team_model_name is not None + else resolved_model + if llm_router is not None and _router_knows_model(llm_router, resolved_model) + else None + ) + if llm_router is not None and router_model is not None: if stream: return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs - model=model, + model=router_model, messages=completion_messages, tools=tools, stream=True, temperature=USAGE_AI_TEMPERATURE, drop_params=True, + metadata={"user_api_key_team_id": team_id}, ) return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs - model=model, + model=router_model, messages=completion_messages, tools=tools, temperature=USAGE_AI_TEMPERATURE, drop_params=True, + metadata={"user_api_key_team_id": team_id}, ) return await litellm.acompletion( - model=model, + model=resolved_model, messages=completion_messages, tools=tools, stream=stream, @@ -650,7 +665,12 @@ async def _process_tool_call( chat_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_result}) -async def _stream_final_response(model: str, chat_messages: list[Mapping[str, object]]) -> AsyncIterator[str]: +async def _stream_final_response( + model: str, + chat_messages: list[Mapping[str, object]], + team_id: str | None, + model_aliases: Mapping[str, str] | None, +) -> AsyncIterator[str]: """Stream the final LLM response after tool results are appended.""" yield _sse({"type": "status", "message": "Analyzing results..."}) @@ -658,6 +678,8 @@ async def _stream_final_response(model: str, chat_messages: list[Mapping[str, ob model=model, messages=chat_messages, stream=True, + team_id=team_id, + model_aliases=model_aliases, ) async for chunk in response: delta = chunk.choices[0].delta.content @@ -670,6 +692,8 @@ async def stream_usage_ai_chat( model: str | None = None, user_id: str | None = None, is_admin: bool = False, + team_id: str | None = None, + model_aliases: Mapping[str, str] | None = None, ) -> AsyncGenerator[str, None]: """Stream SSE events: status → tool_call → chunk → done.""" resolved_model: Final = _resolve_model(model) @@ -686,6 +710,8 @@ async def stream_usage_ai_chat( model=resolved_model, messages=chat_messages, tools=tools, + team_id=team_id, + model_aliases=model_aliases, ) choice: Final = response.choices[0] @@ -699,7 +725,7 @@ async def stream_usage_ai_chat( for tc in choice.message.tool_calls: async for event in _process_tool_call(tc, chat_messages, user_id, is_admin): yield event - async for event in _stream_final_response(resolved_model, chat_messages): + async for event in _stream_final_response(resolved_model, chat_messages, team_id, model_aliases): yield event yield _sse({"type": "done"}) diff --git a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py b/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py index 61f330cc494..8c67a7881ad 100644 --- a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py +++ b/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py @@ -46,6 +46,7 @@ async def usage_ai_chat( AI chat about usage data. Streams SSE events with the AI response. The AI agent has access to tools that query aggregated daily activity data. """ + from litellm.proxy.auth.auth_checks import key_model_aliases_for_auth_check from litellm.proxy.management_endpoints.common_utils import ( require_caller_user_id_for_non_admin, user_api_key_has_admin_view, @@ -71,6 +72,8 @@ async def usage_ai_chat( model=data.model, user_id=user_id, is_admin=is_admin, + team_id=user_api_key_dict.team_id, + model_aliases=key_model_aliases_for_auth_check(user_api_key_dict), ), ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds, ping_chunk=SSE_COMMENT_PING, diff --git a/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py b/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py index d77361da4f2..0675592d07b 100644 --- a/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py +++ b/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py @@ -859,25 +859,30 @@ class TestUsageAiChatCompletionRouting: class TestReasoningModelTemperature: @pytest.mark.asyncio async def test_reasoning_model_temperature_is_dropped(self): - import litellm - - response = await litellm.acompletion( - model="gpt-5.6-terra", - messages=[{"role": "user", "content": "hi"}], - temperature=0.2, - drop_params=True, - mock_response="ok", + from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( + _acompletion, ) - assert response is not None - with pytest.raises(litellm.UnsupportedParamsError): - await litellm.acompletion( + expected_response: Final = MagicMock() + mock_acompletion = AsyncMock(return_value=expected_response) + with ( + patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam + "litellm.proxy.proxy_server.llm_router", None + ), + patch( # test-quality-ok: _acompletion calls the module-level litellm.acompletion directly; no injection seam + "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion", + new=mock_acompletion, + ), + ): + result: Final = await _acompletion( model="gpt-5.6-terra", messages=[{"role": "user", "content": "hi"}], - temperature=0.2, - mock_response="ok", ) + assert result is expected_response + mock_acompletion.assert_awaited_once() + assert mock_acompletion.await_args.kwargs["drop_params"] is True + class TestUsageAiChatKeepalive: async def _collect_endpoint_body(self, monkeypatch, interval, delay=0.3) -> tuple[list[bytes], dict]: @@ -1145,3 +1150,191 @@ class TestUsageAiChatModelAccess: ) assert exc_info.value.code == "403" + + +class TestUsageAiChatTeamModelRouting: + @pytest.mark.asyncio + async def test_team_public_model_name_routes_through_router(self): + from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( + _acompletion, + ) + from litellm.router import Router + + router: Final = Router( + model_list=[ + { + "model_name": "team-a-internal", + "litellm_params": {"model": "gpt-4o", "api_key": "sk-dummy"}, + "model_info": { + "team_id": "team-a", + "team_public_model_name": "my-team-model", + }, + } + ] + ) + expected_response: Final = MagicMock() + router.acompletion = AsyncMock(return_value=expected_response) + mock_litellm_acompletion = AsyncMock(return_value=MagicMock()) + + with ( + patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam + "litellm.proxy.proxy_server.llm_router", router + ), + patch( # test-quality-ok: _acompletion calls the module-level litellm.acompletion directly; no injection seam + "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion", + new=mock_litellm_acompletion, + ), + ): + result: Final = await _acompletion( + model="my-team-model", + messages=[{"role": "user", "content": "hi"}], + team_id="team-a", + ) + + assert result is expected_response + router.acompletion.assert_awaited_once() + assert router.acompletion.await_args.kwargs["model"] == "my-team-model" + assert router.acompletion.await_args.kwargs["metadata"]["user_api_key_team_id"] == "team-a" + assert router.acompletion.await_args.kwargs["drop_params"] is True + mock_litellm_acompletion.assert_not_called() + + @pytest.mark.asyncio + async def test_team_public_model_name_without_team_id_uses_litellm(self): + from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( + _acompletion, + ) + from litellm.router import Router + + router: Final = Router( + model_list=[ + { + "model_name": "team-a-internal", + "litellm_params": {"model": "gpt-4o", "api_key": "sk-dummy"}, + "model_info": { + "team_id": "team-a", + "team_public_model_name": "my-team-model", + }, + } + ] + ) + expected_response: Final = MagicMock() + router.acompletion = AsyncMock(return_value=MagicMock()) + mock_litellm_acompletion = AsyncMock(return_value=expected_response) + + with ( + patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam + "litellm.proxy.proxy_server.llm_router", router + ), + patch( # test-quality-ok: _acompletion calls the module-level litellm.acompletion directly; no injection seam + "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion", + new=mock_litellm_acompletion, + ), + ): + result: Final = await _acompletion( + model="my-team-model", + messages=[{"role": "user", "content": "hi"}], + ) + + assert result is expected_response + mock_litellm_acompletion.assert_awaited_once() + router.acompletion.assert_not_called() + + @pytest.mark.asyncio + async def test_key_alias_resolves_to_router_model(self): + from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( + _acompletion, + ) + from litellm.router import Router + + router: Final = Router( + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "gpt-4o-mini", "api_key": "sk-dummy"}, + } + ] + ) + expected_response: Final = MagicMock() + router.acompletion = AsyncMock(return_value=expected_response) + mock_litellm_acompletion = AsyncMock(return_value=MagicMock()) + + with ( + patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam + "litellm.proxy.proxy_server.llm_router", router + ), + patch( # test-quality-ok: _acompletion calls the module-level litellm.acompletion directly; no injection seam + "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion", + new=mock_litellm_acompletion, + ), + ): + result: Final = await _acompletion( + model="my-alias", + messages=[{"role": "user", "content": "hi"}], + model_aliases={"my-alias": "gpt-4o-mini"}, + ) + + assert result is expected_response + router.acompletion.assert_awaited_once() + assert router.acompletion.await_args.kwargs["model"] == "gpt-4o-mini" + mock_litellm_acompletion.assert_not_called() + + +class TestUsageAiChatEndpointAccessCheck: + @pytest.mark.asyncio + async def test_endpoint_denies_key_without_model_access(self): + from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth + from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( + ChatMessage, + UsageAIChatRequest, + usage_ai_chat, + ) + from litellm.router import Router + + router: Final = Router( + model_list=[ + { + "model_name": "team-a-secret", + "litellm_params": {"model": "gpt-4o", "api_key": "sk-dummy"}, + }, + { + "model_name": "shared-model", + "litellm_params": {"model": "gpt-4o-mini", "api_key": "sk-dummy"}, + }, + ] + ) + mock_stream = MagicMock() + restricted_key: Final = UserAPIKeyAuth( + user_id="user-1", + user_role=LitellmUserRoles.INTERNAL_USER, + models=["shared-model"], + api_key="sk-test", + ) + body: Final = UsageAIChatRequest( + messages=[ChatMessage(role="user", content="hi")], + model="team-a-secret", + ) + + with ( + patch( # test-quality-ok: the check reads the proxy_server.llm_router module global; no injection seam + "litellm.proxy.proxy_server.llm_router", router + ), + patch( # test-quality-ok: the check reads the proxy_server.llm_model_list module global; no injection seam + "litellm.proxy.proxy_server.llm_model_list", None + ), + patch( # test-quality-ok: the check reads the proxy_server.prisma_client module global; no injection seam + "litellm.proxy.proxy_server.prisma_client", None + ), + patch( # test-quality-ok: endpoint lazy-imports stream_usage_ai_chat from the module; no injection seam + "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.stream_usage_ai_chat", + new=mock_stream, + ), + pytest.raises(ProxyException) as exc_info, + ): + await usage_ai_chat( + data=body, + request=MagicMock(), + user_api_key_dict=restricted_key, + ) + + assert exc_info.value.code == "403" + mock_stream.assert_not_called()