fix(proxy): honor team models and key aliases in usage Ask AI routing and test endpoint access wiring

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Devin AI 2026-10-17 22:11:11 +00:00
parent 51cf5960f6
commit 07c547bdbf
3 changed files with 241 additions and 19 deletions

View file

@ -479,6 +479,8 @@ async def _acompletion(
messages: list[Mapping[str, object]], # mutable-ok: callers pass the request message list unchanged
tools: list[_ToolDef] | None = None, # mutable-ok: forwarded verbatim to acompletion
stream: bool = False,
team_id: str | None = None,
model_aliases: Mapping[str, str] | None = None,
) -> ModelResponse | CustomStreamWrapper:
from litellm.proxy.proxy_server import llm_router
from litellm.types.llms.openai import AllMessageValues
@ -486,25 +488,38 @@ async def _acompletion(
completion_messages: Final = cast( # cast-ok: acompletion accepts the same Mapping messages it already receives
list[AllMessageValues], messages
)
if llm_router is not None and _router_knows_model(llm_router, model):
resolved_model: Final = model_aliases.get(model, model) if model_aliases is not None else model
team_model_name: Final = (
llm_router.map_team_model(resolved_model, team_id) if llm_router is not None and team_id is not None else None
)
router_model: Final = (
team_model_name
if team_model_name is not None
else resolved_model
if llm_router is not None and _router_knows_model(llm_router, resolved_model)
else None
)
if llm_router is not None and router_model is not None:
if stream:
return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs
model=model,
model=router_model,
messages=completion_messages,
tools=tools,
stream=True,
temperature=USAGE_AI_TEMPERATURE,
drop_params=True,
metadata={"user_api_key_team_id": team_id},
)
return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs
model=model,
model=router_model,
messages=completion_messages,
tools=tools,
temperature=USAGE_AI_TEMPERATURE,
drop_params=True,
metadata={"user_api_key_team_id": team_id},
)
return await litellm.acompletion(
model=model,
model=resolved_model,
messages=completion_messages,
tools=tools,
stream=stream,
@ -650,7 +665,12 @@ async def _process_tool_call(
chat_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_result})
async def _stream_final_response(model: str, chat_messages: list[Mapping[str, object]]) -> AsyncIterator[str]:
async def _stream_final_response(
model: str,
chat_messages: list[Mapping[str, object]],
team_id: str | None,
model_aliases: Mapping[str, str] | None,
) -> AsyncIterator[str]:
"""Stream the final LLM response after tool results are appended."""
yield _sse({"type": "status", "message": "Analyzing results..."})
@ -658,6 +678,8 @@ async def _stream_final_response(model: str, chat_messages: list[Mapping[str, ob
model=model,
messages=chat_messages,
stream=True,
team_id=team_id,
model_aliases=model_aliases,
)
async for chunk in response:
delta = chunk.choices[0].delta.content
@ -670,6 +692,8 @@ async def stream_usage_ai_chat(
model: str | None = None,
user_id: str | None = None,
is_admin: bool = False,
team_id: str | None = None,
model_aliases: Mapping[str, str] | None = None,
) -> AsyncGenerator[str, None]:
"""Stream SSE events: status → tool_call → chunk → done."""
resolved_model: Final = _resolve_model(model)
@ -686,6 +710,8 @@ async def stream_usage_ai_chat(
model=resolved_model,
messages=chat_messages,
tools=tools,
team_id=team_id,
model_aliases=model_aliases,
)
choice: Final = response.choices[0]
@ -699,7 +725,7 @@ async def stream_usage_ai_chat(
for tc in choice.message.tool_calls:
async for event in _process_tool_call(tc, chat_messages, user_id, is_admin):
yield event
async for event in _stream_final_response(resolved_model, chat_messages):
async for event in _stream_final_response(resolved_model, chat_messages, team_id, model_aliases):
yield event
yield _sse({"type": "done"})

View file

@ -46,6 +46,7 @@ async def usage_ai_chat(
AI chat about usage data. Streams SSE events with the AI response.
The AI agent has access to tools that query aggregated daily activity data.
"""
from litellm.proxy.auth.auth_checks import key_model_aliases_for_auth_check
from litellm.proxy.management_endpoints.common_utils import (
require_caller_user_id_for_non_admin,
user_api_key_has_admin_view,
@ -71,6 +72,8 @@ async def usage_ai_chat(
model=data.model,
user_id=user_id,
is_admin=is_admin,
team_id=user_api_key_dict.team_id,
model_aliases=key_model_aliases_for_auth_check(user_api_key_dict),
),
ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
ping_chunk=SSE_COMMENT_PING,

View file

@ -859,25 +859,30 @@ class TestUsageAiChatCompletionRouting:
class TestReasoningModelTemperature:
@pytest.mark.asyncio
async def test_reasoning_model_temperature_is_dropped(self):
import litellm
response = await litellm.acompletion(
model="gpt-5.6-terra",
messages=[{"role": "user", "content": "hi"}],
temperature=0.2,
drop_params=True,
mock_response="ok",
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
_acompletion,
)
assert response is not None
with pytest.raises(litellm.UnsupportedParamsError):
await litellm.acompletion(
expected_response: Final = MagicMock()
mock_acompletion = AsyncMock(return_value=expected_response)
with (
patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam
"litellm.proxy.proxy_server.llm_router", None
),
patch( # test-quality-ok: _acompletion calls the module-level litellm.acompletion directly; no injection seam
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion",
new=mock_acompletion,
),
):
result: Final = await _acompletion(
model="gpt-5.6-terra",
messages=[{"role": "user", "content": "hi"}],
temperature=0.2,
mock_response="ok",
)
assert result is expected_response
mock_acompletion.assert_awaited_once()
assert mock_acompletion.await_args.kwargs["drop_params"] is True
class TestUsageAiChatKeepalive:
async def _collect_endpoint_body(self, monkeypatch, interval, delay=0.3) -> tuple[list[bytes], dict]:
@ -1145,3 +1150,191 @@ class TestUsageAiChatModelAccess:
)
assert exc_info.value.code == "403"
class TestUsageAiChatTeamModelRouting:
@pytest.mark.asyncio
async def test_team_public_model_name_routes_through_router(self):
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
_acompletion,
)
from litellm.router import Router
router: Final = Router(
model_list=[
{
"model_name": "team-a-internal",
"litellm_params": {"model": "gpt-4o", "api_key": "sk-dummy"},
"model_info": {
"team_id": "team-a",
"team_public_model_name": "my-team-model",
},
}
]
)
expected_response: Final = MagicMock()
router.acompletion = AsyncMock(return_value=expected_response)
mock_litellm_acompletion = AsyncMock(return_value=MagicMock())
with (
patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam
"litellm.proxy.proxy_server.llm_router", router
),
patch( # test-quality-ok: _acompletion calls the module-level litellm.acompletion directly; no injection seam
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion",
new=mock_litellm_acompletion,
),
):
result: Final = await _acompletion(
model="my-team-model",
messages=[{"role": "user", "content": "hi"}],
team_id="team-a",
)
assert result is expected_response
router.acompletion.assert_awaited_once()
assert router.acompletion.await_args.kwargs["model"] == "my-team-model"
assert router.acompletion.await_args.kwargs["metadata"]["user_api_key_team_id"] == "team-a"
assert router.acompletion.await_args.kwargs["drop_params"] is True
mock_litellm_acompletion.assert_not_called()
@pytest.mark.asyncio
async def test_team_public_model_name_without_team_id_uses_litellm(self):
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
_acompletion,
)
from litellm.router import Router
router: Final = Router(
model_list=[
{
"model_name": "team-a-internal",
"litellm_params": {"model": "gpt-4o", "api_key": "sk-dummy"},
"model_info": {
"team_id": "team-a",
"team_public_model_name": "my-team-model",
},
}
]
)
expected_response: Final = MagicMock()
router.acompletion = AsyncMock(return_value=MagicMock())
mock_litellm_acompletion = AsyncMock(return_value=expected_response)
with (
patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam
"litellm.proxy.proxy_server.llm_router", router
),
patch( # test-quality-ok: _acompletion calls the module-level litellm.acompletion directly; no injection seam
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion",
new=mock_litellm_acompletion,
),
):
result: Final = await _acompletion(
model="my-team-model",
messages=[{"role": "user", "content": "hi"}],
)
assert result is expected_response
mock_litellm_acompletion.assert_awaited_once()
router.acompletion.assert_not_called()
@pytest.mark.asyncio
async def test_key_alias_resolves_to_router_model(self):
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
_acompletion,
)
from litellm.router import Router
router: Final = Router(
model_list=[
{
"model_name": "gpt-4o-mini",
"litellm_params": {"model": "gpt-4o-mini", "api_key": "sk-dummy"},
}
]
)
expected_response: Final = MagicMock()
router.acompletion = AsyncMock(return_value=expected_response)
mock_litellm_acompletion = AsyncMock(return_value=MagicMock())
with (
patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam
"litellm.proxy.proxy_server.llm_router", router
),
patch( # test-quality-ok: _acompletion calls the module-level litellm.acompletion directly; no injection seam
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion",
new=mock_litellm_acompletion,
),
):
result: Final = await _acompletion(
model="my-alias",
messages=[{"role": "user", "content": "hi"}],
model_aliases={"my-alias": "gpt-4o-mini"},
)
assert result is expected_response
router.acompletion.assert_awaited_once()
assert router.acompletion.await_args.kwargs["model"] == "gpt-4o-mini"
mock_litellm_acompletion.assert_not_called()
class TestUsageAiChatEndpointAccessCheck:
@pytest.mark.asyncio
async def test_endpoint_denies_key_without_model_access(self):
from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth
from litellm.proxy.management_endpoints.usage_endpoints.endpoints import (
ChatMessage,
UsageAIChatRequest,
usage_ai_chat,
)
from litellm.router import Router
router: Final = Router(
model_list=[
{
"model_name": "team-a-secret",
"litellm_params": {"model": "gpt-4o", "api_key": "sk-dummy"},
},
{
"model_name": "shared-model",
"litellm_params": {"model": "gpt-4o-mini", "api_key": "sk-dummy"},
},
]
)
mock_stream = MagicMock()
restricted_key: Final = UserAPIKeyAuth(
user_id="user-1",
user_role=LitellmUserRoles.INTERNAL_USER,
models=["shared-model"],
api_key="sk-test",
)
body: Final = UsageAIChatRequest(
messages=[ChatMessage(role="user", content="hi")],
model="team-a-secret",
)
with (
patch( # test-quality-ok: the check reads the proxy_server.llm_router module global; no injection seam
"litellm.proxy.proxy_server.llm_router", router
),
patch( # test-quality-ok: the check reads the proxy_server.llm_model_list module global; no injection seam
"litellm.proxy.proxy_server.llm_model_list", None
),
patch( # test-quality-ok: the check reads the proxy_server.prisma_client module global; no injection seam
"litellm.proxy.proxy_server.prisma_client", None
),
patch( # test-quality-ok: endpoint lazy-imports stream_usage_ai_chat from the module; no injection seam
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.stream_usage_ai_chat",
new=mock_stream,
),
pytest.raises(ProxyException) as exc_info,
):
await usage_ai_chat(
data=body,
request=MagicMock(),
user_api_key_dict=restricted_key,
)
assert exc_info.value.code == "403"
mock_stream.assert_not_called()