mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): honor team models and key aliases in usage Ask AI routing and test endpoint access wiring
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
51cf5960f6
commit
07c547bdbf
3 changed files with 241 additions and 19 deletions
|
|
@ -479,6 +479,8 @@ async def _acompletion(
|
|||
messages: list[Mapping[str, object]], # mutable-ok: callers pass the request message list unchanged
|
||||
tools: list[_ToolDef] | None = None, # mutable-ok: forwarded verbatim to acompletion
|
||||
stream: bool = False,
|
||||
team_id: str | None = None,
|
||||
model_aliases: Mapping[str, str] | None = None,
|
||||
) -> ModelResponse | CustomStreamWrapper:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
|
@ -486,25 +488,38 @@ async def _acompletion(
|
|||
completion_messages: Final = cast( # cast-ok: acompletion accepts the same Mapping messages it already receives
|
||||
list[AllMessageValues], messages
|
||||
)
|
||||
if llm_router is not None and _router_knows_model(llm_router, model):
|
||||
resolved_model: Final = model_aliases.get(model, model) if model_aliases is not None else model
|
||||
team_model_name: Final = (
|
||||
llm_router.map_team_model(resolved_model, team_id) if llm_router is not None and team_id is not None else None
|
||||
)
|
||||
router_model: Final = (
|
||||
team_model_name
|
||||
if team_model_name is not None
|
||||
else resolved_model
|
||||
if llm_router is not None and _router_knows_model(llm_router, resolved_model)
|
||||
else None
|
||||
)
|
||||
if llm_router is not None and router_model is not None:
|
||||
if stream:
|
||||
return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs
|
||||
model=model,
|
||||
model=router_model,
|
||||
messages=completion_messages,
|
||||
tools=tools,
|
||||
stream=True,
|
||||
temperature=USAGE_AI_TEMPERATURE,
|
||||
drop_params=True,
|
||||
metadata={"user_api_key_team_id": team_id},
|
||||
)
|
||||
return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs
|
||||
model=model,
|
||||
model=router_model,
|
||||
messages=completion_messages,
|
||||
tools=tools,
|
||||
temperature=USAGE_AI_TEMPERATURE,
|
||||
drop_params=True,
|
||||
metadata={"user_api_key_team_id": team_id},
|
||||
)
|
||||
return await litellm.acompletion(
|
||||
model=model,
|
||||
model=resolved_model,
|
||||
messages=completion_messages,
|
||||
tools=tools,
|
||||
stream=stream,
|
||||
|
|
@ -650,7 +665,12 @@ async def _process_tool_call(
|
|||
chat_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_result})
|
||||
|
||||
|
||||
async def _stream_final_response(model: str, chat_messages: list[Mapping[str, object]]) -> AsyncIterator[str]:
|
||||
async def _stream_final_response(
|
||||
model: str,
|
||||
chat_messages: list[Mapping[str, object]],
|
||||
team_id: str | None,
|
||||
model_aliases: Mapping[str, str] | None,
|
||||
) -> AsyncIterator[str]:
|
||||
"""Stream the final LLM response after tool results are appended."""
|
||||
yield _sse({"type": "status", "message": "Analyzing results..."})
|
||||
|
||||
|
|
@ -658,6 +678,8 @@ async def _stream_final_response(model: str, chat_messages: list[Mapping[str, ob
|
|||
model=model,
|
||||
messages=chat_messages,
|
||||
stream=True,
|
||||
team_id=team_id,
|
||||
model_aliases=model_aliases,
|
||||
)
|
||||
async for chunk in response:
|
||||
delta = chunk.choices[0].delta.content
|
||||
|
|
@ -670,6 +692,8 @@ async def stream_usage_ai_chat(
|
|||
model: str | None = None,
|
||||
user_id: str | None = None,
|
||||
is_admin: bool = False,
|
||||
team_id: str | None = None,
|
||||
model_aliases: Mapping[str, str] | None = None,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""Stream SSE events: status → tool_call → chunk → done."""
|
||||
resolved_model: Final = _resolve_model(model)
|
||||
|
|
@ -686,6 +710,8 @@ async def stream_usage_ai_chat(
|
|||
model=resolved_model,
|
||||
messages=chat_messages,
|
||||
tools=tools,
|
||||
team_id=team_id,
|
||||
model_aliases=model_aliases,
|
||||
)
|
||||
choice: Final = response.choices[0]
|
||||
|
||||
|
|
@ -699,7 +725,7 @@ async def stream_usage_ai_chat(
|
|||
for tc in choice.message.tool_calls:
|
||||
async for event in _process_tool_call(tc, chat_messages, user_id, is_admin):
|
||||
yield event
|
||||
async for event in _stream_final_response(resolved_model, chat_messages):
|
||||
async for event in _stream_final_response(resolved_model, chat_messages, team_id, model_aliases):
|
||||
yield event
|
||||
yield _sse({"type": "done"})
|
||||
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ async def usage_ai_chat(
|
|||
AI chat about usage data. Streams SSE events with the AI response.
|
||||
The AI agent has access to tools that query aggregated daily activity data.
|
||||
"""
|
||||
from litellm.proxy.auth.auth_checks import key_model_aliases_for_auth_check
|
||||
from litellm.proxy.management_endpoints.common_utils import (
|
||||
require_caller_user_id_for_non_admin,
|
||||
user_api_key_has_admin_view,
|
||||
|
|
@ -71,6 +72,8 @@ async def usage_ai_chat(
|
|||
model=data.model,
|
||||
user_id=user_id,
|
||||
is_admin=is_admin,
|
||||
team_id=user_api_key_dict.team_id,
|
||||
model_aliases=key_model_aliases_for_auth_check(user_api_key_dict),
|
||||
),
|
||||
ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
|
||||
ping_chunk=SSE_COMMENT_PING,
|
||||
|
|
|
|||
|
|
@ -859,25 +859,30 @@ class TestUsageAiChatCompletionRouting:
|
|||
class TestReasoningModelTemperature:
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_model_temperature_is_dropped(self):
|
||||
import litellm
|
||||
|
||||
response = await litellm.acompletion(
|
||||
model="gpt-5.6-terra",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
temperature=0.2,
|
||||
drop_params=True,
|
||||
mock_response="ok",
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
_acompletion,
|
||||
)
|
||||
assert response is not None
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError):
|
||||
await litellm.acompletion(
|
||||
expected_response: Final = MagicMock()
|
||||
mock_acompletion = AsyncMock(return_value=expected_response)
|
||||
with (
|
||||
patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam
|
||||
"litellm.proxy.proxy_server.llm_router", None
|
||||
),
|
||||
patch( # test-quality-ok: _acompletion calls the module-level litellm.acompletion directly; no injection seam
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion",
|
||||
new=mock_acompletion,
|
||||
),
|
||||
):
|
||||
result: Final = await _acompletion(
|
||||
model="gpt-5.6-terra",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
temperature=0.2,
|
||||
mock_response="ok",
|
||||
)
|
||||
|
||||
assert result is expected_response
|
||||
mock_acompletion.assert_awaited_once()
|
||||
assert mock_acompletion.await_args.kwargs["drop_params"] is True
|
||||
|
||||
|
||||
class TestUsageAiChatKeepalive:
|
||||
async def _collect_endpoint_body(self, monkeypatch, interval, delay=0.3) -> tuple[list[bytes], dict]:
|
||||
|
|
@ -1145,3 +1150,191 @@ class TestUsageAiChatModelAccess:
|
|||
)
|
||||
|
||||
assert exc_info.value.code == "403"
|
||||
|
||||
|
||||
class TestUsageAiChatTeamModelRouting:
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_public_model_name_routes_through_router(self):
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
_acompletion,
|
||||
)
|
||||
from litellm.router import Router
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "team-a-internal",
|
||||
"litellm_params": {"model": "gpt-4o", "api_key": "sk-dummy"},
|
||||
"model_info": {
|
||||
"team_id": "team-a",
|
||||
"team_public_model_name": "my-team-model",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
expected_response: Final = MagicMock()
|
||||
router.acompletion = AsyncMock(return_value=expected_response)
|
||||
mock_litellm_acompletion = AsyncMock(return_value=MagicMock())
|
||||
|
||||
with (
|
||||
patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam
|
||||
"litellm.proxy.proxy_server.llm_router", router
|
||||
),
|
||||
patch( # test-quality-ok: _acompletion calls the module-level litellm.acompletion directly; no injection seam
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion",
|
||||
new=mock_litellm_acompletion,
|
||||
),
|
||||
):
|
||||
result: Final = await _acompletion(
|
||||
model="my-team-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
team_id="team-a",
|
||||
)
|
||||
|
||||
assert result is expected_response
|
||||
router.acompletion.assert_awaited_once()
|
||||
assert router.acompletion.await_args.kwargs["model"] == "my-team-model"
|
||||
assert router.acompletion.await_args.kwargs["metadata"]["user_api_key_team_id"] == "team-a"
|
||||
assert router.acompletion.await_args.kwargs["drop_params"] is True
|
||||
mock_litellm_acompletion.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_public_model_name_without_team_id_uses_litellm(self):
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
_acompletion,
|
||||
)
|
||||
from litellm.router import Router
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "team-a-internal",
|
||||
"litellm_params": {"model": "gpt-4o", "api_key": "sk-dummy"},
|
||||
"model_info": {
|
||||
"team_id": "team-a",
|
||||
"team_public_model_name": "my-team-model",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
expected_response: Final = MagicMock()
|
||||
router.acompletion = AsyncMock(return_value=MagicMock())
|
||||
mock_litellm_acompletion = AsyncMock(return_value=expected_response)
|
||||
|
||||
with (
|
||||
patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam
|
||||
"litellm.proxy.proxy_server.llm_router", router
|
||||
),
|
||||
patch( # test-quality-ok: _acompletion calls the module-level litellm.acompletion directly; no injection seam
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion",
|
||||
new=mock_litellm_acompletion,
|
||||
),
|
||||
):
|
||||
result: Final = await _acompletion(
|
||||
model="my-team-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
assert result is expected_response
|
||||
mock_litellm_acompletion.assert_awaited_once()
|
||||
router.acompletion.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_alias_resolves_to_router_model(self):
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
_acompletion,
|
||||
)
|
||||
from litellm.router import Router
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4o-mini",
|
||||
"litellm_params": {"model": "gpt-4o-mini", "api_key": "sk-dummy"},
|
||||
}
|
||||
]
|
||||
)
|
||||
expected_response: Final = MagicMock()
|
||||
router.acompletion = AsyncMock(return_value=expected_response)
|
||||
mock_litellm_acompletion = AsyncMock(return_value=MagicMock())
|
||||
|
||||
with (
|
||||
patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam
|
||||
"litellm.proxy.proxy_server.llm_router", router
|
||||
),
|
||||
patch( # test-quality-ok: _acompletion calls the module-level litellm.acompletion directly; no injection seam
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion",
|
||||
new=mock_litellm_acompletion,
|
||||
),
|
||||
):
|
||||
result: Final = await _acompletion(
|
||||
model="my-alias",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
model_aliases={"my-alias": "gpt-4o-mini"},
|
||||
)
|
||||
|
||||
assert result is expected_response
|
||||
router.acompletion.assert_awaited_once()
|
||||
assert router.acompletion.await_args.kwargs["model"] == "gpt-4o-mini"
|
||||
mock_litellm_acompletion.assert_not_called()
|
||||
|
||||
|
||||
class TestUsageAiChatEndpointAccessCheck:
|
||||
@pytest.mark.asyncio
|
||||
async def test_endpoint_denies_key_without_model_access(self):
|
||||
from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.endpoints import (
|
||||
ChatMessage,
|
||||
UsageAIChatRequest,
|
||||
usage_ai_chat,
|
||||
)
|
||||
from litellm.router import Router
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "team-a-secret",
|
||||
"litellm_params": {"model": "gpt-4o", "api_key": "sk-dummy"},
|
||||
},
|
||||
{
|
||||
"model_name": "shared-model",
|
||||
"litellm_params": {"model": "gpt-4o-mini", "api_key": "sk-dummy"},
|
||||
},
|
||||
]
|
||||
)
|
||||
mock_stream = MagicMock()
|
||||
restricted_key: Final = UserAPIKeyAuth(
|
||||
user_id="user-1",
|
||||
user_role=LitellmUserRoles.INTERNAL_USER,
|
||||
models=["shared-model"],
|
||||
api_key="sk-test",
|
||||
)
|
||||
body: Final = UsageAIChatRequest(
|
||||
messages=[ChatMessage(role="user", content="hi")],
|
||||
model="team-a-secret",
|
||||
)
|
||||
|
||||
with (
|
||||
patch( # test-quality-ok: the check reads the proxy_server.llm_router module global; no injection seam
|
||||
"litellm.proxy.proxy_server.llm_router", router
|
||||
),
|
||||
patch( # test-quality-ok: the check reads the proxy_server.llm_model_list module global; no injection seam
|
||||
"litellm.proxy.proxy_server.llm_model_list", None
|
||||
),
|
||||
patch( # test-quality-ok: the check reads the proxy_server.prisma_client module global; no injection seam
|
||||
"litellm.proxy.proxy_server.prisma_client", None
|
||||
),
|
||||
patch( # test-quality-ok: endpoint lazy-imports stream_usage_ai_chat from the module; no injection seam
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.stream_usage_ai_chat",
|
||||
new=mock_stream,
|
||||
),
|
||||
pytest.raises(ProxyException) as exc_info,
|
||||
):
|
||||
await usage_ai_chat(
|
||||
data=body,
|
||||
request=MagicMock(),
|
||||
user_api_key_dict=restricted_key,
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "403"
|
||||
mock_stream.assert_not_called()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue