mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): pass caller auth context to usage Ask AI router calls for access group filtering
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
07c547bdbf
commit
b466afdf61
4 changed files with 89 additions and 26 deletions
|
|
@ -1349,6 +1349,17 @@ def redact_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
|
|||
}
|
||||
|
||||
|
||||
def sanitized_user_api_key_auth(user_api_key_dict: UserAPIKeyAuth) -> UserAPIKeyAuth:
|
||||
return user_api_key_dict.model_copy(
|
||||
update={
|
||||
"metadata": strip_callback_config(user_api_key_dict.metadata),
|
||||
"team_metadata": strip_callback_config(user_api_key_dict.team_metadata),
|
||||
"project_metadata": strip_callback_config(user_api_key_dict.project_metadata),
|
||||
"organization_metadata": strip_callback_config(user_api_key_dict.organization_metadata),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class LiteLLMProxyRequestSetup:
|
||||
@staticmethod
|
||||
def _get_timeout_from_request(headers: dict) -> float | None:
|
||||
|
|
@ -1824,14 +1835,7 @@ class LiteLLMProxyRequestSetup:
|
|||
user_api_key_dict.matched_model_access_groups
|
||||
)
|
||||
# UserAPIKeyAuth object for MCP server access control
|
||||
data[_metadata_variable_name]["user_api_key_auth"] = user_api_key_dict.model_copy(
|
||||
update={
|
||||
"metadata": strip_callback_config(user_api_key_dict.metadata),
|
||||
"team_metadata": strip_callback_config(user_api_key_dict.team_metadata),
|
||||
"project_metadata": strip_callback_config(user_api_key_dict.project_metadata),
|
||||
"organization_metadata": strip_callback_config(user_api_key_dict.organization_metadata),
|
||||
}
|
||||
)
|
||||
data[_metadata_variable_name]["user_api_key_auth"] = sanitized_user_api_key_auth(user_api_key_dict)
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -479,16 +479,29 @@ async def _acompletion(
|
|||
messages: list[Mapping[str, object]], # mutable-ok: callers pass the request message list unchanged
|
||||
tools: list[_ToolDef] | None = None, # mutable-ok: forwarded verbatim to acompletion
|
||||
stream: bool = False,
|
||||
team_id: str | None = None,
|
||||
model_aliases: Mapping[str, str] | None = None,
|
||||
user_api_key_dict: "UserAPIKeyAuth | None" = None,
|
||||
) -> ModelResponse | CustomStreamWrapper:
|
||||
from litellm.proxy.auth.auth_checks import key_model_aliases_for_auth_check
|
||||
from litellm.proxy.litellm_pre_call_utils import sanitized_user_api_key_auth
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
completion_messages: Final = cast( # cast-ok: acompletion accepts the same Mapping messages it already receives
|
||||
list[AllMessageValues], messages
|
||||
)
|
||||
team_id: Final = user_api_key_dict.team_id if user_api_key_dict is not None else None
|
||||
model_aliases: Final = (
|
||||
key_model_aliases_for_auth_check(user_api_key_dict) if user_api_key_dict is not None else None
|
||||
)
|
||||
resolved_model: Final = model_aliases.get(model, model) if model_aliases is not None else model
|
||||
router_metadata: Final = (
|
||||
{
|
||||
"user_api_key_team_id": team_id,
|
||||
"user_api_key_auth": sanitized_user_api_key_auth(user_api_key_dict),
|
||||
}
|
||||
if user_api_key_dict is not None
|
||||
else {"user_api_key_team_id": team_id}
|
||||
)
|
||||
team_model_name: Final = (
|
||||
llm_router.map_team_model(resolved_model, team_id) if llm_router is not None and team_id is not None else None
|
||||
)
|
||||
|
|
@ -508,7 +521,7 @@ async def _acompletion(
|
|||
stream=True,
|
||||
temperature=USAGE_AI_TEMPERATURE,
|
||||
drop_params=True,
|
||||
metadata={"user_api_key_team_id": team_id},
|
||||
metadata=router_metadata,
|
||||
)
|
||||
return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs
|
||||
model=router_model,
|
||||
|
|
@ -516,7 +529,7 @@ async def _acompletion(
|
|||
tools=tools,
|
||||
temperature=USAGE_AI_TEMPERATURE,
|
||||
drop_params=True,
|
||||
metadata={"user_api_key_team_id": team_id},
|
||||
metadata=router_metadata,
|
||||
)
|
||||
return await litellm.acompletion(
|
||||
model=resolved_model,
|
||||
|
|
@ -668,8 +681,7 @@ async def _process_tool_call(
|
|||
async def _stream_final_response(
|
||||
model: str,
|
||||
chat_messages: list[Mapping[str, object]],
|
||||
team_id: str | None,
|
||||
model_aliases: Mapping[str, str] | None,
|
||||
user_api_key_dict: "UserAPIKeyAuth | None",
|
||||
) -> AsyncIterator[str]:
|
||||
"""Stream the final LLM response after tool results are appended."""
|
||||
yield _sse({"type": "status", "message": "Analyzing results..."})
|
||||
|
|
@ -678,8 +690,7 @@ async def _stream_final_response(
|
|||
model=model,
|
||||
messages=chat_messages,
|
||||
stream=True,
|
||||
team_id=team_id,
|
||||
model_aliases=model_aliases,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
async for chunk in response:
|
||||
delta = chunk.choices[0].delta.content
|
||||
|
|
@ -692,8 +703,7 @@ async def stream_usage_ai_chat(
|
|||
model: str | None = None,
|
||||
user_id: str | None = None,
|
||||
is_admin: bool = False,
|
||||
team_id: str | None = None,
|
||||
model_aliases: Mapping[str, str] | None = None,
|
||||
user_api_key_dict: "UserAPIKeyAuth | None" = None,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""Stream SSE events: status → tool_call → chunk → done."""
|
||||
resolved_model: Final = _resolve_model(model)
|
||||
|
|
@ -710,8 +720,7 @@ async def stream_usage_ai_chat(
|
|||
model=resolved_model,
|
||||
messages=chat_messages,
|
||||
tools=tools,
|
||||
team_id=team_id,
|
||||
model_aliases=model_aliases,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
choice: Final = response.choices[0]
|
||||
|
||||
|
|
@ -725,7 +734,7 @@ async def stream_usage_ai_chat(
|
|||
for tc in choice.message.tool_calls:
|
||||
async for event in _process_tool_call(tc, chat_messages, user_id, is_admin):
|
||||
yield event
|
||||
async for event in _stream_final_response(resolved_model, chat_messages, team_id, model_aliases):
|
||||
async for event in _stream_final_response(resolved_model, chat_messages, user_api_key_dict):
|
||||
yield event
|
||||
yield _sse({"type": "done"})
|
||||
|
||||
|
|
|
|||
|
|
@ -46,7 +46,6 @@ async def usage_ai_chat(
|
|||
AI chat about usage data. Streams SSE events with the AI response.
|
||||
The AI agent has access to tools that query aggregated daily activity data.
|
||||
"""
|
||||
from litellm.proxy.auth.auth_checks import key_model_aliases_for_auth_check
|
||||
from litellm.proxy.management_endpoints.common_utils import (
|
||||
require_caller_user_id_for_non_admin,
|
||||
user_api_key_has_admin_view,
|
||||
|
|
@ -72,8 +71,7 @@ async def usage_ai_chat(
|
|||
model=data.model,
|
||||
user_id=user_id,
|
||||
is_admin=is_admin,
|
||||
team_id=user_api_key_dict.team_id,
|
||||
model_aliases=key_model_aliases_for_auth_check(user_api_key_dict),
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
),
|
||||
ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
|
||||
ping_chunk=SSE_COMMENT_PING,
|
||||
|
|
|
|||
|
|
@ -1155,6 +1155,7 @@ class TestUsageAiChatModelAccess:
|
|||
class TestUsageAiChatTeamModelRouting:
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_public_model_name_routes_through_router(self):
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
_acompletion,
|
||||
)
|
||||
|
|
@ -1188,7 +1189,7 @@ class TestUsageAiChatTeamModelRouting:
|
|||
result: Final = await _acompletion(
|
||||
model="my-team-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
team_id="team-a",
|
||||
user_api_key_dict=UserAPIKeyAuth(team_id="team-a"),
|
||||
)
|
||||
|
||||
assert result is expected_response
|
||||
|
|
@ -1200,6 +1201,7 @@ class TestUsageAiChatTeamModelRouting:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_public_model_name_without_team_id_uses_litellm(self):
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
_acompletion,
|
||||
)
|
||||
|
|
@ -1233,6 +1235,7 @@ class TestUsageAiChatTeamModelRouting:
|
|||
result: Final = await _acompletion(
|
||||
model="my-team-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
)
|
||||
|
||||
assert result is expected_response
|
||||
|
|
@ -1241,6 +1244,7 @@ class TestUsageAiChatTeamModelRouting:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_alias_resolves_to_router_model(self):
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
_acompletion,
|
||||
)
|
||||
|
|
@ -1270,7 +1274,7 @@ class TestUsageAiChatTeamModelRouting:
|
|||
result: Final = await _acompletion(
|
||||
model="my-alias",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
model_aliases={"my-alias": "gpt-4o-mini"},
|
||||
user_api_key_dict=UserAPIKeyAuth(aliases={"my-alias": "gpt-4o-mini"}),
|
||||
)
|
||||
|
||||
assert result is expected_response
|
||||
|
|
@ -1278,6 +1282,54 @@ class TestUsageAiChatTeamModelRouting:
|
|||
assert router.acompletion.await_args.kwargs["model"] == "gpt-4o-mini"
|
||||
mock_litellm_acompletion.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_calls_receive_sanitized_caller_auth_for_access_group_filtering(self):
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
_acompletion,
|
||||
)
|
||||
from litellm.router import Router
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "shared-model",
|
||||
"litellm_params": {"model": "gpt-4o", "api_key": "sk-dummy"},
|
||||
"model_info": {"access_groups": ["AG-A"]},
|
||||
},
|
||||
{
|
||||
"model_name": "shared-model",
|
||||
"litellm_params": {"model": "gpt-4o-mini", "api_key": "sk-dummy"},
|
||||
"model_info": {"access_groups": ["AG-B"]},
|
||||
},
|
||||
]
|
||||
)
|
||||
expected_response: Final = MagicMock()
|
||||
router.acompletion = AsyncMock(return_value=expected_response)
|
||||
mock_litellm_acompletion = AsyncMock(return_value=MagicMock())
|
||||
|
||||
with (
|
||||
patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam
|
||||
"litellm.proxy.proxy_server.llm_router", router
|
||||
),
|
||||
patch( # test-quality-ok: _acompletion calls the module-level litellm.acompletion directly; no injection seam
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion",
|
||||
new=mock_litellm_acompletion,
|
||||
),
|
||||
):
|
||||
result: Final = await _acompletion(
|
||||
model="shared-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
user_api_key_dict=UserAPIKeyAuth(models=["AG-A"], api_key="sk-test"),
|
||||
)
|
||||
|
||||
assert result is expected_response
|
||||
router.acompletion.assert_awaited_once()
|
||||
forwarded_auth: Final = router.acompletion.await_args.kwargs["metadata"]["user_api_key_auth"]
|
||||
assert forwarded_auth.models == ["AG-A"]
|
||||
assert router.acompletion.await_args.kwargs["metadata"]["user_api_key_team_id"] is None
|
||||
mock_litellm_acompletion.assert_not_called()
|
||||
|
||||
|
||||
class TestUsageAiChatEndpointAccessCheck:
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue