mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): mirror chat completions routing rule in usage ai chat model gate
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
6a916c2f57
commit
2bd80e89ba
2 changed files with 45 additions and 5 deletions
|
|
@ -447,7 +447,11 @@ def _sse(event: SSEEvent) -> str:
|
|||
|
||||
|
||||
def _router_knows_model(router: "Router", model: str) -> bool:
|
||||
return model in router.get_model_names() or router.pattern_router.route(request=model) is not None # pyright: ignore[reportUnknownMemberType] # PatternMatchDeployments.route returns untyped dicts
|
||||
return (
|
||||
model in router.get_model_names()
|
||||
or router.default_deployment is not None
|
||||
or len(router.pattern_router.patterns) > 0 # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # PatternMatchRouter.patterns values are bare `list`
|
||||
)
|
||||
|
||||
|
||||
async def _acompletion(
|
||||
|
|
|
|||
|
|
@ -544,7 +544,8 @@ class TestUsageAiChatCompletionRouting:
|
|||
async def test_stream_usage_ai_chat_falls_back_to_litellm_for_unknown_model(self):
|
||||
mock_router = MagicMock()
|
||||
mock_router.get_model_names.return_value = ["other"]
|
||||
mock_router.pattern_router.route.return_value = None
|
||||
mock_router.default_deployment = None
|
||||
mock_router.pattern_router.patterns = {}
|
||||
mock_router.acompletion = AsyncMock(return_value=self._plain_content_response())
|
||||
mock_acompletion = AsyncMock(return_value=self._plain_content_response())
|
||||
|
||||
|
|
@ -574,10 +575,46 @@ class TestUsageAiChatCompletionRouting:
|
|||
assert "error" not in event_types
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_usage_ai_chat_uses_router_for_wildcard_matched_model(self):
|
||||
async def test_stream_usage_ai_chat_uses_router_when_wildcard_pattern_exists(self):
|
||||
mock_router = MagicMock()
|
||||
mock_router.get_model_names.return_value = []
|
||||
mock_router.pattern_router.route.return_value = [{"model_name": "openai/*"}]
|
||||
mock_router.default_deployment = None
|
||||
mock_router.pattern_router.patterns = {"openai/.*": [{"model_name": "openai/*"}]}
|
||||
mock_router.acompletion = AsyncMock(return_value=self._plain_content_response())
|
||||
mock_acompletion = AsyncMock(return_value=self._plain_content_response())
|
||||
|
||||
with (
|
||||
patch( # test-quality-ok: ai_usage_chat reads the proxy_server.llm_router module global; no injection seam
|
||||
"litellm.proxy.proxy_server.llm_router", mock_router
|
||||
),
|
||||
patch( # test-quality-ok: the stream calls the module-level litellm.acompletion directly; no injection seam
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion",
|
||||
new=mock_acompletion,
|
||||
),
|
||||
):
|
||||
events = []
|
||||
async for event in stream_usage_ai_chat(
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
model="gpt-5.6-terra",
|
||||
is_admin=True,
|
||||
):
|
||||
events.append(json.loads(event.replace("data: ", "").strip()))
|
||||
|
||||
mock_router.acompletion.assert_awaited_once()
|
||||
assert mock_router.acompletion.await_args.kwargs["model"] == "gpt-5.6-terra"
|
||||
assert mock_router.acompletion.await_args.kwargs["drop_params"] is True
|
||||
mock_acompletion.assert_not_called()
|
||||
|
||||
event_types = [e["type"] for e in events]
|
||||
assert "chunk" in event_types
|
||||
assert "error" not in event_types
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_usage_ai_chat_uses_router_when_default_deployment_exists(self):
|
||||
mock_router = MagicMock()
|
||||
mock_router.get_model_names.return_value = []
|
||||
mock_router.default_deployment = {"model_name": "default", "litellm_params": {}}
|
||||
mock_router.pattern_router.patterns = {}
|
||||
mock_router.acompletion = AsyncMock(return_value=self._plain_content_response())
|
||||
mock_acompletion = AsyncMock(return_value=self._plain_content_response())
|
||||
|
||||
|
|
@ -598,7 +635,6 @@ class TestUsageAiChatCompletionRouting:
|
|||
):
|
||||
events.append(json.loads(event.replace("data: ", "").strip()))
|
||||
|
||||
mock_router.pattern_router.route.assert_called_once_with(request="gpt-5.6-terra")
|
||||
mock_router.acompletion.assert_awaited_once()
|
||||
assert mock_router.acompletion.await_args.kwargs["model"] == "gpt-5.6-terra"
|
||||
assert mock_router.acompletion.await_args.kwargs["drop_params"] is True
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue