mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-08 08:08:37 +00:00
fix(usage): route Ask AI through llm_router for proxy model names
The Ask AI dropdown is populated from the proxy's model_list, but both completion calls went to bare litellm.acompletion, which cannot resolve an alias to its litellm_params. Any deployment whose models carry a custom api_base, or are aliased at all, got LLM Provider NOT provided, swallowed by the broad except into a generic internal error message. Both call sites now go through a small helper that prefers llm_router.acompletion when the model is one the router knows, and falls back to bare litellm.acompletion for native provider strings and for deployments running without a router. Membership is tested with get_model_names(), which covers model_group_alias entries. get_model_group() takes a deployment id rather than a model name, so it would never have matched here.
This commit is contained in:
parent
9bca9dfbb1
commit
fda5d7ae7e
2 changed files with 113 additions and 2 deletions
|
|
@ -495,11 +495,26 @@ async def _process_tool_call(
|
|||
chat_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_result})
|
||||
|
||||
|
||||
async def _acompletion(model: str, **kwargs: Any) -> Any:
|
||||
"""Route through llm_router for proxy model names, bare litellm otherwise.
|
||||
|
||||
The Ask AI dropdown is populated from the proxy's model_list, so the model
|
||||
reaching here is usually an alias whose provider, api_base and api_key live
|
||||
in litellm_params. Bare acompletion cannot see those and fails with
|
||||
"LLM Provider NOT provided".
|
||||
"""
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
if llm_router is not None and model in llm_router.get_model_names():
|
||||
return await llm_router.acompletion(model=model, **kwargs)
|
||||
return await litellm.acompletion(model=model, **kwargs)
|
||||
|
||||
|
||||
async def _stream_final_response(model: str, chat_messages: list[dict[str, Any]]) -> AsyncIterator[str]:
|
||||
"""Stream the final LLM response after tool results are appended."""
|
||||
yield _sse({"type": "status", "message": "Analyzing results..."})
|
||||
|
||||
response: Final = await litellm.acompletion(
|
||||
response: Final = await _acompletion(
|
||||
model=model,
|
||||
messages=chat_messages,
|
||||
stream=True,
|
||||
|
|
@ -528,7 +543,7 @@ async def stream_usage_ai_chat(
|
|||
try:
|
||||
yield _sse({"type": "status", "message": "Thinking..."})
|
||||
tools: Final = get_tools_for_role(is_admin)
|
||||
response: Final = await litellm.acompletion(
|
||||
response: Final = await _acompletion(
|
||||
model=resolved_model,
|
||||
messages=chat_messages,
|
||||
tools=tools,
|
||||
|
|
|
|||
|
|
@ -466,3 +466,99 @@ class TestUsageAiChatServiceAccountGuard:
|
|||
is_admin=False,
|
||||
)
|
||||
assert "Endpoint-level guard missing" in str(exc_info.value)
|
||||
|
||||
|
||||
class TestProxyAliasRouting:
|
||||
"""The selected model comes from the proxy's model_list, so an alias like
|
||||
`kr/gpt-5.6-luna` only resolves through llm_router, never through bare
|
||||
litellm.acompletion."""
|
||||
|
||||
@staticmethod
|
||||
def _single_turn_response():
|
||||
response = MagicMock()
|
||||
response.choices = [MagicMock()]
|
||||
response.choices[0].message.tool_calls = None
|
||||
response.choices[0].message.content = "done"
|
||||
return response
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registered_alias_goes_through_the_router(self):
|
||||
mock_router = MagicMock()
|
||||
mock_router.get_model_names.return_value = ["kr/gpt-5.6-luna"]
|
||||
mock_router.acompletion = AsyncMock(return_value=self._single_turn_response())
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm"
|
||||
) as mock_litellm,
|
||||
patch("litellm.proxy.proxy_server.llm_router", mock_router),
|
||||
):
|
||||
mock_litellm.acompletion = AsyncMock(
|
||||
side_effect=AssertionError("alias must not reach bare litellm.acompletion")
|
||||
)
|
||||
|
||||
events = [
|
||||
json.loads(event.replace("data: ", "").strip())
|
||||
async for event in stream_usage_ai_chat(
|
||||
messages=[{"role": "user", "content": "spend?"}],
|
||||
model="kr/gpt-5.6-luna",
|
||||
user_id="user-123",
|
||||
is_admin=True,
|
||||
)
|
||||
]
|
||||
|
||||
assert mock_router.acompletion.await_count == 1
|
||||
assert mock_router.acompletion.await_args.kwargs["model"] == "kr/gpt-5.6-luna"
|
||||
assert [e["type"] for e in events] == ["status", "chunk", "done"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_provider_model_still_uses_bare_litellm(self):
|
||||
mock_router = MagicMock()
|
||||
mock_router.get_model_names.return_value = ["kr/gpt-5.6-luna"]
|
||||
mock_router.acompletion = AsyncMock(
|
||||
side_effect=AssertionError("unregistered model must not reach the router")
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm"
|
||||
) as mock_litellm,
|
||||
patch("litellm.proxy.proxy_server.llm_router", mock_router),
|
||||
):
|
||||
mock_litellm.acompletion = AsyncMock(return_value=self._single_turn_response())
|
||||
|
||||
events = [
|
||||
json.loads(event.replace("data: ", "").strip())
|
||||
async for event in stream_usage_ai_chat(
|
||||
messages=[{"role": "user", "content": "spend?"}],
|
||||
model="gpt-4o-mini",
|
||||
user_id="user-123",
|
||||
is_admin=True,
|
||||
)
|
||||
]
|
||||
|
||||
assert mock_litellm.acompletion.await_count == 1
|
||||
assert [e["type"] for e in events] == ["status", "chunk", "done"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_router_configured_falls_back_to_bare_litellm(self):
|
||||
with (
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm"
|
||||
) as mock_litellm,
|
||||
patch("litellm.proxy.proxy_server.llm_router", None),
|
||||
):
|
||||
mock_litellm.acompletion = AsyncMock(return_value=self._single_turn_response())
|
||||
|
||||
events = [
|
||||
json.loads(event.replace("data: ", "").strip())
|
||||
async for event in stream_usage_ai_chat(
|
||||
messages=[{"role": "user", "content": "spend?"}],
|
||||
model="kr/gpt-5.6-luna",
|
||||
user_id="user-123",
|
||||
is_admin=True,
|
||||
)
|
||||
]
|
||||
|
||||
assert mock_litellm.acompletion.await_count == 1
|
||||
assert [e["type"] for e in events] == ["status", "chunk", "done"]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue