mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-05 08:07:05 +00:00
fix(router): resolve session.model to the picked deployment in _ageneric_api_call_with_fallbacks_helper
Router._ageneric_api_call_with_fallbacks_helper (used by acreate_realtime_client_secret and friends) resolves model_name to the deployment it picked for the model_group, but left kwargs["session"]["model"] untouched — still the caller's original alias. acreate_realtime_client_secret prioritizes session.model over the model kwarg, so requests routed through a Router model group silently used the alias instead of the resolved deployment, breaking provider inference for anyone whose model group name isn't itself a valid provider-prefixed model string. Fix is scoped to this one helper: rewrite session.model to model_name right before calling the underlying function, whenever a session dict with a model key is present. Leaves acreate_realtime_client_secret's own session-first priority (and its existing tests) untouched for direct callers that don't go through a Router model group. Fixes #36742
This commit is contained in:
parent
0e9cd9893e
commit
ed682aba7b
2 changed files with 46 additions and 0 deletions
|
|
@ -4594,6 +4594,16 @@ class Router:
|
|||
if custom_llm_provider is not None:
|
||||
response_kwargs["custom_llm_provider"] = custom_llm_provider
|
||||
|
||||
# kwargs["session"] (e.g. realtime client_secrets) still has whatever
|
||||
# model string the caller/proxy originally sent (a model_name/alias),
|
||||
# not model_name resolved above. acreate_realtime_client_secret
|
||||
# prioritizes session.model over the model kwarg, so leaving it
|
||||
# unresolved would silently route with the alias instead of the
|
||||
# deployment litellm picked. See BerriAI/litellm#36742.
|
||||
session = response_kwargs.get("session")
|
||||
if isinstance(session, dict) and session.get("model"):
|
||||
response_kwargs["session"] = {**session, "model": model_name}
|
||||
|
||||
response = original_generic_function(**response_kwargs)
|
||||
|
||||
rpm_semaphore: Final = self._get_client(
|
||||
|
|
|
|||
|
|
@ -524,6 +524,42 @@ def test_generic_api_call_with_fallbacks_basic(sync_mode):
|
|||
assert response == mock_response
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ageneric_api_call_with_fallbacks_resolves_session_model():
|
||||
"""
|
||||
Regression test for https://github.com/BerriAI/litellm/issues/36742: when a
|
||||
call has a nested session.model (realtime client_secrets), it must be
|
||||
rewritten to the deployment litellm actually picked for the model_group,
|
||||
not left as the caller's original alias.
|
||||
"""
|
||||
mock_function = AsyncMock()
|
||||
mock_function.__name__ = "acreate_realtime_client_secret"
|
||||
mock_function.return_value = {"value": "ek_test"}
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-realtime-2-1-mini",
|
||||
"litellm_params": {
|
||||
"model": "azure/gpt-realtime-2-1-mini-deployment",
|
||||
"api_key": "fake-api-key",
|
||||
"api_base": "https://fake.openai.azure.com",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
await router._ageneric_api_call_with_fallbacks(
|
||||
model="gpt-realtime-2-1-mini",
|
||||
original_function=mock_function,
|
||||
session={"type": "realtime", "model": "gpt-realtime-2-1-mini"},
|
||||
)
|
||||
|
||||
call_kwargs = mock_function.call_args.kwargs
|
||||
assert call_kwargs["model"] == "azure/gpt-realtime-2-1-mini-deployment"
|
||||
assert call_kwargs["session"]["model"] == "azure/gpt-realtime-2-1-mini-deployment"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aadapter_completion():
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue