mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): check pre-call fallback access with the saved model_group_alias, never the one the client sent (#52077)
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
0ba3d312d4
commit
de20250735
3 changed files with 71 additions and 12 deletions
|
|
@ -3401,14 +3401,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
original_model,
|
||||
fallback_models,
|
||||
)
|
||||
raw_router_settings: Final = _DECODED_JSON.validate_python(
|
||||
_REQUEST_FIELDS.validate_python(rate_limited_data).get("router_settings_override")
|
||||
)
|
||||
group_alias: Final = _DECODED_JSON.validate_python(
|
||||
_REQUEST_FIELDS.validate_python(raw_router_settings).get("model_group_alias")
|
||||
if isinstance(raw_router_settings, Mapping)
|
||||
else None
|
||||
)
|
||||
group_alias: Final = router_settings.get("model_group_alias") if router_settings is not None else None
|
||||
await _release_pre_call_hold(
|
||||
proxy_logging_obj, _REQUEST_FIELDS.validate_python(rate_limited_data), original_exc, user_api_key_dict
|
||||
)
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
|
|||
model_entry(second_fallback, second_fallback_wire),
|
||||
),
|
||||
fallbacks=({primary: [fallback, second_fallback]},),
|
||||
general_settings={},
|
||||
general_settings={"enforce_fallback_model_access": True},
|
||||
)
|
||||
owned: Final = stack.enter_context(
|
||||
owned_proxy_process(
|
||||
|
|
@ -77,3 +77,20 @@ def test_quota_fallback_response_counts_every_candidate_a_limit_rejected(rig: Ri
|
|||
assert_served(response, rig.second_fallback, "served pmrl-")
|
||||
assert response.headers.get("x-litellm-attempted-fallbacks") == "2", dict(response.headers)
|
||||
assert response.headers.get("x-litellm-fallback-reason") == "rate_limit:model_per_key", dict(response.headers)
|
||||
|
||||
|
||||
def test_a_client_sent_model_group_alias_does_not_authorize_a_fallback_the_key_may_not_call(rig: Rig) -> None:
|
||||
with rig.gateway.scenario() as scenario:
|
||||
key: Final = scenario.key(models=[rig.primary], model_tpm_limit={rig.primary: 1})
|
||||
|
||||
response: Final = chat(
|
||||
rig.gateway,
|
||||
key,
|
||||
rig.primary,
|
||||
f"pmrl-{uuid.uuid4().hex}",
|
||||
router_settings_override={
|
||||
"model_group_alias": {rig.fallback: rig.primary, rig.second_fallback: rig.primary}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 429, (response.headers.get("x-litellm-model-group"), response.text)
|
||||
assert "x-litellm-attempted-fallbacks" not in response.headers, dict(response.headers)
|
||||
|
|
|
|||
|
|
@ -8804,7 +8804,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
|
|||
proxy_logging_obj=MagicMock(),
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(),
|
||||
version=None,
|
||||
proxy_config=_proxy_config(),
|
||||
proxy_config=_proxy_config({"model_group_alias": {"embed-alias": "embed-only"}}),
|
||||
user_model=None,
|
||||
user_temperature=None,
|
||||
user_request_timeout=None,
|
||||
|
|
@ -9020,6 +9020,55 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
|
|||
|
||||
assert processor.data["model"] == "claude-3-haiku"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_client_sent_model_group_alias_does_not_authorize_a_fallback_the_key_may_not_call(
|
||||
self, monkeypatch
|
||||
):
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
|
||||
processor = ProxyBaseLLMRequestProcessing(
|
||||
data={"model": "gpt-4", "router_settings_override": {"model_group_alias": {"gpt-3.5-turbo": "gpt-4"}}}
|
||||
)
|
||||
|
||||
async def mock_pre_call_logic(**kwargs):
|
||||
if processor.data.get("model") == "gpt-4":
|
||||
raise ProxyRateLimitError(detail="TPM limit exceeded", headers={"retry-after": "30"})
|
||||
return processor.data, MagicMock()
|
||||
|
||||
async def only_the_requested_model(*, model, valid_token, llm_router):
|
||||
return model == "gpt-4"
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setitem(proxy_server.general_settings, "enforce_fallback_model_access", True)
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.auth.fallback_model_access.is_model_authorized_for_token", only_the_requested_model
|
||||
)
|
||||
mock_router = route_entry_points_to_stubs(MagicMock())
|
||||
mock_router.fallbacks = [{"gpt-4": ["gpt-3.5-turbo"]}]
|
||||
|
||||
with (
|
||||
patch.object(processor, "common_processing_pre_call_logic", side_effect=mock_pre_call_logic),
|
||||
pytest.raises(ProxyRateLimitError),
|
||||
):
|
||||
await processor._pre_call_with_fallbacks(
|
||||
request=MagicMock(),
|
||||
general_settings={},
|
||||
proxy_logging_obj=MagicMock(),
|
||||
user_api_key_dict=MagicMock(router_settings=None, agent_id=None, end_user_id=None),
|
||||
version=None,
|
||||
proxy_config=_proxy_config(),
|
||||
user_model=None,
|
||||
user_temperature=None,
|
||||
user_request_timeout=None,
|
||||
user_max_tokens=None,
|
||||
user_api_base=None,
|
||||
model="gpt-4",
|
||||
route_type="acompletion",
|
||||
llm_router=mock_router,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_agent_owned_key_never_leaves_the_model_it_asked_for(self):
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
|
|
@ -9345,7 +9394,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
|
|||
proxy_logging_obj=MagicMock(),
|
||||
user_api_key_dict=ProxyKey(),
|
||||
version=None,
|
||||
proxy_config=_proxy_config(),
|
||||
proxy_config=_proxy_config({"model_group_alias": {"gpt-3.5-turbo": "pricey"}}),
|
||||
user_model=None,
|
||||
user_temperature=None,
|
||||
user_request_timeout=None,
|
||||
|
|
@ -9394,7 +9443,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
|
|||
proxy_logging_obj=MagicMock(),
|
||||
user_api_key_dict=ProxyKey(),
|
||||
version=None,
|
||||
proxy_config=_proxy_config(),
|
||||
proxy_config=_proxy_config({"model_group_alias": {"gpt-3.5-turbo": "roomy"}}),
|
||||
user_model=None,
|
||||
user_temperature=None,
|
||||
user_request_timeout=None,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue