diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index bf76f99db69..6f91b1693d2 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -64,6 +64,44 @@ def _get_models_from_access_groups( return all_models +def _is_access_group_only_string( + model: str, + proxy_model_list: List[str], + model_access_groups: Dict[str, List[str]], +) -> bool: + """ + Identify entries in a key/team `models` list that are access-group strings + rather than real model identifiers. + + Issue #25550: when a virtual key was created with `models=["foo"]` against + an access group "foo" that has since been removed (or that was never + defined on any model), "foo" remains in the key but is no longer in + `model_access_groups`. The current expansion logic only strips entries + that ARE in the access-groups dict, so the stale string passes through + /v1/models as if it were a model name. + + A string is treated as access-group-only when none of the markers of a + real model identifier are present: + - it is in proxy_model_list (a configured proxy model group) + - it is in model_access_groups (an active access group; expanded above) + - it is in litellm.model_list_set (a known base model id) + - it contains "/" (provider-qualified route, e.g. `openai/gpt-4o`) + - it contains "*" (wildcard route, expanded by `_get_wildcard_models`) + - it starts with "ft:" (OpenAI fine-tune id) + """ + if model in proxy_model_list: + return False + if model in model_access_groups: + return False + if model in litellm.model_list_set: + return False + if "/" in model or "*" in model: + return False + if model.startswith("ft:"): + return False + return True + + async def get_mcp_server_ids( user_api_key_dict: UserAPIKeyAuth, ) -> List[str]: @@ -211,6 +249,20 @@ def get_complete_model_list( valid_models = get_valid_models() append_unique(valid_models) + # Drop stale access-group strings carried in key/team `models` (issue #25550). + # Only filter the key/team paths — the proxy-admin path above is sourced + # from authoritative state (proxy_model_list, model_access_groups keys). + if key_models or team_models: + unique_models = [ + m + for m in unique_models + if not _is_access_group_only_string( + model=m, + proxy_model_list=proxy_model_list, + model_access_groups=model_access_groups, + ) + ] + if only_model_access_groups: model_access_groups_to_return: List[str] = [] for model in unique_models: diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 77aa03032a7..f2f30a88738 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -227,6 +227,155 @@ def test_get_complete_model_list_order( ) +def test_get_complete_model_list_drops_stale_access_group_string(): + """ + Regression for issue #25550. + + A virtual key with `models=["team-sales-api"]` where "team-sales-api" + is neither a configured proxy model nor an active access group should + NOT leak the bare access-group string into /v1/models. + """ + from litellm.proxy.auth.model_checks import get_complete_model_list + + result = get_complete_model_list( + key_models=["team-sales-api"], + team_models=[], + proxy_model_list=["gpt-4o-mini"], + user_model=None, + infer_model_from_keys=False, + model_access_groups={}, + ) + + assert result == [] + + +def test_get_complete_model_list_drops_stale_access_group_string_team(): + """Same regression as above but exercised through the team_models path.""" + from litellm.proxy.auth.model_checks import get_complete_model_list + + result = get_complete_model_list( + key_models=[], + team_models=["team-sales-api"], + proxy_model_list=["gpt-4o-mini"], + user_model=None, + infer_model_from_keys=False, + model_access_groups={}, + ) + + assert result == [] + + +def test_get_complete_model_list_keeps_active_access_group_expansion(): + """ + Active access groups still expand to their member models. The filter + must not interfere with the existing expansion path (i.e., key_models + must arrive already-expanded from get_key_models). + """ + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.model_checks import get_complete_model_list, get_key_models + + user_api_key_dict = UserAPIKeyAuth( + models=["group-engineering"], + api_key="test-key", + ) + proxy_model_list = ["gpt-4o-mini"] + model_access_groups = {"group-engineering": ["gpt-4o-mini"]} + + key_models = get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=proxy_model_list, + model_access_groups=model_access_groups, + ) + + result = get_complete_model_list( + key_models=key_models, + team_models=[], + proxy_model_list=proxy_model_list, + user_model=None, + infer_model_from_keys=False, + model_access_groups=model_access_groups, + ) + + assert result == ["gpt-4o-mini"] + + +def test_get_complete_model_list_keeps_provider_qualified_string(): + """ + Provider-qualified identifiers carry a syntactic marker (`/`) and must + survive the access-group filter — even if the exact model id is not + present in the static litellm.model_list_set. + """ + from litellm.proxy.auth.model_checks import get_complete_model_list + + result = get_complete_model_list( + key_models=["bedrock/very-new-model"], + team_models=[], + proxy_model_list=[], + user_model=None, + infer_model_from_keys=False, + model_access_groups={}, + ) + + assert result == ["bedrock/very-new-model"] + + +def test_get_complete_model_list_keeps_known_base_model(): + """ + A known LiteLLM base model id (in litellm.model_list_set) must survive + the filter even when not configured on the proxy. + """ + from litellm.proxy.auth.model_checks import get_complete_model_list + + result = get_complete_model_list( + key_models=["gpt-4o-mini"], + team_models=[], + proxy_model_list=[], + user_model=None, + infer_model_from_keys=False, + model_access_groups={}, + ) + + assert result == ["gpt-4o-mini"] + + +def test_get_complete_model_list_keeps_finetune_id(): + """OpenAI fine-tune ids (`ft:...`) must survive the filter.""" + from litellm.proxy.auth.model_checks import get_complete_model_list + + ft_id = "ft:gpt-4o:my-org:custom-suffix:abc123" + result = get_complete_model_list( + key_models=[ft_id], + team_models=[], + proxy_model_list=[], + user_model=None, + infer_model_from_keys=False, + model_access_groups={}, + ) + + assert result == [ft_id] + + +def test_get_complete_model_list_does_not_filter_proxy_admin_path(): + """ + The filter must only apply when key_models or team_models is set. When + both are empty (proxy-admin / scope=expand path), unique_models is + sourced from the authoritative proxy_model_list and should pass through + untouched. + """ + from litellm.proxy.auth.model_checks import get_complete_model_list + + result = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=["arbitrary-named-model"], + user_model=None, + infer_model_from_keys=False, + model_access_groups={}, + ) + + assert result == ["arbitrary-named-model"] + + def test_get_complete_model_list_byok_wildcard_expansion(): """ Test that wildcard models (e.g., openai/*) are expanded when the router has