scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-01 07:03:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 129

fix(proxy): list only routable models for a wildcard alias over a provider org (#49152)

Browse source 
* fix(proxy): list only routable models for a wildcard alias over a provider org

A wildcard alias over an org-scoped deployment (together_ai/* over
together_ai/nosuchorg/*) listed the provider's whole catalog on /v1/models
and /model/info, and every listed id routed to a model the provider 404s.
A skipped deployment path segment now counts as a routing prefix only when
the routed model resolves to a catalog id through the get_model_info
lookup order, so responses/, Bedrock converse/, and region paths list as
before.

Together's model list now also carries the cost map's together_ai rows,
skipping the together-ai-* pricing tiers, so an alias over a real org lists
the models Together serves today.

* fix(proxy): keep flat catalog ids under routes the cost map can't parse, drop retired Together rows

* fix(proxy): check skipped path segments only on Together so Hugging Face provider routes keep listing

---------

Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
mika-lt 2026-10-30 00:14:49 -07:00 • committed by GitHub
parent 56388fd13d
commit cfb77ecc3a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 150 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/__init__.py
View file

@ -872,6 +872,11 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
if "-to-" not in key and "fireworks-ai-default" not in key:
fireworks_ai_models.add(key)
elif value.get("litellm_provider") == "together_ai":
if _is_past_deprecation_date(value):
together_ai_models.discard(key)
elif not key.startswith("together-ai-"):
together_ai_models.add(key)
elif value.get("litellm_provider") == "baseten":
if _is_past_deprecation_date(value):
baseten_models.discard(key)

2
litellm/constants.py
View file

@ -1274,7 +1274,7 @@ empower_models: Final = set(
]
)
together_ai_models: Final[set] = set(
together_ai_models: Final[set[str]] = set(
[
# llama llms - chat
"togethercomputer/llama-2-70b-chat",

34
litellm/proxy/auth/model_checks.py
View file

@ -14,14 +14,16 @@ from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
from litellm.responses.main import strip_responses_routing_prefix
from litellm.router import Router
from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager, get_valid_models
from litellm.utils import ProviderConfigManager, get_valid_models, potential_model_names_in_lookup_order
LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders)
_PROVIDERS_SENDING_THE_DEPLOYMENT_PATH_AS_THE_MODEL_ID: Final = frozenset({LlmProviders.TOGETHER_AI.value})
_CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields)
@ -331,6 +333,25 @@ def _routes_back_to(
)
def _resolves_to_catalog_model(model: str, provider: str, provider_model_ids: frozenset[str]) -> bool:
provider_id_prefix: Final = f"{provider}/"
resolved_model: Final = (
f"{provider_id_prefix}{strip_responses_routing_prefix(model.removeprefix(provider_id_prefix))}"
)
return any(
name.removeprefix(provider_id_prefix) in provider_model_ids
for name in potential_model_names_in_lookup_order(resolved_model, provider)
)
def _routes_to_catalog_model(
routes_to_model: Callable[[str, str], bool], provider: str, provider_model_ids: frozenset[str]
) -> Callable[[str, str], bool]:
return lambda listed_id, model: (
routes_to_model(listed_id, model) and _resolves_to_catalog_model(model, provider, provider_model_ids)
)
def _ids_routing_to_catalog_models(
name_pattern: str,
deployment_pattern: str,
@ -344,8 +365,15 @@ def _ids_routing_to_catalog_models(
return ()
provider_id_prefix: Final = f"{provider}/"
provider_model_ids: Final = tuple(catalog_id.removeprefix(provider_id_prefix) for catalog_id in catalog_ids)
deployment_model_pattern_without_provider: Final = deployment_pattern.removeprefix(provider_id_prefix)
catalog_pattern: Final = _without_routing_segments(
deployment_pattern.removeprefix(provider_id_prefix), provider, name_pattern, provider_model_ids
deployment_model_pattern_without_provider, provider, name_pattern, provider_model_ids
)
routing_prefix: Final = deployment_model_pattern_without_provider.removesuffix(catalog_pattern)
reaches_catalog_model: Final = (
_routes_to_catalog_model(routes_to_model, provider, frozenset(provider_model_ids))
if routing_prefix and provider in _PROVIDERS_SENDING_THE_DEPLOYMENT_PATH_AS_THE_MODEL_ID
else routes_to_model
)
deployment_model_pattern: Final = f"{provider_id_prefix}{catalog_pattern}"
deployment_regex: Final = re.compile(PatternMatchRouter().pattern_to_regex(deployment_model_pattern))
@ -369,7 +397,7 @@ def _ids_routing_to_catalog_models(
for listed_id, catalog_id in listed_ids
if listed_id
and _routes_back_to(
listed_id, name_regex, (deployment_model_pattern, catalog_id), deployment_pattern, routes_to_model
listed_id, name_regex, (deployment_model_pattern, catalog_id), deployment_pattern, reaches_catalog_model
)
)
)

8
litellm/responses/main.py
View file

@ -759,7 +759,7 @@ def _dispatched_model_name(
from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata
provider_model, _ = resolve_model_and_provider_for_metadata(model=model, custom_llm_provider=custom_llm_provider)
return _strip_responses_routing_prefix(provider_model)
return strip_responses_routing_prefix(provider_model)
def _will_bridge_to_chat_completions(
@ -1245,7 +1245,7 @@ def _pop_use_chat_completions_api_kw(kwargs: dict[str, object]) -> bool:
_RESPONSES_ROUTING_PREFIX: Final = "responses/"
def _strip_responses_routing_prefix(model: str) -> str:
def strip_responses_routing_prefix(model: str) -> str:
if not model.startswith(_RESPONSES_ROUTING_PREFIX):
return model
return model[len(_RESPONSES_ROUTING_PREFIX) :]
@ -1273,7 +1273,7 @@ def _resolve_model_provider_for_responses(
litellm_params.api_key = dynamic_api_key
if dynamic_api_base is not None:
litellm_params.api_base = dynamic_api_base
return _strip_responses_routing_prefix(provider_model), resolved_provider
return strip_responses_routing_prefix(provider_model), resolved_provider
def _apply_managed_file_id_mapping(
@ -2893,7 +2893,7 @@ async def _aresponses_websocket(
api_key=api_key,
litellm_params=litellm_params,
)
resolved_model: Final = _strip_responses_routing_prefix(provider_model)
resolved_model: Final = strip_responses_routing_prefix(provider_model)
litellm_params_dict["data_residency"] = infer_openai_data_residency(
_custom_llm_provider,

46
litellm/utils.py
View file

@ -6708,6 +6708,26 @@ def _first_registered_match(
return next(matches, (None, None))
def _model_cost_lookup_order(
model: str, potential_model_names: PotentialModelNamesAndCustomLLMProvider
) -> tuple[str, ...]:
return (
potential_model_names["combined_model_name"],
model,
potential_model_names["region_free_combined_model_name"],
potential_model_names["split_model"],
potential_model_names["combined_stripped_model_name"],
potential_model_names["stripped_model_name"],
potential_model_names["provider_prefixed_model_name"],
)
def potential_model_names_in_lookup_order(model: str, custom_llm_provider: str) -> tuple[str, ...]:
return _model_cost_lookup_order(
model, get_potential_model_names(model=model, custom_llm_provider=custom_llm_provider)
)
def _get_model_info_from_generalization(
model: str,
potential_model_names: PotentialModelNamesAndCustomLLMProvider,
@ -6726,15 +6746,7 @@ def _get_model_info_from_generalization(
still had a priced exact name to try. O(number of rules); only call after the
exact lookups have missed.
"""
candidates: Final = (
potential_model_names["combined_model_name"],
model,
potential_model_names["region_free_combined_model_name"],
potential_model_names["split_model"],
potential_model_names["combined_stripped_model_name"],
potential_model_names["stripped_model_name"],
potential_model_names["provider_prefixed_model_name"],
)
candidates: Final = _model_cost_lookup_order(model, potential_model_names)
if any(get_model_cost_key(candidate) is not None for candidate in candidates):
return None
for candidate in candidates:
@ -6973,11 +6985,6 @@ def get_model_info_helper(
model=model, custom_llm_provider=custom_llm_provider or declared_authenticating_provider(model)
)
combined_model_name: Final = potential_model_names["combined_model_name"]
stripped_model_name: Final = potential_model_names["stripped_model_name"]
combined_stripped_model_name: Final = potential_model_names["combined_stripped_model_name"]
provider_prefixed_model_name: Final = potential_model_names["provider_prefixed_model_name"]
split_model: Final = potential_model_names["split_model"]
custom_llm_provider = potential_model_names["custom_llm_provider"]
model_cost_custom_llm_provider: Final = custom_llm_provider
#########################
@ -7044,16 +7051,9 @@ def get_model_info_helper(
and strip it. Tried last so no model that already resolves through 1-6 can change.
"""
lookup_order: Final = (
combined_model_name,
model,
potential_model_names["region_free_combined_model_name"],
split_model,
combined_stripped_model_name,
stripped_model_name,
provider_prefixed_model_name,
lookup: Final = _first_registered_match(
_model_cost_lookup_order(model, potential_model_names), model_cost_custom_llm_provider
)
lookup: Final = _first_registered_match(lookup_order, model_cost_custom_llm_provider)
key: str | None = lookup[0]
_model_info: dict[str, Any] | None = lookup[1]

87
tests/unit/proxy/auth/test_model_checks.py
View file

@ -1057,7 +1057,9 @@ def test_wildcard_lists_each_model_once_under_its_full_owner_id(provider: str):
import litellm
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
full_ids: Final = {f"{provider}/{model.removeprefix(f'{provider}/')}" for model in litellm.models_by_provider[provider]}
full_ids: Final = {
f"{provider}/{model.removeprefix(f'{provider}/')}" for model in litellm.models_by_provider[provider]
}
assert any(model.count("/") > 1 for model in full_ids)
assert sorted(get_known_models_from_wildcard(f"{provider}/*")) == sorted(full_ids)
@ -1372,12 +1374,57 @@ def test_partial_wildcard_lists_catalog_ids_stored_with_the_provider_prefix(
assert sorted(listed) == expected
@pytest.mark.parametrize(
("provider", "routing_segment"), [("bedrock", "converse"), ("bedrock", "us-east-1"), ("openai", "responses")]
)
def test_routing_segment_lists_the_same_catalog_ids_as_the_bare_provider_wildcard(
provider: str, routing_segment: str
) -> None:
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.types.router import LiteLLM_Params
bare_listing: Final = get_known_models_from_wildcard(f"{provider}/*", LiteLLM_Params(model=f"{provider}/*"))
routed_listing: Final = get_known_models_from_wildcard(
f"{provider}/*", LiteLLM_Params(model=f"{provider}/{routing_segment}/*")
)
assert bare_listing
assert sorted(routed_listing) == sorted(bare_listing)
def test_provider_wildcard_lists_cost_map_models_but_not_pricing_tiers() -> None:
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
model_id: Final = "together_ai/partial-wildcard-org/model-t"
pricing_tier: Final = "together-ai-partial-wildcard-tier"
with _registered_chat_models({model_id: "together_ai", pricing_tier: "together_ai"}):
listed: Final = get_known_models_from_wildcard("together_ai/*")
assert model_id in listed
assert [listed_id for listed_id in listed if pricing_tier in listed_id] == []
def test_provider_wildcard_stops_listing_a_cost_map_model_once_it_is_retired() -> None:
import litellm
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
model_id: Final = "together_ai/partial-wildcard-org/model-retired"
with _registered_chat_models({model_id: "together_ai"}):
listed_while_served: Final = get_known_models_from_wildcard("together_ai/*")
litellm.add_known_models({model_id: {"litellm_provider": "together_ai", "deprecation_date": "2000-01-01"}})
listed_once_retired: Final = get_known_models_from_wildcard("together_ai/*")
assert model_id in listed_while_served
assert model_id not in listed_once_retired
_ROUTE_CF_ID: Final = "cloudflare/@cf/partial-route-org/model-a-instruct"
_ROUTE_CF_SUFFIXED_ID: Final = "cloudflare/@cf/partial-route-org/model-b-instruct-v2"
_ROUTE_HF_ID: Final = "cloudflare/@hf/partial-route-org/model-c"
_ROUTE_GEMINI_ID: Final = "gemini-partial-route-9-flash"
_ROUTE_OPENAI_ID: Final = "gpt-partial-route-9-mini"
_ROUTE_BEDROCK_ID: Final = "anthropic.partial-route-9-v1:0"
_ROUTE_TOGETHER_ID: Final = "together_ai/partial-route-org/model-t"
_ROUTE_CF_IDS: Final = {_ROUTE_CF_ID, _ROUTE_CF_SUFFIXED_ID}
@ -1456,6 +1503,29 @@ _ROUTE_CF_IDS: Final = {_ROUTE_CF_ID, _ROUTE_CF_SUFFIXED_ID}
set(),
id="same-name-namespace-the-catalog-lacks",
),
pytest.param(
"together/*", "together_ai/partial-route-missing-org/*", None, set(), id="alias-over-org-the-catalog-lacks"
),
pytest.param(
"openai/*",
"openai/chat_completions/*",
None,
{f"openai/chat_completions/{_ROUTE_OPENAI_ID}"},
id="chat-completions-bridge-route",
),
pytest.param(
"bedrock/*", "bedrock/llama/*", None, {f"bedrock/llama/{_ROUTE_BEDROCK_ID}"}, id="invoke-format-route"
),
pytest.param(
"together/*",
"together_ai/partial-route-org/*",
None,
{_ROUTE_TOGETHER_ID},
id="alias-over-org-a-cost-map-catalog-carries",
),
pytest.param(
"bedrock/*", "bedrock/us-east-1/*", None, {f"bedrock/us-east-1/{_ROUTE_BEDROCK_ID}"}, id="region-route"
),
pytest.param(
"*", "openai/*", "openai/gpt-partial-route-9*", set(), id="provider-prefixed-key-filter-over-catch-all"
),
@ -1501,6 +1571,7 @@ def test_every_listed_partial_wildcard_id_routes_to_the_catalog_model_it_names(
_ROUTE_GEMINI_ID: "gemini",
_ROUTE_OPENAI_ID: "openai",
_ROUTE_BEDROCK_ID: "bedrock",
_ROUTE_TOGETHER_ID: "together_ai",
}
):
v1_models: Final = get_complete_model_list(
@ -1535,6 +1606,20 @@ def test_every_listed_partial_wildcard_id_routes_to_the_catalog_model_it_names(
assert listed_model_info_names == sorted(reachable)
def test_inference_provider_route_lists_every_id_the_plain_provider_wildcard_lists() -> None:
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.types.router import LiteLLM_Params
plain: Final = get_known_models_from_wildcard("huggingface/*", LiteLLM_Params(model="huggingface/*"))
routed: Final = get_known_models_from_wildcard(
"huggingface/*", LiteLLM_Params(model="huggingface/featherless-ai/*")
)
assert plain
assert [model for model in plain if "/" not in model.removeprefix("huggingface/")] == []
assert sorted(routed) == sorted(plain)
def test_catch_all_lists_no_id_a_more_specific_sibling_wildcard_routes_elsewhere() -> None:
from litellm import Router
from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list