mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
fix(proxy): report the widest window across a model group, not the first deployment's
This commit is contained in:
michelligabriele
parent
c60c60e6fe
commit
d5270890c4
6 changed files with 242 additions and 44 deletions
|
|
@ -7496,10 +7496,11 @@ def _safe_get_model_info(model: str, get_model_info: Callable[[str], ModelInfo])
|
|||
def _resolve_listing_model_info(
|
||||
deployment_model: str | None,
|
||||
listed_model: str,
|
||||
listed_info: ModelInfo | None,
|
||||
get_model_info: Callable[[str], ModelInfo],
|
||||
) -> tuple[ModelInfo, ...]:
|
||||
"""
|
||||
Cost-map entries describing a listed model, best source first.
|
||||
Cost-map entries describing one deployment behind a listed model, best source first.
|
||||
|
||||
The name a model is listed under is an arbitrary public alias, so it often misses the
|
||||
cost map and lands on a fallback-generalization rule that answers with a conservative
|
||||
|
|
@ -7508,9 +7509,10 @@ def _resolve_listing_model_info(
|
|||
generalize, and because a deployment's own model is registered into the cost map as a
|
||||
stub that carries no limits of its own. Exact entries are consulted before generalized
|
||||
ones, and each field is then taken from the first entry that has it.
|
||||
"""
|
||||
listed_info: Final = _safe_get_model_info(listed_model, get_model_info)
|
||||
|
||||
``listed_info`` is resolved once by the caller, since a group with several distinct
|
||||
underlying models resolves the same alias for each of them.
|
||||
"""
|
||||
# Fast path, and the only one a wildcard-expanded name takes: with a single name
|
||||
# there is nothing to order, so skip the generalization test entirely. This keeps
|
||||
# the per-model cost of the listing on the hot path #33721 exists to protect.
|
||||
|
|
@ -7539,6 +7541,20 @@ def _first_token_limit(candidates: tuple[ModelInfo, ...], field: str) -> int | N
|
|||
)
|
||||
|
||||
|
||||
def _group_token_limit(candidate_sets: tuple[tuple[ModelInfo, ...], ...], field: str) -> int | None:
|
||||
"""The widest limit any deployment behind the listed name declares for ``field``.
|
||||
|
||||
A model group is normally one model behind several interchangeable deployments, so
|
||||
there is a single value to report. When a group genuinely mixes models, reporting the
|
||||
widest window keeps the listing independent of config order and agreeing with
|
||||
``/model_group/info``, which aggregates the same way for the Admin UI.
|
||||
"""
|
||||
limits: Final = tuple(
|
||||
limit for limit in (_first_token_limit(candidates, field) for candidates in candidate_sets) if limit is not None
|
||||
)
|
||||
return max(limits) if limits else None
|
||||
|
||||
|
||||
def create_model_info_response(
|
||||
model_id: str,
|
||||
provider: str,
|
||||
|
|
@ -7565,19 +7581,30 @@ def create_model_info_response(
|
|||
|
||||
listing_info: Final = llm_router.get_model_listing_info(model_id) if llm_router is not None else None
|
||||
|
||||
candidates: Final = _resolve_listing_model_info(
|
||||
deployment_model=listing_info.cost_map_key if listing_info is not None else None,
|
||||
listed_model=model_id,
|
||||
get_model_info=get_model_info,
|
||||
# One entry per distinct model behind the listed name; (None,) when the router knows
|
||||
# nothing about it, so the listed name is resolved on its own as before.
|
||||
deployment_models: Final[tuple[str | None, ...]] = (
|
||||
listing_info.cost_map_keys if listing_info is not None and listing_info.cost_map_keys else (None,)
|
||||
)
|
||||
listed_info: Final = _safe_get_model_info(model_id, get_model_info)
|
||||
candidate_sets: Final = tuple(
|
||||
_resolve_listing_model_info(
|
||||
deployment_model=deployment_model,
|
||||
listed_model=model_id,
|
||||
listed_info=listed_info,
|
||||
get_model_info=get_model_info,
|
||||
)
|
||||
for deployment_model in deployment_models
|
||||
)
|
||||
|
||||
max_input_tokens: int | None = _first_token_limit(candidates, "max_input_tokens")
|
||||
max_output_tokens: int | None = _first_token_limit(candidates, "max_output_tokens")
|
||||
max_input_tokens: int | None = _group_token_limit(candidate_sets, "max_input_tokens")
|
||||
max_output_tokens: int | None = _group_token_limit(candidate_sets, "max_output_tokens")
|
||||
mode: Final = next(
|
||||
(
|
||||
m
|
||||
for m in (
|
||||
cast("Mapping[str, object]", info).get("mode") # cast-ok: an entry need not carry "mode"
|
||||
for candidates in candidate_sets
|
||||
for info in candidates
|
||||
)
|
||||
if isinstance(m, str)
|
||||
|
|
|
|||
|
|
@ -9729,29 +9729,57 @@ class Router:
|
|||
|
||||
def get_model_listing_info(self, model_name: str) -> DeploymentModelListingInfo | None:
|
||||
"""
|
||||
Return what the concrete deployment behind model_name contributes to its
|
||||
/v1/models entry: the cost-map key for its underlying model, plus any token
|
||||
limits explicitly configured in its model_info. Resolved via O(1) index lookup.
|
||||
Return what the concrete deployments behind model_name contribute to its
|
||||
/v1/models entry: the cost-map keys for their underlying models, plus the widest
|
||||
token limits explicitly configured in their model_info. Resolved via O(1) index
|
||||
lookup.
|
||||
|
||||
Returns None for wildcard-expanded or unknown names, where the listed name is
|
||||
the real model name and no deployment-specific information exists, and treats a
|
||||
malformed configured limit as absent rather than failing the listing. Unlike
|
||||
get_model_group_info, this never triggers pattern matching or deep copies, so it
|
||||
is safe to call per listed model on the /v1/models hot path.
|
||||
Returns None for wildcard-expanded or unknown names, where the listed name is the
|
||||
real model name and no deployment-specific information exists, and treats a
|
||||
malformed configured limit as absent rather than failing the listing.
|
||||
|
||||
The whole group is read rather than just its first deployment, so a group that
|
||||
mixes models does not advertise a window that depends on config order; the widest
|
||||
one is reported, which is what get_model_group_info already shows the Admin UI.
|
||||
Keys are deduplicated, so the ordinary group of interchangeable deployments of one
|
||||
model still costs the caller a single cost-map lookup. Unlike get_model_group_info,
|
||||
this never triggers pattern matching or deep copies, so it is safe to call per
|
||||
listed model on the /v1/models hot path.
|
||||
"""
|
||||
deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name)
|
||||
if deployment is None:
|
||||
indices: Final = self.model_name_to_deployment_indices.get(model_name)
|
||||
if not indices:
|
||||
return None
|
||||
|
||||
model_info: Final = deployment.model_info
|
||||
# base_model is a declared field, so read it as one: an unset or blank value
|
||||
# means the deployment's own model name is the cost-map key.
|
||||
base_model: Final = model_info.base_model
|
||||
return DeploymentModelListingInfo(
|
||||
cost_map_key=base_model or deployment.litellm_params.model,
|
||||
max_input_tokens=coerce_token_limit(model_info.get("max_input_tokens")),
|
||||
max_output_tokens=coerce_token_limit(model_info.get("max_output_tokens")),
|
||||
deployments: Final = tuple(self.model_list[index] for index in indices)
|
||||
model_infos: Final = tuple(deployment.get("model_info") or MappingProxyType({}) for deployment in deployments)
|
||||
params: Final = tuple(deployment.get("litellm_params") or MappingProxyType({}) for deployment in deployments)
|
||||
# base_model resolution mirrors get_router_model_info: unset or blank means the
|
||||
# deployment's own model name is the cost-map key.
|
||||
cost_map_keys: Final = tuple(
|
||||
dict.fromkeys( # deduplicates while preserving config order
|
||||
key
|
||||
for key in (
|
||||
model_info.get("base_model") or litellm_params.get("base_model") or litellm_params.get("model")
|
||||
for model_info, litellm_params in zip(model_infos, params)
|
||||
)
|
||||
if isinstance(key, str) and key
|
||||
)
|
||||
)
|
||||
return DeploymentModelListingInfo(
|
||||
cost_map_keys=cost_map_keys,
|
||||
max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"),
|
||||
max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _widest_configured_limit(model_infos: Sequence[Mapping[str, Any]], field: str) -> int | None:
|
||||
"""The largest usable value of ``field`` across a group's configured model_info blocks."""
|
||||
limits: Final = tuple(
|
||||
limit
|
||||
for limit in (coerce_token_limit(model_info.get(field)) for model_info in model_infos)
|
||||
if limit is not None
|
||||
)
|
||||
return max(limits) if limits else None
|
||||
|
||||
def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]":
|
||||
"""
|
||||
|
|
@ -9760,11 +9788,20 @@ class Router:
|
|||
|
||||
Returns (None, None) for wildcard-expanded or unknown names, and treats a
|
||||
malformed configured value as absent rather than failing the caller.
|
||||
|
||||
Deliberately reads one deployment rather than aggregating the group the way
|
||||
get_model_listing_info does: its caller truncates an embedding input to this
|
||||
value, so the widest window in a mixed group would be the wrong answer there.
|
||||
"""
|
||||
listing_info: Final = self.get_model_listing_info(model_name=model_name)
|
||||
if listing_info is None:
|
||||
deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name)
|
||||
if deployment is None:
|
||||
return (None, None)
|
||||
return (listing_info.max_input_tokens, listing_info.max_output_tokens)
|
||||
|
||||
model_info: Final = deployment.model_info
|
||||
return (
|
||||
coerce_token_limit(model_info.get("max_input_tokens")),
|
||||
coerce_token_limit(model_info.get("max_output_tokens")),
|
||||
)
|
||||
|
||||
def get_deployment_credentials_with_provider(
|
||||
self, model_id: str, team_id: str | None = None
|
||||
|
|
|
|||
|
|
@ -577,17 +577,18 @@ class Deployment(BaseModel):
|
|||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeploymentModelListingInfo:
|
||||
"""What a concrete deployment contributes to its OpenAI-compatible listing entry.
|
||||
"""What the deployments behind a model name contribute to its OpenAI-compatible listing entry.
|
||||
|
||||
``cost_map_key`` is the name the deployment's underlying model is known by in
|
||||
``litellm.model_cost`` (``model_info.base_model`` when set, else
|
||||
``litellm_params.model``), which is what the request actually reaches; the public
|
||||
model name it is listed under is an arbitrary alias and often absent from the cost
|
||||
map. The token limits are the ones explicitly set in ``model_info``, which outrank
|
||||
anything the cost map says.
|
||||
``cost_map_keys`` are the names those deployments' underlying models are known by in
|
||||
``litellm.model_cost`` (``base_model`` when set, else ``litellm_params.model``), which
|
||||
is what a request actually reaches; the public model name they are listed under is an
|
||||
arbitrary alias and often absent from the cost map. Keys are deduplicated in config
|
||||
order, so the ordinary group -- several interchangeable deployments of one model --
|
||||
carries exactly one. The token limits are the widest explicitly set in any
|
||||
deployment's ``model_info``, which outrank anything the cost map says.
|
||||
"""
|
||||
|
||||
cost_map_key: str
|
||||
cost_map_keys: tuple[str, ...]
|
||||
max_input_tokens: int | None
|
||||
max_output_tokens: int | None
|
||||
|
||||
|
|
|
|||
|
|
@ -168,7 +168,7 @@ def test_anthropic_format_carries_router_configured_token_limits(client, auth_as
|
|||
def _configured(model_name):
|
||||
max_input, max_output = (300000, 32000) if model_name == "gpt-4" else (500000, 4096)
|
||||
return DeploymentModelListingInfo(
|
||||
cost_map_key=model_name, max_input_tokens=max_input, max_output_tokens=max_output
|
||||
cost_map_keys=(model_name,), max_input_tokens=max_input, max_output_tokens=max_output
|
||||
)
|
||||
|
||||
def _cost_map_lookup(model_id):
|
||||
|
|
|
|||
|
|
@ -930,7 +930,7 @@ def test_create_model_info_response_does_not_call_router_group_info():
|
|||
def test_create_model_info_response_uses_deployment_limits_when_not_in_cost_map():
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = DeploymentModelListingInfo(
|
||||
cost_map_key="my-custom-deployment", max_input_tokens=32000, max_output_tokens=8000
|
||||
cost_map_keys=("my-custom-deployment",), max_input_tokens=32000, max_output_tokens=8000
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
|
|
@ -948,7 +948,7 @@ def test_create_model_info_response_uses_deployment_limits_when_not_in_cost_map(
|
|||
def test_create_model_info_response_deployment_limits_override_cost_map():
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = DeploymentModelListingInfo(
|
||||
cost_map_key="gpt-4o", max_input_tokens=200000, max_output_tokens=None
|
||||
cost_map_keys=("gpt-4o",), max_input_tokens=200000, max_output_tokens=None
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
|
|
@ -962,6 +962,54 @@ def test_create_model_info_response_deployment_limits_override_cost_map():
|
|||
assert response["max_output_tokens"] == 16384
|
||||
|
||||
|
||||
def test_create_model_info_response_reports_widest_window_in_a_mixed_group():
|
||||
"""A group mixing models advertises the widest window, not whichever is listed first."""
|
||||
limits = {
|
||||
"small-model": _fake_model_info(max_input_tokens=200000, max_output_tokens=4096, mode="chat"),
|
||||
"large-model": _fake_model_info(max_input_tokens=1000000, max_output_tokens=128000, mode="chat"),
|
||||
}
|
||||
|
||||
for keys in (("small-model", "large-model"), ("large-model", "small-model")):
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = DeploymentModelListingInfo(
|
||||
cost_map_keys=keys, max_input_tokens=None, max_output_tokens=None
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="house-claude",
|
||||
provider="openai",
|
||||
llm_router=router,
|
||||
get_model_info=lambda model: limits[model],
|
||||
)
|
||||
|
||||
assert response["max_input_tokens"] == 1000000, keys
|
||||
assert response["max_output_tokens"] == 128000, keys
|
||||
|
||||
|
||||
def test_create_model_info_response_resolves_alias_once_per_listing():
|
||||
"""The alias is the same for every deployment in the group, so it is looked up once."""
|
||||
seen: list[str] = []
|
||||
|
||||
def _tracking_get_model_info(model: str) -> ModelInfo:
|
||||
seen.append(model)
|
||||
return _fake_model_info(max_input_tokens=128000)
|
||||
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = DeploymentModelListingInfo(
|
||||
cost_map_keys=("model-a", "model-b"), max_input_tokens=None, max_output_tokens=None
|
||||
)
|
||||
|
||||
create_model_info_response(
|
||||
model_id="house-model",
|
||||
provider="openai",
|
||||
llm_router=router,
|
||||
get_model_info=_tracking_get_model_info,
|
||||
)
|
||||
|
||||
assert seen.count("house-model") == 1
|
||||
assert sorted(seen) == ["house-model", "model-a", "model-b"]
|
||||
|
||||
|
||||
def test_create_model_info_response_survives_malformed_configured_limits():
|
||||
from litellm import Router
|
||||
|
||||
|
|
|
|||
|
|
@ -7286,7 +7286,7 @@ def test_get_model_listing_info_prefers_base_model_over_litellm_params_model():
|
|||
|
||||
info = router.get_model_listing_info("bedrock-claude-opus-5")
|
||||
assert info is not None
|
||||
assert info.cost_map_key == "eu.anthropic.claude-opus-5"
|
||||
assert info.cost_map_keys == ("eu.anthropic.claude-opus-5",)
|
||||
|
||||
|
||||
def test_get_model_listing_info_falls_back_to_litellm_params_model():
|
||||
|
|
@ -7301,7 +7301,7 @@ def test_get_model_listing_info_falls_back_to_litellm_params_model():
|
|||
|
||||
info = router.get_model_listing_info("bedrock-claude-opus-5")
|
||||
assert info is not None
|
||||
assert info.cost_map_key == "bedrock/eu.anthropic.claude-opus-5"
|
||||
assert info.cost_map_keys == ("bedrock/eu.anthropic.claude-opus-5",)
|
||||
|
||||
|
||||
def test_get_model_listing_info_ignores_blank_base_model():
|
||||
|
|
@ -7318,7 +7318,7 @@ def test_get_model_listing_info_ignores_blank_base_model():
|
|||
|
||||
info = router.get_model_listing_info("bedrock-claude-opus-5")
|
||||
assert info is not None
|
||||
assert info.cost_map_key == "bedrock/eu.anthropic.claude-opus-5"
|
||||
assert info.cost_map_keys == ("bedrock/eu.anthropic.claude-opus-5",)
|
||||
|
||||
|
||||
def test_get_model_listing_info_returns_none_for_unknown_name():
|
||||
|
|
@ -7350,6 +7350,91 @@ def test_get_model_listing_info_carries_configured_limits():
|
|||
assert (info.max_input_tokens, info.max_output_tokens) == (32000, 8000)
|
||||
|
||||
|
||||
def test_get_model_listing_info_dedupes_interchangeable_deployments():
|
||||
"""The ordinary group is N deployments of one model, so it yields exactly one key."""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4o",
|
||||
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-a"},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-4o",
|
||||
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-b"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
info = router.get_model_listing_info("gpt-4o")
|
||||
assert info is not None
|
||||
assert info.cost_map_keys == ("openai/gpt-4o",)
|
||||
|
||||
|
||||
def test_get_model_listing_info_collects_every_model_in_a_mixed_group():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "house-claude",
|
||||
"litellm_params": {"model": "anthropic/claude-3-haiku-20240307"},
|
||||
},
|
||||
{
|
||||
"model_name": "house-claude",
|
||||
"litellm_params": {"model": "bedrock/eu.anthropic.claude-opus-5"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
info = router.get_model_listing_info("house-claude")
|
||||
assert info is not None
|
||||
assert info.cost_map_keys == (
|
||||
"anthropic/claude-3-haiku-20240307",
|
||||
"bedrock/eu.anthropic.claude-opus-5",
|
||||
)
|
||||
|
||||
|
||||
def test_get_model_listing_info_reports_widest_configured_limits_in_a_mixed_group():
|
||||
"""Matches how get_model_group_info aggregates for the Admin UI, so the two agree."""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "house-model",
|
||||
"litellm_params": {"model": "openai/some-unmapped-model"},
|
||||
"model_info": {"max_input_tokens": 32000, "max_output_tokens": 4096},
|
||||
},
|
||||
{
|
||||
"model_name": "house-model",
|
||||
"litellm_params": {"model": "openai/another-unmapped-model"},
|
||||
"model_info": {"max_input_tokens": 128000, "max_output_tokens": 16384},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
info = router.get_model_listing_info("house-model")
|
||||
assert info is not None
|
||||
assert (info.max_input_tokens, info.max_output_tokens) == (128000, 16384)
|
||||
|
||||
|
||||
def test_get_model_listing_info_reads_base_model_from_litellm_params():
|
||||
"""base_model resolution mirrors get_router_model_info, which also accepts it there."""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "azure-deployment",
|
||||
"litellm_params": {
|
||||
"model": "azure/my-azure-deployment-name",
|
||||
"base_model": "azure/gpt-4o",
|
||||
"api_key": "sk-a",
|
||||
"api_base": "https://example.openai.azure.com",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
info = router.get_model_listing_info("azure-deployment")
|
||||
assert info is not None
|
||||
assert info.cost_map_keys == ("azure/gpt-4o",)
|
||||
|
||||
|
||||
def test_get_model_listing_info_skips_wildcard_pattern_matching():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue