scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-04 02:31:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 71

fix(proxy): source /v1/models token limits from the cost map instead of Router.get_model_group_info (#33721)

Browse source 
* fix(proxy): source /v1/models token limits from cost map instead of Router.get_model_group_info

Resolves the per-model get_model_group_info fan-out on GET /v1/models
(and /models) that pegged the event loop on wildcard listings (#33636).
create_model_info_response now reads max_input_tokens/max_output_tokens
from litellm.get_model_info (the static cost map) rather than the router,
which aggregated and deepcopied every deployment in a group per listed
model.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(proxy): inject model-info lookup into create_model_info_response for deterministic coverage

Inject the cost-map lookup (defaulting to litellm.get_model_info) so the
except and max_output_tokens branches are exercised deterministically and
the token-limit tests no longer hardcode mutable cost-map values.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* feat(proxy): surface custom deployment token limits on /v1/models via cheap index lookup

Add Router.get_configured_token_limits, an O(1) model-name index lookup that
reads a concrete deployment's configured max_input_tokens/max_output_tokens
without triggering pattern matching or deep copies. create_model_info_response
layers this over the cost map so custom deployments absent from the cost map
still surface their limits, and admin-configured limits override cost-map
defaults, while wildcard-expanded names stay on the fast path.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: ryan <ryan@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
(cherry picked from commit 8536e3b80e)
This commit is contained in:
devin-ai-integration[bot] 2026-07-17 20:04:18 -07:00 • committed by Yuneng Jiang
parent a50166df1f
commit 44b3de5a83
No known key found for this signature in database
5 changed files with 209 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
litellm/proxy/utils.py
View file

@ -19,6 +19,7 @@ from typing import (
Any,
AsyncGenerator,
Awaitable,
Callable,
ClassVar,
Dict,
List,
@ -49,7 +50,7 @@ from litellm.proxy._types import (
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.proxy.model_listing import ModelInfoResponse
from litellm.types.utils import CallTypes, CallTypesLiteral
from litellm.types.utils import CallTypes, CallTypesLiteral, ModelInfo
try:
from litellm_enterprise.enterprise_callbacks.send_emails.base_email import (
@ -6098,6 +6099,7 @@ def create_model_info_response(
include_metadata: bool = False,
fallback_type: Optional[str] = None,
llm_router: Optional["Router"] = None,
get_model_info: Callable[[str], ModelInfo] = litellm.get_model_info,
) -> ModelInfoResponse:
"""
Create a standardized OpenAI-compatible model object.
@ -6115,25 +6117,37 @@ def create_model_info_response(
"owned_by": provider,
}
# Surface context-window limits for OpenAI-compatible discovery clients.
# Only emitted when known, so wildcard routes and limitless backends stay clean.
# Limits are best-effort enrichment, so a single malformed deployment degrades
# to the base response rather than 500-ing the whole listing.
try:
model_cost_info: ModelInfo | None = get_model_info(model_id)
except Exception as e:
verbose_proxy_logger.debug(
"create_model_info_response: cost map lookup failed for %s: %s",
model_id,
e,
)
model_cost_info = None
max_input_tokens: int | None = None
max_output_tokens: int | None = None
if model_cost_info is not None:
cost_map_input = model_cost_info.get("max_input_tokens")
if cost_map_input is not None:
max_input_tokens = int(cost_map_input)
cost_map_output = model_cost_info.get("max_output_tokens")
if cost_map_output is not None:
max_output_tokens = int(cost_map_output)
if llm_router is not None:
try:
model_group_info = llm_router.get_model_group_info(model_id)
except Exception as e:
verbose_proxy_logger.debug(
"create_model_info_response: get_model_group_info failed for %s: %s",
model_id,
e,
)
model_group_info = None
if model_group_info is not None:
if model_group_info.max_input_tokens is not None:
base["max_input_tokens"] = int(model_group_info.max_input_tokens)
if model_group_info.max_output_tokens is not None:
base["max_output_tokens"] = int(model_group_info.max_output_tokens)
configured_input, configured_output = llm_router.get_configured_token_limits(model_id)
if configured_input is not None:
max_input_tokens = configured_input
if configured_output is not None:
max_output_tokens = configured_output
if max_input_tokens is not None:
base["max_input_tokens"] = max_input_tokens
if max_output_tokens is not None:
base["max_output_tokens"] = max_output_tokens
if not include_metadata:
return base

21
litellm/router.py
View file

@ -8357,6 +8357,27 @@ class Router:
raise Exception("Model Name invalid - {}".format(type(model)))
return None
def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]":
"""
Return (max_input_tokens, max_output_tokens) explicitly configured in a concrete
deployment's model_info for model_name, via O(1) index lookup.
Returns (None, None) for wildcard-expanded or unknown names. Unlike
get_model_group_info, this never triggers pattern matching or deep copies, so it
is safe to call per listed model on the /v1/models hot path.
"""
deployment = self.get_deployment_by_model_group_name(model_group_name=model_name)
if deployment is None:
return (None, None)
model_info = deployment.model_info
max_input = model_info.get("max_input_tokens")
max_output = model_info.get("max_output_tokens")
return (
int(max_input) if max_input is not None else None,
int(max_output) if max_output is not None else None,
)
def get_deployment_credentials_with_provider(self, model_id: str) -> Optional[Dict[str, Any]]:
"""
Get API credentials and provider info from a model name in model_list.

186
tests/test_litellm/proxy/test_proxy_utils.py
View file

@ -476,101 +476,118 @@ class TestPostCallFailureHookLiftsRecoveredPartialSpend:
assert "response_cost" not in request_data
from typing import cast
from litellm.proxy.utils import create_model_info_response
from litellm.types.router import ModelGroupInfo
from litellm.types.utils import ModelInfo
def _router_returning(model_group_info):
router = MagicMock()
router.get_model_group_info = MagicMock(return_value=model_group_info)
return router
def _fake_model_info(**fields: int) -> ModelInfo:
return cast(ModelInfo, dict(fields))
def test_create_model_info_response_includes_max_tokens_when_available():
router = _router_returning(
ModelGroupInfo(
model_group="qwen-vllm",
providers=["hosted_vllm"],
max_input_tokens=32768,
max_output_tokens=8192,
)
def _raise_unmapped(model_id: str) -> ModelInfo:
raise ValueError(f"This model isn't mapped yet: {model_id}")
def test_create_model_info_response_includes_max_tokens_from_lookup():
response = create_model_info_response(
model_id="some-model",
provider="openai",
llm_router=None,
get_model_info=lambda _model: _fake_model_info(
max_input_tokens=128000, max_output_tokens=16384
),
)
assert response["id"] == "some-model"
assert response["object"] == "model"
assert response["max_input_tokens"] == 128000
assert response["max_output_tokens"] == 16384
def test_create_model_info_response_does_not_call_router_group_info():
router = MagicMock()
router.get_configured_token_limits.return_value = (None, None)
response = create_model_info_response(
model_id="qwen-vllm", provider="openai", llm_router=router
model_id="some-model",
provider="openai",
llm_router=router,
get_model_info=lambda _model: _fake_model_info(
max_input_tokens=128000, max_output_tokens=16384
),
)
router.get_model_group_info.assert_called_once_with("qwen-vllm")
assert response["id"] == "qwen-vllm"
assert response["object"] == "model"
assert response["max_input_tokens"] == 32768
assert response["max_output_tokens"] == 8192
router.get_model_group_info.assert_not_called()
assert response["max_input_tokens"] == 128000
def test_create_model_info_response_uses_deployment_limits_when_not_in_cost_map():
router = MagicMock()
router.get_configured_token_limits.return_value = (32000, 8000)
response = create_model_info_response(
model_id="my-custom-deployment",
provider="openai",
llm_router=router,
get_model_info=_raise_unmapped,
)
router.get_model_group_info.assert_not_called()
assert response["max_input_tokens"] == 32000
assert response["max_output_tokens"] == 8000
def test_create_model_info_response_deployment_limits_override_cost_map():
router = MagicMock()
router.get_configured_token_limits.return_value = (200000, None)
response = create_model_info_response(
model_id="gpt-4o",
provider="openai",
llm_router=router,
get_model_info=lambda _model: _fake_model_info(
max_input_tokens=128000, max_output_tokens=16384
),
)
assert response["max_input_tokens"] == 200000
assert response["max_output_tokens"] == 16384
def test_create_model_info_response_emits_integer_token_counts():
# ModelGroupInfo types the limits as float; OpenAI-compatible clients expect
# plain integers, so the response must not leak 128000.0.
router = _router_returning(
ModelGroupInfo(
model_group="gpt-4o",
providers=["openai"],
max_input_tokens=128000.0,
max_output_tokens=16384.0,
)
)
response = create_model_info_response(
model_id="gpt-4o", provider="openai", llm_router=router
model_id="some-model",
provider="openai",
llm_router=None,
get_model_info=lambda _model: _fake_model_info(
max_input_tokens=128000, max_output_tokens=16384
),
)
assert response["max_input_tokens"] == 128000
assert isinstance(response["max_input_tokens"], int)
assert response["max_output_tokens"] == 16384
assert isinstance(response["max_output_tokens"], int)
def test_create_model_info_response_omits_unknown_individual_limit():
router = _router_returning(
ModelGroupInfo(
model_group="partial",
providers=["openai"],
max_input_tokens=4096,
max_output_tokens=None,
)
)
response = create_model_info_response(
model_id="partial", provider="openai", llm_router=router
model_id="some-embedding",
provider="openai",
llm_router=None,
get_model_info=lambda _model: _fake_model_info(max_input_tokens=8191),
)
assert response["max_input_tokens"] == 4096
assert response["max_input_tokens"] == 8191
assert "max_output_tokens" not in response
def test_create_model_info_response_omits_limits_when_both_none():
router = _router_returning(
ModelGroupInfo(
model_group="no-limits",
providers=["openai"],
max_input_tokens=None,
max_output_tokens=None,
)
)
def test_create_model_info_response_omits_limits_when_lookup_raises():
response = create_model_info_response(
model_id="no-limits", provider="openai", llm_router=router
)
assert "max_input_tokens" not in response
assert "max_output_tokens" not in response
def test_create_model_info_response_omits_limits_when_group_unknown():
# Wildcard routes / access groups have no ModelGroupInfo.
router = _router_returning(None)
response = create_model_info_response(
model_id="openai/*", provider="openai", llm_router=router
model_id="openai/*",
provider="openai",
llm_router=None,
get_model_info=_raise_unmapped,
)
assert response["id"] == "openai/*"
@ -578,32 +595,33 @@ def test_create_model_info_response_omits_limits_when_group_unknown():
assert "max_output_tokens" not in response
def test_create_model_info_response_degrades_when_group_info_raises():
# A malformed deployment must not turn the listing into a 500; the entry
# falls back to the base fields without limits.
router = MagicMock()
router.get_model_group_info = MagicMock(side_effect=ValueError("bad deployment"))
response = create_model_info_response(
model_id="broken", provider="openai", llm_router=router
)
assert response["id"] == "broken"
assert "max_input_tokens" not in response
assert "max_output_tokens" not in response
def test_create_model_info_response_no_router_keeps_base_fields():
response = create_model_info_response(
model_id="some-model", provider="openai", llm_router=None
model_id="totally-unknown-model-xyz",
provider="openai",
llm_router=None,
get_model_info=_raise_unmapped,
)
assert response == {
"id": "some-model",
"id": "totally-unknown-model-xyz",
"object": "model",
"created": response["created"],
"owned_by": "openai",
}
def test_create_model_info_response_reads_real_cost_map():
response = create_model_info_response(
model_id="gpt-4o", provider="openai", llm_router=None
)
assert isinstance(response["max_input_tokens"], int)
assert response["max_input_tokens"] > 0
assert isinstance(response["max_output_tokens"], int)
assert response["max_output_tokens"] > 0
class TestPostCallFailureHookLLMExceptionAlerting:
"""The llm_exceptions alert is for infra / LLM-API failures, not user
errors (https://github.com/BerriAI/litellm/issues/3395). Already-normalized

12
tests/test_litellm/proxy/utils/helpers/test_model_access.py
View file

@ -103,18 +103,16 @@ def test_is_known_vector_store_index_error_path_no_registry(monkeypatch):
def test_create_model_info_response_happy_path_no_metadata():
result = create_model_info_response(model_id="gpt-4o", provider="openai")
assert result == {
"id": "gpt-4o",
"object": "model",
"created": result["created"],
"owned_by": "openai",
}
snapshot = {
"id": result["id"],
"object": result["object"],
"owned_by": result["owned_by"],
"created_is_int": isinstance(result["created"], int),
"metadata_absent": "metadata" not in result,
"max_input_tokens_positive_int": isinstance(result["max_input_tokens"], int)
and result["max_input_tokens"] > 0,
"max_output_tokens_positive_int": isinstance(result["max_output_tokens"], int)
and result["max_output_tokens"] > 0,
}
assert snapshot == {
"id": "gpt-4o",
@ -122,6 +120,8 @@ def test_create_model_info_response_happy_path_no_metadata():
"owned_by": "openai",
"created_is_int": True,
"metadata_absent": True,
"max_input_tokens_positive_int": True,
"max_output_tokens_positive_int": True,
}

47
tests/test_litellm/test_router.py
View file

@ -5304,3 +5304,50 @@ class TestRouterRequestTimeoutPropagation:
)
== 60
)
def test_get_configured_token_limits_reads_deployment_model_info():
router = litellm.Router(
model_list=[
{
"model_name": "my-custom-model",
"litellm_params": {"model": "openai/some-unmapped-model"},
"model_info": {"max_input_tokens": 32000, "max_output_tokens": 8000},
}
]
)
assert router.get_configured_token_limits("my-custom-model") == (32000, 8000)
def test_get_configured_token_limits_returns_none_for_unset_or_unknown():
router = litellm.Router(
model_list=[
{
"model_name": "no-limits-model",
"litellm_params": {"model": "openai/some-unmapped-model"},
}
]
)
assert router.get_configured_token_limits("no-limits-model") == (None, None)
assert router.get_configured_token_limits("not-a-real-model") == (None, None)
def test_get_configured_token_limits_skips_wildcard_pattern_matching():
router = litellm.Router(
model_list=[
{
"model_name": "bedrock/*",
"litellm_params": {"model": "bedrock/*"},
"model_info": {"max_input_tokens": 12345},
}
]
)
with patch.object(
router.pattern_router, "route", side_effect=AssertionError("pattern route called")
):
assert router.get_configured_token_limits(
"bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
) == (None, None)