From eb4b847268fb5cf6876d59bb3426104ee004a45d Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Fri, 14 Aug 2026 16:52:39 -0700 Subject: [PATCH] fix(proxy): always emit the Anthropic /v1/models token limits, null when unknown (#36961) Anthropic's Models API declares max_input_tokens and max_tokens as nullable, not optional, and the live vendor endpoint returns both keys on every entry. The merged Anthropic-native listing dropped either key whenever LiteLLM could not resolve a limit, so a client validating against a nullable-but-required schema saw a malformed entry for any model the cost map does not know. --- litellm/llms/anthropic/common_utils.py | 10 +++--- .../anthropic/test_anthropic_common_utils.py | 16 ++++++--- .../proxy/proxy_server/test_routes_models.py | 36 +++++++++++++++++-- 3 files changed, 49 insertions(+), 13 deletions(-) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index b444c77d718..1cdbd60f943 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -1227,16 +1227,13 @@ def process_anthropic_headers(headers: httpx.Headers | dict) -> dict: def _anthropic_model_entry(model: ModelInfoResponse, created_at: str) -> Mapping[str, object]: - token_limits: Final = ( - ("max_input_tokens", model.get("max_input_tokens")), - ("max_tokens", model.get("max_output_tokens")), - ) return { # mutable-ok: JSON response body, serialized by the route and never mutated "type": "model", "id": model["id"], "display_name": model["id"], "created_at": created_at, - **{name: limit for name, limit in token_limits if limit is not None}, # mutable-ok: merged into the body above + "max_input_tokens": model.get("max_input_tokens"), + "max_tokens": model.get("max_output_tokens"), } @@ -1246,7 +1243,8 @@ def create_anthropic_model_list_response(models: Sequence[ModelInfoResponse]) -> Clients that send an anthropic-version header parse the Anthropic Models API shape (type/display_name/created_at plus has_more/first_id/last_id) and filter the list themselves, so every model is returned here. The token limits carry - over from the OpenAI-shaped listing, named as the Messages API names them + over from the OpenAI-shaped listing, named as the Messages API names them, and + are always present because the vendor shape declares them nullable, not optional """ created_at: Final = ( datetime.fromtimestamp(DEFAULT_MODEL_CREATED_AT_TIME, tz=timezone.utc).isoformat().replace("+00:00", "Z") diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index 431030bcf2e..d205a903063 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -2056,11 +2056,14 @@ def test_create_anthropic_model_list_response_shape(): # ISO 8601 with a Z suffix, as the Anthropic Models API returns. assert entry["created_at"].endswith("Z") assert "+00:00" not in entry["created_at"] - assert "max_input_tokens" not in entry - assert "max_tokens" not in entry + assert entry["max_input_tokens"] is None + assert entry["max_tokens"] is None def test_create_anthropic_model_list_response_carries_token_limits(): + """max_input_tokens and max_tokens are nullable in the Anthropic Models shape, + not optional, so both keys are emitted for every entry and carry null when the + limit is unknown.""" from litellm.llms.anthropic.common_utils import ( create_anthropic_model_list_response, ) @@ -2091,9 +2094,12 @@ def test_create_anthropic_model_list_response_carries_token_limits(): assert opus["max_tokens"] == 64000 assert "max_output_tokens" not in opus assert input_only["max_input_tokens"] == 8192 - assert "max_tokens" not in input_only - assert "max_input_tokens" not in unknown - assert "max_tokens" not in unknown + assert input_only["max_tokens"] is None + assert unknown["max_input_tokens"] is None + assert unknown["max_tokens"] is None + for entry in response["data"]: + assert "max_input_tokens" in entry + assert "max_tokens" in entry def test_create_anthropic_model_list_response_empty(): diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_models.py b/tests/test_litellm/proxy/proxy_server/test_routes_models.py index f18c5998b8c..2b126b1ea95 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_models.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_models.py @@ -15,6 +15,8 @@ import pytest import litellm from litellm.proxy import proxy_server +from litellm.proxy import utils as proxy_utils +from litellm.proxy.utils import create_model_info_response from .conftest import normalize # type: ignore[import-not-found] @@ -151,8 +153,38 @@ def test_anthropic_format_exposes_token_limits( assert claude["max_input_tokens"] == 200000 assert claude["max_tokens"] == 64000 assert "max_output_tokens" not in claude - assert "max_input_tokens" not in gpt_4 - assert "max_tokens" not in gpt_4 + assert gpt_4["max_input_tokens"] is None + assert gpt_4["max_tokens"] is None + + +@pytest.mark.parametrize("path", ["/v1/models", "/models"]) +def test_anthropic_format_carries_router_configured_token_limits(client, auth_as, patched_models, monkeypatch, path): + """Pins the whole resolution chain, not just the formatter: a deployment's + configured limits beat the cost map, and the configured output budget is what + lands on the Anthropic ``max_tokens``. All eight limits differ, so an entry + built from another entry's lookup shows up as the wrong numbers.""" + + def _configured(model_name): + return (300000, 32000) if model_name == "gpt-4" else (500000, 4096) + + def _cost_map_lookup(model_id): + max_input, max_output = (200000, 64000) if model_id == "gpt-4" else (100000, 8000) + return {"max_input_tokens": max_input, "max_output_tokens": max_output, "mode": "chat"} + + patched_models.get_configured_token_limits = MagicMock(side_effect=_configured) + + def _resolved(**kwargs): + return create_model_info_response(**kwargs, get_model_info=_cost_map_lookup) + + monkeypatch.setattr(proxy_utils, "create_model_info_response", _resolved) + + with auth_as(): + response = client.get(path, headers={"anthropic-version": "2023-06-01"}) + + assert response.status_code == 200 + gpt_4, claude = response.json()["data"] + assert (gpt_4["max_input_tokens"], gpt_4["max_tokens"]) == (300000, 32000) + assert (claude["max_input_tokens"], claude["max_tokens"]) == (500000, 4096) @pytest.mark.parametrize("path", ["/v1/models", "/models"])