From 73b0c54bcc5de4e500cbc52a1215d5de689c0778 Mon Sep 17 00:00:00 2001 From: hktitof Date: Wed, 9 Sep 2026 06:20:50 +0000 Subject: [PATCH 1/4] fix(vertex_ai): resolve correct default max_tokens for versioned claude ids two defects stacked on the vertex claude path so a request without max_tokens was silently capped at 4096 output tokens the fill in transform_request called AnthropicConfig.get_config directly so get_max_tokens_for_model on VertexAIAnthropicConfig never ran, and the bare versioned id like claude-haiku-4-5@20251001 is not a cost map key, so the fill fell back to the 4096 default now transform_request calls self.get_config and VertexAIAnthropicConfig resolves the vertex_ai prefixed key first, then falls back to the base lookup also the two vertex_ai claude haiku 4 5 map entries said 8192 max output while every sibling claude-haiku-4-5 key says 64000 and google documents 64000, the reporter verified 64000 live on vertex with a 200 and 15290 completion tokens, both entries and the runtime backup map now say 64000 Fixes #40363 --- litellm/llms/anthropic/chat/transformation.py | 25 +++++++++-- .../anthropic/transformation.py | 11 +++++ ...partner_models_anthropic_transformation.py | 41 +++++++++++++++++++ 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 38af6e7998b..8c02fdd569f 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -3,7 +3,7 @@ import json import re import time from collections.abc import Callable, Mapping, Sequence -from types import MappingProxyType +from types import BuiltinFunctionType, FunctionType, MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, cast import httpx @@ -490,7 +490,26 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): @classmethod def get_config(cls, *, model: str | None = None): - config: Final = super().get_config() + # defaults configured on the base class (litellm.AnthropicConfig(max_tokens=...)) live on + # AnthropicConfig itself and must keep reaching subclass requests (vertex/azure claude) + base: Final = { # mutable-ok: get_config returns a plain dict, same contract as the base impl + k: v + for k, v in AnthropicConfig.__dict__.items() + if not k.startswith("_") + and not isinstance( + v, + ( + FunctionType, + BuiltinFunctionType, + classmethod, + staticmethod, + property, + ), + ) + and v is not None + and not callable(v) + } + config: Final = {**base, **super().get_config()} # mutable-ok: one-shot merge, same dict contract # anthropic requires a default value for max_tokens if config.get("max_tokens") is None: @@ -2379,7 +2398,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): optional_params["tools"] = tools ## Load Config - config: Final = litellm.AnthropicConfig.get_config(model=model) + config: Final = self.get_config(model=model) for k, v in config.items(): if ( k not in optional_params diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 29253359003..9b6789a07d4 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -11,6 +11,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMed from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse +from litellm.utils import get_max_tokens from ....anthropic.chat.transformation import AnthropicConfig from .output_params_utils import sanitize_vertex_anthropic_output_params @@ -62,6 +63,16 @@ class VertexAIAnthropicConfig(AnthropicConfig): def should_strip_billing_metadata(self) -> bool: return True + @staticmethod + def get_max_tokens_for_model(model: str | None = None) -> int: + if model is not None: + vertex_key: Final = f"vertex_ai/{model}" + if vertex_key in litellm.model_cost: + vertex_max: Final = get_max_tokens(vertex_key) + if vertex_max is not None: + return vertex_max + return AnthropicConfig.get_max_tokens_for_model(model) + def _add_context_management_beta_headers(self, beta_set: set, context_management: dict) -> None: """ Add context_management beta headers to the beta_set. diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index ace24c8a3f4..dcaa3d3b907 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -1147,3 +1147,44 @@ def test_refusal_stop_details_kept_through_vertex_anthropic_transform(): provider_fields: Final = result.choices[0].message.provider_specific_fields assert provider_fields["stop_reason"] == "refusal" assert provider_fields["stop_details"] == ANTHROPIC_DOCS_REFUSAL_MESSAGE["stop_details"] + +def test_vertex_ai_anthropic_versioned_model_default_max_tokens(local_model_cost_map): + config = VertexAIAnthropicConfig() + data = config.transform_request( + model="claude-haiku-4-5@20251001", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert data["max_tokens"] == 64000 + + unversioned = config.transform_request( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert unversioned["max_tokens"] == 64000 + + +def test_vertex_ai_anthropic_base_config_defaults_reach_subclass(local_model_cost_map): + from litellm import AnthropicConfig + + AnthropicConfig(max_tokens=123, temperature=0.5) + try: + data = VertexAIAnthropicConfig().transform_request( + model="claude-haiku-4-5@20251001", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + finally: + for attr in ("max_tokens", "temperature"): + if attr in AnthropicConfig.__dict__: + delattr(AnthropicConfig, attr) + + assert data["max_tokens"] == 123 + assert data["temperature"] == 0.5 From 6c3884c6c9ce083638ca35bed955b5383fb0c8e2 Mon Sep 17 00:00:00 2001 From: Abdel <62770500+hktitof@users.noreply.github.com> Date: Sun, 18 Oct 2026 18:48:41 +0000 Subject: [PATCH 2/4] chore(anthropic): drop redundant mutable-ok suppressions the type discipline gate flags these two comments as LIT013, both suppress nothing because neither line has an LIT001 violation. no behavior change --- litellm/llms/anthropic/chat/transformation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 8c02fdd569f..707c5465b0f 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -492,7 +492,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): def get_config(cls, *, model: str | None = None): # defaults configured on the base class (litellm.AnthropicConfig(max_tokens=...)) live on # AnthropicConfig itself and must keep reaching subclass requests (vertex/azure claude) - base: Final = { # mutable-ok: get_config returns a plain dict, same contract as the base impl + base: Final = { k: v for k, v in AnthropicConfig.__dict__.items() if not k.startswith("_") @@ -509,7 +509,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): and v is not None and not callable(v) } - config: Final = {**base, **super().get_config()} # mutable-ok: one-shot merge, same dict contract + config: Final = {**base, **super().get_config()} # anthropic requires a default value for max_tokens if config.get("max_tokens") is None: From c88718915f5677a7c3c110b67e75026f1ef2f7d6 Mon Sep 17 00:00:00 2001 From: Abdel <62770500+hktitof@users.noreply.github.com> Date: Sun, 18 Oct 2026 19:07:13 +0000 Subject: [PATCH 3/4] fix(vertex_ai): look up the vertex max tokens without the partially unknown cost map the membership check on litellm.model_cost is a reportUnknownMemberType the basedpyright delta gate counts, and it adds exactly one over the merge-base. get_max_tokens already raises for a model that is not mapped, so catch that and fall back to the parent, same as the parent does for its own lookup. the focused vertex anthropic tests still pass --- .../vertex_ai_partner_models/anthropic/transformation.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 9b6789a07d4..77e4e31465d 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -66,11 +66,12 @@ class VertexAIAnthropicConfig(AnthropicConfig): @staticmethod def get_max_tokens_for_model(model: str | None = None) -> int: if model is not None: - vertex_key: Final = f"vertex_ai/{model}" - if vertex_key in litellm.model_cost: - vertex_max: Final = get_max_tokens(vertex_key) + try: + vertex_max: Final = get_max_tokens(f"vertex_ai/{model}") if vertex_max is not None: return vertex_max + except Exception: + return AnthropicConfig.get_max_tokens_for_model(model) return AnthropicConfig.get_max_tokens_for_model(model) def _add_context_management_beta_headers(self, beta_set: set, context_management: dict) -> None: From 757f7991bfeac99eec95234d14cb8b2691f3fcf6 Mon Sep 17 00:00:00 2001 From: Abdel <62770500+hktitof@users.noreply.github.com> Date: Sun, 18 Oct 2026 19:13:11 +0000 Subject: [PATCH 4/4] fix(vertex_ai): guard the model cost map lookup with a pyright ignore the try/except form added a blind except that the strict ruff gate counts as BLE001, one over the merge-base. keep the membership check and suppress only the reportUnknownMemberType on the dynamically typed registry, the same pattern the complexity router already uses. ruff strict is identical to the base, ruff format and ruff check clean, focused vertex anthropic tests 42 passed --- .../vertex_ai_partner_models/anthropic/transformation.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 77e4e31465d..fde1dd18752 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -66,12 +66,11 @@ class VertexAIAnthropicConfig(AnthropicConfig): @staticmethod def get_max_tokens_for_model(model: str | None = None) -> int: if model is not None: - try: - vertex_max: Final = get_max_tokens(f"vertex_ai/{model}") + vertex_key: Final = f"vertex_ai/{model}" + if vertex_key in litellm.model_cost: # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed + vertex_max: Final = get_max_tokens(vertex_key) if vertex_max is not None: return vertex_max - except Exception: - return AnthropicConfig.get_max_tokens_for_model(model) return AnthropicConfig.get_max_tokens_for_model(model) def _add_context_management_beta_headers(self, beta_set: set, context_management: dict) -> None: