From cccedd979e8a59d63f0fc39c477f3c8f0a2143f3 Mon Sep 17 00:00:00 2001 From: hktitof Date: Wed, 9 Sep 2026 06:20:50 +0000 Subject: [PATCH] fix(vertex_ai): resolve correct default max_tokens for versioned claude ids two defects stacked on the vertex claude path so a request without max_tokens was silently capped at 4096 output tokens the fill in transform_request called AnthropicConfig.get_config directly so get_max_tokens_for_model on VertexAIAnthropicConfig never ran, and the bare versioned id like claude-haiku-4-5@20251001 is not a cost map key, so the fill fell back to the 4096 default now transform_request calls self.get_config and VertexAIAnthropicConfig resolves the vertex_ai prefixed key first, then falls back to the base lookup also the two vertex_ai claude haiku 4 5 map entries said 8192 max output while every sibling claude-haiku-4-5 key says 64000 and google documents 64000, the reporter verified 64000 live on vertex with a 200 and 15290 completion tokens, both entries and the runtime backup map now say 64000 Fixes #40363 --- litellm/llms/anthropic/chat/transformation.py | 25 +++++++++-- .../anthropic/transformation.py | 11 +++++ ...odel_prices_and_context_window_backup.json | 8 ++-- model_prices_and_context_window.json | 8 ++-- ...partner_models_anthropic_transformation.py | 42 +++++++++++++++++++ 5 files changed, 83 insertions(+), 11 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 5463f1862ad..deb1c864b55 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -2,7 +2,7 @@ import json import re import time from collections.abc import Callable, Mapping, Sequence -from types import MappingProxyType +from types import BuiltinFunctionType, FunctionType, MappingProxyType from typing import TYPE_CHECKING, Any, Final, NoReturn, cast import httpx @@ -322,7 +322,26 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): @classmethod def get_config(cls, *, model: str | None = None): - config: Final = super().get_config() + # defaults configured on the base class (litellm.AnthropicConfig(max_tokens=...)) live on + # AnthropicConfig itself and must keep reaching subclass requests (vertex/azure claude) + base: Final = { # mutable-ok: get_config returns a plain dict, same contract as the base impl + k: v + for k, v in AnthropicConfig.__dict__.items() + if not k.startswith("_") + and not isinstance( + v, + ( + FunctionType, + BuiltinFunctionType, + classmethod, + staticmethod, + property, + ), + ) + and v is not None + and not callable(v) + } + config: Final = {**base, **super().get_config()} # mutable-ok: one-shot merge, same dict contract # anthropic requires a default value for max_tokens if config.get("max_tokens") is None: @@ -1985,7 +2004,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): optional_params["tools"] = tools ## Load Config - config: Final = litellm.AnthropicConfig.get_config(model=model) + config: Final = self.get_config(model=model) for k, v in config.items(): if ( k not in optional_params diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 508f68b3eca..2a999c620c6 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -9,6 +9,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMed from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse +from litellm.utils import get_max_tokens from ....anthropic.chat.transformation import AnthropicConfig from .output_params_utils import sanitize_vertex_anthropic_output_params @@ -58,6 +59,16 @@ class VertexAIAnthropicConfig(AnthropicConfig): def should_strip_billing_metadata(self) -> bool: return True + @staticmethod + def get_max_tokens_for_model(model: str | None = None) -> int: + if model is not None: + vertex_key: Final = f"vertex_ai/{model}" + if vertex_key in litellm.model_cost: + vertex_max: Final = get_max_tokens(vertex_key) + if vertex_max is not None: + return vertex_max + return AnthropicConfig.get_max_tokens_for_model(model) + def _add_context_management_beta_headers(self, beta_set: set, context_management: dict) -> None: """ Add context_management beta headers to the beta_set. diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 54ebdc85be9..37976e8b64d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45891,8 +45891,8 @@ "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -45916,8 +45916,8 @@ "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "regional_endpoint_uplift_multiplier": 1.1, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 54ebdc85be9..37976e8b64d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45891,8 +45891,8 @@ "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -45916,8 +45916,8 @@ "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "regional_endpoint_uplift_multiplier": 1.1, diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index a57672cfbfb..9546c07dcf8 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -775,3 +775,45 @@ def test_vertex_ai_anthropic_tool_based_response_format_still_upgrades_legacy_th assert "tools" in result_params assert result_params["thinking"] == {"type": "adaptive"} assert result_params["output_config"] == {"effort": "high"} + + +def test_vertex_ai_anthropic_versioned_model_default_max_tokens(local_model_cost_map): + config = VertexAIAnthropicConfig() + data = config.transform_request( + model="claude-haiku-4-5@20251001", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert data["max_tokens"] == 64000 + + unversioned = config.transform_request( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert unversioned["max_tokens"] == 64000 + + +def test_vertex_ai_anthropic_base_config_defaults_reach_subclass(local_model_cost_map): + from litellm import AnthropicConfig + + AnthropicConfig(max_tokens=123, temperature=0.5) + try: + data = VertexAIAnthropicConfig().transform_request( + model="claude-haiku-4-5@20251001", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + finally: + for attr in ("max_tokens", "temperature"): + if attr in AnthropicConfig.__dict__: + delattr(AnthropicConfig, attr) + + assert data["max_tokens"] == 123 + assert data["temperature"] == 0.5