fix(vertex_ai): resolve correct default max_tokens for versioned claude ids

two defects stacked on the vertex claude path so a request without
max_tokens was silently capped at 4096 output tokens

the fill in transform_request called AnthropicConfig.get_config directly
so get_max_tokens_for_model on VertexAIAnthropicConfig never ran, and the
bare versioned id like claude-haiku-4-5@20251001 is not a cost map key,
so the fill fell back to the 4096 default

now transform_request calls self.get_config and VertexAIAnthropicConfig
resolves the vertex_ai prefixed key first, then falls back to the base
lookup

also the two vertex_ai claude haiku 4 5 map entries said 8192 max output
while every sibling claude-haiku-4-5 key says 64000 and google documents
64000, the reporter verified 64000 live on vertex with a 200 and 15290
completion tokens, both entries and the runtime backup map now say 64000

Fixes #40363
This commit is contained in:
hktitof 2026-09-09 06:20:50 +00:00
parent e8e3172d7d
commit cccedd979e
5 changed files with 83 additions and 11 deletions

View file

@ -2,7 +2,7 @@ import json
import re
import time
from collections.abc import Callable, Mapping, Sequence
from types import MappingProxyType
from types import BuiltinFunctionType, FunctionType, MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NoReturn, cast
import httpx
@ -322,7 +322,26 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
@classmethod
def get_config(cls, *, model: str | None = None):
config: Final = super().get_config()
# defaults configured on the base class (litellm.AnthropicConfig(max_tokens=...)) live on
# AnthropicConfig itself and must keep reaching subclass requests (vertex/azure claude)
base: Final = { # mutable-ok: get_config returns a plain dict, same contract as the base impl
k: v
for k, v in AnthropicConfig.__dict__.items()
if not k.startswith("_")
and not isinstance(
v,
(
FunctionType,
BuiltinFunctionType,
classmethod,
staticmethod,
property,
),
)
and v is not None
and not callable(v)
}
config: Final = {**base, **super().get_config()} # mutable-ok: one-shot merge, same dict contract
# anthropic requires a default value for max_tokens
if config.get("max_tokens") is None:
@ -1985,7 +2004,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
optional_params["tools"] = tools
## Load Config
config: Final = litellm.AnthropicConfig.get_config(model=model)
config: Final = self.get_config(model=model)
for k, v in config.items():
if (
k not in optional_params

View file

@ -9,6 +9,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMed
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
from litellm.utils import get_max_tokens
from ....anthropic.chat.transformation import AnthropicConfig
from .output_params_utils import sanitize_vertex_anthropic_output_params
@ -58,6 +59,16 @@ class VertexAIAnthropicConfig(AnthropicConfig):
def should_strip_billing_metadata(self) -> bool:
return True
@staticmethod
def get_max_tokens_for_model(model: str | None = None) -> int:
if model is not None:
vertex_key: Final = f"vertex_ai/{model}"
if vertex_key in litellm.model_cost:
vertex_max: Final = get_max_tokens(vertex_key)
if vertex_max is not None:
return vertex_max
return AnthropicConfig.get_max_tokens_for_model(model)
def _add_context_management_beta_headers(self, beta_set: set, context_management: dict) -> None:
"""
Add context_management beta headers to the beta_set.

View file

@ -45891,8 +45891,8 @@
"input_cost_per_token": 1e-06,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -45916,8 +45916,8 @@
"input_cost_per_token": 1e-06,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 5e-06,
"regional_endpoint_uplift_multiplier": 1.1,

View file

@ -45891,8 +45891,8 @@
"input_cost_per_token": 1e-06,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -45916,8 +45916,8 @@
"input_cost_per_token": 1e-06,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 5e-06,
"regional_endpoint_uplift_multiplier": 1.1,

View file

@ -775,3 +775,45 @@ def test_vertex_ai_anthropic_tool_based_response_format_still_upgrades_legacy_th
assert "tools" in result_params
assert result_params["thinking"] == {"type": "adaptive"}
assert result_params["output_config"] == {"effort": "high"}
def test_vertex_ai_anthropic_versioned_model_default_max_tokens(local_model_cost_map):
config = VertexAIAnthropicConfig()
data = config.transform_request(
model="claude-haiku-4-5@20251001",
messages=[{"role": "user", "content": "hi"}],
optional_params={},
litellm_params={},
headers={},
)
assert data["max_tokens"] == 64000
unversioned = config.transform_request(
model="claude-haiku-4-5",
messages=[{"role": "user", "content": "hi"}],
optional_params={},
litellm_params={},
headers={},
)
assert unversioned["max_tokens"] == 64000
def test_vertex_ai_anthropic_base_config_defaults_reach_subclass(local_model_cost_map):
from litellm import AnthropicConfig
AnthropicConfig(max_tokens=123, temperature=0.5)
try:
data = VertexAIAnthropicConfig().transform_request(
model="claude-haiku-4-5@20251001",
messages=[{"role": "user", "content": "hi"}],
optional_params={},
litellm_params={},
headers={},
)
finally:
for attr in ("max_tokens", "temperature"):
if attr in AnthropicConfig.__dict__:
delattr(AnthropicConfig, attr)
assert data["max_tokens"] == 123
assert data["temperature"] == 0.5