mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(vertex_ai): resolve correct default max_tokens for versioned claude ids
two defects stacked on the vertex claude path so a request without max_tokens was silently capped at 4096 output tokens the fill in transform_request called AnthropicConfig.get_config directly so get_max_tokens_for_model on VertexAIAnthropicConfig never ran, and the bare versioned id like claude-haiku-4-5@20251001 is not a cost map key, so the fill fell back to the 4096 default now transform_request calls self.get_config and VertexAIAnthropicConfig resolves the vertex_ai prefixed key first, then falls back to the base lookup also the two vertex_ai claude haiku 4 5 map entries said 8192 max output while every sibling claude-haiku-4-5 key says 64000 and google documents 64000, the reporter verified 64000 live on vertex with a 200 and 15290 completion tokens, both entries and the runtime backup map now say 64000 Fixes #40363
This commit is contained in:
parent
e8e3172d7d
commit
cccedd979e
5 changed files with 83 additions and 11 deletions
|
|
@ -2,7 +2,7 @@ import json
|
|||
import re
|
||||
import time
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from types import BuiltinFunctionType, FunctionType, MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, NoReturn, cast
|
||||
|
||||
import httpx
|
||||
|
|
@ -322,7 +322,26 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
|
||||
@classmethod
|
||||
def get_config(cls, *, model: str | None = None):
|
||||
config: Final = super().get_config()
|
||||
# defaults configured on the base class (litellm.AnthropicConfig(max_tokens=...)) live on
|
||||
# AnthropicConfig itself and must keep reaching subclass requests (vertex/azure claude)
|
||||
base: Final = { # mutable-ok: get_config returns a plain dict, same contract as the base impl
|
||||
k: v
|
||||
for k, v in AnthropicConfig.__dict__.items()
|
||||
if not k.startswith("_")
|
||||
and not isinstance(
|
||||
v,
|
||||
(
|
||||
FunctionType,
|
||||
BuiltinFunctionType,
|
||||
classmethod,
|
||||
staticmethod,
|
||||
property,
|
||||
),
|
||||
)
|
||||
and v is not None
|
||||
and not callable(v)
|
||||
}
|
||||
config: Final = {**base, **super().get_config()} # mutable-ok: one-shot merge, same dict contract
|
||||
|
||||
# anthropic requires a default value for max_tokens
|
||||
if config.get("max_tokens") is None:
|
||||
|
|
@ -1985,7 +2004,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
optional_params["tools"] = tools
|
||||
|
||||
## Load Config
|
||||
config: Final = litellm.AnthropicConfig.get_config(model=model)
|
||||
config: Final = self.get_config(model=model)
|
||||
for k, v in config.items():
|
||||
if (
|
||||
k not in optional_params
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMed
|
|||
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import ModelResponse
|
||||
from litellm.utils import get_max_tokens
|
||||
|
||||
from ....anthropic.chat.transformation import AnthropicConfig
|
||||
from .output_params_utils import sanitize_vertex_anthropic_output_params
|
||||
|
|
@ -58,6 +59,16 @@ class VertexAIAnthropicConfig(AnthropicConfig):
|
|||
def should_strip_billing_metadata(self) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_max_tokens_for_model(model: str | None = None) -> int:
|
||||
if model is not None:
|
||||
vertex_key: Final = f"vertex_ai/{model}"
|
||||
if vertex_key in litellm.model_cost:
|
||||
vertex_max: Final = get_max_tokens(vertex_key)
|
||||
if vertex_max is not None:
|
||||
return vertex_max
|
||||
return AnthropicConfig.get_max_tokens_for_model(model)
|
||||
|
||||
def _add_context_management_beta_headers(self, beta_set: set, context_management: dict) -> None:
|
||||
"""
|
||||
Add context_management beta headers to the beta_set.
|
||||
|
|
|
|||
|
|
@ -45891,8 +45891,8 @@
|
|||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
"max_output_tokens": 64000,
|
||||
"max_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -45916,8 +45916,8 @@
|
|||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
"max_output_tokens": 64000,
|
||||
"max_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
|
|||
|
|
@ -45891,8 +45891,8 @@
|
|||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
"max_output_tokens": 64000,
|
||||
"max_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -45916,8 +45916,8 @@
|
|||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
"max_output_tokens": 64000,
|
||||
"max_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
|
|||
|
|
@ -775,3 +775,45 @@ def test_vertex_ai_anthropic_tool_based_response_format_still_upgrades_legacy_th
|
|||
assert "tools" in result_params
|
||||
assert result_params["thinking"] == {"type": "adaptive"}
|
||||
assert result_params["output_config"] == {"effort": "high"}
|
||||
|
||||
|
||||
def test_vertex_ai_anthropic_versioned_model_default_max_tokens(local_model_cost_map):
|
||||
config = VertexAIAnthropicConfig()
|
||||
data = config.transform_request(
|
||||
model="claude-haiku-4-5@20251001",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
assert data["max_tokens"] == 64000
|
||||
|
||||
unversioned = config.transform_request(
|
||||
model="claude-haiku-4-5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
assert unversioned["max_tokens"] == 64000
|
||||
|
||||
|
||||
def test_vertex_ai_anthropic_base_config_defaults_reach_subclass(local_model_cost_map):
|
||||
from litellm import AnthropicConfig
|
||||
|
||||
AnthropicConfig(max_tokens=123, temperature=0.5)
|
||||
try:
|
||||
data = VertexAIAnthropicConfig().transform_request(
|
||||
model="claude-haiku-4-5@20251001",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
finally:
|
||||
for attr in ("max_tokens", "temperature"):
|
||||
if attr in AnthropicConfig.__dict__:
|
||||
delattr(AnthropicConfig, attr)
|
||||
|
||||
assert data["max_tokens"] == 123
|
||||
assert data["temperature"] == 0.5
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue