mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(vertex_ai): resolve correct default max_tokens for versioned claude ids (#40376)
* fix(vertex_ai): resolve correct default max_tokens for versioned claude ids two defects stacked on the vertex claude path so a request without max_tokens was silently capped at 4096 output tokens the fill in transform_request called AnthropicConfig.get_config directly so get_max_tokens_for_model on VertexAIAnthropicConfig never ran, and the bare versioned id like claude-haiku-4-5@20251001 is not a cost map key, so the fill fell back to the 4096 default now transform_request calls self.get_config and VertexAIAnthropicConfig resolves the vertex_ai prefixed key first, then falls back to the base lookup also the two vertex_ai claude haiku 4 5 map entries said 8192 max output while every sibling claude-haiku-4-5 key says 64000 and google documents 64000, the reporter verified 64000 live on vertex with a 200 and 15290 completion tokens, both entries and the runtime backup map now say 64000 Fixes #40363 * chore(anthropic): drop redundant mutable-ok suppressions the type discipline gate flags these two comments as LIT013, both suppress nothing because neither line has an LIT001 violation. no behavior change * fix(vertex_ai): look up the vertex max tokens without the partially unknown cost map the membership check on litellm.model_cost is a reportUnknownMemberType the basedpyright delta gate counts, and it adds exactly one over the merge-base. get_max_tokens already raises for a model that is not mapped, so catch that and fall back to the parent, same as the parent does for its own lookup. the focused vertex anthropic tests still pass * fix(vertex_ai): guard the model cost map lookup with a pyright ignore the try/except form added a blind except that the strict ruff gate counts as BLE001, one over the merge-base. keep the membership check and suppress only the reportUnknownMemberType on the dynamically typed registry, the same pattern the complexity router already uses. ruff strict is identical to the base, ruff format and ruff check clean, focused vertex anthropic tests 42 passed * fix(vertex_ai): read the unversioned Claude row before the Vertex row for a missing max_tokens A Vertex @version id now reads the cost-map row of its id without the version before the vertex_ai row, so claude-sonnet-4-6@default keeps the 128000 default that Vertex accepts while versioned Haiku, Opus, and Sonnet 4.5 ids still get 64000. * fix(anthropic): keep an explicit DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS ahead of the unversioned and provider rows An operator who set DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS, for example to 4096 as the v1.109.0 notes suggest, got that value for every Claude id with no own cost-map row. The new unversioned and provider-row steps ranked above it, so vertex_ai/claude-haiku-4-5@20251001 went from 4096 to 64000 and claude-sonnet-4-6@default from 4096 to 128000 for them. An explicit value now comes right after the model's own row, so only deployments on the built-in fallback change * fix(anthropic): keep DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS as the last fallback after the unversioned and provider rows Reverts the previous commit. With DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS set to any value, versioned Vertex Claude ids ignored their model's row again: at 4096 a Haiku 4.5 reply still stopped at 4096 tokens, the symptom the linked issue reports, and at 128000 Vertex still answered 400. The env var is documented as the default for models missing from the cost map, and a versioned id whose model has a row is not missing, so it now gets the same cap as its unversioned id * fix(anthropic): read the unversioned max_tokens row directly and test the lookup on other routes The unversioned lookup went through get_max_tokens, which prints the provider banner a second time when the id has no row, so SDK calls on ids like claude-3-5-haiku@20241022 printed it twice. Reading the row directly gives the same max_tokens for every cost-map Claude id on every route. New tests drive the provider and unversioned rows on the Anthropic, Azure AI and Claude Platform routes * ci: restart the required checks after their held runs failed at startup --------- Co-authored-by: hktitof <hktitof@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
parent
8366e15be0
commit
74fd46a59f
5 changed files with 157 additions and 27 deletions
|
|
@ -37,6 +37,7 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
drops_unsupported_params,
|
||||
map_finish_reason,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
DEFS_MAX_INLINED_BYTES,
|
||||
invalid_type_message,
|
||||
|
|
@ -645,6 +646,22 @@ def tools_without_unsupported_tool_search(
|
|||
return [_without_deferral(tool) for tool in tools if not _is_openai_tool_search_tool(tool)]
|
||||
|
||||
|
||||
def mapped_max_output_tokens(cost_map_key: str) -> int | None:
|
||||
row: Final = object_mapping(litellm.model_cost.get(cost_map_key)) or {} # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # cost map rows are untyped JSON
|
||||
match row.get("max_output_tokens"):
|
||||
case int() as max_tokens:
|
||||
return max_tokens
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _own_max_output_tokens(model: str) -> int | None:
|
||||
try:
|
||||
return get_max_tokens(model)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
||||
"""
|
||||
Reference: https://docs.anthropic.com/claude/reference/messages_post
|
||||
|
|
@ -691,7 +708,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
)
|
||||
|
||||
@classmethod
|
||||
def get_config(cls, *, model: str | None = None, configured_max_output_tokens: int | None = None):
|
||||
def get_config(
|
||||
cls,
|
||||
*,
|
||||
model: str | None = None,
|
||||
configured_max_output_tokens: int | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
):
|
||||
config: Final = super().get_config()
|
||||
|
||||
# anthropic requires a default value for max_tokens
|
||||
|
|
@ -699,26 +722,32 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
config["max_tokens"] = (
|
||||
configured_max_output_tokens
|
||||
if configured_max_output_tokens is not None
|
||||
else cls.get_max_tokens_for_model(model)
|
||||
else cls.get_max_tokens_for_model(model, custom_llm_provider)
|
||||
)
|
||||
|
||||
return config
|
||||
|
||||
@staticmethod
|
||||
def get_max_tokens_for_model(model: str | None = None) -> int:
|
||||
def get_max_tokens_for_model(model: str | None = None, custom_llm_provider: str | None = None) -> int:
|
||||
"""
|
||||
Get the max output tokens for a given model.
|
||||
Falls back to DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS (configurable via env var) if model is not found.
|
||||
Get the max output tokens for a given model from its own cost-map row, then from its
|
||||
unversioned id's row (claude-sonnet-4-6@default reads claude-sonnet-4-6), else from the
|
||||
serving host's row (e.g. vertex_ai/claude-3-5-haiku@20241022).
|
||||
Falls back to DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS (configurable via env var) if no row exists.
|
||||
"""
|
||||
if model is None:
|
||||
return DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS
|
||||
try:
|
||||
max_tokens: Final = get_max_tokens(model)
|
||||
if max_tokens is None:
|
||||
return DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS
|
||||
return max_tokens
|
||||
except Exception:
|
||||
return DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS
|
||||
own_max_tokens: Final = _own_max_output_tokens(model)
|
||||
if own_max_tokens is not None:
|
||||
return own_max_tokens
|
||||
unversioned_model, version_separator, _ = model.partition("@")
|
||||
unversioned_max_tokens: Final = mapped_max_output_tokens(unversioned_model) if version_separator else None
|
||||
if unversioned_max_tokens is not None:
|
||||
return unversioned_max_tokens
|
||||
host_max_tokens: Final = (
|
||||
None if custom_llm_provider is None else mapped_max_output_tokens(f"{custom_llm_provider}/{model}")
|
||||
)
|
||||
return DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS if host_max_tokens is None else host_max_tokens
|
||||
|
||||
@staticmethod
|
||||
def convert_tool_use_to_openai_format(
|
||||
|
|
@ -2681,7 +2710,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
else None
|
||||
)
|
||||
config: Final[Mapping[str, object]] = MappingProxyType(
|
||||
litellm.AnthropicConfig.get_config(model=model, configured_max_output_tokens=configured_max_output_tokens)
|
||||
litellm.AnthropicConfig.get_config(
|
||||
model=model,
|
||||
configured_max_output_tokens=configured_max_output_tokens,
|
||||
custom_llm_provider=self._resolved_provider,
|
||||
)
|
||||
)
|
||||
for k, v in config.items():
|
||||
if (
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
|||
convert_url_to_base64,
|
||||
inline_every_remote_url,
|
||||
)
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig, mapped_max_output_tokens
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
OBJECT_MAPPING_ADAPTER,
|
||||
AnthropicModelInfo,
|
||||
|
|
@ -72,15 +72,6 @@ def _anthropic_config_max_tokens() -> int | None:
|
|||
return litellm.AnthropicConfig.max_tokens
|
||||
|
||||
|
||||
def _mapped_max_tokens(model: str) -> int | None:
|
||||
row: Final = object_mapping(litellm.model_cost.get(model)) or {} # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # cost map rows are untyped JSON
|
||||
match row.get("max_output_tokens"):
|
||||
case int() as max_tokens:
|
||||
return max_tokens
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
||||
"""
|
||||
Reference:
|
||||
|
|
@ -112,12 +103,20 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
|
||||
@classmethod
|
||||
def get_config(
|
||||
cls, *, model: str | None = None, configured_max_output_tokens: int | None = None
|
||||
cls,
|
||||
*,
|
||||
model: str | None = None,
|
||||
configured_max_output_tokens: int | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
) -> dict[str, object]: # mutable-ok: overrides BaseConfig's dict
|
||||
return {
|
||||
key: value
|
||||
for key, value in super()
|
||||
.get_config(model=model, configured_max_output_tokens=configured_max_output_tokens)
|
||||
.get_config(
|
||||
model=model,
|
||||
configured_max_output_tokens=configured_max_output_tokens,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
.items()
|
||||
if key != "max_tokens"
|
||||
}
|
||||
|
|
@ -326,7 +325,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
(
|
||||
max_tokens
|
||||
for max_tokens in map(
|
||||
_mapped_max_tokens, self._default_max_tokens_models(model, litellm_params)
|
||||
mapped_max_output_tokens, self._default_max_tokens_models(model, litellm_params)
|
||||
)
|
||||
if max_tokens is not None
|
||||
),
|
||||
|
|
|
|||
|
|
@ -398,7 +398,13 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
|
|||
return "databricks"
|
||||
|
||||
@classmethod
|
||||
def get_config(cls, *, model: str | None = None, configured_max_output_tokens: int | None = None):
|
||||
def get_config(
|
||||
cls,
|
||||
*,
|
||||
model: str | None = None,
|
||||
configured_max_output_tokens: int | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
):
|
||||
return super().get_config()
|
||||
|
||||
def get_json_schema_from_pydantic_object(
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import litellm
|
|||
from litellm.llms.anthropic.messages.native import handler as python_messages
|
||||
from litellm.constants import (
|
||||
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS,
|
||||
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS,
|
||||
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
|
||||
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
|
||||
DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET,
|
||||
|
|
@ -2713,6 +2714,48 @@ def test_transform_request_uses_deployment_max_output_tokens_before_catalog() ->
|
|||
assert requested["max_tokens"] == 654
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"config_class",
|
||||
(AnthropicConfig, AzureAnthropicConfig, BedrockClaudePlatformConfig),
|
||||
ids=("anthropic", "azure_ai", "bedrock_claude_platform"),
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("cost_map_rows", "expected_max_tokens"),
|
||||
(
|
||||
pytest.param({"{provider}/claude-unit-test@20990101": 1234}, 1234, id="provider-row-fills-unmapped-id"),
|
||||
pytest.param(
|
||||
{"claude-unit-test": 4321, "{provider}/claude-unit-test@20990101": 1234},
|
||||
4321,
|
||||
id="unversioned-row-before-provider-row",
|
||||
),
|
||||
pytest.param({}, DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS, id="no-row-keeps-default"),
|
||||
),
|
||||
)
|
||||
def test_default_max_tokens_reads_unversioned_and_provider_rows_on_every_route(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
config_class: type[AnthropicConfig],
|
||||
cost_map_rows: Mapping[str, int],
|
||||
expected_max_tokens: int,
|
||||
) -> None:
|
||||
config: Final = config_class()
|
||||
for key, max_output_tokens in cost_map_rows.items():
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
key.format(provider=config.custom_llm_provider),
|
||||
{"max_output_tokens": max_output_tokens},
|
||||
)
|
||||
|
||||
result: Final = config.transform_request(
|
||||
model="claude-unit-test@20990101",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert result["max_tokens"] == expected_max_tokens
|
||||
|
||||
|
||||
def test_get_config_without_model_uses_fallback():
|
||||
"""
|
||||
Test that get_config without model parameter uses 128000 fallback.
|
||||
|
|
|
|||
|
|
@ -1506,3 +1506,52 @@ def test_beta_detection_preserves_unrecognized_thinking(thinking: object) -> Non
|
|||
|
||||
assert result["thinking"] == thinking
|
||||
assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER not in headers.get("anthropic-beta", "").split(",")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cost_map_rows", "litellm_params", "expected_max_tokens"),
|
||||
(
|
||||
pytest.param({"vertex_ai/claude-unit-test@20990101": 1234}, {}, 1234, id="vertex-row-fills-unmapped-bare-id"),
|
||||
pytest.param(
|
||||
{"claude-unit-test@20990101": 4321, "vertex_ai/claude-unit-test@20990101": 1234},
|
||||
{},
|
||||
4321,
|
||||
id="own-row-first",
|
||||
),
|
||||
pytest.param(
|
||||
{"claude-unit-test": 4321, "vertex_ai/claude-unit-test@20990101": 1234},
|
||||
{},
|
||||
4321,
|
||||
id="unversioned-row-before-vertex-row",
|
||||
),
|
||||
pytest.param(
|
||||
{"vertex_ai/claude-unit-test@20990101": 1234},
|
||||
{"model_info": {"max_output_tokens": 500}},
|
||||
500,
|
||||
id="deployment-cap-first",
|
||||
),
|
||||
pytest.param({}, {}, None, id="no-row-keeps-default"),
|
||||
),
|
||||
)
|
||||
def test_vertex_ai_anthropic_default_max_tokens_reads_vertex_row(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
cost_map_rows: Mapping[str, int],
|
||||
litellm_params: Mapping[str, object],
|
||||
expected_max_tokens: int | None,
|
||||
) -> None:
|
||||
from litellm.constants import DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS
|
||||
|
||||
for key, max_output_tokens in cost_map_rows.items():
|
||||
monkeypatch.setitem(litellm.model_cost, key, {"max_output_tokens": max_output_tokens})
|
||||
|
||||
result: Final = VertexAIAnthropicConfig().transform_request(
|
||||
model="claude-unit-test@20990101",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={},
|
||||
litellm_params=dict(litellm_params),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert result["max_tokens"] == (
|
||||
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS if expected_max_tokens is None else expected_max_tokens
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue