diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 1d24cfab131..204efb84d08 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -37,6 +37,7 @@ from litellm.litellm_core_utils.core_helpers import ( drops_unsupported_params, map_finish_reason, ) +from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( DEFS_MAX_INLINED_BYTES, invalid_type_message, @@ -645,6 +646,22 @@ def tools_without_unsupported_tool_search( return [_without_deferral(tool) for tool in tools if not _is_openai_tool_search_tool(tool)] +def mapped_max_output_tokens(cost_map_key: str) -> int | None: + row: Final = object_mapping(litellm.model_cost.get(cost_map_key)) or {} # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # cost map rows are untyped JSON + match row.get("max_output_tokens"): + case int() as max_tokens: + return max_tokens + case _: + return None + + +def _own_max_output_tokens(model: str) -> int | None: + try: + return get_max_tokens(model) + except Exception: + return None + + class AnthropicConfig(AnthropicModelInfo, BaseConfig): """ Reference: https://docs.anthropic.com/claude/reference/messages_post @@ -691,7 +708,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) @classmethod - def get_config(cls, *, model: str | None = None, configured_max_output_tokens: int | None = None): + def get_config( + cls, + *, + model: str | None = None, + configured_max_output_tokens: int | None = None, + custom_llm_provider: str | None = None, + ): config: Final = super().get_config() # anthropic requires a default value for max_tokens @@ -699,26 +722,32 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): config["max_tokens"] = ( configured_max_output_tokens if configured_max_output_tokens is not None - else cls.get_max_tokens_for_model(model) + else cls.get_max_tokens_for_model(model, custom_llm_provider) ) return config @staticmethod - def get_max_tokens_for_model(model: str | None = None) -> int: + def get_max_tokens_for_model(model: str | None = None, custom_llm_provider: str | None = None) -> int: """ - Get the max output tokens for a given model. - Falls back to DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS (configurable via env var) if model is not found. + Get the max output tokens for a given model from its own cost-map row, then from its + unversioned id's row (claude-sonnet-4-6@default reads claude-sonnet-4-6), else from the + serving host's row (e.g. vertex_ai/claude-3-5-haiku@20241022). + Falls back to DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS (configurable via env var) if no row exists. """ if model is None: return DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS - try: - max_tokens: Final = get_max_tokens(model) - if max_tokens is None: - return DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS - return max_tokens - except Exception: - return DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS + own_max_tokens: Final = _own_max_output_tokens(model) + if own_max_tokens is not None: + return own_max_tokens + unversioned_model, version_separator, _ = model.partition("@") + unversioned_max_tokens: Final = mapped_max_output_tokens(unversioned_model) if version_separator else None + if unversioned_max_tokens is not None: + return unversioned_max_tokens + host_max_tokens: Final = ( + None if custom_llm_provider is None else mapped_max_output_tokens(f"{custom_llm_provider}/{model}") + ) + return DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS if host_max_tokens is None else host_max_tokens @staticmethod def convert_tool_use_to_openai_format( @@ -2681,7 +2710,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): else None ) config: Final[Mapping[str, object]] = MappingProxyType( - litellm.AnthropicConfig.get_config(model=model, configured_max_output_tokens=configured_max_output_tokens) + litellm.AnthropicConfig.get_config( + model=model, + configured_max_output_tokens=configured_max_output_tokens, + custom_llm_provider=self._resolved_provider, + ) ) for k, v in config.items(): if ( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py index 6e0cdbcce08..48c6b8d8841 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py @@ -18,7 +18,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( convert_url_to_base64, inline_every_remote_url, ) -from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.anthropic.chat.transformation import AnthropicConfig, mapped_max_output_tokens from litellm.llms.anthropic.common_utils import ( OBJECT_MAPPING_ADAPTER, AnthropicModelInfo, @@ -72,15 +72,6 @@ def _anthropic_config_max_tokens() -> int | None: return litellm.AnthropicConfig.max_tokens -def _mapped_max_tokens(model: str) -> int | None: - row: Final = object_mapping(litellm.model_cost.get(model)) or {} # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # cost map rows are untyped JSON - match row.get("max_output_tokens"): - case int() as max_tokens: - return max_tokens - case _: - return None - - class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): """ Reference: @@ -112,12 +103,20 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): @classmethod def get_config( - cls, *, model: str | None = None, configured_max_output_tokens: int | None = None + cls, + *, + model: str | None = None, + configured_max_output_tokens: int | None = None, + custom_llm_provider: str | None = None, ) -> dict[str, object]: # mutable-ok: overrides BaseConfig's dict return { key: value for key, value in super() - .get_config(model=model, configured_max_output_tokens=configured_max_output_tokens) + .get_config( + model=model, + configured_max_output_tokens=configured_max_output_tokens, + custom_llm_provider=custom_llm_provider, + ) .items() if key != "max_tokens" } @@ -326,7 +325,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): ( max_tokens for max_tokens in map( - _mapped_max_tokens, self._default_max_tokens_models(model, litellm_params) + mapped_max_output_tokens, self._default_max_tokens_models(model, litellm_params) ) if max_tokens is not None ), diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 7917a74247e..61548aab788 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -398,7 +398,13 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): return "databricks" @classmethod - def get_config(cls, *, model: str | None = None, configured_max_output_tokens: int | None = None): + def get_config( + cls, + *, + model: str | None = None, + configured_max_output_tokens: int | None = None, + custom_llm_provider: str | None = None, + ): return super().get_config() def get_json_schema_from_pydantic_object( diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index f434b1f4d9f..5dedbb4d3a8 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -18,6 +18,7 @@ import litellm from litellm.llms.anthropic.messages.native import handler as python_messages from litellm.constants import ( ANTHROPIC_MIN_THINKING_BUDGET_TOKENS, + DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS, DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET, @@ -2713,6 +2714,48 @@ def test_transform_request_uses_deployment_max_output_tokens_before_catalog() -> assert requested["max_tokens"] == 654 +@pytest.mark.parametrize( + "config_class", + (AnthropicConfig, AzureAnthropicConfig, BedrockClaudePlatformConfig), + ids=("anthropic", "azure_ai", "bedrock_claude_platform"), +) +@pytest.mark.parametrize( + ("cost_map_rows", "expected_max_tokens"), + ( + pytest.param({"{provider}/claude-unit-test@20990101": 1234}, 1234, id="provider-row-fills-unmapped-id"), + pytest.param( + {"claude-unit-test": 4321, "{provider}/claude-unit-test@20990101": 1234}, + 4321, + id="unversioned-row-before-provider-row", + ), + pytest.param({}, DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS, id="no-row-keeps-default"), + ), +) +def test_default_max_tokens_reads_unversioned_and_provider_rows_on_every_route( + monkeypatch: pytest.MonkeyPatch, + config_class: type[AnthropicConfig], + cost_map_rows: Mapping[str, int], + expected_max_tokens: int, +) -> None: + config: Final = config_class() + for key, max_output_tokens in cost_map_rows.items(): + monkeypatch.setitem( + litellm.model_cost, + key.format(provider=config.custom_llm_provider), + {"max_output_tokens": max_output_tokens}, + ) + + result: Final = config.transform_request( + model="claude-unit-test@20990101", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert result["max_tokens"] == expected_max_tokens + + def test_get_config_without_model_uses_fallback(): """ Test that get_config without model parameter uses 128000 fallback. diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index 2be6794186e..70abcfc35d8 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -1506,3 +1506,52 @@ def test_beta_detection_preserves_unrecognized_thinking(thinking: object) -> Non assert result["thinking"] == thinking assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER not in headers.get("anthropic-beta", "").split(",") + + +@pytest.mark.parametrize( + ("cost_map_rows", "litellm_params", "expected_max_tokens"), + ( + pytest.param({"vertex_ai/claude-unit-test@20990101": 1234}, {}, 1234, id="vertex-row-fills-unmapped-bare-id"), + pytest.param( + {"claude-unit-test@20990101": 4321, "vertex_ai/claude-unit-test@20990101": 1234}, + {}, + 4321, + id="own-row-first", + ), + pytest.param( + {"claude-unit-test": 4321, "vertex_ai/claude-unit-test@20990101": 1234}, + {}, + 4321, + id="unversioned-row-before-vertex-row", + ), + pytest.param( + {"vertex_ai/claude-unit-test@20990101": 1234}, + {"model_info": {"max_output_tokens": 500}}, + 500, + id="deployment-cap-first", + ), + pytest.param({}, {}, None, id="no-row-keeps-default"), + ), +) +def test_vertex_ai_anthropic_default_max_tokens_reads_vertex_row( + monkeypatch: pytest.MonkeyPatch, + cost_map_rows: Mapping[str, int], + litellm_params: Mapping[str, object], + expected_max_tokens: int | None, +) -> None: + from litellm.constants import DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS + + for key, max_output_tokens in cost_map_rows.items(): + monkeypatch.setitem(litellm.model_cost, key, {"max_output_tokens": max_output_tokens}) + + result: Final = VertexAIAnthropicConfig().transform_request( + model="claude-unit-test@20990101", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params=dict(litellm_params), + headers={}, + ) + + assert result["max_tokens"] == ( + DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS if expected_max_tokens is None else expected_max_tokens + )