mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(bedrock): make supports_output_config gate actually consult cost map
The gate at three call sites was calling _supports_factory with custom_llm_provider=None, which relies on get_llm_provider inferring the provider from the model string. For the invoke path, the model still carries an 'invoke/' routing prefix (e.g. 'invoke/us.anthropic.claude-opus-4-6-v1') that is not a known provider, so inference raises BadRequestError, _supports_factory swallows it and returns False, and the user's output_config.effort gets silently dropped before the Bedrock request. Strip the routing prefix with the existing strip_bedrock_routing_prefix helper and pass custom_llm_provider='bedrock' explicitly so the declarative 'supports_output_config' flag in model_prices_and_context_window.json is the actual source of truth. Also adds a regression test that exercises the full 'invoke/us.anthropic.claude-opus-4-6-v1' path and asserts output_config survives.
This commit is contained in:
parent
d6bbd91362
commit
a22611c415
4 changed files with 71 additions and 3 deletions
|
|
@ -74,6 +74,7 @@ from ..common_utils import (
|
|||
get_anthropic_beta_from_headers,
|
||||
get_bedrock_tool_name,
|
||||
is_claude_4_5_on_bedrock,
|
||||
strip_bedrock_routing_prefix,
|
||||
)
|
||||
|
||||
# Computer use tool prefixes supported by Bedrock
|
||||
|
|
@ -1202,8 +1203,16 @@ class AmazonConverseConfig(BaseConfig):
|
|||
output_config: Optional[OutputConfigBlock] = inference_params.pop(
|
||||
"outputConfig", None
|
||||
)
|
||||
# Strip routing prefixes (e.g. ``converse/``) and pass the provider
|
||||
# explicitly so the declarative ``supports_output_config`` flag in
|
||||
# ``model_prices_and_context_window.json`` is actually consulted.
|
||||
# Passing ``custom_llm_provider=None`` would make ``_supports_factory``
|
||||
# silently return False for any model whose name doesn't resolve to a
|
||||
# known provider prefix on its own.
|
||||
if not _supports_factory(
|
||||
model=model, custom_llm_provider=None, key="supports_output_config"
|
||||
model=strip_bedrock_routing_prefix(model),
|
||||
custom_llm_provider="bedrock",
|
||||
key="supports_output_config",
|
||||
):
|
||||
inference_params.pop("output_config", None)
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
get_anthropic_beta_from_headers,
|
||||
normalize_tool_input_schema_types_for_bedrock_invoke,
|
||||
remove_custom_field_from_tools,
|
||||
strip_bedrock_routing_prefix,
|
||||
)
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
|
@ -170,8 +171,16 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
anthropic_request.pop("model", None)
|
||||
anthropic_request.pop("stream", None)
|
||||
anthropic_request.pop("output_format", None)
|
||||
# ``model`` reaches this transformation with an ``invoke/`` routing
|
||||
# prefix (e.g. ``invoke/us.anthropic.claude-opus-4-6-v1``) which is
|
||||
# not a valid provider and makes ``_supports_factory`` silently
|
||||
# return False. Strip the routing prefix and pass the provider
|
||||
# explicitly so the declarative ``supports_output_config`` flag in
|
||||
# ``model_prices_and_context_window.json`` is actually consulted.
|
||||
if not _supports_factory(
|
||||
model=model, custom_llm_provider=None, key="supports_output_config"
|
||||
model=strip_bedrock_routing_prefix(model),
|
||||
custom_llm_provider="bedrock",
|
||||
key="supports_output_config",
|
||||
):
|
||||
anthropic_request.pop("output_config", None)
|
||||
if "anthropic_version" not in anthropic_request:
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
is_claude_4_5_on_bedrock,
|
||||
normalize_tool_input_schema_types_for_bedrock_invoke,
|
||||
remove_custom_field_from_tools,
|
||||
strip_bedrock_routing_prefix,
|
||||
)
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
|
@ -504,8 +505,15 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
# 5b. Bedrock Invoke supports output_config (effort) for Claude 4.6+ models,
|
||||
# but older models do not — strip it to avoid request rejection.
|
||||
# Ref: https://github.com/BerriAI/litellm/issues/22797
|
||||
# ``model`` still carries the ``invoke/`` routing prefix here, which
|
||||
# is not a valid provider and makes ``_supports_factory`` silently
|
||||
# return False. Strip the prefix and pass the provider explicitly so
|
||||
# the declarative ``supports_output_config`` flag in the cost map
|
||||
# is actually consulted.
|
||||
if not _supports_factory(
|
||||
model=model, custom_llm_provider=None, key="supports_output_config"
|
||||
model=strip_bedrock_routing_prefix(model),
|
||||
custom_llm_provider="bedrock",
|
||||
key="supports_output_config",
|
||||
):
|
||||
anthropic_messages_request.pop("output_config", None)
|
||||
|
||||
|
|
|
|||
|
|
@ -440,6 +440,48 @@ def test_output_config_removed_from_bedrock_chat_invoke_request():
|
|||
assert result["max_tokens"] == 100
|
||||
|
||||
|
||||
def test_output_config_preserved_for_claude_4_6_with_invoke_prefix():
|
||||
"""
|
||||
Regression test: when the proxy routes ``bedrock/invoke/us.anthropic.
|
||||
claude-opus-4-6-v1``, the transformation sees ``model=invoke/us.
|
||||
anthropic.claude-opus-4-6-v1``. The ``invoke/`` prefix is not a valid
|
||||
provider, so a naive ``_supports_factory(model, custom_llm_provider=None)``
|
||||
call silently returns False and drops the user's ``output_config``.
|
||||
|
||||
This test forces the local cost map (so ``supports_output_config: true``
|
||||
from the bundled JSON is visible) and asserts the Anthropic-style
|
||||
``output_config.effort`` is forwarded to Bedrock.
|
||||
"""
|
||||
import os
|
||||
|
||||
import litellm
|
||||
|
||||
old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP")
|
||||
old_cost = litellm.model_cost
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
try:
|
||||
config = AmazonAnthropicClaudeConfig()
|
||||
result = config.transform_request(
|
||||
model="invoke/us.anthropic.claude-opus-4-6-v1",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
optional_params={
|
||||
"max_tokens": 64,
|
||||
"output_config": {"effort": "medium"},
|
||||
},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert result.get("output_config") == {"effort": "medium"}
|
||||
finally:
|
||||
litellm.model_cost = old_cost
|
||||
if old_env is None:
|
||||
os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None)
|
||||
else:
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env
|
||||
|
||||
|
||||
def test_output_format_removed_from_bedrock_invoke_request():
|
||||
"""
|
||||
Test that output_format parameter is removed from Bedrock Invoke requests.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue