scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

fix(vertex_ai): propagate extra_headers anthropic-beta to request body (#20666)

Browse source 
Vertex AI requires Anthropic beta flags in the request body
(anthropic_beta array), not as HTTP headers. The Bedrock handler
already extracts user-specified beta headers from the headers dict,
but the Vertex handler was missing this, causing extra_headers like
interleaved-thinking-2025-05-14 to be silently dropped.

This extracts anthropic-beta values from optional_params extra_headers
and merges them into the anthropic_beta request body field, and also
removes extra_headers from the request body since the parent's
transform_request spreads optional_params into data.
This commit is contained in:
Elias Högbom Aronsson 2026-02-08 08:05:17 +01:00 • committed by GitHub
parent c9c6a5edc9
commit 0458e734b2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 256 additions and 131 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py
View file

@ -56,34 +56,36 @@ class VertexAIAnthropicConfig(AnthropicConfig):
) -> None:
"""
Add context_management beta headers to the beta_set.
- If any edit has type "compact_20260112", add compact-2026-01-12 header
- For all other edits, add context-management-2025-06-27 header
Args:
beta_set: Set of beta headers to modify in-place
context_management: The context_management dict from optional_params
"""
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
edits = context_management.get("edits", [])
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
beta_set.add(
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value
)
def transform_request(
self,
@ -102,10 +104,10 @@ class VertexAIAnthropicConfig(AnthropicConfig):
)
data.pop("model", None) # vertex anthropic doesn't accept 'model' parameter
# VertexAI doesn't support output_format parameter, remove it if present
data.pop("output_format", None)
tools = optional_params.get("tools")
tool_search_used = self.is_tool_search_used(tools)
auto_betas = self.get_anthropic_beta_list(
@ -119,16 +121,30 @@ class VertexAIAnthropicConfig(AnthropicConfig):
beta_set = set(auto_betas)
if tool_search_used:
beta_set.add("tool-search-tool-2025-10-19") # Vertex requires this header for tool search
beta_set.add(
"tool-search-tool-2025-10-19"
) # Vertex requires this header for tool search
# Add context_management beta headers (compact and/or context-management)
context_management = optional_params.get("context_management")
if context_management:
self._add_context_management_beta_headers(beta_set, context_management)
extra_headers = optional_params.get("extra_headers") or {}
anthropic_beta_value = extra_headers.get("anthropic-beta", "")
if isinstance(anthropic_beta_value, str) and anthropic_beta_value:
for beta in anthropic_beta_value.split(","):
beta = beta.strip()
if beta:
beta_set.add(beta)
elif isinstance(anthropic_beta_value, list):
beta_set.update(anthropic_beta_value)
data.pop("extra_headers", None)
if beta_set:
data["anthropic_beta"] = list(beta_set)
return data
def map_openai_params(
@ -148,7 +164,7 @@ class VertexAIAnthropicConfig(AnthropicConfig):
original_model = model
if "response_format" in non_default_params:
model = "claude-3-sonnet-20240229" # Use a model that will use tool-based approach
# Call parent method with potentially modified model name
optional_params = super().map_openai_params(
non_default_params=non_default_params,
@ -156,10 +172,10 @@ class VertexAIAnthropicConfig(AnthropicConfig):
model=model,
drop_params=drop_params,
)
# Restore original model name for any other processing
model = original_model
return optional_params
def transform_response(

341
tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py
View file

@ -45,68 +45,65 @@ def test_vertex_ai_anthropic_web_search_header_in_completion():
# Create the config instance
model_info = AnthropicModelInfo()
# Test the header generation directly
tools = [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}]
# Check if web search tool is detected
web_search_detected = model_info.is_web_search_tool_used(tools=tools)
assert web_search_detected is True, "Web search tool should be detected"
# Generate headers with is_vertex_request=True
headers = model_info.get_anthropic_headers(
api_key="test-key",
web_search_tool_used=web_search_detected,
is_vertex_request=True,
)
# Assert that the anthropic-beta header with web-search is present
assert "anthropic-beta" in headers, "anthropic-beta header should be present"
assert headers["anthropic-beta"] == "web-search-2025-03-05", \
f"anthropic-beta should be 'web-search-2025-03-05', got: {headers['anthropic-beta']}"
assert (
headers["anthropic-beta"] == "web-search-2025-03-05"
), f"anthropic-beta should be 'web-search-2025-03-05', got: {headers['anthropic-beta']}"
# Test that header is NOT added for non-Vertex requests
headers_non_vertex = model_info.get_anthropic_headers(
api_key="test-key",
web_search_tool_used=web_search_detected,
is_vertex_request=False,
)
# For non-Vertex (Anthropic-hosted), the web search header should NOT be in anthropic-beta
# because Anthropic doesn't require it
assert "anthropic-beta" not in headers_non_vertex or "web-search" not in headers_non_vertex.get("anthropic-beta", ""), \
"anthropic-beta with web-search should not be present for non-Vertex requests"
assert (
"anthropic-beta" not in headers_non_vertex
or "web-search" not in headers_non_vertex.get("anthropic-beta", "")
), "anthropic-beta with web-search should not be present for non-Vertex requests"
def test_vertex_ai_anthropic_context_management_compact_beta_header():
"""Test that context_management with compact adds the correct beta header for Vertex AI"""
config = VertexAIAnthropicConfig()
messages = [{"role": "user", "content": "Hello"}]
optional_params = {
"context_management": {
"edits": [
{
"type": "compact_20260112"
}
]
},
"context_management": {"edits": [{"type": "compact_20260112"}]},
"max_tokens": 100,
"is_vertex_request": True
"is_vertex_request": True,
}
result = config.transform_request(
model="claude-opus-4-6",
messages=messages,
optional_params=optional_params,
litellm_params={},
headers={}
headers={},
)
# Verify context_management is included
assert "context_management" in result
assert result["context_management"]["edits"][0]["type"] == "compact_20260112"
# Verify compact beta header is in anthropic_beta field
assert "anthropic_beta" in result
assert "compact-2026-01-12" in result["anthropic_beta"]
@ -115,33 +112,27 @@ def test_vertex_ai_anthropic_context_management_compact_beta_header():
def test_vertex_ai_anthropic_context_management_mixed_edits():
"""Test that context_management with both compact and other edits adds both beta headers"""
config = VertexAIAnthropicConfig()
messages = [{"role": "user", "content": "Hello"}]
optional_params = {
"context_management": {
"edits": [
{
"type": "compact_20260112"
},
{
"type": "replace",
"message_id": "msg_123",
"content": "new content"
}
{"type": "compact_20260112"},
{"type": "replace", "message_id": "msg_123", "content": "new content"},
]
},
"max_tokens": 100,
"is_vertex_request": True
"is_vertex_request": True,
}
result = config.transform_request(
model="claude-opus-4-6",
messages=messages,
optional_params=optional_params,
litellm_params={},
headers={}
headers={},
)
# Verify both beta headers are present
assert "anthropic_beta" in result
assert "compact-2026-01-12" in result["anthropic_beta"]
@ -151,58 +142,65 @@ def test_vertex_ai_anthropic_context_management_mixed_edits():
def test_vertex_ai_anthropic_structured_output_header_not_added():
"""Test that structured output beta headers are NOT added for Vertex AI requests"""
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
config = AnthropicConfig()
# Test case 1: Vertex request with output_format should NOT add beta header
headers_vertex = {}
optional_params_vertex = {
'output_format': {
'type': 'json_schema',
'json_schema': {
'name': 'MathResult',
'schema': {'properties': {'result': {'type': 'integer'}}}
}
"output_format": {
"type": "json_schema",
"json_schema": {
"name": "MathResult",
"schema": {"properties": {"result": {"type": "integer"}}},
},
},
'is_vertex_request': True
"is_vertex_request": True,
}
result_vertex = config.update_headers_with_optional_anthropic_beta(headers_vertex, optional_params_vertex)
assert "anthropic-beta" not in result_vertex, \
f"Vertex request should NOT have anthropic-beta header for structured output, got: {result_vertex.get('anthropic-beta')}"
result_vertex = config.update_headers_with_optional_anthropic_beta(
headers_vertex, optional_params_vertex
)
assert (
"anthropic-beta" not in result_vertex
), f"Vertex request should NOT have anthropic-beta header for structured output, got: {result_vertex.get('anthropic-beta')}"
# Test case 2: Non-Vertex request with output_format SHOULD add beta header
headers_non_vertex = {}
optional_params_non_vertex = {
'output_format': {
'type': 'json_schema',
'json_schema': {
'name': 'MathResult',
'schema': {'properties': {'result': {'type': 'integer'}}}
}
"output_format": {
"type": "json_schema",
"json_schema": {
"name": "MathResult",
"schema": {"properties": {"result": {"type": "integer"}}},
},
},
'is_vertex_request': False
"is_vertex_request": False,
}
result_non_vertex = config.update_headers_with_optional_anthropic_beta(headers_non_vertex, optional_params_non_vertex)
assert "anthropic-beta" in result_non_vertex, \
"Non-Vertex request SHOULD have anthropic-beta header for structured output"
assert result_non_vertex["anthropic-beta"] == "structured-outputs-2025-11-13", \
f"Expected 'structured-outputs-2025-11-13', got: {result_non_vertex.get('anthropic-beta')}"
result_non_vertex = config.update_headers_with_optional_anthropic_beta(
headers_non_vertex, optional_params_non_vertex
)
assert (
"anthropic-beta" in result_non_vertex
), "Non-Vertex request SHOULD have anthropic-beta header for structured output"
assert (
result_non_vertex["anthropic-beta"] == "structured-outputs-2025-11-13"
), f"Expected 'structured-outputs-2025-11-13', got: {result_non_vertex.get('anthropic-beta')}"
def test_vertex_ai_claude_sonnet_4_5_structured_output_fix():
"""
Test fix for issue #18625: Claude Sonnet 4.5 on VertexAI should use tool-based
Test fix for issue #18625: Claude Sonnet 4.5 on VertexAI should use tool-based
structured outputs instead of output_format parameter.
This test verifies that:
1. Claude Sonnet 4.5 uses tool-based structured outputs on VertexAI
2. output_format parameter is removed from the final request
3. The fix prevents "Extra inputs are not permitted" error
"""
config = VertexAIAnthropicConfig()
# Test data matching the issue report
response_format = {
"type": "json_schema",
@ -212,29 +210,23 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix():
"schema": {
"type": "object",
"properties": {
"question": {
"type": "string"
},
"response": {
"type": "string"
}
"question": {"type": "string"},
"response": {"type": "string"},
},
"required": ["question", "response"],
"additionalProperties": False
}
}
"additionalProperties": False,
},
},
}
messages = [
{"role": "user", "content": "Generate a question and answer about AI."}
]
messages = [{"role": "user", "content": "Generate a question and answer about AI."}]
# Test parameters that would trigger the issue
non_default_params = {
"response_format": response_format,
"max_tokens": 1000,
}
# Test 1: Verify map_openai_params forces tool-based approach for Claude Sonnet 4.5
optional_params = {}
result_params = config.map_openai_params(
@ -243,17 +235,19 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix():
model="claude-3-5-sonnet-20241022", # Claude Sonnet 4.5 model
drop_params=False,
)
# Should have tools and tool_choice (tool-based approach)
assert "tools" in result_params, "Tools should be present for structured output"
assert "tool_choice" in result_params, "Tool choice should be present for structured output"
assert (
"tool_choice" in result_params
), "Tool choice should be present for structured output"
assert "json_mode" in result_params, "JSON mode should be enabled"
# Verify the tool is the response format tool
tools = result_params["tools"]
assert len(tools) == 1, "Should have exactly one tool for response format"
assert tools[0]["name"] == "json_tool_call", "Tool should be named json_tool_call"
# Test 2: Verify transform_request removes output_format parameter
# Simulate what would happen if parent class added output_format
test_data = {
@ -264,20 +258,22 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix():
"tool_choice": result_params["tool_choice"],
"output_format": { # This would be added by parent class for Sonnet 4.5
"type": "json_schema",
"schema": response_format["json_schema"]["schema"]
}
"schema": response_format["json_schema"]["schema"],
},
}
# Mock the parent transform_request to return data with output_format
original_transform = config.__class__.__bases__[0].transform_request
def mock_transform_request(self, model, messages, optional_params, litellm_params, headers):
def mock_transform_request(
self, model, messages, optional_params, litellm_params, headers
):
# Return test data that includes output_format
return test_data.copy()
# Temporarily replace parent method
config.__class__.__bases__[0].transform_request = mock_transform_request
try:
final_data = config.transform_request(
model="claude-3-5-sonnet-20241022",
@ -286,13 +282,15 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix():
litellm_params={},
headers={},
)
# Verify that output_format was removed (fixes the "Extra inputs are not permitted" error)
assert "output_format" not in final_data, "output_format should be removed for VertexAI"
assert (
"output_format" not in final_data
), "output_format should be removed for VertexAI"
assert "model" not in final_data, "model should be removed for VertexAI"
assert "tools" in final_data, "tools should still be present"
assert "tool_choice" in final_data, "tool_choice should still be present"
finally:
# Restore original method
config.__class__.__bases__[0].transform_request = original_transform
@ -300,43 +298,149 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix():
def test_vertex_ai_anthropic_other_models_still_use_tools():
"""
Test that other Anthropic models (non-Sonnet 4.5) on VertexAI also use tool-based
Test that other Anthropic models (non-Sonnet 4.5) on VertexAI also use tool-based
structured outputs, ensuring consistency across all models.
"""
config = VertexAIAnthropicConfig()
response_format = {
"type": "json_schema",
"json_schema": {
"name": "test_schema",
"schema": {
"type": "object",
"properties": {
"result": {"type": "string"}
}
}
}
"schema": {"type": "object", "properties": {"result": {"type": "string"}}},
},
}
# Test with Claude 3 Sonnet (not 4.5)
non_default_params = {"response_format": response_format}
optional_params = {}
result_params = config.map_openai_params(
non_default_params=non_default_params,
optional_params=optional_params,
model="claude-3-sonnet-20240229",
drop_params=False,
)
# Should still use tool-based approach
assert "tools" in result_params, "Claude 3 Sonnet should also use tool-based structured output"
assert (
"tools" in result_params
), "Claude 3 Sonnet should also use tool-based structured output"
assert "tool_choice" in result_params, "Tool choice should be present"
assert "json_mode" in result_params, "JSON mode should be enabled"
def test_vertex_ai_anthropic_extra_headers_beta_propagation():
"""Test that anthropic-beta values from extra_headers are propagated to the
anthropic_beta request body field for Vertex AI requests.
Vertex AI requires beta flags in the request body (anthropic_beta array),
not as HTTP headers. This mirrors the Bedrock handler's behavior of
extracting user-specified beta headers.
"""
config = VertexAIAnthropicConfig()
messages = [{"role": "user", "content": "Hello"}]
optional_params = {
"max_tokens": 100,
"is_vertex_request": True,
"extra_headers": {
"anthropic-beta": "interleaved-thinking-2025-05-14",
},
}
result = config.transform_request(
model="claude-sonnet-4-20250514",
messages=messages,
optional_params=optional_params,
litellm_params={},
headers={},
)
assert "anthropic_beta" in result
assert "interleaved-thinking-2025-05-14" in result["anthropic_beta"]
assert "extra_headers" not in result
def test_vertex_ai_anthropic_extra_headers_beta_merged_with_auto_betas():
"""Test that extra_headers betas are merged with auto-detected betas
rather than replacing them."""
config = VertexAIAnthropicConfig()
messages = [{"role": "user", "content": "Hello"}]
optional_params = {
"max_tokens": 100,
"is_vertex_request": True,
"extra_headers": {
"anthropic-beta": "interleaved-thinking-2025-05-14",
},
"context_management": {"edits": [{"type": "compact_20260112"}]},
}
result = config.transform_request(
model="claude-opus-4-6",
messages=messages,
optional_params=optional_params,
litellm_params={},
headers={},
)
assert "anthropic_beta" in result
assert "interleaved-thinking-2025-05-14" in result["anthropic_beta"]
assert "compact-2026-01-12" in result["anthropic_beta"]
def test_vertex_ai_anthropic_extra_headers_comma_separated_betas():
"""Test that comma-separated beta values in extra_headers are all extracted."""
config = VertexAIAnthropicConfig()
messages = [{"role": "user", "content": "Hello"}]
optional_params = {
"max_tokens": 100,
"is_vertex_request": True,
"extra_headers": {
"anthropic-beta": "interleaved-thinking-2025-05-14,dev-full-thinking-2025-05-14",
},
}
result = config.transform_request(
model="claude-sonnet-4-20250514",
messages=messages,
optional_params=optional_params,
litellm_params={},
headers={},
)
assert "anthropic_beta" in result
assert "interleaved-thinking-2025-05-14" in result["anthropic_beta"]
assert "dev-full-thinking-2025-05-14" in result["anthropic_beta"]
def test_vertex_ai_anthropic_no_extra_headers_unchanged():
"""Test that requests without extra_headers still work normally."""
config = VertexAIAnthropicConfig()
messages = [{"role": "user", "content": "Hello"}]
optional_params = {
"max_tokens": 100,
"is_vertex_request": True,
}
result = config.transform_request(
model="claude-sonnet-4-20250514",
messages=messages,
optional_params=optional_params,
litellm_params={},
headers={},
)
assert "anthropic_beta" not in result
assert "extra_headers" not in result
def test_vertex_ai_partner_models_anthropic_remove_prompt_caching_scope_beta_header():
"""
Test that remove_unsupported_beta correctly filters out prompt-caching-scope-2026-01-05
Test that remove_unsupported_beta correctly filters out prompt-caching-scope-2026-01-05
from the anthropic-beta headers.
"""
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import (
@ -352,13 +456,18 @@ def test_vertex_ai_partner_models_anthropic_remove_prompt_caching_scope_beta_hea
headers = update_headers_with_filtered_beta(headers, "vertex_ai")
beta_header = headers.get("anthropic-beta")
assert PROMPT_CACHING_BETA_HEADER not in (beta_header or ""), \
f"{PROMPT_CACHING_BETA_HEADER} should be filtered out"
assert "other-feature" in (beta_header or ""), \
"Other non-excluded beta headers should remain"
assert "web-search-2025-03-05" in (beta_header or ""), \
"Other non-excluded beta headers should remain"
assert PROMPT_CACHING_BETA_HEADER not in (
beta_header or ""
), f"{PROMPT_CACHING_BETA_HEADER} should be filtered out"
assert "other-feature" in (
beta_header or ""
), "Other non-excluded beta headers should remain"
assert "web-search-2025-03-05" in (
beta_header or ""
), "Other non-excluded beta headers should remain"
# If prompt-caching was the only value, header should be removed completely
headers2 = {"anthropic-beta": PROMPT_CACHING_BETA_HEADER}
headers2 = update_headers_with_filtered_beta(headers2, "vertex_ai")
assert "anthropic-beta" not in headers2, "Header should be removed if no supported values remain"
assert (
"anthropic-beta" not in headers2
), "Header should be removed if no supported values remain"