scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 280

fix(anthropic): forward unlisted betas to Foundry and keep Foundry calls off the Rust messages route

Browse source 
An anthropic deployment on a Foundry host now drops only the betas the azure_ai row marks
unsupported on /v1/messages, and forwards betas the table does not list as sent, as main did.
cognitiveservices.azure.com counts as a Foundry host. The Rust messages route sends an async call
with a Foundry api_base to Python, which filters by row. A sync call stays on Rust, since Python has
no sync /v1/messages handler. The json_mode chat response keeps input_transformations,
and the first /v1/messages call with betas loads the beta table off the event loop.
This commit is contained in:
mateo-berri 2026-11-02 11:55:09 -08:00
parent e3a2dce981
commit 55385e483f
10 changed files with 206 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
litellm/anthropic_beta_headers_manager.py
View file

@ -49,6 +49,7 @@ _BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
_DECODED_JSON: Final = TypeAdapter(object)
BetaPlacement: TypeAlias = Literal["header", "body"]
UnlistedBetas: TypeAlias = Literal["drop", "forward"]
_HEADER_ONLY_COLUMN_SUFFIX: Final = "_header_only"
@ -281,12 +282,13 @@ def filter_and_transform_beta_headers(
beta_headers: list[str],
provider: str,
placement: BetaPlacement = "header",
unlisted: UnlistedBetas = "drop",
) -> list[str]:
"""
Filter and transform beta headers based on provider's mapping configuration.
This function:
1. Only allows headers that are present in the provider's mapping keys
1. Only allows headers that are present in the provider's mapping keys, unless ``unlisted`` is "forward"
2. Filters out headers with null values (unsupported)
3. Maps headers to provider-specific names (e.g., advanced-tool-use -> tool-search-tool)
@ -294,6 +296,7 @@ def filter_and_transform_beta_headers(
beta_headers: List of Anthropic beta header values
provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai")
placement: Whether the betas travel as the anthropic-beta header or the anthropic_beta body field
unlisted: Whether a beta missing from the mapping is dropped or forwarded as sent
Returns:
List of filtered and transformed beta headers for the provider
@ -308,6 +311,10 @@ def filter_and_transform_beta_headers(
for header in beta_headers:
header = header.strip()
if header not in provider_mapping and unlisted == "forward":
filtered_headers.add(header)
continue
# Check if header is in the mapping
if header not in provider_mapping:
verbose_logger.debug(
@ -389,7 +396,11 @@ def get_provider_beta_header(
return provider_mapping[anthropic_beta_header]
FOUNDRY_API_BASE_HOST_SUFFIXES: Final[tuple[str, ...]] = ("services.ai.azure.com", "openai.azure.com")
FOUNDRY_API_BASE_HOST_SUFFIXES: Final[tuple[str, ...]] = (
"services.ai.azure.com",
"openai.azure.com",
"cognitiveservices.azure.com",
)
def is_foundry_api_base(api_base: str | None) -> bool:
@ -416,6 +427,7 @@ def update_headers_with_filtered_beta(
headers: dict,
provider: str,
api_base: str | None = None,
unlisted: UnlistedBetas = "drop",
) -> dict:
"""
Update headers dict by filtering and transforming anthropic-beta header values.
@ -425,6 +437,7 @@ def update_headers_with_filtered_beta(
headers: Request headers dict (will be modified in place)
provider: Provider name
api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``)
unlisted: Whether a beta missing from the row is dropped or forwarded as sent
Returns:
Updated headers dict
@ -440,6 +453,7 @@ def update_headers_with_filtered_beta(
filtered_beta_values: Final = filter_and_transform_beta_headers(
beta_headers=beta_values,
provider=resolve_beta_provider(provider, api_base),
unlisted=unlisted,
)
# Update or remove the header

9
litellm/llms/anthropic/chat/transformation.py
View file

@ -3168,8 +3168,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
STOP_REASON_FIELDS.validate_python(completion_response)
)
completion_response["stop_reason"] = "stop"
if stop_reason_fields:
json_mode_message.provider_specific_fields = dict(stop_reason_fields)
input_transformations: Final = provider_specific_fields.get("input_transformations")
json_mode_fields: Final = {
**stop_reason_fields,
**({} if input_transformations is None else {"input_transformations": input_transformations}),
}
if json_mode_fields:
json_mode_message.provider_specific_fields = json_mode_fields
if "refusal" in refusal_fields:
json_mode_message.refusal = refusal_fields["refusal"]
if thinking_blocks:

13
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -2591,9 +2591,8 @@ class BaseLLMHTTPHandler:
takes_another_beta_row: Final = (
resolve_beta_provider(custom_llm_provider, affinity_api_base) != custom_llm_provider
)
filters_beta_headers: Final = (
anthropic_messages_provider_config.should_filter_anthropic_beta_headers() or takes_another_beta_row
)
provider_filters_betas: Final = anthropic_messages_provider_config.should_filter_anthropic_beta_headers()
filters_beta_headers: Final = provider_filters_betas or takes_another_beta_row
if request_params.get("safeguards") is not None and filters_beta_headers:
await aload_beta_headers_config()
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
@ -2614,9 +2613,13 @@ class BaseLLMHTTPHandler:
api_base=api_base,
)
if filters_beta_headers:
if filters_beta_headers and headers.get("anthropic-beta") is not None:
await aload_beta_headers_config()
headers = update_headers_with_filtered_beta(
headers=headers, provider=custom_llm_provider, api_base=affinity_api_base
headers=headers,
provider=custom_llm_provider,
api_base=affinity_api_base,
unlisted="drop" if provider_filters_betas else "forward",
)
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase

1
litellm/messages/dispatch.py
View file

@ -119,6 +119,7 @@ _ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType(
{
**_FEATURES,
Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook,
Feature.FOUNDRY_BETA_ROW: route_detectors.foundry_beta_row,
Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception,
Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted,
Feature.STRICT_TOOLS_RETRY: route_detectors.strict_tools_retry,

3
litellm/rust_bridge/catalog.py
View file

@ -66,6 +66,8 @@ class Feature(str, Enum):
"""Extra body fields that Python transforms and merges into the provider request."""
PROVIDER_AFFINITY = "provider_affinity"
"""A provider affinity header that Python validates and populates from the session id."""
FOUNDRY_BETA_ROW = "foundry_beta_row"
"""A Foundry api_base, where Python drops the anthropic-beta values Foundry rejects."""
PRE_REQUEST_HOOK = "pre_request_hook"
"""A logger overriding async_pre_request_hook, which may rewrite the request."""
WEBSEARCH_INTERCEPTION = "websearch_interception"
@ -170,6 +172,7 @@ CHAT_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
Feature.PROVIDER_AFFINITY,
Feature.FOUNDRY_BETA_ROW,
Feature.SAFEGUARDS_CLASSIFIER,
Feature.PRE_REQUEST_HOOK,
Feature.WEBSEARCH_INTERCEPTION,

7
litellm/rust_bridge/messages/detectors.py
View file

@ -5,6 +5,7 @@ from __future__ import annotations
from typing import Final
import litellm
from litellm.anthropic_beta_headers_manager import is_foundry_api_base
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.llms.opencode.common_utils import is_opencode_api_base
from litellm.rust_bridge.detectors import api_base, overridden, present, streaming
@ -15,6 +16,12 @@ def provider_affinity(request: NativeCall) -> bool:
return is_opencode_api_base(api_base(request))
def foundry_beta_row(request: NativeCall) -> bool:
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
return is_foundry_api_base(AnthropicModelInfo.get_api_base(api_base(request)))
def anthropic_workload_identity(request: NativeCall) -> bool:
"""Python mints an Anthropic federation token when no static credential is configured."""
from litellm.llms.anthropic.common_utils import AnthropicModelInfo

6
tests/test_litellm_rust/messages/test_dispatch.py
View file

@ -569,6 +569,12 @@ async def test_strict_tools_stay_native_unless_python_could_drop_strict(
ASYNC_ONLY_FEATURES: Final = (
pytest.param(Feature.PRE_REQUEST_HOOK, {}, {"callbacks": [_Rewrite()]}, id="pre-request-hook"),
pytest.param(
Feature.FOUNDRY_BETA_ROW,
{"api_base": "https://team.services.ai.azure.com/anthropic"},
{},
id="foundry-beta-row",
),
pytest.param(
Feature.WEBSEARCH_INTERCEPTION, {}, {"callbacks": [WebSearchInterceptionLogger()]}, id="websearch-interception"
),

53
tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -40,6 +40,7 @@ from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation im
VertexAIAnthropicConfig,
)
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
from litellm.types.llms.openai import ChatCompletionToolCallFunctionChunk
from litellm.types.utils import ChatCompletionToolCallChunk, ModelResponse, ServerToolUse, Usage
from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE
@ -10017,18 +10018,18 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key():
_INPUT_TRANSFORMATIONS: Final = [{"type": "thinking_block_binding", "count": 2}]
def _input_transformations_response() -> dict:
return {
"id": "msg_input_transformations",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-6",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 12, "output_tokens": 3},
"input_transformations": _INPUT_TRANSFORMATIONS,
}
def _input_transformations_response() -> AnthropicMessagesResponse:
return AnthropicMessagesResponse(
id="msg_input_transformations",
type="message",
role="assistant",
model="claude-sonnet-4-6",
content=[{"type": "text", "text": "ok"}],
stop_reason="end_turn",
stop_sequence=None,
usage={"input_tokens": 12, "output_tokens": 3},
input_transformations=_INPUT_TRANSFORMATIONS,
)
def test_chat_response_surfaces_input_transformations_in_provider_specific_fields():
@ -10048,8 +10049,9 @@ def test_chat_response_surfaces_input_transformations_in_provider_specific_field
def test_chat_response_without_input_transformations_adds_no_key():
completion_response = _input_transformations_response()
del completion_response["input_transformations"]
completion_response: Final = {
key: value for key, value in _input_transformations_response().items() if key != "input_transformations"
}
result = AnthropicConfig().transform_response(
model="claude-sonnet-4-6",
raw_response=httpx.Response(200, json=completion_response),
@ -10065,6 +10067,29 @@ def test_chat_response_without_input_transformations_adds_no_key():
assert "input_transformations" not in result.choices[0].message.provider_specific_fields
def test_json_mode_chat_response_keeps_input_transformations():
completion_response: Final = {
**_input_transformations_response(),
"content": [{"type": "tool_use", "id": "toolu_json", "name": RESPONSE_FORMAT_TOOL_NAME, "input": {"answer": 42}}],
"stop_reason": "tool_use",
}
result: Final = AnthropicConfig().transform_response(
model="claude-sonnet-4-6",
raw_response=httpx.Response(200, json=completion_response),
model_response=ModelResponse(),
logging_obj=MagicMock(),
request_data={},
messages=[{"role": "user", "content": "hi"}],
optional_params={},
litellm_params={},
encoding=None,
json_mode=True,
)
assert json.loads(result.choices[0].message.content) == {"answer": 42}
assert result.choices[0].message.provider_specific_fields["input_transformations"] == _INPUT_TRANSFORMATIONS
_KEPT_SEARCH_CALL: Final = {
"role": "assistant",
"content": None,

119
tests/unit/llms/custom_httpx/test_llm_http_handler.py
View file

@ -6265,29 +6265,59 @@ async def test_non_chatgpt_responses_preserves_caller_headers_after_validation(
client.client.close()
_BETA_THE_TABLE_DOES_NOT_LIST: Final = "not-in-the-beta-table-2099-01-01"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("custom_llm_provider", "pass_through", "forwarded_betas"),
("custom_llm_provider", "messages_config", "foundry_api_base", "forwarded_betas"),
[
pytest.param("anthropic", False, "thinking-binding-controls-2026-08-01", id="anthropic-takes-the-azure-ai-row"),
pytest.param(
"anthropic",
"anthropic",
"https://my-resource.services.ai.azure.com/anthropic",
f"thinking-binding-controls-2026-08-01,{_BETA_THE_TABLE_DOES_NOT_LIST}",
id="anthropic-drops-what-the-azure-ai-row-rejects",
),
pytest.param(
"anthropic",
"anthropic",
"https://my-resource.cognitiveservices.azure.com/anthropic",
f"thinking-binding-controls-2026-08-01,{_BETA_THE_TABLE_DOES_NOT_LIST}",
id="anthropic-on-a-cognitiveservices-host",
),
pytest.param(
"azure_ai",
"azure_ai",
"https://my-resource.services.ai.azure.com/anthropic",
"thinking-binding-controls-2026-08-01",
id="azure-ai-keeps-dropping-unlisted-betas",
),
pytest.param(
"openai",
True,
"advisor-tool-2026-03-01,thinking-binding-controls-2026-08-01",
"pass_through",
"https://my-resource.services.ai.azure.com/anthropic",
f"advisor-tool-2026-03-01,thinking-binding-controls-2026-08-01,{_BETA_THE_TABLE_DOES_NOT_LIST}",
id="messages-pass-through-forwards-as-sent",
),
],
)
async def test_async_anthropic_messages_handler_picks_the_beta_row_by_deployment_on_a_foundry_api_base(
monkeypatch, custom_llm_provider: str, pass_through: bool, forwarded_betas: str
monkeypatch, custom_llm_provider: str, messages_config: str, foundry_api_base: str, forwarded_betas: str
):
from litellm import anthropic_beta_headers_manager
from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig
from litellm.llms.azure_ai.anthropic.messages_transformation import AzureAnthropicMessagesConfig
from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig
monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True")
anthropic_beta_headers_manager._BETA_HEADERS_CONFIG = None
foundry_api_base: Final = "https://my-resource.services.ai.azure.com/anthropic"
assert _BETA_THE_TABLE_DOES_NOT_LIST not in anthropic_beta_headers_manager._load_beta_headers_config()["azure_ai"]
configs: Final = {
"anthropic": AnthropicMessagesConfig,
"azure_ai": AzureAnthropicMessagesConfig,
"pass_through": OpenAILikeAnthropicMessagesConfig,
}
upstream_response: Final = httpx.Response(
200,
json={
@ -6311,9 +6341,7 @@ async def test_async_anthropic_messages_handler_picks_the_beta_row_by_deployment
await BaseLLMHTTPHandler().async_anthropic_messages_handler(
model="claude-sonnet-4-6",
messages=[{"role": "user", "content": "Hello"}],
anthropic_messages_provider_config=(
OpenAILikeAnthropicMessagesConfig() if pass_through else AnthropicMessagesConfig()
),
anthropic_messages_provider_config=configs[messages_config](),
anthropic_messages_optional_request_params={"max_tokens": 8},
custom_llm_provider=custom_llm_provider,
litellm_params=GenericLiteLLMParams(api_base=foundry_api_base),
@ -6321,11 +6349,80 @@ async def test_async_anthropic_messages_handler_picks_the_beta_row_by_deployment
api_key="sk-test",
api_base=foundry_api_base,
client=mock_client,
kwargs={"extra_headers": {"anthropic-beta": "advisor-tool-2026-03-01,thinking-binding-controls-2026-08-01"}},
kwargs={
"extra_headers": {
"anthropic-beta": (
f"advisor-tool-2026-03-01,thinking-binding-controls-2026-08-01,{_BETA_THE_TABLE_DOES_NOT_LIST}"
)
}
},
)
sent_headers: Final = mock_client.post.call_args.kwargs["headers"]
assert sent_headers["anthropic-beta"] == forwarded_betas
assert set(sent_headers["anthropic-beta"].split(",")) == set(forwarded_betas.split(","))
@pytest.mark.asyncio
async def test_first_foundry_request_with_betas_fetches_the_beta_table_off_the_event_loop(monkeypatch):
from litellm import anthropic_beta_headers_manager, anthropic_beta_headers_url
from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig
monkeypatch.delenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", raising=False)
monkeypatch.delenv("LITELLM_OFFLINE_MODE", raising=False)
monkeypatch.setattr(anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None)
bundled_table: Final = anthropic_beta_headers_manager.GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
fetch_started: Final = threading.Event()
loop_answered: Final = threading.Event()
def table_held_until_the_loop_answers(request: httpx.Request) -> httpx.Response:
fetch_started.set()
return httpx.Response(200 if loop_answered.wait(timeout=2) else 504, json=bundled_table)
foundry_api_base: Final = "https://my-resource.services.ai.azure.com/anthropic"
mock_client: Final = AsyncMock(spec=AsyncHTTPHandler)
mock_client.post = AsyncMock(
return_value=httpx.Response(
200,
json={
"id": "msg_123",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "Hello!"}],
"model": "claude-sonnet-4-6",
"stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 1},
},
request=httpx.Request("POST", f"{foundry_api_base}/v1/messages"),
)
)
mock_logging_obj: Final = Mock()
mock_logging_obj.model_call_details = {}
mock_logging_obj.stream = False
mock_logging_obj.dynamic_success_callbacks = None
with respx.mock(assert_all_called=True) as router:
table_route: Final = router.get(anthropic_beta_headers_url).mock(side_effect=table_held_until_the_loop_answers)
request: Final = asyncio.create_task(
BaseLLMHTTPHandler().async_anthropic_messages_handler(
model="claude-sonnet-4-6",
messages=[{"role": "user", "content": "Hello"}],
anthropic_messages_provider_config=AnthropicMessagesConfig(),
anthropic_messages_optional_request_params={"max_tokens": 8},
custom_llm_provider="anthropic",
litellm_params=GenericLiteLLMParams(api_base=foundry_api_base),
logging_obj=mock_logging_obj,
api_key="sk-test",
api_base=foundry_api_base,
client=mock_client,
kwargs={"extra_headers": {"anthropic-beta": "advisor-tool-2026-03-01,compact-2026-01-12"}},
)
)
assert await asyncio.to_thread(fetch_started.wait, 5)
loop_answered.set()
await request
assert table_route.calls.last.response.status_code == 200
assert mock_client.post.call_args.kwargs["headers"]["anthropic-beta"] == "compact-2026-01-12"
@pytest.mark.asyncio

11
tests/unit/rust_bridge/test_detectors.py
View file

@ -334,6 +334,17 @@ def test_stream_cost_injection_needs_a_stream(monkeypatch: pytest.MonkeyPatch) -
assert not messages_detectors.stream_cost_injection(call())
def test_foundry_beta_row_reads_the_api_base_the_anthropic_handler_reads(monkeypatch: pytest.MonkeyPatch) -> None:
for name in ("ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"):
monkeypatch.delenv(name, raising=False)
assert messages_detectors.foundry_beta_row(call(api_base="https://team.services.ai.azure.com/anthropic"))
assert messages_detectors.foundry_beta_row(call(base_url="https://team.cognitiveservices.azure.com/anthropic"))
assert not messages_detectors.foundry_beta_row(call(api_base="https://api.anthropic.com"))
assert not messages_detectors.foundry_beta_row(call())
monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://team.services.ai.azure.com/anthropic")
assert messages_detectors.foundry_beta_row(call())
def test_model_alias_map_applies_only_to_the_requested_model(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "model_alias_map", {"fast": "openai/gpt-5-mini"})
assert chat_detectors.model_alias_map(call(model="fast"))