fix(anthropic): continue unbound threads off /v1/messages and keep every anthropic-beta on chat and Responses

This commit is contained in:
mateo-berri 2026-11-04 20:45:41 -08:00
parent 020fa93d6a
commit ff857389c5
8 changed files with 119 additions and 30 deletions

View file

@ -517,6 +517,31 @@ def merge_anthropic_beta_headers(existing: str | Sequence[str] | None, new_beta:
return ",".join(sorted(betas))
_HEADER_VALUE_LIST: Final = TypeAdapter(tuple[JsonValue, ...])
def _header_strings(value: object) -> tuple[str, ...]:
if isinstance(value, str):
return (value,)
if not isinstance(value, list | tuple):
return ()
return tuple(entry for entry in _HEADER_VALUE_LIST.validate_python(value) if isinstance(entry, str))
def merge_header_sources(*sources: Mapping[str, object] | None) -> Mapping[str, object]:
"""Later sources win per header, except ``anthropic-beta``, which unions every source so client betas
never drop a deployment's configured ones."""
entries: Final = tuple(chain.from_iterable((source or {}).items() for source in sources))
beta_values: Final = tuple(
chain.from_iterable(_header_strings(value) for name, value in entries if name.lower() == "anthropic-beta")
)
merged_beta: Final = merge_anthropic_beta_headers(beta_values, None)
return MappingProxyType(
{name: value for name, value in entries if name.lower() != "anthropic-beta"}
| ({"anthropic-beta": merged_beta} if merged_beta else {})
)
def optionally_handle_anthropic_oauth(headers: dict, api_key: str | None) -> tuple[dict, str | None]:
"""
Handle Anthropic OAuth token detection and header setup.

View file

@ -6,7 +6,6 @@ import ssl
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence
from contextlib import asynccontextmanager
from functools import lru_cache, partial, reduce
from itertools import chain
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
@ -68,7 +67,7 @@ from litellm.litellm_core_utils.realtime_errors import (
)
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_header_sources
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -649,22 +648,6 @@ def _unwrap_provider_error(error: Exception) -> Exception:
return error
def _merge_messages_header_sources(*sources: Mapping[str, object] | None) -> Mapping[str, object]:
"""Later sources win per header, except ``anthropic-beta``, which unions every source so client betas
never drop a deployment's configured ones."""
entries: Final = tuple(chain.from_iterable((source or {}).items() for source in sources))
beta_values: Final = tuple(
cast("str | Sequence[str]", value) # cast-ok: the merge helper skips non-str list entries
for name, value in entries
if name.lower() == "anthropic-beta" and isinstance(value, str | Sequence)
)
merged_beta: Final = reduce(merge_anthropic_beta_headers, beta_values, "")
return MappingProxyType(
{name: value for name, value in entries if name.lower() != "anthropic-beta"}
| ({"anthropic-beta": merged_beta} if merged_beta else {})
)
class BaseLLMHTTPHandler:
async def _make_common_async_call(
self,
@ -2573,7 +2556,7 @@ class BaseLLMHTTPHandler:
forwarded_headers: Final = kwargs.get("headers", None)
# Also check for extra_headers in kwargs (from config or direct calls)
extra_headers_from_kwargs: Final = kwargs.get("extra_headers", None)
merged_headers: Final = _merge_messages_header_sources(
merged_headers: Final = merge_header_sources(
forwarded_headers, extra_headers_from_kwargs, provider_specific_headers
)
request_params: Final = reduce(

View file

@ -242,7 +242,7 @@ from .litellm_core_utils.streaming_chunk_builder_utils import (
ChunkProcessor as ChunkProcessor, # noqa: PLC0414 # public re-export
)
from .llms.anthropic.chat import AnthropicChatCompletion
from .llms.anthropic.common_utils import is_claude_tool_target
from .llms.anthropic.common_utils import is_claude_tool_target, merge_header_sources
from .llms.azure.audio_transcriptions import AzureAudioTranscription
from .llms.azure.azure import AzureChatCompletion, check_dynamic_azure_params
from .llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion
@ -5957,12 +5957,15 @@ def completion(
tools = _drop_anthropic_only_tool_keys(tools=tools)
if provider_specific_header is not None:
headers.update(
merged_headers: Final = merge_header_sources(
headers,
ProviderSpecificHeaderUtils.get_provider_specific_headers(
provider_specific_header=provider_specific_header,
custom_llm_provider=custom_llm_provider,
)
),
)
headers.clear()
headers.update(merged_headers)
if model_response is not None and hasattr(model_response, HIDDEN_PARAMS_ATTR):
model_response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior

View file

@ -186,11 +186,11 @@ class MessageThreadOwnership(CustomLogger):
if previous_message_id is None:
return None
binding: Final = await self._binding(previous_message_id, user_api_key_dict)
if binding is None and call_type == _PASS_THROUGH_CALL_TYPE:
if binding is None and call_type != _MESSAGES_CALL_TYPE:
return None
if binding is None or binding.owner != _owner(user_api_key_dict):
raise HTTPException(status_code=404, detail=message_thread_not_found_body(_UNOWNED_THREAD_MESSAGE))
if binding.anthropic_beta is None or call_type != _MESSAGES_CALL_TYPE:
if binding.anthropic_beta is None or call_type == _PASS_THROUGH_CALL_TYPE:
return None
return {**data, "extra_headers": _with_anthropic_beta(data.get("extra_headers"), binding.anthropic_beta)}

View file

@ -16,6 +16,7 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, iter_items
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
from litellm.llms.anthropic.common_utils import merge_header_sources
from litellm.llms.base_llm.responses.transformation import (
ANY_REASONING_FIELD_VALUES,
REASONING_SUMMARY_VALUES,
@ -262,7 +263,7 @@ class ResponsesAPIRequestUtils:
)
if extra_headers is None and not scoped_headers:
return None
return {**(extra_headers or {}), **scoped_headers}
return dict(merge_header_sources(extra_headers, scoped_headers))
@staticmethod
def _check_valid_arg(

View file

@ -182,9 +182,13 @@ async def test_a_thread_continue_on_another_route_stays_with_the_key_that_create
assert not await allows(ownership, INTRUDER, "msg_cross_route", call_type)
@pytest.mark.parametrize("call_type", ("pass_through_endpoint", "acompletion", "aresponses"))
@pytest.mark.asyncio
async def test_a_pass_through_continue_of_a_message_the_proxy_never_bound_is_left_to_the_upstream_credential() -> None:
assert await allows(hook(), INTRUDER, "msg_unknown", "pass_through_endpoint")
async def test_a_continue_outside_the_messages_route_of_a_message_the_proxy_never_bound_is_left_to_the_upstream(
call_type: CallTypesLiteral,
) -> None:
assert await allows(hook(), OWNER, "msg_unknown", call_type)
assert await allows(hook(), INTRUDER, "msg_unknown", call_type)
@pytest.mark.asyncio
@ -248,9 +252,27 @@ async def test_the_create_turns_betas_reach_every_later_turn_of_the_thread() ->
assert third["extra_headers"] == {"anthropic-beta": "advisor-tool-2026-03-01,message-threads-2026-08-12"}
@pytest.mark.parametrize("call_type", ("acompletion", "pass_through_endpoint"))
@pytest.mark.parametrize("call_type", ("acompletion", "aresponses"))
@pytest.mark.asyncio
async def test_a_continue_outside_the_messages_route_is_checked_but_left_unchanged(call_type: CallTypesLiteral) -> None:
async def test_a_continue_on_chat_or_responses_resends_the_betas_its_create_turn_sent(
call_type: CallTypesLiteral,
) -> None:
ownership: Final = hook()
await ownership.async_post_call_success_hook(
data=sent_with("advisor-tool-2026-03-01,message-threads-2026-08-12", thread=CREATE),
user_api_key_dict=OWNER,
response=anthropic_message("msg_other_route"),
)
continued: Final = await ownership.async_pre_call_hook(
user_api_key_dict=OWNER, cache=DualCache(), data=continue_from("msg_other_route"), call_type=call_type
)
assert continued is not None
assert continued["extra_headers"] == {"anthropic-beta": "advisor-tool-2026-03-01,message-threads-2026-08-12"}
@pytest.mark.asyncio
async def test_a_pass_through_continue_is_checked_but_left_unchanged() -> None:
ownership: Final = hook()
await ownership.async_post_call_success_hook(
data=sent_with("advisor-tool-2026-03-01", thread=CREATE),
@ -260,7 +282,10 @@ async def test_a_continue_outside_the_messages_route_is_checked_but_left_unchang
assert (
await ownership.async_pre_call_hook(
user_api_key_dict=OWNER, cache=DualCache(), data=continue_from("msg_other_route"), call_type=call_type
user_api_key_dict=OWNER,
cache=DualCache(),
data=continue_from("msg_other_route"),
call_type="pass_through_endpoint",
)
is None
)

View file

@ -968,6 +968,23 @@ class TestMergeProviderSpecificHeaders:
extra_headers=None, provider_specific_header=None, custom_llm_provider="chatgpt"
) is None
def test_client_anthropic_betas_join_the_extra_header_betas(self):
merged = ResponsesAPIRequestUtils.merge_provider_specific_headers(
extra_headers={"anthropic-beta": "advisor-tool-2026-03-01", "x-tenant": "team-a"},
provider_specific_header={
"custom_llm_provider": "anthropic",
"extra_headers": {"anthropic-beta": "message-threads-2026-08-12"},
},
custom_llm_provider="anthropic",
)
assert merged is not None
assert merged["x-tenant"] == "team-a"
assert set(str(merged["anthropic-beta"]).split(",")) == {
"advisor-tool-2026-03-01",
"message-threads-2026-08-12",
}
def test_responses_object_bills_an_explicit_cache_hit_at_the_explicit_rate():
model = "dashscope/qwen-responses-explicit-cache-test"

View file

@ -5384,6 +5384,41 @@ def test_anthropic_oauth_credential_does_not_persist_into_next_provider_hop():
assert "anthropic-version" not in shared_headers
def test_anthropic_chat_sends_the_client_betas_beside_the_extra_header_betas():
from litellm.types.utils import ProviderSpecificHeader
def echo_sent_betas(request: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
json={
"id": "msg_01EchoedBetas",
"type": "message",
"role": "assistant",
"model": "claude-haiku-5-5",
"content": [{"type": "text", "text": request.headers.get("anthropic-beta", "")}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 1},
},
)
client: Final = HTTPHandler()
client.client = httpx.Client(transport=httpx.MockTransport(echo_sent_betas))
response: Final = litellm.completion(
model="anthropic/claude-haiku-5-5",
messages=[{"role": "user", "content": "Say OK"}],
extra_headers={"anthropic-beta": "advisor-tool-2026-03-01"},
provider_specific_header=ProviderSpecificHeader(
custom_llm_provider="anthropic", extra_headers={"anthropic-beta": "message-threads-2026-08-12"}
),
api_key="sk-fake-anthropic-key",
client=client,
)
sent_betas: Final = frozenset(beta.strip() for beta in response.choices[0].message.content.split(","))
assert {"advisor-tool-2026-03-01", "message-threads-2026-08-12"} <= sent_betas
STREAM_COST_MODEL = "gpt-4o"
STREAMED_USAGE = {"prompt_tokens": 137, "completion_tokens": 42, "total_tokens": 179}