mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(anthropic): continue unbound threads off /v1/messages and keep every anthropic-beta on chat and Responses
This commit is contained in:
parent
020fa93d6a
commit
ff857389c5
8 changed files with 119 additions and 30 deletions
|
|
@ -517,6 +517,31 @@ def merge_anthropic_beta_headers(existing: str | Sequence[str] | None, new_beta:
|
|||
return ",".join(sorted(betas))
|
||||
|
||||
|
||||
_HEADER_VALUE_LIST: Final = TypeAdapter(tuple[JsonValue, ...])
|
||||
|
||||
|
||||
def _header_strings(value: object) -> tuple[str, ...]:
|
||||
if isinstance(value, str):
|
||||
return (value,)
|
||||
if not isinstance(value, list | tuple):
|
||||
return ()
|
||||
return tuple(entry for entry in _HEADER_VALUE_LIST.validate_python(value) if isinstance(entry, str))
|
||||
|
||||
|
||||
def merge_header_sources(*sources: Mapping[str, object] | None) -> Mapping[str, object]:
|
||||
"""Later sources win per header, except ``anthropic-beta``, which unions every source so client betas
|
||||
never drop a deployment's configured ones."""
|
||||
entries: Final = tuple(chain.from_iterable((source or {}).items() for source in sources))
|
||||
beta_values: Final = tuple(
|
||||
chain.from_iterable(_header_strings(value) for name, value in entries if name.lower() == "anthropic-beta")
|
||||
)
|
||||
merged_beta: Final = merge_anthropic_beta_headers(beta_values, None)
|
||||
return MappingProxyType(
|
||||
{name: value for name, value in entries if name.lower() != "anthropic-beta"}
|
||||
| ({"anthropic-beta": merged_beta} if merged_beta else {})
|
||||
)
|
||||
|
||||
|
||||
def optionally_handle_anthropic_oauth(headers: dict, api_key: str | None) -> tuple[dict, str | None]:
|
||||
"""
|
||||
Handle Anthropic OAuth token detection and header setup.
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import ssl
|
|||
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence
|
||||
from contextlib import asynccontextmanager
|
||||
from functools import lru_cache, partial, reduce
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
|
|
@ -68,7 +67,7 @@ from litellm.litellm_core_utils.realtime_errors import (
|
|||
)
|
||||
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
|
||||
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_header_sources
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import (
|
||||
BaseAnthropicMessagesConfig,
|
||||
)
|
||||
|
|
@ -649,22 +648,6 @@ def _unwrap_provider_error(error: Exception) -> Exception:
|
|||
return error
|
||||
|
||||
|
||||
def _merge_messages_header_sources(*sources: Mapping[str, object] | None) -> Mapping[str, object]:
|
||||
"""Later sources win per header, except ``anthropic-beta``, which unions every source so client betas
|
||||
never drop a deployment's configured ones."""
|
||||
entries: Final = tuple(chain.from_iterable((source or {}).items() for source in sources))
|
||||
beta_values: Final = tuple(
|
||||
cast("str | Sequence[str]", value) # cast-ok: the merge helper skips non-str list entries
|
||||
for name, value in entries
|
||||
if name.lower() == "anthropic-beta" and isinstance(value, str | Sequence)
|
||||
)
|
||||
merged_beta: Final = reduce(merge_anthropic_beta_headers, beta_values, "")
|
||||
return MappingProxyType(
|
||||
{name: value for name, value in entries if name.lower() != "anthropic-beta"}
|
||||
| ({"anthropic-beta": merged_beta} if merged_beta else {})
|
||||
)
|
||||
|
||||
|
||||
class BaseLLMHTTPHandler:
|
||||
async def _make_common_async_call(
|
||||
self,
|
||||
|
|
@ -2573,7 +2556,7 @@ class BaseLLMHTTPHandler:
|
|||
forwarded_headers: Final = kwargs.get("headers", None)
|
||||
# Also check for extra_headers in kwargs (from config or direct calls)
|
||||
extra_headers_from_kwargs: Final = kwargs.get("extra_headers", None)
|
||||
merged_headers: Final = _merge_messages_header_sources(
|
||||
merged_headers: Final = merge_header_sources(
|
||||
forwarded_headers, extra_headers_from_kwargs, provider_specific_headers
|
||||
)
|
||||
request_params: Final = reduce(
|
||||
|
|
|
|||
|
|
@ -242,7 +242,7 @@ from .litellm_core_utils.streaming_chunk_builder_utils import (
|
|||
ChunkProcessor as ChunkProcessor, # noqa: PLC0414 # public re-export
|
||||
)
|
||||
from .llms.anthropic.chat import AnthropicChatCompletion
|
||||
from .llms.anthropic.common_utils import is_claude_tool_target
|
||||
from .llms.anthropic.common_utils import is_claude_tool_target, merge_header_sources
|
||||
from .llms.azure.audio_transcriptions import AzureAudioTranscription
|
||||
from .llms.azure.azure import AzureChatCompletion, check_dynamic_azure_params
|
||||
from .llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion
|
||||
|
|
@ -5957,12 +5957,15 @@ def completion(
|
|||
tools = _drop_anthropic_only_tool_keys(tools=tools)
|
||||
|
||||
if provider_specific_header is not None:
|
||||
headers.update(
|
||||
merged_headers: Final = merge_header_sources(
|
||||
headers,
|
||||
ProviderSpecificHeaderUtils.get_provider_specific_headers(
|
||||
provider_specific_header=provider_specific_header,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
),
|
||||
)
|
||||
headers.clear()
|
||||
headers.update(merged_headers)
|
||||
|
||||
if model_response is not None and hasattr(model_response, HIDDEN_PARAMS_ATTR):
|
||||
model_response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior
|
||||
|
|
|
|||
|
|
@ -186,11 +186,11 @@ class MessageThreadOwnership(CustomLogger):
|
|||
if previous_message_id is None:
|
||||
return None
|
||||
binding: Final = await self._binding(previous_message_id, user_api_key_dict)
|
||||
if binding is None and call_type == _PASS_THROUGH_CALL_TYPE:
|
||||
if binding is None and call_type != _MESSAGES_CALL_TYPE:
|
||||
return None
|
||||
if binding is None or binding.owner != _owner(user_api_key_dict):
|
||||
raise HTTPException(status_code=404, detail=message_thread_not_found_body(_UNOWNED_THREAD_MESSAGE))
|
||||
if binding.anthropic_beta is None or call_type != _MESSAGES_CALL_TYPE:
|
||||
if binding.anthropic_beta is None or call_type == _PASS_THROUGH_CALL_TYPE:
|
||||
return None
|
||||
return {**data, "extra_headers": _with_anthropic_beta(data.get("extra_headers"), binding.anthropic_beta)}
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, iter_items
|
||||
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path
|
||||
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
|
||||
from litellm.llms.anthropic.common_utils import merge_header_sources
|
||||
from litellm.llms.base_llm.responses.transformation import (
|
||||
ANY_REASONING_FIELD_VALUES,
|
||||
REASONING_SUMMARY_VALUES,
|
||||
|
|
@ -262,7 +263,7 @@ class ResponsesAPIRequestUtils:
|
|||
)
|
||||
if extra_headers is None and not scoped_headers:
|
||||
return None
|
||||
return {**(extra_headers or {}), **scoped_headers}
|
||||
return dict(merge_header_sources(extra_headers, scoped_headers))
|
||||
|
||||
@staticmethod
|
||||
def _check_valid_arg(
|
||||
|
|
|
|||
|
|
@ -182,9 +182,13 @@ async def test_a_thread_continue_on_another_route_stays_with_the_key_that_create
|
|||
assert not await allows(ownership, INTRUDER, "msg_cross_route", call_type)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("call_type", ("pass_through_endpoint", "acompletion", "aresponses"))
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_pass_through_continue_of_a_message_the_proxy_never_bound_is_left_to_the_upstream_credential() -> None:
|
||||
assert await allows(hook(), INTRUDER, "msg_unknown", "pass_through_endpoint")
|
||||
async def test_a_continue_outside_the_messages_route_of_a_message_the_proxy_never_bound_is_left_to_the_upstream(
|
||||
call_type: CallTypesLiteral,
|
||||
) -> None:
|
||||
assert await allows(hook(), OWNER, "msg_unknown", call_type)
|
||||
assert await allows(hook(), INTRUDER, "msg_unknown", call_type)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -248,9 +252,27 @@ async def test_the_create_turns_betas_reach_every_later_turn_of_the_thread() ->
|
|||
assert third["extra_headers"] == {"anthropic-beta": "advisor-tool-2026-03-01,message-threads-2026-08-12"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("call_type", ("acompletion", "pass_through_endpoint"))
|
||||
@pytest.mark.parametrize("call_type", ("acompletion", "aresponses"))
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_continue_outside_the_messages_route_is_checked_but_left_unchanged(call_type: CallTypesLiteral) -> None:
|
||||
async def test_a_continue_on_chat_or_responses_resends_the_betas_its_create_turn_sent(
|
||||
call_type: CallTypesLiteral,
|
||||
) -> None:
|
||||
ownership: Final = hook()
|
||||
await ownership.async_post_call_success_hook(
|
||||
data=sent_with("advisor-tool-2026-03-01,message-threads-2026-08-12", thread=CREATE),
|
||||
user_api_key_dict=OWNER,
|
||||
response=anthropic_message("msg_other_route"),
|
||||
)
|
||||
continued: Final = await ownership.async_pre_call_hook(
|
||||
user_api_key_dict=OWNER, cache=DualCache(), data=continue_from("msg_other_route"), call_type=call_type
|
||||
)
|
||||
|
||||
assert continued is not None
|
||||
assert continued["extra_headers"] == {"anthropic-beta": "advisor-tool-2026-03-01,message-threads-2026-08-12"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_pass_through_continue_is_checked_but_left_unchanged() -> None:
|
||||
ownership: Final = hook()
|
||||
await ownership.async_post_call_success_hook(
|
||||
data=sent_with("advisor-tool-2026-03-01", thread=CREATE),
|
||||
|
|
@ -260,7 +282,10 @@ async def test_a_continue_outside_the_messages_route_is_checked_but_left_unchang
|
|||
|
||||
assert (
|
||||
await ownership.async_pre_call_hook(
|
||||
user_api_key_dict=OWNER, cache=DualCache(), data=continue_from("msg_other_route"), call_type=call_type
|
||||
user_api_key_dict=OWNER,
|
||||
cache=DualCache(),
|
||||
data=continue_from("msg_other_route"),
|
||||
call_type="pass_through_endpoint",
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
|
|
|||
|
|
@ -968,6 +968,23 @@ class TestMergeProviderSpecificHeaders:
|
|||
extra_headers=None, provider_specific_header=None, custom_llm_provider="chatgpt"
|
||||
) is None
|
||||
|
||||
def test_client_anthropic_betas_join_the_extra_header_betas(self):
|
||||
merged = ResponsesAPIRequestUtils.merge_provider_specific_headers(
|
||||
extra_headers={"anthropic-beta": "advisor-tool-2026-03-01", "x-tenant": "team-a"},
|
||||
provider_specific_header={
|
||||
"custom_llm_provider": "anthropic",
|
||||
"extra_headers": {"anthropic-beta": "message-threads-2026-08-12"},
|
||||
},
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
assert merged is not None
|
||||
assert merged["x-tenant"] == "team-a"
|
||||
assert set(str(merged["anthropic-beta"]).split(",")) == {
|
||||
"advisor-tool-2026-03-01",
|
||||
"message-threads-2026-08-12",
|
||||
}
|
||||
|
||||
|
||||
def test_responses_object_bills_an_explicit_cache_hit_at_the_explicit_rate():
|
||||
model = "dashscope/qwen-responses-explicit-cache-test"
|
||||
|
|
|
|||
|
|
@ -5384,6 +5384,41 @@ def test_anthropic_oauth_credential_does_not_persist_into_next_provider_hop():
|
|||
assert "anthropic-version" not in shared_headers
|
||||
|
||||
|
||||
def test_anthropic_chat_sends_the_client_betas_beside_the_extra_header_betas():
|
||||
from litellm.types.utils import ProviderSpecificHeader
|
||||
|
||||
def echo_sent_betas(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "msg_01EchoedBetas",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-haiku-5-5",
|
||||
"content": [{"type": "text", "text": request.headers.get("anthropic-beta", "")}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1},
|
||||
},
|
||||
)
|
||||
|
||||
client: Final = HTTPHandler()
|
||||
client.client = httpx.Client(transport=httpx.MockTransport(echo_sent_betas))
|
||||
|
||||
response: Final = litellm.completion(
|
||||
model="anthropic/claude-haiku-5-5",
|
||||
messages=[{"role": "user", "content": "Say OK"}],
|
||||
extra_headers={"anthropic-beta": "advisor-tool-2026-03-01"},
|
||||
provider_specific_header=ProviderSpecificHeader(
|
||||
custom_llm_provider="anthropic", extra_headers={"anthropic-beta": "message-threads-2026-08-12"}
|
||||
),
|
||||
api_key="sk-fake-anthropic-key",
|
||||
client=client,
|
||||
)
|
||||
|
||||
sent_betas: Final = frozenset(beta.strip() for beta in response.choices[0].message.content.split(","))
|
||||
assert {"advisor-tool-2026-03-01", "message-threads-2026-08-12"} <= sent_betas
|
||||
|
||||
|
||||
STREAM_COST_MODEL = "gpt-4o"
|
||||
STREAMED_USAGE = {"prompt_tokens": 137, "completion_tokens": 42, "total_tokens": 179}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue