diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index bdfddb48522..1b659770181 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -21,6 +21,7 @@ "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14", "mcp-client-2025-11-20": "mcp-client-2025-11-20", "mcp-client-2025-04-04": "mcp-client-2025-04-04", + "message-threads-2026-08-12": "message-threads-2026-08-12", "mcp-servers-2025-12-04": null, "oauth-2025-04-20": "oauth-2025-04-20", "output-128k-2025-02-19": "output-128k-2025-02-19", @@ -56,7 +57,6 @@ "mcp-client-2026-09-15": "mcp-client-2026-09-15", "mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22", "message-batches-2024-09-24": "message-batches-2024-09-24", - "message-threads-2026-08-12": "message-threads-2026-08-12", "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21", "model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26", "output-300k-2026-03-24": "output-300k-2026-03-24", @@ -98,6 +98,7 @@ "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14", "mcp-client-2025-11-20": "mcp-client-2025-11-20", "mcp-client-2025-04-04": "mcp-client-2025-04-04", + "message-threads-2026-08-12": "message-threads-2026-08-12", "mcp-servers-2025-12-04": null, "output-128k-2025-02-19": "output-128k-2025-02-19", "structured-output-2024-03-01": null, @@ -132,7 +133,6 @@ "mcp-client-2026-09-15": "mcp-client-2026-09-15", "mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22", "message-batches-2024-09-24": "message-batches-2024-09-24", - "message-threads-2026-08-12": "message-threads-2026-08-12", "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01", "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21", "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01", @@ -177,6 +177,7 @@ "interleaved-thinking-2025-05-14": null, "mcp-client-2025-11-20": null, "mcp-client-2025-04-04": null, + "message-threads-2026-08-12": null, "mcp-servers-2025-12-04": null, "output-128k-2025-02-19": null, "structured-output-2024-03-01": null, @@ -212,6 +213,7 @@ "interleaved-thinking-2025-05-14": null, "mcp-client-2025-11-20": null, "mcp-client-2025-04-04": null, + "message-threads-2026-08-12": null, "mcp-servers-2025-12-04": null, "output-128k-2025-02-19": null, "structured-output-2024-03-01": null, @@ -249,6 +251,7 @@ "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14", "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14", "mcp-client-2025-04-04": null, + "message-threads-2026-08-12": null, "mcp-client-2025-11-20": null, "mcp-servers-2025-12-04": null, "output-128k-2025-02-19": "output-128k-2025-02-19", @@ -286,6 +289,7 @@ "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14", "mcp-client-2025-11-20": null, "mcp-client-2025-04-04": null, + "message-threads-2026-08-12": null, "mcp-servers-2025-12-04": null, "output-128k-2025-02-19": "output-128k-2025-02-19", "structured-output-2024-03-01": null, @@ -321,7 +325,6 @@ "mcp-client-2026-09-15": null, "mcp-tunnels-2026-06-22": null, "message-batches-2024-09-24": null, - "message-threads-2026-08-12": null, "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01", "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21", "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01", @@ -386,6 +389,7 @@ "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14", "mcp-client-2025-11-20": null, "mcp-client-2025-04-04": null, + "message-threads-2026-08-12": null, "mcp-servers-2025-12-04": null, "oauth-2025-04-20": null, "output-128k-2025-02-19": "output-128k-2025-02-19", diff --git a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py index b4c95fafff3..058ac4dcf4c 100644 --- a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py +++ b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py @@ -5,15 +5,18 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop """ import json +from collections.abc import Mapping from typing import Final -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicErrorType _PARSED_ERROR_BODY: Final = TypeAdapter(object) +_ERROR_DETAILS: Final = TypeAdapter(Mapping[str, object]) +_JSON_DECODER: Final = json.JSONDecoder() # HTTP status code -> Anthropic error type # Source: https://platform.claude.com/docs/en/api/errors @@ -166,12 +169,41 @@ class AnthropicExceptionMapping: else: message = raw_message - return AnthropicExceptionMapping.create_error_response( + response: Final = AnthropicExceptionMapping.create_error_response( status_code=status_code, message=message, request_id=request_id, error_type=error_type, ) + details: Final = AnthropicExceptionMapping.embedded_error_details(raw_message) + if details is None: + return response + with_details: Final[AnthropicErrorResponse] = {**response, "error": {**response["error"], "details": details}} + return with_details + + @staticmethod + def embedded_error_details(raw_message: str) -> Mapping[str, object] | None: + """ + ``error.details`` of the Anthropic error body quoted inside ``raw_message``, when there is one. + + litellm wraps an upstream error as ``" - {upstream json}\n\nLiteLLM: ..."``. Anthropic + puts machine-readable codes such as ``thread_not_found`` under ``error.details``, and the SDKs + branch on them, so the code is lifted next to the wrapped message instead of staying only inside it. + """ + start: Final = raw_message.find("{") + if start < 0: + return None + try: + embedded, _ = _JSON_DECODER.raw_decode(raw_message, start) + except ValueError: + return None + try: + envelope: Final = _ERROR_DETAILS.validate_python(embedded, strict=True) + error: Final = _ERROR_DETAILS.validate_python(envelope.get("error"), strict=True) + details: Final = _ERROR_DETAILS.validate_python(error.get("details"), strict=True) + except ValidationError: + return None + return details if envelope.get("type") == "error" else None class AnthropicErrorSseFrame(str): diff --git a/litellm/anthropic_interface/exceptions/exceptions.py b/litellm/anthropic_interface/exceptions/exceptions.py index 18b3ca37a3b..71d4de0029d 100644 --- a/litellm/anthropic_interface/exceptions/exceptions.py +++ b/litellm/anthropic_interface/exceptions/exceptions.py @@ -27,6 +27,7 @@ class AnthropicErrorDetail(TypedDict): type: AnthropicErrorType message: str + details: NotRequired[ReadOnly[Mapping[str, object]]] provider_specific_fields: NotRequired[ReadOnly[Mapping[str, object]]] litellm_error_type: NotRequired[ReadOnly[str]] litellm_call_id: NotRequired[ReadOnly[str]] diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 922de386fe4..c304c947383 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -29,6 +29,7 @@ from litellm.litellm_core_utils.reasoning_effort_capability import ( resolve_deployment_reasoning_effort, ) from litellm.llms.anthropic.common_utils import ( + AnthropicError, AnthropicModelInfo, strip_provider_specific_fields_from_anthropic_messages, tools_marked_for_a_tool_config_point, @@ -58,7 +59,12 @@ from ..safeguards import ( with_safeguard_results, ) from .interceptors import get_messages_interceptors -from .utils import AnthropicMessagesRequestUtils, mock_response, prepare_native_messages +from .utils import ( + AnthropicMessagesRequestUtils, + mock_response, + prepare_native_messages, + supports_message_threads, +) __all__ = ("anthropic_messages", "anthropic_messages_handler") @@ -760,6 +766,14 @@ def anthropic_messages_handler( anthropic_messages_provider_config: Final = _native_messages_config( model, custom_llm_provider, kwargs.get("model_info"), api_base ) + if kwargs.get("thread") is not None and not supports_message_threads( # pyright: ignore[reportUnknownMemberType] # **kwargs of the handler is untyped + custom_llm_provider, has_native_config=anthropic_messages_provider_config is not None + ): + raise AnthropicError( + status_code=400, + message=f"thread: message threads are not supported for provider {custom_llm_provider}. " + "Send the full conversation without the thread field", + ) if anthropic_messages_provider_config is None: # Route to Responses API for OpenAI / Azure, chat/completions for everything else. if kwargs.get("compaction") is None and _should_route_to_responses_api( diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index 5281ad99620..e232390d7e6 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -59,6 +59,7 @@ from .mid_conversation_system import ( convert_mid_conversation_system_turns, is_system_role_message, ) +from .utils import continues_message_thread DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01" @@ -137,6 +138,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): "output_config", "reasoning_effort", "safeguards", + "thread", # TODO: Add Anthropic `metadata` support # "metadata", ] @@ -767,10 +769,13 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): # Auto-strip advisor blocks from history if advisor tool is absent. # Prevents Anthropic 400: advisor_tool_result in history requires advisor tool. + # A thread continue turn omits tools by contract and the stored tools still apply, so it keeps them. tools: Final = anthropic_messages_optional_request_params.get("tools") _tools: Final = tools or [] _has_advisor: Final = any(isinstance(t, dict) and t.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for t in _tools) - if not _has_advisor: + if not _has_advisor and not continues_message_thread( + anthropic_messages_optional_request_params.get("thread") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # the request params arrive as a bare dict + ): messages = strip_advisor_blocks_from_messages(messages) gated: Final = ( @@ -793,7 +798,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): ), max_tokens=max_tokens, model=model, - **anthropic_messages_optional_request_params, + **anthropic_messages_optional_request_params, # pyright: ignore[reportUnknownArgumentType] # the request params arrive as a bare dict ) request: Final = ( {**dict(anthropic_messages_request), "thinking": gated.thinking} @@ -910,6 +915,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if messages_carry_output_config(messages): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value) + if optional_params.get("thread") is not None: # pyright: ignore[reportUnknownMemberType] # the optional params arrive as a bare dict + beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.MESSAGE_THREADS_2026_08_12.value) + tools: Final = optional_params.get("tools") if any(isinstance(tool, dict) and tool.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for tool in tools or ()): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.ADVISOR_TOOL_2026_03_01.value) diff --git a/litellm/llms/anthropic/messages/native/utils.py b/litellm/llms/anthropic/messages/native/utils.py index ea3afdb9a8f..77bd1b65f11 100644 --- a/litellm/llms/anthropic/messages/native/utils.py +++ b/litellm/llms/anthropic/messages/native/utils.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Final, cast, get_type_hints from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError, with_config +from litellm.anthropic_beta_headers_manager import is_beta_header_supported_by_bundled_config from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.anthropic.common_utils import ( @@ -13,7 +14,9 @@ from litellm.llms.anthropic.common_utils import ( ) from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enabled from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, AnthropicMessagesRequestOptionalParams, + AnthropicMessageThread, AnthropicStopDetails, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( @@ -279,6 +282,32 @@ def prepare_native_messages( ) +_THREAD_ADAPTER: Final = TypeAdapter(AnthropicMessageThread) + + +def continues_message_thread(thread: object) -> bool: + """True when ``thread`` is a well-formed ``{"type": "continue", ...}`` body field.""" + try: + return _THREAD_ADAPTER.validate_python(thread)["type"] == "continue" + except ValidationError: + return False + + +def supports_message_threads(custom_llm_provider: str | None, *, has_native_config: bool) -> bool: + """ + Whether a ``thread`` body field can reach the upstream platform. Only a native Anthropic Messages + leg whose beta header column maps the message-threads beta can, since a translated leg has no + server-side conversation to continue and a platform whose column lacks the beta rejects the field. + """ + return ( + has_native_config + and custom_llm_provider is not None + and is_beta_header_supported_by_bundled_config( + ANTHROPIC_BETA_HEADER_VALUES.MESSAGE_THREADS_2026_08_12.value, custom_llm_provider + ) + ) + + @lru_cache(maxsize=1) def anthropic_messages_optional_param_keys() -> frozenset[str]: """ diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index c0c5d74a88a..1bb49cb2661 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -310,6 +310,7 @@ from litellm.router_utils.cooldown_handlers import ( is_caller_scoped_auth_failure, is_caller_timeout_408, is_failure_outside_selected_deployment, + is_message_thread_not_found, is_video_lookup_not_found, mark_deployment_selection_failure, set_cooldown_deployments, @@ -10309,6 +10310,13 @@ class PythonCore(DeprecatedRouterMembers): ) return False + if is_message_thread_not_found(exception): # pyright: ignore[reportUnknownArgumentType] # router callback kwargs arrive untyped + verbose_router_logger.debug( + "Router: Exiting 'deployment_callback_on_failure' without cooldown. " + "Anthropic 404 says the caller's message thread is gone, not that the deployment is down." + ) + return False + exception_status: Final = getattr(exception, "status_code", "") failure_call_details: Final = _CALL_KWARGS_ADAPTER.validate_python(kwargs) diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 7a8a645ad13..48f6e260ed2 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -32,6 +32,7 @@ from litellm.litellm_core_utils.litellm_logging import mask_api_base_credentials from litellm.router_deprecated import renamed_router_param from litellm.router_utils.cooldown_cache import ROUTER_COOLDOWNS_TARGET, CooldownCacheValue from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback +from litellm.types.llms.anthropic import MESSAGE_THREAD_NOT_FOUND_ERROR_CODE from litellm.types.router import Deployment from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, @@ -108,6 +109,12 @@ def is_video_lookup_not_found(exception: object, failure_kwargs: Mapping[str, ob return getattr(exception, "status_code", None) == 404 and failure_kwargs.get("call_type") in VIDEO_ID_CALL_TYPES +def is_message_thread_not_found(exception: object) -> bool: + """Whether a message-thread continue failed with Anthropic's 404 ``thread_not_found``: the caller's + ``previous_message_id`` is stale and the SDK replays the conversation, the deployment is healthy.""" + return getattr(exception, "status_code", None) == 404 and MESSAGE_THREAD_NOT_FOUND_ERROR_CODE in str(exception) + + _EXCEPTION_POLICY_FIELDS: Final[tuple[tuple[type, str], ...]] = ( # ContentPolicyViolationError subclasses BadRequestError, so it must be checked first. (litellm.ContentPolicyViolationError, "ContentPolicyViolationErrorAllowedFails"), diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 802f6794839..e9a5573f49c 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -416,6 +416,14 @@ class AnthropicCompaction(TypedDict, total=False): instructions: ReadOnly[str] +class AnthropicMessageThread(TypedDict, total=False): + """``thread`` body field of the message-threads beta: ``{"type": "create"}`` opens a server-side + thread, ``{"type": "continue", "previous_message_id": "msg_..."}`` appends only the new messages to it.""" + + type: Required[ReadOnly[Literal["create", "continue"]]] + previous_message_id: ReadOnly[str] + + class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): max_tokens: int | None metadata: AnthropicMetadata | dict | None @@ -440,6 +448,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): reasoning_effort: str | None safeguards: ReadOnly[list[dict[str, object]] | None] service_tier: ReadOnly[str | None] + thread: ReadOnly[AnthropicMessageThread | None] class AnthropicMessagesRequest(AnthropicMessagesRequestOptionalParams, total=False): @@ -851,6 +860,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): PER_TURN_CONTROL_2026_07_01 = "per-turn-control-2026-07-01" DANGEROUS_TOOL_USE_2026_09_03 = "dangerous-tool-use-2026-09-03" MCP_CLIENT_2025_04_04 = "mcp-client-2025-04-04" + MESSAGE_THREADS_2026_08_12 = "message-threads-2026-08-12" # Tool search beta header constant (for Anthropic direct API and Microsoft Foundry) @@ -868,6 +878,8 @@ ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24" ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: Final = "mid-conversation-output-config-2026-07-01" ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: Final = "thinking-display-updates-2026-08-18" + +MESSAGE_THREAD_NOT_FOUND_ERROR_CODE: Final = "thread_not_found" ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER: Final = "thinking-binding-controls-2026-08-01" ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: Final = "mid-conversation-tool-changes-2026-07-01" diff --git a/tests/integration/messages_endpoint/test_message_threads_wire.py b/tests/integration/messages_endpoint/test_message_threads_wire.py new file mode 100644 index 00000000000..b924d2b4a4a --- /dev/null +++ b/tests/integration/messages_endpoint/test_message_threads_wire.py @@ -0,0 +1,194 @@ +import json +import re +import uuid +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import pytest +from integration._support.client import Gateway, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.shares_worker_state + +ANTHROPIC_MODEL: Final = "threads-anthropic" +BRIDGED_MODEL: Final = "threads-bridged" +MESSAGE_THREADS_BETA: Final = "message-threads-2026-08-12" +STALE_PREVIOUS_MESSAGE_ID: Final = "msg_01stale" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_NONCE: Final = re.compile(rb"nonce-([0-9a-f]{32})") + + +def nonce() -> str: + return f"nonce-{uuid.uuid4().hex}" + + +def thread_not_found_reply() -> Reply: + return Reply( + status=404, + body=json.dumps( + { + "type": "error", + "error": { + "type": "not_found_error", + "message": "No thread state was found for the requested `previous_message_id`.", + "details": {"error_code": "thread_not_found"}, + }, + "request_id": "req_upstream", + } + ).encode(), + ) + + +def respond(request: Request) -> Reply: + if request.method != "POST": + return Reply(body=b'{"object":"list","data":[]}') + match: Final = _NONCE.search(request.body) + assert match is not None, request.body[:200] + identity: Final = match.group(1).decode() + if STALE_PREVIOUS_MESSAGE_ID.encode() in request.body: + return thread_not_found_reply() + return Reply( + body=json.dumps( + { + "id": f"msg_{identity}", + "type": "message", + "role": "assistant", + "model": "claude-opus-4-8", + "content": [{"type": "text", "text": f"answered {identity}"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 3}, + } + ).encode() + ) + + +def write_config(directory: Path, wire_url: str) -> Path: + path: Final = directory / f"threads-{uuid.uuid4().hex}.json" + path.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": ANTHROPIC_MODEL, + "litellm_params": { + "model": "anthropic/claude-opus-4-8", + "api_base": f"{wire_url}/anthropic", + "api_key": "synthetic-anthropic-key", + }, + }, + { + "model_name": BRIDGED_MODEL, + "litellm_params": { + "model": "openai/gpt-5.5", + "api_base": f"{wire_url}/openai", + "api_key": "synthetic-openai-key", + }, + }, + ], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + }, + } + ) + ) + return path + + +@dataclass(frozen=True, slots=True) +class Rig: + gateway: Gateway + wire: Wire + + def send(self, model: str, request_nonce: str, thread: Mapping[str, str], **fields: JsonValue) -> tuple[int, bytes]: + response: Final = self.gateway.client.post( + "/v1/messages", + json={ + "model": model, + "max_tokens": 64, + "messages": [{"role": "user", "content": f"hi {request_nonce}"}], + "thread": dict(thread), + **fields, + }, + headers={"Authorization": f"Bearer {self.gateway.key}"}, + timeout=120, + ) + return response.status_code, response.content + + def received(self, request_nonce: str) -> tuple[Request, ...]: + return tuple(request for request in self.wire.drain() if request_nonce.encode() in request.body) + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("threads") + with ( + gateway_from_environment() as shared, + wire_server(respond) as wire, + owned_proxy( + shared, + directory, + {"LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS": "true"}, + config=write_config(directory, wire.url), + workers=1, + ) as gateway, + ): + yield Rig(gateway, wire) + + +def beta_header(headers: Mapping[str, str]) -> frozenset[str]: + return frozenset( + piece for key, value in headers.items() if key.lower() == "anthropic-beta" for piece in value.split(",") + ) # comprehension-ok: one header line may carry several comma-separated betas + + +def test_a_create_turn_reaches_anthropic_with_the_thread_field_and_its_beta(rig: Rig) -> None: + request_nonce: Final = nonce() + status, raw = rig.send(ANTHROPIC_MODEL, request_nonce, {"type": "create"}, system="be terse") + assert status == 200, raw[:500] + assert _JSON_OBJECT.validate_json(raw)["id"] == f"msg_{request_nonce.removeprefix('nonce-')}" + (upstream,) = rig.received(request_nonce) + body: Final = _JSON_OBJECT.validate_json(upstream.body) + assert body["thread"] == {"type": "create"} + assert body["system"] == "be terse" + assert MESSAGE_THREADS_BETA in beta_header(upstream.headers), upstream.headers + + +def test_a_continue_turn_reaches_anthropic_with_only_the_new_messages_and_the_same_beta(rig: Rig) -> None: + request_nonce: Final = nonce() + thread: Final = {"type": "continue", "previous_message_id": "msg_01previous"} + status, raw = rig.send(ANTHROPIC_MODEL, request_nonce, thread) + assert status == 200, raw[:500] + (upstream,) = rig.received(request_nonce) + body: Final = _JSON_OBJECT.validate_json(upstream.body) + assert body["thread"] == thread + assert body["messages"] == [{"role": "user", "content": f"hi {request_nonce}"}] + assert "system" not in body and "tools" not in body + assert MESSAGE_THREADS_BETA in beta_header(upstream.headers), upstream.headers + + +def test_a_thread_on_a_translated_leg_is_a_400_and_never_reaches_the_provider(rig: Rig) -> None: + request_nonce: Final = nonce() + status, raw = rig.send(BRIDGED_MODEL, request_nonce, {"type": "continue", "previous_message_id": "msg_01p"}) + assert status == 400, raw[:500] + error: Final = _JSON_OBJECT.validate_json(raw) + assert "message threads are not supported" in json.dumps(error), error + assert rig.received(request_nonce) == () + + +def test_a_thread_not_found_from_anthropic_keeps_its_error_code_for_the_sdk_to_replay_on(rig: Rig) -> None: + request_nonce: Final = nonce() + status, raw = rig.send( + ANTHROPIC_MODEL, request_nonce, {"type": "continue", "previous_message_id": STALE_PREVIOUS_MESSAGE_ID} + ) + assert status == 404, raw[:500] + error: Final = _JSON_OBJECT.validate_json(raw) + detail: Final = error["error"] + assert isinstance(detail, dict), error + assert detail["type"] == "not_found_error" + assert detail["details"] == {"error_code": "thread_not_found"} diff --git a/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py b/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py index 210ab6717aa..0e0e1c8c53e 100644 --- a/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py +++ b/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py @@ -258,3 +258,48 @@ class TestAnthropicErrorSseFrame: "type": "error", "error": {"type": "api_error", "message": "upstream unavailable"}, } + + +THREAD_NOT_FOUND_UPSTREAM: Final = json.dumps( + { + "type": "error", + "error": { + "type": "not_found_error", + "message": "No thread state was found for the requested `previous_message_id`.", + "details": {"error_code": "thread_not_found"}, + }, + "request_id": "req_upstream", + } +) + + +class TestEmbeddedErrorDetails: + def test_a_wrapped_upstream_error_keeps_its_details_next_to_the_wrapped_message(self) -> None: + wrapped: Final = ( + f"litellm.NotFoundError: AnthropicException - {THREAD_NOT_FOUND_UPSTREAM}\n\n" + "LiteLLM: model group 'haiku' failed with the error above. No fallback was attempted." + ) + response: Final = AnthropicExceptionMapping.transform_to_anthropic_error(status_code=404, raw_message=wrapped) + assert response["error"]["details"] == {"error_code": "thread_not_found"} + assert response["error"]["type"] == "not_found_error" + assert response["error"]["message"] == wrapped + + def test_the_error_event_of_a_failed_stream_carries_the_same_details(self) -> None: + wrapped: Final = f"litellm.NotFoundError: AnthropicException - {THREAD_NOT_FOUND_UPSTREAM}" + frame: Final = anthropic_error_sse_frame(404, wrapped) + assert frame.error_response["error"]["details"] == {"error_code": "thread_not_found"} + + @pytest.mark.parametrize( + "raw_message", + ( + "plain text with a { brace and no json", + 'litellm.BadRequestError: OpenAIException - {"error": {"message": "bad", "type": "invalid_request_error"}}', + 'AnthropicException - {"type": "error", "error": {"type": "api_error", "message": "no details here"}}', + ), + ids=("no_json", "openai_shape", "anthropic_without_details"), + ) + def test_a_message_without_anthropic_details_gets_no_details_key(self, raw_message: str) -> None: + response: Final = AnthropicExceptionMapping.transform_to_anthropic_error( + status_code=400, raw_message=raw_message + ) + assert "details" not in response["error"] diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index 8a924dd32d6..4287e90b043 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -2716,3 +2716,23 @@ async def test_anthropic_messages_strict_tools_rejection_is_sent_again_without_s assert first["tools"] == [tool] assert second == {**first, "tools": [{"name": "lookup_record", "input_schema": {"type": "object"}}]} assert _JSON_BODY.validate_python(response)["id"] == "msg_strict_retry" + + +@pytest.mark.parametrize( + "model", + ("openai/gpt-5.5", "vertex_ai/claude-opus-4-8", "bedrock/anthropic.claude-opus-4-8-v1:0"), + ids=("chat_bridge", "vertex_native", "bedrock_native"), +) +def test_a_thread_on_a_leg_without_message_threads_is_a_400_before_any_upstream_call(model: str) -> None: + from litellm.llms.anthropic.common_utils import AnthropicError + from litellm.llms.anthropic.messages.native.handler import anthropic_messages_handler + + with pytest.raises(AnthropicError, match="thread: message threads are not supported") as raised: + anthropic_messages_handler( + max_tokens=64, + messages=[{"role": "user", "content": "hi"}], + model=model, + api_key="synthetic-key", + thread={"type": "continue", "previous_message_id": "msg_01previous"}, + ) + assert raised.value.status_code == 400 diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py index 54b29ca6aa7..b5e9f22d946 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py @@ -721,3 +721,93 @@ def test_a_first_party_messages_request_never_fetches_the_remote_beta_headers_co assert request["messages"][1] == {"role": "system", "content": "be terse"} assert not remote_config.called + + +MESSAGE_THREADS_BETA: Final = "message-threads-2026-08-12" + + +@pytest.mark.parametrize( + "thread", + ({"type": "create"}, {"type": "continue", "previous_message_id": "msg_01previous"}), + ids=("create", "continue"), +) +def test_messages_request_forwards_the_thread_field_to_anthropic(thread: dict[str, str]) -> None: + request: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType] # bare-dict signature + model="claude-opus-4-8", + messages=[{"role": "user", "content": "what did I just say"}], + anthropic_messages_optional_request_params={"max_tokens": 64, "thread": dict(thread)}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert request["thread"] == thread + assert request["messages"] == [{"role": "user", "content": "what did I just say"}] + + +def test_messages_headers_add_the_message_threads_beta_next_to_the_callers_betas() -> None: + headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType] # bare-dict signature + headers={"anthropic-beta": "context-1m-2025-08-07"}, + optional_params={"thread": {"type": "continue", "previous_message_id": "msg_01previous"}}, + messages=[], + ) + assert headers["anthropic-beta"] == f"context-1m-2025-08-07,{MESSAGE_THREADS_BETA}" + + +def test_messages_headers_add_no_message_threads_beta_without_a_thread() -> None: + headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType] # bare-dict signature + headers={}, optional_params={"max_tokens": 64}, messages=[] + ) + assert MESSAGE_THREADS_BETA not in headers.get("anthropic-beta", "") # pyright: ignore[reportUnknownMemberType] # bare-dict result + + +def test_the_thread_beta_is_the_same_on_the_create_and_the_continue_turn() -> None: + config: Final = AnthropicMessagesConfig() + create: Final = config._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType] # bare-dict signature + headers={}, optional_params={"thread": {"type": "create"}, "system": "be terse"}, messages=[] + ) + cont: Final = config._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType] # bare-dict signature + headers={}, optional_params={"thread": {"type": "continue", "previous_message_id": "msg_01p"}}, messages=[] + ) + assert create["anthropic-beta"] == cont["anthropic-beta"] == MESSAGE_THREADS_BETA + + +def _advisor_turn() -> list[dict[str, object]]: + return [ + { + "role": "assistant", + "content": [ + {"type": "server_tool_use", "id": "srvtoolu_01", "name": "advisor", "input": {}}, + { + "type": "advisor_tool_result", + "tool_use_id": "srvtoolu_01", + "content": [{"type": "text", "text": "ok"}], + }, + {"type": "text", "text": "done"}, + ], + }, + {"role": "user", "content": "and now?"}, + ] + + +def test_a_thread_continue_turn_keeps_advisor_blocks_the_stored_tools_still_explain() -> None: + request: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType] # bare-dict signature + model="claude-opus-4-8", + messages=_advisor_turn(), + anthropic_messages_optional_request_params={ + "max_tokens": 64, + "thread": {"type": "continue", "previous_message_id": "msg_01previous"}, + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert request["messages"][0]["content"] == _advisor_turn()[0]["content"] + + +def test_a_request_without_a_thread_still_strips_advisor_blocks_when_the_tool_is_absent() -> None: + request: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType] # bare-dict signature + model="claude-opus-4-8", + messages=_advisor_turn(), + anthropic_messages_optional_request_params={"max_tokens": 64}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert request["messages"][0]["content"] == [{"type": "text", "text": "done"}] diff --git a/tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py b/tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py index d525ccfec37..3dfd476f2c9 100644 --- a/tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py +++ b/tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py @@ -6,6 +6,8 @@ Regression tests for the /v1/messages request-parse fast paths: while resolving the (static) type hints only once per process. """ +from typing import Final + import pytest import litellm @@ -133,3 +135,11 @@ def test_sampling_param_raises_clean_400_without_drop_params(monkeypatch): model="claude-opus-4-7", drop_params=False, ) + + +def test_the_thread_field_survives_optional_param_filtering(): + thread: Final = {"type": "continue", "previous_message_id": "msg_01previous"} + result: Final = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + {"max_tokens": 64, "thread": thread} + ) + assert result == {"max_tokens": 64, "thread": thread} diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 71c37b5b1d5..0386260b729 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -3672,3 +3672,72 @@ def test_the_refusal_names_each_cooled_deployment_some_cooldown_state_routes_the with pytest.raises(RouterRateLimitError) as refusal: _select(_tenant_router(EVERY_TENANT, **BOTH_ON), "async", "tenants", copy.deepcopy(request_kwargs)) assert {deployment["model_id"] for deployment in refusal.value.cooldown_deployments or ()} == reachable + + +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +class TestMessageThreadNotFoundCooldown: + THREAD_NOT_FOUND_BODY: Final = ( + '{"type":"error","error":{"type":"not_found_error","message":"No thread state was found for the ' + 'requested `previous_message_id`.","details":{"error_code":"thread_not_found"}}}' + ) + + @staticmethod + def _not_found(message: str) -> Exception: + return litellm.NotFoundError( + message=f"AnthropicException - {message}", + model="claude-opus-4-8", + llm_provider="anthropic", + ) + + @pytest.mark.asyncio + async def test_a_stale_thread_404_leaves_the_deployment_out_of_cooldown(self) -> None: + model_id: Final = "anthropic-threads" + router: Final = _make_router( + model_list=[ + { + "model_name": "claude", + "litellm_params": {"model": "anthropic/claude-opus-4-8", "api_key": "synthetic-key"}, + "model_info": {"id": model_id}, + } + ], + ) + + result: Final = router.deployment_callback_on_failure( + kwargs={ + "exception": self._not_found(self.THREAD_NOT_FOUND_BODY), + "litellm_params": {"model_info": {"id": model_id}}, + }, + completion_response=None, + start_time=0, + end_time=1, + ) + + assert result is False + assert get_cooldown_deployments(RoutingContext(router), parent_otel_span=None) == [] + + @pytest.mark.asyncio + async def test_a_404_without_the_thread_code_still_cools_the_deployment_down(self) -> None: + model_id: Final = "anthropic-missing-model" + router: Final = _make_router( + model_list=[ + { + "model_name": "claude", + "litellm_params": {"model": "anthropic/claude-opus-4-8", "api_key": "synthetic-key"}, + "model_info": {"id": model_id}, + } + ], + ) + + router.deployment_callback_on_failure( + kwargs={ + "exception": self._not_found( + '{"type":"error","error":{"type":"not_found_error","message":"model: x"}}' + ), + "litellm_params": {"model_info": {"id": model_id}}, + }, + completion_response=None, + start_time=0, + end_time=1, + ) + + assert get_cooldown_deployments(RoutingContext(router), parent_otel_span=None) == [model_id]