mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-14 09:03:16 +00:00
Merge 2ed0e5bfa6 into 732a50318b
This commit is contained in:
Mateo Wang
GitHub
commit
b157d822a1
15 changed files with 551 additions and 8 deletions
|
|
@ -21,6 +21,7 @@
|
|||
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
|
||||
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
|
||||
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
|
||||
"message-threads-2026-08-12": "message-threads-2026-08-12",
|
||||
"mcp-servers-2025-12-04": null,
|
||||
"oauth-2025-04-20": "oauth-2025-04-20",
|
||||
"output-128k-2025-02-19": "output-128k-2025-02-19",
|
||||
|
|
@ -56,7 +57,6 @@
|
|||
"mcp-client-2026-09-15": "mcp-client-2026-09-15",
|
||||
"mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
|
||||
"message-batches-2024-09-24": "message-batches-2024-09-24",
|
||||
"message-threads-2026-08-12": "message-threads-2026-08-12",
|
||||
"mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
|
||||
"model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
|
||||
"output-300k-2026-03-24": "output-300k-2026-03-24",
|
||||
|
|
@ -98,6 +98,7 @@
|
|||
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
|
||||
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
|
||||
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
|
||||
"message-threads-2026-08-12": "message-threads-2026-08-12",
|
||||
"mcp-servers-2025-12-04": null,
|
||||
"output-128k-2025-02-19": "output-128k-2025-02-19",
|
||||
"structured-output-2024-03-01": null,
|
||||
|
|
@ -132,7 +133,6 @@
|
|||
"mcp-client-2026-09-15": "mcp-client-2026-09-15",
|
||||
"mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
|
||||
"message-batches-2024-09-24": "message-batches-2024-09-24",
|
||||
"message-threads-2026-08-12": "message-threads-2026-08-12",
|
||||
"mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
|
||||
"mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
|
||||
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
|
||||
|
|
@ -177,6 +177,7 @@
|
|||
"interleaved-thinking-2025-05-14": null,
|
||||
"mcp-client-2025-11-20": null,
|
||||
"mcp-client-2025-04-04": null,
|
||||
"message-threads-2026-08-12": null,
|
||||
"mcp-servers-2025-12-04": null,
|
||||
"output-128k-2025-02-19": null,
|
||||
"structured-output-2024-03-01": null,
|
||||
|
|
@ -212,6 +213,7 @@
|
|||
"interleaved-thinking-2025-05-14": null,
|
||||
"mcp-client-2025-11-20": null,
|
||||
"mcp-client-2025-04-04": null,
|
||||
"message-threads-2026-08-12": null,
|
||||
"mcp-servers-2025-12-04": null,
|
||||
"output-128k-2025-02-19": null,
|
||||
"structured-output-2024-03-01": null,
|
||||
|
|
@ -249,6 +251,7 @@
|
|||
"fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
|
||||
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
|
||||
"mcp-client-2025-04-04": null,
|
||||
"message-threads-2026-08-12": null,
|
||||
"mcp-client-2025-11-20": null,
|
||||
"mcp-servers-2025-12-04": null,
|
||||
"output-128k-2025-02-19": "output-128k-2025-02-19",
|
||||
|
|
@ -286,6 +289,7 @@
|
|||
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
|
||||
"mcp-client-2025-11-20": null,
|
||||
"mcp-client-2025-04-04": null,
|
||||
"message-threads-2026-08-12": null,
|
||||
"mcp-servers-2025-12-04": null,
|
||||
"output-128k-2025-02-19": "output-128k-2025-02-19",
|
||||
"structured-output-2024-03-01": null,
|
||||
|
|
@ -321,7 +325,6 @@
|
|||
"mcp-client-2026-09-15": null,
|
||||
"mcp-tunnels-2026-06-22": null,
|
||||
"message-batches-2024-09-24": null,
|
||||
"message-threads-2026-08-12": null,
|
||||
"mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
|
||||
"mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
|
||||
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
|
||||
|
|
@ -386,6 +389,7 @@
|
|||
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
|
||||
"mcp-client-2025-11-20": null,
|
||||
"mcp-client-2025-04-04": null,
|
||||
"message-threads-2026-08-12": null,
|
||||
"mcp-servers-2025-12-04": null,
|
||||
"oauth-2025-04-20": null,
|
||||
"output-128k-2025-02-19": "output-128k-2025-02-19",
|
||||
|
|
|
|||
|
|
@ -5,15 +5,18 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop
|
|||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
|
||||
from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicErrorType
|
||||
|
||||
_PARSED_ERROR_BODY: Final = TypeAdapter(object)
|
||||
_ERROR_DETAILS: Final = TypeAdapter(Mapping[str, object])
|
||||
_JSON_DECODER: Final = json.JSONDecoder()
|
||||
|
||||
# HTTP status code -> Anthropic error type
|
||||
# Source: https://platform.claude.com/docs/en/api/errors
|
||||
|
|
@ -166,12 +169,41 @@ class AnthropicExceptionMapping:
|
|||
else:
|
||||
message = raw_message
|
||||
|
||||
return AnthropicExceptionMapping.create_error_response(
|
||||
response: Final = AnthropicExceptionMapping.create_error_response(
|
||||
status_code=status_code,
|
||||
message=message,
|
||||
request_id=request_id,
|
||||
error_type=error_type,
|
||||
)
|
||||
details: Final = AnthropicExceptionMapping.embedded_error_details(raw_message)
|
||||
if details is None:
|
||||
return response
|
||||
with_details: Final[AnthropicErrorResponse] = {**response, "error": {**response["error"], "details": details}}
|
||||
return with_details
|
||||
|
||||
@staticmethod
|
||||
def embedded_error_details(raw_message: str) -> Mapping[str, object] | None:
|
||||
"""
|
||||
``error.details`` of the Anthropic error body quoted inside ``raw_message``, when there is one.
|
||||
|
||||
litellm wraps an upstream error as ``"<Exception> - {upstream json}\n\nLiteLLM: ..."``. Anthropic
|
||||
puts machine-readable codes such as ``thread_not_found`` under ``error.details``, and the SDKs
|
||||
branch on them, so the code is lifted next to the wrapped message instead of staying only inside it.
|
||||
"""
|
||||
start: Final = raw_message.find("{")
|
||||
if start < 0:
|
||||
return None
|
||||
try:
|
||||
embedded, _ = _JSON_DECODER.raw_decode(raw_message, start)
|
||||
except ValueError:
|
||||
return None
|
||||
try:
|
||||
envelope: Final = _ERROR_DETAILS.validate_python(embedded, strict=True)
|
||||
error: Final = _ERROR_DETAILS.validate_python(envelope.get("error"), strict=True)
|
||||
details: Final = _ERROR_DETAILS.validate_python(error.get("details"), strict=True)
|
||||
except ValidationError:
|
||||
return None
|
||||
return details if envelope.get("type") == "error" else None
|
||||
|
||||
|
||||
class AnthropicErrorSseFrame(str):
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ class AnthropicErrorDetail(TypedDict):
|
|||
|
||||
type: AnthropicErrorType
|
||||
message: str
|
||||
details: NotRequired[ReadOnly[Mapping[str, object]]]
|
||||
provider_specific_fields: NotRequired[ReadOnly[Mapping[str, object]]]
|
||||
litellm_error_type: NotRequired[ReadOnly[str]]
|
||||
litellm_call_id: NotRequired[ReadOnly[str]]
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from litellm.litellm_core_utils.reasoning_effort_capability import (
|
|||
resolve_deployment_reasoning_effort,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
strip_provider_specific_fields_from_anthropic_messages,
|
||||
tools_marked_for_a_tool_config_point,
|
||||
|
|
@ -58,7 +59,12 @@ from ..safeguards import (
|
|||
with_safeguard_results,
|
||||
)
|
||||
from .interceptors import get_messages_interceptors
|
||||
from .utils import AnthropicMessagesRequestUtils, mock_response, prepare_native_messages
|
||||
from .utils import (
|
||||
AnthropicMessagesRequestUtils,
|
||||
mock_response,
|
||||
prepare_native_messages,
|
||||
supports_message_threads,
|
||||
)
|
||||
|
||||
__all__ = ("anthropic_messages", "anthropic_messages_handler")
|
||||
|
||||
|
|
@ -760,6 +766,14 @@ def anthropic_messages_handler(
|
|||
anthropic_messages_provider_config: Final = _native_messages_config(
|
||||
model, custom_llm_provider, kwargs.get("model_info"), api_base
|
||||
)
|
||||
if kwargs.get("thread") is not None and not supports_message_threads( # pyright: ignore[reportUnknownMemberType] # **kwargs of the handler is untyped
|
||||
custom_llm_provider, has_native_config=anthropic_messages_provider_config is not None
|
||||
):
|
||||
raise AnthropicError(
|
||||
status_code=400,
|
||||
message=f"thread: message threads are not supported for provider {custom_llm_provider}. "
|
||||
"Send the full conversation without the thread field",
|
||||
)
|
||||
if anthropic_messages_provider_config is None:
|
||||
# Route to Responses API for OpenAI / Azure, chat/completions for everything else.
|
||||
if kwargs.get("compaction") is None and _should_route_to_responses_api(
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ from .mid_conversation_system import (
|
|||
convert_mid_conversation_system_turns,
|
||||
is_system_role_message,
|
||||
)
|
||||
from .utils import continues_message_thread
|
||||
|
||||
DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01"
|
||||
|
||||
|
|
@ -137,6 +138,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
"output_config",
|
||||
"reasoning_effort",
|
||||
"safeguards",
|
||||
"thread",
|
||||
# TODO: Add Anthropic `metadata` support
|
||||
# "metadata",
|
||||
]
|
||||
|
|
@ -767,10 +769,13 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
|
||||
# Auto-strip advisor blocks from history if advisor tool is absent.
|
||||
# Prevents Anthropic 400: advisor_tool_result in history requires advisor tool.
|
||||
# A thread continue turn omits tools by contract and the stored tools still apply, so it keeps them.
|
||||
tools: Final = anthropic_messages_optional_request_params.get("tools")
|
||||
_tools: Final = tools or []
|
||||
_has_advisor: Final = any(isinstance(t, dict) and t.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for t in _tools)
|
||||
if not _has_advisor:
|
||||
if not _has_advisor and not continues_message_thread(
|
||||
anthropic_messages_optional_request_params.get("thread") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # the request params arrive as a bare dict
|
||||
):
|
||||
messages = strip_advisor_blocks_from_messages(messages)
|
||||
|
||||
gated: Final = (
|
||||
|
|
@ -793,7 +798,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
),
|
||||
max_tokens=max_tokens,
|
||||
model=model,
|
||||
**anthropic_messages_optional_request_params,
|
||||
**anthropic_messages_optional_request_params, # pyright: ignore[reportUnknownArgumentType] # the request params arrive as a bare dict
|
||||
)
|
||||
request: Final = (
|
||||
{**dict(anthropic_messages_request), "thinking": gated.thinking}
|
||||
|
|
@ -910,6 +915,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
if messages_carry_output_config(messages):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
|
||||
|
||||
if optional_params.get("thread") is not None: # pyright: ignore[reportUnknownMemberType] # the optional params arrive as a bare dict
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.MESSAGE_THREADS_2026_08_12.value)
|
||||
|
||||
tools: Final = optional_params.get("tools")
|
||||
if any(isinstance(tool, dict) and tool.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for tool in tools or ()):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.ADVISOR_TOOL_2026_03_01.value)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Final, cast, get_type_hints
|
|||
|
||||
from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError, with_config
|
||||
|
||||
from litellm.anthropic_beta_headers_manager import is_beta_header_supported_by_bundled_config
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
|
|
@ -13,7 +14,9 @@ from litellm.llms.anthropic.common_utils import (
|
|||
)
|
||||
from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enabled
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
AnthropicMessagesRequestOptionalParams,
|
||||
AnthropicMessageThread,
|
||||
AnthropicStopDetails,
|
||||
)
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
|
|
@ -279,6 +282,32 @@ def prepare_native_messages(
|
|||
)
|
||||
|
||||
|
||||
_THREAD_ADAPTER: Final = TypeAdapter(AnthropicMessageThread)
|
||||
|
||||
|
||||
def continues_message_thread(thread: object) -> bool:
|
||||
"""True when ``thread`` is a well-formed ``{"type": "continue", ...}`` body field."""
|
||||
try:
|
||||
return _THREAD_ADAPTER.validate_python(thread)["type"] == "continue"
|
||||
except ValidationError:
|
||||
return False
|
||||
|
||||
|
||||
def supports_message_threads(custom_llm_provider: str | None, *, has_native_config: bool) -> bool:
|
||||
"""
|
||||
Whether a ``thread`` body field can reach the upstream platform. Only a native Anthropic Messages
|
||||
leg whose beta header column maps the message-threads beta can, since a translated leg has no
|
||||
server-side conversation to continue and a platform whose column lacks the beta rejects the field.
|
||||
"""
|
||||
return (
|
||||
has_native_config
|
||||
and custom_llm_provider is not None
|
||||
and is_beta_header_supported_by_bundled_config(
|
||||
ANTHROPIC_BETA_HEADER_VALUES.MESSAGE_THREADS_2026_08_12.value, custom_llm_provider
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def anthropic_messages_optional_param_keys() -> frozenset[str]:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -310,6 +310,7 @@ from litellm.router_utils.cooldown_handlers import (
|
|||
is_caller_scoped_auth_failure,
|
||||
is_caller_timeout_408,
|
||||
is_failure_outside_selected_deployment,
|
||||
is_message_thread_not_found,
|
||||
is_video_lookup_not_found,
|
||||
mark_deployment_selection_failure,
|
||||
set_cooldown_deployments,
|
||||
|
|
@ -10309,6 +10310,13 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
)
|
||||
return False
|
||||
|
||||
if is_message_thread_not_found(exception): # pyright: ignore[reportUnknownArgumentType] # router callback kwargs arrive untyped
|
||||
verbose_router_logger.debug(
|
||||
"Router: Exiting 'deployment_callback_on_failure' without cooldown. "
|
||||
"Anthropic 404 says the caller's message thread is gone, not that the deployment is down."
|
||||
)
|
||||
return False
|
||||
|
||||
exception_status: Final = getattr(exception, "status_code", "")
|
||||
failure_call_details: Final = _CALL_KWARGS_ADAPTER.validate_python(kwargs)
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ from litellm.litellm_core_utils.litellm_logging import mask_api_base_credentials
|
|||
from litellm.router_deprecated import renamed_router_param
|
||||
from litellm.router_utils.cooldown_cache import ROUTER_COOLDOWNS_TARGET, CooldownCacheValue
|
||||
from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback
|
||||
from litellm.types.llms.anthropic import MESSAGE_THREAD_NOT_FOUND_ERROR_CODE
|
||||
from litellm.types.router import Deployment
|
||||
from litellm.types.utils import (
|
||||
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
|
||||
|
|
@ -108,6 +109,12 @@ def is_video_lookup_not_found(exception: object, failure_kwargs: Mapping[str, ob
|
|||
return getattr(exception, "status_code", None) == 404 and failure_kwargs.get("call_type") in VIDEO_ID_CALL_TYPES
|
||||
|
||||
|
||||
def is_message_thread_not_found(exception: object) -> bool:
|
||||
"""Whether a message-thread continue failed with Anthropic's 404 ``thread_not_found``: the caller's
|
||||
``previous_message_id`` is stale and the SDK replays the conversation, the deployment is healthy."""
|
||||
return getattr(exception, "status_code", None) == 404 and MESSAGE_THREAD_NOT_FOUND_ERROR_CODE in str(exception)
|
||||
|
||||
|
||||
_EXCEPTION_POLICY_FIELDS: Final[tuple[tuple[type, str], ...]] = (
|
||||
# ContentPolicyViolationError subclasses BadRequestError, so it must be checked first.
|
||||
(litellm.ContentPolicyViolationError, "ContentPolicyViolationErrorAllowedFails"),
|
||||
|
|
|
|||
|
|
@ -416,6 +416,14 @@ class AnthropicCompaction(TypedDict, total=False):
|
|||
instructions: ReadOnly[str]
|
||||
|
||||
|
||||
class AnthropicMessageThread(TypedDict, total=False):
|
||||
"""``thread`` body field of the message-threads beta: ``{"type": "create"}`` opens a server-side
|
||||
thread, ``{"type": "continue", "previous_message_id": "msg_..."}`` appends only the new messages to it."""
|
||||
|
||||
type: Required[ReadOnly[Literal["create", "continue"]]]
|
||||
previous_message_id: ReadOnly[str]
|
||||
|
||||
|
||||
class AnthropicMessagesRequestOptionalParams(TypedDict, total=False):
|
||||
max_tokens: int | None
|
||||
metadata: AnthropicMetadata | dict | None
|
||||
|
|
@ -440,6 +448,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False):
|
|||
reasoning_effort: str | None
|
||||
safeguards: ReadOnly[list[dict[str, object]] | None]
|
||||
service_tier: ReadOnly[str | None]
|
||||
thread: ReadOnly[AnthropicMessageThread | None]
|
||||
|
||||
|
||||
class AnthropicMessagesRequest(AnthropicMessagesRequestOptionalParams, total=False):
|
||||
|
|
@ -851,6 +860,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum):
|
|||
PER_TURN_CONTROL_2026_07_01 = "per-turn-control-2026-07-01"
|
||||
DANGEROUS_TOOL_USE_2026_09_03 = "dangerous-tool-use-2026-09-03"
|
||||
MCP_CLIENT_2025_04_04 = "mcp-client-2025-04-04"
|
||||
MESSAGE_THREADS_2026_08_12 = "message-threads-2026-08-12"
|
||||
|
||||
|
||||
# Tool search beta header constant (for Anthropic direct API and Microsoft Foundry)
|
||||
|
|
@ -868,6 +878,8 @@ ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24"
|
|||
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: Final = "mid-conversation-output-config-2026-07-01"
|
||||
|
||||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: Final = "thinking-display-updates-2026-08-18"
|
||||
|
||||
MESSAGE_THREAD_NOT_FOUND_ERROR_CODE: Final = "thread_not_found"
|
||||
ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER: Final = "thinking-binding-controls-2026-08-01"
|
||||
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: Final = "mid-conversation-tool-changes-2026-07-01"
|
||||
|
||||
|
|
|
|||
194
tests/integration/messages_endpoint/test_message_threads_wire.py
Normal file
194
tests/integration/messages_endpoint/test_message_threads_wire.py
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
import json
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway, gateway_from_environment
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
pytestmark: Final = pytest.mark.shares_worker_state
|
||||
|
||||
ANTHROPIC_MODEL: Final = "threads-anthropic"
|
||||
BRIDGED_MODEL: Final = "threads-bridged"
|
||||
MESSAGE_THREADS_BETA: Final = "message-threads-2026-08-12"
|
||||
STALE_PREVIOUS_MESSAGE_ID: Final = "msg_01stale"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_NONCE: Final = re.compile(rb"nonce-([0-9a-f]{32})")
|
||||
|
||||
|
||||
def nonce() -> str:
|
||||
return f"nonce-{uuid.uuid4().hex}"
|
||||
|
||||
|
||||
def thread_not_found_reply() -> Reply:
|
||||
return Reply(
|
||||
status=404,
|
||||
body=json.dumps(
|
||||
{
|
||||
"type": "error",
|
||||
"error": {
|
||||
"type": "not_found_error",
|
||||
"message": "No thread state was found for the requested `previous_message_id`.",
|
||||
"details": {"error_code": "thread_not_found"},
|
||||
},
|
||||
"request_id": "req_upstream",
|
||||
}
|
||||
).encode(),
|
||||
)
|
||||
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
if request.method != "POST":
|
||||
return Reply(body=b'{"object":"list","data":[]}')
|
||||
match: Final = _NONCE.search(request.body)
|
||||
assert match is not None, request.body[:200]
|
||||
identity: Final = match.group(1).decode()
|
||||
if STALE_PREVIOUS_MESSAGE_ID.encode() in request.body:
|
||||
return thread_not_found_reply()
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": f"msg_{identity}",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-opus-4-8",
|
||||
"content": [{"type": "text", "text": f"answered {identity}"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 12, "output_tokens": 3},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
|
||||
def write_config(directory: Path, wire_url: str) -> Path:
|
||||
path: Final = directory / f"threads-{uuid.uuid4().hex}.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model_list": [
|
||||
{
|
||||
"model_name": ANTHROPIC_MODEL,
|
||||
"litellm_params": {
|
||||
"model": "anthropic/claude-opus-4-8",
|
||||
"api_base": f"{wire_url}/anthropic",
|
||||
"api_key": "synthetic-anthropic-key",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": BRIDGED_MODEL,
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.5",
|
||||
"api_base": f"{wire_url}/openai",
|
||||
"api_key": "synthetic-openai-key",
|
||||
},
|
||||
},
|
||||
],
|
||||
"general_settings": {
|
||||
"master_key": "os.environ/LITELLM_MASTER_KEY",
|
||||
"database_url": "os.environ/DATABASE_URL",
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Rig:
|
||||
gateway: Gateway
|
||||
wire: Wire
|
||||
|
||||
def send(self, model: str, request_nonce: str, thread: Mapping[str, str], **fields: JsonValue) -> tuple[int, bytes]:
|
||||
response: Final = self.gateway.client.post(
|
||||
"/v1/messages",
|
||||
json={
|
||||
"model": model,
|
||||
"max_tokens": 64,
|
||||
"messages": [{"role": "user", "content": f"hi {request_nonce}"}],
|
||||
"thread": dict(thread),
|
||||
**fields,
|
||||
},
|
||||
headers={"Authorization": f"Bearer {self.gateway.key}"},
|
||||
timeout=120,
|
||||
)
|
||||
return response.status_code, response.content
|
||||
|
||||
def received(self, request_nonce: str) -> tuple[Request, ...]:
|
||||
return tuple(request for request in self.wire.drain() if request_nonce.encode() in request.body)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
|
||||
directory: Final = tmp_path_factory.mktemp("threads")
|
||||
with (
|
||||
gateway_from_environment() as shared,
|
||||
wire_server(respond) as wire,
|
||||
owned_proxy(
|
||||
shared,
|
||||
directory,
|
||||
{"LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS": "true"},
|
||||
config=write_config(directory, wire.url),
|
||||
workers=1,
|
||||
) as gateway,
|
||||
):
|
||||
yield Rig(gateway, wire)
|
||||
|
||||
|
||||
def beta_header(headers: Mapping[str, str]) -> frozenset[str]:
|
||||
return frozenset(
|
||||
piece for key, value in headers.items() if key.lower() == "anthropic-beta" for piece in value.split(",")
|
||||
) # comprehension-ok: one header line may carry several comma-separated betas
|
||||
|
||||
|
||||
def test_a_create_turn_reaches_anthropic_with_the_thread_field_and_its_beta(rig: Rig) -> None:
|
||||
request_nonce: Final = nonce()
|
||||
status, raw = rig.send(ANTHROPIC_MODEL, request_nonce, {"type": "create"}, system="be terse")
|
||||
assert status == 200, raw[:500]
|
||||
assert _JSON_OBJECT.validate_json(raw)["id"] == f"msg_{request_nonce.removeprefix('nonce-')}"
|
||||
(upstream,) = rig.received(request_nonce)
|
||||
body: Final = _JSON_OBJECT.validate_json(upstream.body)
|
||||
assert body["thread"] == {"type": "create"}
|
||||
assert body["system"] == "be terse"
|
||||
assert MESSAGE_THREADS_BETA in beta_header(upstream.headers), upstream.headers
|
||||
|
||||
|
||||
def test_a_continue_turn_reaches_anthropic_with_only_the_new_messages_and_the_same_beta(rig: Rig) -> None:
|
||||
request_nonce: Final = nonce()
|
||||
thread: Final = {"type": "continue", "previous_message_id": "msg_01previous"}
|
||||
status, raw = rig.send(ANTHROPIC_MODEL, request_nonce, thread)
|
||||
assert status == 200, raw[:500]
|
||||
(upstream,) = rig.received(request_nonce)
|
||||
body: Final = _JSON_OBJECT.validate_json(upstream.body)
|
||||
assert body["thread"] == thread
|
||||
assert body["messages"] == [{"role": "user", "content": f"hi {request_nonce}"}]
|
||||
assert "system" not in body and "tools" not in body
|
||||
assert MESSAGE_THREADS_BETA in beta_header(upstream.headers), upstream.headers
|
||||
|
||||
|
||||
def test_a_thread_on_a_translated_leg_is_a_400_and_never_reaches_the_provider(rig: Rig) -> None:
|
||||
request_nonce: Final = nonce()
|
||||
status, raw = rig.send(BRIDGED_MODEL, request_nonce, {"type": "continue", "previous_message_id": "msg_01p"})
|
||||
assert status == 400, raw[:500]
|
||||
error: Final = _JSON_OBJECT.validate_json(raw)
|
||||
assert "message threads are not supported" in json.dumps(error), error
|
||||
assert rig.received(request_nonce) == ()
|
||||
|
||||
|
||||
def test_a_thread_not_found_from_anthropic_keeps_its_error_code_for_the_sdk_to_replay_on(rig: Rig) -> None:
|
||||
request_nonce: Final = nonce()
|
||||
status, raw = rig.send(
|
||||
ANTHROPIC_MODEL, request_nonce, {"type": "continue", "previous_message_id": STALE_PREVIOUS_MESSAGE_ID}
|
||||
)
|
||||
assert status == 404, raw[:500]
|
||||
error: Final = _JSON_OBJECT.validate_json(raw)
|
||||
detail: Final = error["error"]
|
||||
assert isinstance(detail, dict), error
|
||||
assert detail["type"] == "not_found_error"
|
||||
assert detail["details"] == {"error_code": "thread_not_found"}
|
||||
|
|
@ -258,3 +258,48 @@ class TestAnthropicErrorSseFrame:
|
|||
"type": "error",
|
||||
"error": {"type": "api_error", "message": "upstream unavailable"},
|
||||
}
|
||||
|
||||
|
||||
THREAD_NOT_FOUND_UPSTREAM: Final = json.dumps(
|
||||
{
|
||||
"type": "error",
|
||||
"error": {
|
||||
"type": "not_found_error",
|
||||
"message": "No thread state was found for the requested `previous_message_id`.",
|
||||
"details": {"error_code": "thread_not_found"},
|
||||
},
|
||||
"request_id": "req_upstream",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class TestEmbeddedErrorDetails:
|
||||
def test_a_wrapped_upstream_error_keeps_its_details_next_to_the_wrapped_message(self) -> None:
|
||||
wrapped: Final = (
|
||||
f"litellm.NotFoundError: AnthropicException - {THREAD_NOT_FOUND_UPSTREAM}\n\n"
|
||||
"LiteLLM: model group 'haiku' failed with the error above. No fallback was attempted."
|
||||
)
|
||||
response: Final = AnthropicExceptionMapping.transform_to_anthropic_error(status_code=404, raw_message=wrapped)
|
||||
assert response["error"]["details"] == {"error_code": "thread_not_found"}
|
||||
assert response["error"]["type"] == "not_found_error"
|
||||
assert response["error"]["message"] == wrapped
|
||||
|
||||
def test_the_error_event_of_a_failed_stream_carries_the_same_details(self) -> None:
|
||||
wrapped: Final = f"litellm.NotFoundError: AnthropicException - {THREAD_NOT_FOUND_UPSTREAM}"
|
||||
frame: Final = anthropic_error_sse_frame(404, wrapped)
|
||||
assert frame.error_response["error"]["details"] == {"error_code": "thread_not_found"}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw_message",
|
||||
(
|
||||
"plain text with a { brace and no json",
|
||||
'litellm.BadRequestError: OpenAIException - {"error": {"message": "bad", "type": "invalid_request_error"}}',
|
||||
'AnthropicException - {"type": "error", "error": {"type": "api_error", "message": "no details here"}}',
|
||||
),
|
||||
ids=("no_json", "openai_shape", "anthropic_without_details"),
|
||||
)
|
||||
def test_a_message_without_anthropic_details_gets_no_details_key(self, raw_message: str) -> None:
|
||||
response: Final = AnthropicExceptionMapping.transform_to_anthropic_error(
|
||||
status_code=400, raw_message=raw_message
|
||||
)
|
||||
assert "details" not in response["error"]
|
||||
|
|
|
|||
|
|
@ -2716,3 +2716,23 @@ async def test_anthropic_messages_strict_tools_rejection_is_sent_again_without_s
|
|||
assert first["tools"] == [tool]
|
||||
assert second == {**first, "tools": [{"name": "lookup_record", "input_schema": {"type": "object"}}]}
|
||||
assert _JSON_BODY.validate_python(response)["id"] == "msg_strict_retry"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
("openai/gpt-5.5", "vertex_ai/claude-opus-4-8", "bedrock/anthropic.claude-opus-4-8-v1:0"),
|
||||
ids=("chat_bridge", "vertex_native", "bedrock_native"),
|
||||
)
|
||||
def test_a_thread_on_a_leg_without_message_threads_is_a_400_before_any_upstream_call(model: str) -> None:
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError
|
||||
from litellm.llms.anthropic.messages.native.handler import anthropic_messages_handler
|
||||
|
||||
with pytest.raises(AnthropicError, match="thread: message threads are not supported") as raised:
|
||||
anthropic_messages_handler(
|
||||
max_tokens=64,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
model=model,
|
||||
api_key="synthetic-key",
|
||||
thread={"type": "continue", "previous_message_id": "msg_01previous"},
|
||||
)
|
||||
assert raised.value.status_code == 400
|
||||
|
|
|
|||
|
|
@ -721,3 +721,93 @@ def test_a_first_party_messages_request_never_fetches_the_remote_beta_headers_co
|
|||
|
||||
assert request["messages"][1] == {"role": "system", "content": "be terse"}
|
||||
assert not remote_config.called
|
||||
|
||||
|
||||
MESSAGE_THREADS_BETA: Final = "message-threads-2026-08-12"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"thread",
|
||||
({"type": "create"}, {"type": "continue", "previous_message_id": "msg_01previous"}),
|
||||
ids=("create", "continue"),
|
||||
)
|
||||
def test_messages_request_forwards_the_thread_field_to_anthropic(thread: dict[str, str]) -> None:
|
||||
request: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType] # bare-dict signature
|
||||
model="claude-opus-4-8",
|
||||
messages=[{"role": "user", "content": "what did I just say"}],
|
||||
anthropic_messages_optional_request_params={"max_tokens": 64, "thread": dict(thread)},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
assert request["thread"] == thread
|
||||
assert request["messages"] == [{"role": "user", "content": "what did I just say"}]
|
||||
|
||||
|
||||
def test_messages_headers_add_the_message_threads_beta_next_to_the_callers_betas() -> None:
|
||||
headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType] # bare-dict signature
|
||||
headers={"anthropic-beta": "context-1m-2025-08-07"},
|
||||
optional_params={"thread": {"type": "continue", "previous_message_id": "msg_01previous"}},
|
||||
messages=[],
|
||||
)
|
||||
assert headers["anthropic-beta"] == f"context-1m-2025-08-07,{MESSAGE_THREADS_BETA}"
|
||||
|
||||
|
||||
def test_messages_headers_add_no_message_threads_beta_without_a_thread() -> None:
|
||||
headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType] # bare-dict signature
|
||||
headers={}, optional_params={"max_tokens": 64}, messages=[]
|
||||
)
|
||||
assert MESSAGE_THREADS_BETA not in headers.get("anthropic-beta", "") # pyright: ignore[reportUnknownMemberType] # bare-dict result
|
||||
|
||||
|
||||
def test_the_thread_beta_is_the_same_on_the_create_and_the_continue_turn() -> None:
|
||||
config: Final = AnthropicMessagesConfig()
|
||||
create: Final = config._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType] # bare-dict signature
|
||||
headers={}, optional_params={"thread": {"type": "create"}, "system": "be terse"}, messages=[]
|
||||
)
|
||||
cont: Final = config._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType] # bare-dict signature
|
||||
headers={}, optional_params={"thread": {"type": "continue", "previous_message_id": "msg_01p"}}, messages=[]
|
||||
)
|
||||
assert create["anthropic-beta"] == cont["anthropic-beta"] == MESSAGE_THREADS_BETA
|
||||
|
||||
|
||||
def _advisor_turn() -> list[dict[str, object]]:
|
||||
return [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "server_tool_use", "id": "srvtoolu_01", "name": "advisor", "input": {}},
|
||||
{
|
||||
"type": "advisor_tool_result",
|
||||
"tool_use_id": "srvtoolu_01",
|
||||
"content": [{"type": "text", "text": "ok"}],
|
||||
},
|
||||
{"type": "text", "text": "done"},
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "and now?"},
|
||||
]
|
||||
|
||||
|
||||
def test_a_thread_continue_turn_keeps_advisor_blocks_the_stored_tools_still_explain() -> None:
|
||||
request: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType] # bare-dict signature
|
||||
model="claude-opus-4-8",
|
||||
messages=_advisor_turn(),
|
||||
anthropic_messages_optional_request_params={
|
||||
"max_tokens": 64,
|
||||
"thread": {"type": "continue", "previous_message_id": "msg_01previous"},
|
||||
},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
assert request["messages"][0]["content"] == _advisor_turn()[0]["content"]
|
||||
|
||||
|
||||
def test_a_request_without_a_thread_still_strips_advisor_blocks_when_the_tool_is_absent() -> None:
|
||||
request: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType] # bare-dict signature
|
||||
model="claude-opus-4-8",
|
||||
messages=_advisor_turn(),
|
||||
anthropic_messages_optional_request_params={"max_tokens": 64},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
assert request["messages"][0]["content"] == [{"type": "text", "text": "done"}]
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ Regression tests for the /v1/messages request-parse fast paths:
|
|||
while resolving the (static) type hints only once per process.
|
||||
"""
|
||||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
|
|
@ -133,3 +135,11 @@ def test_sampling_param_raises_clean_400_without_drop_params(monkeypatch):
|
|||
model="claude-opus-4-7",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
|
||||
def test_the_thread_field_survives_optional_param_filtering():
|
||||
thread: Final = {"type": "continue", "previous_message_id": "msg_01previous"}
|
||||
result: Final = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param(
|
||||
{"max_tokens": 64, "thread": thread}
|
||||
)
|
||||
assert result == {"max_tokens": 64, "thread": thread}
|
||||
|
|
|
|||
|
|
@ -3672,3 +3672,72 @@ def test_the_refusal_names_each_cooled_deployment_some_cooldown_state_routes_the
|
|||
with pytest.raises(RouterRateLimitError) as refusal:
|
||||
_select(_tenant_router(EVERY_TENANT, **BOTH_ON), "async", "tenants", copy.deepcopy(request_kwargs))
|
||||
assert {deployment["model_id"] for deployment in refusal.value.cooldown_deployments or ()} == reachable
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestMessageThreadNotFoundCooldown:
|
||||
THREAD_NOT_FOUND_BODY: Final = (
|
||||
'{"type":"error","error":{"type":"not_found_error","message":"No thread state was found for the '
|
||||
'requested `previous_message_id`.","details":{"error_code":"thread_not_found"}}}'
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _not_found(message: str) -> Exception:
|
||||
return litellm.NotFoundError(
|
||||
message=f"AnthropicException - {message}",
|
||||
model="claude-opus-4-8",
|
||||
llm_provider="anthropic",
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_stale_thread_404_leaves_the_deployment_out_of_cooldown(self) -> None:
|
||||
model_id: Final = "anthropic-threads"
|
||||
router: Final = _make_router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude",
|
||||
"litellm_params": {"model": "anthropic/claude-opus-4-8", "api_key": "synthetic-key"},
|
||||
"model_info": {"id": model_id},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
result: Final = router.deployment_callback_on_failure(
|
||||
kwargs={
|
||||
"exception": self._not_found(self.THREAD_NOT_FOUND_BODY),
|
||||
"litellm_params": {"model_info": {"id": model_id}},
|
||||
},
|
||||
completion_response=None,
|
||||
start_time=0,
|
||||
end_time=1,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
assert get_cooldown_deployments(RoutingContext(router), parent_otel_span=None) == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_404_without_the_thread_code_still_cools_the_deployment_down(self) -> None:
|
||||
model_id: Final = "anthropic-missing-model"
|
||||
router: Final = _make_router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude",
|
||||
"litellm_params": {"model": "anthropic/claude-opus-4-8", "api_key": "synthetic-key"},
|
||||
"model_info": {"id": model_id},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
router.deployment_callback_on_failure(
|
||||
kwargs={
|
||||
"exception": self._not_found(
|
||||
'{"type":"error","error":{"type":"not_found_error","message":"model: x"}}'
|
||||
),
|
||||
"litellm_params": {"model_info": {"id": model_id}},
|
||||
},
|
||||
completion_response=None,
|
||||
start_time=0,
|
||||
end_time=1,
|
||||
)
|
||||
|
||||
assert get_cooldown_deployments(RoutingContext(router), parent_otel_span=None) == [model_id]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue