diff --git a/litellm/llms/anthropic/pass_through/messages/handler.py b/litellm/llms/anthropic/pass_through/messages/handler.py index 949d81f3c1b..079e3639b54 100644 --- a/litellm/llms/anthropic/pass_through/messages/handler.py +++ b/litellm/llms/anthropic/pass_through/messages/handler.py @@ -11,7 +11,7 @@ from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequenc from functools import partial from typing import Final, cast -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError import litellm from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook @@ -56,6 +56,15 @@ __all__ = ("anthropic_messages", "anthropic_messages_handler") _RESPONSES_API_PROVIDERS: Final = frozenset({"openai"}) _SAFEGUARD_KWARGS: Final = TypeAdapter(Mapping[str, object]) _SAFEGUARD_MESSAGES: Final = TypeAdapter(Sequence[Mapping[str, object]]) +_DROP_PATHS: Final = TypeAdapter(list[object]) + + +def additional_drop_paths(kwargs: Mapping[str, object]) -> tuple[str, ...]: + try: + configured: Final = _DROP_PATHS.validate_python(kwargs.get("additional_drop_params")) + except ValidationError: + return () + return tuple(path for path in configured if isinstance(path, str)) def _bridges_to_responses_api(model: str, custom_llm_provider: str) -> bool: @@ -679,6 +688,7 @@ def anthropic_messages_handler( model=model, drop_params=litellm_params.get("drop_params") is True, custom_llm_provider=custom_llm_provider, + additional_drop_params=additional_drop_paths(request_kwargs), ) ) resolved_api_base: Final = ( diff --git a/litellm/llms/anthropic/pass_through/messages/utils.py b/litellm/llms/anthropic/pass_through/messages/utils.py index d8b96c1b7a0..78e6fca4f3f 100644 --- a/litellm/llms/anthropic/pass_through/messages/utils.py +++ b/litellm/llms/anthropic/pass_through/messages/utils.py @@ -182,6 +182,7 @@ class AnthropicMessagesRequestUtils: model: str | None = None, drop_params: bool = False, custom_llm_provider: str | None = None, + additional_drop_params: Iterable[str] = (), ) -> AnthropicMessagesRequestOptionalParams: """ Filter parameters to only include those defined in AnthropicMessagesRequestOptionalParams. @@ -191,11 +192,13 @@ class AnthropicMessagesRequestUtils: model: Resolved model id; when set, unsupported params may be dropped drop_params: Per-request drop_params flag (also respects litellm.drop_params) custom_llm_provider: Routed provider; fast mode is gated to direct Anthropic + additional_drop_params: Paths the deployment drops. A listed top-level key is left out + before the unsupported-param checks run, so it never raises Returns: AnthropicMessagesRequestOptionalParams instance with only the valid parameters """ - valid_keys: Final = anthropic_messages_optional_param_keys() + valid_keys: Final = anthropic_messages_optional_param_keys().difference(additional_drop_params) filtered_params: Final = {k: v for k, v in params.items() if k in valid_keys and v is not None} if model is not None: from litellm.llms.anthropic.chat.transformation import AnthropicConfig diff --git a/litellm/llms/anthropic/prompt_cache_prediction.py b/litellm/llms/anthropic/prompt_cache_prediction.py index 4c363dba860..701b5283e4f 100644 --- a/litellm/llms/anthropic/prompt_cache_prediction.py +++ b/litellm/llms/anthropic/prompt_cache_prediction.py @@ -650,17 +650,15 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M custom_llm_provider="anthropic", tools=_MESSAGES.validate_python(owned.get("tools") or []), ) + drop_paths: Final = TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()) options: Final = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( {**owned, "system": system}, model=resolved_model, custom_llm_provider="anthropic", drop_params=owned.get("drop_params") is True, + additional_drop_params=drop_paths, ) - filtered: Final = reduce( - delete_nested_value, - TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()), - dict(options), - ) + filtered: Final = reduce(delete_nested_value, drop_paths, dict(options)) body: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( model=resolved_model, messages=strip_provider_specific_fields_from_anthropic_messages(messages), diff --git a/tests/integration/messages_endpoint/test_messages_additional_drop_params_wire.py b/tests/integration/messages_endpoint/test_messages_additional_drop_params_wire.py new file mode 100644 index 00000000000..f19976bda74 --- /dev/null +++ b/tests/integration/messages_endpoint/test_messages_additional_drop_params_wire.py @@ -0,0 +1,119 @@ +import json +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final + +import pytest +from integration._support.client import Gateway, list_value, object_value +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_JSON: Final = TypeAdapter(dict[str, JsonValue]) +_ANSWER: Final = "sampling params dropped" +_SAMPLING_SPELLINGS: Final = ("top_k", "top_p", "topK", "topP") +_AWS: Final[dict[str, JsonValue]] = { + "aws_access_key_id": "AKIASCRIPTEDPROVIDER", + "aws_secret_access_key": "scripted-secret", + "aws_region_name": "us-east-1", +} + + +@dataclass(frozen=True, slots=True) +class Backend: + model: str + path_suffix: str + credentials: Mapping[str, JsonValue] + + +_BEDROCK_INVOKE: Final = Backend("bedrock/invoke/us.anthropic.claude-opus-4-8", "/invoke", _AWS) +_ANTHROPIC: Final = Backend("anthropic/claude-opus-4-8", "/v1/messages", {}) +_BEDROCK_CONVERSE: Final = Backend("bedrock/converse/us.anthropic.claude-opus-4-8", "/converse", _AWS) +_NATIVE_BACKENDS: Final = ( + pytest.param(_BEDROCK_INVOKE, id="bedrock_invoke"), + pytest.param(_ANTHROPIC, id="anthropic"), +) + + +def _answer(request: Request) -> Reply: + if request.target.endswith("/converse"): + return Reply( + body=json.dumps( + { + "output": {"message": {"role": "assistant", "content": [{"text": _ANSWER}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 5, "outputTokens": 3, "totalTokens": 8}, + "metrics": {"latencyMs": 1}, + } + ).encode() + ) + return Reply( + body=json.dumps( + { + "id": "msg_dropped_sampling_params", + "type": "message", + "role": "assistant", + "model": "claude-opus-4-8", + "content": [{"type": "text", "text": _ANSWER}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 5, "output_tokens": 3}, + } + ).encode() + ) + + +def _messages(gateway: Gateway, model: str, **sampling: JsonValue) -> tuple[int, str]: + response: Final = gateway.request( + "POST", + "/v1/messages", + {"model": model, "max_tokens": 32, "messages": [{"role": "user", "content": "hi"}], **sampling}, + ) + return response.status_code, response.text + + +def _answered(status: int, text: str) -> None: + assert status == 200, text + content: Final = list_value(_JSON.validate_json(text)["content"]) + assert object_value(content[0])["text"] == _ANSWER, text + + +def _sent_without_sampling_params(wire: Wire, backend: Backend) -> None: + (request,) = wire.drain() + assert request.target.endswith(backend.path_suffix), request.target + sent: Final = request.body.decode() + assert not any(spelling in sent for spelling in _SAMPLING_SPELLINGS), sent + + +@pytest.mark.parametrize( + "backend", (*_NATIVE_BACKENDS, pytest.param(_BEDROCK_CONVERSE, id="bedrock_converse_chat_bridge")) +) +def test_messages_drops_the_sampling_params_a_deployment_lists_on_a_model_without_them( + gateway: Gateway, backend: Backend +) -> None: + with wire_server(_answer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=backend.model, + api_base=wire.url, + additional_drop_params=["top_k", "top_p"], + **backend.credentials, + ) + _answered(*_messages(gateway, model, top_k=5, top_p=0.9)) + _sent_without_sampling_params(wire, backend) + + +@pytest.mark.parametrize("backend", _NATIVE_BACKENDS) +def test_messages_rejects_a_sampling_param_the_deployment_does_not_list_before_calling_the_provider( + gateway: Gateway, backend: Backend +) -> None: + with wire_server(_answer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=backend.model, + api_base=wire.url, + additional_drop_params=["top_k"], + **backend.credentials, + ) + rejected_status, rejected_text = _messages(gateway, model, top_k=5, top_p=0.9) + assert rejected_status == 400, rejected_text + assert "does not support top_p=0.9" in rejected_text, rejected_text + _answered(*_messages(gateway, model, top_k=5)) + _sent_without_sampling_params(wire, backend) diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index bdc144b91cc..15e535b7916 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -1635,6 +1635,89 @@ _CLIENT_BETA_HEADERS: Final = ( ) +_OPUS_4_8_ON_BEDROCK_INVOKE: Final = "bedrock/invoke/us.anthropic.claude-opus-4-8" + + +def _bedrock_invoke_recording(bodies: list[dict[str, object]]) -> AsyncHTTPHandler: + def record_and_answer(request: httpx.Request) -> httpx.Response: + bodies.append(json.loads(request.content)) + return httpx.Response( + 200, + json={ + "id": "msg_drop_top_k", + "type": "message", + "role": "assistant", + "model": "claude-opus-4-8", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 3, "output_tokens": 1}, + }, + request=request, + ) + + upstream = AsyncHTTPHandler() + upstream.client = httpx.AsyncClient(transport=httpx.MockTransport(record_and_answer)) + return upstream + + +async def _opus_4_8_messages_with_top_k(bodies: list[dict[str, object]], **deployment: object) -> object: + return await messages.acreate( + max_tokens=16, + messages=[{"role": "user", "content": "hi"}], + model=_OPUS_4_8_ON_BEDROCK_INVOKE, + top_k=5, + aws_access_key_id="test-access-key", + aws_secret_access_key="test-secret-key", + aws_region_name="us-east-1", + client=_bedrock_invoke_recording(bodies), + **deployment, + ) + + +@pytest.mark.asyncio +async def test_bedrock_invoke_messages_drops_top_k_the_deployment_lists_on_a_model_without_sampling_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + bodies: list[dict[str, object]] = [] + + response = await _opus_4_8_messages_with_top_k(bodies, additional_drop_params=["top_k"]) + + assert response["content"] == [{"type": "text", "text": "ok"}] + (body,) = bodies + assert "top_k" not in body + assert body["max_tokens"] == 16 + + +@pytest.mark.asyncio +async def test_bedrock_invoke_messages_still_rejects_top_k_the_deployment_does_not_list( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + bodies: list[dict[str, object]] = [] + + with pytest.raises(litellm.UnsupportedParamsError, match="does not support top_k=5") as raised: + await _opus_4_8_messages_with_top_k(bodies, additional_drop_params=["top_p"]) + + assert raised.value.status_code == 400 + assert bodies == [] + + +@pytest.mark.asyncio +async def test_bedrock_invoke_messages_answers_the_top_k_400_for_a_drop_list_that_is_not_a_list( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + bodies: list[dict[str, object]] = [] + + with pytest.raises(litellm.UnsupportedParamsError, match="does not support top_k=5") as raised: + await _opus_4_8_messages_with_top_k(bodies, additional_drop_params="top_k") + + assert raised.value.status_code == 400 + assert bodies == [] + + @pytest.mark.asyncio @pytest.mark.parametrize("client_headers", _CLIENT_BETA_HEADERS) async def test_anthropic_messages_forwards_safeguards_and_dangerous_tool_use_beta_to_bedrock_invoke( diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_request_optional_param_utils.py b/tests/unit/llms/anthropic/pass_through/messages/test_request_optional_param_utils.py index 7a4f3aa5f90..cee265655c0 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_request_optional_param_utils.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_request_optional_param_utils.py @@ -133,3 +133,37 @@ def test_sampling_param_raises_clean_400_without_drop_params(monkeypatch): model="claude-opus-4-7", drop_params=False, ) + + +@pytest.mark.parametrize( + ("model", "param", "value"), + [ + ("claude-opus-4-8", "temperature", 0.3), + ("claude-opus-4-8", "top_p", 0.9), + ("claude-opus-4-8", "top_k", 5), + ("claude-sonnet-4-6", "speed", "fast"), + ], +) +def test_deployment_listed_param_is_dropped_before_the_unsupported_param_gate(monkeypatch, model, param, value): + monkeypatch.setattr(litellm, "drop_params", False) + result = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params={param: value, "stream": True}, + model=model, + drop_params=False, + additional_drop_params=(param,), + ) + + assert result == {"stream": True} + + +def test_param_missing_from_the_drop_list_still_raises_the_unsupported_param_400(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.utils.UnsupportedParamsError, match="does not support top_p") as raised: + AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params={"top_k": 5, "top_p": 0.9}, + model="claude-opus-4-8", + drop_params=False, + additional_drop_params=("top_k", "top_p.nested"), + ) + + assert raised.value.status_code == 400 diff --git a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py index df89774951f..9e19b661ffa 100644 --- a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py +++ b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py @@ -26,6 +26,7 @@ from litellm.llms.anthropic.prompt_cache_prediction import ( parse_cache_plan, parse_observed_cache, parse_prompt, + prepare_native_baseline_body, resolve_baseline_prediction_target, resolve_prediction_target, supported_prediction_headers, @@ -452,3 +453,23 @@ async def test_native_count_preserves_settings_and_requires_every_prefix( assert tuple(marker.prefix_tokens for marker in result.breakpoints) == (5000, 5800) assert tuple(route.call_count for route in routes) == (1, 1, 1) assert unexpected.call_count == 0 + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_native_baseline_body_leaves_out_a_listed_sampling_param_the_model_does_not_support( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + request: Final = { + "model": "claude-opus-4-8", + "max_tokens": 64, + "messages": [{"role": "user", "content": "hi"}], + "top_k": 5, + "additional_drop_params": ["top_k"], + } + + body: Final = prepare_native_baseline_body(request, "anthropic/claude-opus-4-8") + + assert body is not None + assert "top_k" not in body + assert body["max_tokens"] == 64