diff --git a/litellm-rust/crates/inference-messages/src/handler.rs b/litellm-rust/crates/inference-messages/src/handler.rs index 6a8ed7fff8e..c7a7d93bf98 100644 --- a/litellm-rust/crates/inference-messages/src/handler.rs +++ b/litellm-rust/crates/inference-messages/src/handler.rs @@ -133,10 +133,10 @@ async fn provider_call( )?; let transform_context = MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params); - let filtered = requested_optional_params(sanitized, provider, &transform_context)?; - let requested_params = serde_json::to_value(&filtered.params).map_err(serialize_failure)?; - let trimmed = - without_additional_drop_params(filtered, &shaping.settings.additional_drop_params)?; + let without_listed = + without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?; + let trimmed = requested_optional_params(without_listed, provider, &transform_context)?; + let requested_params = serde_json::to_value(&trimmed.params).map_err(serialize_failure)?; let scoped = get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str()); diff --git a/litellm-rust/crates/inference-messages/tests/messages/host.rs b/litellm-rust/crates/inference-messages/tests/messages/host.rs index 20f6a97aa37..063e9d18738 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/host.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/host.rs @@ -456,6 +456,34 @@ async fn the_request_context_carries_the_shaped_params_without_model_or_messages assert_eq!(optional_params, json!({"max_tokens": 16})); } +#[rstest] +#[tokio::test] +async fn the_request_context_leaves_out_a_listed_param_the_model_refuses(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let host = RecordingHost::passthrough(authenticated( + MessagesCall { + shaping: MessagesShaping { + settings: MessagesSettings { + additional_drop_params: vec!["temperature".into()], + ..MessagesSettings::default() + }, + capabilities: AnthropicModelCapabilities { + supports_sampling_params: false, + ..AnthropicModelCapabilities::default() + }, + }, + ..with_fields(call, json!({"temperature": 0.2})) + }, + upstream.uri(), + )); + + run_through(&host).await.expect("messages call succeeds"); + + let [optional_params] = <[Value; 1]>::try_from(host.optional_params.into_inner().unwrap()) + .unwrap_or_else(|seen| panic!("before_provider_request runs once, saw {}", seen.len())); + assert_eq!(optional_params, json!({"max_tokens": 16})); +} + #[rstest] #[case::reasoning_effort_on_an_adaptive_model( AnthropicModelCapabilities { diff --git a/litellm-rust/crates/inference-messages/tests/messages/request.rs b/litellm-rust/crates/inference-messages/tests/messages/request.rs index 216aa96fbe6..3784eb6cc68 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/request.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/request.rs @@ -530,6 +530,30 @@ async fn unsupported_params_are_dropped_under_drop_params_and_rejected_without_i assert_eq!(sent["max_tokens"], 16); } +#[rstest] +#[tokio::test] +async fn additional_drop_params_clear_a_param_the_model_does_not_support(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + + run_message(MessagesCall { + api_key: Some("sk".into()), + api_base: Some(upstream.uri()), + shaping: MessagesShaping { + capabilities: sampling_removed(), + settings: MessagesSettings { + additional_drop_params: vec!["top_k".into()], + ..MessagesSettings::default() + }, + }, + ..with_fields(call, json!({"top_k": 5})) + }) + .await; + + let sent = only_request(&upstream).await.json(); + assert_eq!(sent.get("top_k"), None); + assert_eq!(sent["max_tokens"], 16); +} + #[rstest] #[case::adaptive_thinking(json!({"type": "adaptive"}), json!({"type": "adaptive", "display": "summarized"}))] #[case::disabled_thinking(json!({"type": "disabled"}), json!({"type": "disabled"}))] diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 922de386fe4..59ffb6fef04 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -12,7 +12,7 @@ from dataclasses import dataclass from functools import partial from typing import Final, cast -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError import litellm from litellm.anthropic_beta_headers_manager import aload_beta_headers_config @@ -67,6 +67,15 @@ __all__ = ("anthropic_messages", "anthropic_messages_handler") _RESPONSES_API_PROVIDERS: Final = frozenset({"openai"}) _SAFEGUARD_KWARGS: Final = TypeAdapter(Mapping[str, object]) _SAFEGUARD_MESSAGES: Final = TypeAdapter(Sequence[Mapping[str, object]]) +_DROP_PATHS: Final = TypeAdapter(list[object]) + + +def additional_drop_paths(kwargs: Mapping[str, object]) -> tuple[str, ...]: + try: + configured: Final = _DROP_PATHS.validate_python(kwargs.get("additional_drop_params")) + except ValidationError: + return () + return tuple(path for path in configured if isinstance(path, str)) def _bridges_to_responses_api(model: str, custom_llm_provider: str, deployment_model_info: object) -> bool: @@ -837,6 +846,7 @@ def anthropic_messages_handler( model=model, drop_params=litellm_params.get("drop_params") is True, custom_llm_provider=custom_llm_provider, + additional_drop_params=additional_drop_paths(request_kwargs), ) ) resolved_api_base: Final = ( diff --git a/litellm/llms/anthropic/messages/native/utils.py b/litellm/llms/anthropic/messages/native/utils.py index ea3afdb9a8f..8f33d000c7c 100644 --- a/litellm/llms/anthropic/messages/native/utils.py +++ b/litellm/llms/anthropic/messages/native/utils.py @@ -299,6 +299,7 @@ class AnthropicMessagesRequestUtils: model: str | None = None, drop_params: bool = False, custom_llm_provider: str | None = None, + additional_drop_params: Iterable[str] = (), ) -> AnthropicMessagesRequestOptionalParams: """ Filter parameters to only include those defined in AnthropicMessagesRequestOptionalParams. @@ -308,11 +309,13 @@ class AnthropicMessagesRequestUtils: model: Resolved model id; when set, unsupported params may be dropped drop_params: Per-request drop_params flag (also respects litellm.drop_params) custom_llm_provider: Routed provider; fast mode is gated to direct Anthropic + additional_drop_params: Paths the deployment drops. A listed top-level key is left out + before the unsupported-param checks run, so it never raises Returns: AnthropicMessagesRequestOptionalParams instance with only the valid parameters """ - valid_keys: Final = anthropic_messages_optional_param_keys() + valid_keys: Final = anthropic_messages_optional_param_keys().difference(additional_drop_params) filtered_params: Final = {k: v for k, v in params.items() if k in valid_keys and v is not None} if model is not None: from litellm.llms.anthropic.chat.transformation import AnthropicConfig diff --git a/litellm/llms/anthropic/prompt_cache_prediction.py b/litellm/llms/anthropic/prompt_cache_prediction.py index 2da071ebdd2..28896c99eaf 100644 --- a/litellm/llms/anthropic/prompt_cache_prediction.py +++ b/litellm/llms/anthropic/prompt_cache_prediction.py @@ -654,17 +654,15 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M custom_llm_provider="anthropic", tools=_MESSAGES.validate_python(owned.get("tools") or []), ) + drop_paths: Final = TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()) options: Final = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( {**owned, "system": system}, model=resolved_model, custom_llm_provider="anthropic", drop_params=owned.get("drop_params") is True, + additional_drop_params=drop_paths, ) - filtered: Final = reduce( - delete_nested_value, - TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()), - dict(options), - ) + filtered: Final = reduce(delete_nested_value, drop_paths, dict(options)) config: Final = _BaselineAnthropicMessagesConfig() body: Final = config.transform_anthropic_messages_request( model=resolved_model, diff --git a/litellm/rust_bridge/host/routes/messages.py b/litellm/rust_bridge/host/routes/messages.py index c50791f8a9c..c6568264edb 100644 --- a/litellm/rust_bridge/host/routes/messages.py +++ b/litellm/rust_bridge/host/routes/messages.py @@ -35,7 +35,6 @@ from litellm.rust_bridge.host import callbacks_legacy_python, http_transport from litellm.rust_bridge.public_call import known_params, optional_str from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse -_DROP_PATHS: Final = TypeAdapter(list[object]) _METADATA: Final = TypeAdapter(dict[str, object]) _BODY_FIELDS: Final = frozenset( { @@ -92,7 +91,7 @@ def inferred_provider(model: str, api_base: str | None) -> str | None: def body_params(request: Mapping[str, object]) -> Mapping[str, object]: - params: Final = known_params(_BODY_FIELDS, request) + params: Final = known_params(_BODY_FIELDS.difference(handler.additional_drop_paths(request)), request) if deployment_reasoning_effort_levels(request.get("model_info")) is None: return params return _with_deployment_effort(params, request) @@ -386,20 +385,12 @@ def _drop_params(kwargs: Mapping[str, object]) -> bool: return bool(litellm.drop_params) or normalize_drop_params(kwargs.get("drop_params")) is True -def _additional_drop_params(kwargs: Mapping[str, object]) -> tuple[str, ...]: - try: - configured: Final = _DROP_PATHS.validate_python(kwargs.get("additional_drop_params")) - except ValidationError: - return () - return tuple(path for path in configured if isinstance(path, str)) - - def settings(kwargs: Mapping[str, object]) -> dict[str, object]: return asdict( MessagesSettings( drop_params=_drop_params(kwargs), reasoning_auto_summary=is_reasoning_auto_summary_enabled(), - additional_drop_params=_additional_drop_params(kwargs), + additional_drop_params=handler.additional_drop_paths(kwargs), request_timeout=get_configured_request_timeout(), ) ) diff --git a/tests/integration/messages_endpoint/test_messages_additional_drop_params_wire.py b/tests/integration/messages_endpoint/test_messages_additional_drop_params_wire.py new file mode 100644 index 00000000000..f19976bda74 --- /dev/null +++ b/tests/integration/messages_endpoint/test_messages_additional_drop_params_wire.py @@ -0,0 +1,119 @@ +import json +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final + +import pytest +from integration._support.client import Gateway, list_value, object_value +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_JSON: Final = TypeAdapter(dict[str, JsonValue]) +_ANSWER: Final = "sampling params dropped" +_SAMPLING_SPELLINGS: Final = ("top_k", "top_p", "topK", "topP") +_AWS: Final[dict[str, JsonValue]] = { + "aws_access_key_id": "AKIASCRIPTEDPROVIDER", + "aws_secret_access_key": "scripted-secret", + "aws_region_name": "us-east-1", +} + + +@dataclass(frozen=True, slots=True) +class Backend: + model: str + path_suffix: str + credentials: Mapping[str, JsonValue] + + +_BEDROCK_INVOKE: Final = Backend("bedrock/invoke/us.anthropic.claude-opus-4-8", "/invoke", _AWS) +_ANTHROPIC: Final = Backend("anthropic/claude-opus-4-8", "/v1/messages", {}) +_BEDROCK_CONVERSE: Final = Backend("bedrock/converse/us.anthropic.claude-opus-4-8", "/converse", _AWS) +_NATIVE_BACKENDS: Final = ( + pytest.param(_BEDROCK_INVOKE, id="bedrock_invoke"), + pytest.param(_ANTHROPIC, id="anthropic"), +) + + +def _answer(request: Request) -> Reply: + if request.target.endswith("/converse"): + return Reply( + body=json.dumps( + { + "output": {"message": {"role": "assistant", "content": [{"text": _ANSWER}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 5, "outputTokens": 3, "totalTokens": 8}, + "metrics": {"latencyMs": 1}, + } + ).encode() + ) + return Reply( + body=json.dumps( + { + "id": "msg_dropped_sampling_params", + "type": "message", + "role": "assistant", + "model": "claude-opus-4-8", + "content": [{"type": "text", "text": _ANSWER}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 5, "output_tokens": 3}, + } + ).encode() + ) + + +def _messages(gateway: Gateway, model: str, **sampling: JsonValue) -> tuple[int, str]: + response: Final = gateway.request( + "POST", + "/v1/messages", + {"model": model, "max_tokens": 32, "messages": [{"role": "user", "content": "hi"}], **sampling}, + ) + return response.status_code, response.text + + +def _answered(status: int, text: str) -> None: + assert status == 200, text + content: Final = list_value(_JSON.validate_json(text)["content"]) + assert object_value(content[0])["text"] == _ANSWER, text + + +def _sent_without_sampling_params(wire: Wire, backend: Backend) -> None: + (request,) = wire.drain() + assert request.target.endswith(backend.path_suffix), request.target + sent: Final = request.body.decode() + assert not any(spelling in sent for spelling in _SAMPLING_SPELLINGS), sent + + +@pytest.mark.parametrize( + "backend", (*_NATIVE_BACKENDS, pytest.param(_BEDROCK_CONVERSE, id="bedrock_converse_chat_bridge")) +) +def test_messages_drops_the_sampling_params_a_deployment_lists_on_a_model_without_them( + gateway: Gateway, backend: Backend +) -> None: + with wire_server(_answer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=backend.model, + api_base=wire.url, + additional_drop_params=["top_k", "top_p"], + **backend.credentials, + ) + _answered(*_messages(gateway, model, top_k=5, top_p=0.9)) + _sent_without_sampling_params(wire, backend) + + +@pytest.mark.parametrize("backend", _NATIVE_BACKENDS) +def test_messages_rejects_a_sampling_param_the_deployment_does_not_list_before_calling_the_provider( + gateway: Gateway, backend: Backend +) -> None: + with wire_server(_answer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=backend.model, + api_base=wire.url, + additional_drop_params=["top_k"], + **backend.credentials, + ) + rejected_status, rejected_text = _messages(gateway, model, top_k=5, top_p=0.9) + assert rejected_status == 400, rejected_text + assert "does not support top_p=0.9" in rejected_text, rejected_text + _answered(*_messages(gateway, model, top_k=5)) + _sent_without_sampling_params(wire, backend) diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index 3296163c64e..a83e98afb89 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -2021,6 +2021,89 @@ async def test_anthropic_messages_forwards_safeguards_and_dangerous_tool_use_bet assert response["safeguard_results"] == safeguard_results +_OPUS_4_8_ON_BEDROCK_INVOKE: Final = "bedrock/invoke/us.anthropic.claude-opus-4-8" + + +def _bedrock_invoke_recording(bodies: list[dict[str, object]]) -> AsyncHTTPHandler: + def record_and_answer(request: httpx.Request) -> httpx.Response: + bodies.append(json.loads(request.content)) + return httpx.Response( + 200, + json={ + "id": "msg_drop_top_k", + "type": "message", + "role": "assistant", + "model": "claude-opus-4-8", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 3, "output_tokens": 1}, + }, + request=request, + ) + + upstream: Final = AsyncHTTPHandler() + upstream.client = httpx.AsyncClient(transport=httpx.MockTransport(record_and_answer)) + return upstream + + +async def _opus_4_8_messages_with_top_k(bodies: list[dict[str, object]], **deployment: object) -> object: + return await messages.acreate( + max_tokens=16, + messages=[{"role": "user", "content": "hi"}], + model=_OPUS_4_8_ON_BEDROCK_INVOKE, + top_k=5, + aws_access_key_id="test-access-key", + aws_secret_access_key="test-secret-key", + aws_region_name="us-east-1", + client=_bedrock_invoke_recording(bodies), + **deployment, + ) + + +@pytest.mark.asyncio +async def test_bedrock_invoke_messages_drops_top_k_the_deployment_lists_on_a_model_without_sampling_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + bodies: Final[list[dict[str, object]]] = [] + + response: Final = await _opus_4_8_messages_with_top_k(bodies, additional_drop_params=["top_k"]) + + assert response["content"] == [{"type": "text", "text": "ok"}] + (body,) = bodies + assert "top_k" not in body + assert body["max_tokens"] == 16 + + +@pytest.mark.asyncio +async def test_bedrock_invoke_messages_still_rejects_top_k_the_deployment_does_not_list( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + bodies: Final[list[dict[str, object]]] = [] + + with pytest.raises(litellm.UnsupportedParamsError, match="does not support top_k=5") as raised: + await _opus_4_8_messages_with_top_k(bodies, additional_drop_params=["top_p"]) + + assert raised.value.status_code == 400 + assert bodies == [] + + +@pytest.mark.asyncio +async def test_bedrock_invoke_messages_answers_the_top_k_400_for_a_drop_list_that_is_not_a_list( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + bodies: Final[list[dict[str, object]]] = [] + + with pytest.raises(litellm.UnsupportedParamsError, match="does not support top_k=5") as raised: + await _opus_4_8_messages_with_top_k(bodies, additional_drop_params="top_k") + + assert raised.value.status_code == 400 + assert bodies == [] + + @pytest.mark.asyncio @pytest.mark.parametrize("client_headers", _CLIENT_BETA_HEADERS) async def test_anthropic_messages_forwards_safeguards_and_dangerous_tool_use_beta_to_vertex( diff --git a/tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py b/tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py index d525ccfec37..8104c0cef56 100644 --- a/tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py +++ b/tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py @@ -6,6 +6,8 @@ Regression tests for the /v1/messages request-parse fast paths: while resolving the (static) type hints only once per process. """ +from typing import Final + import pytest import litellm @@ -133,3 +135,37 @@ def test_sampling_param_raises_clean_400_without_drop_params(monkeypatch): model="claude-opus-4-7", drop_params=False, ) + + +@pytest.mark.parametrize( + ("model", "param", "value"), + [ + ("claude-opus-4-8", "temperature", 0.3), + ("claude-opus-4-8", "top_p", 0.9), + ("claude-opus-4-8", "top_k", 5), + ("claude-sonnet-4-6", "speed", "fast"), + ], +) +def test_deployment_listed_param_is_dropped_before_the_unsupported_param_gate(monkeypatch, model, param, value): + monkeypatch.setattr(litellm, "drop_params", False) + result: Final = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params={param: value, "stream": True}, + model=model, + drop_params=False, + additional_drop_params=(param,), + ) + + assert result == {"stream": True} + + +def test_param_missing_from_the_drop_list_still_raises_the_unsupported_param_400(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.utils.UnsupportedParamsError, match="does not support top_p") as raised: + AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params={"top_k": 5, "top_p": 0.9}, + model="claude-opus-4-8", + drop_params=False, + additional_drop_params=("top_k", "top_p.nested"), + ) + + assert raised.value.status_code == 400 diff --git a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py index d95f831df3e..89ea412af0c 100644 --- a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py +++ b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py @@ -618,6 +618,26 @@ def test_native_baseline_body_drops_a_non_numeric_budget_without_logging_a_drop( assert [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] == [] +@pytest.mark.usefixtures("local_model_cost_map") +def test_native_baseline_body_leaves_out_a_listed_sampling_param_the_model_does_not_support( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + request: Final = { + "model": "claude-opus-4-8", + "max_tokens": 64, + "messages": [{"role": "user", "content": "hi"}], + "top_k": 5, + "additional_drop_params": ["top_k"], + } + + body: Final = prepare_native_baseline_body(request, "anthropic/claude-opus-4-8") + + assert body is not None + assert "top_k" not in body + assert body["max_tokens"] == 64 + + @pytest.mark.usefixtures("local_model_cost_map") def test_native_baseline_body_drops_a_non_string_reasoning_effort_without_logging_a_drop( caplog: pytest.LogCaptureFixture, diff --git a/tests/unit/rust_bridge/host/routes/test_messages.py b/tests/unit/rust_bridge/host/routes/test_messages.py index 704d52386a8..aa7dcc74841 100644 --- a/tests/unit/rust_bridge/host/routes/test_messages.py +++ b/tests/unit/rust_bridge/host/routes/test_messages.py @@ -184,6 +184,28 @@ def test_body_params_keep_the_requested_effort_without_deployment_levels() -> No assert messages_host.body_params(request)["output_config"] == {"effort": "max"} +@pytest.mark.parametrize( + ("additional_drop_params", "expected_keys"), + [ + (["thinking"], {"max_tokens"}), + (["top_k"], {"max_tokens", "thinking", "output_config"}), + ], +) +def test_body_params_derive_no_effort_from_a_param_the_deployment_lists( + additional_drop_params: list[str], expected_keys: set[str] +) -> None: + request: Final[dict[str, object]] = { + "model": _ADAPTIVE_OPUS, + "messages": [], + "max_tokens": 8192, + "thinking": {"type": "enabled", "budget_tokens": 4096}, + "additional_drop_params": additional_drop_params, + "model_info": {"id": "deployment-1", "reasoning_effort_levels": ["low", "medium", "high"]}, + } + + assert set(messages_host.body_params(request)).difference({"messages", "model"}) == expected_keys + + def _marked(phase: str | None = None, *args: object) -> ValueError: error: Final = ValueError(*args) if phase is not None: