diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 118b22859ab..a1ecb8bcdb7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14775,7 +14775,7 @@ _DEPLOYMENT_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object]) def _deployment_with_request_drop_params( - deployment: Mapping[str, object] | None, drop_params: bool | None + deployment: Mapping[str, object] | None, drop_params: object ) -> Mapping[str, object] | None: if drop_params is None: return deployment @@ -15475,9 +15475,7 @@ async def _count_request_tokens( model_to_use = _model sent_params: Final[Mapping[str, object]] = {**_router_default_litellm_params(llm_router), **request_litellm_params} - count_deployment: Final = _deployment_with_request_drop_params( - deployment, normalize_drop_params(sent_params.get("drop_params")) - ) + count_deployment: Final = _deployment_with_request_drop_params(deployment, sent_params.get("drop_params")) conversation_empty: Final = not messages and not contents if prompt and not conversation_empty and not deployment_drops_params(count_deployment): raise ProxyException( diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py index bf3627ff7f7..1450a6711af 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py @@ -485,9 +485,13 @@ _CLEAR_THINKING_EDITS: Final[dict[str, JsonValue]] = { @pytest.mark.asyncio -@pytest.mark.parametrize("request_options", (_DISPLAY_THINKING, _EFFORT), ids=("thinking_display", "effort")) +@pytest.mark.parametrize( + ("request_options", "expected_betas"), + ((_DISPLAY_THINKING, (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,)), (_EFFORT, ())), + ids=("thinking_display", "effort"), +) async def test_invoke_count_retried_on_mantle_sends_the_betas_invoke_inference_sends( - request_options: dict[str, JsonValue], + request_options: dict[str, JsonValue], expected_betas: tuple[str, ...] ) -> None: inference, bedrock = await _messages_inference_and_count( "bedrock/invoke/us.anthropic.claude-opus-5-5", @@ -499,7 +503,7 @@ async def test_invoke_count_retried_on_mantle_sends_the_betas_invoke_inference_s assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST) sent_betas: Final = _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta", []) assert isinstance(sent_betas, list) - assert _sent_betas(bedrock.requests[1]) == tuple(sorted(sent_betas)) + assert _sent_betas(bedrock.requests[1]) == tuple(sorted(sent_betas)) == expected_betas @pytest.mark.asyncio @@ -531,6 +535,21 @@ async def test_runtime_count_of_an_effort_sends_the_effort_beta_once_beside_the_ assert set(counted_betas) - {ANTHROPIC_EFFORT_BETA_HEADER} == set(sent_betas) - {ANTHROPIC_EFFORT_BETA_HEADER} +@pytest.mark.asyncio +async def test_runtime_count_of_an_output_format_sends_only_the_betas_invoke_inference_sends() -> None: + inference, bedrock = await _messages_inference_and_count( + "bedrock/invoke/us.anthropic.claude-sonnet-4-6", + "us.anthropic.claude-sonnet-4-6", + {**_CITY_FORMAT, "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}, + ) + + assert bedrock.posted_hosts() == (RUNTIME_HOST,) + counted: Final = _invoke_body(bedrock.requests[0]) + sent: Final = _JSON_BODY.validate_json(inference.requests[0].content) + assert counted.get("output_config") == sent.get("output_config") + assert counted.get("anthropic_beta") == sent.get("anthropic_beta") + + _CONVERSE_REPLY: Final = { "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, "stopReason": "end_turn", @@ -547,10 +566,9 @@ _NO_PARALLEL_TOOL_USE: Final[dict[str, JsonValue]] = { ("deployment_model", "inference_reply"), ( ("bedrock/converse/us.anthropic.claude-sonnet-4-6", _CONVERSE_REPLY), - ("bedrock/converse/us.anthropic.claude-opus-4-6-v1", _CONVERSE_REPLY), ("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _CLAUDE_REPLY), ), - ids=("converse", "converse_parallel_tool_use_config_model", "invoke"), + ids=("converse", "invoke"), ) async def test_runtime_count_sends_disable_parallel_tool_use_only_when_messages_inference_sends_it( deployment_model: str, inference_reply: Mapping[str, object] diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 6fa522331a4..5b6bbe09ec2 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -1744,20 +1744,28 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_i _CALLER_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value +def _claude_router(deployment_model: str, deployment_params: Mapping[str, JsonValue]) -> Router: + return Router( + model_list=[ + { + "model_name": "claude-agent", + "litellm_params": {"model": deployment_model, "api_key": "sk-ant-test", **deployment_params}, + } + ] + ) + + async def _count_on_a_claude_deployment( monkeypatch: pytest.MonkeyPatch, deployment_model: str, body: Mapping[str, JsonValue], reply: Callable[[httpx.Request], httpx.Response], user_agent: str = "python-httpx/0.28.1", + deployment_params: Mapping[str, JsonValue] = MappingProxyType({}), ) -> tuple[object, tuple[httpx.Request, ...]]: import litellm.proxy.proxy_server as proxy_server - router: Final = Router( - model_list=[ - {"model_name": "claude-agent", "litellm_params": {"model": deployment_model, "api_key": "sk-ant-test"}} - ] - ) + router: Final = _claude_router(deployment_model, deployment_params) request: Final = _json_request( {"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body}, user_agent, @@ -2078,7 +2086,6 @@ async def _claude_inference_body(model: str, options: Mapping[str, JsonValue]) - ("request_drop_params", "user_agent"), ( ({"drop_params": True}, "python-httpx/0.28.1"), - ({"drop_params": " true "}, "python-httpx/0.28.1"), ({}, "claude-cli/2.1.0 (external, cli)"), ), ) @@ -2106,6 +2113,44 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_thinking_ } +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_drop_params", "request_drop_params"), + ((True, "maybe"), (True, False), (None, " true ")), +) +async def test_anthropic_count_tokens_reads_the_request_drop_params_over_the_deployments_like_inference( + monkeypatch: pytest.MonkeyPatch, deployment_drop_params: bool | None, request_drop_params: JsonValue +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + thinking: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000, "summary": "auto"} + deployment_params: Final[Mapping[str, JsonValue]] = ( + {} if deployment_drop_params is None else {"drop_params": deployment_drop_params} + ) + with respx.mock: + inference_route: Final = respx.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY) + ) + await _claude_router("anthropic/claude-opus-4-7", deployment_params).aanthropic_messages( + model="claude-agent", + messages=[{"role": "user", "content": "Weather in SF?"}], + max_tokens=4096, + thinking=thinking, + drop_params=request_drop_params, + ) + sent: Final = json.loads(inference_route.calls.last.request.content) + + response, (count,) = await _count_on_a_claude_deployment( + monkeypatch, + "anthropic/claude-opus-4-7", + {"thinking": thinking, "drop_params": request_drop_params}, + lambda _: httpx.Response(200, json={"input_tokens": 7}), + deployment_params=deployment_params, + ) + + assert not isinstance(response, JSONResponse) + assert json.loads(count.content).get("thinking") == sent.get("thinking") + + @pytest.mark.asyncio async def test_anthropic_count_tokens_on_a_claude_deployment_counts_the_output_format_as_its_output_config_format( monkeypatch: pytest.MonkeyPatch,