From e7a09590928d9199c7a7d4e3d015358cbf88592d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:36:03 -0700 Subject: [PATCH 1/3] fix(anthropic_messages): keep the client's max_tokens and other params for agentic follow-up calls The /v1/messages request transform pops max_tokens, thinking, temperature and similar keys out of the params dict in place, and the same dict was then handed to the agentic hooks, so web search interception and the other follow-up builders fell back to max_tokens=1024. Give the transform its own copy. --- litellm/llms/custom_httpx/llm_http_handler.py | 2 +- .../test_websearch_interception_wire.py | 85 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index afa12b58b29..4edd98b05f8 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2579,7 +2579,7 @@ class BaseLLMHTTPHandler: request_body: Final = anthropic_messages_provider_config.transform_anthropic_messages_request( model=model, messages=messages, - anthropic_messages_optional_request_params=request_params, + anthropic_messages_optional_request_params=dict(request_params), litellm_params=litellm_params, headers=headers, ) diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py index 5ec860910e5..a38722b54c5 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py @@ -487,3 +487,88 @@ def test_empty_exa_search_reaches_the_model_without_the_exa_response_envelope(ga assert envelope["requestId"] not in response.text, response.text targets: Final = tuple((request.method, urlsplit(request.target).path) for request in wire.drain()) assert targets[-3:] == (("POST", "/v1/messages"), ("POST", "/exa/search"), ("POST", "/v1/messages")), targets + + +@pytest.mark.parametrize("stream", [False, True]) +def test_web_search_follow_up_call_keeps_the_client_max_tokens(gateway: Gateway, tmp_path: Path, stream: bool) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST", request.target + body: Final = json.loads(request.body) + if request.target == SEARCH_TARGET: + return Reply(body=json.dumps({"query": body["query"], "results": [SEARCH_RESULT]}).encode()) + assert request.target == INVOKE_TARGET, request.target + searched: Final = any( + isinstance(message["content"], list) + and any(block["type"] == "tool_result" for block in message["content"]) + for message in body["messages"] + ) + content: Final = ( + [{"type": "text", "text": "the long answer"}] + if searched + else [{"type": "tool_use", "id": "toolu_0", "name": "litellm_web_search", "input": {"query": "q"}}] + ) + return Reply( + body=json.dumps( + { + "id": "msg_max_tokens", + "type": "message", + "role": "assistant", + "model": BEDROCK_MODEL, + "content": content, + "stop_reason": "end_turn" if searched else "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 4}, + } + ).encode() + ) + + with wire_server(respond) as wire: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["search_tools"] = [ + { + "search_tool_name": "integration-search", + "litellm_params": { + "search_provider": "tavily", + "api_key": "synthetic-tavily-key", + "api_base": wire.url + "/tavily", + }, + } + ] + config["litellm_settings"].update( + { + "callbacks": ["websearch_interception"], + "websearch_interception_params": { + "enabled_providers": ["bedrock"], + "search_tool_name": "integration-search", + }, + } + ) + path: Final = tmp_path / "websearch-max-tokens.yaml" + path.write_text(yaml.safe_dump(config)) + with owned_proxy(gateway, tmp_path, {}, config=path) as candidate, candidate.scenario() as scenario: + model: Final = scenario.model( + model=f"bedrock/{BEDROCK_MODEL}", + api_key="synthetic-bedrock-token", + api_base=wire.url, + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint=wire.url, + ) + response: Final = candidate.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 4000, + "stream": stream, + "messages": [{"role": "user", "content": "search and answer at length"}], + "tools": [{"type": "web_search_20250305", "name": "web_search"}], + }, + ) + assert response.status_code == 200, response.text + assert "the long answer" in response.text, response.text + upstream: Final = wire.drain() + assert [request.target for request in upstream] == [INVOKE_TARGET, SEARCH_TARGET, INVOKE_TARGET] + assert [json.loads(request.body)["max_tokens"] for request in upstream if request.target == INVOKE_TARGET] == [ + 4000, + 4000, + ] From a4a140a12230e24d2cee362fcc07d04731ca99a4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 04:06:08 -0700 Subject: [PATCH 2/3] fix(anthropic_messages): pass the client's max_tokens to agentic hooks in kwargs Revert the shallow copy of the request params and add the client's max_tokens to the hook kwargs instead, where the built-in hooks already fall back. A hook that passes its own max_tokens while spreading the params dict no longer fails, and the web search hook no longer sees the pre-transform thinking budget on adaptive models. A plan's own max_tokens still wins in the plan runner. --- litellm/integrations/custom_logger.py | 6 +- .../websearch_interception/handler.py | 4 +- litellm/llms/custom_httpx/llm_http_handler.py | 21 +-- .../test_websearch_interception_wire.py | 31 +++- .../test_websearch_thinking_constraint.py | 29 ++++ .../custom_httpx/test_llm_http_handler.py | 132 ++++++++++++++++++ 6 files changed, 204 insertions(+), 19 deletions(-) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index a4de1be1842..f9303ce6602 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -703,7 +703,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac anthropic_messages_optional_request_params: Request parameters (tools, etc.) logging_obj: LiteLLM logging object stream: Whether response is streaming - kwargs: Additional request parameters + kwargs: Additional request parameters, including the client's ``max_tokens`` Returns: Final response after executing agentic loop @@ -727,8 +727,8 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac final_response = await messages.acreate( model=model, messages=messages + [assistant_msg, user_msg], - max_tokens=anthropic_messages_optional_request_params.get("max_tokens"), - **anthropic_messages_optional_request_params + max_tokens=kwargs["max_tokens"], + **{k: v for k, v in anthropic_messages_optional_request_params.items() if k != "max_tokens"} ) return final_response diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 6e1ba3eea08..9f53ddca614 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -1377,7 +1377,9 @@ class WebSearchInterceptionLogger(CustomLogger): if max_tokens is None: max_tokens = cast(int, kwargs.get("max_tokens", 1024)) - patch_kwargs: Final = dict[str, object](request_patch.kwargs) + patch_kwargs: Final = { + key: value for key, value in dict[str, object](request_patch.kwargs).items() if key != "max_tokens" + } response: AnthropicMessagesResponse | AsyncIterator[object] = await anthropic_messages.acreate( max_tokens=max_tokens, messages=request_patch.messages, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 77b9dfe90ee..c5170a17c7c 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2486,7 +2486,12 @@ class BaseLLMHTTPHandler: # Prepare headers kwargs = kwargs or {} - kwargs_for_agentic: Final = self._agentic_hook_kwargs(kwargs=kwargs, api_key=api_key, api_base=api_base) + kwargs_for_agentic: Final = self._agentic_hook_kwargs( + kwargs=kwargs, + api_key=api_key, + api_base=api_base, + max_tokens=_OPTIONAL_PARAMS.validate_python(anthropic_messages_optional_request_params).get("max_tokens"), + ) provider_specific_header: Final = cast( litellm.types.utils.ProviderSpecificHeader | Sequence[litellm.types.utils.ProviderSpecificHeader] | None, kwargs.get("provider_specific_header", None), @@ -2580,7 +2585,7 @@ class BaseLLMHTTPHandler: request_body: Final = anthropic_messages_provider_config.transform_anthropic_messages_request( model=model, messages=messages, - anthropic_messages_optional_request_params=dict(request_params), + anthropic_messages_optional_request_params=request_params, litellm_params=litellm_params, headers=headers, ) @@ -5781,12 +5786,12 @@ class BaseLLMHTTPHandler: @staticmethod def _agentic_hook_kwargs( - kwargs: Mapping[str, object], api_key: str | None, api_base: str | None + kwargs: Mapping[str, object], api_key: str | None, api_base: str | None, max_tokens: object ) -> dict[str, object]: - """``api_key`` and ``api_base`` are named parameters of ``anthropic_messages`` rather than kwargs, so the - follow-up call an agentic hook makes only reaches the same deployment if they are re-added here.""" - deployment_params: Final = {"api_key": api_key, "api_base": api_base} - return {**kwargs, **{key: value for key, value in deployment_params.items() if value}} + """The follow-up call an agentic hook makes needs ``api_key``, ``api_base`` and the client's ``max_tokens``. + None of them are in ``kwargs``, and the request transform pops ``max_tokens`` from the hooks' params dict.""" + named_params: Final = {"api_key": api_key, "api_base": api_base, "max_tokens": max_tokens} + return {**kwargs, **{key: value for key, value in named_params.items() if value}} @staticmethod def _has_agentic_completion_hook(logging_obj: LiteLLMLoggingObj) -> bool: @@ -6010,12 +6015,12 @@ class BaseLLMHTTPHandler: response: AnthropicMessagesResponse | AsyncIterator[object] = await anthropic_messages.acreate( **{ - "max_tokens": max_tokens, "messages": patch.messages, "model": patch.model or full_model_name, "stream": stream, **optional_params, **kwargs_for_followup, + "max_tokens": max_tokens, } ) diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py index a38722b54c5..7d8d61b02df 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py @@ -489,14 +489,30 @@ def test_empty_exa_search_reaches_the_model_without_the_exa_response_envelope(ga assert targets[-3:] == (("POST", "/v1/messages"), ("POST", "/exa/search"), ("POST", "/v1/messages")), targets -@pytest.mark.parametrize("stream", [False, True]) -def test_web_search_follow_up_call_keeps_the_client_max_tokens(gateway: Gateway, tmp_path: Path, stream: bool) -> None: +@pytest.mark.parametrize( + ("stream", "bedrock_model", "thinking"), + [ + pytest.param(False, BEDROCK_MODEL, None, id="non-streaming"), + pytest.param(True, BEDROCK_MODEL, None, id="streaming"), + pytest.param( + False, + "us.anthropic.claude-opus-5-5", + {"type": "enabled", "budget_tokens": 8000}, + id="adaptive-model-with-a-thinking-budget-above-max-tokens", + ), + ], +) +def test_web_search_follow_up_call_keeps_the_client_max_tokens( + gateway: Gateway, tmp_path: Path, stream: bool, bedrock_model: str, thinking: dict[str, object] | None +) -> None: + invoke_target: Final = f"/model/{bedrock_model}/invoke" + def respond(request: Request) -> Reply: assert request.method == "POST", request.target body: Final = json.loads(request.body) if request.target == SEARCH_TARGET: return Reply(body=json.dumps({"query": body["query"], "results": [SEARCH_RESULT]}).encode()) - assert request.target == INVOKE_TARGET, request.target + assert request.target == invoke_target, request.target searched: Final = any( isinstance(message["content"], list) and any(block["type"] == "tool_result" for block in message["content"]) @@ -513,7 +529,7 @@ def test_web_search_follow_up_call_keeps_the_client_max_tokens(gateway: Gateway, "id": "msg_max_tokens", "type": "message", "role": "assistant", - "model": BEDROCK_MODEL, + "model": bedrock_model, "content": content, "stop_reason": "end_turn" if searched else "tool_use", "stop_sequence": None, @@ -547,7 +563,7 @@ def test_web_search_follow_up_call_keeps_the_client_max_tokens(gateway: Gateway, path.write_text(yaml.safe_dump(config)) with owned_proxy(gateway, tmp_path, {}, config=path) as candidate, candidate.scenario() as scenario: model: Final = scenario.model( - model=f"bedrock/{BEDROCK_MODEL}", + model=f"bedrock/{bedrock_model}", api_key="synthetic-bedrock-token", api_base=wire.url, aws_region_name="us-east-1", @@ -562,13 +578,14 @@ def test_web_search_follow_up_call_keeps_the_client_max_tokens(gateway: Gateway, "stream": stream, "messages": [{"role": "user", "content": "search and answer at length"}], "tools": [{"type": "web_search_20250305", "name": "web_search"}], + **({"thinking": thinking} if thinking else {}), }, ) assert response.status_code == 200, response.text assert "the long answer" in response.text, response.text upstream: Final = wire.drain() - assert [request.target for request in upstream] == [INVOKE_TARGET, SEARCH_TARGET, INVOKE_TARGET] - assert [json.loads(request.body)["max_tokens"] for request in upstream if request.target == INVOKE_TARGET] == [ + assert [request.target for request in upstream] == [invoke_target, SEARCH_TARGET, invoke_target] + assert [json.loads(request.body)["max_tokens"] for request in upstream if request.target == invoke_target] == [ 4000, 4000, ] diff --git a/tests/unit/integrations/websearch_interception/test_websearch_thinking_constraint.py b/tests/unit/integrations/websearch_interception/test_websearch_thinking_constraint.py index b2d5225070c..8c467b051a0 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_thinking_constraint.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_thinking_constraint.py @@ -125,6 +125,35 @@ class TestThinkingBudgetTokensConstraint: assert captured_kwargs["max_tokens"] == 5000 + 1024 + @pytest.mark.asyncio + async def test_client_max_tokens_passed_in_kwargs_reaches_the_follow_up(self) -> None: + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + captured_kwargs: Dict[str, Any] = {} + + async def _fake_acreate(**kw): + captured_kwargs.update(kw) + return MagicMock() + + with ( + patch( + "litellm.integrations.websearch_interception.handler.anthropic_messages.acreate", + side_effect=_fake_acreate, + ), + patch.object(logger, "_execute_search", return_value=("search result", None)), + ): + await logger._execute_agentic_loop( + model="us.anthropic.claude-opus-4-6-v1", + messages=[{"role": "user", "content": "hi"}], + tool_calls=_make_tool_calls(), + thinking_blocks=[], + anthropic_messages_optional_request_params={}, + logging_obj=_make_logging_obj(), + stream=False, + kwargs={"max_tokens": 4000}, + ) + + assert captured_kwargs["max_tokens"] == 4000 + @pytest.mark.asyncio async def test_max_tokens_unchanged_when_greater_than_budget(self): """max_tokens > thinking.budget_tokens → no adjustment needed.""" diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index f10f5a67b2f..5c07f4da5ee 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -64,6 +64,7 @@ from litellm.responses.streaming_iterator import ( BaseResponsesAPIStreamingIterator, MockResponsesAPIStreamingIterator, ) +from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( @@ -2666,6 +2667,137 @@ class _FakeWebsocketsModule: return outcome +_LOOKUP_CALL: Final = {"type": "tool_use", "id": "toolu_lookup", "name": "lookup", "input": {"q": "answer"}} +_LOOKUP_TOOL: Final = { + "name": "lookup", + "description": "look a fact up", + "input_schema": {"type": "object", "properties": {"q": {"type": "string"}}}, +} + + +def _answer_after_the_lookup(request: httpx.Request) -> httpx.Response: + looked_up: Final = b'"tool_result"' in request.content + return httpx.Response( + 200, + json={ + "id": "msg_lookup", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "the answer"}] if looked_up else [_LOOKUP_CALL], + "stop_reason": "end_turn" if looked_up else "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 5, "output_tokens": 3}, + }, + ) + + +def _messages_with_the_lookup_result(messages: list[dict]) -> list[dict]: + return [ + *messages, + {"role": "assistant", "content": [_LOOKUP_CALL]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_lookup", "content": "42"}]}, + ] + + +class _LookupLoop(CustomLogger): + async def async_should_run_agentic_loop( + self, + response: object, + model: str, + messages: list[dict], + tools: list[dict] | None, + stream: bool, + custom_llm_provider: str, + kwargs: dict, + ) -> tuple[bool, dict]: + return isinstance(response, dict) and response.get("stop_reason") == "tool_use", {} + + +class _OwnMaxTokensLoop(_LookupLoop): + async def async_run_agentic_loop( + self, + tools: dict, + model: str, + messages: list[dict], + response: object, + anthropic_messages_provider_config: object, + anthropic_messages_optional_request_params: dict, + logging_obj: object, + stream: bool, + kwargs: dict, + ) -> object: + return await litellm.anthropic.messages.acreate( + model="anthropic/claude-haiku-5-5", + messages=_messages_with_the_lookup_result(messages), + max_tokens=300, + api_key="sk-ant-test", + **anthropic_messages_optional_request_params, + ) + + +class _PlannedLoop(_LookupLoop): + def __init__(self, max_tokens: int | None) -> None: + super().__init__() + self.max_tokens: Final = max_tokens + + async def async_build_agentic_loop_plan( + self, + tools: dict, + model: str, + messages: list[dict], + response: object, + anthropic_messages_provider_config: object, + anthropic_messages_optional_request_params: dict, + logging_obj: object, + stream: bool, + kwargs: dict, + ) -> AgenticLoopPlan: + return AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + model="anthropic/claude-haiku-5-5", + messages=_messages_with_the_lookup_result(messages), + max_tokens=self.max_tokens, + kwargs={key: value for key, value in kwargs.items() if key != "litellm_logging_obj"}, + ), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("hook", "follow_up_max_tokens"), + [ + pytest.param(_OwnMaxTokensLoop(), 300, id="hook-passes-its-own-max-tokens"), + pytest.param(_PlannedLoop(max_tokens=None), 4000, id="plan-leaves-max-tokens-to-the-client"), + pytest.param(_PlannedLoop(max_tokens=300), 300, id="plan-sets-its-own-max-tokens"), + ], +) +async def test_agentic_follow_up_uses_the_client_max_tokens_unless_the_hook_sets_its_own( + hook: CustomLogger, + follow_up_max_tokens: int, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "callbacks", [hook]) + upstream: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + side_effect=_answer_after_the_lookup + ) + + result: Final = await litellm.anthropic.messages.acreate( + model="anthropic/claude-haiku-5-5", + messages=[{"role": "user", "content": "look the answer up"}], + max_tokens=4000, + api_key="sk-ant-test", + tools=[_LOOKUP_TOOL], + ) + + assert isinstance(result, dict) + assert result["content"] == [{"type": "text", "text": "the answer"}] + assert [json.loads(call.request.content)["max_tokens"] for call in upstream.calls] == [4000, follow_up_max_tokens] + + @pytest.mark.asyncio async def test_realtime_backend_open_retries_then_succeeds(): """A hung/slow open handshake is retried; a later fresh attempt connects. From d8913d812763f1ed730b9cb3fb8a5b73bbff57ed Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:01:04 -0700 Subject: [PATCH 3/3] fix(anthropic_messages): give agentic follow-ups the client's max_tokens through the agentic loop params Hooks get the same params and kwargs as before, so a custom hook that spreads its kwargs keeps working and a plan's own kwargs max_tokens still wins. --- litellm/integrations/custom_logger.py | 6 +-- .../websearch_interception/handler.py | 23 ++++---- .../llms/anthropic/messages/native/handler.py | 1 + litellm/llms/custom_httpx/llm_http_handler.py | 29 +++++----- litellm/types/utils.py | 3 ++ .../test_websearch_thinking_constraint.py | 29 ---------- .../custom_httpx/test_llm_http_handler.py | 54 ++++++++++++------- 7 files changed, 66 insertions(+), 79 deletions(-) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index f9303ce6602..a4de1be1842 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -703,7 +703,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac anthropic_messages_optional_request_params: Request parameters (tools, etc.) logging_obj: LiteLLM logging object stream: Whether response is streaming - kwargs: Additional request parameters, including the client's ``max_tokens`` + kwargs: Additional request parameters Returns: Final response after executing agentic loop @@ -727,8 +727,8 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac final_response = await messages.acreate( model=model, messages=messages + [assistant_msg, user_msg], - max_tokens=kwargs["max_tokens"], - **{k: v for k, v in anthropic_messages_optional_request_params.items() if k != "max_tokens"} + max_tokens=anthropic_messages_optional_request_params.get("max_tokens"), + **anthropic_messages_optional_request_params ) return final_response diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 9f53ddca614..27e16f44afa 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -1296,6 +1296,7 @@ class WebSearchInterceptionLogger(CustomLogger): def _resolve_max_tokens( optional_params: dict, kwargs: dict, + client_max_tokens: int | None = None, ) -> int: """Extract max_tokens and validate against thinking.budget_tokens. @@ -1304,7 +1305,7 @@ class WebSearchInterceptionLogger(CustomLogger): """ max_tokens: int = optional_params.get( "max_tokens", - kwargs.get("max_tokens", 1024), + kwargs.get("max_tokens", 1024 if client_max_tokens is None else client_max_tokens), ) thinking_param: Final = optional_params.get("thinking") if thinking_param and isinstance(thinking_param, dict): @@ -1377,9 +1378,7 @@ class WebSearchInterceptionLogger(CustomLogger): if max_tokens is None: max_tokens = cast(int, kwargs.get("max_tokens", 1024)) - patch_kwargs: Final = { - key: value for key, value in dict[str, object](request_patch.kwargs).items() if key != "max_tokens" - } + patch_kwargs: Final = dict[str, object](request_patch.kwargs) response: AnthropicMessagesResponse | AsyncIterator[object] = await anthropic_messages.acreate( max_tokens=max_tokens, messages=request_patch.messages, @@ -1449,9 +1448,16 @@ class WebSearchInterceptionLogger(CustomLogger): # Correlation context for structured logging _call_id: Final = getattr(logging_obj, "litellm_call_id", None) or kwargs.get("litellm_call_id", "unknown") - full_model_name = model # safe default before try block + agentic_view: Final[_AgenticLoopParamsView] = { + "agentic_loop_params": {} + if logging_obj is None + else logging_obj.model_call_details.get("agentic_loop_params", {}) + } + full_model_name: Final = agentic_view["agentic_loop_params"].get("model", model) - max_tokens: Final = self._resolve_max_tokens(anthropic_messages_optional_request_params, kwargs) + max_tokens: Final = self._resolve_max_tokens( + anthropic_messages_optional_request_params, kwargs, agentic_view["agentic_loop_params"].get("max_tokens") + ) verbose_logger.debug("WebSearchInterception: Using max_tokens=%s for follow-up request", max_tokens) @@ -1460,11 +1466,6 @@ class WebSearchInterceptionLogger(CustomLogger): } kwargs_for_followup: Final = self._prepare_followup_kwargs(kwargs) - if logging_obj is not None: - agentic_view: Final[_AgenticLoopParamsView] = { - "agentic_loop_params": logging_obj.model_call_details.get("agentic_loop_params", {}) - } - full_model_name = agentic_view["agentic_loop_params"].get("model", model) verbose_logger.debug( "WebSearchInterception: Built anthropic request patch [call_id=%s model=%s messages=%d searches=%d]", _call_id, diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 6f97a019a96..02803242f97 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -636,6 +636,7 @@ def anthropic_messages_handler( litellm_logging_obj.model_call_details["agentic_loop_params"] = { "model": original_model, "custom_llm_provider": custom_llm_provider, + "max_tokens": max_tokens, } litellm_logging_obj.model_call_details.setdefault("litellm_params", {})[CallTypes.aanthropic_messages.value] = ( is_async diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index c5170a17c7c..cee04f71324 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2486,12 +2486,7 @@ class BaseLLMHTTPHandler: # Prepare headers kwargs = kwargs or {} - kwargs_for_agentic: Final = self._agentic_hook_kwargs( - kwargs=kwargs, - api_key=api_key, - api_base=api_base, - max_tokens=_OPTIONAL_PARAMS.validate_python(anthropic_messages_optional_request_params).get("max_tokens"), - ) + kwargs_for_agentic: Final = self._agentic_hook_kwargs(kwargs=kwargs, api_key=api_key, api_base=api_base) provider_specific_header: Final = cast( litellm.types.utils.ProviderSpecificHeader | Sequence[litellm.types.utils.ProviderSpecificHeader] | None, kwargs.get("provider_specific_header", None), @@ -5786,12 +5781,12 @@ class BaseLLMHTTPHandler: @staticmethod def _agentic_hook_kwargs( - kwargs: Mapping[str, object], api_key: str | None, api_base: str | None, max_tokens: object + kwargs: Mapping[str, object], api_key: str | None, api_base: str | None ) -> dict[str, object]: - """The follow-up call an agentic hook makes needs ``api_key``, ``api_base`` and the client's ``max_tokens``. - None of them are in ``kwargs``, and the request transform pops ``max_tokens`` from the hooks' params dict.""" - named_params: Final = {"api_key": api_key, "api_base": api_base, "max_tokens": max_tokens} - return {**kwargs, **{key: value for key, value in named_params.items() if value}} + """``api_key`` and ``api_base`` are named parameters of ``anthropic_messages`` rather than kwargs, so the + follow-up call an agentic hook makes only reaches the same deployment if they are re-added here.""" + deployment_params: Final = {"api_key": api_key, "api_base": api_base} + return {**kwargs, **{key: value for key, value in deployment_params.items() if value}} @staticmethod def _has_agentic_completion_hook(logging_obj: LiteLLMLoggingObj) -> bool: @@ -5981,10 +5976,10 @@ class BaseLLMHTTPHandler: if patch.messages is None: raise ValueError("Agentic loop plan missing patched messages") - full_model_name = model - if logging_obj is not None: - agentic_params: Final[Mapping[str, object]] = logging_obj.model_call_details.get("agentic_loop_params", {}) - full_model_name = cast(str, agentic_params.get("model", model)) + agentic_params: Final[Mapping[str, object]] = ( + {} if logging_obj is None else logging_obj.model_call_details.get("agentic_loop_params", {}) + ) + full_model_name: Final = cast(str, agentic_params.get("model", model)) optional_params: Final = dict(anthropic_messages_optional_request_params) optional_params.update(patch.optional_params) @@ -5997,7 +5992,7 @@ class BaseLLMHTTPHandler: else: optional_params.pop("max_tokens", None) if max_tokens is None: - max_tokens = cast(int, kwargs.get("max_tokens", 1024)) + max_tokens = cast(int, kwargs.get("max_tokens", agentic_params.get("max_tokens", 1024))) internal_keys: Final = {"litellm_logging_obj"} kwargs_for_followup: Final = { @@ -6015,12 +6010,12 @@ class BaseLLMHTTPHandler: response: AnthropicMessagesResponse | AsyncIterator[object] = await anthropic_messages.acreate( **{ + "max_tokens": max_tokens, "messages": patch.messages, "model": patch.model or full_model_name, "stream": stream, **optional_params, **kwargs_for_followup, - "max_tokens": max_tokens, } ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 70ca05904ef..aaf0ff990b8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -268,6 +268,9 @@ class AgenticLoopParams(TypedDict, total=False): custom_llm_provider: str """The LLM provider name (e.g., 'bedrock', 'anthropic')""" + max_tokens: ReadOnly[int] + """The client's max_tokens, which the request transform removes from the params the hooks receive""" + class OffPeakWindow(TypedDict, total=False): """One off-peak rule: UTC time-of-day windows, optionally restricted to weekdays. diff --git a/tests/unit/integrations/websearch_interception/test_websearch_thinking_constraint.py b/tests/unit/integrations/websearch_interception/test_websearch_thinking_constraint.py index 8c467b051a0..b2d5225070c 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_thinking_constraint.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_thinking_constraint.py @@ -125,35 +125,6 @@ class TestThinkingBudgetTokensConstraint: assert captured_kwargs["max_tokens"] == 5000 + 1024 - @pytest.mark.asyncio - async def test_client_max_tokens_passed_in_kwargs_reaches_the_follow_up(self) -> None: - logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) - captured_kwargs: Dict[str, Any] = {} - - async def _fake_acreate(**kw): - captured_kwargs.update(kw) - return MagicMock() - - with ( - patch( - "litellm.integrations.websearch_interception.handler.anthropic_messages.acreate", - side_effect=_fake_acreate, - ), - patch.object(logger, "_execute_search", return_value=("search result", None)), - ): - await logger._execute_agentic_loop( - model="us.anthropic.claude-opus-4-6-v1", - messages=[{"role": "user", "content": "hi"}], - tool_calls=_make_tool_calls(), - thinking_blocks=[], - anthropic_messages_optional_request_params={}, - logging_obj=_make_logging_obj(), - stream=False, - kwargs={"max_tokens": 4000}, - ) - - assert captured_kwargs["max_tokens"] == 4000 - @pytest.mark.asyncio async def test_max_tokens_unchanged_when_greater_than_budget(self): """max_tokens > thinking.budget_tokens → no adjustment needed.""" diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 5c07f4da5ee..c05c52daf94 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -7,7 +7,7 @@ import threading import time from collections.abc import Awaitable, Callable, Mapping from types import MappingProxyType -from typing import Final +from typing import Final, Literal from unittest.mock import AsyncMock, Mock, patch import httpx @@ -2692,7 +2692,7 @@ def _answer_after_the_lookup(request: httpx.Request) -> httpx.Response: ) -def _messages_with_the_lookup_result(messages: list[dict]) -> list[dict]: +def _messages_with_the_lookup_result(messages: list[dict[str, object]]) -> list[dict[str, object]]: return [ *messages, {"role": "assistant", "content": [_LOOKUP_CALL]}, @@ -2700,66 +2700,80 @@ def _messages_with_the_lookup_result(messages: list[dict]) -> list[dict]: ] +def _without_the_logging_obj(kwargs: Mapping[str, object]) -> dict[str, object]: + return {key: value for key, value in kwargs.items() if key != "litellm_logging_obj"} + + class _LookupLoop(CustomLogger): async def async_should_run_agentic_loop( self, response: object, model: str, - messages: list[dict], - tools: list[dict] | None, + messages: list[dict[str, object]], + tools: list[dict[str, object]] | None, stream: bool, custom_llm_provider: str, - kwargs: dict, - ) -> tuple[bool, dict]: + kwargs: dict[str, object], + ) -> tuple[bool, dict[str, object]]: return isinstance(response, dict) and response.get("stop_reason") == "tool_use", {} class _OwnMaxTokensLoop(_LookupLoop): + def __init__(self, spreads: Literal["params", "kwargs"]) -> None: + super().__init__() + self.spreads: Final = spreads + async def async_run_agentic_loop( self, - tools: dict, + tools: dict[str, object], model: str, - messages: list[dict], + messages: list[dict[str, object]], response: object, anthropic_messages_provider_config: object, - anthropic_messages_optional_request_params: dict, + anthropic_messages_optional_request_params: dict[str, object], logging_obj: object, stream: bool, - kwargs: dict, + kwargs: dict[str, object], ) -> object: + spread: Final = ( + {"api_key": "sk-ant-test", **anthropic_messages_optional_request_params} + if self.spreads == "params" + else _without_the_logging_obj(kwargs) + ) return await litellm.anthropic.messages.acreate( model="anthropic/claude-haiku-5-5", messages=_messages_with_the_lookup_result(messages), max_tokens=300, - api_key="sk-ant-test", - **anthropic_messages_optional_request_params, + **spread, # pyright: ignore[reportArgumentType] # a hook forwards the request's params as they arrive ) class _PlannedLoop(_LookupLoop): - def __init__(self, max_tokens: int | None) -> None: + def __init__(self, max_tokens: int | None, kwargs_max_tokens: int | None = None) -> None: super().__init__() self.max_tokens: Final = max_tokens + self.kwargs_max_tokens: Final = kwargs_max_tokens async def async_build_agentic_loop_plan( self, - tools: dict, + tools: dict[str, object], model: str, - messages: list[dict], + messages: list[dict[str, object]], response: object, anthropic_messages_provider_config: object, - anthropic_messages_optional_request_params: dict, + anthropic_messages_optional_request_params: dict[str, object], logging_obj: object, stream: bool, - kwargs: dict, + kwargs: dict[str, object], ) -> AgenticLoopPlan: + own_kwargs: Final = {} if self.kwargs_max_tokens is None else {"max_tokens": self.kwargs_max_tokens} return AgenticLoopPlan( run_agentic_loop=True, request_patch=AgenticLoopRequestPatch( model="anthropic/claude-haiku-5-5", messages=_messages_with_the_lookup_result(messages), max_tokens=self.max_tokens, - kwargs={key: value for key, value in kwargs.items() if key != "litellm_logging_obj"}, + kwargs={**_without_the_logging_obj(kwargs), **own_kwargs}, ), ) @@ -2768,9 +2782,11 @@ class _PlannedLoop(_LookupLoop): @pytest.mark.parametrize( ("hook", "follow_up_max_tokens"), [ - pytest.param(_OwnMaxTokensLoop(), 300, id="hook-passes-its-own-max-tokens"), + pytest.param(_OwnMaxTokensLoop(spreads="params"), 300, id="hook-spreads-the-params-with-its-own-max-tokens"), + pytest.param(_OwnMaxTokensLoop(spreads="kwargs"), 300, id="hook-spreads-the-kwargs-with-its-own-max-tokens"), pytest.param(_PlannedLoop(max_tokens=None), 4000, id="plan-leaves-max-tokens-to-the-client"), pytest.param(_PlannedLoop(max_tokens=300), 300, id="plan-sets-its-own-max-tokens"), + pytest.param(_PlannedLoop(max_tokens=None, kwargs_max_tokens=300), 300, id="plan-sets-max-tokens-in-kwargs"), ], ) async def test_agentic_follow_up_uses_the_client_max_tokens_unless_the_hook_sets_its_own(