scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-03 02:22:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 81

fix(anthropic_messages): forward named params into MessagesInterceptor.handle

Browse source 
When ``anthropic_messages`` dispatches to a registered ``MessagesInterceptor``
(e.g. ``AdvisorOrchestrationHandler``), it currently splats only ``**kwargs``
plus a handful of explicit positional/named args. Top-level parameters bound
as named arguments on ``anthropic_messages`` — ``thinking``, ``metadata``,
``stop_sequences``, ``system``, ``temperature``, ``tool_choice``, ``top_k``,
``top_p`` — are silently dropped, because they live in local variables, not
in ``kwargs``.

This loses request fields on every interceptor sub-call. The most visible
breakage: ``thinking={"type": "adaptive"}`` sent by clients (Claude Code,
Anthropic SDK callers, etc.) is dropped on the executor sub-call, so
downstream providers whose validation depends on ``thinking`` reject the
request. Concretely, Vertex AI returns:

    invalid_request_error: ``clear_thinking_20251015`` strategy requires
    ``thinking`` to be enabled or adaptive

even though the caller correctly sent ``thinking: {type: adaptive}``.

Fix
---
1. Extend the existing ``request_kwargs.pop()`` extraction (already used for
   ``tools`` and ``stream``) to cover all named params we forward to the
   interceptor. This honors pre-request hook overrides for any of those
   fields and prevents duplicate-keyword conflicts when ``**kwargs`` is
   splatted into ``interceptor.handle(...)``.
2. Forward every named parameter explicitly into ``interceptor.handle``, so
   the advisor (and any future interceptor) preserves the full request
   shape on its internal sub-calls.

Tests
-----
- ``test_named_params_forwarded_into_advisor_executor_subcall`` — drives the
  full ``anthropic_messages`` -> interceptor -> executor path and asserts
  all 8 named params arrive in the executor sub-call. Verified to fail on
  master (None vs caller-supplied values) and pass with this fix.
- ``test_pre_request_hook_override_does_not_collide_with_explicit_kwargs`` —
  simulates a ``CustomLogger.async_pre_request_hook`` returning ``thinking``,
  ``system``, ``temperature``. Without the new pops, the explicit-kwarg
  forwarding raises ``TypeError: got multiple values for keyword argument``.
  This test locks in the pop extraction.

All 5 tests in ``test_advisor_integration.py`` pass.
This commit is contained in:
Samarth Maganahalli 2026-05-12 20:29:28 -07:00
parent 8c6625216b
commit cb820163d6
2 changed files with 192 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -219,9 +219,20 @@ async def anthropic_messages(
**kwargs,
)
# Extract modified parameters
# Extract modified parameters. Pop every named param of `anthropic_messages`
# that we may forward explicitly downstream, so we (a) honor pre-request hook
# overrides and (b) avoid duplicate-keyword conflicts when splatting `kwargs`
# into call sites that already pass these as named arguments.
tools = request_kwargs.pop("tools", tools)
stream = request_kwargs.pop("stream", stream)
metadata = request_kwargs.pop("metadata", metadata)
stop_sequences = request_kwargs.pop("stop_sequences", stop_sequences)
system = request_kwargs.pop("system", system)
temperature = request_kwargs.pop("temperature", temperature)
thinking = request_kwargs.pop("thinking", thinking)
tool_choice = request_kwargs.pop("tool_choice", tool_choice)
top_k = request_kwargs.pop("top_k", top_k)
top_p = request_kwargs.pop("top_p", top_p)
# Propagate the provider derived inside pre-request hooks, if not already set.
# The litellm_params dict may have been overwritten by **kwargs in
# _execute_pre_request_hooks, so fall back to get_llm_provider() if needed.
@ -255,8 +266,8 @@ async def anthropic_messages(
return short_circuit_response
# Run registered MessagesInterceptors (e.g. advisor orchestration loop).
# api_key and api_base are explicit params (not in **kwargs) so pass them
# explicitly so interceptor sub-calls can route to the same backend.
# Named params on `anthropic_messages` are bound to locals, not `**kwargs`,
# so forward them explicitly — otherwise interceptor sub-calls drop them.
for interceptor in get_messages_interceptors():
if interceptor.can_handle(tools, custom_llm_provider):
return await interceptor.handle(
@ -268,6 +279,14 @@ async def anthropic_messages(
custom_llm_provider=custom_llm_provider,
api_key=api_key,
api_base=api_base,
metadata=metadata,
stop_sequences=stop_sequences,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
top_k=top_k,
top_p=top_p,
**kwargs,
)

170
tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_advisor_integration.py
View file

@ -194,3 +194,173 @@ async def test_anthropic_provider_bypasses_interceptor():
content = result.get("content", []) if isinstance(result, dict) else []
text_blocks = [b for b in content if b.get("type") == "text"]
assert any("Native anthropic" in b.get("text", "") for b in text_blocks)
# ---------------------------------------------------------------------------
# 4. Regression: top-level named params must be forwarded into executor sub-call
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_named_params_forwarded_into_advisor_executor_subcall():
"""
Regression test: ``thinking``, ``metadata``, ``system``, ``temperature``,
``stop_sequences``, ``tool_choice``, ``top_k``, ``top_p`` are bound as named
parameters on ``anthropic_messages``. They must be forwarded to the
interceptor handler so the advisor executor sub-call carries them through
to the underlying provider.
Without this forwarding, ``thinking={"type": "adaptive"}`` (and others)
are silently dropped, causing 400s on providers whose validation depends on
them, e.g. Vertex AI rejecting ``clear_thinking_20251015`` context_management
edits with: ``strategy requires thinking to be enabled or adaptive``.
"""
from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
anthropic_messages,
)
captured_executor_kwargs: Dict = {}
async def mock_handler(
model, messages, tools, stream, max_tokens, custom_llm_provider, **kwargs
):
# First call is the executor sub-call (returns advisor tool_use).
# Capture its kwargs so we can assert the forwarded params.
if not captured_executor_kwargs:
captured_executor_kwargs.update(
{
"thinking": kwargs.get("thinking"),
"metadata": kwargs.get("metadata"),
"system": kwargs.get("system"),
"temperature": kwargs.get("temperature"),
"stop_sequences": kwargs.get("stop_sequences"),
"tool_choice": kwargs.get("tool_choice"),
"top_k": kwargs.get("top_k"),
"top_p": kwargs.get("top_p"),
}
)
return _advisor_call_resp()
# Subsequent calls — terminate the loop.
if tools is None:
return _text_resp("Some advice.", model="claude-opus-4-6")
return _text_resp("Final answer.")
with patch(
"litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor._call_messages_handler",
side_effect=mock_handler,
):
await anthropic_messages(
model="openai/gpt-4o-mini",
messages=MESSAGES,
tools=[ADVISOR_TOOL],
stream=False,
max_tokens=512,
custom_llm_provider="openai",
thinking={"type": "adaptive"},
metadata={"caller_field": "preserve_me"},
system="You are a helpful assistant.",
temperature=0.7,
stop_sequences=["STOP"],
tool_choice={"type": "auto"},
top_k=40,
top_p=0.9,
)
assert captured_executor_kwargs["thinking"] == {"type": "adaptive"}, (
"thinking must be forwarded into executor sub-call — see "
"anthropic_messages.handler interceptor invocation."
)
# The advisor enriches metadata with `advisor_sub_call` / `parent_request_id`,
# but the original caller fields must survive into the executor sub-call.
assert isinstance(captured_executor_kwargs["metadata"], dict)
assert captured_executor_kwargs["metadata"].get("caller_field") == "preserve_me"
assert captured_executor_kwargs["system"] == "You are a helpful assistant."
assert captured_executor_kwargs["temperature"] == 0.7
assert captured_executor_kwargs["stop_sequences"] == ["STOP"]
assert captured_executor_kwargs["tool_choice"] == {"type": "auto"}
assert captured_executor_kwargs["top_k"] == 40
assert captured_executor_kwargs["top_p"] == 0.9
# ---------------------------------------------------------------------------
# 5. Regression: pre-request hook returning a named param must not cause
# "got multiple values for keyword argument" at the interceptor dispatch.
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_pre_request_hook_override_does_not_collide_with_explicit_kwargs():
"""
``_execute_pre_request_hooks`` may return any subset of params. After
extraction those values are also propagated as named kwargs into the
interceptor, so the same key must not also appear in ``**kwargs`` (or the
splat raises ``TypeError: got multiple values for keyword argument``).
Regression for Greptile P2 on PR #27810.
"""
from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
anthropic_messages,
)
captured: Dict = {}
async def mock_handler(
model, messages, tools, stream, max_tokens, custom_llm_provider, **kwargs
):
if not captured:
captured.update(
{
"thinking": kwargs.get("thinking"),
"system": kwargs.get("system"),
"temperature": kwargs.get("temperature"),
}
)
return _advisor_call_resp()
if tools is None:
return _text_resp("Some advice.", model="claude-opus-4-6")
return _text_resp("Final answer.")
async def fake_pre_request_hooks(
model, messages, tools, stream, custom_llm_provider, **hook_kwargs
):
# Simulate a CustomLogger.async_pre_request_hook that overrides several
# named params on its way through. Without the request_kwargs.pop()
# extraction in handler.py, these would collide with the explicit
# kwargs passed to interceptor.handle() (TypeError: got multiple
# values for keyword argument).
return {
"tools": tools,
"stream": stream,
"litellm_params": {"custom_llm_provider": custom_llm_provider},
"thinking": {"type": "enabled", "budget_tokens": 2048},
"system": "Hook overrode the system prompt.",
"temperature": 0.1,
}
with (
patch(
"litellm.llms.anthropic.experimental_pass_through.messages.handler._execute_pre_request_hooks",
side_effect=fake_pre_request_hooks,
),
patch(
"litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor._call_messages_handler",
side_effect=mock_handler,
),
):
# Should not raise TypeError.
await anthropic_messages(
model="openai/gpt-4o-mini",
messages=MESSAGES,
tools=[ADVISOR_TOOL],
stream=False,
max_tokens=512,
custom_llm_provider="openai",
thinking={"type": "adaptive"},
system="Original system prompt.",
temperature=0.9,
)
# Hook overrides win and reach the executor sub-call.
assert captured["thinking"] == {"type": "enabled", "budget_tokens": 2048}
assert captured["system"] == "Hook overrode the system prompt."
assert captured["temperature"] == 0.1