fix(shadow-eval): skip hosted web search samples (#40827)

(cherry picked from commit a78cd2fe02)

Co-authored-by: Tin Chi Lo <tin@berri.ai>

litellm/integrations/shadow_eval_logger.py

@@ -27,6 +27,7 @@ from litellm._logging import verbose_logger
 from litellm.caching.in_memory_cache import InMemoryCache
 from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.integrations.websearch_interception.tools import is_web_search_tool_responses
 from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
 from litellm.litellm_core_utils.internal_call_metadata import sanitized_forwardable_call_metadata
 from litellm.litellm_core_utils.llm_judge import (
@@ -335,6 +336,16 @@ def _forwards_nothing(value: object) -> bool:
     return value is None or (isinstance(value, list) and len(value) == 0)
 
 
+def _request_has_hosted_web_search(request: Mapping[str, object]) -> bool:
+    if request.get("web_search_options") is not None:
+        return True
+    tools: Final = request.get("tools")
+    return isinstance(tools, Sequence) and any(
+        isinstance(tool, Mapping) and tool.get("type") != "function" and is_web_search_tool_responses(tool)
+        for tool in tools
+    )
+
+
 def _judgeable_sample(
     ops: _SurfaceOps,
     kwargs: Mapping[str, object],
@@ -343,9 +354,14 @@ def _judgeable_sample(
 ) -> tuple[tuple[Mapping[str, object], ...], Mapping[str, object], str] | None:
     """The normalized chat conversation, the forwardable generation params, and the
     judgeable final text; None when this request's shapes cannot be sampled (no text and no
-    tool call to serialize, or a shape the owner transformations reject)."""
+    tool call to serialize, hosted web search the shadow cannot replay comparably,
+    or a shape the owner transformations reject)."""
+    if _request_has_hosted_web_search(_proxy_wire_body(kwargs) if ops.wire_params else model_parameters):
+        return None
     try:
         request: Final = ops.chat_request(kwargs, model_parameters)
+        if _request_has_hosted_web_search(request):
+            return None
         items: Final = _MESSAGE_ITEMS_ADAPTER.validate_python(request.get("messages"))
         messages: Final = _CHAT_MESSAGES_ADAPTER.validate_python(
             tuple(m.model_dump(exclude_none=True) if isinstance(m, BaseModel) else m for m in items)

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -400,7 +400,7 @@ class LiteLLMAnthropicMessagesAdapter:
 
         Anthropic web search tools have:
         - type starting with "web_search" (e.g., "web_search_20260209")
-        - name = "web_search"
+        - legacy name = "web_search" without a client input_schema
 
         Args:
             tool: Tool definition dict
@@ -410,7 +410,9 @@ class LiteLLMAnthropicMessagesAdapter:
         """
         tool_type: Final = tool.get("type", "")
         tool_name: Final = tool.get("name", "")
-        return (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search"
+        return (isinstance(tool_type, str) and tool_type.startswith("web_search")) or (
+            tool_name == "web_search" and "input_schema" not in tool
+        )
 
     def translate_anthropic_messages_to_openai(
         self,

tests/test_litellm/integrations/test_shadow_eval_logger.py

@@ -2,6 +2,7 @@
 the detached pipeline's single attempt-row write, and the cache-first job lookup."""
 
 import asyncio
+from collections.abc import Mapping
 from datetime import datetime, timedelta, timezone
 from typing import Final
 from unittest.mock import AsyncMock, MagicMock
@@ -421,6 +422,183 @@ class TestSurfaceNormalization:
         assert "previous_response_id" not in shadow_call
         assert "instructions" not in shadow_call
 
+    @pytest.mark.parametrize(
+        "call_type,search_params,model",
+        [
+            ("completion", {"web_search_options": {}}, "anthropic/claude-fable-5"),
+            (
+                "acompletion",
+                {"web_search_options": {"search_context_size": "high"}},
+                "anthropic/claude-fable-5",
+            ),
+            (
+                "acompletion",
+                {"tools": [{"type": "web_search_20260209", "name": "web_search"}]},
+                "anthropic/claude-fable-5",
+            ),
+            (
+                "anthropic_messages",
+                {"tools": [{"type": "web_search_20250305", "name": "web_search"}]},
+                "anthropic/claude-fable-5",
+            ),
+            (
+                "anthropic_messages",
+                {"tools": [{"type": "web_search_20260209", "name": "web_search"}]},
+                "anthropic/claude-fable-5",
+            ),
+            (
+                "anthropic_messages",
+                {"tools": [{"name": "web_search"}]},
+                "anthropic/claude-fable-5",
+            ),
+            ("aresponses", {"tools": [{"type": "web_search"}]}, "anthropic/claude-fable-5"),
+            ("responses", {"tools": [{"type": "web_search_preview"}]}, "anthropic/claude-fable-5"),
+            ("aresponses", {"tools": [{"type": "web_search_2025_08_26"}]}, "anthropic/claude-fable-5"),
+            (
+                "responses",
+                {"tools": [{"type": "web_search_preview_2025_03_11"}]},
+                "anthropic/claude-fable-5",
+            ),
+            ("aresponses", {"tools": [{"type": "web_search"}]}, "bedrock/us.anthropic.claude-fable-5"),
+            (
+                "responses",
+                {"tools": [{"type": "web_search_preview"}]},
+                "bedrock/us.anthropic.claude-fable-5",
+            ),
+            (
+                "acompletion",
+                {
+                    "tools": [
+                        {"type": "function", "function": {"name": "WebSearch", "parameters": {"type": "object"}}},
+                        {"type": "web_search_20260209", "name": "web_search"},
+                    ]
+                },
+                "anthropic/claude-fable-5",
+            ),
+        ],
+        ids=[
+            "chat-empty-options",
+            "chat-configured-options",
+            "chat-provider-transformed-tools",
+            "messages-native-search",
+            "messages-dated-search",
+            "messages-legacy-search-normalized",
+            "responses-search",
+            "responses-preview",
+            "responses-dated-search",
+            "responses-dated-preview",
+            "responses-bedrock-erases-search",
+            "responses-bedrock-erases-preview",
+            "chat-mixed-client-and-hosted-tools",
+        ],
+    )
+    async def test_hosted_web_search_skips_shadow_calls_and_spend(
+        self, call_type: str, search_params: Mapping[str, object], model: str
+    ) -> None:
+        base_kwargs: Final = _success_kwargs(call_type=call_type, model=model)
+        is_chat: Final = call_type in ("completion", "acompletion")
+        is_responses: Final = call_type in ("responses", "aresponses")
+        hook_kwargs: Final = {
+            **base_kwargs,
+            "model": model,
+            "messages": "what is new" if is_responses else base_kwargs["messages"],
+            "standard_logging_object": {
+                **base_kwargs["standard_logging_object"],
+                "model_parameters": search_params if is_chat else {},
+            },
+            "litellm_params": {
+                **base_kwargs["litellm_params"],
+                "proxy_server_request": {"body": {} if is_chat else search_params},
+            },
+        }
+        prisma: Final = _prisma()
+        router: Final = _router()
+        counter: Final = {"spend:shadow_eval:job-1": 0.1, "spend:shadow_eval:job-2": 0.1}
+        logger: Final = _logger(
+            router=router,
+            prisma=prisma,
+            jobs=(_job(max_budget=0.2), _job(id="job-2", max_budget=0.2)),
+            counter_store=counter,
+        )
+
+        await logger.async_log_success_event(
+            hook_kwargs, RESPONSES_API_RESPONSE if is_responses else RESPONSE, None, None
+        )
+        await _drain(logger)
+
+        router.acompletion.assert_not_called()
+        prisma.db.litellm_shadowevalattempt.create.assert_not_called()
+        assert logger._test_funnel == [("job-1", "unjudgeable"), ("job-2", "unjudgeable")]
+        assert logger._job_starts == {}
+        assert logger._test_counter == {"spend:shadow_eval:job-1": 0.1, "spend:shadow_eval:job-2": 0.1}
+
+    @pytest.mark.parametrize(
+        "call_type,tool_name",
+        [
+            (call_type, tool_name)
+            for call_type in ("completion", "acompletion", "anthropic_messages", "responses", "aresponses")
+            for tool_name in ("WebSearch", "litellm_web_search", "web_search")
+        ],
+    )
+    async def test_client_web_search_tools_remain_sampled(self, call_type: str, tool_name: str) -> None:
+        is_chat: Final = call_type in ("completion", "acompletion")
+        is_responses: Final = call_type in ("responses", "aresponses")
+        tool: Final = (
+            {"type": "function", "function": {"name": tool_name, "parameters": {"type": "object"}}}
+            if is_chat
+            else {"type": "function", "name": tool_name, "parameters": {"type": "object"}}
+            if is_responses
+            else {"name": tool_name, "input_schema": {"type": "object", "properties": {}}}
+        )
+        source: Final = {"tools": [tool], "web_search_options": None}
+        base_kwargs: Final = _success_kwargs(call_type=call_type)
+        hook_kwargs: Final = {
+            **base_kwargs,
+            "messages": "search for current news" if is_responses else base_kwargs["messages"],
+            "standard_logging_object": {
+                **base_kwargs["standard_logging_object"],
+                "model_parameters": source if is_chat else {},
+            },
+            "litellm_params": {
+                **base_kwargs["litellm_params"],
+                "proxy_server_request": {"body": {} if is_chat else source},
+            },
+        }
+
+        prisma, router = await self._drive(hook_kwargs, RESPONSES_API_RESPONSE if is_responses else RESPONSE)
+
+        assert router.acompletion.call_count == 2
+        shadow_call: Final = router.acompletion.call_args_list[0].kwargs
+        assert shadow_call["tools"][0]["function"]["name"] == tool_name
+        assert "web_search_options" not in shadow_call
+        prisma.db.litellm_shadowevalattempt.create.assert_called_once()
+
+    @pytest.mark.parametrize("call_type", ["completion", "acompletion"])
+    async def test_chat_search_removed_by_guardrail_still_samples(self, call_type: str) -> None:
+        base_kwargs: Final = _success_kwargs(
+            call_type=call_type,
+            request_metadata={
+                "standard_logging_guardrail_information": [{"guardrail_name": "g", "guardrail_mode": "pre_call"}]
+            },
+        )
+        hook_kwargs: Final = {
+            **base_kwargs,
+            "litellm_params": {
+                **base_kwargs["litellm_params"],
+                "proxy_server_request": {
+                    "body": {"web_search_options": {}, "tools": [{"type": "web_search_20260209"}]}
+                },
+            },
+        }
+
+        prisma, router = await self._drive(hook_kwargs, RESPONSE)
+
+        shadow_call: Final = router.acompletion.call_args_list[0].kwargs
+        assert "web_search_options" not in shadow_call
+        assert "tools" not in shadow_call
+        assert router.acompletion.call_count == 2
+        prisma.db.litellm_shadowevalattempt.create.assert_called_once()
+
     @pytest.mark.parametrize("payload_shape", ["typed", "dict"])
     @pytest.mark.parametrize("call_type", ["aresponses", "responses"])
     async def test_responses_arms_normalize_bare_string_input_and_instructions(self, call_type, payload_shape):

tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py

@@ -3358,6 +3358,27 @@ def test_is_web_search_tool():
     assert adapter._is_web_search_tool(regular_tool) is False
 
 
+@pytest.mark.parametrize("schema", [{}, {"type": "object", "properties": {"query": {"type": "string"}}}])
+def test_translate_anthropic_client_web_search_preserves_schema_and_choice(schema: dict[str, object]) -> None:
+    from litellm.types.llms.anthropic import AnthropicMessagesRequest
+
+    request: Final = AnthropicMessagesRequest(
+        model="gpt-5.4-mini",
+        max_tokens=128,
+        messages=[{"role": "user", "content": "Search for current news"}],
+        tools=[{"name": "web_search", "input_schema": schema}],
+        tool_choice={"type": "tool", "name": "web_search"},
+    )
+
+    translated, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(request)
+
+    assert "web_search_options" not in translated
+    assert translated["tools"] == [
+        {"type": "function", "function": {"name": "web_search", "parameters": schema}}
+    ]
+    assert translated["tool_choice"] == {"type": "function", "function": {"name": "web_search"}}
+
+
 def test_translate_anthropic_to_openai_with_web_search_tool():
     """
     Test that Anthropic web search tools are converted to web_search_options parameter.