Merge remote-tracking branch 'origin/main' into litellm_mcp_options_without_auth

.github/PULL_REQUEST_TEMPLATE/general.md

@@ -9,10 +9,16 @@
 <!-- The PR author rewrites this in their own words before asking for review. Being able to sum up the PR in a few
      plain sentences proves the author understands what it does, and human-written technical English is far easier
      for a reviewer to parse than AI-written English
-     An AI agent may draft a starting point here, but the author replaces it with their own handwritten version
+     An AI agent may draft a starting point here, but always wraps its draft in the make-me-human-readable tags below,
+     backticks included so the tags stay visible on GitHub, so nobody mistakes it for the author's words. The author rewrites it in their own words and deletes the tags.
+     Tags still present at review time mean the summary is AI-written and has not been rewritten yet
      Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
      No bullets, tables, or code names here. Everything else goes inside the Details block below -->
 
+`<make-me-human-readable>`
+<blah>
+`</make-me-human-readable>`
+
 <details>
 <summary>Details</summary>
 

AGENTS.md

@@ -41,6 +41,8 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM
 
 If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section
 
+Run QA proxies with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --num_workers 2 --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`, and swap its `--config` for a copy of `litellm/proxy/dev_config.yaml` that points both `router_settings` (`redis_host`, `redis_port`) and `general_settings.coordination_redis` (`host`, `port`) at one dedicated Redis instance, with credentials when required, so the workers share router state and spend, budget, and rate-limit counters. Check that two worker PIDs are serving, and use the same worker count and Redis setup for the before and after proof. When the PR is about `--run_gunicorn`, `--run_hypercorn`, or `--run_granian`, which `--reload` does not support, drop `--reload`. When it is about behavior without Redis, drop Redis. When the proxy prints `"workers" flag is ignored when reloading is enabled`, that checkout or its uvicorn reloads only one worker, so drop the two-worker check for it. Say in the proof which part you dropped and why
+
 Simple PRs may skip before/after e2e when the full diff is mechanical and focused checks completely prove it, such as #49030 restoring an existing helper name. Trace affected callers and run relevant tests/static checks at the tip. For a bug fix, establish that the regression catches the original defect. Size alone never qualifies. New behavior, API/provider semantics, auth, billing, persistence, concurrency or uncertain integration risk need live e2e. In Screenshots / Proof of Fix, record `E2E exception at <tip SHA>`, scope, rationale, commands and results. Tests count as proof only for this exception. Reassess changed diffs, retain CI/bot gates, and honor explicit e2e requests (2026-10-28, unnecessary live QA for #49030)
 
 For other PRs, proof must show the end user's real flow before and after, with commands and output against a live proxy and real provider calls where applicable. Pytest alone is insufficient. For UI or interactive coding-tool flows, drive that surface and embed before/after screenshots, with URLs, clicks and input fields a reviewer can reproduce

README.md

@@ -696,7 +696,7 @@ For companies that need better security, user management and professional suppor
 [Talk to founders](https://enterprise.litellm.ai/demo)
 
 This covers:
-- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**
+- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/enterprise):**
 - ✅ **Feature Prioritization**
 - ✅ **Custom Integrations**
 - ✅ **Professional Support - Dedicated discord + slack**

litellm/__init__.py

@@ -1647,6 +1647,7 @@ def set_global_gitlab_config(config: Mapping[str, object]) -> None:
 # Lazy loading system for heavy modules to reduce initial import time and memory usage
 
 if TYPE_CHECKING:
+    from . import guardrails as guardrails
     from litellm.types.utils import ModelInfo as _ModelInfoType
     from litellm.types.utils import PriorityReservationSettings
     from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@@ -2499,10 +2500,10 @@ def __getattr__(name: str) -> Any:
         handler_func: Final = registry[name]
         return handler_func(name)
 
-    if name == "proxy":
+    if name in ("proxy", "guardrails"):
         import importlib
 
-        return importlib.import_module("litellm.proxy")
+        return importlib.import_module(f"litellm.{name}")
 
     if name == "harness" or name in _AGENT_EXPORTS:
         import importlib

litellm/guardrails.py

@@ -0,0 +1,66 @@
+"""Invoke configured gateway guardrails without generating a completion."""
+
+from collections.abc import Mapping, Sequence
+from enum import Enum
+from typing import Final
+
+import httpx
+from pydantic import TypeAdapter
+
+import litellm
+from litellm.llms.custom_httpx.http_handler import (
+    get_async_httpx_client,  # pyright: ignore[reportUnknownVariableType]  # shared factory has untyped optional parameters
+)
+from litellm.types.guardrails import NativeGuardrailResponse
+
+_NATIVE_RESPONSE: Final[TypeAdapter[NativeGuardrailResponse]] = TypeAdapter(NativeGuardrailResponse)
+
+
+class _InputType(Enum):
+    REQUEST = "request"
+    RESPONSE = "response"
+
+
+async def apply(
+    guardrail_name: str,
+    *,
+    input: str | None = None,
+    response: str | None = None,
+    messages: Sequence[Mapping[str, object]] | None = None,
+    metadata: Mapping[str, object] | None = None,
+    api_base: str | None = None,
+    api_key: str | None = None,
+    timeout: float | httpx.Timeout | None = None,
+) -> NativeGuardrailResponse:
+    """Return a configured guardrail's native structured findings.
+
+    Supply exactly one of ``input`` or ``response``. Grounding guardrails can
+    use ``messages`` as evidence. Connection settings default to
+    ``litellm.api_base`` and ``litellm.api_key``.
+    """
+    if (input is None) == (response is None):
+        raise ValueError("Supply exactly one of input or response.")
+    resolved_base: Final = api_base if api_base is not None else litellm.api_base
+    if not resolved_base:
+        raise ValueError("Set api_base or litellm.api_base to your LiteLLM gateway URL.")
+    resolved_key: Final = api_key if api_key is not None else litellm.api_key
+    input_type: Final = _InputType.RESPONSE if response is not None else _InputType.REQUEST
+    payload: Final = dict[str, object](
+        guardrail_name=guardrail_name,
+        text=response if response is not None else input,
+        input_type=input_type.value,
+        response_format="native",
+    )
+    if messages is not None:
+        payload["messages"] = [dict(message) for message in messages]
+    if metadata is not None:
+        payload["metadata"] = dict(metadata)
+    gateway_root: Final = resolved_base.rstrip("/").removesuffix("/v1")
+    client: Final = get_async_httpx_client(llm_provider="litellm_guardrails")
+    result: Final = await client.post(  # pyright: ignore[reportUnknownMemberType]  # shared handler has untyped optional parameters
+        url=f"{gateway_root}/guardrails/apply_guardrail",
+        json=payload,
+        headers={"Authorization": f"Bearer {resolved_key}"} if resolved_key else {},
+        timeout=timeout,
+    )
+    return _NATIVE_RESPONSE.validate_json(result.content)

litellm/integrations/anthropic_cache_control_hook.py

@@ -32,6 +32,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
     tool_call_ids_answered_by_client,
     tool_call_ids_answered_in,
 )
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import string_field
 from litellm.llms.anthropic.common_utils import (
     is_claude_code_one_shot_subagent_request,
     supports_anthropic_cache_control,
@@ -128,7 +129,7 @@ def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool:
         return False
     return (
         not isinstance(fields.get("content"), list)
-        or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
+        or string_field(fields, "role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
     )
 
 

litellm/litellm_core_utils/core_helpers.py

@@ -315,6 +315,8 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
     "model_context_window_exceeded": "length",
     # Cohere
     "COMPLETE": "stop",
+    "STOP_SEQUENCE": "stop",
+    "TOOL_CALL": "tool_calls",
     "ERROR_TOXIC": "content_filter",
     "ERROR": "stop",
     # HuggingFace / Together AI

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -8,7 +8,7 @@ import io
 import json
 import mimetypes
 import re
-from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from graphlib import CycleError, TopologicalSorter
 from itertools import chain, groupby, islice
@@ -3235,3 +3235,15 @@ def text_completion_prompt_to_messages(prompt: object) -> tuple[AllMessageValues
     if string_entries and len(string_entries) == len(entries):
         return tuple(ChatCompletionUserMessage(role="user", content=entry) for entry in string_entries)
     raise ValueError(f"`prompt` must be a non-empty string or a non-empty list of strings. Got: {prompt_type_name}.")
+
+
+def is_present_non_string(value: object) -> bool:
+    return value is not None and not isinstance(value, str)
+
+
+def invalid_type_message(field_path: str, value: object) -> str:
+    return f"Invalid type for '{field_path}': expected a string, got {type(value).__name__}"
+
+
+def is_unhashable_value_behind(error: TypeError, value: object) -> bool:
+    return not isinstance(value, Hashable) and f"unhashable type: '{type(value).__name__}'" in str(error)

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -32,6 +32,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
     parts_of,
     place_mid_conversation_system,
     split_leading_system_run,
+    string_field,
 )
 from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
 from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client, header_value
@@ -2023,7 +2024,7 @@ def _server_tool_answers_replaced_after(messages: Sequence[object], index: int)
     following: Final = _messages_until_the_next_assistant(messages, index)
     a_user_turn_remains: Final = any(
         message_field(message, "role") in ("user", "tool", "function")
-        and message_field(message, "tool_call_id") not in kept_server_result_ids
+        and string_field(message, "tool_call_id") not in kept_server_result_ids
         for message in following
     )
     return _tool_call_ids_in_tool_messages(following) & kept_server_result_ids if a_user_turn_remains else frozenset()

litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py

@@ -85,6 +85,11 @@ def message_field(message: object, key: str) -> object:
     return mapping.get(key) if mapping is not None else getattr(message, key, None)
 
 
+def string_field(message: object, key: str) -> str | None:
+    value: Final = message_field(message, key)
+    return value if isinstance(value, str) else None
+
+
 def is_system_message(message: object) -> bool:
     return message_field(message, "role") == "system"
 

litellm/litellm_core_utils/streaming_handler.py

@@ -905,13 +905,21 @@ class CustomStreamWrapper:
         except Exception as e:
             raise e
 
+    def _served_openrouter_model(self) -> str | None:
+        if self.custom_llm_provider == LlmProviders.OPENROUTER.value or (
+            self.custom_llm_provider == "cached_response"
+            and self.logging_obj.model_call_details.get("custom_llm_provider") == LlmProviders.OPENROUTER.value
+        ):
+            return self._provider_response_model
+        return None
+
     def model_response_creator(
         self,
         chunk: dict | None = None,
         hidden_params: Mapping[str, object] | None = None,
         delta: Delta | None = None,
     ) -> ModelResponseStream:
-        _model: Final = self._cached_model_name
+        _model: Final = self._served_openrouter_model() or self._cached_model_name
         _logging_obj_llm_provider: Final = self._cached_logging_llm_provider
 
         if chunk is None:
@@ -1699,7 +1707,7 @@ class CustomStreamWrapper:
                 return dispatch_result.value
             response_obj = dispatch_result.response_obj
 
-            model_response.model = self.model
+            model_response.model = self._served_openrouter_model() or self.model
             ## FUNCTION CALL PARSING
             original_chunk: Final[_UpstreamChunkLike | None] = (
                 response_obj.get("original_chunk") if response_obj is not None else None
@@ -2561,7 +2569,7 @@ class CustomStreamWrapper:
         if partial is None or not isinstance(usage, Usage):
             return None
         if self.model:
-            partial.model = self.model
+            partial.model = self._served_openrouter_model() or self.model
         backfill_missing_cache_usage_fields(usage)
         return partial
 

litellm/llms/anthropic/chat/transformation.py

@@ -2,7 +2,9 @@ import copy
 import json
 import re
 import time
-from collections.abc import Callable, Mapping, Sequence
+from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from functools import partial
+from itertools import chain
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, cast
 
@@ -27,7 +29,10 @@ from litellm.constants import (
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     DEFS_MAX_INLINED_BYTES,
+    invalid_type_message,
     is_object_schema,
+    is_present_non_string,
+    is_unhashable_value_behind,
     sanitize_input_schema_for_anthropic,
     schema_fragment_ref,
     schema_grammar_branch_count,
@@ -45,6 +50,8 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
     inline_remote_image_urls,
 )
 from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
+    message_field,
+    parts_of,
     place_mid_conversation_system,
     split_leading_system_run,
 )
@@ -179,6 +186,76 @@ NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS: Final = 24
 NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES: Final = 2048
 
 
+def _tool_call_ids_failing(
+    index: int, tool_calls: object, fails: Callable[[object], bool]
+) -> Iterator[tuple[str, object]]:
+    ids: Final = ((position, message_field(tool_call, "id")) for position, tool_call in enumerate(parts_of(tool_calls)))
+    return (
+        (f"messages[{index}].tool_calls[{position}].id", tool_call_id)
+        for position, tool_call_id in ids
+        if fails(tool_call_id)
+    )
+
+
+def _roles_failing(messages: Sequence[object], fails: Callable[[object], bool]) -> Iterator[tuple[str, object]]:
+    roles: Final = ((index, message_field(message, "role")) for index, message in enumerate(messages))
+    return ((f"messages[{index}].role", role) for index, role in roles if fails(role))
+
+
+def _is_non_empty_non_string(value: object) -> bool:
+    return bool(value) and is_present_non_string(value)
+
+
+def _assistant_tool_call_ids_failing(
+    messages: Sequence[object], fails: Callable[[object], bool]
+) -> Iterator[tuple[str, object]]:
+    assistant_turns: Final = (
+        (index, message) for index, message in enumerate(messages) if message_field(message, "role") == "assistant"
+    )
+    return chain.from_iterable(
+        _tool_call_ids_failing(index, message_field(message, "tool_calls"), fails) for index, message in assistant_turns
+    )
+
+
+def _tool_answer_ids_failing(
+    messages: Sequence[object], fails: Callable[[object], bool]
+) -> Iterator[tuple[str, object]]:
+    answers: Final = (
+        (index, message_field(message, "tool_call_id"))
+        for index, message in enumerate(messages)
+        if message_field(message, "role") in ("tool", "function")
+    )
+    return ((f"messages[{index}].tool_call_id", tool_call_id) for index, tool_call_id in answers if fails(tool_call_id))
+
+
+def _is_non_string_value_behind(error: AttributeError, value: object) -> bool:
+    return is_present_non_string(value) and value is error.obj
+
+
+def _fields_behind(messages: Sequence[object], error: Exception) -> Iterable[tuple[str, object]]:
+    match error:
+        case TypeError():
+            caused_error: Final = partial(is_unhashable_value_behind, error)
+            return chain(
+                _assistant_tool_call_ids_failing(messages, caused_error),
+                _tool_answer_ids_failing(messages, caused_error),
+                _roles_failing(messages, caused_error),
+            )
+        case AttributeError():
+            return _assistant_tool_call_ids_failing(messages, partial(_is_non_string_value_behind, error))
+        case _:
+            return ()
+
+
+def non_string_message_field_error(messages: Sequence[object], error: Exception) -> AnthropicError | None:
+    malformed_field: Final = next(
+        chain(_roles_failing(messages, _is_non_empty_non_string), _fields_behind(messages, error)), None
+    )
+    if malformed_field is None:
+        return None
+    return AnthropicError(status_code=400, message=invalid_type_message(*malformed_field))
+
+
 def _tool_type(tool: object) -> str | None:
     match tool:
         case {"type": str() as tool_type}:
@@ -2370,12 +2447,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         # Handling anthropic API Prompt Caching
         if len(anthropic_system_message_list) > 0:
             optional_params["system"] = anthropic_system_message_list
-        conversation: Final = place_mid_conversation_system(
-            later_messages,
-            supports_mid_conversation_system=supports_mid_conversation_system(
-                model=model, custom_llm_provider=self.custom_llm_provider
-            ),
-        )
+        try:
+            conversation: Final = place_mid_conversation_system(
+                later_messages,
+                supports_mid_conversation_system=supports_mid_conversation_system(
+                    model=model, custom_llm_provider=self.custom_llm_provider
+                ),
+            )
+        except TypeError as error:
+            field_error: Final = non_string_message_field_error(messages, error)
+            if field_error is None:
+                raise
+            raise field_error from error
         # Format rest of message according to anthropic guidelines
         try:
             anthropic_messages = anthropic_messages_pt(
@@ -2385,7 +2468,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 drop_params=drop_params,
             )
         except Exception as e:
-            raise AnthropicError(
+            conversion_field_error: Final = non_string_message_field_error(messages, e)
+            raise conversion_field_error or AnthropicError(
                 status_code=400,
                 message=f"{e}\nReceived Messages={request_messages}",
             )  # don't use verbose_logger.exception, if exception is raised

litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py

@@ -74,6 +74,7 @@ class _ClearToolUsesFields(BaseModel):
     clear_tool_inputs: bool | tuple[str, ...] | None = None
 
 
+_BLOCK: Final = TypeAdapter(Mapping[str, object])
 _BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
 # api.anthropic.com, measured 2026-10-19 and 2026-10-28: its wording for the shapes pydantic words differently
 _API_MESSAGES: Final = MappingProxyType(
@@ -317,12 +318,10 @@ def _clear_tool_results(
         new_blocks: list[object] = []
         mutated = False
         for block in content:
-            if (
-                isinstance(block, dict)
-                and block.get("type") == "tool_result"
-                and block.get("tool_use_id") in ids_to_clear
-            ):
-                new_blocks.append(_cleared_result(block))
+            if (parsed := _parse(_BLOCK, block)) is not None and _block_id(
+                parsed, "tool_result", "tool_use_id"
+            ) in ids_to_clear:
+                new_blocks.append(_cleared_result(parsed))
                 mutated = True
                 cleared += 1
             else:

litellm/llms/base_llm/files/batch_records.py

@@ -19,6 +19,8 @@ def responses_batch_body_to_chat_body(
     openai_request_body: Mapping[str, object],
     custom_llm_provider: str | None = None,
     drop_params: bool | None = None,
+    *,
+    caller_input: str | ResponseInputParam | None = None,
 ) -> dict[str, object]:  # mutable-ok: provider transforms take the bridged chat body as a plain dict
     """
     Rewrite the body of an OpenAI `/v1/responses` batch record as a Chat Completions body.
@@ -52,5 +54,6 @@ def responses_batch_body_to_chat_body(
         responses_api_request=responses_request,
         custom_llm_provider=custom_llm_provider,
         drop_params=drop_params,
+        caller_input=caller_input,
         metadata=openai_request_body.get("metadata"),
     )

litellm/llms/bedrock/files/transformation.py

@@ -42,6 +42,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     extract_file_data,
     text_completion_prompt_to_messages,
 )
+from litellm.llms.anthropic.common_utils import AnthropicError
 from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body
@@ -347,7 +348,14 @@ class _BatchInputLine:
     record: _OpenAIBatchRecord
 
 
-_BATCH_RECORD_CONVERSION_ERRORS: Final = (AttributeError, LookupError, TypeError, ValueError, BadRequestError)
+_BATCH_RECORD_CONVERSION_ERRORS: Final = (
+    AttributeError,
+    LookupError,
+    TypeError,
+    ValueError,
+    BadRequestError,
+    AnthropicError,
+)
 
 
 def _batch_line_error(line_number: int, problem: str) -> BedrockError:

litellm/llms/cohere/common_utils.py

@@ -3,6 +3,7 @@ from typing import Final, Literal
 
 from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.types.llms.cohere import CohereV2StreamedToolCall
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import (
     ChatCompletionToolCallChunk,
@@ -222,6 +223,7 @@ class CohereV2ModelResponseIterator:
         self.content_blocks: list = []
         self.tool_index = -1
         self.json_mode = json_mode
+        self.tool_call_indexes_with_arguments: frozenset[int] = frozenset()
 
     def _parse_content_delta(self, chunk: dict) -> str:
         """Parse content-delta chunks to extract text."""
@@ -234,20 +236,23 @@ class CohereV2ModelResponseIterator:
             return content
         return ""
 
-    def _parse_tool_call_delta(self, chunk: dict) -> ChatCompletionToolCallChunk | None:
-        """Parse tool-call-delta chunks to extract tool calls."""
-        delta: Final = chunk.get("delta", {})
-        tool_calls: Final = delta.get("tool_calls", [])
-        if tool_calls:
-            return {
-                "id": tool_calls[0].get("id", ""),
-                "type": "function",
-                "function": {
-                    "name": tool_calls[0].get("name", ""),
-                    "arguments": tool_calls[0].get("arguments", ""),
-                },
-            }
-        return None
+    def _parse_tool_call_delta(self, chunk: dict, index: int) -> ChatCompletionToolCallChunk:
+        """Parse tool-call-start and tool-call-delta chunks to extract tool calls."""
+        tool_call: Final = CohereV2StreamedToolCall.model_validate(chunk)
+        if tool_call.arguments:
+            self.tool_call_indexes_with_arguments = self.tool_call_indexes_with_arguments | {index}
+        return {
+            "id": tool_call.id,
+            "type": "function",
+            "function": {"name": tool_call.name, "arguments": tool_call.arguments},
+            "index": index,
+        }
+
+    def _parse_tool_call_end(self, index: int) -> ChatCompletionToolCallChunk | None:
+        """Close a tool call that streamed no arguments with an empty JSON object, as OpenAI streams one."""
+        if index in self.tool_call_indexes_with_arguments:
+            return None
+        return {"id": None, "type": "function", "function": {"name": None, "arguments": "{}"}, "index": index}
 
     def _parse_tool_plan_delta(self, chunk: dict) -> dict | None:
         """Parse tool-plan-delta events to extract tool plan."""
@@ -301,7 +306,8 @@ class CohereV2ModelResponseIterator:
 
         v2 format:
         - Content: chunk.type == "content-delta" -> chunk.delta.message.content.text
-        - Tool calls: chunk.type == "tool-call-delta" -> chunk.delta.tool_calls
+        - Tool calls: chunk.type in ("tool-call-start", "tool-call-delta") -> chunk.delta.message.tool_calls
+        - Tool call end: chunk.type == "tool-call-end" -> "{}" arguments when the call streamed none
         - Tool plan: chunk.event == "tool-plan-delta" -> chunk.data.delta.message.tool_plan
         - Citations: chunk.event == "citation-start" -> chunk.data.delta.message.citations
         - Finish: chunk.event == "message-end" -> chunk.data.delta.finish_reason
@@ -321,8 +327,12 @@ class CohereV2ModelResponseIterator:
             # Handle different chunk types
             if chunk_type == "content-delta":
                 text = self._parse_content_delta(chunk)
-            elif chunk_type == "tool-call-delta":
-                tool_use = self._parse_tool_call_delta(chunk)
+            elif chunk_type in ("tool-call-start", "tool-call-delta", "tool-call-end"):
+                tool_use = (
+                    self._parse_tool_call_end(index)
+                    if chunk_type == "tool-call-end"
+                    else self._parse_tool_call_delta(chunk, index)
+                )
             elif event_type == "tool-plan-delta":
                 provider_specific_fields = self._parse_tool_plan_delta(chunk)
             elif event_type == "citation-start":

litellm/llms/custom_httpx/aiohttp_handler.py

@@ -14,6 +14,7 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig
 from litellm.llms.base_llm.image_variations.transformation import (
     BaseImageVariationConfig,
 )
+from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import build_pool_wait_trace_config
 from litellm.llms.custom_httpx.aiohttp_transport import LiteLLMAiohttpTransport
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -90,7 +91,7 @@ class BaseLLMAIOHTTPHandler:
         if self.transport is None:
             connector: Final = self._get_connector()
             if connector:
-                return aiohttp.ClientSession(connector=connector)
+                return aiohttp.ClientSession(connector=connector, trace_configs=[build_pool_wait_trace_config()])
 
         transport: Final = self.transport or self._get_or_create_transport()
         if transport is not None and hasattr(transport, "_get_valid_client_session"):
@@ -99,7 +100,7 @@ class BaseLLMAIOHTTPHandler:
             except RuntimeError:
                 pass
 
-        return aiohttp.ClientSession()
+        return aiohttp.ClientSession(trace_configs=[build_pool_wait_trace_config()])
 
     def _get_async_client_session(self, dynamic_client_session: ClientSession | None = None) -> ClientSession:
         if dynamic_client_session:

litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py

@@ -0,0 +1,130 @@
+import asyncio
+from types import SimpleNamespace
+from typing import Final
+
+from aiohttp import (
+    ClientSession,
+    TraceConfig,
+    TraceConnectionQueuedEndParams,
+    TraceConnectionQueuedStartParams,
+    TraceRequestExceptionParams,
+    TraceRequestRedirectParams,
+    TraceRequestStartParams,
+)
+from yarl import URL
+
+from litellm._logging import verbose_logger
+
+_POOL_WAIT_WARNING_SECONDS: Final = 1.0
+
+
+class _PoolWaitContext(SimpleNamespace):
+    def __init__(self, trace_request_ctx: object = None) -> None:
+        super().__init__()
+        self.host: str | None = None
+        self.port: int | None = None
+        self.started_at: float | None = None
+        self.waited: float = 0.0
+        self.warning: asyncio.TimerHandle | None = None
+        self.warned: bool = False
+
+    def set_destination(self, params: TraceRequestStartParams) -> None:
+        self.host = params.url.host
+        self.port = params.url.port
+
+    def redirect(self, params: TraceRequestRedirectParams) -> None:
+        location: Final = params.response.headers.get("Location") or params.response.headers.get("URI")
+        if location is None:
+            return
+        try:
+            destination: Final = params.url.join(URL(location))
+            host: Final = destination.host
+            port: Final = destination.port
+        except ValueError:
+            return
+        self.host = host
+        self.port = port
+
+    def start(self, session: ClientSession) -> None:
+        loop: Final = asyncio.get_running_loop()
+        self.started_at = loop.time()
+        if not self.warned:
+            self.warning = loop.call_later(max(0.0, _POOL_WAIT_WARNING_SECONDS - self.waited), self.warn, session)
+
+    def elapsed(self) -> float:
+        return self.waited + (
+            asyncio.get_running_loop().time() - self.started_at if self.started_at is not None else 0.0
+        )
+
+    def warn(self, session: ClientSession) -> None:
+        if self.warned or self.started_at is None:
+            return
+        self.warned = True
+        connector: Final = session.connector
+        if connector is None:
+            return
+        verbose_logger.warning(
+            "aiohttp connection pool saturated: host=%s port=%s waited=%.3fs for a connection slot "
+            "(acquired_total=%s, limit=%s, limit_per_host=%s). "
+            "Wait occurred before sending the request upstream.",
+            self.host,
+            self.port,
+            self.elapsed(),
+            len(connector._acquired),  # pyright: ignore[reportPrivateUsage]  # aiohttp exposes no public acquired count
+            connector.limit,
+            connector.limit_per_host,
+        )
+
+    def finish(self, session: ClientSession, failed: bool = False) -> None:
+        if self.warning is not None:
+            self.warning.cancel()
+            self.warning = None
+        if self.started_at is None:
+            return
+        if failed or self.elapsed() >= _POOL_WAIT_WARNING_SECONDS:
+            self.warn(session)
+        self.waited = self.elapsed()
+        self.started_at = None
+
+
+async def _on_request_start(session: ClientSession, context: SimpleNamespace, params: TraceRequestStartParams) -> None:
+    if isinstance(context, _PoolWaitContext):
+        context.set_destination(params)
+
+
+async def _on_queued_start(
+    session: ClientSession, context: SimpleNamespace, params: TraceConnectionQueuedStartParams
+) -> None:
+    if isinstance(context, _PoolWaitContext):
+        context.start(session)
+
+
+async def _on_queued_end(
+    session: ClientSession, context: SimpleNamespace, params: TraceConnectionQueuedEndParams
+) -> None:
+    if isinstance(context, _PoolWaitContext):
+        context.finish(session)
+
+
+async def _on_request_exception(
+    session: ClientSession, context: SimpleNamespace, params: TraceRequestExceptionParams
+) -> None:
+    if isinstance(context, _PoolWaitContext):
+        context.finish(session, failed=True)
+
+
+async def _on_request_redirect(
+    session: ClientSession, context: SimpleNamespace, params: TraceRequestRedirectParams
+) -> None:
+    if isinstance(context, _PoolWaitContext):
+        context.redirect(params)
+
+
+def build_pool_wait_trace_config() -> TraceConfig:
+    trace: Final = TraceConfig(trace_config_ctx_factory=_PoolWaitContext)
+    trace.on_request_start.append(_on_request_start)
+    trace.on_connection_queued_start.append(_on_queued_start)
+    trace.on_connection_queued_end.append(_on_queued_end)
+    trace.on_request_exception.append(_on_request_exception)
+    trace.on_request_redirect.append(_on_request_redirect)
+    return trace

litellm/llms/custom_httpx/http_handler.py

@@ -44,6 +44,7 @@ from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
 from litellm.litellm_core_utils.request_timeout_resolver import (
     get_configured_request_timeout,
 )
+from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import build_pool_wait_trace_config
 from litellm.llms.custom_httpx.http2_connection_spread import AsyncHTTP2ConnectionSpread
 from litellm.types.llms.custom_http import *
 
@@ -1456,6 +1457,7 @@ class AsyncHTTPHandler:
                 connector=TCPConnector(**transport_connector_kwargs),
                 cookie_jar=DummyCookieJar(),
                 trust_env=trust_env,
+                trace_configs=[build_pool_wait_trace_config()],
             )
 
         # Use shared session if provided and valid

litellm/llms/litellm_proxy/skills/skill_search.py

@@ -131,7 +131,7 @@ async def search_skills(
     user_api_key_dict: UserAPIKeyAuth,
     proxy_logging_obj: ProxyLogging,
 ) -> SkillSearchOutcome:
-    if embedding_model is None:
+    if embedding_model is None or not embedding_model.strip():
         return SkillSearchNotConfigured(
             reason="skill search needs litellm_settings.skill_search_embedding_model set to an embedding model from model_list"
         )

litellm/model_prices_and_context_window_backup.json

@@ -15555,18 +15555,20 @@
         "input_cost_per_token": 2.65e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8000,
-        "max_output_tokens": 2048,
+        "max_output_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 3.5e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md",
         "supports_pdf_input": true
     },
     "bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": {
         "input_cost_per_token": 3e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8000,
-        "max_output_tokens": 2048,
+        "max_output_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 6e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md",
         "supports_pdf_input": true
     },
     "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": {
@@ -48955,6 +48957,7 @@
     },
     "together_ai/Qwen/Qwen3.8-2.4T-A95B": {
         "cache_read_input_token_cost": 2.5e-07,
+        "deprecation_date": "2026-11-04",
         "input_cost_per_token": 2e-06,
         "litellm_provider": "together_ai",
         "max_input_tokens": 1010000,

litellm/proxy/_lazy_features.py

@@ -105,10 +105,10 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
     LazyFeature(
         name="a2a",
         module_path="litellm.proxy.agent_endpoints.a2a_endpoints",
-        # ``/v1/a2a/{agent_id}/message/send`` is caught via the suffix so the
+        # ``/v1/a2a/{agent_id}/message/send`` and ``/message/stream`` are caught via suffixes so the
         # ``/v1/a2a`` prefix doesn't subsume the discover prefix below.
         path_prefixes=("/a2a",),
-        path_suffixes=("/message/send",),
+        path_suffixes=("/message/send", "/message/stream"),
     ),
     LazyFeature(
         name="a2a_registration",

litellm/proxy/_lazy_openapi_snapshot.json

@@ -54,6 +54,57 @@
           ],
           "title": "ValidationError",
           "type": "object"
+        },
+        "_A2AJsonRpcResponse": {
+          "additionalProperties": true,
+          "properties": {
+            "error": {
+              "anyOf": [
+                {
+                  "additionalProperties": true,
+                  "type": "object"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Error"
+            },
+            "id": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "integer"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Id"
+            },
+            "jsonrpc": {
+              "const": "2.0",
+              "title": "Jsonrpc",
+              "type": "string"
+            },
+            "result": {
+              "anyOf": [
+                {},
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Result"
+            }
+          },
+          "required": [
+            "jsonrpc",
+            "id"
+          ],
+          "title": "_A2AJsonRpcResponse",
+          "type": "object"
         }
       }
     },
@@ -242,6 +293,119 @@
           ]
         }
       },
+      "/a2a/{agent_id}/message/stream": {
+        "post": {
+          "description": "Invoke an agent using the A2A protocol (JSON-RPC 2.0).\n\nSupported methods:\n- message/send: Send a message and get a response\n- message/stream: Send a message and stream the response",
+          "operationId": "invoke_agent_a2a_a2a__agent_id__message_stream_post",
+          "parameters": [
+            {
+              "in": "path",
+              "name": "agent_id",
+              "required": true,
+              "schema": {
+                "title": "Agent Id",
+                "type": "string"
+              }
+            }
+          ],
+          "requestBody": {
+            "content": {
+              "application/json": {
+                "example": {
+                  "id": "stream-1",
+                  "jsonrpc": "2.0",
+                  "method": "message/stream",
+                  "params": {
+                    "message": {
+                      "kind": "message",
+                      "messageId": "message-1",
+                      "parts": [
+                        {
+                          "kind": "text",
+                          "text": "Hello"
+                        }
+                      ],
+                      "role": "user"
+                    }
+                  }
+                },
+                "schema": {
+                  "additionalProperties": true,
+                  "properties": {
+                    "id": {
+                      "anyOf": [
+                        {
+                          "type": "string"
+                        },
+                        {
+                          "type": "integer"
+                        },
+                        {
+                          "type": "null"
+                        }
+                      ],
+                      "title": "Id"
+                    },
+                    "jsonrpc": {
+                      "const": "2.0",
+                      "title": "Jsonrpc",
+                      "type": "string"
+                    },
+                    "method": {
+                      "title": "Method",
+                      "type": "string"
+                    },
+                    "params": {
+                      "additionalProperties": true,
+                      "title": "Params",
+                      "type": "object"
+                    }
+                  },
+                  "required": [
+                    "jsonrpc",
+                    "method",
+                    "params"
+                  ],
+                  "title": "_A2AJsonRpcRequest",
+                  "type": "object"
+                }
+              }
+            },
+            "required": true
+          },
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/_A2AJsonRpcResponse"
+                  }
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Invoke Agent A2A",
+          "tags": [
+            "a2a"
+          ]
+        }
+      },
       "/v1/a2a/{agent_id}/message/send": {
         "post": {
           "description": "Invoke an agent using the A2A protocol (JSON-RPC 2.0).\n\nSupported methods:\n- message/send: Send a message and get a response\n- message/stream: Send a message and stream the response",
@@ -287,6 +451,119 @@
             "a2a"
           ]
         }
+      },
+      "/v1/a2a/{agent_id}/message/stream": {
+        "post": {
+          "description": "Invoke an agent using the A2A protocol (JSON-RPC 2.0).\n\nSupported methods:\n- message/send: Send a message and get a response\n- message/stream: Send a message and stream the response",
+          "operationId": "invoke_agent_a2a_v1_a2a__agent_id__message_stream_post",
+          "parameters": [
+            {
+              "in": "path",
+              "name": "agent_id",
+              "required": true,
+              "schema": {
+                "title": "Agent Id",
+                "type": "string"
+              }
+            }
+          ],
+          "requestBody": {
+            "content": {
+              "application/json": {
+                "example": {
+                  "id": "stream-1",
+                  "jsonrpc": "2.0",
+                  "method": "message/stream",
+                  "params": {
+                    "message": {
+                      "kind": "message",
+                      "messageId": "message-1",
+                      "parts": [
+                        {
+                          "kind": "text",
+                          "text": "Hello"
+                        }
+                      ],
+                      "role": "user"
+                    }
+                  }
+                },
+                "schema": {
+                  "additionalProperties": true,
+                  "properties": {
+                    "id": {
+                      "anyOf": [
+                        {
+                          "type": "string"
+                        },
+                        {
+                          "type": "integer"
+                        },
+                        {
+                          "type": "null"
+                        }
+                      ],
+                      "title": "Id"
+                    },
+                    "jsonrpc": {
+                      "const": "2.0",
+                      "title": "Jsonrpc",
+                      "type": "string"
+                    },
+                    "method": {
+                      "title": "Method",
+                      "type": "string"
+                    },
+                    "params": {
+                      "additionalProperties": true,
+                      "title": "Params",
+                      "type": "object"
+                    }
+                  },
+                  "required": [
+                    "jsonrpc",
+                    "method",
+                    "params"
+                  ],
+                  "title": "_A2AJsonRpcRequest",
+                  "type": "object"
+                }
+              }
+            },
+            "required": true
+          },
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/_A2AJsonRpcResponse"
+                  }
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Invoke Agent A2A",
+          "tags": [
+            "a2a"
+          ]
+        }
       }
     }
   },
@@ -15326,6 +15603,10 @@
               ],
               "title": "Metadata"
             },
+            "response_format": {
+              "$ref": "#/components/schemas/GuardrailResponseFormat",
+              "default": "standard"
+            },
             "text": {
               "title": "Text",
               "type": "string"
@@ -16203,6 +16484,14 @@
           "title": "BedrockChecksSensitiveInformationModel",
           "type": "object"
         },
+        "BedrockGuardrailOutputScope": {
+          "enum": [
+            "INTERVENTIONS",
+            "FULL"
+          ],
+          "title": "BedrockGuardrailOutputScope",
+          "type": "string"
+        },
         "BlockedWord": {
           "description": "Represents a blocked word with its action and optional description",
           "properties": {
@@ -16746,6 +17035,14 @@
           "title": "GuardrailInfoResponse",
           "type": "object"
         },
+        "GuardrailResponseFormat": {
+          "enum": [
+            "standard",
+            "native"
+          ],
+          "title": "GuardrailResponseFormat",
+          "type": "string"
+        },
         "GuardrailSubmissionItem": {
           "properties": {
             "created_at": {
@@ -16998,6 +17295,7 @@
           "title": "HTTPValidationError",
           "type": "object"
         },
+        "JsonValue": {},
         "LakeraCategoryThresholds": {
           "additionalProperties": true,
           "properties": {
@@ -17924,6 +18222,18 @@
               "description": "Whether to fail the request if the guardrail encounters an error. Implemented by guardrail='model_armor', 'generic_guardrail_api' and 'crowdstrike_aidr'. True (default) raises the error. False logs a critical error and lets the request proceed, so only a valid guardrail response can block or modify it.",
               "title": "Fail On Error"
             },
+            "file_checkpoint_id": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "File-checkpoint ID for the Ovalix Tracker service (falls back to pre/post).",
+              "title": "File Checkpoint Id"
+            },
             "file_guardrail_timeout": {
               "anyOf": [
                 {
@@ -18426,6 +18736,17 @@
               ],
               "description": "Optional parameters for the guardrail"
             },
+            "outputScope": {
+              "anyOf": [
+                {
+                  "$ref": "#/components/schemas/BedrockGuardrailOutputScope"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "ApplyGuardrail response scope. FULL includes detected and non-detected findings. Omit to retain Bedrock's default of returning detected findings only."
+            },
             "output_parse_pii": {
               "anyOf": [
                 {
@@ -20308,7 +20629,24 @@
               "content": {
                 "application/json": {
                   "schema": {
-                    "$ref": "#/components/schemas/ApplyGuardrailResponse"
+                    "anyOf": [
+                      {
+                        "$ref": "#/components/schemas/ApplyGuardrailResponse"
+                      },
+                      {
+                        "additionalProperties": {
+                          "$ref": "#/components/schemas/JsonValue"
+                        },
+                        "type": "object"
+                      },
+                      {
+                        "items": {
+                          "$ref": "#/components/schemas/JsonValue"
+                        },
+                        "type": "array"
+                      }
+                    ],
+                    "title": "Response Apply Guardrail Apply Guardrail Post"
                   }
                 }
               },
@@ -20400,7 +20738,24 @@
               "content": {
                 "application/json": {
                   "schema": {
-                    "$ref": "#/components/schemas/ApplyGuardrailResponse"
+                    "anyOf": [
+                      {
+                        "$ref": "#/components/schemas/ApplyGuardrailResponse"
+                      },
+                      {
+                        "additionalProperties": {
+                          "$ref": "#/components/schemas/JsonValue"
+                        },
+                        "type": "object"
+                      },
+                      {
+                        "items": {
+                          "$ref": "#/components/schemas/JsonValue"
+                        },
+                        "type": "array"
+                      }
+                    ],
+                    "title": "Response Apply Guardrail Guardrails Apply Guardrail Post"
                   }
                 }
               },

litellm/proxy/_types.py

@@ -501,6 +501,7 @@ class LiteLLMRoutes(enum.Enum):
         "/a2a/{agent_id}/message/send",
         "/v1/a2a/{agent_id}/message/send",
         "/a2a/{agent_id}/message/stream",
+        "/v1/a2a/{agent_id}/message/stream",
         "/a2a/{agent_id}/.well-known/agent-card.json",
     )
 

litellm/proxy/a2a/agent_card.py

@@ -12,6 +12,8 @@ from collections.abc import Mapping
 from copy import deepcopy
 from typing import Final, Literal
 
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
+
 SupportedA2AVersion = Literal["0.3", "1.0"]
 
 # Protocol versions LiteLLM can serve to A2A clients. The admin pins one per agent;
@@ -21,7 +23,6 @@ SUPPORTED_A2A_PROTOCOL_VERSIONS: Final[tuple[SupportedA2AVersion, ...]] = ("0.3"
 # Default served version when the agent card does not pin one.
 LITELLM_A2A_PROTOCOL_VERSION: Final = "1.0"
 
-
 _PROTOCOL_VERSION_PATTERN: Final = re.compile(
     r"^(\d+\.\d+)(?:\.\d+(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?)?$"
 )
@@ -50,6 +51,24 @@ def resolve_served_protocol_version(card: Mapping[str, object] | None) -> str:
     return normalized if normalized is not None else LITELLM_A2A_PROTOCOL_VERSION
 
 
+def point_agent_card_at_proxy(
+    agent_card: Mapping[str, object],
+    proxy_url: str,
+) -> dict[str, object]:  # mutable-ok: the public helper contract requires a dictionary
+    interfaces: Final = agent_card.get("supportedInterfaces")
+    interface_items: Final = (
+        tuple(iter_items(agent_card.get("supportedInterfaces"))) if isinstance(interfaces, list) else ()
+    )
+    first: Final[object] = interface_items[0] if interface_items else None
+    first_mapping: Final = as_str_mapping(first)
+    pointed_interfaces: Final = (
+        {"supportedInterfaces": [{**first_mapping, "url": proxy_url}, *interface_items[1:]]}
+        if isinstance(first, dict) and first_mapping is not None
+        else {}
+    )
+    return deepcopy({**agent_card, "url": proxy_url, **pointed_interfaces})
+
+
 # Security scheme exposed by the LiteLLM-fronted agent card. Always replaces
 # whatever upstream advertised — the client must authenticate to the proxy,
 # not the upstream agent.

litellm/proxy/agent_endpoints/a2a_endpoints.py

@@ -12,9 +12,8 @@ The A2A SDK can point to LiteLLM's URL and invoke agents registered with LiteLLM
 
 import json
 from collections.abc import AsyncGenerator, Callable, Coroutine, Mapping
-from copy import deepcopy
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Protocol
+from typing import TYPE_CHECKING, Any, Final, Literal, Protocol
 from urllib.parse import urlparse
 
 from fastapi import APIRouter, Depends, HTTPException, Request, Response
@@ -29,6 +28,7 @@ from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
 from litellm.llms.a2a.common_utils import resolve_a2a_hop_auth_header
 from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
 from litellm.proxy._types import ProxyException, UserAPIKeyAuth
+from litellm.proxy.a2a.agent_card import point_agent_card_at_proxy
 from litellm.proxy.a2a.version_convert import (
     A2AVersion,
     normalize_agent_card,
@@ -43,6 +43,7 @@ from litellm.proxy.agent_endpoints.databricks_oauth import (
 )
 from litellm.proxy.agent_endpoints.utils import merge_agent_headers
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body
 from litellm.proxy.common_utils.openai_error_payload import status_typed_proxy_exception
 from litellm.proxy.common_utils.sse_keepalive import (
     SSE_COMMENT_PING,
@@ -59,6 +60,42 @@ if TYPE_CHECKING:
 
 router: Final = APIRouter()
 
+
+class _A2AJsonRpcRequest(BaseModel):
+    model_config = ConfigDict(extra="allow", frozen=True)
+
+    jsonrpc: Literal["2.0"]
+    id: str | int | None = None
+    method: str
+    params: Mapping[str, object]
+
+
+class _A2AJsonRpcResponse(BaseModel):
+    model_config = ConfigDict(extra="allow", frozen=True)
+
+    jsonrpc: Literal["2.0"]
+    id: str | int | None
+    result: object | None = None
+    error: Mapping[str, object] | None = None
+
+
+_A2A_MESSAGE_STREAM_REQUEST_BODY: Final = inline_request_body(
+    _A2AJsonRpcRequest,
+    {
+        "jsonrpc": "2.0",
+        "id": "stream-1",
+        "method": "message/stream",
+        "params": {
+            "message": {
+                "messageId": "message-1",
+                "role": "user",
+                "parts": [{"kind": "text", "text": "Hello"}],
+                "kind": "message",
+            }
+        },
+    },
+)
+
 # Mirrors the native seam's own headers: a reverse proxy that batches the whole
 # stream would swallow the keepalives this route sends to defeat idle timeouts.
 _SSE_KEEPALIVE_HEADERS: Final[Mapping[str, str]] = MappingProxyType(
@@ -206,6 +243,35 @@ def _forwarding_headers(
     return merged or None
 
 
+async def _log_agent_rejection(
+    *,
+    proxy_logging_obj: ProxyLogging,
+    user_api_key_dict: UserAPIKeyAuth,
+    request_id: object,
+    method: str | None,
+    rejection: HTTPException,
+    target_agent_id: str,
+    model: str,
+) -> None:
+    request_data: Final[dict[str, object]] = {  # mutable-ok: the failure hook stamps its logging state onto it
+        "jsonrpc": "2.0",
+        "id": request_id,
+        "method": method,
+        "model": model,
+        "agent_id": target_agent_id,
+        "metadata": {"agent_id": target_agent_id},
+    }
+    try:
+        await proxy_logging_obj.post_call_failure_hook(  # pyright: ignore[reportUnknownMemberType]  # bare dict in sig
+            request_data=request_data,
+            original_exception=rejection,
+            user_api_key_dict=user_api_key_dict,
+            route="/a2a/{agent_id}",
+        )
+    except Exception:  # noqa: BLE001  # a logging failure must not change the rejection the caller gets
+        verbose_proxy_logger.exception("Failed to log rejected A2A request")
+
+
 def _jsonrpc_error(
     request_id: object,
     code: int,
@@ -684,16 +750,12 @@ async def get_agent_card(
             )
 
         proxy_url: Final = get_custom_url(str(request.base_url), route=f"a2a/{agent_id}")
-        agent_card = deepcopy(agent.agent_card_params)
-        agent_card["url"] = proxy_url
-        interfaces: Final = agent_card.get("supportedInterfaces")
-        if isinstance(interfaces, list) and interfaces:
-            interfaces[0]["url"] = proxy_url
+        agent_card: Final = point_agent_card_at_proxy(agent.agent_card_params, proxy_url)
         served_version: Final = _served_version(agent, request)
-        agent_card = normalize_agent_card(agent_card, served_version)
+        normalized_agent_card: Final = normalize_agent_card(agent_card, served_version)
 
         verbose_proxy_logger.debug("Returning agent card for '%s' with proxy URL: %s", agent_id, proxy_url)
-        return JSONResponse(content=agent_card)
+        return JSONResponse(content=normalized_agent_card)
 
     except HTTPException:
         raise
@@ -717,6 +779,20 @@ async def get_agent_card(
     tags=["[beta] A2A Agents"],
     dependencies=[Depends(user_api_key_auth)],
 )
+@router.post(
+    "/a2a/{agent_id}/message/stream",
+    tags=["[beta] A2A Agents"],
+    dependencies=[Depends(user_api_key_auth)],
+    openapi_extra={"requestBody": _A2A_MESSAGE_STREAM_REQUEST_BODY},
+    responses={200: {"model": _A2AJsonRpcResponse}},
+)
+@router.post(
+    "/v1/a2a/{agent_id}/message/stream",
+    tags=["[beta] A2A Agents"],
+    dependencies=[Depends(user_api_key_auth)],
+    openapi_extra={"requestBody": _A2A_MESSAGE_STREAM_REQUEST_BODY},
+    responses={200: {"model": _A2AJsonRpcResponse}},
+)
 async def invoke_agent_a2a(
     agent_id: str,
     request: Request,
@@ -778,6 +854,15 @@ async def invoke_agent_a2a(
         # Find the agent
         agent: Final = await _get_agent(agent_id)
         if agent is None:
+            await _log_agent_rejection(
+                proxy_logging_obj=proxy_logging_obj,
+                user_api_key_dict=user_api_key_dict,
+                request_id=request_id,
+                method=method,
+                rejection=HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found"),
+                target_agent_id=agent_id,
+                model=f"a2a_agent/{agent_id}",
+            )
             return _jsonrpc_error(request_id, -32000, f"Agent '{agent_id}' not found", 404)
 
         served_version: Final = _served_version(agent, request, original_method)
@@ -787,10 +872,20 @@ async def invoke_agent_a2a(
             user_api_key_auth=user_api_key_dict,
         )
         if not is_allowed:
-            raise HTTPException(
+            denial: Final = HTTPException(
                 status_code=403,
                 detail=f"Agent '{agent_id}' is not allowed for your key/team. Contact proxy admin for access.",
             )
+            await _log_agent_rejection(
+                proxy_logging_obj=proxy_logging_obj,
+                user_api_key_dict=user_api_key_dict,
+                request_id=request_id,
+                method=method,
+                rejection=denial,
+                target_agent_id=agent.agent_id,
+                model=f"a2a_agent/{(as_str_mapping(agent.agent_card_params or {}) or {}).get('name', agent_id)}",
+            )
+            raise denial
 
         user_api_key_dict.invoked_agent_id = agent.agent_id
         strict_policy: Final = agent.identity_managed or bool(user_api_key_dict.requires_fresh_policy)

litellm/proxy/agent_endpoints/agent_search.py

@@ -140,7 +140,7 @@ async def search_agents(
     user_api_key_dict: UserAPIKeyAuth,
     proxy_logging_obj: ProxyLogging,
 ) -> AgentSearchOutcome:
-    if embedding_model is None:
+    if embedding_model is None or not embedding_model.strip():
         return AgentSearchNotConfigured(
             reason="agent search needs litellm_settings.agent_search_embedding_model set to an embedding model from model_list"
         )

litellm/proxy/agent_endpoints/endpoints.py

@@ -22,6 +22,7 @@ from typing_extensions import ReadOnly, Required, assert_never
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import AGENT_HEALTH_CHECK_GATHER_TIMEOUT_SECONDS, AGENT_HEALTH_CHECK_TIMEOUT_SECONDS
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
 from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 from litellm.proxy._types import (
     CommonProxyErrors,
@@ -33,6 +34,7 @@ from litellm.proxy.a2a.agent_card import (
     SUPPORTED_A2A_PROTOCOL_VERSIONS,
     merge_agent_card,
     normalize_protocol_version,
+    point_agent_card_at_proxy,
 )
 from litellm.proxy.agent_endpoints.agent_registry import (
     AgentIdWhere,
@@ -180,10 +182,36 @@ def _redact_agent_litellm_params_dict(
     return dict(parse_agent_litellm_params(redact_sensitive_agent_litellm_params(litellm_params)))
 
 
+def _non_admin_agent_card(
+    card: Mapping[str, object], proxy_url: str
+) -> dict[str, object]:  # mutable-ok: AgentResponse.agent_card_params requires a mutable dictionary
+    pointed_card: Final = point_agent_card_at_proxy(card, proxy_url)
+    interfaces: Final = pointed_card.get("supportedInterfaces")
+    interface_items: Final = (
+        tuple(iter_items(pointed_card.get("supportedInterfaces"))) if isinstance(interfaces, list) else ()
+    )
+    first_interface: Final[object] = interface_items[0] if interface_items else None
+    first_mapping: Final = as_str_mapping(first_interface)
+    proxy_interface: Final = (
+        {"supportedInterfaces": [{**first_mapping, "url": proxy_url}]}
+        if isinstance(first_interface, dict) and first_mapping is not None and first_mapping.get("url") == proxy_url
+        else {}
+    )
+    return {
+        **{
+            key: value
+            for key, value in pointed_card.items()
+            if key not in ("additionalInterfaces", "supportedInterfaces")
+        },
+        **proxy_interface,
+    }
+
+
 def _redact_sensitive_agent_fields(
     agents: Sequence[AgentResponse],
     *,
     is_admin: bool,
+    request_base_url: str,
 ) -> list[AgentResponse]:
     """
     Return copies of the given agents with credential-bearing litellm_params
@@ -207,6 +235,11 @@ def _redact_sensitive_agent_fields(
             copy.keys = None
             copy.kill_switch = None
             copy.identity = None
+            if copy.agent_card_params:
+                copy.agent_card_params = _non_admin_agent_card(
+                    copy.agent_card_params,
+                    get_custom_url(request_base_url, route=f"a2a/{copy.agent_id}"),
+                )
         if copy.litellm_params:
             copy.litellm_params = _redact_agent_litellm_params_dict(copy.litellm_params)
         copy.kill_switch = redact_kill_switch(copy.kill_switch)
@@ -461,14 +494,6 @@ async def get_agents(
                 global_agent_registry.ids_for_agent(agent.agent_id).isdisjoint(litellm.public_agent_groups)
             )
 
-        # litellm_params secrets are always redacted; keys/headers stay
-        # admin-only.
-        is_admin: Final = (
-            user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
-            or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value
-        )
-        returned_agents = _redact_sensitive_agent_fields(returned_agents, is_admin=is_admin)
-
         if health_check:
             agents_with_url: Final = [agent for agent in returned_agents if (agent.agent_card_params or {}).get("url")]
             agents_without_url = [agent for agent in returned_agents if not (agent.agent_card_params or {}).get("url")]
@@ -493,6 +518,18 @@ async def get_agents(
             healthy_ids: Final = {result["agent_id"] for result in health_results if result["healthy"]}
             returned_agents = [agent for agent in agents_with_url if agent.agent_id in healthy_ids] + agents_without_url
 
+        # litellm_params secrets are always redacted; keys/headers stay
+        # admin-only.
+        is_admin: Final = (
+            user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
+            or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value
+        )
+        returned_agents = _redact_sensitive_agent_fields(
+            returned_agents,
+            is_admin=is_admin,
+            request_base_url=str(request.base_url),
+        )
+
         if query is None:
             return returned_agents
         return await _rank_agents_by_query(query, returned_agents, top_k, user_api_key_dict)
@@ -691,7 +728,11 @@ async def create_agent(
 
         # The caller is a proxy admin (enforced above); litellm_params
         # secrets are still never echoed back in the response.
-        return _redact_sensitive_agent_fields((result,), is_admin=True)[0]
+        return _redact_sensitive_agent_fields(
+            (result,),
+            is_admin=True,
+            request_base_url=str(http_request.base_url),
+        )[0]
 
     except HTTPException:
         raise
@@ -708,6 +749,7 @@ async def create_agent(
 )
 async def get_agent_by_id(
     agent_id: str,
+    request: Request,
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
 ):
     """
@@ -770,7 +812,11 @@ async def get_agent_by_id(
             user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
             or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value
         )
-        agent = _redact_sensitive_agent_fields((agent,), is_admin=is_admin)[0]
+        agent = _redact_sensitive_agent_fields(
+            (agent,),
+            is_admin=is_admin,
+            request_base_url=str(request.base_url),
+        )[0]
 
         return agent
     except HTTPException:
@@ -879,7 +925,11 @@ async def update_agent(
             "Successfully updated agent '%s' (ID: %s) in memory", existing_agent.get("agent_name"), agent_id
         )
 
-        return _redact_sensitive_agent_fields((result,), is_admin=True)[0]
+        return _redact_sensitive_agent_fields(
+            (result,),
+            is_admin=True,
+            request_base_url=str(http_request.base_url),
+        )[0]
     except HTTPException:
         raise
     except Exception as e:
@@ -987,7 +1037,11 @@ async def patch_agent(
             "Successfully updated agent '%s' (ID: %s) in memory", existing_agent.get("agent_name"), agent_id
         )
 
-        return _redact_sensitive_agent_fields((result,), is_admin=True)[0]
+        return _redact_sensitive_agent_fields(
+            (result,),
+            is_admin=True,
+            request_base_url=str(http_request.base_url),
+        )[0]
     except HTTPException:
         raise
     except Exception as e:

litellm/proxy/dev_reload.py

@@ -0,0 +1,172 @@
+import importlib
+import logging
+import multiprocessing
+import os
+import signal
+import sys
+import threading
+from collections.abc import Callable, Mapping, Sequence
+from functools import partial
+from inspect import signature
+from multiprocessing.process import BaseProcess
+from socket import socket
+from typing import Final, Literal
+
+from pydantic import ConfigDict, ImportString, JsonValue, TypeAdapter, with_config
+from typing_extensions import ReadOnly, Required, TypedDict
+from uvicorn import Config, Server
+from uvicorn import config as uvicorn_config
+from uvicorn.supervisors import ChangeReload, Multiprocess
+
+
+@with_config(ConfigDict(extra="allow"))
+class ServerOptions(TypedDict, total=False):
+    host: ReadOnly[str]
+    port: ReadOnly[int]
+    server_header: ReadOnly[bool]
+    timeout_graceful_shutdown: ReadOnly[float]
+    log_config: ReadOnly[  # mutable-ok: uvicorn.Config requires a concrete logging dict
+        str | dict[str, JsonValue] | None
+    ]
+    log_level: ReadOnly[str | int]
+    timeout_keep_alive: ReadOnly[int]
+    timeout_worker_healthcheck: ReadOnly[int]
+    limit_max_requests: ReadOnly[int]
+    limit_max_requests_jitter: ReadOnly[int]
+    limit_concurrency: ReadOnly[int]
+    ssl_keyfile: ReadOnly[str]
+    ssl_certfile: ReadOnly[str]
+    loop: ReadOnly[Literal["none", "auto", "asyncio", "uvloop"]]
+
+
+@with_config(ConfigDict(extra="allow"))
+class ReloadOptions(ServerOptions):
+    app: Required[ReadOnly[str]]
+    reload: ReadOnly[bool]
+    reload_dirs: ReadOnly[tuple[str, ...]]
+    reload_includes: ReadOnly[tuple[str, ...]]
+
+
+RELOAD_OPTIONS: Final = TypeAdapter(ReloadOptions)
+SERVER_OPTIONS: Final = TypeAdapter(ServerOptions)
+ASGI_APP: Final = TypeAdapter(ImportString[Callable[..., object]])
+LOGGER: Final = logging.getLogger("uvicorn.error")
+RELOAD_WORKER_HEALTHCHECK_SECONDS: Final = 30
+RELOADER_ONLY_KEYS: Final = frozenset(ReloadOptions.__annotations__) - frozenset(ServerOptions.__annotations__)
+
+
+def multi_worker_reload_supported(platform: str = sys.platform) -> bool:
+    return platform != "win32" and hasattr(uvicorn_config, "STARTUP_FAILURE")
+
+
+def server_options(options: ReloadOptions) -> ServerOptions:
+    return SERVER_OPTIONS.validate_python(
+        {key: value for key, value in options.items() if key not in RELOADER_ONLY_KEYS}
+    )
+
+
+def load_app_or_exit(import_path: str, factory: bool = False) -> Callable[..., object]:
+    module_name: Final = import_path.partition(":")[0]
+    try:
+        importlib.import_module(module_name)
+        app: Final = ASGI_APP.validate_python(import_path)
+        return ASGI_APP.validate_python(app()) if factory else app
+    except Exception:
+        LOGGER.exception("Error loading ASGI app %s", import_path)
+        sys.exit(uvicorn_config.STARTUP_FAILURE)
+
+
+def stop_worker_when_pool_exits(pool_parent: BaseProcess) -> None:
+    pool_parent.join()
+    LOGGER.info("Worker pool process [%s] exited, stopping worker [%s].", pool_parent.pid, os.getpid())
+    os.kill(os.getpid(), signal.SIGTERM)
+
+
+def load_worker_app(import_path: str, worker_app_factory: bool) -> Callable[..., object]:
+    pool_parent: Final = multiprocessing.parent_process()
+    if pool_parent is not None:
+        threading.Thread(target=stop_worker_when_pool_exits, args=(pool_parent,), daemon=True).start()
+    return load_app_or_exit(import_path, worker_app_factory)
+
+
+def pool_config(options: ReloadOptions, workers: int, worker_overrides: Mapping[str, object] | None = None) -> Config:
+    overrides: Final = worker_overrides or {}
+    worker_app: Final = TypeAdapter(str).validate_python(overrides.get("app", options["app"]))
+    worker_app_factory: Final = TypeAdapter(bool).validate_python(overrides.get("factory", False))
+    return Config(
+        partial(load_worker_app, worker_app, worker_app_factory),
+        factory=True,
+        workers=workers,
+        **SERVER_OPTIONS.validate_python(
+            {
+                "timeout_worker_healthcheck": RELOAD_WORKER_HEALTHCHECK_SECONDS,
+                **server_options(options),
+                **{key: value for key, value in overrides.items() if key not in ("app", "factory")},
+            }
+        ),  # pyright: ignore[reportArgumentType]  # uvicorn types the float graceful shutdown timeout as int
+    )
+
+
+def stop_pool_when_reloader_exits(reloader: BaseProcess, pool_should_exit: threading.Event) -> None:
+    reloader.join()
+    LOGGER.info("Reloader process [%s] exited, stopping the worker pool.", reloader.pid)
+    pool_should_exit.set()
+
+
+class ReloadWorkerPool(Multiprocess):
+    quit_received: bool = False
+
+    def handle_quit(self) -> None:
+        LOGGER.info("Received SIGQUIT, exiting.")
+        self.quit_received = True
+        self.should_exit.set()
+
+    def terminate_all(self) -> None:
+        quit_signal: Final = getattr(signal, "SIGQUIT", None)
+        if quit_signal is not None and quit_signal in self.signal_queue:
+            self.handle_quit()
+        if not self.quit_received:
+            super().terminate_all()
+            return
+        for process in self.processes:
+            process.kill()
+
+
+def _run_worker_pool(config: Config, sockets: Sequence[socket] | None) -> None:
+    shared_sockets: Final = list(sockets or ())
+    pool: Final = (
+        ReloadWorkerPool(
+            config=config,
+            target=Server(config=config).run,  # pyright: ignore[reportCallIssue]  # Older uvicorn versions require target
+            sockets=shared_sockets,
+        )
+        if "target" in signature(Multiprocess).parameters
+        else ReloadWorkerPool(config=config, sockets=shared_sockets)
+    )
+    reloader: Final = multiprocessing.parent_process()
+    if reloader is not None:
+        threading.Thread(target=stop_pool_when_reloader_exits, args=(reloader, pool.should_exit), daemon=True).start()
+    pool.run()
+
+
+def run_multi_worker_reload(
+    options: ReloadOptions, workers: int, worker_overrides: Mapping[str, object] | None = None
+) -> None:
+    worker_config: Final = pool_config(options, workers, worker_overrides)
+    reload_config: Final = Config(
+        options["app"],
+        workers=1,
+        reload=options["reload"],
+        reload_dirs=list(options["reload_dirs"]),
+        reload_includes=list(options["reload_includes"]),
+        **server_options(options),  # pyright: ignore[reportArgumentType]  # uvicorn types the float graceful shutdown timeout as int
+    )
+    shared_socket: Final = worker_config.bind_socket()
+    try:
+        ChangeReload(
+            config=reload_config,
+            target=partial(_run_worker_pool, worker_config),
+            sockets=[shared_socket],
+        ).run()
+    finally:
+        shared_socket.close()

litellm/proxy/guardrails/guardrail_endpoints.py

@@ -9,7 +9,7 @@ import os
 from collections.abc import Awaitable, Callable, Mapping, Sequence
 from datetime import datetime, timezone
 from types import MappingProxyType, UnionType
-from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, Union, cast, get_args, get_origin
+from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, Union, cast, get_args, get_origin
 from urllib.parse import urlparse
 
 from fastapi import APIRouter, Depends, HTTPException, Request
@@ -43,6 +43,11 @@ from litellm.proxy.guardrails.guardrail_registry import (
     encrypt_guardrail_litellm_params,
     parse_tolerant_litellm_params,
 )
+from litellm.proxy.guardrails.response_handlers import (
+    GUARDRAIL_RESPONSE_HANDLERS,
+    GuardrailEndpointResponse,
+    StandardGuardrailResponseHandler,
+)
 from litellm.proxy.guardrails.usage_endpoints import router as guardrails_usage_router
 from litellm.proxy.management_endpoints.common_utils import (  # noqa: F401  # legacy module exports
     _user_has_admin_view,  # pyright: ignore[reportPrivateUsage,reportUnusedImport]  # backwards-compatible package export
@@ -53,7 +58,6 @@ from litellm.repositories.table_repositories import GuardrailsRepository
 from litellm.types.guardrails import (
     PII_ENTITY_CATEGORIES_MAP,
     ApplyGuardrailRequest,
-    ApplyGuardrailResponse,
     BaseLitellmParams,
     BedrockGuardrailConfigModel,
     BedrockGuardrailStreamingParams,
@@ -2350,13 +2354,7 @@ async def _run_test_guardrail(execute_guardrail: Callable[[], object], timeout:
     return await await_with_timeout(raw_result, remaining, label="test")
 
 
-def _resolve_guardrail_input_type(active_guardrail: CustomGuardrail, input_type: str) -> Literal["request", "response"]:
-    """Return the effective input_type, auto-upgrading to 'response' for post_call guardrails."""
-    if input_type == "request":
-        hook: Final = getattr(active_guardrail, "event_hook", None)
-        if hook == GuardrailEventHooks.post_call or hook == "post_call":
-            return "response"
-    return "response" if input_type == "response" else "request"
+_RESPONSE: Final[TypeAdapter[GuardrailEndpointResponse]] = TypeAdapter(GuardrailEndpointResponse)
 
 
 class _GuardrailLoggingObj(Protocol):
@@ -2384,35 +2382,43 @@ def _patch_logging_obj_for_guardrail(litellm_logging_obj: _GuardrailLoggingObj,
     )
 
 
-async def _emit_guardrail_success_logs(
+async def _guardrail_response_after_success_hook(
+    response_handler: StandardGuardrailResponseHandler,
     proxy_logging_obj: _GuardrailProxyLogging,
-    litellm_logging_obj: _GuardrailLoggingObj | None,
-    data: dict,
+    data: Mapping[str, object],
     user_api_key_dict: UserAPIKeyAuth,
-    response: ApplyGuardrailResponse,
-    start_time: datetime,
-) -> ApplyGuardrailResponse:
-    """Fire proxy and LiteLLM success hooks after a successful guardrail run.
-
-    Each hook is wrapped defensively so a callback failure never prevents the
-    caller from receiving the guardrail response.  Returns the (possibly
-    hook-modified) response.
-    """
+    response: GuardrailEndpointResponse,
+) -> GuardrailEndpointResponse:
     try:
         modified: Final = await proxy_logging_obj.post_call_success_hook(
             data=data,
             user_api_key_dict=user_api_key_dict,
             response=response,
         )
-        if isinstance(modified, ApplyGuardrailResponse):
-            response = modified
+        return response_handler.accept_hook_result(response, modified)
     except Exception:
         verbose_proxy_logger.exception("apply_guardrail: post_call_success_hook failed")
+    return response
+
+
+async def _emit_guardrail_success_logs(
+    response_handler: StandardGuardrailResponseHandler,
+    proxy_logging_obj: _GuardrailProxyLogging,
+    litellm_logging_obj: _GuardrailLoggingObj | None,
+    data: Mapping[str, object],
+    user_api_key_dict: UserAPIKeyAuth,
+    response: GuardrailEndpointResponse,
+    start_time: datetime,
+) -> GuardrailEndpointResponse:
+    """Apply the success hook and log the response returned to the caller."""
+    updated_response: Final = await _guardrail_response_after_success_hook(
+        response_handler, proxy_logging_obj, data, user_api_key_dict, response
+    )
 
     # Build the logging payload after post_call_success_hook so that logged
     # data matches what the caller actually receives if the hook modified
     # the response.
-    response_for_logging: Final = {"response": response.model_dump(exclude_none=True)}
+    response_for_logging: Final = {"response": _RESPONSE.dump_python(updated_response, mode="json")}
 
     if litellm_logging_obj is not None:
         end_time: Final = datetime.now(timezone.utc)
@@ -2426,11 +2432,11 @@ async def _emit_guardrail_success_logs(
         except Exception:
             verbose_proxy_logger.exception("apply_guardrail: async_success_handler failed")
 
-    return response
+    return updated_response
 
 
-@router.post("/guardrails/apply_guardrail", response_model=ApplyGuardrailResponse)
-@router.post("/apply_guardrail", response_model=ApplyGuardrailResponse)
+@router.post("/guardrails/apply_guardrail", response_model=GuardrailEndpointResponse)
+@router.post("/apply_guardrail", response_model=GuardrailEndpointResponse)
 async def apply_guardrail(
     fastapi_request: Request,
     request: ApplyGuardrailRequest,
@@ -2455,7 +2461,7 @@ async def apply_guardrail(
     )
     from litellm.proxy.utils import handle_exception_on_proxy
 
-    data: dict = {
+    data: dict[str, object] = {
         "guardrail_name": request.guardrail_name,
         "input": [request.text],
         "messages": request.messages or [],
@@ -2495,18 +2501,15 @@ async def apply_guardrail(
         if litellm_logging_obj is not None:
             _patch_logging_obj_for_guardrail(litellm_logging_obj, request)
 
-        request_data: Final[dict] = {
-            **({"messages": request.messages} if request.messages is not None else {}),
-            **({"metadata": request.metadata} if request.metadata is not None else {}),
-        }
-        _input_type: Final = _resolve_guardrail_input_type(active_guardrail, request.input_type)
+        response_handler: Final = GUARDRAIL_RESPONSE_HANDLERS[request.response_format]
+        request_data: Final = dict(response_handler.prepare_request_data(request))
+        input_type: Final = response_handler.resolve_input_type(request, active_guardrail)
         guardrailed_inputs: Final = await active_guardrail.apply_guardrail(
             inputs={"texts": [request.text]},
             request_data=request_data,
-            input_type=_input_type,
+            input_type=input_type,
         )
-        response_text: Final = guardrailed_inputs.get("texts", [])
-        response = ApplyGuardrailResponse(response_text=response_text[0] if response_text else request.text)
+        response: Final = response_handler.build_response(request, guardrailed_inputs, request_data)
     except Exception as e:
         if litellm_logging_obj is not None and not isinstance(e, HTTPException):
             try:
@@ -2537,7 +2540,8 @@ async def apply_guardrail(
         raise handle_exception_on_proxy(status_typed_proxy_exception(e) if isinstance(e, HTTPException) else e)
 
     # Success logging outside except so a hook error never triggers failure handlers.
-    response = await _emit_guardrail_success_logs(
+    logged_response: Final = await _emit_guardrail_success_logs(
+        response_handler=response_handler,
         proxy_logging_obj=proxy_logging_obj,
         litellm_logging_obj=litellm_logging_obj,
         data=data,
@@ -2545,7 +2549,7 @@ async def apply_guardrail(
         response=response,
         start_time=start_time,
     )
-    return response
+    return logged_response
 
 
 # Usage (dashboard) endpoints: overview, detail, logs

litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py

@@ -65,6 +65,7 @@ from litellm.proxy.guardrails.anthropic_sse import (
 )
 from litellm.types.guardrails import (
     BedrockChecksConfigModel,
+    BedrockGuardrailOutputScope,
     BedrockGuardrailStreamingParams,
     GuardrailEventHooks,
     LitellmParams,
@@ -289,6 +290,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         streaming_buffer_release_on_scan: bool | None = None,
         streaming_scan_in_background: bool | None = None,
         async_handler: AsyncHTTPHandler | None = None,
+        outputScope: BedrockGuardrailOutputScope | None = None,
         **kwargs,
     ):
         self.async_handler = async_handler or get_async_httpx_client(
@@ -309,6 +311,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         )
         self.guardrailIdentifier = guardrailIdentifier
         self.guardrailVersion = guardrailVersion
+        self.outputScope = outputScope
         self.guardrail_provider = "bedrock"
         self.chunk_budget_chars = chunk_budget_chars
         self.contextual_grounding_from_messages = contextual_grounding_from_messages
@@ -1280,10 +1283,10 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         AWS billed them to ``completed_chunk_usages``, and the attempt log sums those
         with the blocking call's own usage.
         """
-        bedrock_request_data: Final = {
-            **base_request_data,
-            "content": content,
-        }
+        bedrock_request_data: Final = dict(base_request_data)
+        bedrock_request_data["content"] = content
+        if self.outputScope is not None and "outputScope" not in bedrock_request_data:
+            bedrock_request_data["outputScope"] = self.outputScope.value
         prepared_request: Final = await run_aws_signing(
             self._prepare_request,
             credentials=credentials,

litellm/proxy/guardrails/guardrail_hooks/ovalix/__init__.py

@@ -19,6 +19,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
     application_id: Final = getattr(litellm_params, "application_id", None)
     pre_checkpoint_id: Final = getattr(litellm_params, "pre_checkpoint_id", None)
     post_checkpoint_id: Final = getattr(litellm_params, "post_checkpoint_id", None)
+    file_checkpoint_id: Final = getattr(litellm_params, "file_checkpoint_id", None)
 
     _ovalix_callback: Final = OvalixGuardrail(
         guardrail_name=guardrail.get("guardrail_name", ""),
@@ -27,6 +28,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
         application_id=application_id,
         pre_checkpoint_id=pre_checkpoint_id,
         post_checkpoint_id=post_checkpoint_id,
+        file_checkpoint_id=file_checkpoint_id,
         event_hook=litellm_params.mode,
         default_on=litellm_params.default_on,
         timeout=litellm_params.timeout,

litellm/proxy/guardrails/guardrail_hooks/ovalix/ovalix.py

@@ -4,13 +4,19 @@ Use Ovalix Guardrails for your LLM calls. Supports pre_call (user input) and
 post_call (model output) checkpoints with optional correction/blocking.
 """
 
+import asyncio
+import base64
 import datetime
+import gzip
 import hashlib
+import mimetypes
 import os
-from typing import TYPE_CHECKING, Any, Final, Literal
+from collections.abc import Mapping, Sequence
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, Literal
 
 import httpx
-from typing_extensions import ReadOnly, TypedDict, Unpack
+from typing_extensions import TypedDict, Unpack
 
 from litellm._logging import verbose_proxy_logger
 from litellm.exceptions import GuardrailRaisedException
@@ -18,10 +24,23 @@ from litellm.integrations.custom_guardrail import (
     CustomGuardrail,
     log_guardrail_information,
 )
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
 )
+from litellm.proxy.guardrails.guardrail_hooks.ovalix.ovalix_extraction import (
+    FilePart,
+    extract_file_parts_from_images,
+    extract_file_parts_from_messages,
+    extract_tool_calls_from_messages,
+    extract_tool_results,
+    make_tool_data,
+    str_or_none,
+    tool_call_to_tool_data,
+    tool_data_key,
+    tool_result_text_indices,
+)
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.utils import GenericGuardrailAPIInputs
 
@@ -32,13 +51,34 @@ if TYPE_CHECKING:
 
 BLOCKED_BY_OVALIX_FALLBACK_MESSAGE: Final = "This message was blocked by Ovalix"
 BLOCKED_ACTION_TYPE: Final = "block"
+_MODIFY_ACTION_TYPES: Final = ("anonymize", "sanitize")
+_DEFAULT_FILE_SIZE_LIMIT: Final = 64 * 1024 * 1024
+_NO_METADATA: Final[Mapping[str, object]] = MappingProxyType({})
+_FILE_BLOCK_ESCALATION_REASON: Final = (
+    "This message was blocked by Ovalix because file content anonymization isn't possible via LiteLLM"
+)
+_UNDECODABLE_FILE_BLOCK_REASON: Final = (
+    "This message was blocked by Ovalix because an attached file couldn't be decoded for inspection"
+)
+_TOOL_BLOCK_ESCALATION_REASON: Final = (
+    "This message was blocked by Ovalix because tool call anonymization isn't possible via LiteLLM"
+)
+_TOOL_RESULT_BLOCK_ESCALATION_REASON: Final = (
+    "This message was blocked by Ovalix because tool result anonymization isn't possible via LiteLLM"
+)
+
+
+def _encode_file_wire_format(raw: bytes) -> str:
+    return base64.b64encode(gzip.compress(raw)).decode()
+
+
+def _mapping_or_empty(value: object) -> Mapping[str, object]:
+    return as_str_mapping(value) or _NO_METADATA
 
 
 class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object):
     """Base-class constructor options this guardrail forwards untouched to CustomGuardrail."""
 
-    supported_event_hooks: ReadOnly[list[GuardrailEventHooks]]
-
 
 class OvalixGuardrailMissingSecrets(Exception):
     """Raised when required Ovalix config (API base, key, application/checkpoint IDs) is missing."""
@@ -87,6 +127,8 @@ class OvalixGuardrail(CustomGuardrail):
         application_id: str | None = None,
         pre_checkpoint_id: str | None = None,
         post_checkpoint_id: str | None = None,
+        file_checkpoint_id: str | None = None,
+        supported_event_hooks: list[GuardrailEventHooks] | None = None,
         **kwargs: Unpack[_CustomGuardrailOptions],
     ):
         self._tracker_api_base = tracker_api_base or os.environ.get("OVALIX_TRACKER_API_BASE")
@@ -94,23 +136,26 @@ class OvalixGuardrail(CustomGuardrail):
         self._application_id = application_id or os.environ.get("OVALIX_APPLICATION_ID")
         self._pre_checkpoint_id = pre_checkpoint_id or os.environ.get("OVALIX_PRE_CHECKPOINT_ID")
         self._post_checkpoint_id = post_checkpoint_id or os.environ.get("OVALIX_POST_CHECKPOINT_ID")
+        self._file_checkpoint_id = file_checkpoint_id or os.environ.get("OVALIX_FILE_CHECKPOINT_ID")
 
-        supported_event_hooks: Final = kwargs.get("supported_event_hooks", [])
+        event_hooks: Final = self._validated_event_hooks(supported_event_hooks or ())
 
-        self._validate_config(supported_event_hooks)
-
-        self._tracker_headers = httpx.Headers(
-            {
-                "Authorization": f"Bearer {self._tracker_api_key}",
-                "Content-Type": "application/json",
-            },
-            encoding="utf-8",
+        self._tracker_headers = dict(
+            httpx.Headers(
+                MappingProxyType(
+                    {
+                        "Authorization": f"Bearer {self._tracker_api_key}",
+                        "x-api-key": self._tracker_api_key or "",
+                        "Content-Type": "application/json",
+                    }
+                ),
+                encoding="utf-8",
+            )
         )
 
         self._async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback)
 
-        forwarded: Final[_CustomGuardrailOptions] = {**kwargs, "supported_event_hooks": supported_event_hooks}
-        super().__init__(**forwarded)
+        super().__init__(supported_event_hooks=event_hooks, **kwargs)
         verbose_proxy_logger.debug(
             "Ovalix Guardrail initialized: tracker=%s, application_id=%s, pre_checkpoint_id=%s, post_checkpoint_id=%s",
             self._tracker_api_base,
@@ -119,44 +164,53 @@ class OvalixGuardrail(CustomGuardrail):
             self._post_checkpoint_id,
         )
 
-    def _validate_config(self, supported_event_hooks: list[GuardrailEventHooks]) -> None:
-        """Ensure required secrets and checkpoint IDs are set; auto-add hooks when IDs are present."""
-        errors: Final[list[str]] = []
-
-        if not self._tracker_api_base:
-            errors.append("Tracker API base, set OVALIX_TRACKER_API_BASE or pass tracker_api_base")
-        if not self._tracker_api_key:
-            errors.append("Tracker API key, set OVALIX_TRACKER_API_KEY or pass tracker_api_key")
-        if not self._application_id:
-            errors.append("Application ID, set OVALIX_APPLICATION_ID or pass application_id")
-        if not self._pre_checkpoint_id and GuardrailEventHooks.pre_call in supported_event_hooks:
-            errors.append("Pre-checkpoint ID, set OVALIX_PRE_CHECKPOINT_ID or pass pre_checkpoint_id")
-        if not self._post_checkpoint_id and GuardrailEventHooks.post_call in supported_event_hooks:
-            errors.append("Post-checkpoint ID, set OVALIX_POST_CHECKPOINT_ID or pass post_checkpoint_id")
-        if not self._pre_checkpoint_id and not self._post_checkpoint_id:
-            errors.append(
-                "Pre-checkpoint ID or Post-checkpoint ID, set OVALIX_PRE_CHECKPOINT_ID or OVALIX_POST_CHECKPOINT_ID or pass pre_checkpoint_id or post_checkpoint_id"
+    def _validated_event_hooks(self, requested: Sequence[GuardrailEventHooks]) -> list[GuardrailEventHooks]:
+        """Ensure required secrets and checkpoint IDs are set; return the requested hooks plus one per configured checkpoint."""
+        errors: Final = tuple(
+            message
+            for missing, message in (
+                (not self._tracker_api_base, "Tracker API base, set OVALIX_TRACKER_API_BASE or pass tracker_api_base"),
+                (not self._tracker_api_key, "Tracker API key, set OVALIX_TRACKER_API_KEY or pass tracker_api_key"),
+                (not self._application_id, "Application ID, set OVALIX_APPLICATION_ID or pass application_id"),
+                (
+                    not self._pre_checkpoint_id and GuardrailEventHooks.pre_call in requested,
+                    "Pre-checkpoint ID, set OVALIX_PRE_CHECKPOINT_ID or pass pre_checkpoint_id",
+                ),
+                (
+                    not self._post_checkpoint_id and GuardrailEventHooks.post_call in requested,
+                    "Post-checkpoint ID, set OVALIX_POST_CHECKPOINT_ID or pass post_checkpoint_id",
+                ),
+                (
+                    not self._pre_checkpoint_id and not self._post_checkpoint_id,
+                    "Pre-checkpoint ID or Post-checkpoint ID, set OVALIX_PRE_CHECKPOINT_ID or OVALIX_POST_CHECKPOINT_ID or pass pre_checkpoint_id or post_checkpoint_id",
+                ),
             )
+            if missing
+        )
 
         if errors:
             raise OvalixGuardrailMissingSecrets("Missing Ovalix guardrail configuration errors: " + ". ".join(errors))
 
-        # auto-add hooks when checkpoint IDs are present
-        if self._pre_checkpoint_id and GuardrailEventHooks.pre_call not in supported_event_hooks:
-            supported_event_hooks.append(GuardrailEventHooks.pre_call)
-        if self._post_checkpoint_id and GuardrailEventHooks.post_call not in supported_event_hooks:
-            supported_event_hooks.append(GuardrailEventHooks.post_call)
+        auto_added: Final = tuple(
+            hook
+            for configured, hook in (
+                (self._pre_checkpoint_id, GuardrailEventHooks.pre_call),
+                (self._post_checkpoint_id, GuardrailEventHooks.post_call),
+            )
+            if configured and hook not in requested
+        )
+        return [*requested, *auto_added]
 
-    def _get_actor(self, data: dict) -> str:
+    def _get_actor(self, data: Mapping[str, object]) -> str:
         """Return a stable actor identifier from request metadata (e.g. user email or id)."""
-        metadata: Final = data.get("metadata") or data.get("litellm_metadata") or {}
-        if metadata.get("user_api_key_user_email"):
-            return metadata["user_api_key_user_email"]
-        if metadata.get("user_api_key_user_id"):
-            return metadata["user_api_key_user_id"]
-        return "unknown"
+        metadata: Final = _mapping_or_empty(data.get("metadata") or data.get("litellm_metadata"))
+        return (
+            str_or_none(metadata.get("user_api_key_user_email"))
+            or str_or_none(metadata.get("user_api_key_user_id"))
+            or "unknown"
+        )
 
-    def _get_tracker_actor_id(self, data: dict) -> str:
+    def _get_tracker_actor_id(self, data: Mapping[str, object]) -> str:
         """Normalize the actor string into a short, stable id for Tracker API payloads."""
         # NOTE: this hash is purely for normalization — it collapses an arbitrary actor
         # string (email, user id, or "unknown") into a compact, fixed-length, consistent
@@ -166,7 +220,7 @@ class OvalixGuardrail(CustomGuardrail):
         normalized_actor_id: Final = hashlib.sha256(actor_id).hexdigest()[:8]
         return normalized_actor_id
 
-    def _get_session_id(self, data: dict) -> str:
+    def _get_session_id(self, data: Mapping[str, object]) -> str:
         """Return a unique identifier for the chat/session (actor + date + application_id)."""
         actor_hash: Final = self._get_tracker_actor_id(data)
         today: Final = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d")
@@ -174,106 +228,254 @@ class OvalixGuardrail(CustomGuardrail):
 
     async def _call_checkpoint(
         self,
-        content: str,
+        data_type: str,
+        data: Mapping[str, object],
         checkpoint_id: str,
         actor: str,
         session_id: str,
-    ) -> dict[str, Any]:
-        """Call the Ovalix Tracker checkpoint API and return the JSON response."""
+        oversize: bool = False,
+    ) -> Mapping[str, object]:
+        """Call the Ovalix Tracker checkpoint API and return the JSON response.
+
+        Both routes live on the tracker's /beta litellm router, which accepts the api key this
+        guardrail already sends.
+        """
         application_id: Final = self._application_id
         if not application_id or not checkpoint_id:
             raise ValueError("Ovalix: application_id or checkpoint_id not resolved")
 
-        url: Final = f"{self._tracker_api_base}/tracking/custom_application/checkpoint"
-        headers: Final = dict(self._tracker_headers)
+        route: Final = "file_checkpoint" if data_type == "FILE" else "checkpoint"
         payload: Final = {
             "application_id": application_id,
             "checkpoint_id": checkpoint_id,
             "actor": actor,
             "session_id": session_id,
-            "data_type": "TEXT",
-            "data": {"content": content},
+            "data_type": data_type,
+            "data": data,
+            "tool": "LiteLLM",
+            **(MappingProxyType({"oversize": True}) if oversize else _NO_METADATA),
         }
-        response: Final = await self._async_handler.post(url, headers=headers, json=payload, timeout=self.timeout)
+        response: Final = await self._async_handler.post(
+            f"{self._tracker_api_base}/tracking/beta/{route}",
+            headers=self._tracker_headers,
+            json=payload,
+            timeout=self.timeout,
+        )
         response.raise_for_status()
         return response.json()
 
+    def _verdict(self, resp: Mapping[str, object]) -> tuple[str, str | None]:
+        return (str_or_none(resp.get("action_type")) or "").lower(), self._get_trackers_corrected_message(resp)
+
+    async def _block_reason_for_item(
+        self,
+        data_type: str,
+        data: Mapping[str, object],
+        checkpoint_id: str,
+        actor: str,
+        session_id: str,
+        escalation_reason: str,
+        oversize: bool = False,
+    ) -> str | None:
+        try:
+            resp: Final = await self._call_checkpoint(
+                data_type, data, checkpoint_id, actor, session_id, oversize=oversize
+            )
+        except Exception as e:
+            verbose_proxy_logger.exception("Ovalix checkpoint call failed: %s", e)
+            raise GuardrailRaisedException(
+                guardrail_name=self.guardrail_name,
+                message=f"Ovalix guardrail error: {e}",
+                should_wrap_with_default_message=False,
+            ) from e
+        action, corrected = self._verdict(resp)
+        if action == BLOCKED_ACTION_TYPE:
+            return corrected or BLOCKED_BY_OVALIX_FALLBACK_MESSAGE
+        if action in _MODIFY_ACTION_TYPES:
+            return escalation_reason
+        return None
+
+    async def _check_items_block_only(
+        self,
+        items: Sequence[tuple[str, Mapping[str, object]]],
+        checkpoint_id: str,
+        actor: str,
+        session_id: str,
+        escalation_reason: str,
+    ) -> str | None:
+        for data_type, data in items:
+            reason = await self._block_reason_for_item(
+                data_type, data, checkpoint_id, actor, session_id, escalation_reason
+            )
+            if reason is not None:
+                return reason
+        return None
+
+    async def _check_files_for_block(
+        self,
+        file_parts: Sequence[FilePart],
+        checkpoint_id: str,
+        actor: str,
+        session_id: str,
+    ) -> str | None:
+        if any(part.undecodable for part in file_parts):
+            return _UNDECODABLE_FILE_BLOCK_REASON
+        for part in sorted(file_parts, key=lambda p: p.message_index, reverse=True):
+            data = await self._file_part_to_data(part)
+            reason = await self._block_reason_for_item(
+                "FILE",
+                data,
+                checkpoint_id,
+                actor,
+                session_id,
+                _FILE_BLOCK_ESCALATION_REASON,
+                oversize=part.oversize,
+            )
+            if reason is not None:
+                return reason
+        return None
+
     @log_guardrail_information
     async def apply_guardrail(
         self,
         inputs: GenericGuardrailAPIInputs,
-        request_data: dict,
+        request_data: Mapping[str, object],
         input_type: Literal["request", "response"],
         logging_obj: "LiteLLMLoggingObj | None" = None,
     ) -> GenericGuardrailAPIInputs:
-        """
-        Apply Ovalix guardrail to the given inputs (request or response text).
-
-        Used by the unified guardrail flow and the /apply_guardrail API.
-        For "request", uses the pre-checkpoint; for "response", uses the post-checkpoint.
-
-        Args:
-            inputs: Guardrail API inputs (e.g. texts to check).
-            request_data: Full request payload (messages, metadata, response).
-            input_type: "request" (pre_call) or "response" (post_call).
-            logging_obj: Optional logging context.
-
-        Returns:
-            Updated inputs (e.g. with replaced/corrected texts, or unchanged).
-        """
-        if not self._pre_checkpoint_id and not self._post_checkpoint_id:
-            return inputs
-
-        tracker_actor_id: Final = self._get_tracker_actor_id(request_data)
+        actor: Final = self._get_actor(request_data)
         session_id: Final = self._get_session_id(request_data)
-        texts: Final = inputs.get("texts") or []
-        if not texts or not isinstance(texts, list):
+        is_response: Final = input_type == "response"
+
+        prompt_checkpoint: Final = self._post_checkpoint_id if is_response else self._pre_checkpoint_id
+        file_checkpoint: Final = self._file_checkpoint_id or prompt_checkpoint
+        if not file_checkpoint:
+            verbose_proxy_logger.debug(
+                "Ovalix guardrail: application %s has no %s checkpoint, leaving this direction uninspected",
+                self._application_id,
+                input_type,
+            )
             return inputs
 
-        if input_type == "response":
-            if not self._post_checkpoint_id:
-                return inputs
-            corrected_llm_responses: Final = await self._generate_post_guardrail_llm_texts(
-                texts, tracker_actor_id, session_id, self._post_checkpoint_id
+        structured_messages: Final = inputs.get("structured_messages") or ()
+        file_parts: Final = (
+            extract_file_parts_from_images(inputs.get("images"), size_limit=_DEFAULT_FILE_SIZE_LIMIT)
+            if is_response
+            else extract_file_parts_from_messages(structured_messages, size_limit=_DEFAULT_FILE_SIZE_LIMIT)
+        )
+        file_block: Final = await self._check_files_for_block(file_parts, file_checkpoint, actor, session_id)
+        if file_block is not None:
+            self._block_current_message(file_block)
+
+        if not prompt_checkpoint:
+            verbose_proxy_logger.debug(
+                "Ovalix guardrail: application %s has only a %s file checkpoint, skipping text and tool inspection",
+                self._application_id,
+                input_type,
             )
-            return {**inputs, "texts": corrected_llm_responses}
+            return inputs
 
-        if self._pre_checkpoint_id:
-            post_guardrail_texts: Final = await self._generate_post_guardrail_llm_texts(
-                texts, tracker_actor_id, session_id, self._pre_checkpoint_id
-            )
-            return {**inputs, "texts": post_guardrail_texts}
-        return inputs
+        tool_calls: Final = (
+            *(inputs.get("tool_calls") or ()),
+            *extract_tool_calls_from_messages(structured_messages),
+        )
+        unique_tool_data: Final = MappingProxyType(
+            {tool_data_key(data): data for data in (tool_call_to_tool_data(tc) for tc in tool_calls) if data}
+        )
+        tool_call_items: Final = tuple(("TOOL", data) for data in unique_tool_data.values())
+        tool_block: Final = await self._check_items_block_only(
+            tool_call_items,
+            prompt_checkpoint,
+            actor,
+            session_id,
+            _TOOL_BLOCK_ESCALATION_REASON,
+        )
+        if tool_block is not None:
+            self._block_current_message(tool_block)
 
-    async def _generate_post_guardrail_llm_texts(
-        self, texts: list[str], actor: str, session_id: str, checkpoint_id: str
-    ) -> list[str]:
-        """Generate post-guardrail LLM responses for the given LLM responses."""
-        post_guardrail_texts: Final[list[str]] = []
+        tool_results: Final = extract_tool_results(structured_messages)
+        tool_result_items: Final = tuple(("TOOL", make_tool_data(name, content)) for name, content, _ in tool_results)
+        tool_result_block: Final = await self._check_items_block_only(
+            tool_result_items,
+            prompt_checkpoint,
+            actor,
+            session_id,
+            _TOOL_RESULT_BLOCK_ESCALATION_REASON,
+        )
+        if tool_result_block is not None:
+            self._block_current_message(tool_result_block)
 
-        is_first_response = True
-        for llm_response in reversed(texts):
-            try:
-                resp = await self._call_checkpoint(llm_response, checkpoint_id, actor, session_id)
-            except Exception as e:
-                verbose_proxy_logger.exception("Ovalix apply_guardrail checkpoint call failed: %s", e)
-                raise GuardrailRaisedException(
-                    guardrail_name=self.guardrail_name,
-                    message=f"Ovalix guardrail error: {e}",
-                    should_wrap_with_default_message=False,
-                ) from e
+        texts: Final = inputs.get("texts") or ()
+        if not texts:
+            return inputs
+        output_texts: Final = await self._check_texts(
+            texts,
+            prompt_checkpoint,
+            actor,
+            session_id,
+            tool_result_text_indices(structured_messages, texts),
+        )
+        if output_texts is None:
+            return inputs
+        return {**inputs, "texts": output_texts}
 
-            action_type = (resp.get("action_type") or "").lower()
-            blocking_message = self._get_trackers_corrected_message(resp) or BLOCKED_BY_OVALIX_FALLBACK_MESSAGE
-            if action_type == BLOCKED_ACTION_TYPE and is_first_response:
-                self._block_current_message(blocking_message)
-            elif action_type == BLOCKED_ACTION_TYPE:
-                post_guardrail_texts.insert(0, blocking_message)
-            else:
-                corrected_text = self._get_trackers_corrected_message(resp) or llm_response
-                post_guardrail_texts.insert(0, corrected_text)
-            is_first_response = False
-        return post_guardrail_texts
+    async def _file_part_to_data(self, part: FilePart) -> Mapping[str, object]:
+        extension: Final = mimetypes.guess_extension(part.mime_hint) if part.mime_hint else None
+        name: Final = part.name or (f"file{extension}" if extension else "file")
+        content: Final = (
+            await asyncio.get_event_loop().run_in_executor(None, _encode_file_wire_format, part.data)
+            if part.data
+            else None
+        )
+        return {"name": name, "content": content}
+
+    async def _check_texts(
+        self,
+        texts: Sequence[str],
+        checkpoint_id: str,
+        actor: str,
+        session_id: str,
+        skip_indices: frozenset[int],
+    ) -> list[str] | None:
+        newest_first: Final = tuple(index for index in reversed(range(len(texts))) if index not in skip_indices)
+        checked: Final = MappingProxyType(
+            {
+                index: await self._check_text(
+                    texts[index], checkpoint_id, actor, session_id, is_newest=index == len(texts) - 1
+                )
+                for index in newest_first
+            }
+        )
+        output: Final = [checked.get(index, text) for index, text in enumerate(texts)]
+        return output if output != list(texts) else None
+
+    async def _check_text(
+        self,
+        content: str,
+        checkpoint_id: str,
+        actor: str,
+        session_id: str,
+        is_newest: bool,
+    ) -> str:
+        """Return the text that replaces `content` after Ovalix checks it; a blocked newest text raises."""
+        try:
+            resp: Final = await self._call_checkpoint("TEXT", {"content": content}, checkpoint_id, actor, session_id)
+        except Exception as e:
+            verbose_proxy_logger.exception("Ovalix checkpoint call failed: %s", e)
+            raise GuardrailRaisedException(
+                guardrail_name=self.guardrail_name,
+                message=f"Ovalix guardrail error: {e}",
+                should_wrap_with_default_message=False,
+            ) from e
+        action, corrected = self._verdict(resp)
+        if action == BLOCKED_ACTION_TYPE:
+            block_message: Final = corrected or BLOCKED_BY_OVALIX_FALLBACK_MESSAGE
+            if is_newest:
+                self._block_current_message(block_message)
+            return block_message
+        if action in _MODIFY_ACTION_TYPES and corrected is not None:
+            return corrected
+        return content
 
     def _block_current_message(self, blocking_message: str) -> None:
         """Raise OvalixGuardrailBlockedException with the given message (no default wrapper)."""
@@ -283,12 +485,13 @@ class OvalixGuardrail(CustomGuardrail):
             should_wrap_with_default_message=False,
         )
 
-    def _get_trackers_corrected_message(self, resp: dict) -> str | None:
+    def _get_trackers_corrected_message(self, resp: Mapping[str, object]) -> str | None:
         """Extract corrected/blocking message content from Tracker checkpoint response."""
-        modified: Final = resp.get("modified_data")
-        if isinstance(modified, dict) and "content" in modified:
-            return modified["content"]
-        return None
+        modified: Final = as_str_mapping(resp.get("modified_data"))
+        if modified is None:
+            return None
+        content: Final = modified.get("content")
+        return content if isinstance(content, str) else None
 
     @staticmethod
     def get_config_model() -> type["GuardrailConfigModel"] | None:

litellm/proxy/guardrails/guardrail_hooks/ovalix/ovalix_extraction.py

@@ -0,0 +1,396 @@
+import base64
+import json
+import posixpath
+import re
+from bisect import bisect_left
+from collections.abc import Callable, Iterator, Mapping, Sequence
+from itertools import chain, groupby
+from operator import attrgetter
+from types import MappingProxyType
+from typing import Final, NamedTuple
+from urllib.parse import unquote, urlparse
+
+from pydantic import TypeAdapter, ValidationError
+
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
+
+_TOOL_NAME_MAX_LENGTH: Final = 100
+_DEFAULT_TOOL_RESULT_NAME: Final = "tool_result"
+_NO_TOOL_INPUT: Final[Mapping[str, object]] = MappingProxyType({})
+
+_DATA_URL_RE: Final = re.compile(r"^data:(?P<mime>[^;,]+)?(?P<params>(?:;[^;,]+)*?)(?P<b64>;base64)?,", re.IGNORECASE)
+_URLSAFE_TO_STANDARD_B64: Final = str.maketrans("-_", "+/")
+_OBJECT_LIST: Final = TypeAdapter(list[object])
+
+
+def str_or_none(value: object) -> str | None:
+    return value if isinstance(value, str) and value else None
+
+
+def _list_items(value: object) -> Sequence[object]:
+    try:
+        return _OBJECT_LIST.validate_python(value, strict=True)
+    except ValidationError:
+        return ()
+
+
+class FilePart(NamedTuple):
+    name: str | None
+    data: bytes | None
+    mime_hint: str | None
+    inline: bool
+    oversize: bool
+    message_index: int
+    undecodable: bool = False
+
+
+def _split_data_url(value: str) -> tuple[str | None, str | None]:
+    match: Final = _DATA_URL_RE.match(value)
+    if not match:
+        if value.lower().startswith("data:"):
+            return None, None
+        return None, value
+    mime: Final = match.group("mime") or None
+    if not match.group("b64"):
+        return mime, None
+    return mime, value[match.end() :]
+
+
+def _b64decode_or_none(payload: str) -> bytes | None:
+    try:
+        return base64.b64decode(payload, validate=True)
+    except ValueError:
+        return None
+
+
+def _decoded_base64(cleaned: str) -> bytes | None:
+    strict: Final = _b64decode_or_none(cleaned)
+    if strict is not None or ("-" not in cleaned and "_" not in cleaned):
+        return strict
+    return _b64decode_or_none(cleaned.translate(_URLSAFE_TO_STANDARD_B64))
+
+
+def _inline_part(name: str | None, value: str, size_limit: int | None, message_index: int) -> FilePart | None:
+    mime_hint, payload = _split_data_url(value)
+    if payload is None:
+        return FilePart(name, None, mime_hint, True, False, message_index, undecodable=True)
+    cleaned: Final = "".join(payload.split())
+    if not cleaned:
+        return None
+    if size_limit is not None and (len(cleaned) * 3) // 4 - 2 > size_limit:
+        return FilePart(name, None, mime_hint, True, True, message_index)
+    data: Final = _decoded_base64(cleaned)
+    if data is None:
+        return FilePart(name, None, mime_hint, True, False, message_index, undecodable=True)
+    if size_limit is not None and len(data) > size_limit:
+        return FilePart(name, None, mime_hint, True, True, message_index)
+    return FilePart(name, data, mime_hint, True, False, message_index)
+
+
+def _name_from_url(url: str) -> str | None:
+    try:
+        return unquote(posixpath.basename(urlparse(url).path)) or None
+    except ValueError:
+        return None
+
+
+def _part_from_file_block(block: Mapping[str, object], size_limit: int | None, message_index: int) -> FilePart | None:
+    file_obj: Final = as_str_mapping(block.get("file"))
+    if file_obj is None:
+        return None
+    name: Final = str_or_none(file_obj.get("filename")) or str_or_none(file_obj.get("file_id"))
+    file_data: Final = str_or_none(file_obj.get("file_data"))
+    inline: Final = _inline_part(name, file_data, size_limit, message_index) if file_data else None
+    return inline or FilePart(name, None, None, False, False, message_index)
+
+
+def _part_from_image_url_block(
+    block: Mapping[str, object], size_limit: int | None, message_index: int
+) -> FilePart | None:
+    image_url: Final = block.get("image_url")
+    image_url_fields: Final = as_str_mapping(image_url)
+    url: Final = image_url_fields.get("url") if image_url_fields is not None else image_url
+    if not isinstance(url, str) or not url:
+        return None
+    if url.startswith("data:"):
+        return _inline_part(None, url, size_limit, message_index)
+    return FilePart(_name_from_url(url), None, None, False, False, message_index)
+
+
+def _part_from_input_file_block(
+    block: Mapping[str, object], size_limit: int | None, message_index: int
+) -> FilePart | None:
+    declared_name: Final = str_or_none(block.get("filename")) or str_or_none(block.get("file_id"))
+    file_data: Final = str_or_none(block.get("file_data"))
+    inline: Final = _inline_part(declared_name, file_data, size_limit, message_index) if file_data else None
+    if inline is not None:
+        return inline
+    file_url: Final = block.get("file_url")
+    name: Final = (
+        _name_from_url(file_url) if isinstance(file_url, str) and file_url and not declared_name else declared_name
+    )
+    return FilePart(name, None, None, False, False, message_index)
+
+
+def _part_from_input_audio_block(
+    block: Mapping[str, object], size_limit: int | None, message_index: int
+) -> FilePart | None:
+    audio: Final = as_str_mapping(block.get("input_audio"))
+    if audio is None:
+        return None
+    data_b64: Final = audio.get("data")
+    if not isinstance(data_b64, str) or not data_b64:
+        return None
+    name: Final = f"audio.{audio.get('format') or 'bin'}"
+    return _inline_part(name, data_b64, size_limit, message_index) or FilePart(
+        name, None, None, False, False, message_index
+    )
+
+
+_BLOCK_PARSERS: Final[Mapping[str, Callable[[Mapping[str, object], int | None, int], FilePart | None]]] = (
+    MappingProxyType(
+        {
+            "file": _part_from_file_block,
+            "image_url": _part_from_image_url_block,
+            "input_image": _part_from_image_url_block,
+            "input_file": _part_from_input_file_block,
+            "input_audio": _part_from_input_audio_block,
+        }
+    )
+)
+
+
+def _file_parts_of_message(
+    message: Mapping[str, object], size_limit: int | None, message_index: int
+) -> Iterator[FilePart]:
+    for block in map(as_str_mapping, _list_items(message.get("content"))):
+        if block is None:
+            continue
+        block_type = block.get("type")
+        if not isinstance(block_type, str):
+            continue
+        parser = _BLOCK_PARSERS.get(block_type)
+        if parser is None:
+            continue
+        try:
+            part = parser(block, size_limit, message_index)
+        except (TypeError, ValueError, AttributeError, KeyError):
+            continue
+        if part is not None and (part.inline or part.name):
+            yield part
+
+
+def _file_parts_of_messages(structured_messages: Sequence[object], size_limit: int | None) -> Iterator[FilePart]:
+    for message_index, message in enumerate(map(as_str_mapping, structured_messages)):
+        if message is not None:
+            yield from _file_parts_of_message(message, size_limit, message_index)
+
+
+def extract_file_parts_from_messages(
+    structured_messages: Sequence[object] | None, size_limit: int | None = None
+) -> tuple[FilePart, ...]:
+    return tuple(_file_parts_of_messages(structured_messages or (), size_limit))
+
+
+def _file_part_of_image(value: str, size_limit: int | None, index: int) -> FilePart | None:
+    if value.startswith(("http://", "https://")):
+        name: Final = _name_from_url(value)
+        return FilePart(name, None, None, False, False, index) if name else None
+    return _inline_part(None, value, size_limit, index)
+
+
+def extract_file_parts_from_images(
+    images: Sequence[object] | None, size_limit: int | None = None
+) -> tuple[FilePart, ...]:
+    candidates: Final = (
+        _file_part_of_image(value, size_limit, index)
+        for index, value in enumerate(images or ())
+        if isinstance(value, str) and value
+    )
+    return tuple(part for part in candidates if part is not None)
+
+
+def make_tool_data(
+    name: str, content: str | None, tool_input: Mapping[str, object] | None = None
+) -> Mapping[str, object]:
+    action_name: Final = str(name) if str(name).strip() else _DEFAULT_TOOL_RESULT_NAME
+    truncated: Final = action_name[:_TOOL_NAME_MAX_LENGTH]
+    tool_name: Final = truncated if truncated.strip() else _DEFAULT_TOOL_RESULT_NAME
+    return {
+        "content": content,
+        "tool_name": tool_name,
+        "action_name": action_name,
+        "tool_input": dict(tool_input or ()),
+    }
+
+
+def _tool_call_field(tool_call: object, key: str) -> object:
+    fields: Final = as_str_mapping(tool_call)
+    return fields.get(key) if fields is not None else getattr(tool_call, key, None)
+
+
+def _json_or_str(value: object) -> str:
+    try:
+        return json.dumps(value)
+    except (TypeError, ValueError):
+        return str(value)
+
+
+def _parsed_tool_input(raw_arguments: str) -> Mapping[str, object]:
+    if not raw_arguments:
+        return _NO_TOOL_INPUT
+    try:
+        parsed: Final = json.loads(raw_arguments)
+    except (ValueError, TypeError):
+        return _NO_TOOL_INPUT
+    return as_str_mapping(parsed) or _NO_TOOL_INPUT
+
+
+def _tool_content_and_input(raw_arguments: object) -> tuple[str, Mapping[str, object]]:
+    if isinstance(raw_arguments, str):
+        return raw_arguments, _parsed_tool_input(raw_arguments)
+    if raw_arguments is None:
+        return "", _NO_TOOL_INPUT
+    arguments: Final = as_str_mapping(raw_arguments)
+    return _json_or_str(raw_arguments), arguments if arguments is not None else _NO_TOOL_INPUT
+
+
+def tool_call_to_tool_data(tool_call: object) -> Mapping[str, object] | None:
+    function: Final = _tool_call_field(tool_call, "function")
+    name: Final = _tool_call_field(function, "name")
+    if not name or not str(name).strip():
+        return None
+    content, tool_input = _tool_content_and_input(_tool_call_field(function, "arguments"))
+    return make_tool_data(str(name), content, tool_input)
+
+
+def _message_tool_calls(message: object) -> Sequence[object]:
+    fields: Final = as_str_mapping(message)
+    return _list_items(fields.get("tool_calls")) if fields is not None else ()
+
+
+def extract_tool_calls_from_messages(structured_messages: Sequence[object] | None) -> tuple[object, ...]:
+    """Tool calls declared on the messages themselves.
+
+    Surfaces such as the Anthropic request path populate ``structured_messages`` but leave the
+    top-level ``tool_calls`` input empty, so calls made in prior assistant turns are only visible here.
+    """
+    return tuple(chain.from_iterable(_message_tool_calls(message) for message in structured_messages or ()))
+
+
+def tool_data_key(tool_data: Mapping[str, object]) -> str:
+    """Stable identity for a tool payload, so a call reached from two sources is only scanned once."""
+    return json.dumps(tool_data, sort_keys=True, default=str)
+
+
+def _tool_content_block_text(block: object) -> str | None:
+    if isinstance(block, str):
+        return block
+    fields: Final = as_str_mapping(block)
+    return str_or_none(fields.get("text")) if fields is not None else None
+
+
+def _tool_content_text(content: object) -> str:
+    if isinstance(content, str):
+        return content
+    fields: Final = as_str_mapping(content)
+    if fields is not None:
+        return _json_or_str(fields)
+    texts: Final = (_tool_content_block_text(block) for block in _list_items(content))
+    return "\n".join(text for text in texts if text is not None)
+
+
+def _extract_tool_content(content: object) -> str | None:
+    text: Final = _tool_content_text(content)
+    return text if text.strip() else None
+
+
+class _Declaration(NamedTuple):
+    call_id: str
+    message_index: int
+    name: str
+
+
+def _declarations_in(message_index: int, message: object) -> Iterator[_Declaration]:
+    fields: Final = as_str_mapping(message)
+    if fields is None or fields.get("role") != "assistant":
+        return
+    for tool_call in map(as_str_mapping, _list_items(fields.get("tool_calls"))):
+        if tool_call is None:
+            continue
+        call_id = tool_call.get("id")
+        function = as_str_mapping(tool_call.get("function"))
+        name = str_or_none(function.get("name")) if function is not None else None
+        if isinstance(call_id, str) and call_id and name and name.strip():
+            yield _Declaration(call_id, message_index, name)
+
+
+def _declarations_by_call_id(messages: Sequence[object]) -> Mapping[str, tuple[_Declaration, ...]]:
+    """Every named declaration of each tool call id, in message order, built in one pass."""
+    declarations: Final = chain.from_iterable(
+        _declarations_in(index, message) for index, message in enumerate(messages)
+    )
+    ordered: Final = sorted(declarations, key=attrgetter("call_id", "message_index"))
+    return MappingProxyType({call_id: tuple(group) for call_id, group in groupby(ordered, key=attrgetter("call_id"))})
+
+
+def _resolve_tool_name(
+    declarations: Mapping[str, tuple[_Declaration, ...]], tool_index: int, tool_call_id: object
+) -> str:
+    """Name from the nearest declaration of ``tool_call_id`` before ``tool_index``."""
+    if not isinstance(tool_call_id, str) or not tool_call_id:
+        return _DEFAULT_TOOL_RESULT_NAME
+    declared: Final = declarations.get(tool_call_id, ())
+    earlier: Final = bisect_left(declared, tool_index, key=attrgetter("message_index"))
+    return declared[earlier - 1].name if earlier else _DEFAULT_TOOL_RESULT_NAME
+
+
+def extract_tool_results(
+    structured_messages: Sequence[object] | None,
+) -> tuple[tuple[str, str, object], ...]:
+    messages: Final = tuple(structured_messages or ())
+    declarations: Final = _declarations_by_call_id(messages)
+
+    def _results() -> Iterator[tuple[str, str, object]]:
+        for index, message in enumerate(map(as_str_mapping, messages)):
+            if message is None or message.get("role") != "tool":
+                continue
+            content = _extract_tool_content(message.get("content"))
+            if content is None:
+                continue
+            tool_call_id = message.get("tool_call_id")
+            yield _resolve_tool_name(declarations, index, tool_call_id), content, tool_call_id
+
+    return tuple(_results())
+
+
+def _block_texts(content: object) -> Iterator[object]:
+    blocks: Final = (as_str_mapping(block) for block in _list_items(content))
+    return (block.get("text") for block in blocks if block is not None and block.get("text") is not None)
+
+
+def _message_text_origins(structured_messages: Sequence[object] | None) -> Iterator[tuple[object, bool]]:
+    for message in map(as_str_mapping, structured_messages or ()):
+        if message is None:
+            continue
+        content = message.get("content")
+        from_tool_result = message.get("role") == "tool" and _extract_tool_content(content) is not None
+        if isinstance(content, str):
+            yield content, from_tool_result
+        else:
+            yield from ((text, from_tool_result) for text in _block_texts(content))
+
+
+def tool_result_text_indices(structured_messages: Sequence[object] | None, texts: Sequence[str]) -> frozenset[int]:
+    """Positions in ``texts`` that hold content already submitted under the TOOL policy.
+
+    The chat-completions guardrail flow builds ``texts`` and ``structured_messages`` from the
+    same message list, so tool-role content lands in both and would otherwise be checked twice.
+    Other surfaces (e.g. Anthropic messages) build ``texts`` from a differently shaped payload,
+    so the mapping is only trusted when replaying it reproduces ``texts`` exactly; anything else
+    falls back to checking every text.
+    """
+    origins: Final = tuple(_message_text_origins(structured_messages))
+    if tuple(text for text, _ in origins) != tuple(texts):
+        return frozenset()
+    return frozenset(index for index, (_, from_tool_result) in enumerate(origins) if from_tool_result)

litellm/proxy/guardrails/guardrail_initializers.py

@@ -18,6 +18,7 @@ def initialize_bedrock(litellm_params: LitellmParams, guardrail: Guardrail):
         event_hook=litellm_params.mode,
         guardrailIdentifier=litellm_params.guardrailIdentifier,
         guardrailVersion=litellm_params.guardrailVersion,
+        outputScope=litellm_params.outputScope,
         checks=litellm_params.checks if isinstance(litellm_params.checks, BedrockChecksConfigModel) else None,
         content_filter_threshold=litellm_params.content_filter_threshold,
         prompt_attack_threshold=litellm_params.prompt_attack_threshold,

litellm/proxy/guardrails/response_handlers.py

@@ -0,0 +1,117 @@
+"""Response policies for the standalone guardrail endpoint."""
+
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final, Literal, TypeAlias
+
+from fastapi import HTTPException
+from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
+
+from litellm.integrations.custom_guardrail import CustomGuardrail
+from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
+from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.types.guardrails import (
+    ApplyGuardrailRequest,
+    ApplyGuardrailResponse,
+    GuardrailEventHooks,
+    GuardrailResponseFormat,
+    NativeGuardrailResponse,
+)
+from litellm.types.utils import GenericGuardrailAPIInputs
+
+GuardrailEndpointResponse: TypeAlias = ApplyGuardrailResponse | NativeGuardrailResponse
+_NATIVE_RESPONSE: Final[TypeAdapter[NativeGuardrailResponse]] = TypeAdapter(NativeGuardrailResponse)
+
+
+class StandardGuardrailResponseHandler:
+    """Preserve the endpoint's existing text response and hook behavior."""
+
+    def prepare_request_data(self, request: ApplyGuardrailRequest) -> Mapping[str, object]:
+        return request.model_dump(include={"messages", "metadata"}, exclude_none=True)
+
+    def resolve_input_type(
+        self, request: ApplyGuardrailRequest, guardrail: CustomGuardrail
+    ) -> Literal["request", "response"]:
+        if request.input_type == "request" and guardrail.event_hook in (GuardrailEventHooks.post_call, "post_call"):
+            return "response"
+        return "response" if request.input_type == "response" else "request"
+
+    def build_response(
+        self,
+        request: ApplyGuardrailRequest,
+        processed_inputs: GenericGuardrailAPIInputs,
+        request_data: Mapping[str, object],
+    ) -> GuardrailEndpointResponse:
+        texts: Final = processed_inputs.get("texts", [])
+        return ApplyGuardrailResponse(response_text=texts[0] if texts else request.text)
+
+    def accept_hook_result(self, original: GuardrailEndpointResponse, modified: object) -> GuardrailEndpointResponse:
+        return modified if isinstance(modified, ApplyGuardrailResponse) else original
+
+
+class _NativeGuardrailRecord(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    guardrail_name: str | None = None
+    guardrail_response: JsonValue = None
+
+
+class _NativeGuardrailMetadata(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    standard_logging_guardrail_information: tuple[_NativeGuardrailRecord, ...] = ()
+
+
+class NativeGuardrailResponseHandler(StandardGuardrailResponseHandler):
+    """Return structured findings recorded by this guardrail invocation."""
+
+    def prepare_request_data(self, request: ApplyGuardrailRequest) -> Mapping[str, object]:
+        return request.model_dump(
+            include={"messages", "metadata"},
+            exclude={"metadata": {"standard_logging_guardrail_information"}},
+            exclude_none=True,
+        )
+
+    def resolve_input_type(
+        self, request: ApplyGuardrailRequest, guardrail: CustomGuardrail
+    ) -> Literal["request", "response"]:
+        if "input_type" in request.model_fields_set:
+            return "response" if request.input_type == "response" else "request"
+        return super().resolve_input_type(request, guardrail)
+
+    def build_response(
+        self,
+        request: ApplyGuardrailRequest,
+        processed_inputs: GenericGuardrailAPIInputs,
+        request_data: Mapping[str, object],
+    ) -> GuardrailEndpointResponse:
+        metadata: Final = request_data.get(get_metadata_variable_name_from_kwargs(request_data))
+        recorded: Final = _NativeGuardrailMetadata.model_validate_json(
+            safe_dumps(metadata if metadata is not None else {})
+        )
+        findings: Final = tuple(
+            record.guardrail_response
+            for record in recorded.standard_logging_guardrail_information
+            if record.guardrail_name == request.guardrail_name and isinstance(record.guardrail_response, (dict, list))
+        )
+        if not findings:
+            raise HTTPException(
+                status_code=422,
+                detail=f"Guardrail '{request.guardrail_name}' did not report structured native findings.",
+            )
+        return findings[0] if len(findings) == 1 else findings
+
+    def accept_hook_result(self, original: GuardrailEndpointResponse, modified: object) -> GuardrailEndpointResponse:
+        if modified is None:
+            return original
+        return _NATIVE_RESPONSE.validate_python(modified)
+
+
+GUARDRAIL_RESPONSE_HANDLERS: Final[Mapping[GuardrailResponseFormat, StandardGuardrailResponseHandler]] = (
+    MappingProxyType(
+        {
+            GuardrailResponseFormat.STANDARD: StandardGuardrailResponseHandler(),
+            GuardrailResponseFormat.NATIVE: NativeGuardrailResponseHandler(),
+        }
+    )
+)

litellm/proxy/hooks/dynamic_rate_limiter.py

@@ -118,7 +118,7 @@ class _PROXY_DynamicRateLimitHandler(CustomLogger):
             elif priority is not None and litellm.priority_reservation is not None:
                 if os.getenv("LITELLM_LICENSE", None) is None:
                     verbose_proxy_logger.error(
-                        "PREMIUM FEATURE: Reserving tpm/rpm by priority is a premium feature. Please add a 'LITELLM_LICENSE' to your .env to enable this.\nGet a license: https://docs.litellm.ai/docs/proxy/enterprise."
+                        "PREMIUM FEATURE: Reserving tpm/rpm by priority is a premium feature. Please add a 'LITELLM_LICENSE' to your .env to enable this.\nGet a license: https://docs.litellm.ai/docs/enterprise."
                     )
                 else:
                     value: Final = litellm.priority_reservation[priority]

litellm/proxy/hooks/dynamic_rate_limiter_v3.py

@@ -133,7 +133,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
         elif priority is not None and litellm.priority_reservation is not None:
             if os.getenv("LITELLM_LICENSE", None) is None:
                 verbose_proxy_logger.error(
-                    "PREMIUM FEATURE: Reserving tpm/rpm by priority is a premium feature. Please add a 'LITELLM_LICENSE' to your .env to enable this.\nGet a license: https://docs.litellm.ai/docs/proxy/enterprise."
+                    "PREMIUM FEATURE: Reserving tpm/rpm by priority is a premium feature. Please add a 'LITELLM_LICENSE' to your .env to enable this.\nGet a license: https://docs.litellm.ai/docs/enterprise."
                 )
             else:
                 value: Final = litellm.priority_reservation[priority]

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -2405,7 +2405,7 @@ async def generate_key_fn(
     - soft_budget: Optional[float] - Specify soft budget for a given key. Will trigger a slack alert when this soft budget is reached.
     - tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing).
     - prompts: Optional[List[str]] - List of prompts that the key is allowed to use.
-    - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/proxy/enterprise#enforce-required-params-for-llm-requests)
+    - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/enterprise#governance-and-cost)
     - prompts: Optional[List[str]] - List of prompts that the key is allowed to use.
     - allowed_routes: Optional[list] - List of allowed routes for the key. Store the actual route or store a wildcard pattern for a set of routes. Example - ["/chat/completions", "/embeddings", "/keys/*"]
     - allowed_passthrough_routes: Optional[list] - List of allowed pass through endpoints for the key. Store the actual endpoint or store a wildcard pattern for a set of endpoints. Example - ["/my-custom-endpoint"]. Use this instead of allowed_routes, if you just want to specify which pass through endpoints the key can access, without specifying the routes. If allowed_routes is specified, allowed_pass_through_endpoints is ignored.
@@ -2613,7 +2613,7 @@ async def generate_service_account_key_fn(
     - tpd_limit: Optional[int] - Specify tpd limit for a given key (Tokens per day). Charged by batch submissions instead of tpm_limit/rpm_limit.
     - soft_budget: Optional[float] - Specify soft budget for a given key. Will trigger a slack alert when this soft budget is reached.
     - tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing).
-    - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/proxy/enterprise#enforce-required-params-for-llm-requests)
+    - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/enterprise#governance-and-cost)
     - allowed_routes: Optional[list] - List of allowed routes for the key. Store the actual route or store a wildcard pattern for a set of routes. Example - ["/chat/completions", "/embeddings", "/keys/*"]
     - object_permission: Optional[LiteLLM_ObjectPermissionBase] - key-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"], "agents": ["agent_1", "agent_2"], "agent_access_groups": ["dev_group"]}. IF null or {} then no object permission.
     Examples:
@@ -4045,7 +4045,7 @@ async def update_key_fn(
     - models: Optional[list] - Model_name's a user is allowed to call
     - tags: Optional[List[str]] - Tags for organizing keys (Enterprise only)
     - prompts: Optional[List[str]] - List of prompts that the key is allowed to use.
-    - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/proxy/enterprise#enforce-required-params-for-llm-requests)
+    - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/enterprise#governance-and-cost)
     - spend: Optional[float] - Amount spent by key
     - max_budget: Optional[float] - Max budget for key
     - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets keyed by model name, `*` pattern like "claude-opus*", or model access group, e.g. {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}

litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py

@@ -3614,7 +3614,9 @@ async def deepgram_listen_websocket_route(
     if not deepgram_listen_is_priced(target):
         await websocket.close(
             code=1008,
-            reason=_DEEPGRAM_WS_UNPRICED_REASON.format(registry_key=deepgram_listen_registry_key(target)),
+            reason=truncated_close_reason(
+                _DEEPGRAM_WS_UNPRICED_REASON.format(registry_key=deepgram_listen_registry_key(target))
+            ),
         )
         return
 

litellm/proxy/proxy_cli.py

@@ -1193,7 +1193,7 @@ class ProxyInitializationHelpers:
     "--reload",
     is_flag=True,
     default=False,
-    help="Enable uvicorn hot reload (dev only). Also reloads when the --config YAML file changes. Incompatible with --num_workers>1, --run_gunicorn, and --run_hypercorn.",
+    help="Enable uvicorn hot reload (dev only), including every --num_workers worker on uvicorn>=0.50.0 outside Windows. Also reloads when the --config YAML or .env changes. Incompatible with --run_gunicorn, --run_hypercorn, and --run_granian.",
 )
 @click.option(
     "--prometheus_metrics_port",
@@ -1785,9 +1785,28 @@ def run_server(
             if num_workers > 1:
                 start_query_engine_reaper()
             if num_workers > 1 or reload:
-                uvicorn_args.update(
-                    ProxyInitializationHelpers.child_process_uvicorn_overrides(uvicorn_args.get("log_config"))
+                worker_overrides: Final = ProxyInitializationHelpers.child_process_uvicorn_overrides(
+                    uvicorn_args.get("log_config")
                 )
+                if reload and num_workers > 1:
+                    from litellm.proxy.dev_reload import (
+                        RELOAD_OPTIONS,
+                        multi_worker_reload_supported,
+                        run_multi_worker_reload,
+                    )
+
+                    if multi_worker_reload_supported():
+                        run_multi_worker_reload(
+                            RELOAD_OPTIONS.validate_python(uvicorn_args),
+                            workers=TypeAdapter(int).validate_python(num_workers),
+                            worker_overrides=worker_overrides,
+                        )
+                        return
+                    print(
+                        "\033[1;33mLiteLLM Proxy: --reload with --num_workers>1 requires uvicorn>=0.50.0 "
+                        "and an OS other than Windows, so only a single worker reloads.\033[0m"
+                    )
+                uvicorn_args.update(worker_overrides)
             uvicorn.run(
                 **uvicorn_args,
                 workers=num_workers,

litellm/proxy/proxy_server.py

@@ -392,6 +392,7 @@ from litellm.llms.anthropic.messages.convertible_blocks import (
     drop_non_string_tool_result_texts,
     malformed_tool_result_count_error,
 )
+from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import build_pool_wait_trace_config
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.openai.live import parse_live_session_start
 from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS
@@ -1209,7 +1210,7 @@ ui_message += "\n\n💸 [```LiteLLM Model Cost Map```](https://models.litellm.ai
 ui_message += f"\n\n🔎 [```LiteLLM Model Hub```]({model_hub_link}). See available models on the proxy. [**Docs**](https://docs.litellm.ai/docs/proxy/ai_hub)"
 
 custom_swagger_message: Final = (
-    "[**Customize Swagger Docs**](https://docs.litellm.ai/docs/proxy/enterprise#swagger-docs---custom-routes--branding)"
+    "[**Customize Swagger Docs**](https://docs.litellm.ai/docs/enterprise#operations--branding)"
 )
 
 ### CUSTOM BRANDING [ENTERPRISE FEATURE] ###
@@ -1460,7 +1461,11 @@ async def initialize_shared_aiohttp_session() -> "ClientSession | None":
             connector_kwargs["socket_factory"] = socket_factory
 
         connector: Final = TCPConnector(**connector_kwargs)
-        session: Final = ClientSession(connector=connector, cookie_jar=DummyCookieJar())
+        session: Final = ClientSession(
+            connector=connector,
+            cookie_jar=DummyCookieJar(),
+            trace_configs=[build_pool_wait_trace_config()],
+        )
 
         verbose_proxy_logger.info(
             "SESSION REUSE: Created shared aiohttp session for connection pooling (ID: %s, limit=%s, limit_per_host=%s)",
@@ -3070,6 +3075,29 @@ redis_usage_cache: RedisCache | None = None  # redis cache used for tracking spe
 polling_via_cache_enabled: Literal["all"] | list[str] | bool = False
 native_background_mode: list[str] = []  # Models that should use native provider background mode instead of polling
 polling_cache_ttl: int = 3600  # Default 1 hour TTL for polling cache
+
+
+class _ResponseBackgroundSettings(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    polling_via_cache: Literal["all"] | tuple[str, ...] | bool = False
+    native_background_mode: tuple[str, ...] = ()
+    ttl: int = 3600
+
+
+def _apply_background_response_settings(value: SettingsJsonValue) -> None:
+    global polling_via_cache_enabled, native_background_mode, polling_cache_ttl
+
+    responses: Final = _as_settings_mapping(value)
+    settings: Final = _ResponseBackgroundSettings.model_validate(responses.get("background_mode") or {})
+    polling: Final = settings.polling_via_cache
+    polling_via_cache_enabled, native_background_mode, polling_cache_ttl = (  # rebind-ok: Refresh runtime config.
+        list(polling) if isinstance(polling, tuple) else polling,
+        list(settings.native_background_mode),
+        settings.ttl,
+    )
+
+
 user_custom_auth = None
 user_custom_key_generate = None
 # Sentinel: prevents PKCE-no-Redis advisory from re-logging on config hot-reload.
@@ -6932,20 +6960,7 @@ class ProxyConfig:
                     # see usage here: https://docs.litellm.ai/docs/proxy/caching
                     pass
                 elif key == "responses":
-                    # Initialize global polling via cache settings
-                    global polling_via_cache_enabled, native_background_mode, polling_cache_ttl
-                    background_mode = value.get("background_mode", {})
-                    polling_via_cache_enabled = background_mode.get("polling_via_cache", False)
-                    native_background_mode = background_mode.get("native_background_mode", [])
-                    polling_cache_ttl = background_mode.get("ttl", 3600)
-                    verbose_proxy_logger.debug(
-                        "%s Initialized polling via cache: enabled=%s, native_background_mode=%s, ttl=%s%s",
-                        blue_color_code,
-                        polling_via_cache_enabled,
-                        native_background_mode,
-                        polling_cache_ttl,
-                        reset_color_code,
-                    )
+                    _apply_background_response_settings(value)
                 elif key == "max_ui_session_budget":
                     litellm.max_ui_session_budget = float(value) if value is not None else None
                     verbose_proxy_logger.debug(
@@ -8653,6 +8668,7 @@ class ProxyConfig:
 
     def _apply_litellm_settings_db_values(self, db_values: Mapping[str, SettingsJsonValue]) -> None:
         self.litellm_settings.apply_db_row("litellm_settings", db_values)
+        _apply_background_response_settings(self.litellm_settings.get("responses", {}))
         for key in LITELLM_SETTINGS_SAFE_DB_OVERRIDES:
             if key in db_values and (value := self.litellm_settings.get(key)) is not None:
                 setattr(litellm, key, value)
@@ -18110,6 +18126,8 @@ async def model_deprecations(
 def get_model_group_info(
     llm_router: Router, all_models_str: Sequence[str], model_group: str | None, team_id: str | None = None
 ) -> list[ModelGroupInfoProxy]:
+    from litellm.proxy.response_polling.polling_handler import should_use_polling_for_request
+
     model_groups: Final[list[ModelGroupInfoProxy]] = []
 
     unique_models: Final = tuple(dict.fromkeys(all_models_str))
@@ -18119,6 +18137,14 @@ def get_model_group_info(
         if model_group is not None and model_group != model:
             continue
 
+        supports_background_chat = should_use_polling_for_request(
+            background_mode=True,
+            polling_via_cache_enabled=polling_via_cache_enabled,
+            redis_cache=redis_usage_cache,
+            model=model,
+            llm_router=llm_router,
+            native_background_mode=native_background_mode,
+        )
         _model_group_info = registry.group(model, team_id=team_id)
 
         if _model_group_info is not None:
@@ -18126,6 +18152,7 @@ def get_model_group_info(
                 ModelGroupInfoProxy.model_validate(
                     {
                         **_model_group_info.model_dump(),
+                        "supports_background_chat": supports_background_chat,
                         **model_group_custom_fields(
                             registry.deployments(
                                 resolve_model_group_alias(registry.aliases(), model) or model,
@@ -18140,6 +18167,7 @@ def get_model_group_info(
             model_group_info = ModelGroupInfoProxy(
                 model_group=model,
                 providers=[],
+                supports_background_chat=supports_background_chat,
             )
             model_groups.append(model_group_info)
 

litellm/proxy/response_polling/background_streaming.py

@@ -11,6 +11,7 @@ https://platform.openai.com/docs/api-reference/responses-streaming
 import asyncio
 import json
 from collections.abc import AsyncIterable, Callable, Mapping, Sequence
+from time import perf_counter
 from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, runtime_checkable
 
 from fastapi import Request, Response
@@ -128,6 +129,16 @@ async def background_streaming_task(
     https://platform.openai.com/docs/api-reference/responses/object
     """
 
+    started_at: Final = perf_counter()
+    first_token_at: float | None = None  # rebind-ok: first content event closes the TTFT interval
+    finished_at: float | None = None  # rebind-ok: terminal event closes the execution interval
+
+    def timing_snapshot() -> Mapping[str, float]:
+        return {
+            **({"time_to_first_token_ms": (first_token_at - started_at) * 1000} if first_token_at is not None else {}),
+            **({"total_latency_ms": (finished_at - started_at) * 1000} if finished_at is not None else {}),
+        }
+
     try:
         verbose_proxy_logger.info("Starting background streaming for %s", polling_id)
 
@@ -191,7 +202,7 @@ async def background_streaming_task(
         incomplete_details_data = None
 
         state_dirty = False  # Track if state needs to be synced
-        last_update_time = asyncio.get_event_loop().time()
+        last_update_time = perf_counter()
         UPDATE_INTERVAL: Final = 0.150  # 150ms batching interval
 
         # Track the terminal event from the stream (may not be "completed")
@@ -210,13 +221,14 @@ async def background_streaming_task(
             """Flush accumulated state to Redis if interval elapsed or forced"""
             nonlocal state_dirty, last_update_time
 
-            current_time: Final = asyncio.get_event_loop().time()
+            current_time: Final = perf_counter()
             if state_dirty and (force or (current_time - last_update_time) >= UPDATE_INTERVAL):
                 # Convert output_items dict to list for update
                 output_list: Final = list(output_items.values())
                 await polling_handler.update_state(
                     polling_id=polling_id,
                     output=output_list,
+                    timing=timing_snapshot(),
                 )
                 state_dirty = False
                 last_update_time = current_time
@@ -242,6 +254,20 @@ async def background_streaming_task(
                         event: _StreamEvent = _StreamEventParser.parse(chunk_data)
                         event_type = event.get("type", "")
 
+                        if (
+                            first_token_at is None
+                            and event_type
+                            in (
+                                "response.output_text.delta",
+                                "response.reasoning_text.delta",
+                                "response.reasoning_summary_text.delta",
+                                "response.reasoning.delta",
+                            )
+                            and event.get("delta")
+                        ):
+                            first_token_at = perf_counter()
+                            state_dirty = True
+
                         # Process different event types based on OpenAI streaming spec
                         if event_type == "response.output_item.added":
                             # New output item added
@@ -334,6 +360,7 @@ async def background_streaming_task(
                         ):
                             # Terminal event - extract all ResponsesAPIResponse fields
                             # https://platform.openai.com/docs/api-reference/responses-streaming
+                            finished_at = perf_counter()
                             response_data = event.get("response", {})
                             terminal_status = response_data.get(
                                 "status",
@@ -384,10 +411,13 @@ async def background_streaming_task(
 
         # Use the terminal status from the stream, default to "completed"
         final_status: Final = terminal_status or "completed"
+        if finished_at is None:
+            finished_at = perf_counter()  # rebind-ok: close the interval when the stream has no terminal event
 
         await polling_handler.update_state(
             polling_id=polling_id,
             status=final_status,
+            timing=timing_snapshot(),
             usage=usage_data,
             error=terminal_error,
             reasoning=reasoning_data,
@@ -417,6 +447,7 @@ async def background_streaming_task(
         )
 
     except Exception as e:
+        finished_at = perf_counter()  # rebind-ok: capture execution time at failure
         verbose_proxy_logger.error("Error in background streaming task for %s: %s", polling_id, e)
         import traceback
 
@@ -425,6 +456,7 @@ async def background_streaming_task(
         await polling_handler.update_state(
             polling_id=polling_id,
             status="failed",
+            timing=timing_snapshot(),
             error={
                 "type": "internal_error",
                 "message": str(e),

litellm/proxy/response_polling/polling_handler.py

@@ -3,9 +3,9 @@ Response Polling Handler for Background Responses with Cache
 """
 
 import json
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from datetime import datetime, timezone
-from typing import Final
+from typing import TYPE_CHECKING, Final
 
 from pydantic import TypeAdapter
 
@@ -15,6 +15,9 @@ from litellm._uuid import uuid4
 from litellm.caching.redis_cache import RedisCache
 from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStatus
 
+if TYPE_CHECKING:
+    from litellm.router import Router
+
 _RESPONSE_POLLING_TARGET: Final = "response_polling"
 _JSON_OBJECT: Final = TypeAdapter(dict[str, object])
 _JSON_LIST: Final = TypeAdapter(list[object])
@@ -116,6 +119,7 @@ class ResponsePollingHandler:
         parallel_tool_calls: bool | None = None,
         user: str | None = None,
         store: bool | None = None,
+        timing: Mapping[str, float] | None = None,
     ) -> None:
         """
         Update the polling state in Redis
@@ -144,6 +148,7 @@ class ResponsePollingHandler:
             parallel_tool_calls: Whether parallel tool calls are enabled
             user: User identifier
             store: Whether to store the response
+            timing: Server execution timings in milliseconds
         """
         if not self.redis_cache:
             return
@@ -167,6 +172,9 @@ class ResponsePollingHandler:
         if output is not None:
             state["output"] = output
 
+        if timing is not None:
+            state["litellm_response_timing"] = timing
+
         # Update usage
         if usage:
             state["usage"] = usage
@@ -264,10 +272,10 @@ class ResponsePollingHandler:
 
 def should_use_polling_for_request(
     background_mode: bool,
-    polling_via_cache_enabled,  # Can be False, "all", or List[str]
-    redis_cache,  # RedisCache or None
+    polling_via_cache_enabled: bool | str | Sequence[str],
+    redis_cache: RedisCache | None,
     model: str,
-    llm_router,  # Router instance or None
+    llm_router: "Router | None",
     native_background_mode: list[str] | None = None,  # List of models that should use native background mode
 ) -> bool:
     """

litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py

@@ -216,6 +216,20 @@ class _SettingsWithSchema(LiteLLMBaseModel):
     field_schema: dict[str, object]
 
 
+class BackgroundChatSettings(LiteLLMBaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    enabled: bool = Field(description="Enable cache-backed background Responses for Chat and API requests.")
+
+
+class BackgroundChatSettingsResponse(BackgroundChatSettings):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    redis_configured: bool = Field(description="Whether this proxy has coordination Redis configured.")
+    managed_by_config: bool = Field(description="Whether the responses configuration is owned by the config file.")
+    can_save: bool = Field(description="Whether database-backed configuration updates are available.")
+
+
 class SSOSettingsResponse(SettingsResponse):
     """Response model for SSO settings"""
 
@@ -1815,6 +1829,87 @@ async def sync_ui_settings_to_general_settings(
     return apply_runtime_general_settings_flags(parsed)
 
 
+def _background_chat_settings_response(config: Mapping[str, object]) -> BackgroundChatSettingsResponse:
+    from litellm.proxy import proxy_server
+
+    settings: Final = _as_settings_section(config.get("litellm_settings"))
+    responses: Final = _as_settings_section(settings.get("responses"))
+    background: Final = _as_settings_section(responses.get("background_mode"))
+    polling: Final = background.get("polling_via_cache", False)
+    return BackgroundChatSettingsResponse(
+        enabled=bool(polling),
+        redis_configured=proxy_server.redis_usage_cache is not None,
+        managed_by_config=proxy_server.proxy_config.litellm_settings.owned_by_config("responses"),
+        can_save=proxy_server.prisma_client is not None and proxy_server.store_model_in_db is True,
+    )
+
+
+@router.get(
+    "/get/background_chat_settings",
+    tags=["Settings"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=BackgroundChatSettingsResponse,
+)
+async def get_background_chat_settings() -> BackgroundChatSettingsResponse:
+    """Get saved background-response settings and the prerequisites for changing them."""
+    from litellm.proxy.proxy_server import proxy_config
+
+    config: Final = _as_settings_section(_DECODED_JSON.validate_python(await proxy_config.get_config()))
+    return _background_chat_settings_response(config)
+
+
+@router.patch(
+    "/update/background_chat_settings",
+    tags=["Settings"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=BackgroundChatSettingsResponse,
+)
+async def update_background_chat_settings(
+    settings: BackgroundChatSettings,
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+) -> BackgroundChatSettingsResponse:
+    """Save background-response polling settings for the proxy configuration refresh."""
+    from litellm.proxy.proxy_server import (
+        create_config_audit_log,
+        prisma_client,
+        proxy_config,
+        redis_usage_cache,
+        store_model_in_db,
+    )
+
+    if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
+        raise HTTPException(status_code=403, detail="Only proxy admins can update background chat settings.")
+    if prisma_client is None or store_model_in_db is not True:
+        raise HTTPException(status_code=400, detail="Enable database-backed configuration to save this setting.")
+    if settings.enabled and redis_usage_cache is None:
+        raise HTTPException(status_code=400, detail="Configure coordination Redis and restart the proxy first.")
+
+    raw_config: Final = await proxy_config.get_config()
+    config: Final = _as_settings_section(_DECODED_JSON.validate_python(raw_config))
+    litellm_settings: Final = _as_settings_section(config.get("litellm_settings"))
+    responses: Final = _UI_SETTINGS_OBJECT.validate_python(_as_settings_section(litellm_settings.get("responses")))
+    background: Final = _UI_SETTINGS_OBJECT.validate_python(_as_settings_section(responses.get("background_mode")))
+    updated_responses: Final[Mapping[str, JsonValue]] = {
+        **responses,
+        "background_mode": {**background, "polling_via_cache": "all" if settings.enabled else False},
+    }
+    proxy_config.reject_config_owned_writes(
+        section_name="litellm_settings", changed_keys={"responses": dict(updated_responses)}
+    )
+    raw_config["litellm_settings"] = {**litellm_settings, "responses": dict(updated_responses)}
+    await proxy_config.save_config(new_config=config)
+    asyncio.create_task(
+        create_config_audit_log(
+            param_name="responses.background_mode.polling_via_cache",
+            action="updated",
+            before_value=background.get("polling_via_cache", False),
+            after_value="all" if settings.enabled else False,
+            user_api_key_dict=user_api_key_dict,
+        )
+    )
+    return _background_chat_settings_response(config)
+
+
 @router.get(
     "/get/ui_settings",
     tags=["UI Settings"],

litellm/responses/litellm_completion_transformation/count_request.py

@@ -128,6 +128,15 @@ def _with_hoisted_additional_tools(responses_request: Mapping[str, object], drop
     return {**responses_request, "input": hoisted.input, "tools": list(hoisted.tools)}
 
 
+def _caller_input(responses_request: Mapping[str, object]) -> ResponseInputParam | None:
+    responses_input: Final = responses_request.get("input")
+    return (
+        cast(ResponseInputParam, responses_input)  # cast-ok: forwarded as sent, as inference does
+        if isinstance(responses_input, list)
+        else None
+    )
+
+
 def _with_emulated_file_search(responses_request: Mapping[str, object]) -> Mapping[str, object]:
     tools: Final = responses_request.get("tools")
     if tools is None:
@@ -200,6 +209,7 @@ async def chat_count_request(
                 ),
                 custom_llm_provider,
                 policy.drop_params,
+                caller_input=_caller_input(responses_request),
             ),
             model,
             custom_llm_provider,

litellm/responses/litellm_completion_transformation/handler.py

@@ -4,6 +4,7 @@ Handler for transforming responses api requests to litellm.completion requests
 
 from collections.abc import Coroutine, Mapping, Sequence
 from dataclasses import dataclass
+from types import MappingProxyType
 from typing import Final
 
 from pydantic import TypeAdapter
@@ -32,6 +33,13 @@ from litellm.types.llms.openai import (
 from litellm.types.utils import ModelResponse
 
 _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object])
+_BRIDGE_TRANSFORM_KEYWORDS: Final = frozenset({"caller_input", "names_non_string_fields"})
+
+
+def _without_bridge_transform_keywords(request_kwargs: Mapping[str, object]) -> Mapping[str, object]:
+    return MappingProxyType(
+        {key: value for key, value in request_kwargs.items() if key not in _BRIDGE_TRANSFORM_KEYWORDS}
+    )
 
 
 def _chat_verbosity(responses_api_request: Mapping[str, object]) -> Mapping[str, object]:
@@ -95,6 +103,7 @@ class LiteLLMCompletionTransformationHandler:
                 previous_response_id,
                 hoisted,
                 model=model,
+                caller_input=input,
                 responses_api_request=responses_api_request,
                 custom_llm_provider=custom_llm_provider,
                 stream=stream,
@@ -108,11 +117,13 @@ class LiteLLMCompletionTransformationHandler:
             **LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
                 model=model,
                 input=bridged_input,
+                caller_input=input,
+                names_non_string_fields=True,
                 responses_api_request=bridged_request,
                 custom_llm_provider=custom_llm_provider,
                 stream=stream,
                 extra_headers=extra_headers,
-                **kwargs,
+                **_without_bridge_transform_keywords(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)),
             ),
             **_chat_verbosity(bridged_request),
         }
@@ -163,6 +174,7 @@ class LiteLLMCompletionTransformationHandler:
         /,
         *,
         model: str,
+        caller_input: str | ResponseInputParam,
         responses_api_request: ResponsesAPIOptionalRequestParams,
         custom_llm_provider: str | None,
         stream: bool | None,
@@ -186,11 +198,13 @@ class LiteLLMCompletionTransformationHandler:
                 **LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
                     model=model,
                     input=bridged.input,
+                    caller_input=caller_input,
+                    names_non_string_fields=True,
                     responses_api_request=bridged.responses_api_request,
                     custom_llm_provider=custom_llm_provider,
                     stream=stream,
                     extra_headers=extra_headers,
-                    **request_kwargs,
+                    **_without_bridge_transform_keywords(request_kwargs),
                 ),
                 **_chat_verbosity(bridged.responses_api_request),
             },

litellm/responses/litellm_completion_transformation/transformation.py

@@ -45,6 +45,7 @@ from litellm.constants import (
     REDACTED_BY_LITELLM,
     REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER,
 )
+from litellm.exceptions import BadRequestError
 from litellm.integrations.anthropic_cache_control_hook import (
     AnthropicCacheControlHook,
     block_with_cache_control,
@@ -65,6 +66,12 @@ from litellm.litellm_core_utils.hidden_params import (
 )
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    invalid_type_message,
+    is_present_non_string,
+    is_unhashable_value_behind,
+)
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
 from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control
 from litellm.llms.anthropic.messages.native.utils import anthropic_stop_reason, openai_chat_refusal_text
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
@@ -268,6 +275,7 @@ _IMAGE_URL_ADAPTER: Final = TypeAdapter(ChatCompletionImageUrlObject)
 _FILE_PART_ADAPTER: Final = TypeAdapter(_ResponsesFilePart)
 _MessageT = TypeVar("_MessageT")
 _MediaT: Final = TypeVar("_MediaT")
+_InputItemT: Final = TypeVar("_InputItemT")
 _CACHED_CONTENT_ADAPTER: Final = TypeAdapter(ChatCompletionCachedContent)
 _DICT_ITEMS_LIST_ADAPTER: Final = TypeAdapter(list[dict[object, object]])
 _TEXT_ADAPTER: Final = TypeAdapter(str)
@@ -445,6 +453,86 @@ TOOL_CALL_OUTPUT_ITEM_TYPES: Final = frozenset(
 )
 
 
+def _read_positions(
+    input_items: Sequence[_InputItemT], read_input: str | ResponseInputParam
+) -> Iterator[tuple[int, _InputItemT]]:
+    read_ids: Final = frozenset(map(id, parts_of(read_input)))
+    return ((index, item) for index, item in enumerate(input_items) if id(item) in read_ids)
+
+
+def _first_invalid_input_type(fields: Iterator[tuple[str, object]]) -> BadRequestError | None:
+    first: Final = next(fields, None)
+    return None if first is None else _invalid_input_type(*first)
+
+
+def _is_tool_output_item(item: object) -> bool:
+    item_type: Final = message_field(item, "type")
+    return isinstance(item_type, str) and item_type in TOOL_CALL_OUTPUT_ITEM_TYPES
+
+
+def _converts_before(
+    convert_input: Callable[[str | ResponseInputParam], object], sent_input: str | ResponseInputParam, position: int
+) -> bool:
+    try:
+        convert_input(sent_input[:position])
+    except TypeError:
+        return False
+    return True
+
+
+def _unhashable_tool_output_call_id_error(
+    input: str | ResponseInputParam,
+    read_input: str | ResponseInputParam,
+    error: TypeError,
+    converts_before: Callable[[int], bool],
+) -> BadRequestError | None:
+    call_ids: Final = (
+        (f"input[{index}].call_id", message_field(item, "call_id"), item)
+        for index, item in _read_positions(parts_of(input), read_input)
+        if _is_tool_output_item(item)
+    )
+    first: Final = next((candidate for candidate in call_ids if is_unhashable_value_behind(error, candidate[1])), None)
+    if first is None:
+        return None
+    path, call_id, item = first
+    read_position: Final = next(
+        position for position, read_item in enumerate(parts_of(read_input)) if read_item is item
+    )
+    return _invalid_input_type(path, call_id) if converts_before(read_position) else None
+
+
+def _carries_non_string_field(message: object) -> bool:
+    tool_call_ids: Final = (
+        message_field(tool_call, "id") for tool_call in parts_of(message_field(message, "tool_calls"))
+    )
+    return is_present_non_string(message_field(message, "role")) or any(map(is_present_non_string, tool_call_ids))
+
+
+def _non_string_item_fields(
+    item: Mapping[str, object], convert_item: Callable[[Mapping[str, object]], Sequence[object]]
+) -> Iterator[tuple[str, object]]:
+    messages: Final = convert_item(item)
+    tool_call_id_field: Final = "call_id" if item.get("call_id") else "id"
+    roles: Final = (("role", message_field(message, "role")) for message in messages)
+    tool_calls: Final = chain.from_iterable(parts_of(message_field(message, "tool_calls")) for message in messages)
+    tool_call_ids: Final = ((tool_call_id_field, message_field(tool_call, "id")) for tool_call in tool_calls)
+    return ((field, value) for field, value in chain(roles, tool_call_ids) if is_present_non_string(value))
+
+
+def _indexed_non_string_item_fields(
+    input: ResponseInputParam,
+    read_input: str | ResponseInputParam,
+    convert_item: Callable[[Mapping[str, object]], Sequence[object]],
+) -> Iterator[tuple[str, object]]:
+    for index, item in _read_positions(input, read_input):
+        for field, value in _non_string_item_fields(item, convert_item):
+            yield f"input[{index}].{field}", value
+
+
+def _invalid_input_type(field_path: str, value: object) -> BadRequestError:
+    return BadRequestError(message=invalid_type_message(field_path, value), model="", llm_provider="")
+
+
 def _input_item_role(input_item: Mapping[str, object]) -> str:
     return cast(str, input_item.get("role") or "user")  # cast-ok: client-supplied role forwarded verbatim, unvalidated
 
@@ -1101,6 +1189,9 @@ class LiteLLMCompletionResponsesConfig:
         stream: bool | None = None,
         extra_headers: Mapping[str, object] | None = None,
         drop_params: object = None,
+        *,
+        caller_input: str | ResponseInputParam | None = None,
+        names_non_string_fields: bool = False,
         **kwargs,
     ) -> dict:
         """
@@ -1149,6 +1240,7 @@ class LiteLLMCompletionResponsesConfig:
         native_compaction: Final = _replays_compaction_block(
             request_input, context_management
         ) and provider_takes_compaction_blocks(model, custom_llm_provider)
+        numbered_input: Final = request_input if caller_input is None else caller_input
         bridged_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
             input=request_input,
             responses_api_request=responses_api_request,
@@ -1157,7 +1249,28 @@ class LiteLLMCompletionResponsesConfig:
             tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs),
             native_compaction=native_compaction,
             only_since_compaction=not native_compaction,
+            caller_input=numbered_input,
         )
+        malformed_field_error: Final = (
+            _first_invalid_input_type(
+                _indexed_non_string_item_fields(
+                    numbered_input,
+                    _input_to_read(request_input, not native_compaction, native_compaction),
+                    partial(
+                        LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message,
+                        replay_reasoning=True,
+                        native_compaction=native_compaction,
+                    ),
+                )
+            )
+            if names_non_string_fields
+            and isinstance(numbered_input, list)
+            and any(map(_carries_non_string_field, bridged_messages))
+            and provider_builds_anthropic_request(model, custom_llm_provider)
+            else None
+        )
+        if malformed_field_error is not None:
+            raise malformed_field_error
 
         litellm_completion_request: dict = {
             "messages": (
@@ -1221,6 +1334,7 @@ class LiteLLMCompletionResponsesConfig:
         tool_call_provider_fields: bool = False,
         native_compaction: bool = False,
         only_since_compaction: bool = False,
+        caller_input: str | ResponseInputParam | None = None,
     ) -> list[
         AllMessageValues
         | GenericChatCompletionMessage
@@ -1274,13 +1388,27 @@ class LiteLLMCompletionResponsesConfig:
 
         dumped_input: Final = with_compaction_items_as_params(input)
         read_input: Final = _input_to_read(dumped_input, only_since_compaction, native_compaction)
-        messages.extend(
-            LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message(
-                input=_without_item_keys(read_input, _dropped_item_keys(item_cache_control, tool_call_provider_fields)),
-                replay_reasoning=replay_reasoning,
-                native_compaction=native_compaction,
-            )
+        sent_input: Final = _without_item_keys(
+            read_input, _dropped_item_keys(item_cache_control, tool_call_provider_fields)
         )
+        convert_input: Final = partial(
+            LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message,
+            replay_reasoning=replay_reasoning,
+            native_compaction=native_compaction,
+        )
+        try:
+            converted: Final = convert_input(sent_input)
+        except TypeError as error:
+            call_id_error: Final = _unhashable_tool_output_call_id_error(
+                dumped_input if caller_input is None else caller_input,
+                read_input,
+                error,
+                partial(_converts_before, convert_input, sent_input),
+            )
+            if call_id_error is None:
+                raise
+            raise call_id_error from error
+        messages.extend(converted)
 
         return messages
 

litellm/router_strategy/auto_router/litellm_encoder.py

@@ -14,13 +14,13 @@ else:
     RoutingContext = Any
 
 
-def litellm_to_list(embeds: litellm.EmbeddingResponse) -> list[list[float]]:
+def litellm_to_list(embeds: object) -> list[list[float]]:
     """Convert a LiteLLM embedding response to a list of embeddings.
 
     :param embeds: The LiteLLM embedding response.
     :return: A list of embeddings.
     """
-    if not embeds or not embeds.data:
+    if not embeds or not isinstance(embeds, litellm.EmbeddingResponse) or not embeds.data:
         raise ValueError("No embeddings found in LiteLLM embedding response.")
     return [x["embedding"] for x in embeds.data]
 

litellm/router_strategy/complexity_router/jev_classifier.py

@@ -68,6 +68,7 @@ class JevBudgetedRequest:
     telemetry: StandardLoggingRoutingDecision
 
 
+@renamed_router_param("router", "context")
 def jev_input_targets(context: "RoutingContext", config: "OpenSourceClassifierConfig") -> tuple[JevInputTarget, ...]:
     from litellm.types.router import Deployment
 

litellm/router_strategy/tag_based_routing.py

@@ -392,9 +392,11 @@ def _resolve_constraint_only_pool(
 
 def _all_deployments_or_fallback(
     ctx: "RoutingContext",
-    model: str,
+    model: str | None,
     fallback: _DeploymentPool,
 ) -> Sequence[_DeploymentLike | DeploymentTypedDict] | Mapping[_DeploymentLike, object]:
+    if model is None:  # a group of no name has no deployments, it is not a failed lookup
+        return ()
     try:
         return ctx.registry().deployments(model, source="all")
     except Exception:  # noqa: BLE001  # fail safe toward today's healthy-only behavior on lookup errors

litellm/router_utils/handle_error.py

@@ -23,7 +23,7 @@ else:
 
 @renamed_router_param("litellm_router_instance")
 async def send_llm_exception_alert(
-    ctx: "RoutingContext",
+    ctx: "RoutingContext | None",
     request_kwargs: dict,
     error_traceback_str: str,
     original_exception,
@@ -39,6 +39,9 @@ async def send_llm_exception_alert(
     Returns:
         None
     """
+    if ctx is None:
+        return
+
     slack_alerting_logger: Final = ctx.observers().slack_alerting_logger
     if slack_alerting_logger is None:
         return

litellm/types/guardrails.py

@@ -1,10 +1,10 @@
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from datetime import datetime
 from enum import Enum
 from types import MappingProxyType
 from typing import Annotated, Final, Literal, TypeAlias, cast
 
-from pydantic import ConfigDict, Field, field_validator, model_validator
+from pydantic import ConfigDict, Field, JsonValue, field_validator, model_validator
 from pydantic.fields import FieldInfo
 from typing_extensions import ReadOnly, Required, TypedDict
 
@@ -604,6 +604,11 @@ class BedrockChecksConfigModel(LiteLLMBaseModel):
         return self
 
 
+class BedrockGuardrailOutputScope(Enum):
+    INTERVENTIONS = "INTERVENTIONS"
+    FULL = "FULL"
+
+
 class BedrockGuardrailConfigModel(LiteLLMBaseModel):
     """Configuration parameters for the AWS Bedrock guardrail"""
 
@@ -612,6 +617,11 @@ class BedrockGuardrailConfigModel(LiteLLMBaseModel):
         default=None,
         description="The version of your Bedrock guardrail (e.g., DRAFT or version number)",
     )
+    outputScope: BedrockGuardrailOutputScope | None = Field(
+        default=None,
+        description="ApplyGuardrail response scope. FULL includes detected and non-detected findings. "
+        "Omit to retain Bedrock's default of returning detected findings only.",
+    )
     disable_exception_on_block: bool | None = Field(
         default=False,
         description="If True, will not raise an exception when the guardrail is blocked. Useful for OpenWebUI where exceptions can end the chat flow.",
@@ -1540,6 +1550,14 @@ class PresidioPerRequestConfig(LiteLLMBaseModel):
     entities: list[PiiEntityType] | None = None
 
 
+NativeGuardrailResponse: TypeAlias = Mapping[str, JsonValue] | Sequence[JsonValue]
+
+
+class GuardrailResponseFormat(Enum):
+    STANDARD = "standard"
+    NATIVE = "native"
+
+
 class ApplyGuardrailRequest(LiteLLMBaseModel):
     guardrail_name: str
     text: str
@@ -1548,6 +1566,7 @@ class ApplyGuardrailRequest(LiteLLMBaseModel):
     input_type: str = "request"
     messages: list[dict[str, object]] | None = None
     metadata: dict[str, object] | None = None
+    response_format: GuardrailResponseFormat = GuardrailResponseFormat.STANDARD
 
 
 class ApplyGuardrailResponse(LiteLLMBaseModel):

litellm/types/llms/cohere.py

@@ -1,5 +1,6 @@
 from typing import Literal
 
+from pydantic import AliasPath, BaseModel, ConfigDict, Field
 from typing_extensions import Required, TypedDict
 
 
@@ -98,3 +99,15 @@ class CohereV2ChatResponse(TypedDict):
     message: CohereV2ChatResponseMessage
     usage: CohereV2ChatResponseUsage
     logprobs: CohereV2ChatResponseLogProbs
+
+
+class CohereV2StreamedToolCall(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    id: str | None = Field(default=None, validation_alias=AliasPath("delta", "message", "tool_calls", "id"))
+    name: str | None = Field(
+        default=None, validation_alias=AliasPath("delta", "message", "tool_calls", "function", "name")
+    )
+    arguments: str = Field(
+        default="", validation_alias=AliasPath("delta", "message", "tool_calls", "function", "arguments")
+    )

litellm/types/proxy/guardrails/guardrail_hooks/ovalix.py

@@ -28,6 +28,10 @@ class OvalixGuardrailConfigModel(GuardrailConfigModel):
         default=None,
         description="Post-checkpoint ID for the Ovalix Tracker service.",
     )
+    file_checkpoint_id: str | None = Field(
+        default=None,
+        description="File-checkpoint ID for the Ovalix Tracker service (falls back to pre/post).",
+    )
 
     @staticmethod
     def ui_friendly_name() -> str:

litellm/types/proxy/management_endpoints/model_management_endpoints.py

@@ -35,6 +35,9 @@ class PatchModelRequest(updateDeployment):
 
 
 class ModelGroupInfoProxy(ModelGroupInfo):
+    supports_background_chat: bool = Field(
+        default=False, description="Whether cache-backed background Responses runs are enabled for this model group"
+    )
     is_public_model_group: bool = Field(default=False)
     health_status: str | None = Field(default=None)
     health_response_time: float | None = Field(default=None)

litellm/types/utils.py

@@ -1151,6 +1151,8 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = {
     # A2A (Agent-to-Agent)
     "/a2a/{agent_id}": [CallTypes.asend_message, CallTypes.send_message],
     "/a2a/{agent_id}/message/send": [CallTypes.asend_message, CallTypes.send_message],
+    "/a2a/{agent_id}/message/stream": [CallTypes.asend_message, CallTypes.send_message],
+    "/v1/a2a/{agent_id}/message/stream": [CallTypes.asend_message, CallTypes.send_message],
     # Passthrough endpoints
     "/llm_passthrough": [
         CallTypes.allm_passthrough_route,

model_prices_and_context_window.json

@@ -15555,18 +15555,20 @@
         "input_cost_per_token": 2.65e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8000,
-        "max_output_tokens": 2048,
+        "max_output_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 3.5e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md",
         "supports_pdf_input": true
     },
     "bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": {
         "input_cost_per_token": 3e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8000,
-        "max_output_tokens": 2048,
+        "max_output_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 6e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md",
         "supports_pdf_input": true
     },
     "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": {
@@ -48955,6 +48957,7 @@
     },
     "together_ai/Qwen/Qwen3.8-2.4T-A95B": {
         "cache_read_input_token_cost": 2.5e-07,
+        "deprecation_date": "2026-11-04",
         "input_cost_per_token": 2e-06,
         "litellm_provider": "together_ai",
         "max_input_tokens": 1010000,

tests/base_sdk_tests/check_sdk_http.py

@@ -70,7 +70,9 @@ def enqueue(body: object, status: int = 200) -> None:
 
 def enqueue_stream() -> None:
     chunk: Final = {**CHAT, "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": {"content": "pong"}}]}
-    RESPONSES.put((200, f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n".encode()))
+    terminal: Final = {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}
+    stream: Final = f"data: {json.dumps(chunk)}\n\ndata: {json.dumps(terminal)}\n\ndata: [DONE]\n\n"
+    RESPONSES.put((200, stream.encode()))
 
 
 def bedrock_event(event: str, payload: object) -> bytes:

tests/e2e/CONTRIBUTING.md

@@ -90,7 +90,7 @@ The suites run against a live proxy, so bring one up first by running the litell
 
    They also need a proxy whose bundled UI contains the change under test, so run the proxy from your branch (an editable install serves the UI your checkout builds)
 
-Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy. The presidio guardrail tests need a running Presidio analyzer and anonymizer the proxy can reach, addressed by `PRESIDIO_ANALYZER_API_BASE` / `PRESIDIO_ANONYMIZER_API_BASE`
+Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and start the proxy with `LITELLM_OTEL_V2=true`, the `otel` callback, and `OTEL_EXPORTER_OTLP_ENDPOINT` pointing at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy. The presidio guardrail tests need a running Presidio analyzer and anonymizer the proxy can reach, addressed by `PRESIDIO_ANALYZER_API_BASE` / `PRESIDIO_ANONYMIZER_API_BASE`
 
 The Presidio spend-log audit test also requires prompt storage on the proxy, because its assertions inspect the detected entities and their scores. Add the following to the proxy config before starting it:
 

tests/e2e/gateway/stage_mirror_ci_config.yml

@@ -32,7 +32,7 @@ litellm_settings:
       - host: 127.0.0.1
         port: 6379
     ssl: true
-  callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel", "websearch_interception"]
+  callbacks: ["datadog", "smtp_email", "prometheus", "otel", "websearch_interception"]
   websearch_interception_params:
     enabled_providers: ["bedrock"]
     search_tool_name: e2e-search

tests/e2e/logging/test_otel_trace_e2e.py

@@ -54,7 +54,7 @@ def _assert_otel_destination_configured(client: LoggingClient) -> None:
     details = _ReadinessDetails.model_validate_json(readiness_details_body(client))
     assert OTEL_V2_LOGGER_NAME in details.success_callbacks, (
         f"the proxy must report the {OTEL_V2_LOGGER_NAME} callback active "
-        f"(LITELLM_OTEL_V2 + arize_phoenix preset in the compose config); got: {details.success_callbacks}"
+        f"(LITELLM_OTEL_V2=true + the otel callback in the stack config); got: {details.success_callbacks}"
     )
 
 

tests/integration/_support/a2a_denial_probe.py

@@ -0,0 +1,145 @@
+import json
+import os
+from collections.abc import Mapping
+from datetime import datetime
+from typing import Final
+
+import httpx
+from fastapi import HTTPException
+
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.proxy._types import UserAPIKeyAuth
+
+
+def _mapping(value: object) -> Mapping[str, object]:
+    return value if isinstance(value, Mapping) else {}
+
+
+def _logged(hook: str, kwargs: Mapping[str, object]) -> Mapping[str, object]:
+    logged: Final = _mapping(kwargs.get("standard_logging_object"))
+    metadata: Final = _mapping(logged.get("metadata"))
+    return {
+        "hook": hook,
+        "call_type": kwargs.get("call_type"),
+        "litellm_call_id": kwargs.get("litellm_call_id"),
+        "model": logged.get("model"),
+        "status": logged.get("status"),
+        "key_alias": metadata.get("user_api_key_alias"),
+        "key_hash": metadata.get("user_api_key_hash"),
+        "user_id": metadata.get("user_api_key_user_id"),
+        "team_id": metadata.get("user_api_key_team_id"),
+    }
+
+
+def _status(exception: Exception) -> int | None:
+    status: Final = getattr(exception, "status_code", None)
+    if isinstance(status, int):
+        return status
+    code: Final = getattr(exception, "code", None)
+    return int(code) if isinstance(code, str) and code.isdigit() else None
+
+
+def _hooked(
+    request_data: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth, original_exception: Exception
+) -> Mapping[str, object]:
+    return {
+        "hook": "post_call_failure",
+        "call_type": request_data.get("call_type"),
+        "litellm_call_id": request_data.get("litellm_call_id"),
+        "key_alias": user_api_key_dict.key_alias,
+        "key_hash": user_api_key_dict.api_key,
+        "user_id": user_api_key_dict.user_id,
+        "team_id": user_api_key_dict.team_id,
+        "agent_id": request_data.get("agent_id"),
+        "metadata_agent_id": _mapping(request_data.get("metadata")).get("agent_id"),
+        "status_code": _status(original_exception),
+    }
+
+
+def _send(record: Mapping[str, object]) -> None:
+    with httpx.Client(trust_env=False) as client:
+        client.post(os.environ["A2A_DENIAL_PROBE_URL"], content=json.dumps(record, default=str)).raise_for_status()
+
+
+async def _send_async(record: Mapping[str, object]) -> None:
+    async with httpx.AsyncClient(trust_env=False) as client:
+        response: Final = await client.post(os.environ["A2A_DENIAL_PROBE_URL"], content=json.dumps(record, default=str))
+        response.raise_for_status()
+
+
+class A2ADenialProbe(CustomLogger):
+    """Reports the identity, target and outcome every logging and failure hook receives, one record per call."""
+
+    async def async_log_success_event(
+        self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
+    ) -> None:
+        await _send_async(_logged("async_success", kwargs))
+
+    def log_failure_event(
+        self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
+    ) -> None:
+        _send(_logged("sync_failure", kwargs))
+
+    async def async_log_failure_event(
+        self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
+    ) -> None:
+        await _send_async(_logged("async_failure", kwargs))
+
+    async def async_post_call_failure_hook(
+        self,
+        request_data: dict[str, object],
+        original_exception: Exception,
+        user_api_key_dict: UserAPIKeyAuth,
+        traceback_str: str | None = None,
+    ) -> None:
+        await _send_async(_hooked(request_data, user_api_key_dict, original_exception))
+
+
+class RaisingFailureLogger(CustomLogger):
+    """Fails inside both failure hooks."""
+
+    async def async_log_failure_event(
+        self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
+    ) -> None:
+        raise RuntimeError("failure logger is broken")
+
+    async def async_post_call_failure_hook(
+        self,
+        request_data: dict[str, object],
+        original_exception: Exception,
+        user_api_key_dict: UserAPIKeyAuth,
+        traceback_str: str | None = None,
+    ) -> None:
+        raise RuntimeError("failure hook is broken")
+
+
+class ReturningTeapotLogger(CustomLogger):
+    """Hands a replacement 418 back from the failure hook."""
+
+    async def async_post_call_failure_hook(
+        self,
+        request_data: dict[str, object],
+        original_exception: Exception,
+        user_api_key_dict: UserAPIKeyAuth,
+        traceback_str: str | None = None,
+    ) -> HTTPException:
+        return HTTPException(status_code=418, detail="transformed")
+
+
+class RaisingTeapotLogger(CustomLogger):
+    """Raises a replacement 418 from the failure hook."""
+
+    async def async_post_call_failure_hook(
+        self,
+        request_data: dict[str, object],
+        original_exception: Exception,
+        user_api_key_dict: UserAPIKeyAuth,
+        traceback_str: str | None = None,
+    ) -> None:
+        raise HTTPException(status_code=418, detail="transformed")
+
+
+probe: Final = A2ADenialProbe()
+raising_failure_logger: Final = RaisingFailureLogger()
+returning_teapot_logger: Final = ReturningTeapotLogger()
+raising_teapot_logger: Final = RaisingTeapotLogger()

tests/integration/_support/mcp.py

@@ -10,6 +10,7 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Final, Literal
 
+import anyio
 import httpx
 from integration._support.asgi import asgi_server
 from integration._support.client import Gateway, Scenario
@@ -63,7 +64,9 @@ class Confirmation(BaseModel):
     confirmed: bool
 
 
-def math_service(name: str = "integration-math", *, rich: bool = False) -> MCPServer:
+def math_service(
+    name: str = "integration-math", *, rich: bool = False, on_slow: Callable[[], None] | None = None
+) -> MCPServer:
     service: Final = MCPServer(name)
     service.add_tool(add)
     service.add_tool(multiply)
@@ -77,6 +80,8 @@ def math_service(name: str = "integration-math", *, rich: bool = False) -> MCPSe
 
     @service.tool()
     async def slow(seconds: float) -> str:
+        if on_slow is not None:
+            on_slow()
         await asyncio.sleep(seconds)
         return "slept"
 
@@ -114,6 +119,29 @@ def math_service(name: str = "integration-math", *, rich: bool = False) -> MCPSe
     return service
 
 
+def _cancel_on_disconnect(app: ASGIApp) -> ASGIApp:
+    async def connected(scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] != "http":
+            await app(scope, receive, send)
+            return
+        sender, reader = anyio.create_memory_object_stream[Message]()
+        async with sender, reader, anyio.create_task_group() as tasks:
+
+            async def relay() -> None:
+                while True:
+                    message: Final = await receive()
+                    if message["type"] == "http.disconnect":
+                        tasks.cancel_scope.cancel()
+                        return
+                    await sender.send(message)
+
+            tasks.start_soon(relay)
+            await app(scope, reader.receive, send)
+            tasks.cancel_scope.cancel()
+
+    return connected
+
+
 def _capturing(app: Callable[[Scope, Receive, Send], object], observed: queue.Queue[dict[str, object]]):
     async def capture(scope: Scope, receive: Receive, send: Send) -> None:
         if scope["type"] != "http":
@@ -247,8 +275,10 @@ def _draining_sse_watcher(app: Callable[[Scope, Receive, Send], object]):
 
 
 @contextmanager
-def mcp_peer(transport: Literal["http", "sse"] = "http", *, rich: bool = False) -> Iterator[McpPeer]:
-    service: Final = math_service(rich=rich)
+def mcp_peer(
+    transport: Literal["http", "sse"] = "http", *, rich: bool = False, on_slow: Callable[[], None] | None = None
+) -> Iterator[McpPeer]:
+    service: Final = math_service(rich=rich, on_slow=on_slow)
     security: Final = TransportSecuritySettings(enable_dns_rebinding_protection=False)
     app: Final = (
         _draining_sse_watcher(service.sse_app(transport_security=security))
@@ -256,7 +286,11 @@ def mcp_peer(transport: Literal["http", "sse"] = "http", *, rich: bool = False)
         else service.streamable_http_app(stateless_http=True, json_response=True, transport_security=security)
     )
     observed: Final[queue.Queue[dict[str, object]]] = queue.Queue()
-    with asgi_server(_capturing(app, observed), before_stop=_drain_sse_streams if transport == "sse" else None) as url:
+    captured: Final = _capturing(app, observed)
+    with asgi_server(
+        _cancel_on_disconnect(captured) if transport == "http" else captured,
+        before_stop=_drain_sse_streams if transport == "sse" else None,
+    ) as url:
         yield McpPeer(url + ("/sse" if transport == "sse" else "/mcp"), observed, transport)
 
 

tests/integration/authorization/test_agent_access.py

@@ -66,6 +66,13 @@ def _peer_card(url: str, name: str) -> dict[str, JsonValue]:
         "description": "Synthetic peer",
         "version": "1.0.0",
         "url": url + "/",
+        "supportedInterfaces": [
+            {
+                "url": url + "/",
+                "protocolBinding": "JSONRPC",
+                "protocolVersion": "0.3",
+            }
+        ],
         "capabilities": {"streaming": True},
         "defaultInputModes": ["text"],
         "defaultOutputModes": ["text"],
@@ -73,10 +80,15 @@ def _peer_card(url: str, name: str) -> dict[str, JsonValue]:
     }
 
 
-def _register_agent(gateway: Gateway, scenario: Scenario, name: str, url: str) -> str:
-    created: Final = gateway.request(
-        "POST", "/v1/agents", {"agent_name": name, "agent_card_params": _peer_card(url, name)}
-    )
+def _register_agent(
+    gateway: Gateway,
+    scenario: Scenario,
+    name: str,
+    url: str,
+    agent_card: dict[str, JsonValue] | None = None,
+) -> str:
+    card: Final = _peer_card(url, name) if agent_card is None else agent_card
+    created: Final = gateway.request("POST", "/v1/agents", {"agent_name": name, "agent_card_params": card})
     assert created.status_code == 200, created.text
     identity: Final = created.json()["agent_id"]
 
@@ -416,3 +428,68 @@ def test_legacy_agent_json_card_path_applies_agent_permissions_to_non_admin_keys
         )
         assert (denied_legacy.status_code, denied_legacy.text) == (403, refusal(denied)), denied_legacy.text
         assert wire.drain() == ()
+
+
+def test_agent_list_and_detail_cards_use_proxy_urls_for_scoped_keys(gateway: Gateway) -> None:
+    marker: Final = "a2acardurl" + uuid.uuid4().hex[:12]
+
+    def upstream(request: Request) -> Reply:
+        assert request.method == "GET", request.method
+        return Reply(body=json.dumps(_peer_card(wire.url, marker)).encode())
+
+    with wire_server(upstream) as wire, gateway.scenario() as scenario:
+        backend_url: Final = wire.url + "/"
+        peer_card: Final = _peer_card(wire.url, marker)
+        peer_interfaces: Final = peer_card["supportedInterfaces"]
+        assert isinstance(peer_interfaces, list)
+        agent_card: Final = {
+            **peer_card,
+            "additionalInterfaces": [{"url": backend_url, "transport": "GRPC"}],
+            "supportedInterfaces": [
+                *peer_interfaces,
+                {
+                    "url": backend_url,
+                    "protocolBinding": "GRPC",
+                    "protocolVersion": "0.3",
+                },
+            ],
+        }
+        agent_id: Final = _register_agent(gateway, scenario, marker, wire.url, agent_card)
+        user_id: Final = scenario.user(user_role="internal_user")
+        scoped_key: Final = scenario.key(user_id=user_id, object_permission={"agents": [agent_id]})
+
+        public_card: Final = gateway.request(
+            "GET",
+            f"/a2a/{agent_id}/.well-known/agent-card.json",
+            key=scoped_key,
+        )
+        listed: Final = gateway.request("GET", "/v1/agents", key=scoped_key)
+        detailed: Final = gateway.request("GET", f"/v1/agents/{agent_id}", key=scoped_key)
+
+        assert public_card.status_code == 200, public_card.text
+        assert listed.status_code == 200, listed.text
+        assert detailed.status_code == 200, detailed.text
+        assert wire.url not in listed.text
+        assert wire.url not in detailed.text
+        assert "additionalInterfaces" not in listed.text
+        assert "additionalInterfaces" not in detailed.text
+        public_urls: Final = (
+            public_card.json()["url"],
+            public_card.json()["supportedInterfaces"][0]["url"],
+        )
+        listed_agent: Final = next(item for item in listed.json() if item["agent_id"] == agent_id)
+        for body in (listed_agent, detailed.json()):
+            card: Final = body["agent_card_params"]
+            assert (card["url"], card["supportedInterfaces"][0]["url"]) == public_urls
+            assert card["supportedInterfaces"] == [
+                {
+                    "url": public_urls[1],
+                    "protocolBinding": "JSONRPC",
+                    "protocolVersion": "0.3",
+                }
+            ]
+
+        admin_listed: Final = gateway.request("GET", "/v1/agents")
+        assert admin_listed.status_code == 200, admin_listed.text
+        admin_agent: Final = next(item for item in admin_listed.json() if item["agent_id"] == agent_id)
+        assert admin_agent["agent_card_params"]["url"] == backend_url

tests/integration/authorization/test_agent_denial_logging.py

@@ -0,0 +1,809 @@
+import hashlib
+import json
+import uuid
+from collections.abc import Iterator, Mapping
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from types import MappingProxyType
+from typing import Final
+
+import httpx
+import pytest
+import yaml
+from integration._support.client import (
+    JSON_OBJECT,
+    Gateway,
+    Scenario,
+    eventually,
+    gateway_from_environment,
+    held,
+    object_value,
+    string_value,
+)
+from integration._support.process import owned_proxy
+from integration._support.wire import Reply, Request, Wire, wire_server
+from pydantic import BaseModel, ConfigDict, JsonValue
+
+pytestmark = pytest.mark.timeout(900)
+
+_PROBE: Final = "integration._support.a2a_denial_probe."
+_SETTLE: Final = 3.0
+_CONVERGE: Final = 30.0
+_NO_SETTINGS: Final[Mapping[str, JsonValue]] = MappingProxyType({})
+_ROWS: Final = (
+    'SELECT request_id, call_type, status, model, agent_id, spend, api_key, "user", team_id, metadata '
+    'FROM "LiteLLM_SpendLogs" WHERE api_key = %s'
+)
+_ONE_FAILURE: Final = ("async_failure", "post_call_failure")
+_STREAM_METHODS: Final = frozenset({"message/stream", "SendStreamingMessage"})
+
+
+class _Event(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    hook: str
+    call_type: str | None = None
+    litellm_call_id: str | None = None
+    model: str | None = None
+    status: str | None = None
+    key_alias: str | None = None
+    key_hash: str | None = None
+    user_id: str | None = None
+    team_id: str | None = None
+    agent_id: str | None = None
+    metadata_agent_id: str | None = None
+    status_code: int | None = None
+
+
+class _Events:
+    def __init__(self, sink: Wire) -> None:
+        self.sink: Final = sink
+        self.seen: tuple[_Event, ...] = ()
+
+    def of(self, alias: str) -> tuple[_Event, ...]:
+        self.seen = (*self.seen, *(_Event.model_validate_json(request.body) for request in self.sink.drain()))
+        return tuple(event for event in self.seen if event.key_alias == alias)
+
+
+@dataclass(frozen=True, slots=True)
+class _Agent:
+    agent_id: str
+    name: str
+
+
+@dataclass(frozen=True, slots=True)
+class _Rig:
+    proxy: Gateway
+    events: _Events
+    upstream: Wire
+    allowed: _Agent
+    denied: _Agent
+    denied_legacy: _Agent
+    bridge: _Agent
+
+
+@dataclass(frozen=True, slots=True)
+class _Caller:
+    key: str
+    alias: str
+    user_id: str
+    team_id: str | None
+
+    @property
+    def hashed(self) -> str:
+        return hashlib.sha256(self.key.encode()).hexdigest()
+
+
+def _compact(content: Mapping[str, JsonValue]) -> bytes:
+    return json.dumps(content, separators=(",", ":")).encode()
+
+
+def _refusal(agent: str, response: httpx.Response | None = None) -> bytes:
+    message: Final = f"Agent '{agent}' is not allowed for your key/team. Contact proxy admin for access."
+    if response is None:
+        return _compact({"detail": message})
+    return _compact(
+        {
+            "jsonrpc": "2.0",
+            "id": JSON_OBJECT.validate_json(response.request.content)["id"],
+            "error": {
+                "code": -32000,
+                "message": message,
+                "data": {"type": "permission_error", "param": None, "code": "403"},
+            },
+        }
+    )
+
+
+def _not_found(request_id: JsonValue, agent: str) -> bytes:
+    return _compact(
+        {"jsonrpc": "2.0", "id": request_id, "error": {"code": -32000, "message": f"Agent '{agent}' not found"}}
+    )
+
+
+def _card(url: str, name: str, version: str) -> dict[str, JsonValue]:
+    card: Final[dict[str, JsonValue]] = {
+        "protocolVersion": version,
+        "name": name,
+        "description": "Synthetic peer",
+        "version": "1.0.0",
+        "url": url + "/",
+        "capabilities": {"streaming": True},
+        "defaultInputModes": ["text"],
+        "defaultOutputModes": ["text"],
+        "skills": [],
+    }
+    if version == "0.3":
+        return card
+    return {**card, "supportedInterfaces": [{"url": url + "/", "protocolBinding": "JSONRPC", "protocolVersion": "1.0"}]}
+
+
+def _peer(request: Request) -> Reply:
+    if request.method == "GET":
+        return Reply(body=_compact(_card("http://" + request.headers["host"], "peer", "0.3")))
+    body: Final = JSON_OBJECT.validate_json(request.body)
+    if body.get("method") == "message/stream":
+        frame: Final = {
+            "jsonrpc": "2.0",
+            "id": body["id"],
+            "result": {
+                "kind": "status-update",
+                "taskId": "t1",
+                "contextId": "c1",
+                "status": {"state": "completed"},
+                "final": True,
+            },
+        }
+        return Reply(content_type="text/event-stream", chunks=(f"data: {json.dumps(frame)}\n\n".encode(),))
+    return Reply(
+        body=_compact(
+            {
+                "jsonrpc": "2.0",
+                "id": body.get("id"),
+                "result": {
+                    "kind": "message",
+                    "role": "agent",
+                    "messageId": "peer-message",
+                    "parts": [{"kind": "text", "text": "granted"}],
+                },
+            }
+        )
+    )
+
+
+def _register(proxy: Gateway, scenario: Scenario, body: Mapping[str, JsonValue]) -> _Agent:
+    created: Final = proxy.request("POST", "/v1/agents", body)
+    assert created.status_code == 200, created.text
+    agent: Final = _Agent(string_value(JSON_OBJECT.validate_json(created.content)["agent_id"]), str(body["agent_name"]))
+
+    def cleanup() -> None:
+        deleted: Final = proxy.request("DELETE", f"/v1/agents/{agent.agent_id}")
+        assert deleted.status_code == 200, deleted.text
+
+    scenario.cleanups.callback(cleanup)
+    return agent
+
+
+def _config(directory: Path, callbacks: tuple[str, ...], general: Mapping[str, JsonValue]) -> Path:
+    base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
+    config: Final = {
+        **base,
+        "general_settings": {**object_value(base["general_settings"]), **general},
+        "litellm_settings": {
+            **object_value(base["litellm_settings"]),
+            "callbacks": [_PROBE + "probe", *(_PROBE + name for name in callbacks)],
+        },
+    }
+    path: Final = directory / "a2a-denials.yaml"
+    path.write_text(yaml.safe_dump(config))
+    return path
+
+
+@contextmanager
+def _owned_rig(
+    directory: Path,
+    callbacks: tuple[str, ...] = (),
+    general: Mapping[str, JsonValue] = _NO_SETTINGS,
+    environment: Mapping[str, str] = MappingProxyType({}),
+) -> Iterator[_Rig]:
+    marker: Final = "a2adeny" + uuid.uuid4().hex[:10]
+    with (
+        wire_server(_peer) as upstream,
+        wire_server(lambda request: Reply()) as sink,
+        gateway_from_environment() as shared,
+        owned_proxy(
+            shared,
+            directory,
+            {"A2A_DENIAL_PROBE_URL": sink.url, **environment},
+            config=_config(directory, callbacks, general),
+        ) as proxy,
+        proxy.scenario() as scenario,
+    ):
+
+        def agent(suffix: str, version: str, **extra: JsonValue) -> _Agent:
+            name: Final = f"{marker}-{suffix}"
+            return _register(
+                proxy, scenario, {"agent_name": name, "agent_card_params": _card(upstream.url, name, version), **extra}
+            )
+
+        yield _Rig(
+            proxy,
+            _Events(sink),
+            upstream,
+            allowed=agent("allowed", "0.3"),
+            denied=agent("denied", "1.0"),
+            denied_legacy=agent("legacy", "0.3"),
+            bridge=agent(
+                "bridge",
+                "0.3",
+                litellm_params={
+                    "custom_llm_provider": "openai",
+                    "model": "openai/gpt-5.4-mini",
+                    "api_base": upstream.url + "/v1",
+                    "api_key": "synthetic-bridge-key",
+                },
+            ),
+        )
+
+
+@pytest.fixture(scope="module")
+def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
+    with _owned_rig(tmp_path_factory.mktemp("a2a-denials")) as owned:
+        yield owned
+
+
+@pytest.fixture(scope="module", params=["raising_failure_logger,returning_teapot_logger", "raising_teapot_logger"])
+def hostile(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
+    callbacks: Final = tuple(str(request.param).split(","))
+    with _owned_rig(tmp_path_factory.mktemp("a2a-denials-hostile"), callbacks) as owned:
+        yield owned
+
+
+@pytest.fixture(scope="module", params=["disable_pre_routing_error_logs", "disable_error_logs"])
+def flagged(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
+    with _owned_rig(tmp_path_factory.mktemp("a2a-denials-flagged"), general={str(request.param): True}) as owned:
+        yield owned
+
+
+@pytest.fixture(scope="module")
+def alerting(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[_Rig, Wire]]:
+    with wire_server(lambda request: Reply(body=b"ok", content_type="text/plain")) as alerts:
+        with _owned_rig(
+            tmp_path_factory.mktemp("a2a-denials-alerting"),
+            general={"alerting": ["slack"], "alert_types": ["llm_exceptions"]},
+            environment={"SLACK_WEBHOOK_URL": alerts.url, "DEFAULT_FLUSH_INTERVAL_SECONDS": "1"},
+        ) as owned:
+            yield owned, alerts
+
+
+def _caller(rig: _Rig, scenario: Scenario, grant: str = "team") -> _Caller:
+    alias: Final = "a2adeny-" + uuid.uuid4().hex[:12]
+    user: Final = scenario.user()
+    allowed: Final[list[JsonValue]] = [rig.allowed.agent_id]
+    if grant == "personal":
+        return _Caller(
+            scenario.key(user_id=user, key_alias=alias, object_permission={"agents": allowed}), alias, user, None
+        )
+    if grant == "access group":
+        created: Final = rig.proxy.request(
+            "POST", "/v1/access_group", {"access_group_name": alias, "access_agent_ids": allowed}
+        )
+        assert created.status_code == 201, created.text
+        group: Final = string_value(JSON_OBJECT.validate_json(created.content)["access_group_id"])
+        scenario.cleanups.callback(rig.proxy.request, "DELETE", f"/v1/access_group/{group}")
+        return _Caller(scenario.key(user_id=user, key_alias=alias, access_group_ids=[group]), alias, user, None)
+    team: Final = scenario.team(object_permission={"agents": allowed})
+    return _Caller(scenario.key(user_id=user, team_id=team, key_alias=alias), alias, user, team)
+
+
+def _params(method: str, message_id: str, version: str | None) -> dict[str, JsonValue]:
+    if method.startswith("tasks/pushNotificationConfig"):
+        return {"taskId": "t1", "pushNotificationConfig": {"url": "https://callback.example.com/hook"}}
+    if method in {"tasks/get", "tasks/cancel", "GetTask", "CancelTask"}:
+        return {"id": "t1"}
+    if version == "1.0":
+        return {"message": {"messageId": message_id, "role": "ROLE_USER", "parts": [{"text": "x"}]}}
+    return {
+        "message": {
+            "kind": "message",
+            "messageId": message_id,
+            "role": "user",
+            "parts": [{"kind": "text", "text": "x"}],
+        }
+    }
+
+
+def _send(
+    rig: _Rig,
+    key: str,
+    path: str,
+    *,
+    method: str = "SendMessage",
+    version: str | None = "1.0",
+    request_id: JsonValue = None,
+    message_id: str | None = None,
+    extra: Mapping[str, JsonValue] = _NO_SETTINGS,
+) -> httpx.Response:
+    headers: Final = {
+        "Authorization": f"Bearer {key}",
+        **({"A2A-Version": version} if version is not None else {}),
+        **({"Accept": "text/event-stream"} if method in _STREAM_METHODS else {}),
+    }
+    body: Final = {
+        "jsonrpc": "2.0",
+        "id": "r-" + uuid.uuid4().hex if request_id is None else request_id,
+        "method": method,
+        "params": _params(method, message_id or uuid.uuid4().hex, version),
+        **extra,
+    }
+    return rig.proxy.client.post(path, headers=headers, json=body)
+
+
+def _assert_never_dispatched(rig: _Rig, scenario: Scenario, denied_message: str) -> None:
+    control_key: Final = scenario.key(object_permission={"agents": [rig.allowed.agent_id]})
+    control_message: Final = "control-" + uuid.uuid4().hex
+    granted: Final = _send(
+        rig,
+        control_key,
+        f"/a2a/{rig.allowed.agent_id}",
+        method="message/send",
+        version=None,
+        message_id=control_message,
+    )
+    assert granted.status_code == 200, granted.text
+    received: Final = tuple(request.body.decode() for request in rig.upstream.drain())
+    assert any(control_message in body for body in received), received
+    assert not any(denied_message in body for body in received), received
+
+
+def _metadata(row: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
+    value: Final = row["metadata"]
+    return JSON_OBJECT.validate_json(value) if isinstance(value, str) else object_value(value)
+
+
+def _rows(rig: _Rig, caller: _Caller) -> list[dict[str, JsonValue]]:
+    return rig.proxy.rows(_ROWS, (caller.hashed,))
+
+
+def _hooks(events: tuple[_Event, ...]) -> tuple[str, ...]:
+    return tuple(sorted(event.hook for event in events))
+
+
+def _spend(rig: _Rig, caller: _Caller) -> tuple[JsonValue, ...]:
+    key: Final = object_value(rig.proxy.get("/key/info", {"key": caller.key})["info"])["spend"]
+    user: Final = object_value(rig.proxy.get("/user/info", {"user_id": caller.user_id})["user_info"])["spend"]
+    if caller.team_id is None:
+        return key, user
+    team: Final = object_value(rig.proxy.get("/team/info", {"team_id": caller.team_id})["team_info"])["spend"]
+    return key, user, team
+
+
+def _assert_one_failure(rig: _Rig, caller: _Caller, *, status: int, agent_id: str, model: str) -> dict[str, JsonValue]:
+    events: Final = held(
+        lambda: rig.events.of(caller.alias),
+        lambda found: _hooks(found) == _ONE_FAILURE,
+        holding=_SETTLE,
+        seconds=_CONVERGE,
+    )
+    logged: Final = next(event for event in events if event.hook == "async_failure")
+    assert (logged.call_type, logged.status, logged.model) == ("asend_message", "failure", model), logged
+    assert (logged.key_hash, logged.user_id, logged.team_id) == (caller.hashed, caller.user_id, caller.team_id), logged
+    hooked: Final = next(event for event in events if event.hook == "post_call_failure")
+    assert (hooked.status_code, hooked.agent_id, hooked.key_hash) == (status, agent_id, caller.hashed), hooked
+    assert (hooked.user_id, hooked.team_id) == (caller.user_id, caller.team_id), hooked
+    rows: Final = held(lambda: _rows(rig, caller), lambda found: len(found) == 1, holding=_SETTLE, seconds=_CONVERGE)
+    row: Final = rows[0]
+    observed: Final = (row["status"], row["call_type"], row["agent_id"], row["api_key"], row["user"], row["spend"])
+    assert observed == ("failure", "asend_message", agent_id, caller.hashed, caller.user_id, 0.0), row
+    assert (row["team_id"] or None) == caller.team_id, row
+    assert object_value(_metadata(row)["error_information"])["error_code"] == str(status), row
+    assert all(spend == 0.0 for spend in _spend(rig, caller)), _spend(rig, caller)
+    return row
+
+
+@pytest.mark.parametrize(
+    ("route", "method", "version", "legacy"),
+    [
+        pytest.param("/a2a/{}", "SendMessage", "1.0", False, id="regression-send-1.0"),
+        pytest.param("/a2a/{}/message/send", "SendMessage", "1.0", False, id="regression-send-alias"),
+        pytest.param("/v1/a2a/{}/message/send", "SendMessage", "1.0", False, id="regression-send-v1-alias"),
+        pytest.param("/a2a/{}", "SendStreamingMessage", "1.0", False, id="regression-stream-1.0"),
+        pytest.param("/a2a/{}", "GetTask", "1.0", False, id="regression-get-task-1.0"),
+        pytest.param("/a2a/{}", "CancelTask", "1.0", False, id="regression-cancel-task-1.0"),
+        pytest.param("/a2a/{}", "message/send", None, True, id="regression-send-0.3"),
+        pytest.param("/a2a/{}", "message/stream", None, True, id="regression-stream-0.3"),
+        pytest.param("/a2a/{}", "tasks/get", None, True, id="regression-tasks-get-0.3"),
+        pytest.param("/a2a/{}", "tasks/cancel", None, True, id="regression-tasks-cancel-0.3"),
+        pytest.param("/a2a/{}", "tasks/pushNotificationConfig/set", None, True, id="regression-push-set-0.3"),
+    ],
+)
+def test_denied_agent_keeps_its_403_logs_one_zero_spend_failure_for_the_caller_and_never_dispatches(
+    rig: _Rig, route: str, method: str, version: str | None, legacy: bool
+) -> None:
+    target: Final = rig.denied_legacy if legacy else rig.denied
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario)
+        message: Final = "denied-" + uuid.uuid4().hex
+        response: Final = _send(
+            rig, caller.key, route.format(target.agent_id), method=method, version=version, message_id=message
+        )
+        assert (response.status_code, response.content) == (403, _refusal(target.agent_id, response))
+        assert response.headers["content-type"] == "application/json"
+        _assert_never_dispatched(rig, scenario, message)
+        _assert_one_failure(rig, caller, status=403, agent_id=target.agent_id, model=f"a2a_agent/{target.name}")
+
+
+@pytest.mark.parametrize("grant", ["personal", "team", "access group"], ids=lambda grant: "regression-" + grant)
+def test_denied_agent_failure_is_attributed_to_the_authenticated_key_for_every_grant_kind(
+    rig: _Rig, grant: str
+) -> None:
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario, grant)
+        granted: Final = _send(rig, caller.key, f"/a2a/{rig.allowed.agent_id}", method="message/send", version=None)
+        assert granted.status_code == 200, granted.text
+        response: Final = _send(rig, caller.key, f"/a2a/{rig.denied.agent_id}")
+        assert (response.status_code, response.content) == (403, _refusal(rig.denied.agent_id, response))
+        events: Final = held(
+            lambda: rig.events.of(caller.alias),
+            lambda found: _hooks(found) == tuple(sorted((*_ONE_FAILURE, "async_success"))),
+            holding=_SETTLE,
+            seconds=_CONVERGE,
+        )
+        failure: Final = next(event for event in events if event.hook == "async_failure")
+        assert (failure.key_hash, failure.user_id, failure.team_id) == (caller.hashed, caller.user_id, caller.team_id)
+        rows: Final = held(
+            lambda: _rows(rig, caller), lambda found: len(found) == 2, holding=_SETTLE, seconds=_CONVERGE
+        )
+        failed: Final = tuple(row for row in rows if row["status"] == "failure")
+        assert [(row["agent_id"], row["user"], row["spend"]) for row in failed] == [
+            (rig.denied.agent_id, caller.user_id, 0.0)
+        ], rows
+
+
+def test_denied_completion_bridge_agent_logs_one_failure_and_never_reaches_the_bridged_model(rig: _Rig) -> None:
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario)
+        message: Final = "bridge-" + uuid.uuid4().hex
+        response: Final = _send(
+            rig, caller.key, f"/a2a/{rig.bridge.agent_id}", method="message/send", version=None, message_id=message
+        )
+        assert (response.status_code, response.content) == (403, _refusal(rig.bridge.agent_id, response))
+        _assert_never_dispatched(rig, scenario, message)
+        _assert_one_failure(rig, caller, status=403, agent_id=rig.bridge.agent_id, model=f"a2a_agent/{rig.bridge.name}")
+
+
+@pytest.mark.parametrize(
+    ("route", "method", "version"),
+    [
+        pytest.param("/a2a/{}", "SendMessage", "1.0", id="regression-send"),
+        pytest.param("/a2a/{}/message/send", "SendMessage", "1.0", id="regression-send-alias"),
+        pytest.param("/v1/a2a/{}/message/send", "SendMessage", "1.0", id="regression-send-v1-alias"),
+        pytest.param("/a2a/{}", "message/stream", None, id="regression-stream-0.3"),
+    ],
+)
+def test_unknown_agent_keeps_its_jsonrpc_404_and_logs_one_zero_spend_failure_for_the_caller(
+    rig: _Rig, route: str, method: str, version: str | None
+) -> None:
+    unknown: Final = "no-such-" + uuid.uuid4().hex
+    request_id: Final = "u-" + uuid.uuid4().hex
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario)
+        message: Final = "unknown-" + uuid.uuid4().hex
+        response: Final = _send(
+            rig,
+            caller.key,
+            route.format(unknown),
+            method=method,
+            version=version,
+            request_id=request_id,
+            message_id=message,
+        )
+        assert (response.status_code, response.content) == (404, _not_found(request_id, unknown))
+        assert response.headers["content-type"] == "application/json"
+        _assert_never_dispatched(rig, scenario, message)
+        _assert_one_failure(rig, caller, status=404, agent_id=unknown, model=f"a2a_agent/{unknown}")
+
+
+@pytest.mark.parametrize("request_id", [0, 7, "abc"], ids=lambda value: f"regression-id-{value!r}")
+def test_unknown_agent_echoes_the_jsonrpc_id_with_its_type_and_logs_one_failure(
+    rig: _Rig, request_id: JsonValue
+) -> None:
+    unknown: Final = "no-such-" + uuid.uuid4().hex
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario)
+        response: Final = _send(rig, caller.key, f"/a2a/{unknown}", request_id=request_id)
+        assert (response.status_code, response.content) == (404, _not_found(request_id, unknown))
+        assert type(JSON_OBJECT.validate_json(response.content)["id"]) is type(request_id)
+        _assert_one_failure(rig, caller, status=404, agent_id=unknown, model=f"a2a_agent/{unknown}")
+
+
+@pytest.mark.parametrize("outcome", ["denied", "unknown"], ids=lambda outcome: "regression-" + outcome)
+def test_client_supplied_attribution_cannot_rewrite_a_denial_failure_row(rig: _Rig, outcome: str) -> None:
+    unknown: Final = "no-such-" + uuid.uuid4().hex
+    target: Final = rig.denied.agent_id if outcome == "denied" else unknown
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario)
+        other: Final = _caller(rig, scenario)
+        success_id: Final = "s-" + uuid.uuid4().hex
+        granted: Final = _send(
+            rig, other.key, f"/a2a/{rig.allowed.agent_id}", method="message/send", version=None, request_id=success_id
+        )
+        assert granted.status_code == 200, granted.text
+        success_rows: Final = eventually(lambda: _rows(rig, other), lambda found: len(found) == 1, seconds=_CONVERGE)
+        success_request: Final = success_rows[0]["request_id"]
+        request_id: Final = "spoof-" + uuid.uuid4().hex
+        spoof: Final[dict[str, JsonValue]] = {
+            "agent_id": "spoof",
+            "metadata": {
+                "agent_id": "spoof",
+                "user_api_key_alias": "spoof",
+                "user_api_key_team_id": "spoof",
+                "tags": ["x"],
+            },
+            "litellm_call_id": success_request,
+        }
+        body: Final = {
+            "jsonrpc": "2.0",
+            "id": request_id,
+            "method": "SendMessage",
+            "params": {**_params("SendMessage", uuid.uuid4().hex, "1.0"), "litellm_call_id": "spoof2"},
+            **spoof,
+        }
+        response: Final = rig.proxy.client.post(
+            f"/a2a/{target}", headers={"Authorization": f"Bearer {caller.key}", "A2A-Version": "1.0"}, json=body
+        )
+        expected: Final = _refusal(target, response) if outcome == "denied" else _not_found(request_id, unknown)
+        assert (response.status_code, response.content) == (403 if outcome == "denied" else 404, expected)
+        model: Final = f"a2a_agent/{rig.denied.name}" if outcome == "denied" else f"a2a_agent/{unknown}"
+        row: Final = _assert_one_failure(rig, caller, status=response.status_code, agent_id=target, model=model)
+        assert row["request_id"] not in {success_request, "spoof2", request_id}, row
+        metadata: Final = _metadata(row)
+        assert (metadata["user_api_key_alias"], metadata["user_api_key_team_id"]) == (caller.alias, caller.team_id)
+        assert rig.events.of("spoof") == ()
+        after: Final = _rows(rig, other)
+        assert [(item["request_id"], item["status"]) for item in after] == [(success_request, "success")], after
+
+
+def test_two_keys_reusing_one_jsonrpc_id_get_one_denial_row_each(rig: _Rig) -> None:
+    shared_id: Final = "same-" + uuid.uuid4().hex
+    with rig.proxy.scenario() as scenario:
+        callers: Final = (_caller(rig, scenario), _caller(rig, scenario))
+        for caller in callers:
+            response = _send(rig, caller.key, f"/a2a/{rig.denied.agent_id}", request_id=shared_id)
+            assert (response.status_code, response.content) == (403, _refusal(rig.denied.agent_id, response))
+        rows: Final = tuple(
+            _assert_one_failure(
+                rig, caller, status=403, agent_id=rig.denied.agent_id, model=f"a2a_agent/{rig.denied.name}"
+            )
+            for caller in callers
+        )
+        assert rows[0]["request_id"] != rows[1]["request_id"], rows
+
+
+@pytest.mark.parametrize("outcome", ["denied", "unknown"])
+def test_failing_or_transforming_failure_callbacks_never_change_the_denial_response(
+    hostile: _Rig, outcome: str
+) -> None:
+    unknown: Final = "no-such-" + uuid.uuid4().hex
+    request_id: Final = "h-" + uuid.uuid4().hex
+    target: Final = hostile.denied.agent_id if outcome == "denied" else unknown
+    with hostile.proxy.scenario() as scenario:
+        caller: Final = _caller(hostile, scenario)
+        response: Final = _send(hostile, caller.key, f"/a2a/{target}", request_id=request_id)
+        expected: Final = (403, _refusal(target, response)) if outcome == "denied" else (404, _not_found(request_id, unknown))
+        assert (response.status_code, response.content) == expected
+
+
+@pytest.mark.parametrize("outcome", ["denied", "unknown"], ids=lambda outcome: "regression-" + outcome)
+def test_failing_or_transforming_failure_callbacks_do_not_stop_the_denial_from_being_logged(
+    hostile: _Rig, outcome: str
+) -> None:
+    unknown: Final = "no-such-" + uuid.uuid4().hex
+    target: Final = hostile.denied.agent_id if outcome == "denied" else unknown
+    with hostile.proxy.scenario() as scenario:
+        caller: Final = _caller(hostile, scenario)
+        response: Final = _send(hostile, caller.key, f"/a2a/{target}")
+        assert response.status_code == (403 if outcome == "denied" else 404), response.text
+        model: Final = f"a2a_agent/{hostile.denied.name}" if outcome == "denied" else f"a2a_agent/{unknown}"
+        _assert_one_failure(hostile, caller, status=response.status_code, agent_id=target, model=model)
+
+
+def test_denials_follow_the_model_denied_chat_logging_under_error_log_flags(flagged: _Rig) -> None:
+    unknown: Final = "no-such-" + uuid.uuid4().hex
+    request_id: Final = "f-" + uuid.uuid4().hex
+    with flagged.proxy.scenario() as scenario:
+        chat_caller: Final = _caller(flagged, scenario)
+        denied_caller: Final = _caller(flagged, scenario)
+        unknown_caller: Final = _caller(flagged, scenario)
+        chat_alias: Final = "a2adeny-chat-" + uuid.uuid4().hex[:8]
+        chat_key: Final = scenario.key(
+            user_id=chat_caller.user_id, team_id=chat_caller.team_id, key_alias=chat_alias, models=["openai/gpt-5-mini"]
+        )
+        chat: Final = flagged.proxy.request(
+            "POST",
+            "/v1/chat/completions",
+            {"model": "openai/gpt-5.4-mini", "messages": [{"role": "user", "content": "x"}]},
+            key=chat_key,
+        )
+        assert chat.status_code in {401, 403}, chat.text
+        denied: Final = _send(flagged, denied_caller.key, f"/a2a/{flagged.denied.agent_id}")
+        assert (denied.status_code, denied.content) == (403, _refusal(flagged.denied.agent_id, denied))
+        missing: Final = _send(flagged, unknown_caller.key, f"/a2a/{unknown}", request_id=request_id)
+        assert (missing.status_code, missing.content) == (404, _not_found(request_id, unknown))
+        _assert_never_dispatched(flagged, scenario, "never-sent")
+        chat_hooks: Final = held(
+            lambda: _hooks(flagged.events.of(chat_alias)),
+            lambda found: len(found) > 0,
+            holding=_SETTLE,
+            seconds=_CONVERGE,
+        )
+        assert _hooks(flagged.events.of(denied_caller.alias)) == chat_hooks
+        assert _hooks(flagged.events.of(unknown_caller.alias)) == chat_hooks
+        control: Final = _caller(flagged, scenario)
+        granted: Final = _send(
+            flagged, control.key, f"/a2a/{flagged.allowed.agent_id}", method="message/send", version=None
+        )
+        assert granted.status_code == 200, granted.text
+        eventually(lambda: _rows(flagged, control), lambda found: len(found) == 1, seconds=_CONVERGE)
+        assert (_rows(flagged, denied_caller), _rows(flagged, unknown_caller)) == ([], [])
+
+
+def test_client_error_denials_never_raise_an_llm_exceptions_alert(alerting: tuple[_Rig, Wire]) -> None:
+    rig, alerts = alerting
+    unknown: Final = "no-such-" + uuid.uuid4().hex
+    marker: Final = "boom-" + uuid.uuid4().hex
+
+    def failing(request: Request) -> Reply:
+        return Reply(
+            status=500, body=_compact({"error": {"message": f"upstream exploded {marker}", "type": "server_error"}})
+        )
+
+    with rig.proxy.scenario() as scenario, wire_server(failing) as broken:
+        caller: Final = _caller(rig, scenario)
+        assert _send(rig, caller.key, f"/a2a/{rig.denied.agent_id}").status_code == 403
+        assert _send(rig, caller.key, f"/a2a/{unknown}").status_code == 404
+        model: Final = scenario.model(api_base=broken.url + "/v1", num_retries=0)
+        failed: Final = rig.proxy.request(
+            "POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": "x"}]}
+        )
+        assert failed.status_code >= 500, failed.text
+        texts: Final = eventually(
+            lambda: tuple(JSON_OBJECT.validate_json(request.body).get("text") for request in alerts.drain()),
+            lambda found: any(marker in str(text) for text in found),
+            seconds=_CONVERGE,
+        )
+        assert not any(rig.denied.agent_id in str(text) or unknown in str(text) for text in texts), texts
+
+
+def test_guard_model_denied_chat_logs_one_acompletion_failure(rig: _Rig) -> None:
+    with rig.proxy.scenario() as scenario:
+        user: Final = scenario.user()
+        alias: Final = "a2adeny-chat-" + uuid.uuid4().hex[:8]
+        key: Final = scenario.key(user_id=user, key_alias=alias, models=["openai/gpt-5-mini"])
+        caller: Final = _Caller(key, alias, user, None)
+        response: Final = rig.proxy.request(
+            "POST",
+            "/v1/chat/completions",
+            {"model": "openai/gpt-5.4-mini", "messages": [{"role": "user", "content": "x"}]},
+            key=key,
+        )
+        assert response.status_code == 403, response.text
+        assert "key_model_access_denied" in response.text
+        events: Final = held(
+            lambda: rig.events.of(alias),
+            lambda found: _hooks(found) == _ONE_FAILURE,
+            holding=_SETTLE,
+            seconds=_CONVERGE,
+        )
+        assert {event.status_code for event in events if event.hook == "post_call_failure"} == {403}
+        rows: Final = held(
+            lambda: _rows(rig, caller), lambda found: len(found) == 1, holding=_SETTLE, seconds=_CONVERGE
+        )
+        assert (rows[0]["status"], rows[0]["call_type"], rows[0]["spend"]) == ("failure", "acompletion", 0.0), rows
+
+
+@pytest.mark.parametrize(
+    ("method", "call_type"), [("message/send", "asend_message"), ("message/stream", "asend_message_streaming")]
+)
+def test_guard_granted_agent_succeeds_with_one_success_row_and_no_failure(
+    rig: _Rig, method: str, call_type: str
+) -> None:
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario)
+        request_id: Final = "g-" + uuid.uuid4().hex
+        response: Final = _send(
+            rig, caller.key, f"/a2a/{rig.allowed.agent_id}", method=method, version=None, request_id=request_id
+        )
+        assert response.status_code == 200, response.text
+        if method == "message/stream":
+            assert response.text.startswith("data: ") and request_id in response.text, response.text
+        else:
+            assert object_value(JSON_OBJECT.validate_json(response.content)["result"])["parts"] == [
+                {"kind": "text", "text": "granted"}
+            ]
+        held(
+            lambda: rig.events.of(caller.alias),
+            lambda found: _hooks(found) == ("async_success",),
+            holding=_SETTLE,
+            seconds=_CONVERGE,
+        )
+        rows: Final = held(
+            lambda: _rows(rig, caller), lambda found: len(found) == 1, holding=_SETTLE, seconds=_CONVERGE
+        )
+        assert (rows[0]["status"], rows[0]["call_type"], rows[0]["agent_id"]) == (
+            "success",
+            call_type,
+            rig.allowed.agent_id,
+        ), rows
+
+
+def test_guard_zero_budget_key_is_refused_once_without_double_logging(rig: _Rig) -> None:
+    with rig.proxy.scenario() as scenario:
+        user: Final = scenario.user()
+        alias: Final = "a2adeny-budget-" + uuid.uuid4().hex[:8]
+        key: Final = scenario.key(
+            user_id=user, key_alias=alias, max_budget=0, object_permission={"agents": [rig.allowed.agent_id]}
+        )
+        caller: Final = _Caller(key, alias, user, None)
+        message: Final = "budget-" + uuid.uuid4().hex
+        response: Final = _send(
+            rig, key, f"/a2a/{rig.allowed.agent_id}", method="message/send", version=None, message_id=message
+        )
+        assert response.status_code == 422, response.text
+        assert "Budget has been exceeded" in response.text
+        _assert_never_dispatched(rig, scenario, message)
+        held(
+            lambda: rig.events.of(alias),
+            lambda found: _hooks(found) == _ONE_FAILURE,
+            holding=_SETTLE,
+            seconds=_CONVERGE,
+        )
+        rows: Final = held(
+            lambda: _rows(rig, caller), lambda found: len(found) == 1, holding=_SETTLE, seconds=_CONVERGE
+        )
+        assert (rows[0]["status"], rows[0]["call_type"]) == ("failure", "asend_message"), rows
+
+
+def test_guard_agent_card_discovery_keeps_its_refusals_and_writes_no_failure_row(rig: _Rig) -> None:
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario)
+        headers: Final = {"Authorization": f"Bearer {caller.key}"}
+        denied_card: Final = rig.proxy.client.get(
+            f"/a2a/{rig.denied.agent_id}/.well-known/agent-card.json", headers=headers
+        )
+        assert (denied_card.status_code, denied_card.content) == (403, _refusal(rig.denied.agent_id))
+        missing: Final = rig.proxy.client.get("/a2a/no-such-card/.well-known/agent-card.json", headers=headers)
+        assert (missing.status_code, missing.content) == (404, _compact({"detail": "Agent 'no-such-card' not found"}))
+        granted: Final = _send(rig, caller.key, f"/a2a/{rig.allowed.agent_id}", method="message/send", version=None)
+        assert granted.status_code == 200, granted.text
+        rows: Final = held(
+            lambda: _rows(rig, caller), lambda found: len(found) == 1, holding=_SETTLE, seconds=_CONVERGE
+        )
+        assert [row["status"] for row in rows] == ["success"], rows
+        assert _hooks(rig.events.of(caller.alias)) == ("async_success",)
+
+
+def test_guard_jsonrpc_1_0_envelope_keeps_its_invalid_request_body(rig: _Rig) -> None:
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario)
+        request_id: Final = "v-" + uuid.uuid4().hex
+        response: Final = _send(
+            rig,
+            caller.key,
+            f"/a2a/{rig.allowed.agent_id}",
+            method="message/send",
+            version=None,
+            request_id=request_id,
+            extra={"jsonrpc": "1.0"},
+        )
+        assert (response.status_code, response.content) == (
+            400,
+            _compact(
+                {
+                    "jsonrpc": "2.0",
+                    "id": request_id,
+                    "error": {"code": -32600, "message": "Invalid Request: jsonrpc must be '2.0'"},
+                }
+            ),
+        )

tests/integration/compatibility/test_a2a_wire_versions.py

@@ -452,8 +452,13 @@ def test_guard_no_agent_key_or_proxy_caller_key_is_sent(gateway: Gateway) -> Non
     (("0.3", "message/stream", _MESSAGE_V03), ("1.0", "SendStreamingMessage", _MESSAGE_V10)),
     ids=("v0.3", "v1.0"),
 )
+@pytest.mark.parametrize(
+    "path",
+    ("/a2a/{identity}", "/a2a/{identity}/message/stream", "/v1/a2a/{identity}/message/stream"),
+    ids=("bare", "message_stream", "v1_message_stream"),
+)
 def test_a2a_stream_relays_each_upstream_event_as_its_own_sse_frame_in_the_pinned_dialect(
-    gateway: Gateway, version: str, method: str, message: dict[str, JsonValue]
+    gateway: Gateway, version: str, method: str, message: dict[str, JsonValue], path: str
 ) -> None:
     marker: Final = "a2astream" + uuid.uuid4().hex
     first_frame_seen: Final = threading.Event()
@@ -482,7 +487,7 @@ def test_a2a_stream_relays_each_upstream_event_as_its_own_sse_frame_in_the_pinne
         )
         with gateway.client.stream(
             "POST",
-            f"/a2a/{identity}",
+            path.format(identity=identity),
             headers={
                 "Authorization": f"Bearer {key}",
                 "a2a-version": version,

tests/integration/configuration/test_multi_worker_reload.py

@@ -0,0 +1,95 @@
+import json
+import os
+import re
+from pathlib import Path
+from typing import Final
+
+import psutil
+import pytest
+
+from tests.integration._support.client import Gateway, eventually, object_value
+from tests.integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
+
+STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]")
+
+
+def _worker_pids(owned: OwnedProxy) -> frozenset[int]:
+    return frozenset(int(match.group(1)) for match in STARTED_WORKER.finditer(owned.log.read_text()))
+
+
+def _launched_config(owned: OwnedProxy) -> Path:
+    arguments: Final = psutil.Process(owned.process.pid).cmdline()
+    return Path(arguments[arguments.index("--config") + 1])
+
+
+def _config(model: str) -> str:
+    return json.dumps(
+        {
+            "model_list": [{"model_name": model, "litellm_params": {"model": "openai/reload-test"}}],
+            "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY"},
+            "router_settings": {"redis_host": "os.environ/REDIS_HOST", "redis_port": "os.environ/REDIS_PORT"},
+        }
+    )
+
+
+def _assert_replaced(owned: OwnedProxy, previous: frozenset[int], workers: int) -> frozenset[int]:
+    started: Final = eventually(
+        lambda: _worker_pids(owned),
+        lambda pids: len(pids - previous) == workers,
+        seconds=graceful_stop_seconds(),
+    )
+    assert all(not psutil.pid_exists(pid) for pid in previous), owned.log.read_text()
+    assert owned.process.poll() is None, owned.log.read_text()
+    assert len(started) == len(previous) + workers, owned.log.read_text()
+    return started
+
+
+def _assert_models(gateway: Gateway) -> None:
+    data: Final = gateway.get("/v1/models")["data"]
+    assert isinstance(data, list), data
+    assert tuple(object_value(model)["id"] for model in data) == ("after-reload",)
+
+
+@pytest.mark.parametrize("workers", (1, 2))
+@pytest.mark.timeout(6 * graceful_stop_seconds())
+def test_reload_replaces_every_worker_and_shutdown_leaves_none(
+    gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, workers: int
+) -> None:
+    root: Final = Path(__file__).resolve().parents[3]
+    monkeypatch.setenv("INTEGRATION_PROXY_ROOT", str(tmp_path))
+    config: Final = tmp_path / "config.yaml"
+    source: Final = tmp_path / "reload_source.py"
+    env: Final = tmp_path / ".env"
+    config.write_text(_config("before-reload"))
+    source.write_text("VALUE = 1\n")
+    env.write_text("RELOAD_TEST_VALUE=before\n")
+    with owned_proxy_process(
+        gateway,
+        tmp_path,
+        {
+            "PYTHONPATH": os.pathsep.join((str(root), str(root / "tests"), os.environ.get("PYTHONPATH", ""))),
+            "STORE_MODEL_IN_DB": "False",
+        },
+        config=config,
+        remove_environment=("DATABASE_URL", "DATABASE_URL_READ_REPLICA", "PYTHON_DOTENV_DISABLED"),
+        workers=workers,
+        database_setup=(),
+        extra_arguments=("--reload",),
+    ) as owned:
+        initial: Final = eventually(
+            lambda: _worker_pids(owned), lambda pids: len(pids) == workers, seconds=graceful_stop_seconds()
+        )
+        _launched_config(owned).write_text(_config("after-reload"))
+        after_config: Final = _assert_replaced(owned, initial, workers)
+        for _ in range(2 * workers):
+            _assert_models(owned.gateway)
+        source.write_text("VALUE = 2\n")
+        after_source: Final = _assert_replaced(owned, after_config, workers)
+        env.write_text("RELOAD_TEST_VALUE=after\n")
+        after_env: Final = _assert_replaced(owned, after_source, workers)
+        response: Final = owned.gateway.request("GET", "/health/readiness")
+        assert response.status_code == 200, response.text
+        owned.begin_shutdown()
+        owned.process.wait(timeout=graceful_stop_seconds())
+        assert all(not psutil.pid_exists(pid) for pid in after_env), owned.log.read_text()
+        assert owned.process.returncode == 0, owned.log.read_text()

tests/integration/configuration/test_reload_worker_pool.py

@@ -0,0 +1,246 @@
+import multiprocessing
+import re
+import signal
+import socket
+import subprocess
+import sys
+import threading
+import time
+from collections.abc import Callable
+from pathlib import Path
+from typing import Final
+
+import httpx
+import psutil
+import pytest
+
+from litellm.proxy.dev_reload import multi_worker_reload_supported, stop_pool_when_reloader_exits
+from tests.integration._support.client import eventually
+from tests.integration._support.ports import allocate_port
+from tests.integration._support.process import group_members, kill_stray_process, signal_group, stop_processes
+
+POOL_WORKERS: Final = 2
+POOL_TIMEOUT_SECONDS: Final = 120
+WORKER_HEALTHCHECK_SECONDS: Final = 30
+WORKER_STARTED: Final = re.compile(r"Started server process \[(\d+)\]")
+POOL_STOPPED: Final = "failed to start, stopping the parent process"
+APP_LOAD_ERROR: Final = "Error loading ASGI app"
+PID_APP: Final = """import os
+
+
+async def app(scope, receive, send):
+    if scope["type"] != "http":
+        return
+    await send({"type": "http.response.start", "status": 200, "headers": []})
+    await send({"type": "http.response.body", "body": str(os.getpid()).encode()})
+"""
+REQUIRES_UVICORN_STARTUP_FAILURE: Final = pytest.mark.skipif(
+    not multi_worker_reload_supported(), reason="uvicorn<0.50.0 has no startup failure exit code"
+)
+CORE_DUMP_SECONDS: Final = 15
+SLOW_CORE_DUMP_ON_QUIT: Final = f"""
+import os
+import signal
+import subprocess
+
+
+def freeze_until_killed(signum, frame):
+    subprocess.Popen(("sh", "-c", f"sleep {CORE_DUMP_SECONDS}; kill -9 {{os.getpid()}}"))
+    os.kill(os.getpid(), signal.SIGSTOP)
+
+
+signal.signal(signal.SIGQUIT, freeze_until_killed)
+"""
+POOL_RUNNER: Final = """import sys
+
+sys.path.insert(0, sys.argv[1])
+
+from litellm.proxy.dev_reload import run_multi_worker_reload
+
+run_multi_worker_reload(
+    {
+        "app": "reload_pool_app:app",
+        "host": "127.0.0.1",
+        "port": int(sys.argv[2]),
+        "timeout_worker_healthcheck": int(sys.argv[3]),
+        "reload": True,
+        "reload_dirs": (sys.argv[1],),
+        "reload_includes": ("*.py",),
+    },
+    workers=int(sys.argv[4]),
+)
+"""
+
+
+def _started_workers(log_text: str) -> frozenset[int]:
+    return frozenset(int(match.group(1)) for match in WORKER_STARTED.finditer(log_text))
+
+
+def _until(done: Callable[[], bool], log: Path, seconds: float = POOL_TIMEOUT_SECONDS) -> None:
+    eventually(lambda: (done(), log.read_text()), lambda observed: observed[0], seconds=seconds)
+
+
+def _wait_for_log(log: Path, process: subprocess.Popen[bytes], seen: Callable[[str], bool]) -> str:
+    log_text, _ = eventually(
+        lambda: (log.read_text(), process.poll()),
+        lambda observed: seen(observed[0]) or observed[1] is not None,
+        seconds=POOL_TIMEOUT_SECONDS,
+    )
+    assert seen(log_text), log_text
+    return log_text
+
+
+def _start_pool(app_dir: Path, port: int, log: Path, runner: str = POOL_RUNNER) -> subprocess.Popen[bytes]:
+    with log.open("wb") as output:
+        return subprocess.Popen(
+            (
+                sys.executable,
+                "-I",
+                "-c",
+                runner,
+                str(app_dir),
+                str(port),
+                str(WORKER_HEALTHCHECK_SECONDS),
+                str(POOL_WORKERS),
+            ),
+            stdout=output,
+            stderr=subprocess.STDOUT,
+            start_new_session=True,
+        )
+
+
+def _port_refuses_connections(port: int) -> bool:
+    with socket.socket() as sock:
+        return sock.connect_ex(("127.0.0.1", port)) != 0
+
+
+def _next_pool(log: Path, process: subprocess.Popen[bytes], port: int, previous: frozenset[int]) -> frozenset[int]:
+    log_text: Final = _wait_for_log(log, process, lambda text: len(_started_workers(text) - previous) == POOL_WORKERS)
+    pool: Final = _started_workers(log_text) - previous
+    assert not any(psutil.pid_exists(pid) for pid in previous), log_text
+    served: Final = httpx.get(f"http://127.0.0.1:{port}/", timeout=POOL_TIMEOUT_SECONDS)
+    assert int(served.text) in pool, log_text
+    return pool
+
+
+def test_the_pool_is_told_to_stop_once_the_reloader_process_exits() -> None:
+    reloader: Final = multiprocessing.get_context("spawn").Process(target=time.sleep, args=(POOL_TIMEOUT_SECONDS,))
+    reloader.start()
+    pool_should_exit: Final = threading.Event()
+    threading.Thread(target=stop_pool_when_reloader_exits, args=(reloader, pool_should_exit), daemon=True).start()
+    try:
+        assert not pool_should_exit.wait(0.5)
+        kill_stray_process(reloader.pid)
+        assert pool_should_exit.wait(POOL_TIMEOUT_SECONDS)
+    finally:
+        if reloader.is_alive():
+            kill_stray_process(reloader.pid)
+        reloader.join()
+
+
+@REQUIRES_UVICORN_STARTUP_FAILURE
+@pytest.mark.timeout(5 * POOL_TIMEOUT_SECONDS)
+def test_a_save_replaces_every_worker_and_a_broken_save_stops_the_pool_until_the_next_good_one(
+    tmp_path: Path,
+) -> None:
+    app: Final = tmp_path / "reload_pool_app.py"
+    app.write_text(PID_APP)
+    log: Final = tmp_path / "reload.log"
+    port: Final = allocate_port()
+    process: Final = _start_pool(tmp_path, port, log)
+    try:
+        first_pool: Final = _next_pool(log, process, port, frozenset())
+        app.write_text("async def app(\n")
+        _wait_for_log(log, process, lambda text: POOL_STOPPED in text)
+        assert not any(psutil.pid_exists(pid) for pid in first_pool), log.read_text()
+        app.write_text(PID_APP)
+        last_pool: Final = _next_pool(log, process, port, first_pool)
+        signal_group(process.pid, signal.SIGINT)
+        assert process.wait(timeout=POOL_TIMEOUT_SECONDS) == 0, log.read_text()
+        assert not any(psutil.pid_exists(pid) for pid in last_pool), log.read_text()
+        assert 1 <= log.read_text().count(APP_LOAD_ERROR) <= POOL_WORKERS, log.read_text()
+    finally:
+        stop_processes(group_members(process.pid))
+        process.wait()
+
+
+@REQUIRES_UVICORN_STARTUP_FAILURE
+@pytest.mark.timeout(3 * POOL_TIMEOUT_SECONDS)
+@pytest.mark.parametrize(
+    ("kill_reloader", "slow_core_dump", "shutdown_timeout"),
+    (
+        pytest.param(kill_stray_process, False, POOL_TIMEOUT_SECONDS, id="sigkill-reloader"),
+        pytest.param(
+            lambda reloader: signal_group(reloader, signal.SIGHUP), False, POOL_TIMEOUT_SECONDS, id="terminal-closed"
+        ),
+        pytest.param(lambda reloader: signal_group(reloader, signal.SIGQUIT), True, 10, id="ctrl-backslash"),
+    ),
+)
+def test_the_pool_stops_and_frees_the_port_when_the_reloader_dies(
+    tmp_path: Path, kill_reloader: Callable[[int], None], slow_core_dump: bool, shutdown_timeout: float
+) -> None:
+    app_source: Final = PID_APP + (SLOW_CORE_DUMP_ON_QUIT if slow_core_dump else "")
+    (tmp_path / "reload_pool_app.py").write_text(app_source)
+    log: Final = tmp_path / "reload.log"
+    port: Final = allocate_port()
+    process: Final = _start_pool(tmp_path, port, log)
+    try:
+        workers: Final = _next_pool(log, process, port, frozenset())
+        pool_parent: Final = psutil.Process(min(workers)).ppid()
+        assert psutil.Process(pool_parent).ppid() == process.pid, log.read_text()
+        kill_reloader(process.pid)
+        process.wait(timeout=POOL_TIMEOUT_SECONDS)
+        _until(
+            lambda: not psutil.pid_exists(pool_parent) and _port_refuses_connections(port),
+            log,
+            seconds=shutdown_timeout,
+        )
+    finally:
+        stop_processes(group_members(process.pid))
+        process.wait()
+
+
+@REQUIRES_UVICORN_STARTUP_FAILURE
+@pytest.mark.timeout(3 * POOL_TIMEOUT_SECONDS)
+def test_a_killed_pool_parent_takes_its_workers_with_it_so_the_next_save_serves_only_new_workers(
+    tmp_path: Path,
+) -> None:
+    app: Final = tmp_path / "reload_pool_app.py"
+    app.write_text(PID_APP)
+    log: Final = tmp_path / "reload.log"
+    port: Final = allocate_port()
+    process: Final = _start_pool(tmp_path, port, log)
+    try:
+        orphans: Final = _next_pool(log, process, port, frozenset())
+        kill_stray_process(psutil.Process(min(orphans)).ppid())
+        _until(lambda: not any(psutil.pid_exists(pid) for pid in orphans), log)
+        app.write_text(f"{PID_APP}\n")
+        _next_pool(log, process, port, orphans)
+    finally:
+        stop_processes(group_members(process.pid))
+        process.wait()
+
+
+@REQUIRES_UVICORN_STARTUP_FAILURE
+@pytest.mark.timeout(3 * POOL_TIMEOUT_SECONDS)
+def test_ctrl_backslash_frees_the_port_without_waiting_out_the_worker_healthcheck_when_core_dumps_are_slow(
+    tmp_path: Path,
+) -> None:
+    (tmp_path / "reload_pool_app.py").write_text(PID_APP + SLOW_CORE_DUMP_ON_QUIT)
+    log: Final = tmp_path / "reload.log"
+    port: Final = allocate_port()
+    process: Final = _start_pool(tmp_path, port, log)
+    try:
+        workers: Final = _next_pool(log, process, port, frozenset())
+        pool_parent: Final = psutil.Process(min(workers)).ppid()
+        signal_group(process.pid, signal.SIGQUIT)
+        _until(
+            lambda: (
+                process.poll() is not None and not psutil.pid_exists(pool_parent) and _port_refuses_connections(port)
+            ),
+            log,
+            seconds=WORKER_HEALTHCHECK_SECONDS / 2,
+        )
+    finally:
+        stop_processes(group_members(process.pid))
+        process.wait()

tests/integration/configuration/test_search_embedding_model_settings.py

@@ -0,0 +1,299 @@
+from __future__ import annotations
+
+import json
+import uuid
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+from io import BytesIO
+from pathlib import Path
+from typing import Final
+from zipfile import ZipFile
+
+import pytest
+from integration._support.client import (
+    JSON_OBJECT,
+    Gateway,
+    gateway_from_environment,
+    list_value,
+    object_value,
+    string_value,
+)
+from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
+from integration._support.wire import Reply, Request, Wire, wire_server
+from pydantic import JsonValue, TypeAdapter
+
+pytestmark: Final = pytest.mark.timeout(int(2 * graceful_stop_seconds() + 120))
+
+_MODEL_NAME: Final = "search-embedding"
+_AGENT_NOT_CONFIGURED: Final = (
+    "agent search needs litellm_settings.agent_search_embedding_model set to an embedding model from model_list"
+)
+_SKILL_NOT_CONFIGURED: Final = (
+    "skill search needs litellm_settings.skill_search_embedding_model set to an embedding model from model_list"
+)
+_SKILL_MARKDOWN: Final = """---
+name: echo-sample
+description: Echo text for search validation
+---
+
+# Echo sample
+
+Return the supplied text unchanged
+"""
+_JSON_ARRAY: Final = TypeAdapter(list[JsonValue])
+
+
+@dataclass(frozen=True, slots=True)
+class _SearchProxy:
+    owned: OwnedProxy
+    wire: Wire
+    skill_id: str
+    agent_id: str
+
+
+def _embedding_reply(request: Request) -> Reply:
+    if request.method != "POST" or request.target != "/v1/embeddings":
+        return Reply(body=b'{"object":"list","data":[]}')
+    body: Final = JSON_OBJECT.validate_json(request.body)
+    input_value: Final = body.get("input")
+    input_count: Final = len(input_value) if isinstance(input_value, list) else 1
+    embeddings: Final = [
+        {"object": "embedding", "index": index, "embedding": [0.25, 0.75]} for index in range(input_count)
+    ]
+    return Reply(
+        body=json.dumps(
+            {
+                "object": "list",
+                "data": embeddings,
+                "model": "text-embedding-3-small",
+                "usage": {"prompt_tokens": input_count, "total_tokens": input_count},
+            }
+        ).encode()
+    )
+
+
+def _write_config(directory: Path, upstream_url: str, agent_model: str, skill_model: str) -> Path:
+    config: Final = directory / f"search_embedding_{uuid.uuid4().hex}.yaml"
+    contents: Final[dict[str, JsonValue]] = {
+        "model_list": [
+            {
+                "model_name": _MODEL_NAME,
+                "litellm_params": {
+                    "model": "openai/text-embedding-3-small",
+                    "api_base": f"{upstream_url}/v1",
+                    "api_key": "synthetic-openai-key",
+                },
+            }
+        ],
+        "litellm_settings": {
+            "agent_search_embedding_model": agent_model,
+            "skill_search_embedding_model": skill_model,
+        },
+        "general_settings": {
+            "master_key": "os.environ/LITELLM_MASTER_KEY",
+            "database_url": "os.environ/DATABASE_URL",
+            "store_model_in_db": True,
+        },
+        "agents": [
+            {
+                "agent_name": "plain",
+                "agent_card_params": {
+                    "protocolVersion": "1.0",
+                    "name": "plain",
+                    "description": "echo agent",
+                    "url": "http://agent:9000/plain/",
+                    "supportedInterfaces": [
+                        {
+                            "url": "http://agent:9000/plain/",
+                            "protocolBinding": "JSONRPC",
+                            "protocolVersion": "1.0",
+                        }
+                    ],
+                    "version": "1.0.0",
+                    "capabilities": {"streaming": True},
+                    "defaultInputModes": ["text/plain"],
+                    "defaultOutputModes": ["text/plain"],
+                    "skills": [],
+                },
+            }
+        ],
+    }
+    config.write_text(json.dumps(contents))
+    return config
+
+
+def _create_skill(candidate: OwnedProxy) -> str:
+    archive_data: Final = BytesIO()
+    with ZipFile(archive_data, "w") as archive:
+        archive.writestr("echo-sample/SKILL.md", _SKILL_MARKDOWN)
+    response: Final = candidate.gateway.request_multipart(
+        "/v1/skills?beta=true&custom_llm_provider=litellm_proxy",
+        {
+            "display_title": "Echo search fixture",
+            "description": "Echo sample skill for search validation",
+            "instructions": "Return the supplied text unchanged",
+        },
+        {"files[]": ("echo-skill.zip", archive_data.getvalue(), "application/zip")},
+    )
+    assert response.status_code == 200, response.text
+    return string_value(JSON_OBJECT.validate_json(response.content)["id"])
+
+
+def _agent_id(candidate: OwnedProxy) -> str:
+    response: Final = candidate.gateway.request("GET", "/v1/agents")
+    assert response.status_code == 200, response.text
+    agents: Final = _JSON_ARRAY.validate_json(response.content)
+    assert len(agents) == 1
+    return string_value(object_value(agents[0])["agent_id"])
+
+
+@pytest.fixture(scope="module")
+def empty_search_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_SearchProxy]:
+    directory: Final = tmp_path_factory.mktemp("search-empty-model")
+    with wire_server(_embedding_reply) as upstream, gateway_from_environment() as gateway:
+        config: Final = _write_config(
+            directory,
+            upstream.url,
+            "os.environ/AGENT_SEARCH_EMBEDDING_MODEL",
+            "os.environ/SKILL_SEARCH_EMBEDDING_MODEL",
+        )
+        with owned_proxy_process(
+            gateway,
+            directory,
+            {"AGENT_SEARCH_EMBEDDING_MODEL": "", "SKILL_SEARCH_EMBEDDING_MODEL": ""},
+            config=config,
+        ) as owned:
+            yield _SearchProxy(owned, upstream, _create_skill(owned), _agent_id(owned))
+
+
+@pytest.fixture(scope="module")
+def configured_search_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_SearchProxy]:
+    directory: Final = tmp_path_factory.mktemp("search-configured-model")
+    with wire_server(_embedding_reply) as upstream, gateway_from_environment() as gateway:
+        config: Final = _write_config(directory, upstream.url, _MODEL_NAME, _MODEL_NAME)
+        with owned_proxy_process(
+            gateway,
+            directory,
+            {"AGENT_SEARCH_EMBEDDING_MODEL": _MODEL_NAME, "SKILL_SEARCH_EMBEDDING_MODEL": _MODEL_NAME},
+            config=config,
+        ) as owned:
+            yield _SearchProxy(owned, upstream, _create_skill(owned), _agent_id(owned))
+
+
+def _assert_not_configured(response_body: JsonValue, expected_error: str, reason: str) -> None:
+    detail: Final = object_value(object_value(response_body)["detail"])
+    assert detail == {"error": expected_error, "message": reason}
+
+
+def _assert_no_embedding_requests(candidate: _SearchProxy) -> None:
+    embedding_requests: Final = tuple(
+        request for request in candidate.wire.drain() if request.method == "POST" and request.target == "/v1/embeddings"
+    )
+    assert embedding_requests == ()
+
+
+def _assert_mcp_search_not_configured(gateway: Gateway, key: str, tool_name: str, reason: str) -> None:
+    response: Final = gateway.request(
+        "POST",
+        "/mcp-rest/tools/call",
+        {"name": tool_name, "arguments": {"query": "echo"}},
+        key=key,
+    )
+    assert response.status_code == 200, response.text
+    result: Final = JSON_OBJECT.validate_json(response.content)
+    assert result.get("isError") is True, response.text
+    content: Final = object_value(list_value(result["content"])[0])
+    text: Final = string_value(content["text"])
+    assert reason in text
+    assert "You passed in model=" not in text
+
+
+def test_empty_agent_embedding_setting_returns_not_configured(empty_search_proxy: _SearchProxy) -> None:
+    response: Final = empty_search_proxy.owned.gateway.request("GET", "/v1/agents?query=echo")
+    assert response.status_code == 400, response.text
+    _assert_not_configured(
+        JSON_OBJECT.validate_json(response.content),
+        "agent_search_not_configured",
+        _AGENT_NOT_CONFIGURED,
+    )
+    _assert_no_embedding_requests(empty_search_proxy)
+
+
+def test_empty_skill_embedding_setting_returns_not_configured(empty_search_proxy: _SearchProxy) -> None:
+    response: Final = empty_search_proxy.owned.gateway.request(
+        "GET", "/v1/skills?custom_llm_provider=litellm_proxy&query=echo"
+    )
+    assert response.status_code == 400, response.text
+    _assert_not_configured(
+        JSON_OBJECT.validate_json(response.content),
+        "skill_search_not_configured",
+        _SKILL_NOT_CONFIGURED,
+    )
+    _assert_no_embedding_requests(empty_search_proxy)
+
+
+def test_empty_agent_embedding_setting_returns_not_configured_through_mcp(empty_search_proxy: _SearchProxy) -> None:
+    with empty_search_proxy.owned.gateway.scenario() as scenario:
+        key: Final = scenario.key(
+            object_permission={
+                "agents": [empty_search_proxy.agent_id],
+                "mcp_tool_search_enabled": True,
+            },
+        )
+        _assert_mcp_search_not_configured(
+            empty_search_proxy.owned.gateway,
+            key,
+            "agent_search",
+            _AGENT_NOT_CONFIGURED,
+        )
+    _assert_no_embedding_requests(empty_search_proxy)
+
+
+def test_empty_skill_embedding_setting_returns_not_configured_through_mcp(empty_search_proxy: _SearchProxy) -> None:
+    with empty_search_proxy.owned.gateway.scenario() as scenario:
+        user_id: Final = scenario.user(user_role="proxy_admin")
+        key: Final = scenario.key(user_id=user_id, object_permission={"mcp_tool_search_enabled": True})
+        _assert_mcp_search_not_configured(
+            empty_search_proxy.owned.gateway,
+            key,
+            "skill_search",
+            _SKILL_NOT_CONFIGURED,
+        )
+    _assert_no_embedding_requests(empty_search_proxy)
+
+
+def test_configured_agent_embedding_model_returns_search_score(configured_search_proxy: _SearchProxy) -> None:
+    configured_search_proxy.wire.drain()
+    response: Final = configured_search_proxy.owned.gateway.request("GET", "/v1/agents?query=echo")
+    assert response.status_code == 200, response.text
+    agents: Final = _JSON_ARRAY.validate_json(response.content)
+    assert len(agents) == 1
+    agent: Final = object_value(agents[0])
+    assert string_value(agent["agent_name"]) == "plain"
+    assert isinstance(agent["search_score"], (int, float))
+    embedding_requests: Final = tuple(
+        request
+        for request in configured_search_proxy.wire.drain()
+        if request.method == "POST" and request.target == "/v1/embeddings"
+    )
+    assert embedding_requests
+
+
+def test_configured_skill_embedding_model_returns_search_score(configured_search_proxy: _SearchProxy) -> None:
+    configured_search_proxy.wire.drain()
+    response: Final = configured_search_proxy.owned.gateway.request(
+        "GET", "/v1/skills?custom_llm_provider=litellm_proxy&query=echo"
+    )
+    assert response.status_code == 200, response.text
+    skills: Final[Sequence[JsonValue]] = list_value(JSON_OBJECT.validate_json(response.content)["data"])
+    matching_skills: Final = tuple(
+        object_value(skill) for skill in skills if object_value(skill)["id"] == configured_search_proxy.skill_id
+    )
+    assert len(matching_skills) == 1
+    assert isinstance(matching_skills[0]["search_score"], (int, float))
+    embedding_requests: Final = tuple(
+        request
+        for request in configured_search_proxy.wire.drain()
+        if request.method == "POST" and request.target == "/v1/embeddings"
+    )
+    assert embedding_requests

tests/integration/mcp/test_mcp_lifecycle.py

@@ -6,6 +6,7 @@ from collections.abc import Mapping
 from contextlib import ExitStack
 from hashlib import sha256
 from pathlib import Path
+from threading import Event
 from typing import Final, cast
 
 import httpx
@@ -24,6 +25,7 @@ from integration._support.generation import LIFECYCLE_SETTINGS, bounded_http_req
 from integration._support.mcp import (
     JsonRpc,
     McpCaller,
+    McpPeer,
     Outcome,
     ScriptedTool,
     call_tool,
@@ -46,6 +48,34 @@ _OBJECTS: Final = TypeAdapter(Mapping[str, object])
 _STRINGS: Final = TypeAdapter(Mapping[str, str])
 
 
+@pytest.mark.timeout(30)
+def test_owned_http_peer_cleans_up_a_disconnected_tool_call() -> None:
+    started: Final = Event()
+
+    async def hang_up(peer: McpPeer) -> None:
+        async with (
+            streamable_http_client(peer.url, terminate_on_close=False) as streams,
+            ClientSession(streams[0], streams[1]) as session,
+        ):
+            await session.initialize()
+            peer.drain()
+            async with asyncio.TaskGroup() as tasks:
+                running: Final = tasks.create_task(session.call_tool("slow", {"seconds": 60}))
+                received: Final = await asyncio.to_thread(
+                    eventually, lambda: tool_calls(peer.drain()), bool, seconds=30
+                )
+                assert len(received) == 1, received
+                params: Final = _OBJECTS.validate_python(_OBJECTS.validate_python(received[0]["body"])["params"])
+                assert (params["name"], params["arguments"]) == ("slow", {"seconds": 60}), received
+                assert await asyncio.to_thread(started.wait, 5), "The MCP tool handler never started"
+                assert not running.done(), running.result().model_dump_json()
+                running.cancel()
+            assert running.cancelled()
+
+    with mcp_peer(rich=True, on_slow=started.set) as peer:
+        asyncio.run(hang_up(peer))
+
+
 @pytest.mark.covers("mcp.call_tool.saved_headers.reach_actual_transport")
 def test_saved_headers_reach_real_mcp_tool_and_survive_unrelated_edit(gateway: Gateway) -> None:
     with mcp_peer() as peer, gateway.scenario() as scenario:

tests/integration/observability/test_ovalix_guardrail.py

@@ -0,0 +1,338 @@
+"""Ovalix guardrail driven through a real proxy.
+
+The Ovalix tracker is the only guardrail double: an owned wire peer that answers the `/tracking/beta` routes
+the way the live tracker does, including allowing a checkpoint call whose `actor` is empty without inspecting
+it. The provider is a second owned peer. The proxy, Postgres and Redis run for real with two workers.
+"""
+
+import json
+import threading
+import uuid
+from collections.abc import Iterator
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from itertools import product
+from pathlib import Path
+from typing import Final
+from urllib.parse import urlsplit
+
+import httpx
+import pytest
+import yaml
+from integration._support.client import Gateway, JsonValue, gateway_from_environment
+from integration._support.process import owned_proxy_process
+from integration._support.wire import Reply, Request, Wire, wire_server
+
+pytestmark: Final = pytest.mark.shares_worker_state
+
+API_KEY: Final = "synthetic-ovalix-key"
+APP_ID: Final = "synthetic-ovalix-app"
+OTHER_APP: Final = "Synthetic Other App"
+PRE_CHECKPOINT: Final = "synthetic-pre-checkpoint"
+POST_CHECKPOINT: Final = "synthetic-post-checkpoint"
+KEYWORD: Final = "Project Falcon"
+NAME: Final = "David"
+DENIAL: Final = "synthetic ovalix denial"
+REPLY: Final = "synthetic provider reply"
+PINNED: Final = "ovalix-pinned"
+ENDPOINTS: Final = ("/v1/chat/completions", "/v1/messages", "/v1/responses")
+CHECKPOINT: Final = "/tracking/beta/checkpoint"
+OUTAGE_AFTER: Final = 6
+
+
+class _Outage:
+    """Takes the tracker double down once it has answered a set number of checkpoint calls."""
+
+    def __init__(self) -> None:
+        self._lock: Final = threading.Lock()
+        self._remaining: int | None = None
+
+    def arm(self, answered: int) -> None:
+        with self._lock:
+            self._remaining = answered
+
+    def clear(self) -> None:
+        with self._lock:
+            self._remaining = None
+
+    def admits(self, path: str) -> bool:
+        with self._lock:
+            if self._remaining is None:
+                return True
+            if self._remaining == 0:
+                return False
+            if path == CHECKPOINT:
+                self._remaining -= 1
+            return True
+
+
+OUTAGE: Final = _Outage()
+
+
+def _checkpoint_verdict(body: dict[str, object]) -> dict[str, object]:
+    data: Final = body["data"]
+    content: Final = str(data.get("content", "")) if isinstance(data, dict) else ""
+    if not body["actor"]:
+        return {"action_type": "allow"}
+    if KEYWORD in content:
+        return {"action_type": "block", "modified_data": {"content": DENIAL}}
+    if NAME in content:
+        return {"action_type": "anonymize", "modified_data": {"content": content.replace(NAME, "{Name}")}}
+    return {"action_type": "allow"}
+
+
+def _tracker(request: Request) -> Reply:
+    if request.headers.get("x-api-key") != API_KEY:
+        return Reply(status=401, body=b'{"detail": "missing api key"}')
+    path: Final = urlsplit(request.target).path
+    if not OUTAGE.admits(path):
+        return Reply(status=503, body=b'{"detail": "tracker unavailable"}')
+    if path != CHECKPOINT:
+        return Reply(status=404, body=b'{"detail": "not found"}')
+    return Reply(body=json.dumps(_checkpoint_verdict(json.loads(request.body))).encode())
+
+
+def _responses_reply(marker: str) -> dict[str, object]:
+    return {
+        "id": "resp_" + marker,
+        "object": "response",
+        "created_at": 1,
+        "status": "completed",
+        "model": "gpt-4o-mini",
+        "output": [
+            {
+                "type": "message",
+                "id": "msg_" + marker,
+                "status": "completed",
+                "role": "assistant",
+                "content": [{"type": "output_text", "text": REPLY, "annotations": []}],
+            }
+        ],
+        "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
+    }
+
+
+def _chat_reply(marker: str) -> dict[str, object]:
+    return {
+        "id": "chatcmpl-" + marker,
+        "object": "chat.completion",
+        "created": 1,
+        "model": "gpt-4o-mini",
+        "choices": [{"index": 0, "message": {"role": "assistant", "content": REPLY}, "finish_reason": "stop"}],
+        "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+    }
+
+
+def _provider(request: Request) -> Reply:
+    marker: Final = uuid.uuid4().hex
+    is_responses: Final = urlsplit(request.target).path.endswith("/responses")
+    return Reply(body=json.dumps(_responses_reply(marker) if is_responses else _chat_reply(marker)).encode())
+
+
+def _alias() -> str:
+    return f"[{OTHER_APP}] integration key {uuid.uuid4().hex}"
+
+
+def _ovalix(name: str, tracker: str, **routing: str) -> dict[str, object]:
+    return {
+        "guardrail_name": name,
+        "litellm_params": {
+            "guardrail": "ovalix",
+            "mode": "pre_call",
+            "default_on": False,
+            "tracker_api_base": tracker,
+            "tracker_api_key": API_KEY,
+            **routing,
+        },
+    }
+
+
+@dataclass(frozen=True, slots=True)
+class Keys:
+    no_user: str
+    user_id: str
+    with_user_id: str
+    email: str
+    with_email: str
+
+
+@dataclass(frozen=True, slots=True)
+class Rig:
+    proxy: Gateway
+    tracker: Wire
+    provider: Wire
+    model: str
+    keys: Keys
+
+
+@pytest.fixture(scope="module")
+def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
+    root: Final = tmp_path_factory.mktemp("ovalix")
+    with (
+        gateway_from_environment() as gateway,
+        wire_server(_tracker) as tracker,
+        wire_server(_provider) as provider,
+    ):
+        config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
+        config["litellm_settings"]["cache"] = False
+        config["guardrails"] = [
+            _ovalix(
+                PINNED,
+                tracker.url,
+                application_id=APP_ID,
+                pre_checkpoint_id=PRE_CHECKPOINT,
+                post_checkpoint_id=POST_CHECKPOINT,
+            ),
+        ]
+        path: Final = root / "ovalix.yaml"
+        path.write_text(yaml.safe_dump(config))
+        with (
+            owned_proxy_process(gateway, root, {}, config=path, workers=2) as owned,
+            owned.gateway.scenario() as scenario,
+        ):
+            model: Final = scenario.model(
+                model="openai/gpt-4o-mini", api_base=provider.url + "/v1", api_key="synthetic-openai-key"
+            )
+            user_id: Final = scenario.user()
+            email: Final = f"ovalix-{uuid.uuid4().hex}@example.com"
+            keys: Final = Keys(
+                no_user=scenario.key(key_alias=_alias()),
+                user_id=user_id,
+                with_user_id=scenario.key(key_alias=_alias(), user_id=user_id),
+                email=email,
+                with_email=scenario.key(key_alias=_alias(), user_id=scenario.user(user_email=email)),
+            )
+            yield Rig(owned.gateway, tracker, provider, model, keys)
+
+
+def _post(rig: Rig, endpoint: str, key: str, text: str, extra: dict[str, JsonValue] | None = None) -> httpx.Response:
+    prompt: Final[dict[str, JsonValue]] = (
+        {"input": text}
+        if endpoint == "/v1/responses"
+        else {"max_tokens": 64, "messages": [{"role": "user", "content": text}]}
+        if endpoint == "/v1/messages"
+        else {"messages": [{"role": "user", "content": text}]}
+    )
+    return rig.proxy.request(
+        "POST", endpoint, {"model": rig.model, "guardrails": [PINNED], **prompt, **(extra or {})}, key=key
+    )
+
+
+def _send(rig: Rig, endpoint: str, key: str, text: str, extra: dict[str, JsonValue] | None = None) -> httpx.Response:
+    rig.tracker.drain()
+    rig.provider.drain()
+    return _post(rig, endpoint, key, text, extra)
+
+
+def _checkpoint_actors(rig: Rig) -> list[object]:
+    return [
+        json.loads(request.body)["actor"]
+        for request in rig.tracker.drain()
+        if urlsplit(request.target).path == CHECKPOINT
+    ]
+
+
+@pytest.mark.parametrize("stream", (False, True))
+@pytest.mark.parametrize("endpoint", ENDPOINTS)
+def test_a_prompt_from_a_key_with_no_user_is_inspected(rig: Rig, endpoint: str, stream: bool) -> None:
+    response: Final = _send(
+        rig, endpoint, rig.keys.no_user, f"Tell me about {KEYWORD} {uuid.uuid4()}", {"stream": stream}
+    )
+
+    assert response.status_code == 400 and DENIAL in response.text, response.text
+    assert rig.provider.drain() == ()
+    assert _checkpoint_actors(rig) == ["unknown"]
+
+
+@pytest.mark.parametrize(
+    ("endpoint", "client_metadata"),
+    (("/v1/messages", {"user_id": "client-chosen"}), ("/v1/responses", {"client": "chosen"})),
+)
+def test_a_client_metadata_object_does_not_switch_inspection_off(
+    rig: Rig, endpoint: str, client_metadata: dict[str, JsonValue]
+) -> None:
+    response: Final = _send(
+        rig,
+        endpoint,
+        rig.keys.with_user_id,
+        f"Tell me about {KEYWORD} {uuid.uuid4()}",
+        {"metadata": client_metadata},
+    )
+
+    assert response.status_code == 400 and DENIAL in response.text, response.text
+    assert rig.provider.drain() == ()
+    assert _checkpoint_actors(rig) == ["unknown"]
+
+
+def test_a_name_from_a_key_with_no_user_reaches_the_provider_anonymized(rig: Rig) -> None:
+    marker: Final = uuid.uuid4().hex
+    response: Final = _send(rig, "/v1/chat/completions", rig.keys.no_user, f"My name is {NAME} {marker}")
+
+    assert response.status_code == 200, response.text
+    sent: Final = [json.loads(request.body)["messages"][-1]["content"] for request in rig.provider.drain()]
+    assert sent == [f"My name is {{Name}} {marker}"]
+
+
+def test_the_actor_is_the_key_users_email_then_their_id(rig: Rig) -> None:
+    by_id: Final = _send(rig, "/v1/chat/completions", rig.keys.with_user_id, f"hello {uuid.uuid4()}")
+    assert by_id.status_code == 200, by_id.text
+    assert _checkpoint_actors(rig) == [rig.keys.user_id]
+
+    by_email: Final = _send(rig, "/v1/chat/completions", rig.keys.with_email, f"hello {uuid.uuid4()}")
+    assert by_email.status_code == 200, by_email.text
+    assert _checkpoint_actors(rig) == [rig.keys.email]
+
+
+@pytest.mark.parametrize("endpoint", ENDPOINTS)
+def test_a_key_alias_naming_another_app_does_not_change_the_application_checked(rig: Rig, endpoint: str) -> None:
+    response: Final = _send(rig, endpoint, rig.keys.with_email, f"Tell me about {KEYWORD} {uuid.uuid4()}")
+
+    assert response.status_code == 400 and DENIAL in response.text, response.text
+    assert rig.provider.drain() == ()
+    calls: Final = [(urlsplit(request.target).path, json.loads(request.body)) for request in rig.tracker.drain()]
+    assert [
+        (path, body["application_id"], body["checkpoint_id"], "application_name" in body) for path, body in calls
+    ] == [(CHECKPOINT, APP_ID, PRE_CHECKPOINT, False)]
+
+
+def _burst(rig: Rig) -> tuple[httpx.Response, ...]:
+    plan: Final = tuple(product(ENDPOINTS, (False, True), range(4)))
+    with ThreadPoolExecutor(max_workers=12) as pool:
+        return tuple(
+            pool.map(
+                lambda cell: _post(
+                    rig,
+                    cell[0],
+                    rig.keys.no_user,
+                    f"Tell me about {KEYWORD} {uuid.uuid4()}",
+                    {"stream": cell[1]},
+                ),
+                plan,
+            )
+        )
+
+
+def test_a_tracker_outage_mid_burst_fails_closed_and_recovers(rig: Rig) -> None:
+    rig.tracker.drain()
+    rig.provider.drain()
+    OUTAGE.arm(OUTAGE_AFTER)
+    try:
+        during: Final = _burst(rig)
+    finally:
+        OUTAGE.clear()
+
+    denied: Final = [response for response in during if response.status_code == 400 and DENIAL in response.text]
+    failed: Final = [response for response in during if response.status_code >= 400 and DENIAL not in response.text]
+    assert len(denied) == OUTAGE_AFTER, [response.text for response in during]
+    assert len(failed) == len(during) - OUTAGE_AFTER, [response.text for response in during]
+    assert all("Ovalix guardrail error" in response.text for response in failed), [response.text for response in failed]
+    assert rig.provider.drain() == ()
+    assert set(_checkpoint_actors(rig)) == {"unknown"}
+
+    after: Final = _burst(rig)
+
+    assert all(response.status_code == 400 and DENIAL in response.text for response in after), [
+        response.text for response in after
+    ]
+    assert rig.provider.drain() == ()
+    assert _checkpoint_actors(rig) == ["unknown"] * len(after)

tests/integration/sdk/test_router_concurrent_deployment_delete_sdk.py

@@ -8,7 +8,7 @@ import threading
 import uuid
 from collections.abc import AsyncIterable, Awaitable, Callable, Iterable
 from concurrent.futures import ThreadPoolExecutor
-from types import FrameType
+from types import FrameType, MappingProxyType
 from typing import Final, TypeVar
 
 import pytest
@@ -28,12 +28,23 @@ _BURST_THREADS: Final = 8
 _BURST_CALLS: Final = 24
 _CHURN_UPSERTS: Final = 120
 _UNREPORTED_MODEL_ID: Final = frozenset({"aresponses-stream", "aanthropic-messages"})
-_ROUTER_FILE: Final = Router.__init__.__code__.co_filename
-_INDEX_LOCK_LINES: Final = frozenset(
-    number
-    for number, line in enumerate(linecache.getlines(_ROUTER_FILE), start=1)
-    if "with self._deployment_index_lock" in line
-)
+_REINDEX: Final = Router._update_deployment_indices_after_removal.__code__
+
+
+def _router_source_files() -> frozenset[str]:
+    facade_files: Final = (sys.modules[cls.__module__].__file__ for cls in Router.__mro__ if cls is not object)
+    return frozenset({_REINDEX.co_filename, *(path for path in facade_files if path is not None)})
+
+
+def _lock_lines(path: str) -> frozenset[int]:
+    return frozenset(
+        number
+        for number, line in enumerate(linecache.getlines(path), start=1)
+        if "with self._deployment_index_lock" in line
+    )
+
+
+_INDEX_LOCK_LINES: Final = MappingProxyType({path: _lock_lines(path) for path in _router_source_files()})
 
 T = TypeVar("T")
 
@@ -80,7 +91,6 @@ def _read_during_held_write(read: Callable[[], T], write: Callable[[], object])
     """Run `read` on its own thread while `write` waits where the list has shrunk but the index maps have not caught
     up. The write resumes once `read` returns or reaches the router's index lock, so a read that skips the lock meets
     the half-updated list and a read that takes the lock waits for the write"""
-    reindex: Final = Router._update_deployment_indices_after_removal.__code__
     held: Final = threading.Event()
     release: Final = threading.Event()
     settled: Final = threading.Event()
@@ -89,7 +99,7 @@ def _read_during_held_write(read: Callable[[], T], write: Callable[[], object])
     failure: Final[list[BaseException]] = []
 
     def hold_at_reindex(frame: FrameType, event: str, _arg: object) -> None:
-        if event == "call" and frame.f_code is reindex and not held.is_set():
+        if event == "call" and frame.f_code is _REINDEX and not held.is_set():
             held.set()
             released_in_time.append(release.wait(timeout=_SYNC_SECONDS))
 
@@ -102,9 +112,10 @@ def _read_during_held_write(read: Callable[[], T], write: Callable[[], object])
             sys.setprofile(previous)
 
     def watch_read(frame: FrameType, event: str, _arg: object) -> Callable[..., object] | None:
-        if frame.f_code.co_filename != _ROUTER_FILE:
+        lock_lines: Final = _INDEX_LOCK_LINES.get(frame.f_code.co_filename)
+        if lock_lines is None:
             return None
-        if event == "line" and frame.f_lineno in _INDEX_LOCK_LINES:
+        if event == "line" and frame.f_lineno in lock_lines:
             settled.set()
         return watch_read
 

tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py

@@ -152,15 +152,14 @@ def _place_delta(partial_json: str) -> SSEFrame:
     )
 
 
-def _place_chunk(delta: dict[str, JsonValue], *, finish_reason: str | None = None) -> SSEFrame:
-    finish: Final = {} if finish_reason is None else {"finish_reason": finish_reason}
+def _place_chunk(delta: dict[str, JsonValue]) -> SSEFrame:
     return SSEFrame(
         data={
             "id": ANY,
             "object": "chat.completion.chunk",
             "created": ANY,
             "model": "anthropic/claude-haiku-5-5",
-            "choices": [{"index": 0, "delta": delta, **finish}],
+            "choices": [{"index": 0, "delta": delta}],
         }
     )
 
@@ -245,7 +244,19 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationT
         _place_chunk({"content": 'try": "'}),
         _place_chunk({"content": 'France"'}),
         _place_chunk({"content": "}"}),
-        _place_chunk({}, finish_reason="stop"),
+        SSEFrame(
+            data={
+                "error": {
+                    "message": (
+                        "litellm.APIConnectionError: AnthropicException - "
+                        "Stream ended before a finish_reason was received"
+                    ),
+                    "type": "internal_server_error",
+                    "param": None,
+                    "code": "500",
+                }
+            }
+        ),
         SSEFrame(data="[DONE]"),
     ),
     expected_litellm_response={
@@ -257,7 +268,6 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationT
             {
                 "index": 0,
                 "message": {"role": "assistant", "content": '{"city": "Paris", "country": "France"}'},
-                "finish_reason": "stop",
             }
         ],
     },

tests/integration/translation/runner.py

@@ -129,8 +129,9 @@ def _join_stream_frames(endpoint: str, frames: tuple[SSEFrame, ...]) -> Mapping[
         case "/v1/chat/completions":
             state: Final = ChatCompletionStreamState()
             for frame in frames:
-                if frame.data != "[DONE]":
-                    state.handle_chunk(ChatCompletionChunk.model_validate(frame.data))
+                if frame.data == "[DONE]" or (isinstance(frame.data, Mapping) and "error" in frame.data):
+                    continue
+                state.handle_chunk(ChatCompletionChunk.model_validate(frame.data))
             return JSON_OBJECT.validate_python(state.get_final_completion().model_dump(mode="json", exclude_none=True))
         case _:
             raise AssertionError(f"no stream accumulator for {endpoint}")

tests/unit/caching/test_request_redis_batch_pre_call.py

@@ -290,7 +290,11 @@ def _router(redis_cache: FakeRedisCache, routing_strategy: str = "usage-based-ro
 
 
 @pytest.mark.asyncio
-async def test_armed_routing_read_rides_the_admission_pipeline_and_routing_issues_no_read_of_its_own():
+async def test_armed_routing_read_rides_the_admission_pipeline_and_routing_issues_no_read_of_its_own(monkeypatch):
+    monkeypatch.setattr(
+        "litellm.router_strategy.lowest_tpm_rpm_v2.get_utc_datetime",
+        lambda: datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc),
+    )
     client = FakeClient(_lua_ok_replies)
     redis_cache = FakeRedisCache(client)
     router = _router(redis_cache)

tests/unit/enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py

@@ -29,7 +29,7 @@ async def test_apply_guardrail_endpoint_returns_correct_response(
         mock_proxy_logging_ctx(),
     ):
         # Create a mock guardrail
-        mock_guardrail = Mock(spec=CustomGuardrail)
+        mock_guardrail = Mock(spec=CustomGuardrail, event_hook=None)
         # Apply guardrail returns GenericGuardrailAPIInputs (dict with texts key)
         mock_guardrail.apply_guardrail = AsyncMock(
             return_value={"texts": ["Redacted text: [REDACTED] and [REDACTED]"]}
@@ -116,7 +116,7 @@ async def test_apply_guardrail_endpoint_with_presidio_guardrail(mock_proxy_loggi
         mock_proxy_logging_ctx(),
     ):
         # Create a mock guardrail that simulates Presidio behavior
-        mock_guardrail = Mock(spec=CustomGuardrail)
+        mock_guardrail = Mock(spec=CustomGuardrail, event_hook=None)
         # Simulate masking PII entities - returns GenericGuardrailAPIInputs (dict with texts key)
         mock_guardrail.apply_guardrail = AsyncMock(
             return_value={
@@ -168,7 +168,7 @@ async def test_apply_guardrail_endpoint_without_optional_params(mock_proxy_loggi
         mock_proxy_logging_ctx(),
     ):
         # Create a mock guardrail
-        mock_guardrail = Mock(spec=CustomGuardrail)
+        mock_guardrail = Mock(spec=CustomGuardrail, event_hook=None)
         # Returns GenericGuardrailAPIInputs (dict with texts key)
         mock_guardrail.apply_guardrail = AsyncMock(
             return_value={"texts": ["Processed text"]}

tests/unit/integration_support/test_upstream.py

@@ -112,6 +112,39 @@ async def test_mcp_notification_ordering_preserves_other_responses(
     assert outgoing == list(messages)
 
 
+@pytest.mark.asyncio
+async def test_mcp_http_disconnect_cancels_an_in_flight_handler(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2]))
+    from integration._support.mcp import _cancel_on_disconnect
+    from starlette.types import Receive, Send
+
+    started: Final = asyncio.Event()
+    stopped: Final = asyncio.Event()
+    body: Final[Message] = {"type": "http.request", "body": b"tool-call", "more_body": False}
+    pending: Final = iter((body,))
+
+    async def receive() -> Message:
+        message: Final = next(pending, None)
+        if message is not None:
+            return message
+        await started.wait()
+        return {"type": "http.disconnect"}
+
+    async def send(message: Message) -> None:
+        pytest.fail(f"Disconnected handler sent a response: {message}")
+
+    async def app(scope: Scope, receive: Receive, send: Send) -> None:
+        assert await receive() == body
+        started.set()
+        try:
+            await asyncio.Future[None]()
+        finally:
+            stopped.set()
+
+    await asyncio.wait_for(_cancel_on_disconnect(app)({"type": "http"}, receive, send), 2)
+    assert stopped.is_set(), "The disconnected HTTP handler was left running"
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("failure", (RuntimeError("dispatch failed"), asyncio.CancelledError()))
 async def test_mcp_notification_dispatch_failure_is_not_acknowledged(

tests/unit/integrations/test_anthropic_cache_control_hook.py

@@ -5042,3 +5042,50 @@ def test_cache_control_hook_injects_next_to_a_malformed_message_instead_of_raisi
         {"type": "ephemeral"},
         {"type": "ephemeral"},
     ]
+
+
+def _search_conversation_with(follow_up: dict[str, object]) -> list[AllMessageValues]:
+    return [
+        cast(AllMessageValues, {"role": "user", "content": "Weather in Paris?"}),
+        _replayed_search_call(_SEARCH_CALL),
+        cast(AllMessageValues, follow_up),
+        cast(AllMessageValues, _GO_ON),
+    ]
+
+
+def _cache_marks_after_injection(messages: list[AllMessageValues]) -> tuple[int, tuple[int, ...]]:
+    _, processed, _ = AnthropicCacheControlHook().get_chat_completion_prompt(
+        model="anthropic/claude-opus-5-5",
+        messages=messages,
+        non_default_params={"cache_control_injection_points": [{"location": "message", "role": "user"}]},
+        prompt_id=None,
+        prompt_variables=None,
+        dynamic_callback_params={},
+    )
+    return (
+        AnthropicCacheControlHook.count_request_cache_breakpoints(messages),
+        tuple(_marks_in(message) for message in processed),
+    )
+
+
+@pytest.mark.parametrize(
+    ("malformed", "well_formed"),
+    [
+        pytest.param(
+            {**_SEARCH_ANSWER, "tool_call_id": {"id": "srvtoolu_01Search"}},
+            {**_SEARCH_ANSWER, "tool_call_id": "toolu_01Unrelated"},
+            id="object_tool_call_id",
+        ),
+        pytest.param(
+            {**_SEARCH_ANSWER, "tool_call_id": ["srvtoolu_01Search"]},
+            {**_SEARCH_ANSWER, "tool_call_id": "toolu_01Unrelated"},
+            id="list_tool_call_id",
+        ),
+    ],
+)
+def test_cache_control_hook_reads_a_non_string_answer_id_as_answering_no_tool_call(
+    malformed: dict[str, object], well_formed: dict[str, object]
+):
+    assert _cache_marks_after_injection(_search_conversation_with(malformed)) == _cache_marks_after_injection(
+        _search_conversation_with(well_formed)
+    )

tests/unit/litellm_core_utils/test_api_route_to_call_types.py

@@ -8,6 +8,8 @@ Regression coverage for the guardrail route table bugs:
   take call_types[0] to a handler-less type
 """
 
+from typing import Final
+
 from litellm.litellm_core_utils.api_route_to_call_types import (
     get_call_types_for_route,
 )
@@ -103,6 +105,16 @@ class TestExistingRouteResolutionUnchanged:
         assert call_types is not None
         assert list(call_types) == [CallTypes.asend_message, CallTypes.send_message]
 
+    def test_single_segment_placeholder_message_stream_resolves(self):
+        call_types: Final = get_call_types_for_route("/a2a/my-agent/message/stream")
+        assert call_types is not None
+        assert list(call_types) == [CallTypes.asend_message, CallTypes.send_message]
+
+    def test_v1_single_segment_placeholder_message_stream_resolves(self):
+        call_types: Final = get_call_types_for_route("/v1/a2a/my-agent/message/stream")
+        assert call_types is not None
+        assert list(call_types) == [CallTypes.asend_message, CallTypes.send_message]
+
     def test_longer_route_does_not_collapse_into_bare_placeholder_pattern(self):
         call_types = get_call_types_for_route("/responses/resp_123/input_items")
         assert call_types is not None

tests/unit/litellm_core_utils/test_core_helpers.py

@@ -319,6 +319,12 @@ class TestMapFinishReasonCohere:
     def test_error(self):
         assert map_finish_reason("ERROR") == "stop"
 
+    def test_tool_call(self):
+        assert map_finish_reason("TOOL_CALL") == "tool_calls"
+
+    def test_stop_sequence(self):
+        assert map_finish_reason("STOP_SEQUENCE") == "stop"
+
 
 class TestMapFinishReasonHuggingFace:
     def test_eos_token(self):

tests/unit/litellm_core_utils/test_json_fragment_accumulator.py

@@ -1,6 +1,5 @@
 import json
 import time
-import tracemalloc
 from collections.abc import Iterator
 from typing import Final
 from unittest.mock import patch
@@ -193,34 +192,15 @@ def _streamed(fragments: tuple[str, ...]) -> JSONFragmentAccumulator:
     return accumulator
 
 
-def _drain_peak_bytes(fragments: tuple[str, ...]) -> int:
-    accumulator: Final = _streamed(fragments)
-    first_found, _ = accumulator.pop_next_value()
-    assert first_found is True
-    already_tracing: Final = tracemalloc.is_tracing()
-    if not already_tracing:
-        tracemalloc.start()
-    tracemalloc.reset_peak()
-    baseline, _ = tracemalloc.get_traced_memory()
-    try:
-        drained: Final = sum(1 for _ in _popped_values(accumulator))
-        _, peak = tracemalloc.get_traced_memory()
-    finally:
-        if not already_tracing:
-            tracemalloc.stop()
-    assert drained == len(fragments) - 1
-    return peak - baseline
-
-
-def test_draining_many_concatenated_values_never_copies_the_remainder():
+def test_draining_many_concatenated_values_yields_every_value_in_order():
     value: Final = '{"text": "' + "x" * 20 + '"}'
     fragments: Final = tuple(value if index % 2 else f" {value}" for index in range(40_000))
-    payload_bytes: Final = sum(len(fragment) for fragment in fragments)
-    peak_bytes: Final = _drain_peak_bytes(fragments)
-    assert peak_bytes < payload_bytes // 4, (
-        f"draining {len(fragments)} values allocated a peak of {peak_bytes} bytes over a {payload_bytes} byte payload, "
-        "copying the remainder on every pop?"
-    )
+    accumulator: Final = _streamed(fragments)
+
+    drained: Final = tuple(_popped_values(accumulator))
+
+    assert drained == (json.loads(value),) * len(fragments)
+    assert accumulator.pop_next_value() == (False, None)
 
 
 def test_could_close_json_after_many_blank_fragments_is_not_quadratic():

tests/unit/litellm_core_utils/test_streaming_handler.py

@@ -7176,6 +7176,80 @@ def test_rebuilding_a_priced_stream_bills_the_margin_once(monkeypatch: pytest.Mo
     )
 
 
+@pytest.mark.parametrize("sync_mode", [True, False])
+@pytest.mark.parametrize("has_served_model", [True, False])
+@pytest.mark.parametrize(
+    ("wrapper_provider", "logging_provider"),
+    [("openrouter", "openrouter"), ("cached_response", "openrouter"),
+     ("openai", "openai"), ("cached_response", "openai")],
+)
+@pytest.mark.asyncio
+async def test_openrouter_served_model_survives_blank_chunks_usage_and_cache_replay(
+    sync_mode: bool, has_served_model: bool, wrapper_provider: str, logging_provider: str
+) -> None:
+    selected_model: Final = "selected-model" if has_served_model else ""
+    expected_model: Final = selected_model if logging_provider == "openrouter" and selected_model else "auto"
+    logging_obj: Final = Logging(
+        model="auto",
+        messages=[{"role": "user", "content": "Say hi"}],
+        stream=True,
+        call_type="completion" if sync_mode else "acompletion",
+        start_time=1.0,
+        litellm_call_id="openrouter-model-identity",
+        function_id="test",
+    )
+    logging_obj.stream_options = {"include_usage": True}
+    logging_obj.model_call_details["custom_llm_provider"] = (
+        None if wrapper_provider == "cached_response" else logging_provider
+    )
+    chunks: Final = [
+        ModelResponseStream(
+            id="served-stream", created=1, model=None,
+            choices=[StreamingChoices(index=0, delta=Delta(role="assistant"))],
+        ),
+        ModelResponseStream(
+            id="served-stream", created=1, model=selected_model,
+            choices=[StreamingChoices(index=0, delta=Delta(content="hello"))],
+            usage=Usage(prompt_tokens=3, completion_tokens=2, total_tokens=5),
+        ),
+        ModelResponseStream(
+            id="served-stream", created=1, model=None,
+            choices=[StreamingChoices(index=0, delta=Delta(content=" world"))],
+        ),
+        ModelResponseStream(
+            id="served-stream", created=1, model="",
+            choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")],
+        ),
+        ModelResponseStream(
+            id="served-stream", created=1, model=None, choices=[],
+            usage=Usage(prompt_tokens=3, completion_tokens=2, total_tokens=5),
+        ),
+    ]
+    wrapper: Final = CustomStreamWrapper(
+        completion_stream=ModelResponseListIterator(model_responses=chunks),
+        model="auto", logging_obj=logging_obj, custom_llm_provider=wrapper_provider,
+        stream_options={"include_usage": True},
+    )
+    logging_obj.update_environment_variables(
+        litellm_params={}, optional_params={}, model="auto", custom_llm_provider=logging_provider,
+    )
+    first: Final = next(wrapper) if sync_mode else await wrapper.__anext__()
+    second: Final = next(wrapper) if sync_mode else await wrapper.__anext__()
+    partial: Final = wrapper.partial_response()
+    emitted: Final = [first, second, *(await _collect(wrapper, sync_mode))]
+    assembled: Final = litellm.stream_chunk_builder(chunks=emitted)
+
+    assert first.model == "auto"
+    assert {chunk.model for chunk in emitted[1:]} == {expected_model}
+    assert partial is not None and partial.model == expected_model
+    assert assembled is not None and assembled.model == expected_model
+    assert assembled.choices[0].message.content == "hello world"
+    assert emitted[-1].choices == []
+    assert emitted[-1].usage is not None and emitted[-1].usage.total_tokens == 5
+    assert second._hidden_params.get("provider_response_model") == (selected_model or None)
+    assert wrapper.model == logging_obj.model == logging_obj.model_call_details["model"] == "auto"
+
+
 def test_chunk_creator_keeps_provider_model_private_across_stream():
     from litellm.litellm_core_utils.add_retry_fallback_headers import (
         get_hidden_params_dict,

tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py

@@ -25,7 +25,7 @@ from litellm.constants import (
 from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
 from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
 from litellm.llms.anthropic.chat import ModelResponseIterator
-from litellm.llms.anthropic.common_utils import process_anthropic_headers
+from litellm.llms.anthropic.common_utils import AnthropicError, process_anthropic_headers
 from litellm.llms.anthropic.chat.transformation import AnthropicConfig
 from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema
 from litellm.llms.anthropic.messages.native.transformation import (
@@ -9922,6 +9922,382 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key():
     assert litellm_params["metadata"] == {}
 
 
+_KEPT_SEARCH_CALL: Final = {
+    "role": "assistant",
+    "content": None,
+    "tool_calls": [
+        {
+            "id": "srvtoolu_01Search",
+            "type": "function",
+            "function": {"name": "tool_search_tool_bm25", "arguments": "{}"},
+        }
+    ],
+    "provider_specific_fields": {
+        "tool_results": [
+            {
+                "type": "tool_search_tool_result",
+                "tool_use_id": "srvtoolu_01Search",
+                "content": {"type": "tool_search_tool_search_result", "tool_references": []},
+            }
+        ]
+    },
+}
+_WEATHER_CALL: Final = {
+    "role": "assistant",
+    "content": None,
+    "tool_calls": [{"id": "toolu_01", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}],
+}
+_ASK: Final = {"role": "user", "content": "Weather in Paris?"}
+
+
+def _search_answered_with(tool_call_id: object) -> list[dict[str, object]]:
+    return [
+        _ASK,
+        _KEPT_SEARCH_CALL,
+        {"role": "tool", "tool_call_id": tool_call_id, "content": "found get_weather"},
+        {"role": "user", "content": "Go on"},
+    ]
+
+
+def _orphan_answer_with(tool_call_id: object) -> list[dict[str, object]]:
+    return [
+        _ASK,
+        {"role": "assistant", "content": "Checking"},
+        {"role": "tool", "tool_call_id": tool_call_id, "content": "sunny"},
+        {"role": "user", "content": "Go on"},
+    ]
+
+
+@pytest.mark.parametrize(
+    ("messages", "field_path", "type_name"),
+    [
+        pytest.param(
+            [
+                {"role": "user", "content": "Hi"},
+                {"role": "assistant", "content": "Hello"},
+                {"role": ["user"], "content": "Go on"},
+                {"role": "system", "content": "Answer in one sentence."},
+            ],
+            "messages[2].role",
+            "list",
+            id="list_role_before_a_later_system_message",
+        ),
+        pytest.param(
+            [
+                {"role": "user", "content": "Hi"},
+                {"role": "assistant", "content": "Hello"},
+                {"role": ["user"], "content": "Go on"},
+            ],
+            "messages[2].role",
+            "list",
+            id="list_role",
+        ),
+        pytest.param(
+            [
+                {"role": "user", "content": "Hi"},
+                {"role": "assistant", "content": "Hello"},
+                {"role": 123, "content": "Go on"},
+                {"role": "system", "content": "Answer in one sentence."},
+            ],
+            "messages[2].role",
+            "int",
+            id="int_role_before_a_later_system_message",
+        ),
+        pytest.param(
+            [*_orphan_answer_with({"id": "toolu_01"})[:3], {"role": True, "content": "Go on"}],
+            "messages[3].role",
+            "bool",
+            id="bool_role_after_an_orphan_object_tool_call_id",
+        ),
+        pytest.param(
+            _search_answered_with({"id": "srvtoolu_01Search"}),
+            "messages[2].tool_call_id",
+            "dict",
+            id="object_tool_call_id_after_a_kept_server_result",
+        ),
+        pytest.param(
+            _search_answered_with(["srvtoolu_01Search"]),
+            "messages[2].tool_call_id",
+            "list",
+            id="list_tool_call_id_after_a_kept_server_result",
+        ),
+        pytest.param(
+            [_ASK, _WEATHER_CALL, {"role": "tool", "tool_call_id": {"id": "toolu_01"}, "content": "sunny"}],
+            "messages[2].tool_call_id",
+            "dict",
+            id="object_tool_call_id_answering_a_client_tool_call",
+        ),
+        pytest.param(
+            [
+                _ASK,
+                _WEATHER_CALL,
+                {"role": "function", "name": "get_weather", "tool_call_id": {"id": "toolu_01"}, "content": "sunny"},
+            ],
+            "messages[2].tool_call_id",
+            "dict",
+            id="object_tool_call_id_on_a_function_message",
+        ),
+        pytest.param(
+            [
+                _ASK,
+                {**_WEATHER_CALL, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "toolu_01"}}]},
+                {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"},
+            ],
+            "messages[1].tool_calls[0].id",
+            "dict",
+            id="object_assistant_tool_call_id",
+        ),
+        pytest.param(
+            [
+                _ASK,
+                {**_WEATHER_CALL, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": 7}]},
+                {"role": "tool", "tool_call_id": "7", "content": "sunny"},
+            ],
+            "messages[1].tool_calls[0].id",
+            "int",
+            id="int_assistant_tool_call_id",
+        ),
+        pytest.param(
+            [*_orphan_answer_with({"id": "toolu_01"})[:3], {"role": ["user"], "content": "Go on"}],
+            "messages[3].role",
+            "list",
+            id="list_role_after_an_orphan_object_tool_call_id",
+        ),
+        pytest.param(
+            [
+                *_orphan_answer_with({"id": "toolu_01"})[:3],
+                {**_WEATHER_CALL, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "toolu_01"}}]},
+                {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"},
+            ],
+            "messages[3].tool_calls[0].id",
+            "dict",
+            id="object_assistant_tool_call_id_after_an_orphan_object_tool_call_id",
+        ),
+        pytest.param(
+            [
+                {**_ASK, "tool_call_id": {"id": "stray"}},
+                _WEATHER_CALL,
+                {"role": "tool", "tool_call_id": {"id": "toolu_01"}, "content": "sunny"},
+            ],
+            "messages[2].tool_call_id",
+            "dict",
+            id="object_tool_call_id_after_a_stray_id_on_a_user_message",
+        ),
+        pytest.param(
+            [
+                {**_ASK, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "stray"}}]},
+                {**_WEATHER_CALL, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "toolu_01"}}]},
+                {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"},
+            ],
+            "messages[1].tool_calls[0].id",
+            "dict",
+            id="object_assistant_tool_call_id_after_stray_tool_calls_on_a_user_message",
+        ),
+        pytest.param(
+            [_ASK, {"role": [], "content": "ok"}, {"role": "system", "content": "Be terse."}],
+            "messages[1].role",
+            "list",
+            id="empty_list_role_before_a_system_message",
+        ),
+        pytest.param(
+            [_ASK, {"role": {}, "content": "ok"}, {"role": "user", "content": "again"}],
+            "messages[1].role",
+            "dict",
+            id="empty_object_role",
+        ),
+    ],
+)
+@pytest.mark.parametrize("modify_params", [False, True])
+@pytest.mark.parametrize("model", [UNFLAGGED_CLAUDE, FLAGGED_CLAUDE])
+def test_chat_transform_request_answers_400_naming_a_non_string_role_or_tool_call_id(
+    local_model_cost_map,
+    monkeypatch: pytest.MonkeyPatch,
+    messages: list[dict[str, object]],
+    field_path: str,
+    type_name: str,
+    modify_params: bool,
+    model: str,
+):
+    monkeypatch.setattr(litellm, "modify_params", modify_params)
+
+    with pytest.raises(AnthropicError) as raised:
+        _chat_request(AnthropicConfig(), model, copy.deepcopy(messages))
+
+    assert (raised.value.status_code, raised.value.message) == (
+        400,
+        f"Invalid type for '{field_path}': expected a string, got {type_name}",
+    )
+
+
+@pytest.mark.parametrize(("role", "type_name"), [(["user"], "list"), ({"name": "user"}, "dict"), ([], "list")])
+def test_chat_transform_request_names_a_non_string_role_on_a_cache_marked_message_under_drop_params(
+    local_model_cost_map, role: object, type_name: str
+):
+    marked: Final = {"role": role, "content": [{"type": "text", "text": "ok"}], "cache_control": {"type": "ephemeral"}}
+
+    with pytest.raises(AnthropicError) as raised:
+        AnthropicConfig().transform_request(
+            model=UNFLAGGED_CLAUDE,
+            messages=[_ASK, marked],
+            optional_params={},
+            litellm_params={"drop_params": True},
+            headers={},
+        )
+
+    assert (raised.value.status_code, raised.value.message) == (
+        400,
+        f"Invalid type for 'messages[1].role': expected a string, got {type_name}",
+    )
+
+
+@pytest.mark.parametrize(
+    ("malformed", "well_formed", "modify_params"),
+    [
+        pytest.param(
+            [{**_ASK, "tool_call_id": {"id": "stray"}}], [_ASK], False, id="stray_tool_call_id_on_a_user_message"
+        ),
+        pytest.param(
+            [{**_ASK, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "toolu_01"}}]}],
+            [_ASK],
+            False,
+            id="stray_tool_calls_on_a_user_message",
+        ),
+        pytest.param(
+            _orphan_answer_with({"id": "toolu_01"}),
+            _orphan_answer_with("toolu_01"),
+            True,
+            id="orphan_answer_dropped_under_modify_params",
+        ),
+    ],
+)
+def test_chat_transform_request_keeps_converting_a_non_string_id_it_never_reads(
+    local_model_cost_map,
+    monkeypatch: pytest.MonkeyPatch,
+    malformed: list[dict[str, object]],
+    well_formed: list[dict[str, object]],
+    modify_params: bool,
+):
+    monkeypatch.setattr(litellm, "modify_params", modify_params)
+
+    converted: Final = _chat_request(AnthropicConfig(), UNFLAGGED_CLAUDE, copy.deepcopy(malformed))
+
+    assert (
+        converted["messages"]
+        == _chat_request(AnthropicConfig(), UNFLAGGED_CLAUDE, copy.deepcopy(well_formed))["messages"]
+    )
+
+
+def test_chat_transform_request_keeps_a_conversion_error_that_a_dropped_orphan_answer_did_not_cause(
+    local_model_cost_map, monkeypatch: pytest.MonkeyPatch
+):
+    monkeypatch.setattr(litellm, "modify_params", True)
+
+    with pytest.raises(AnthropicError) as raised:
+        _chat_request(
+            AnthropicConfig(),
+            UNFLAGGED_CLAUDE,
+            [*_orphan_answer_with({"id": "toolu_01"})[:3], {"role": "bogus", "content": "Go on"}],
+        )
+
+    assert raised.value.status_code == 400
+    assert "Invalid Message passed in {'role': 'bogus'" in raised.value.message
+
+
+@pytest.mark.parametrize(
+    ("later_turn", "real_cause"),
+    [
+        pytest.param(
+            [{"role": "user", "content": ["hi", "there"]}],
+            "'str' object has no attribute 'get'",
+            id="string_content_parts",
+        ),
+        pytest.param(
+            [
+                {
+                    **_WEATHER_CALL,
+                    "tool_calls": [
+                        {
+                            **_WEATHER_CALL["tool_calls"][0],
+                            "function": {"name": "get_weather", "arguments": {"city": "Paris"}},
+                        }
+                    ],
+                },
+                {"role": "tool", "tool_call_id": "toolu_01", "content": "rain"},
+            ],
+            "'dict' object has no attribute 'strip'",
+            id="object_tool_call_arguments",
+        ),
+        pytest.param(
+            [{"role": "user", "content": [{"type": "image_url", "image_url": ["x"]}]}],
+            "list indices must be integers or slices, not str",
+            id="list_image_url",
+        ),
+    ],
+)
+@pytest.mark.parametrize("dropped_answer_id", [7, {"id": "toolu_01"}], ids=["int", "object"])
+def test_chat_transform_request_never_blames_a_dropped_tool_answer_id_for_another_failure(
+    local_model_cost_map,
+    monkeypatch: pytest.MonkeyPatch,
+    later_turn: list[dict[str, object]],
+    real_cause: str,
+    dropped_answer_id: object,
+):
+    monkeypatch.setattr(litellm, "modify_params", True)
+
+    with pytest.raises(AnthropicError) as raised:
+        _chat_request(AnthropicConfig(), UNFLAGGED_CLAUDE, [*_orphan_answer_with(dropped_answer_id)[:3], *later_turn])
+
+    assert raised.value.status_code == 400
+    assert raised.value.message.startswith(real_cause)
+
+
+def test_chat_transform_request_never_blames_a_numeric_assistant_tool_call_id_for_an_earlier_failure(
+    local_model_cost_map,
+):
+    with pytest.raises(AnthropicError) as raised:
+        _chat_request(
+            AnthropicConfig(),
+            UNFLAGGED_CLAUDE,
+            [
+                {"role": "user", "content": ["hi", "there"]},
+                {
+                    "role": "assistant",
+                    "content": None,
+                    "tool_calls": [{"id": 5, "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}],
+                },
+                {"role": "tool", "tool_call_id": "5", "content": "rain"},
+            ],
+        )
+
+    assert raised.value.status_code == 400
+    assert raised.value.message.startswith("'str' object has no attribute 'get'")
+
+
+@pytest.mark.parametrize("empty_role", [0, False])
+def test_chat_transform_request_never_blames_an_empty_role_for_another_failure(local_model_cost_map, empty_role):
+    list_part_type_after_an_empty_role: Final = [
+        _ASK,
+        {"role": empty_role, "content": "ok"},
+        {"role": "user", "content": [{"type": ["text"], "text": "again"}]},
+        {"role": "system", "content": "Be terse."},
+    ]
+
+    with pytest.raises(TypeError):
+        _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, list_part_type_after_an_empty_role)
+
+
+def test_chat_transform_request_names_no_field_for_a_failure_outside_roles_and_tool_call_ids(local_model_cost_map):
+    list_part_type: Final = [
+        _ASK,
+        {"role": "assistant", "content": "ok"},
+        {"role": "user", "content": [{"type": ["text"], "text": "again"}]},
+        {"role": "system", "content": "Be terse."},
+    ]
+
+    with pytest.raises(TypeError):
+        _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, list_part_type)
+
+
 def _five_cache_hint_conversation() -> list[dict[str, object]]:
     return [
         {"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]},

tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py

@@ -614,9 +614,18 @@ def test_clear_tool_inputs_list_clears_only_the_named_tools():
     }
 
 
+def _history_with_first_block_field(index: int, field: str, value: object) -> list[dict[str, object]]:
+    history: Final = _make_history(n_pairs=4)
+    return [
+        {**message, "content": [{**message["content"][0], field: value}, *message["content"][1:]]}
+        if position == index
+        else message
+        for position, message in enumerate(history)
+    ]
+
+
 def test_a_tool_use_with_a_non_string_id_is_left_for_the_provider_to_reject():
-    messages = _make_history(n_pairs=4)
-    messages[1]["content"][0]["id"] = ["toolu_00"]
+    messages: Final = _history_with_first_block_field(1, "id", ["toolu_00"])
     new_messages, applied = apply_clear_tool_uses_20250919(
         model=MODEL,
         messages=messages,
@@ -635,6 +644,24 @@ def test_a_tool_use_with_a_non_string_id_is_left_for_the_provider_to_reject():
     assert (new_messages[3]["content"][0]["input"], new_messages[5]["content"][0]["input"]) == ({}, {})
 
 
+def test_a_tool_result_with_a_non_string_tool_use_id_is_left_for_the_provider_to_reject():
+    messages: Final = _history_with_first_block_field(2, "tool_use_id", ["toolu_00"])
+    new_messages, applied = apply_clear_tool_uses_20250919(
+        model=MODEL,
+        messages=messages,
+        tools=None,
+        system=None,
+        edit_spec={
+            "type": "clear_tool_uses_20250919",
+            "trigger": {"type": "tool_uses", "value": 0},
+            "keep": {"type": "tool_uses", "value": 1},
+        },
+    )
+    assert applied is not None
+    assert new_messages[2] == messages[2]
+    assert _cleared_ids(new_messages[3:]) == {"toolu_01", "toolu_02"}
+
+
 async def _malformed_spec_error(edits: list[dict[str, object]]) -> AnthropicContextManagementError:
     with pytest.raises(AnthropicContextManagementError) as raised:
         await apply_context_management(

tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py

@@ -4472,6 +4472,17 @@ def test_file_content_conversion_keeps_a_line_whose_json_holds_unicode_line_sepa
     assert bedrock_record["modelInput"]["messages"] == [{"role": "user", "content": [{"text": text}]}]
 
 
+_CLAUDE_BODY_WITH_A_LIST_ROLE_BEFORE_A_SYSTEM_MESSAGE: Final = {
+    "model": "bedrock/us.anthropic.claude-haiku-5-5",
+    "max_tokens": 20,
+    "messages": [
+        {"role": "user", "content": "hi"},
+        {"role": ["user"], "content": "ok"},
+        {"role": "system", "content": "Be terse."},
+    ],
+}
+
+
 def _batch_line(body: object) -> str:
     return json.dumps({**_VALID_BATCH_RECORD, "body": body})
 
@@ -4516,6 +4527,11 @@ def _batch_line_with_body(field: str, value: object) -> str:
             "Batch input file line 2 could not be converted to a Bedrock batch record",
             id="string-messages",
         ),
+        pytest.param(
+            f"{_VALID_BATCH_LINE}\n{_batch_line(_CLAUDE_BODY_WITH_A_LIST_ROLE_BEFORE_A_SYSTEM_MESSAGE)}\n".encode(),
+            "Batch input file line 2 could not be converted to a Bedrock batch record",
+            id="claude-list-role-before-a-system-message",
+        ),
         pytest.param(
             f"{_VALID_BATCH_LINE}\n{'[' * 1_000_000}{']' * 1_000_000}\n".encode(),
             "Batch input file line 2 could not be parsed",

tests/unit/llms/cohere/chat/test_cohere_transformation.py

@@ -1,3 +1,4 @@
+import itertools
 import json
 from collections.abc import Callable, Iterator
 from typing import Final
@@ -653,9 +654,7 @@ async def test_cohere_v2_annotations(respx_mock: respx.MockRouter) -> None:
 def test_cohere_v2_parameter_mapping(respx_mock: respx.MockRouter) -> None:
     messages: Final = [{"role": "user", "content": "Generate a creative story."}]
     route: Final = respx_mock.post(COHERE_V2_CHAT_URL).mock(
-        return_value=httpx.Response(
-            200, json=_cohere_v2_response({"content": [{"type": "text", "text": "A story."}]})
-        )
+        return_value=httpx.Response(200, json=_cohere_v2_response({"content": [{"type": "text", "text": "A story."}]}))
     )
 
     litellm.completion(
@@ -705,3 +704,123 @@ def test_cohere_v2_error_handling(respx_mock: respx.MockRouter) -> None:
     assert error.value.status_code == 400
     assert error.value.llm_provider == "cohere"
     assert "invalid request: model 'invalid-model' not found" in error.value.message
+
+
+def _cohere_v2_tool_call_events(index: int, call_id: str, city: str) -> tuple[dict[str, object], ...]:
+    return (
+        {
+            "type": "tool-call-start",
+            "index": index,
+            "delta": {
+                "message": {
+                    "tool_calls": {
+                        "id": call_id,
+                        "type": "function",
+                        "function": {"name": "get_weather", "arguments": ""},
+                    }
+                }
+            },
+        },
+        {
+            "type": "tool-call-delta",
+            "index": index,
+            "delta": {"message": {"tool_calls": {"function": {"arguments": '{"city": '}}}},
+        },
+        {
+            "type": "tool-call-delta",
+            "index": index,
+            "delta": {"message": {"tool_calls": {"function": {"arguments": json.dumps(city) + "}"}}}},
+        },
+        {"type": "tool-call-end", "index": index},
+    )
+
+
+@pytest.mark.usefixtures("_cohere_httpx_transport")
+@pytest.mark.respx(assert_all_called=True)
+async def test_cohere_v2_stream_returns_parallel_tool_calls(respx_mock: respx.MockRouter) -> None:
+    stream_events: Final = (
+        {"type": "message-start", "id": "c0a2b6e1", "delta": {"message": {"role": "assistant"}}},
+        *_cohere_v2_tool_call_events(0, "get_weather_paris", "Paris"),
+        *_cohere_v2_tool_call_events(1, "get_weather_tokyo", "Tokyo"),
+        {
+            "type": "message-end",
+            "delta": {"finish_reason": "TOOL_CALL", "usage": {"tokens": {"input_tokens": 40, "output_tokens": 30}}},
+        },
+    )
+    respx_mock.post(COHERE_V2_CHAT_URL).mock(
+        return_value=httpx.Response(200, content="".join(json.dumps(event) + "\n" for event in stream_events).encode())
+    )
+
+    response: Final = await litellm.acompletion(
+        model="cohere_chat/v2/command-a-plus-05-2026",
+        messages=[{"role": "user", "content": "What is the weather in Paris and in Tokyo?"}],
+        tools=[
+            {
+                "type": "function",
+                "function": {
+                    "name": "get_weather",
+                    "parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
+                },
+            }
+        ],
+        stream=True,
+        api_key="cohere-test-key",
+    )
+    chunks: Final = [chunk async for chunk in response]
+    rebuilt: Final = litellm.stream_chunk_builder(chunks)
+
+    assert rebuilt is not None
+    assert [
+        (call.id, call.function.name, json.loads(call.function.arguments))
+        for call in rebuilt.choices[0].message.tool_calls
+    ] == [
+        ("get_weather_paris", "get_weather", {"city": "Paris"}),
+        ("get_weather_tokyo", "get_weather", {"city": "Tokyo"}),
+    ]
+    assert chunks[-1].choices[0].finish_reason == "tool_calls"
+
+
+@pytest.mark.usefixtures("_cohere_httpx_transport")
+@pytest.mark.respx(assert_all_called=True)
+async def test_cohere_v2_stream_closes_argument_less_tool_call_with_empty_object(respx_mock: respx.MockRouter) -> None:
+    stream_events: Final = (
+        {"type": "message-start", "id": "5d1e9f20", "delta": {"message": {"role": "assistant"}}},
+        {
+            "type": "tool-call-start",
+            "index": 0,
+            "delta": {
+                "message": {
+                    "tool_calls": {
+                        "id": "get_server_time_0",
+                        "type": "function",
+                        "function": {"name": "get_server_time", "arguments": ""},
+                    }
+                }
+            },
+        },
+        {"type": "tool-call-end", "index": 0},
+        {"type": "message-end", "delta": {"finish_reason": "TOOL_CALL"}},
+    )
+    respx_mock.post(COHERE_V2_CHAT_URL).mock(
+        return_value=httpx.Response(200, content="".join(json.dumps(event) + "\n" for event in stream_events).encode())
+    )
+
+    response: Final = await litellm.acompletion(
+        model="cohere_chat/v2/command-a-plus-05-2026",
+        messages=[{"role": "user", "content": "What time is it on the server?"}],
+        tools=[
+            {
+                "type": "function",
+                "function": {"name": "get_server_time", "parameters": {"type": "object", "properties": {}}},
+            }
+        ],
+        stream=True,
+        api_key="cohere-test-key",
+    )
+    deltas_per_chunk: Final = [chunk.choices[0].delta.tool_calls or () async for chunk in response]
+    streamed_calls: Final = tuple(itertools.chain.from_iterable(deltas_per_chunk))
+
+    assert [(call.id, call.function.name) for call in streamed_calls if call.id] == [
+        ("get_server_time_0", "get_server_time")
+    ]
+    assert json.loads("".join(call.function.arguments for call in streamed_calls)) == {}

tests/unit/llms/custom_httpx/test_aiohttp_handler.py

@@ -307,20 +307,12 @@ class TestBaseLLMAIOHTTPHandler:
         # Should not call aiohttp.ClientSession directly
         mock_client_session.assert_not_called()
 
-    @patch("aiohttp.ClientSession")
-    def test_create_client_session_with_connector(self, mock_client_session):
-        """Test session creation using connector"""
-        mock_connector = Mock(spec=aiohttp.BaseConnector)
-        mock_session_instance = Mock()
-        mock_client_session.return_value = mock_session_instance
-
-        handler = BaseLLMAIOHTTPHandler(connector=mock_connector)
-
-        result = handler._create_client_session_with_transport()
-
-        # Should create session with connector
-        mock_client_session.assert_called_once_with(connector=mock_connector)
-        assert result is mock_session_instance
+    @pytest.mark.asyncio
+    async def test_create_client_session_with_connector(self) -> None:
+        connector: Final = aiohttp.TCPConnector()
+        handler: Final = BaseLLMAIOHTTPHandler(connector=connector)
+        async with handler._create_client_session_with_transport() as session:
+            assert session.connector is connector
 
     @pytest.mark.asyncio
     async def test_create_client_session_default_honors_global_ssl_verify_false(

tests/unit/llms/custom_httpx/test_http_handler.py

@@ -1,13 +1,16 @@
 import asyncio, importlib, json, time
 import gc
 import io
+import logging
 import os
 import pathlib
 import ssl
 import threading
 import weakref
-from collections.abc import Callable, Mapping
+from collections.abc import AsyncIterator, Callable, Mapping
 from concurrent.futures import ThreadPoolExecutor
+from contextlib import asynccontextmanager
+from contextvars import Context
 from typing import Final
 from unittest.mock import MagicMock, patch
 
@@ -15,7 +18,10 @@ import certifi
 import httpx
 import pytest
 import respx
-from aiohttp import ClientSession, TCPConnector
+from aiohttp import ClientResponse, ClientSession, RequestInfo, TCPConnector
+from aiohttp.tracing import Trace
+from multidict import CIMultiDict, CIMultiDictProxy
+from yarl import URL
 
 import litellm
 from litellm.caching.llm_caching_handler import LLMClientCache
@@ -2304,3 +2310,217 @@ async def test_async_timeout_reports_the_per_call_timeout_as_a_float_when_one_is
         await handler.close()
 
     assert "Read timed out. Timeout passed=30.0, time taken=" in str(raised.value), str(raised.value)
+
+
+class _PoolDiagnosticClock:
+    def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
+        self.now = 1000.0
+        self.loop = loop
+        self.schedule = loop.call_later
+        self.timers: list[asyncio.TimerHandle] = []
+
+    def time(self) -> float:
+        return self.now
+
+    def call_later(
+        self, delay: float, callback: Callable[..., object],
+        *args: object, context: Context | None = None,
+    ) -> asyncio.TimerHandle:
+        timer: Final = self.schedule(delay, callback, *args, context=context)
+        self.timers.append(timer)
+        return timer
+
+    async def advance(self, seconds: float) -> None:
+        self.now += seconds
+        checkpoint: Final[asyncio.Future[None]] = self.loop.create_future()
+        self.loop.call_soon(checkpoint.set_result, None)
+        await checkpoint
+
+
+def _pool_diagnostic_clock(monkeypatch: pytest.MonkeyPatch) -> _PoolDiagnosticClock:
+    loop: Final = asyncio.get_running_loop()
+    clock: Final = _PoolDiagnosticClock(loop)
+    monkeypatch.setattr(loop, "time", clock.time)
+    monkeypatch.setattr(loop, "call_later", clock.call_later)
+    return clock
+
+
+@asynccontextmanager
+async def _pool_diagnostic_session(source: str) -> AsyncIterator[ClientSession]:
+    if source == "proxy":
+        from litellm.proxy.proxy_server import initialize_shared_aiohttp_session
+
+        session: Final = await initialize_shared_aiohttp_session()
+        assert session is not None
+        async with session:
+            yield session
+        return
+    if source == "native-connector":
+        from litellm.llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler
+
+        handler: Final = BaseLLMAIOHTTPHandler(connector=TCPConnector(limit=7, limit_per_host=2))
+        try:
+            yield handler._get_async_client_session()
+        finally:
+            await handler.close()
+        return
+    transport: Final = AsyncHTTPHandler.create_aiohttp_transport()
+    if source == "rebuilt":
+        await transport.get_valid_client_session().close()
+    try:
+        yield transport.get_valid_client_session()
+    finally:
+        await transport.aclose()
+
+
+async def _pool_diagnostic_request(session: ClientSession, url: URL) -> tuple[Trace, ...]:
+    traces: Final = tuple(Trace(session, config, config.trace_config_ctx()) for config in session.trace_configs)
+    for trace in traces:
+        await trace.send_request_start("POST", url, CIMultiDict())
+    return traces
+
+
+def _pool_diagnostic_messages(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]:
+    return tuple(record.getMessage() for record in caplog.records if "connection pool" in record.getMessage())
+
+
+@pytest.mark.parametrize("source", ["sdk", "proxy", "native-connector", "rebuilt"])
+@pytest.mark.parametrize("error", [TimeoutError("private timeout"), asyncio.CancelledError("private cancellation")])
+async def test_aiohttp_pool_failure_identifies_queue_wait_and_cleans_up_timer(
+    source: str, error: BaseException, monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    pool_diagnostic_clock: Final = _pool_diagnostic_clock(monkeypatch)
+    caplog.set_level(logging.WARNING, logger="LiteLLM")
+    url: Final = URL("https://private-user:private-password@pool.test:8443/private-path?token=private-query")
+    async with _pool_diagnostic_session(source) as session:
+        connector: Final = session.connector
+        assert connector is not None
+        traces: Final = await _pool_diagnostic_request(session, url)
+        for trace in traces:
+            await trace.send_connection_queued_start()
+        await pool_diagnostic_clock.advance(0.25)
+        for trace in traces:
+            await trace.send_request_exception("POST", url, CIMultiDict(), error)
+        messages: Final = _pool_diagnostic_messages(caplog)
+        assert len(messages) == 1, messages
+        message: Final = messages[0]
+        assert "host=pool.test" in message
+        assert "port=8443" in message
+        assert "0.25" in message
+        assert "acquired_total=0" in message
+        assert f"limit={connector.limit}" in message
+        assert f"limit_per_host={connector.limit_per_host}" in message
+        assert "private" not in message
+        assert all(timer.cancelled() for timer in pool_diagnostic_clock.timers)
+        await pool_diagnostic_clock.advance(2.0)
+        assert _pool_diagnostic_messages(caplog) == messages
+
+
+@pytest.mark.parametrize("queued", [False, True])
+async def test_aiohttp_pool_diagnostics_do_not_label_slow_upstream_as_pool_saturation(
+    queued: bool, monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    pool_diagnostic_clock: Final = _pool_diagnostic_clock(monkeypatch)
+    caplog.set_level(logging.WARNING, logger="LiteLLM")
+    url: Final = URL("https://healthy.test/completions")
+    async with _pool_diagnostic_session("sdk") as session:
+        traces: Final = await _pool_diagnostic_request(session, url)
+        if queued:
+            for trace in traces:
+                await trace.send_connection_queued_start()
+            await pool_diagnostic_clock.advance(0.25)
+            for trace in traces:
+                await trace.send_connection_queued_end()
+        await pool_diagnostic_clock.advance(2.0)
+        for trace in traces:
+            await trace.send_request_exception("POST", url, CIMultiDict(), TimeoutError("slow upstream"))
+        assert _pool_diagnostic_messages(caplog) == ()
+        assert all(timer.cancelled() for timer in pool_diagnostic_clock.timers)
+
+
+async def test_aiohttp_pool_warns_while_queued_once_per_request_with_isolated_concurrent_waits(
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture,
+) -> None:
+    pool_diagnostic_clock: Final = _pool_diagnostic_clock(monkeypatch)
+    caplog.set_level(logging.WARNING, logger="LiteLLM")
+    slow_url: Final = URL("https://blocked.test/completions")
+    fast_url: Final = URL("https://available.test/completions")
+    async with _pool_diagnostic_session("sdk") as session:
+        blocked: Final = await _pool_diagnostic_request(session, slow_url)
+        available: Final = await _pool_diagnostic_request(session, fast_url)
+        for trace in blocked + available:
+            await trace.send_connection_queued_start()
+        await pool_diagnostic_clock.advance(0.25)
+        for trace in available:
+            await trace.send_connection_queued_end()
+        for trace in blocked:
+            await trace.send_connection_queued_end()
+            await trace.send_connection_queued_start()
+        await pool_diagnostic_clock.advance(0.8)
+        messages: Final = _pool_diagnostic_messages(caplog)
+        assert len(messages) == 1, messages
+        assert "host=blocked.test" in messages[0]
+        assert "1.05" in messages[0]
+        assert "available.test" not in messages[0]
+        for trace in blocked:
+            await trace.send_connection_queued_end()
+            await trace.send_connection_queued_start()
+        await pool_diagnostic_clock.advance(2.0)
+        for trace in blocked:
+            await trace.send_request_exception("POST", slow_url, CIMultiDict(), TimeoutError())
+        for trace in available:
+            await trace.send_request_exception("POST", fast_url, CIMultiDict(), TimeoutError())
+        await pool_diagnostic_clock.advance(2.0)
+        assert _pool_diagnostic_messages(caplog) == messages
+        assert all(timer.cancelled() for timer in pool_diagnostic_clock.timers)
+
+
+@pytest.mark.parametrize(("header", "location", "host", "port"), [
+    ("Location", "https://redirected.test:9443/private-path?token=private-query", "redirected.test", 9443),
+    ("Location", "/private-path?token=private-query", "initial.test", 8443),
+    ("URI", "https://redirected.test/private-path", "redirected.test", 443),
+    ("Location", "http://[invalid", None, None),
+])
+async def test_aiohttp_pool_warning_tracks_redirect_destination_without_masking_invalid_redirects(
+    header: str, location: str, host: str | None, port: int | None,
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture,
+) -> None:
+    from aiohttp.client_proto import ResponseHandler
+    from aiohttp.helpers import TimerNoop
+    from aiohttp.http_writer import StreamWriter
+
+    clock: Final = _pool_diagnostic_clock(monkeypatch)
+    caplog.set_level(logging.WARNING, logger="LiteLLM")
+    url: Final = URL("https://initial.test:8443/completions")
+    async with _pool_diagnostic_session("sdk") as session:
+        traces: Final = await _pool_diagnostic_request(session, url)
+        loop: Final = asyncio.get_running_loop()
+        response: Final = ClientResponse(
+            "POST", url, writer=None, continue100=None, timer=TimerNoop(),
+            request_info=RequestInfo(url, "POST", CIMultiDictProxy(CIMultiDict()), url),
+            traces=[], loop=loop, session=session,
+            stream_writer=StreamWriter(ResponseHandler(loop), loop),
+        )
+        response._headers = CIMultiDictProxy(CIMultiDict({header: location}))
+        try:
+            for trace in traces:
+                await trace.send_request_redirect("POST", url, CIMultiDict(), response)
+            if host is None:
+                for trace in traces:
+                    await trace.send_request_exception("POST", url, CIMultiDict(), ValueError("invalid redirect"))
+                assert _pool_diagnostic_messages(caplog) == ()
+                return
+            for trace in traces:
+                await trace.send_connection_queued_start()
+            await clock.advance(0.25)
+            for trace in traces:
+                await trace.send_request_exception("POST", url, CIMultiDict(), TimeoutError())
+            messages: Final = _pool_diagnostic_messages(caplog)
+            assert len(messages) == 1, messages
+            assert f"host={host}" in messages[0]
+            assert f"port={port}" in messages[0]
+            assert "private" not in messages[0]
+        finally:
+            response.close()

tests/unit/llms/litellm_proxy/skills/test_skill_search.py

@@ -281,6 +281,36 @@ class TestSearchSkills:
         assert isinstance(outcome, SkillSearchNotConfigured)
         assert "skill_search_embedding_model" in outcome.reason
 
+    @pytest.mark.parametrize("embedding_model", ("", "   "), ids=("empty", "whitespace"))
+    @pytest.mark.asyncio
+    async def test_empty_embedding_model_is_not_configured(self, embedding_model: str) -> None:
+        router: Final = _embedding_router()
+        not_configured: Final = await search_skills(
+            "language translation",
+            SKILLS,
+            5,
+            router=router,
+            embedding_model=None,
+            index=SkillSearchIndex(),
+            user_api_key_dict=CALLER,
+            proxy_logging_obj=_pass_through_key_limits(),
+        )
+        outcome: Final = await search_skills(
+            "language translation",
+            SKILLS,
+            5,
+            router=router,
+            embedding_model=embedding_model,
+            index=SkillSearchIndex(),
+            user_api_key_dict=CALLER,
+            proxy_logging_obj=_pass_through_key_limits(),
+        )
+
+        assert isinstance(not_configured, SkillSearchNotConfigured)
+        assert isinstance(outcome, SkillSearchNotConfigured)
+        assert outcome.reason == not_configured.reason
+        router.aembedding.assert_not_awaited()
+
     @pytest.mark.asyncio
     async def test_no_router_is_not_configured(self) -> None:
         outcome = await search_skills(

tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py

@@ -11,7 +11,7 @@ import pytest
 import respx
 
 import litellm
-from litellm.llms.custom_httpx.http_handler import HTTPHandler
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
 from litellm.llms.openrouter.chat.transformation import (
     OpenRouterChatCompletionStreamingHandler,
@@ -742,7 +742,11 @@ def test_openrouter_completion_with_non_object_usage_raises_api_connection_error
 
 
 @respx.mock
-def test_openrouter_streaming_completion_assembles_text_and_finish_reason() -> None:
+@pytest.mark.parametrize("async_stream", [False, True])
+@pytest.mark.parametrize("requested_model", ["openrouter/openai/gpt-4o-mini", "openrouter/auto"])
+async def test_openrouter_streaming_completion_assembles_text_and_finish_reason(
+    async_stream: bool, requested_model: str
+) -> None:
     chunks: Final = (
         {
             "id": "gen-stream-model",
@@ -785,14 +789,20 @@ def test_openrouter_streaming_completion_assembles_text_and_finish_reason() -> N
         {"role": "system", "content": "You are a helpful assistant."},
         {"role": "user", "content": "How does a case reach the Supreme Court?"},
     ]
-    stream: Final = litellm.completion(
-        model="openrouter/openai/gpt-4o-mini",
+    kwargs: Final = dict(
+        model=requested_model,
         messages=messages,
         stream=True,
         max_tokens=50,
         api_key="test-openrouter-key",
     )
-    stream_chunks: Final = tuple(stream)
+    client: Final = AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport())
+    try:
+        stream: Final = await litellm.acompletion(**kwargs, client=client) if async_stream else litellm.completion(**kwargs)
+        stream_chunks: Final = tuple([chunk async for chunk in stream]) if async_stream else tuple(stream)
+    finally:
+        await client.client.aclose()
+    assert {chunk.model for chunk in stream_chunks} == {chunks[0]["model"]}
     assert stream_chunks[0].choices[0].delta.role == "assistant"
     assert tuple(
         chunk.choices[0].finish_reason for chunk in stream_chunks if chunk.choices[0].finish_reason is not None
@@ -802,6 +812,7 @@ def test_openrouter_streaming_completion_assembles_text_and_finish_reason() -> N
     ) == "The Supreme Court hears federal cases."
     assembled: Final = litellm.stream_chunk_builder(chunks=list(stream_chunks), messages=messages)
     assert assembled is not None
+    assert assembled.model == chunks[0]["model"]
     assert assembled.choices[0].message.content == "The Supreme Court hears federal cases."
     assert assembled.choices[0].finish_reason == "stop"
 

tests/unit/proxy/a2a/test_agent_card.py

@@ -8,6 +8,7 @@ from litellm.proxy.a2a.agent_card import (
     LITELLM_SECURITY_SCHEMES,
     merge_agent_card,
     normalize_protocol_version,
+    point_agent_card_at_proxy,
     resolve_served_protocol_version,
 )
 
@@ -15,6 +16,32 @@ PROXY_URL = "https://proxy.example/a2a/agent-xyz"
 PROXY_BASE = "https://proxy.example"
 
 
+def test_point_agent_card_at_proxy_returns_a_deep_copy() -> None:
+    card: dict[str, object] = {
+        "url": "http://internal/",
+        "metadata": {"nested": ["original"]},
+        "supportedInterfaces": [
+            {"url": "http://internal/", "nested": {"value": "original"}},
+            {"url": "http://secondary/"},
+        ],
+    }
+
+    pointed: dict[str, object] = point_agent_card_at_proxy(card, PROXY_URL)
+
+    assert pointed["url"] == PROXY_URL
+    assert pointed["metadata"] == card["metadata"]
+    assert pointed["metadata"] is not card["metadata"]
+    assert pointed["supportedInterfaces"] == [
+        {"url": PROXY_URL, "nested": {"value": "original"}},
+        {"url": "http://secondary/"},
+    ]
+    assert card["url"] == "http://internal/"
+    assert card["supportedInterfaces"] == [
+        {"url": "http://internal/", "nested": {"value": "original"}},
+        {"url": "http://secondary/"},
+    ]
+
+
 def _full_upstream_card() -> dict:
     return {
         "protocolVersion": "0.9",

tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py

@@ -12,19 +12,22 @@ import sys
 from collections.abc import AsyncIterator, Awaitable, Callable, Mapping
 from contextlib import AbstractContextManager, ExitStack
 from dataclasses import dataclass
+from datetime import datetime
+from types import MappingProxyType
 from typing import Final
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import httpx
 import pytest
 import respx
+from fastapi import HTTPException
 from fastapi.testclient import TestClient
 from pydantic import BaseModel, ConfigDict
 
 import litellm
 from litellm.integrations.custom_logger import CustomLogger
-from litellm.proxy._types import UserAPIKeyAuth
-from litellm.types.agents import AgentCaller
+from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth
+from litellm.types.agents import AgentCaller, AgentResponse
 
 AddLiteLLMData = Callable[..., Awaitable[dict[str, object]]]
 
@@ -2782,7 +2785,9 @@ async def test_forward_jsonrpc_sse_is_untouched_while_keepalives_are_unconfigure
 
 
 @pytest.mark.asyncio
-async def test_forward_jsonrpc_sse_keeps_unicode_line_separators_inside_an_event(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_forward_jsonrpc_sse_keeps_unicode_line_separators_inside_an_event(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
     monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", None)
     separated_text: Final = "before\u2028between\u2029after"
     event: Final = {
@@ -2944,6 +2949,202 @@ async def test_failed_upstream_stream_relays_its_body_only_when_that_is_a_json_o
     }
 
 
+class _RejectionRecorder(CustomLogger):
+    def __init__(self) -> None:
+        super().__init__()
+        self.failure_payloads: Final[list[Mapping[str, object]]] = []  # mutable-ok: collects callback events
+        self.hook_calls: Final[list[tuple[Mapping[str, object], Exception]]] = []  # mutable-ok: collects hook calls
+
+    async def async_log_failure_event(
+        self,
+        kwargs: dict[str, object],
+        response_obj: object,
+        start_time: datetime | None,
+        end_time: datetime | None,
+    ) -> None:
+        payload: Final = kwargs["standard_logging_object"]
+        assert isinstance(payload, Mapping)
+        self.failure_payloads.append(payload)
+
+    async def async_post_call_failure_hook(
+        self,
+        request_data: dict[str, object],
+        original_exception: Exception,
+        user_api_key_dict: UserAPIKeyAuth,
+        traceback_str: str | None = None,
+    ) -> HTTPException:
+        self.hook_calls.append((dict(request_data), original_exception))
+        return HTTPException(status_code=418, detail="transformed")
+
+
+_SPOOFED_BODY_FIELDS: Final = MappingProxyType(
+    {
+        "agent_id": "spoof",
+        "litellm_call_id": "spoofed-call-id",
+        "metadata": {"agent_id": "spoof", "user_api_key_alias": "spoof", "user_api_key_team_id": "spoof"},
+    }
+)
+
+
+def _rejection_request(path_agent_id: str, request_id: object) -> MagicMock:
+    req: Final = MagicMock()
+    req.headers = {}
+    req.json = AsyncMock(
+        return_value={
+            "jsonrpc": "2.0",
+            "id": request_id,
+            "method": "SendMessage",
+            "params": {**_HELLO_MESSAGE_PARAMS, "litellm_call_id": "spoofed-param-call-id"},
+            **_SPOOFED_BODY_FIELDS,
+        }
+    )
+    return req
+
+
+async def _invoke_rejected(
+    monkeypatch: pytest.MonkeyPatch,
+    *,
+    registered: bool,
+    proxy_logging_obj: object,
+    request_id: object = "rej-1",
+) -> tuple[int, object]:
+    from litellm.proxy import proxy_server
+    from litellm.proxy.agent_endpoints import agent_registry
+    from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a
+
+    path_agent_id: Final = "denied-agent" if registered else "no-such-agent"
+    registry: Final = agent_registry.AgentRegistry()
+    if registered:
+        registry.register_agent(
+            AgentResponse(
+                agent_id="registered-denied-id",
+                agent_name=path_agent_id,
+                agent_card_params={"name": "Test Agent", "url": "https://agent.example.com"},
+            )
+        )
+    monkeypatch.setattr(agent_registry, "global_agent_registry", registry)
+    monkeypatch.setattr(proxy_server, "prisma_client", None)
+    monkeypatch.setattr(proxy_server, "proxy_logging_obj", proxy_logging_obj)
+    auth: Final = UserAPIKeyAuth(
+        api_key="hashed-caller",
+        key_alias="caller-alias",
+        user_id="caller-user",
+        team_id="caller-team",
+        object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="caller-grants", agents=["other-agent"]),
+        request_route=f"/a2a/{path_agent_id}",
+    )
+    with respx.mock as upstream:
+        try:
+            response: Final = await invoke_agent_a2a(
+                agent_id=path_agent_id,
+                request=_rejection_request(path_agent_id, request_id),
+                fastapi_response=MagicMock(),
+                user_api_key_dict=auth,
+            )
+        except HTTPException as e:
+            assert not upstream.calls
+            return e.status_code, e.detail
+        assert not upstream.calls
+    return response.status_code, json.loads(response.body.decode())
+
+
+_EXPECTED_REJECTIONS: Final = (
+    pytest.param(
+        True,
+        403,
+        "Agent 'denied-agent' is not allowed for your key/team. Contact proxy admin for access.",
+        "registered-denied-id",
+        "a2a_agent/Test Agent",
+        id="not-granted",
+    ),
+    pytest.param(
+        False,
+        404,
+        {"jsonrpc": "2.0", "id": "rej-1", "error": {"code": -32000, "message": "Agent 'no-such-agent' not found"}},
+        "no-such-agent",
+        "a2a_agent/no-such-agent",
+        id="unknown",
+    ),
+)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(("registered", "status", "body", "target", "model"), _EXPECTED_REJECTIONS)
+async def test_rejected_agent_call_is_logged_once_as_the_caller_without_dispatch(
+    monkeypatch: pytest.MonkeyPatch, registered: bool, status: int, body: object, target: str, model: str
+) -> None:
+    from litellm.caching.caching import DualCache
+    from litellm.proxy.utils import ProxyLogging
+
+    recorder: Final = _RejectionRecorder()
+    for name in ("callbacks", "failure_callback", "_async_failure_callback"):
+        monkeypatch.setattr(litellm, name, [recorder])
+
+    got_status, got_body = await _invoke_rejected(
+        monkeypatch, registered=registered, proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache())
+    )
+
+    assert (got_status, got_body) == (status, body)
+    assert len(recorder.failure_payloads) == 1
+    payload: Final = recorder.failure_payloads[0]
+    metadata: Final = payload["metadata"]
+    assert (payload["status"], payload["call_type"], payload["model"], payload["response_cost"]) == (
+        "failure",
+        "asend_message",
+        model,
+        0,
+    )
+    assert (
+        metadata["user_api_key_alias"],
+        metadata["user_api_key_user_id"],
+        metadata["user_api_key_team_id"],
+        metadata["user_api_key_hash"],
+    ) == ("caller-alias", "caller-user", "caller-team", "hashed-caller")
+    assert payload["id"] not in {"spoofed-call-id", "spoofed-param-call-id", "rej-1"}
+    assert len(recorder.hook_calls) == 1
+    hook_data, hook_exception = recorder.hook_calls[0]
+    assert (hook_data["agent_id"], hook_data["metadata"]["agent_id"]) == (target, target)
+    assert isinstance(hook_exception, HTTPException) and hook_exception.status_code == status
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(("registered", "status", "body", "target", "model"), _EXPECTED_REJECTIONS)
+async def test_rejected_agent_call_response_survives_a_failing_failure_hook(
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+    registered: bool,
+    status: int,
+    body: object,
+    target: str,
+    model: str,
+) -> None:
+    proxy_logging_obj: Final = MagicMock()
+    proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=RuntimeError("logging down"))
+
+    got_status, got_body = await _invoke_rejected(
+        monkeypatch, registered=registered, proxy_logging_obj=proxy_logging_obj
+    )
+
+    assert (got_status, got_body) == (status, body)
+    proxy_logging_obj.post_call_failure_hook.assert_awaited_once()
+    assert any(record.getMessage() == "Failed to log rejected A2A request" for record in caplog.records)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("request_id", [0, 7, "abc"])
+async def test_unknown_agent_rejection_echoes_the_jsonrpc_id(
+    monkeypatch: pytest.MonkeyPatch, request_id: object
+) -> None:
+    proxy_logging_obj: Final = MagicMock()
+    proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
+
+    _, got_body = await _invoke_rejected(
+        monkeypatch, registered=False, proxy_logging_obj=proxy_logging_obj, request_id=request_id
+    )
+
+    assert isinstance(got_body, dict) and got_body["id"] == request_id and type(got_body["id"]) is type(request_id)
+
+
 def _a2a_app_refusing_with(refusal: Exception) -> TestClient:
     from fastapi import FastAPI
 

tests/unit/proxy/agent_endpoints/test_agent_search.py

@@ -261,6 +261,42 @@ class TestSearchAgents:
         assert isinstance(outcome, AgentSearchNotConfigured)
         assert "agent_search_embedding_model" in outcome.reason
 
+    @pytest.mark.parametrize("embedding_model", ("", "   "), ids=("empty", "whitespace"))
+    @pytest.mark.asyncio
+    async def test_empty_embedding_model_is_not_configured(self, embedding_model: str) -> None:
+        router: Final = MagicMock()
+        router.aembedding = AsyncMock(
+            side_effect=lambda model, input, metadata: litellm.EmbeddingResponse(
+                model=model,
+                data=[{"object": "embedding", "index": i, "embedding": list(VECTORS[t])} for i, t in enumerate(input)],
+            )
+        )
+        not_configured: Final = await search_agents(
+            "language translation",
+            AGENTS,
+            5,
+            router=router,
+            embedding_model=None,
+            index=AgentSearchIndex(),
+            user_api_key_dict=CALLER,
+            proxy_logging_obj=_pass_through_key_limits(),
+        )
+        outcome: Final = await search_agents(
+            "language translation",
+            AGENTS,
+            5,
+            router=router,
+            embedding_model=embedding_model,
+            index=AgentSearchIndex(),
+            user_api_key_dict=CALLER,
+            proxy_logging_obj=_pass_through_key_limits(),
+        )
+
+        assert isinstance(not_configured, AgentSearchNotConfigured)
+        assert isinstance(outcome, AgentSearchNotConfigured)
+        assert outcome.reason == not_configured.reason
+        router.aembedding.assert_not_awaited()
+
     @pytest.mark.asyncio
     async def test_no_router_is_not_configured(self) -> None:
         outcome = await search_agents(

tests/unit/proxy/agent_endpoints/test_endpoints.py

@@ -1,7 +1,6 @@
 import json
 from collections.abc import Mapping
 from datetime import datetime, timezone
-
 from types import SimpleNamespace
 from typing import Final
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -13,7 +12,12 @@ from fastapi.testclient import TestClient
 from prisma.models import LiteLLM_AgentsTable
 
 from litellm.constants import REDACTED_BY_LITELM_STRING
-from litellm.proxy._types import LiteLLM_AuditLogs, LitellmTableNames, LitellmUserRoles, UserAPIKeyAuth
+from litellm.proxy._types import (
+    LiteLLM_AuditLogs,
+    LitellmTableNames,
+    LitellmUserRoles,
+    UserAPIKeyAuth,
+)
 from litellm.proxy.agent_endpoints import endpoints as agent_endpoints
 from litellm.proxy.agent_endpoints.auth.agent_permission_handler import (
     RestrictedAgentAccess,
@@ -43,6 +47,29 @@ def _sample_agent_card_params() -> dict:
     }
 
 
+_AGENT_CARD_BACKEND_URL: Final = "http://agent.internal:9000/plain/"
+
+
+def _sample_agent_card_params_with_backend_url() -> dict[str, object]:
+    return {
+        **_sample_agent_card_params(),
+        "url": _AGENT_CARD_BACKEND_URL,
+        "supportedInterfaces": [
+            {
+                "url": _AGENT_CARD_BACKEND_URL,
+                "protocolBinding": "JSONRPC",
+                "protocolVersion": "1.0",
+            },
+            {
+                "url": _AGENT_CARD_BACKEND_URL,
+                "protocolBinding": "GRPC",
+                "protocolVersion": "1.0",
+            },
+        ],
+        "additionalInterfaces": [{"url": _AGENT_CARD_BACKEND_URL, "transport": "GRPC"}],
+    }
+
+
 def _sample_agent_config() -> dict:
     return {
         "agent_name": "Test Agent",
@@ -641,7 +668,7 @@ class TestAgentRBACProxyAdmin:
         monkeypatch.setattr(agent_endpoints, "AGENT_REGISTRY", self.mock_registry)
 
     def test_should_allow_admin_to_create_agent(self, monkeypatch):
-        with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma:
+        with patch("litellm.proxy.proxy_server.prisma_client"):
             self.mock_registry.get_agent_by_name = MagicMock(return_value=None)
             self.mock_registry.add_agent_to_db = AsyncMock(
                 return_value=_sample_agent_response()
@@ -1015,6 +1042,129 @@ class TestAgentHealthCheck:
         assert len(resp.json()) == 2
 
 
+def _client_with_agent_card(
+    monkeypatch: pytest.MonkeyPatch,
+    role: LitellmUserRoles,
+    agent: AgentResponse,
+) -> TestClient:
+    from litellm.proxy import proxy_server
+    from litellm.proxy.agent_endpoints import agent_registry
+
+    registry: Final = MagicMock()
+    registry.get_agent_by_id = MagicMock(return_value=agent)
+    registry.get_agent_list = MagicMock(return_value=(agent,))
+    registry.ids_for_agent = MagicMock(return_value=frozenset({agent.agent_id}))
+    registry.stable_agent_id = MagicMock(side_effect=lambda agent_id: agent_id)
+    monkeypatch.setattr(agent_endpoints, "AGENT_REGISTRY", registry)
+    monkeypatch.setattr(agent_registry, "global_agent_registry", registry)
+    prisma: Final = MagicMock()
+    prisma.writer_db = prisma.db
+    prisma.db.litellm_agentstable.find_many = AsyncMock(return_value=[])
+    prisma.db.litellm_agentstable.find_unique = AsyncMock(return_value=None)
+    prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[])
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma)
+    test_app: Final = FastAPI()
+    test_app.include_router(router)
+    test_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
+        user_id="test-user",
+        user_role=role,
+        object_permission={"object_permission_id": "agent-card-test", "agents": [agent.agent_id]},
+    )
+    return TestClient(test_app)
+
+
+@pytest.mark.parametrize("path", ["/v1/agents", "/v1/agents/agent-123"])
+def test_non_admin_agent_card_urls_point_at_proxy(
+    monkeypatch: pytest.MonkeyPatch,
+    path: str,
+) -> None:
+    agent: Final = AgentResponse(
+        agent_id="agent-123",
+        agent_name="Test Agent",
+        agent_card_params=_sample_agent_card_params_with_backend_url(),
+    )
+    test_client: Final = _client_with_agent_card(monkeypatch, LitellmUserRoles.INTERNAL_USER, agent)
+
+    response: Final = test_client.get(path, headers={"Authorization": "Bearer k"})
+
+    assert response.status_code == 200, response.text
+    agents: Final = response.json()
+    result: Final = agents[0] if path == "/v1/agents" else agents
+    card: Final = result["agent_card_params"]
+    proxy_url: Final = "http://testserver/a2a/agent-123"
+    assert card["url"] == proxy_url
+    assert "agent.internal" not in response.text
+    assert "additionalInterfaces" not in card
+    assert card["supportedInterfaces"] == [
+        {
+            "url": proxy_url,
+            "protocolBinding": "JSONRPC",
+            "protocolVersion": "1.0",
+        }
+    ]
+
+
+def test_proxy_admin_agent_listing_keeps_backend_card_urls(monkeypatch: pytest.MonkeyPatch) -> None:
+    agent: Final = AgentResponse(
+        agent_id="agent-123",
+        agent_name="Test Agent",
+        agent_card_params=_sample_agent_card_params_with_backend_url(),
+    )
+    test_client: Final = _client_with_agent_card(monkeypatch, LitellmUserRoles.PROXY_ADMIN, agent)
+
+    response: Final = test_client.get("/v1/agents", headers={"Authorization": "Bearer k"})
+
+    assert response.status_code == 200, response.text
+    card: Final = response.json()[0]["agent_card_params"]
+    assert card["url"] == _AGENT_CARD_BACKEND_URL
+    assert card["supportedInterfaces"] == [
+        {
+            "url": _AGENT_CARD_BACKEND_URL,
+            "protocolBinding": "JSONRPC",
+            "protocolVersion": "1.0",
+        },
+        {
+            "url": _AGENT_CARD_BACKEND_URL,
+            "protocolBinding": "GRPC",
+            "protocolVersion": "1.0",
+        },
+    ]
+    assert card["additionalInterfaces"] == [{"url": _AGENT_CARD_BACKEND_URL, "transport": "GRPC"}]
+
+
+def test_non_admin_health_check_probes_backend_and_returns_proxy_card(monkeypatch: pytest.MonkeyPatch) -> None:
+    agent: Final = AgentResponse(
+        agent_id="agent-123",
+        agent_name="Test Agent",
+        agent_card_params=_sample_agent_card_params_with_backend_url(),
+    )
+    test_client: Final = _client_with_agent_card(monkeypatch, LitellmUserRoles.INTERNAL_USER, agent)
+    http_get: Final = AsyncMock(return_value=httpx.Response(200))
+    http_client: Final = MagicMock(get=http_get)
+    monkeypatch.setattr(
+        agent_endpoints,
+        "get_async_httpx_client",
+        MagicMock(return_value=http_client),
+    )
+
+    response: Final = test_client.get("/v1/agents?health_check=true", headers={"Authorization": "Bearer k"})
+
+    assert response.status_code == 200, response.text
+    assert http_get.await_args.args[0] == _AGENT_CARD_BACKEND_URL
+    assert response.json()[0]["agent_card_params"]["url"] == "http://testserver/a2a/agent-123"
+
+
+def test_non_admin_agent_card_without_interfaces_does_not_add_them(monkeypatch: pytest.MonkeyPatch) -> None:
+    agent: Final = _sample_agent_response()
+    test_client: Final = _client_with_agent_card(monkeypatch, LitellmUserRoles.INTERNAL_USER, agent)
+
+    response: Final = test_client.get("/v1/agents/agent-123", headers={"Authorization": "Bearer k"})
+
+    assert response.status_code == 200, response.text
+    assert response.json()["agent_card_params"]["url"] == "http://testserver/a2a/agent-123"
+    assert "supportedInterfaces" not in response.json()["agent_card_params"]
+
+
 class TestCheckAgentUrlHealth:
     """Unit tests for the _check_agent_url_health helper."""
 
@@ -1230,7 +1380,11 @@ def test_jwt_authentication_status_does_not_require_virtual_keys(
     monkeypatch.setattr(proxy_server, "general_settings", {"enable_jwt_auth": enabled})
     monkeypatch.setattr(proxy_server, "jwt_handler", handler)
     agent: Final = _sample_agent_response()
-    response: Final = agent_endpoints._redact_sensitive_agent_fields((agent,), is_admin=True)[0]
+    response: Final = agent_endpoints._redact_sensitive_agent_fields(
+        (agent,),
+        is_admin=True,
+        request_base_url="http://testserver",
+    )[0]
     assert response.jwt_auth_configured is expected
     assert agent.jwt_auth_configured is False
 

tests/unit/proxy/auth/test_route_checks.py

@@ -3925,6 +3925,7 @@ AGENT_INFERENCE_ROUTES = [
     "/a2a/abc-123",
     "/a2a/abc-123/message/send",
     "/a2a/abc-123/message/stream",
+    "/v1/a2a/abc-123/message/stream",
     "/a2a/abc-123/.well-known/agent-card.json",
 ]
 

tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py

@@ -27,8 +27,9 @@ from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import (
     BedrockGuardrail,
     _redact_pii_matches,
 )
+from litellm.proxy.guardrails.guardrail_initializers import initialize_bedrock
 from litellm.proxy.utils import ProxyLogging
-from litellm.types.guardrails import GuardrailEventHooks
+from litellm.types.guardrails import BedrockGuardrailOutputScope, Guardrail, GuardrailEventHooks, LitellmParams
 from litellm.types.proxy.guardrails.guardrail_hooks.bedrock_guardrails import (
     BedrockContentItem,
     BedrockTextContent,
@@ -1020,6 +1021,64 @@ def _anthropic_tool_result_conversation(
     ]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("scope", [None, "INTERVENTIONS", "FULL"])
+async def test_configured_output_scope_reaches_bedrock_and_preserves_healthy_assessments(
+    monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, scope: str | None
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    params: Final = LitellmParams.model_validate(
+        {
+            "guardrail": "bedrock",
+            "mode": "post_call",
+            "guardrailIdentifier": "test-guardrail",
+            "guardrailVersion": "DRAFT",
+            "contextual_grounding_from_messages": True,
+            "aws_access_key_id": "test-access-key",
+            "aws_secret_access_key": "test-secret-key",
+            "aws_region_name": "us-east-1",
+            "aws_bedrock_runtime_endpoint": "https://bedrock.example",
+            **({"outputScope": scope} if scope is not None else {}),
+        }
+    )
+    guardrail: Final = initialize_bedrock(params, Guardrail(guardrail_name="grounding", litellm_params=params))
+    assessment: Final = {
+        "contextualGroundingPolicy": {
+            "filters": [{"type": "GROUNDING", "score": 0.96, "threshold": 0.8, "detected": False, "action": "NONE"}]
+        }
+    }
+
+    def provider(request: httpx.Request) -> httpx.Response:
+        body: Final = json.loads(request.content)
+        return httpx.Response(
+            200,
+            json={"action": "NONE", "assessments": [assessment] if body.get("outputScope") == "FULL" else []},
+        )
+
+    route: Final = respx_mock.post("https://bedrock.example/guardrail/test-guardrail/version/DRAFT/apply").mock(
+        side_effect=provider
+    )
+    request_data: Final = {
+        "messages": [
+            {"role": "system", "content": "The tower is 330 metres tall."},
+            {"role": "user", "content": "How tall is the tower?"},
+        ]
+    }
+    result: Final = await guardrail.apply_guardrail(
+        inputs={"texts": ["The tower is 330 metres tall."]}, request_data=request_data, input_type="response"
+    )
+
+    sent: Final = json.loads(route.calls.last.request.content)
+    if scope is None:
+        assert "outputScope" not in sent
+    else:
+        assert sent["outputScope"] == scope
+    assert sent["source"] == "OUTPUT"
+    assert result["texts"] == ["The tower is 330 metres tall."]
+    recorded: Final = request_data["metadata"]["standard_logging_guardrail_information"][-1]["guardrail_response"]
+    assert recorded["assessments"] == ([assessment] if scope == "FULL" else [])
+
+
 @pytest.mark.asyncio
 async def test_during_call_hook_skips_bedrock_call_for_tool_result_only_turn():
     """A tool_result-only latest user turn must not post an empty content list to Bedrock.
@@ -1805,7 +1864,11 @@ async def test_make_bedrock_api_request_logging_event_type_for_spend_logs():
 
 @pytest.mark.asyncio
 async def test_make_bedrock_api_request_filters_dynamic_evaluation_overrides():
-    guardrail = BedrockGuardrail(guardrailIdentifier="test-guardrail", guardrailVersion="DRAFT")
+    guardrail = BedrockGuardrail(
+        guardrailIdentifier="test-guardrail",
+        guardrailVersion="DRAFT",
+        outputScope=BedrockGuardrailOutputScope.INTERVENTIONS,
+    )
     mock_credentials = MagicMock()
     mock_credentials.access_key = "test-access-key"
     mock_credentials.secret_key = "test-secret-key"

tests/unit/proxy/guardrails/guardrail_hooks/test_ovalix_extraction.py

@@ -0,0 +1,443 @@
+import base64
+from typing import Final
+
+from litellm.proxy.guardrails.guardrail_hooks.ovalix.ovalix_extraction import (
+    extract_file_parts_from_images,
+    extract_file_parts_from_messages,
+    extract_tool_results,
+    make_tool_data,
+    tool_call_to_tool_data,
+    tool_result_text_indices,
+)
+
+
+def _b64(raw: bytes) -> str:
+    return base64.b64encode(raw).decode()
+
+
+def test_file_block_data_url_decoded():
+    msgs = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "file", "file": {"filename": "a.txt", "file_data": f"data:text/plain;base64,{_b64(b'hi')}"}}
+            ],
+        }
+    ]
+    parts = extract_file_parts_from_messages(msgs, size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == b"hi" and parts[0].name == "a.txt" and parts[0].inline
+
+
+def test_image_url_reference_tracked_by_name():
+    msgs = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://x.test/pic.png"}}]}]
+    parts = extract_file_parts_from_messages(msgs, size_limit=1000)
+    assert len(parts) == 1 and parts[0].data is None and parts[0].inline is False and parts[0].name == "pic.png"
+
+
+def test_oversize_file_flagged_no_data():
+    big = _b64(b"x" * 100)
+    msgs = [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "file",
+                    "file": {"filename": "big.bin", "file_data": f"data:application/octet-stream;base64,{big}"},
+                }
+            ],
+        }
+    ]
+    parts = extract_file_parts_from_messages(msgs, size_limit=10)
+    assert len(parts) == 1 and parts[0].oversize is True and parts[0].data is None
+
+
+def test_images_field_data_url():
+    parts = extract_file_parts_from_images([f"data:image/png;base64,{_b64(b'png')}"], size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == b"png" and parts[0].inline
+
+
+def test_tool_call_to_tool_data_parses_arguments():
+    td = tool_call_to_tool_data(
+        {"id": "c1", "type": "function", "function": {"name": "get_weather", "arguments": '{"city": "TLV"}'}}
+    )
+    assert (
+        td["content"] == '{"city": "TLV"}'
+        and td["tool_name"] == "get_weather"
+        and td["action_name"] == "get_weather"
+        and td["tool_input"] == {"city": "TLV"}
+    )
+
+
+def test_tool_call_malformed_dropped():
+    assert tool_call_to_tool_data({"id": "c1", "type": "function", "function": {"name": ""}}) is None
+    assert tool_call_to_tool_data({"id": "c1"}) is None
+
+
+def test_extract_tool_results_correlates_name():
+    msgs = [
+        {"role": "assistant", "tool_calls": [{"id": "c1", "function": {"name": "get_weather"}}]},
+        {"role": "tool", "tool_call_id": "c1", "content": "sunny"},
+        {"role": "tool", "tool_call_id": "unknown", "content": "orphan"},
+    ]
+    results = extract_tool_results(msgs)
+    assert ("get_weather", "sunny", "c1") in results
+    assert ("tool_result", "orphan", "unknown") in results
+
+
+def test_make_tool_data_truncates_and_defaults_name():
+    td = make_tool_data("   ", "content")
+    assert td["tool_name"] == "tool_result" and td["action_name"] == "tool_result"
+    long = "x" * 200
+    td2 = make_tool_data(long, "c")
+    assert len(td2["tool_name"]) == 100 and td2["action_name"] == long
+
+
+def test_unhashable_block_type_skipped_without_raising():
+    msgs = [{"role": "user", "content": [{"type": ["file"], "file": {"filename": "a.txt"}}]}]
+    parts = extract_file_parts_from_messages(msgs, size_limit=1000)
+    assert parts == ()
+
+
+def test_unhashable_tool_call_id_skipped_without_raising():
+    msgs = [
+        {"role": "assistant", "tool_calls": [{"id": ["c1"], "function": {"name": "get_weather"}}]},
+        {"role": "tool", "tool_call_id": ["c1"], "content": "sunny"},
+    ]
+    results = extract_tool_results(msgs)
+    assert results == (("tool_result", "sunny", ["c1"]),)
+
+
+def test_extract_tool_results_list_form_content():
+    msgs = [{"role": "tool", "tool_call_id": "c1", "content": [{"type": "text", "text": "part1"}, "part2"]}]
+    results = extract_tool_results(msgs)
+    assert ("tool_result", "part1\npart2", "c1") in results
+
+
+def test_extract_tool_results_dict_form_content():
+    msgs = [{"role": "tool", "tool_call_id": "c1", "content": {"city": "TLV"}}]
+    results = extract_tool_results(msgs)
+    assert ("tool_result", '{"city": "TLV"}', "c1") in results
+
+
+def test_images_field_oversize_flagged_no_data():
+    big = _b64(b"x" * 100)
+    parts = extract_file_parts_from_images([f"data:image/png;base64,{big}"], size_limit=10)
+    assert len(parts) == 1 and parts[0].oversize is True and parts[0].data is None
+
+
+class _StubFunction:
+    def __init__(self, name, arguments):
+        self.name = name
+        self.arguments = arguments
+
+
+class _StubToolCall:
+    def __init__(self, function):
+        self.function = function
+
+
+def test_tool_call_to_tool_data_accepts_object_style_tool_call():
+    tool_call = _StubToolCall(_StubFunction("get_weather", '{"city": "TLV"}'))
+    td = tool_call_to_tool_data(tool_call)
+    assert td["tool_name"] == "get_weather" and td["tool_input"] == {"city": "TLV"}
+
+
+def _msgs(block):
+    return [{"role": "user", "content": [block]}]
+
+
+def test_image_url_block_data_url_decoded_from_messages():
+    block = {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_b64(b'png')}"}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == b"png" and parts[0].inline and parts[0].name is None
+
+
+def test_image_url_block_non_string_url_skipped():
+    block = {"type": "image_url", "image_url": {"url": 123}}
+    assert extract_file_parts_from_messages(_msgs(block), size_limit=1000) == ()
+
+
+def test_input_image_block_data_url_decoded():
+    block = {"type": "input_image", "image_url": f"data:image/png;base64,{_b64(b'img')}"}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == b"img" and parts[0].inline
+
+
+def test_file_block_non_dict_file_skipped():
+    block = {"type": "file", "file": "not-a-dict"}
+    assert extract_file_parts_from_messages(_msgs(block), size_limit=1000) == ()
+
+
+def test_file_block_reference_without_bytes_is_name_only():
+    block = {"type": "file", "file": {"file_id": "file-abc"}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].name == "file-abc" and parts[0].data is None and parts[0].inline is False
+
+
+def test_file_block_urlsafe_base64_decoded_via_fallback():
+    raw = b"\xff\xff\xfe"  # encodes with url-unsafe chars '+'/'/' in standard b64
+    urlsafe = base64.urlsafe_b64encode(raw).decode()
+    assert "-" in urlsafe or "_" in urlsafe
+    block = {
+        "type": "file",
+        "file": {"filename": "b.bin", "file_data": f"data:application/octet-stream;base64,{urlsafe}"},
+    }
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == raw
+
+
+def test_file_block_data_url_without_base64_marker_is_undecodable():
+    block = {"type": "file", "file": {"filename": "n.txt", "file_data": "data:text/plain,hello"}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].undecodable and parts[0].data is None and parts[0].name == "n.txt"
+
+
+def test_input_file_block_data_url_decoded():
+    block = {"type": "input_file", "filename": "doc.pdf", "file_data": f"data:application/pdf;base64,{_b64(b'pdf')}"}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == b"pdf" and parts[0].name == "doc.pdf" and parts[0].inline
+
+
+def test_input_file_block_file_url_reference_name_only():
+    block = {"type": "input_file", "file_url": "https://x.test/report.csv"}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].name == "report.csv" and parts[0].data is None and parts[0].inline is False
+
+
+def test_input_audio_block_decoded():
+    block = {"type": "input_audio", "input_audio": {"data": _b64(b"wav"), "format": "wav"}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == b"wav" and parts[0].name == "audio.wav" and parts[0].inline
+
+
+def test_input_audio_block_undecodable_is_flagged():
+    block = {"type": "input_audio", "input_audio": {"data": "!!!not-base64!!!"}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].undecodable and parts[0].data is None and parts[0].name == "audio.bin"
+
+
+def test_input_audio_block_non_dict_skipped():
+    block = {"type": "input_audio", "input_audio": "nope"}
+    assert extract_file_parts_from_messages(_msgs(block), size_limit=1000) == ()
+
+
+def test_tool_call_dict_arguments_serialized_and_parsed():
+    td = tool_call_to_tool_data({"function": {"name": "f", "arguments": {"a": 1}}})
+    assert td["content"] == '{"a": 1}' and td["tool_input"] == {"a": 1}
+
+
+def test_tool_call_none_arguments_yields_empty_content():
+    td = tool_call_to_tool_data({"function": {"name": "f", "arguments": None}})
+    assert td["content"] == "" and td["tool_input"] == {}
+
+
+def test_tool_call_non_string_non_dict_arguments_serialized():
+    td = tool_call_to_tool_data({"function": {"name": "f", "arguments": [1, 2]}})
+    assert td["content"] == "[1, 2]" and td["tool_input"] == {}
+
+
+def test_tool_call_invalid_json_string_arguments_kept_as_content():
+    td = tool_call_to_tool_data({"function": {"name": "f", "arguments": "{not json"}})
+    assert td["content"] == "{not json" and td["tool_input"] == {}
+
+
+def test_tool_result_with_non_string_tool_call_id_uses_default_name():
+    msgs = [{"role": "tool", "tool_call_id": ["c1"], "content": "orphan"}]
+    results = extract_tool_results(msgs)
+    assert results == (("tool_result", "orphan", ["c1"]),)
+
+
+def test_images_field_http_url_is_name_only_reference():
+    parts = extract_file_parts_from_images(["https://x.test/pic.png"], size_limit=1000)
+    assert len(parts) == 1 and parts[0].name == "pic.png" and parts[0].data is None and parts[0].inline is False
+
+
+def test_messages_skip_non_dict_and_unknown_blocks():
+    msgs = [
+        "not-a-message",
+        {
+            "role": "user",
+            "content": [
+                "bare-string-block",
+                {"type": "text", "text": "hi"},
+                {"type": "file", "file": {"filename": "a.txt", "file_data": f"data:text/plain;base64,{_b64(b'x')}"}},
+            ],
+        },
+    ]
+    parts = extract_file_parts_from_messages(msgs, size_limit=1000)
+    assert len(parts) == 1 and parts[0].name == "a.txt" and parts[0].data == b"x"
+
+
+def test_image_url_block_invalid_data_url_is_flagged():
+    block = {"type": "image_url", "image_url": {"url": "data:image/png;base64,%%%invalid%%%"}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].undecodable and parts[0].data is None
+
+
+def test_tool_message_with_empty_content_is_skipped():
+    msgs = [{"role": "tool", "tool_call_id": "c1", "content": "   "}]
+    assert extract_tool_results(msgs) == ()
+
+
+def test_extract_tool_results_skips_non_dict_messages_and_tool_calls():
+    msgs = [
+        "junk",
+        {"role": "assistant", "tool_calls": ["not-a-dict", {"id": "c1", "function": {"name": "f"}}]},
+        {"role": "tool", "tool_call_id": "c1", "content": "ok"},
+    ]
+    assert extract_tool_results(msgs) == (("f", "ok", "c1"),)
+
+
+def test_tool_result_takes_its_name_from_the_nearest_earlier_declaration():
+    msgs: Final = [
+        {"role": "assistant", "tool_calls": [{"id": "c1", "function": {"name": "first"}}]},
+        {"role": "tool", "tool_call_id": "c1", "content": "r1"},
+        {
+            "role": "assistant",
+            "tool_calls": [{"id": "c1", "function": {"name": "second"}}, {"id": "c1", "function": {"name": "third"}}],
+        },
+        {"role": "tool", "tool_call_id": "c1", "content": "r2"},
+        {"role": "tool", "tool_call_id": "c2", "content": "r3"},
+        {"role": "assistant", "tool_calls": [{"id": "c2", "function": {"name": "too-late"}}]},
+    ]
+    assert extract_tool_results(msgs) == (("first", "r1", "c1"), ("third", "r2", "c1"), ("tool_result", "r3", "c2"))
+
+
+class _ReadCountingMessage(dict):
+    def __init__(self, **fields):
+        super().__init__(**fields)
+        self.reads = 0
+
+    def get(self, key, default=None):
+        self.reads += 1
+        return super().get(key, default)
+
+
+def _reads_of_a_declaration_followed_by(tool_results):
+    declaration: Final = _ReadCountingMessage(role="assistant", tool_calls=[{"id": "c0", "function": {"name": "f"}}])
+    tool_messages: Final = [
+        {"role": "tool", "tool_call_id": f"c{index}", "content": "x"} for index in range(tool_results)
+    ]
+    extract_tool_results([declaration, *tool_messages])
+    return declaration.reads
+
+
+def test_tool_result_names_resolve_without_rescanning_earlier_messages():
+    assert _reads_of_a_declaration_followed_by(1000) == _reads_of_a_declaration_followed_by(1)
+
+
+def test_malformed_data_url_is_undecodable():
+    block = {"type": "file", "file": {"filename": "x.bin", "file_data": "data:garbage-no-comma"}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].undecodable and parts[0].data is None and parts[0].name == "x.bin"
+
+
+def test_input_audio_block_without_data_skipped():
+    block = {"type": "input_audio", "input_audio": {"format": "wav"}}
+    assert extract_file_parts_from_messages(_msgs(block), size_limit=1000) == ()
+
+
+def test_images_field_non_string_entries_skipped():
+    assert extract_file_parts_from_images([123, None, ""], size_limit=1000) == ()
+
+
+def test_make_tool_data_whitespace_after_truncation_defaults_name():
+    name = " " * 100 + "x"
+    td = make_tool_data(name, "c")
+    assert td["tool_name"] == "tool_result" and td["action_name"] == name
+
+
+def test_raw_base64_file_data_without_data_url_prefix_decoded():
+    block = {"type": "file", "file": {"filename": "a.bin", "file_data": _b64(b"rawbytes")}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == b"rawbytes" and parts[0].mime_hint is None
+
+
+def test_images_field_raw_base64_without_data_url_prefix_decoded():
+    parts = extract_file_parts_from_images([_b64(b"rawimg")], size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == b"rawimg"
+
+
+def test_whitespace_only_base64_payload_yields_no_bytes():
+    block = {"type": "file", "file": {"filename": "n.txt", "file_data": "data:text/plain;base64,    "}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].data is None and parts[0].inline is False and parts[0].name == "n.txt"
+
+
+def test_base64_invalid_even_after_urlsafe_translate_is_undecodable():
+    block = {"type": "file", "file": {"filename": "b.bin", "file_data": "_@@@"}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].undecodable and parts[0].data is None and parts[0].name == "b.bin"
+
+
+def test_file_oversize_detected_after_decode_when_estimate_passes():
+    block = {"type": "file", "file": {"filename": "s.bin", "file_data": _b64(b"abc")}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=2)
+    assert len(parts) == 1 and parts[0].oversize is True and parts[0].data is None
+
+
+def test_image_http_url_that_urlparse_rejects_is_dropped():
+    parts = extract_file_parts_from_images(["http://["], size_limit=1000)
+    assert parts == ()
+
+
+def test_message_content_not_a_list_is_skipped():
+    parts = extract_file_parts_from_messages(
+        [{"role": "user", "content": "just a plain string prompt"}], size_limit=1000
+    )
+    assert parts == ()
+
+
+def test_tool_call_dict_arguments_non_serializable_falls_back_to_str():
+    td = tool_call_to_tool_data({"function": {"name": "f", "arguments": {"a": {1, 2}}}})
+    assert isinstance(td["content"], str) and td["tool_input"] == {"a": {1, 2}}
+
+
+def test_tool_call_non_serializable_other_arguments_falls_back_to_str():
+    td = tool_call_to_tool_data({"function": {"name": "f", "arguments": {1, 2}}})
+    assert isinstance(td["content"], str) and td["tool_input"] == {}
+
+
+def test_tool_result_dict_content_non_serializable_falls_back_to_str():
+    results = extract_tool_results([{"role": "tool", "tool_call_id": "c1", "content": {"x": {1, 2}}}])
+    assert len(results) == 1 and isinstance(results[0][1], str) and results[0][1].strip()
+
+
+def test_tool_result_text_indices_marks_only_tool_role_positions():
+    messages = [
+        {"role": "user", "content": "ask"},
+        {"role": "assistant", "tool_calls": [{"id": "c1", "function": {"name": "f"}}]},
+        {"role": "tool", "tool_call_id": "c1", "content": "result"},
+    ]
+    assert tool_result_text_indices(messages, ["ask", "result"]) == frozenset({1})
+
+
+def test_tool_result_text_indices_covers_every_text_block_of_a_tool_message():
+    messages = [
+        {"role": "user", "content": "ask"},
+        {
+            "role": "tool",
+            "tool_call_id": "c1",
+            "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}],
+        },
+    ]
+    assert tool_result_text_indices(messages, ["ask", "a", "b"]) == frozenset({1, 2})
+
+
+def test_tool_result_text_indices_empty_when_texts_do_not_replay_messages():
+    messages = [
+        {"role": "user", "content": "ask"},
+        {"role": "tool", "tool_call_id": "c1", "content": "result"},
+    ]
+    assert tool_result_text_indices(messages, ["result"]) == frozenset()
+    assert tool_result_text_indices(messages, ["ask", "tampered"]) == frozenset()
+
+
+def test_tool_result_text_indices_skips_blank_tool_content_never_submitted_as_tool():
+    messages = [{"role": "tool", "tool_call_id": "c1", "content": "   "}]
+    assert extract_tool_results(messages) == ()
+    assert tool_result_text_indices(messages, ["   "]) == frozenset()
+
+
+def test_tool_result_text_indices_empty_without_structured_messages():
+    assert tool_result_text_indices(None, ["ask"]) == frozenset()

tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py

@@ -2025,6 +2025,28 @@ class TestStreamingHttpErrorFrames:
     def _use_real_mappings(self, monkeypatch):
         _patch_translation_mappings(monkeypatch, load_guardrail_translation_mappings())
 
+    @pytest.mark.asyncio
+    async def test_v1_a2a_stream_route_scans_response_chunks(self):
+        guardrail: Final = RecordingGuardrail()
+        guardrail.streaming_sampling_rate = 1
+        chunk: Final = {
+            "jsonrpc": "2.0",
+            "id": "stream-1",
+            "result": {
+                "kind": "task",
+                "artifacts": [{"parts": [{"kind": "text", "text": "streamed text"}]}],
+            },
+        }
+
+        await _drive_stream(
+            UnifiedLLMGuardrails(),
+            guardrail,
+            [chunk],
+            request_route="/v1/a2a/my-agent/message/stream",
+        )
+
+        assert guardrail.apply_calls[0]["inputs"]["texts"] == ["streamed text"]
+
     @pytest.mark.asyncio
     async def test_chat_eos_block_emits_data_error_frame(self):
         import json as _json