Merge remote-tracking branch 'origin/main' into litellm_usage_export_key_user_metadata

.github/PULL_REQUEST_TEMPLATE/general.md

@@ -9,10 +9,16 @@
 <!-- The PR author rewrites this in their own words before asking for review. Being able to sum up the PR in a few
      plain sentences proves the author understands what it does, and human-written technical English is far easier
      for a reviewer to parse than AI-written English
-     An AI agent may draft a starting point here, but the author replaces it with their own handwritten version
+     An AI agent may draft a starting point here, but always wraps its draft in the make-me-human-readable tags below,
+     backticks included so the tags stay visible on GitHub, so nobody mistakes it for the author's words. The author rewrites it in their own words and deletes the tags.
+     Tags still present at review time mean the summary is AI-written and has not been rewritten yet
      Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
      No bullets, tables, or code names here. Everything else goes inside the Details block below -->
 
+`<make-me-human-readable>`
+<blah>
+`</make-me-human-readable>`
+
 <details>
 <summary>Details</summary>
 

AGENTS.md

@@ -41,6 +41,8 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM
 
 If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section
 
+Run QA proxies with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --num_workers 2 --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`, and swap its `--config` for a copy of `litellm/proxy/dev_config.yaml` that points both `router_settings` (`redis_host`, `redis_port`) and `general_settings.coordination_redis` (`host`, `port`) at one dedicated Redis instance, with credentials when required, so the workers share router state and spend, budget, and rate-limit counters. Check that two worker PIDs are serving, and use the same worker count and Redis setup for the before and after proof. When the PR is about `--run_gunicorn`, `--run_hypercorn`, or `--run_granian`, which `--reload` does not support, drop `--reload`. When it is about behavior without Redis, drop Redis. When the proxy prints `"workers" flag is ignored when reloading is enabled`, that checkout or its uvicorn reloads only one worker, so drop the two-worker check for it. Say in the proof which part you dropped and why
+
 Simple PRs may skip before/after e2e when the full diff is mechanical and focused checks completely prove it, such as #49030 restoring an existing helper name. Trace affected callers and run relevant tests/static checks at the tip. For a bug fix, establish that the regression catches the original defect. Size alone never qualifies. New behavior, API/provider semantics, auth, billing, persistence, concurrency or uncertain integration risk need live e2e. In Screenshots / Proof of Fix, record `E2E exception at <tip SHA>`, scope, rationale, commands and results. Tests count as proof only for this exception. Reassess changed diffs, retain CI/bot gates, and honor explicit e2e requests (2026-10-28, unnecessary live QA for #49030)
 
 For other PRs, proof must show the end user's real flow before and after, with commands and output against a live proxy and real provider calls where applicable. Pytest alone is insufficient. For UI or interactive coding-tool flows, drive that surface and embed before/after screenshots, with URLs, clicks and input fields a reviewer can reproduce

README.md

@@ -696,7 +696,7 @@ For companies that need better security, user management and professional suppor
 [Talk to founders](https://enterprise.litellm.ai/demo)
 
 This covers:
-- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**
+- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/enterprise):**
 - ✅ **Feature Prioritization**
 - ✅ **Custom Integrations**
 - ✅ **Professional Support - Dedicated discord + slack**

litellm/__init__.py

@@ -1647,6 +1647,7 @@ def set_global_gitlab_config(config: Mapping[str, object]) -> None:
 # Lazy loading system for heavy modules to reduce initial import time and memory usage
 
 if TYPE_CHECKING:
+    from . import guardrails as guardrails
     from litellm.types.utils import ModelInfo as _ModelInfoType
     from litellm.types.utils import PriorityReservationSettings
     from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@@ -2499,10 +2500,10 @@ def __getattr__(name: str) -> Any:
         handler_func: Final = registry[name]
         return handler_func(name)
 
-    if name == "proxy":
+    if name in ("proxy", "guardrails"):
         import importlib
 
-        return importlib.import_module("litellm.proxy")
+        return importlib.import_module(f"litellm.{name}")
 
     if name == "harness" or name in _AGENT_EXPORTS:
         import importlib

litellm/guardrails.py

@@ -0,0 +1,66 @@
+"""Invoke configured gateway guardrails without generating a completion."""
+
+from collections.abc import Mapping, Sequence
+from enum import Enum
+from typing import Final
+
+import httpx
+from pydantic import TypeAdapter
+
+import litellm
+from litellm.llms.custom_httpx.http_handler import (
+    get_async_httpx_client,  # pyright: ignore[reportUnknownVariableType]  # shared factory has untyped optional parameters
+)
+from litellm.types.guardrails import NativeGuardrailResponse
+
+_NATIVE_RESPONSE: Final[TypeAdapter[NativeGuardrailResponse]] = TypeAdapter(NativeGuardrailResponse)
+
+
+class _InputType(Enum):
+    REQUEST = "request"
+    RESPONSE = "response"
+
+
+async def apply(
+    guardrail_name: str,
+    *,
+    input: str | None = None,
+    response: str | None = None,
+    messages: Sequence[Mapping[str, object]] | None = None,
+    metadata: Mapping[str, object] | None = None,
+    api_base: str | None = None,
+    api_key: str | None = None,
+    timeout: float | httpx.Timeout | None = None,
+) -> NativeGuardrailResponse:
+    """Return a configured guardrail's native structured findings.
+
+    Supply exactly one of ``input`` or ``response``. Grounding guardrails can
+    use ``messages`` as evidence. Connection settings default to
+    ``litellm.api_base`` and ``litellm.api_key``.
+    """
+    if (input is None) == (response is None):
+        raise ValueError("Supply exactly one of input or response.")
+    resolved_base: Final = api_base if api_base is not None else litellm.api_base
+    if not resolved_base:
+        raise ValueError("Set api_base or litellm.api_base to your LiteLLM gateway URL.")
+    resolved_key: Final = api_key if api_key is not None else litellm.api_key
+    input_type: Final = _InputType.RESPONSE if response is not None else _InputType.REQUEST
+    payload: Final = dict[str, object](
+        guardrail_name=guardrail_name,
+        text=response if response is not None else input,
+        input_type=input_type.value,
+        response_format="native",
+    )
+    if messages is not None:
+        payload["messages"] = [dict(message) for message in messages]
+    if metadata is not None:
+        payload["metadata"] = dict(metadata)
+    gateway_root: Final = resolved_base.rstrip("/").removesuffix("/v1")
+    client: Final = get_async_httpx_client(llm_provider="litellm_guardrails")
+    result: Final = await client.post(  # pyright: ignore[reportUnknownMemberType]  # shared handler has untyped optional parameters
+        url=f"{gateway_root}/guardrails/apply_guardrail",
+        json=payload,
+        headers={"Authorization": f"Bearer {resolved_key}"} if resolved_key else {},
+        timeout=timeout,
+    )
+    return _NATIVE_RESPONSE.validate_json(result.content)

litellm/litellm_core_utils/streaming_handler.py

@@ -905,13 +905,21 @@ class CustomStreamWrapper:
         except Exception as e:
             raise e
 
+    def _served_openrouter_model(self) -> str | None:
+        if self.custom_llm_provider == LlmProviders.OPENROUTER.value or (
+            self.custom_llm_provider == "cached_response"
+            and self.logging_obj.model_call_details.get("custom_llm_provider") == LlmProviders.OPENROUTER.value
+        ):
+            return self._provider_response_model
+        return None
+
     def model_response_creator(
         self,
         chunk: dict | None = None,
         hidden_params: Mapping[str, object] | None = None,
         delta: Delta | None = None,
     ) -> ModelResponseStream:
-        _model: Final = self._cached_model_name
+        _model: Final = self._served_openrouter_model() or self._cached_model_name
         _logging_obj_llm_provider: Final = self._cached_logging_llm_provider
 
         if chunk is None:
@@ -1699,7 +1707,7 @@ class CustomStreamWrapper:
                 return dispatch_result.value
             response_obj = dispatch_result.response_obj
 
-            model_response.model = self.model
+            model_response.model = self._served_openrouter_model() or self.model
             ## FUNCTION CALL PARSING
             original_chunk: Final[_UpstreamChunkLike | None] = (
                 response_obj.get("original_chunk") if response_obj is not None else None
@@ -2561,7 +2569,7 @@ class CustomStreamWrapper:
         if partial is None or not isinstance(usage, Usage):
             return None
         if self.model:
-            partial.model = self.model
+            partial.model = self._served_openrouter_model() or self.model
         backfill_missing_cache_usage_fields(usage)
         return partial
 

litellm/llms/custom_httpx/aiohttp_handler.py

@@ -14,6 +14,7 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig
 from litellm.llms.base_llm.image_variations.transformation import (
     BaseImageVariationConfig,
 )
+from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import build_pool_wait_trace_config
 from litellm.llms.custom_httpx.aiohttp_transport import LiteLLMAiohttpTransport
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -90,7 +91,7 @@ class BaseLLMAIOHTTPHandler:
         if self.transport is None:
             connector: Final = self._get_connector()
             if connector:
-                return aiohttp.ClientSession(connector=connector)
+                return aiohttp.ClientSession(connector=connector, trace_configs=[build_pool_wait_trace_config()])
 
         transport: Final = self.transport or self._get_or_create_transport()
         if transport is not None and hasattr(transport, "_get_valid_client_session"):
@@ -99,7 +100,7 @@ class BaseLLMAIOHTTPHandler:
             except RuntimeError:
                 pass
 
-        return aiohttp.ClientSession()
+        return aiohttp.ClientSession(trace_configs=[build_pool_wait_trace_config()])
 
     def _get_async_client_session(self, dynamic_client_session: ClientSession | None = None) -> ClientSession:
         if dynamic_client_session:

litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py

@@ -0,0 +1,130 @@
+import asyncio
+from types import SimpleNamespace
+from typing import Final
+
+from aiohttp import (
+    ClientSession,
+    TraceConfig,
+    TraceConnectionQueuedEndParams,
+    TraceConnectionQueuedStartParams,
+    TraceRequestExceptionParams,
+    TraceRequestRedirectParams,
+    TraceRequestStartParams,
+)
+from yarl import URL
+
+from litellm._logging import verbose_logger
+
+_POOL_WAIT_WARNING_SECONDS: Final = 1.0
+
+
+class _PoolWaitContext(SimpleNamespace):
+    def __init__(self, trace_request_ctx: object = None) -> None:
+        super().__init__()
+        self.host: str | None = None
+        self.port: int | None = None
+        self.started_at: float | None = None
+        self.waited: float = 0.0
+        self.warning: asyncio.TimerHandle | None = None
+        self.warned: bool = False
+
+    def set_destination(self, params: TraceRequestStartParams) -> None:
+        self.host = params.url.host
+        self.port = params.url.port
+
+    def redirect(self, params: TraceRequestRedirectParams) -> None:
+        location: Final = params.response.headers.get("Location") or params.response.headers.get("URI")
+        if location is None:
+            return
+        try:
+            destination: Final = params.url.join(URL(location))
+            host: Final = destination.host
+            port: Final = destination.port
+        except ValueError:
+            return
+        self.host = host
+        self.port = port
+
+    def start(self, session: ClientSession) -> None:
+        loop: Final = asyncio.get_running_loop()
+        self.started_at = loop.time()
+        if not self.warned:
+            self.warning = loop.call_later(max(0.0, _POOL_WAIT_WARNING_SECONDS - self.waited), self.warn, session)
+
+    def elapsed(self) -> float:
+        return self.waited + (
+            asyncio.get_running_loop().time() - self.started_at if self.started_at is not None else 0.0
+        )
+
+    def warn(self, session: ClientSession) -> None:
+        if self.warned or self.started_at is None:
+            return
+        self.warned = True
+        connector: Final = session.connector
+        if connector is None:
+            return
+        verbose_logger.warning(
+            "aiohttp connection pool saturated: host=%s port=%s waited=%.3fs for a connection slot "
+            "(acquired_total=%s, limit=%s, limit_per_host=%s). "
+            "Wait occurred before sending the request upstream.",
+            self.host,
+            self.port,
+            self.elapsed(),
+            len(connector._acquired),  # pyright: ignore[reportPrivateUsage]  # aiohttp exposes no public acquired count
+            connector.limit,
+            connector.limit_per_host,
+        )
+
+    def finish(self, session: ClientSession, failed: bool = False) -> None:
+        if self.warning is not None:
+            self.warning.cancel()
+            self.warning = None
+        if self.started_at is None:
+            return
+        if failed or self.elapsed() >= _POOL_WAIT_WARNING_SECONDS:
+            self.warn(session)
+        self.waited = self.elapsed()
+        self.started_at = None
+
+
+async def _on_request_start(session: ClientSession, context: SimpleNamespace, params: TraceRequestStartParams) -> None:
+    if isinstance(context, _PoolWaitContext):
+        context.set_destination(params)
+
+
+async def _on_queued_start(
+    session: ClientSession, context: SimpleNamespace, params: TraceConnectionQueuedStartParams
+) -> None:
+    if isinstance(context, _PoolWaitContext):
+        context.start(session)
+
+
+async def _on_queued_end(
+    session: ClientSession, context: SimpleNamespace, params: TraceConnectionQueuedEndParams
+) -> None:
+    if isinstance(context, _PoolWaitContext):
+        context.finish(session)
+
+
+async def _on_request_exception(
+    session: ClientSession, context: SimpleNamespace, params: TraceRequestExceptionParams
+) -> None:
+    if isinstance(context, _PoolWaitContext):
+        context.finish(session, failed=True)
+
+
+async def _on_request_redirect(
+    session: ClientSession, context: SimpleNamespace, params: TraceRequestRedirectParams
+) -> None:
+    if isinstance(context, _PoolWaitContext):
+        context.redirect(params)
+
+
+def build_pool_wait_trace_config() -> TraceConfig:
+    trace: Final = TraceConfig(trace_config_ctx_factory=_PoolWaitContext)
+    trace.on_request_start.append(_on_request_start)
+    trace.on_connection_queued_start.append(_on_queued_start)
+    trace.on_connection_queued_end.append(_on_queued_end)
+    trace.on_request_exception.append(_on_request_exception)
+    trace.on_request_redirect.append(_on_request_redirect)
+    return trace

litellm/llms/custom_httpx/http_handler.py

@@ -44,6 +44,7 @@ from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
 from litellm.litellm_core_utils.request_timeout_resolver import (
     get_configured_request_timeout,
 )
+from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import build_pool_wait_trace_config
 from litellm.llms.custom_httpx.http2_connection_spread import AsyncHTTP2ConnectionSpread
 from litellm.types.llms.custom_http import *
 
@@ -1456,6 +1457,7 @@ class AsyncHTTPHandler:
                 connector=TCPConnector(**transport_connector_kwargs),
                 cookie_jar=DummyCookieJar(),
                 trust_env=trust_env,
+                trace_configs=[build_pool_wait_trace_config()],
             )
 
         # Use shared session if provided and valid

litellm/llms/litellm_proxy/skills/skill_search.py

@@ -131,7 +131,7 @@ async def search_skills(
     user_api_key_dict: UserAPIKeyAuth,
     proxy_logging_obj: ProxyLogging,
 ) -> SkillSearchOutcome:
-    if embedding_model is None:
+    if embedding_model is None or not embedding_model.strip():
         return SkillSearchNotConfigured(
             reason="skill search needs litellm_settings.skill_search_embedding_model set to an embedding model from model_list"
         )

litellm/model_prices_and_context_window_backup.json

@@ -15555,18 +15555,20 @@
         "input_cost_per_token": 2.65e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8000,
-        "max_output_tokens": 2048,
+        "max_output_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 3.5e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md",
         "supports_pdf_input": true
     },
     "bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": {
         "input_cost_per_token": 3e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8000,
-        "max_output_tokens": 2048,
+        "max_output_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 6e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md",
         "supports_pdf_input": true
     },
     "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": {
@@ -48955,6 +48957,7 @@
     },
     "together_ai/Qwen/Qwen3.8-2.4T-A95B": {
         "cache_read_input_token_cost": 2.5e-07,
+        "deprecation_date": "2026-11-04",
         "input_cost_per_token": 2e-06,
         "litellm_provider": "together_ai",
         "max_input_tokens": 1010000,

litellm/proxy/_lazy_features.py

@@ -105,10 +105,10 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
     LazyFeature(
         name="a2a",
         module_path="litellm.proxy.agent_endpoints.a2a_endpoints",
-        # ``/v1/a2a/{agent_id}/message/send`` is caught via the suffix so the
+        # ``/v1/a2a/{agent_id}/message/send`` and ``/message/stream`` are caught via suffixes so the
         # ``/v1/a2a`` prefix doesn't subsume the discover prefix below.
         path_prefixes=("/a2a",),
-        path_suffixes=("/message/send",),
+        path_suffixes=("/message/send", "/message/stream"),
     ),
     LazyFeature(
         name="a2a_registration",

litellm/proxy/_lazy_openapi_snapshot.json

@@ -54,6 +54,57 @@
           ],
           "title": "ValidationError",
           "type": "object"
+        },
+        "_A2AJsonRpcResponse": {
+          "additionalProperties": true,
+          "properties": {
+            "error": {
+              "anyOf": [
+                {
+                  "additionalProperties": true,
+                  "type": "object"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Error"
+            },
+            "id": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "integer"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Id"
+            },
+            "jsonrpc": {
+              "const": "2.0",
+              "title": "Jsonrpc",
+              "type": "string"
+            },
+            "result": {
+              "anyOf": [
+                {},
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Result"
+            }
+          },
+          "required": [
+            "jsonrpc",
+            "id"
+          ],
+          "title": "_A2AJsonRpcResponse",
+          "type": "object"
         }
       }
     },
@@ -242,6 +293,119 @@
           ]
         }
       },
+      "/a2a/{agent_id}/message/stream": {
+        "post": {
+          "description": "Invoke an agent using the A2A protocol (JSON-RPC 2.0).\n\nSupported methods:\n- message/send: Send a message and get a response\n- message/stream: Send a message and stream the response",
+          "operationId": "invoke_agent_a2a_a2a__agent_id__message_stream_post",
+          "parameters": [
+            {
+              "in": "path",
+              "name": "agent_id",
+              "required": true,
+              "schema": {
+                "title": "Agent Id",
+                "type": "string"
+              }
+            }
+          ],
+          "requestBody": {
+            "content": {
+              "application/json": {
+                "example": {
+                  "id": "stream-1",
+                  "jsonrpc": "2.0",
+                  "method": "message/stream",
+                  "params": {
+                    "message": {
+                      "kind": "message",
+                      "messageId": "message-1",
+                      "parts": [
+                        {
+                          "kind": "text",
+                          "text": "Hello"
+                        }
+                      ],
+                      "role": "user"
+                    }
+                  }
+                },
+                "schema": {
+                  "additionalProperties": true,
+                  "properties": {
+                    "id": {
+                      "anyOf": [
+                        {
+                          "type": "string"
+                        },
+                        {
+                          "type": "integer"
+                        },
+                        {
+                          "type": "null"
+                        }
+                      ],
+                      "title": "Id"
+                    },
+                    "jsonrpc": {
+                      "const": "2.0",
+                      "title": "Jsonrpc",
+                      "type": "string"
+                    },
+                    "method": {
+                      "title": "Method",
+                      "type": "string"
+                    },
+                    "params": {
+                      "additionalProperties": true,
+                      "title": "Params",
+                      "type": "object"
+                    }
+                  },
+                  "required": [
+                    "jsonrpc",
+                    "method",
+                    "params"
+                  ],
+                  "title": "_A2AJsonRpcRequest",
+                  "type": "object"
+                }
+              }
+            },
+            "required": true
+          },
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/_A2AJsonRpcResponse"
+                  }
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Invoke Agent A2A",
+          "tags": [
+            "a2a"
+          ]
+        }
+      },
       "/v1/a2a/{agent_id}/message/send": {
         "post": {
           "description": "Invoke an agent using the A2A protocol (JSON-RPC 2.0).\n\nSupported methods:\n- message/send: Send a message and get a response\n- message/stream: Send a message and stream the response",
@@ -287,6 +451,119 @@
             "a2a"
           ]
         }
+      },
+      "/v1/a2a/{agent_id}/message/stream": {
+        "post": {
+          "description": "Invoke an agent using the A2A protocol (JSON-RPC 2.0).\n\nSupported methods:\n- message/send: Send a message and get a response\n- message/stream: Send a message and stream the response",
+          "operationId": "invoke_agent_a2a_v1_a2a__agent_id__message_stream_post",
+          "parameters": [
+            {
+              "in": "path",
+              "name": "agent_id",
+              "required": true,
+              "schema": {
+                "title": "Agent Id",
+                "type": "string"
+              }
+            }
+          ],
+          "requestBody": {
+            "content": {
+              "application/json": {
+                "example": {
+                  "id": "stream-1",
+                  "jsonrpc": "2.0",
+                  "method": "message/stream",
+                  "params": {
+                    "message": {
+                      "kind": "message",
+                      "messageId": "message-1",
+                      "parts": [
+                        {
+                          "kind": "text",
+                          "text": "Hello"
+                        }
+                      ],
+                      "role": "user"
+                    }
+                  }
+                },
+                "schema": {
+                  "additionalProperties": true,
+                  "properties": {
+                    "id": {
+                      "anyOf": [
+                        {
+                          "type": "string"
+                        },
+                        {
+                          "type": "integer"
+                        },
+                        {
+                          "type": "null"
+                        }
+                      ],
+                      "title": "Id"
+                    },
+                    "jsonrpc": {
+                      "const": "2.0",
+                      "title": "Jsonrpc",
+                      "type": "string"
+                    },
+                    "method": {
+                      "title": "Method",
+                      "type": "string"
+                    },
+                    "params": {
+                      "additionalProperties": true,
+                      "title": "Params",
+                      "type": "object"
+                    }
+                  },
+                  "required": [
+                    "jsonrpc",
+                    "method",
+                    "params"
+                  ],
+                  "title": "_A2AJsonRpcRequest",
+                  "type": "object"
+                }
+              }
+            },
+            "required": true
+          },
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/_A2AJsonRpcResponse"
+                  }
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Invoke Agent A2A",
+          "tags": [
+            "a2a"
+          ]
+        }
       }
     }
   },
@@ -15326,6 +15603,10 @@
               ],
               "title": "Metadata"
             },
+            "response_format": {
+              "$ref": "#/components/schemas/GuardrailResponseFormat",
+              "default": "standard"
+            },
             "text": {
               "title": "Text",
               "type": "string"
@@ -16203,6 +16484,14 @@
           "title": "BedrockChecksSensitiveInformationModel",
           "type": "object"
         },
+        "BedrockGuardrailOutputScope": {
+          "enum": [
+            "INTERVENTIONS",
+            "FULL"
+          ],
+          "title": "BedrockGuardrailOutputScope",
+          "type": "string"
+        },
         "BlockedWord": {
           "description": "Represents a blocked word with its action and optional description",
           "properties": {
@@ -16746,6 +17035,14 @@
           "title": "GuardrailInfoResponse",
           "type": "object"
         },
+        "GuardrailResponseFormat": {
+          "enum": [
+            "standard",
+            "native"
+          ],
+          "title": "GuardrailResponseFormat",
+          "type": "string"
+        },
         "GuardrailSubmissionItem": {
           "properties": {
             "created_at": {
@@ -16998,6 +17295,7 @@
           "title": "HTTPValidationError",
           "type": "object"
         },
+        "JsonValue": {},
         "LakeraCategoryThresholds": {
           "additionalProperties": true,
           "properties": {
@@ -17924,6 +18222,18 @@
               "description": "Whether to fail the request if the guardrail encounters an error. Implemented by guardrail='model_armor', 'generic_guardrail_api' and 'crowdstrike_aidr'. True (default) raises the error. False logs a critical error and lets the request proceed, so only a valid guardrail response can block or modify it.",
               "title": "Fail On Error"
             },
+            "file_checkpoint_id": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "File-checkpoint ID for the Ovalix Tracker service (falls back to pre/post).",
+              "title": "File Checkpoint Id"
+            },
             "file_guardrail_timeout": {
               "anyOf": [
                 {
@@ -18426,6 +18736,17 @@
               ],
               "description": "Optional parameters for the guardrail"
             },
+            "outputScope": {
+              "anyOf": [
+                {
+                  "$ref": "#/components/schemas/BedrockGuardrailOutputScope"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "ApplyGuardrail response scope. FULL includes detected and non-detected findings. Omit to retain Bedrock's default of returning detected findings only."
+            },
             "output_parse_pii": {
               "anyOf": [
                 {
@@ -20308,7 +20629,24 @@
               "content": {
                 "application/json": {
                   "schema": {
-                    "$ref": "#/components/schemas/ApplyGuardrailResponse"
+                    "anyOf": [
+                      {
+                        "$ref": "#/components/schemas/ApplyGuardrailResponse"
+                      },
+                      {
+                        "additionalProperties": {
+                          "$ref": "#/components/schemas/JsonValue"
+                        },
+                        "type": "object"
+                      },
+                      {
+                        "items": {
+                          "$ref": "#/components/schemas/JsonValue"
+                        },
+                        "type": "array"
+                      }
+                    ],
+                    "title": "Response Apply Guardrail Apply Guardrail Post"
                   }
                 }
               },
@@ -20400,7 +20738,24 @@
               "content": {
                 "application/json": {
                   "schema": {
-                    "$ref": "#/components/schemas/ApplyGuardrailResponse"
+                    "anyOf": [
+                      {
+                        "$ref": "#/components/schemas/ApplyGuardrailResponse"
+                      },
+                      {
+                        "additionalProperties": {
+                          "$ref": "#/components/schemas/JsonValue"
+                        },
+                        "type": "object"
+                      },
+                      {
+                        "items": {
+                          "$ref": "#/components/schemas/JsonValue"
+                        },
+                        "type": "array"
+                      }
+                    ],
+                    "title": "Response Apply Guardrail Guardrails Apply Guardrail Post"
                   }
                 }
               },

litellm/proxy/_types.py

@@ -501,6 +501,7 @@ class LiteLLMRoutes(enum.Enum):
         "/a2a/{agent_id}/message/send",
         "/v1/a2a/{agent_id}/message/send",
         "/a2a/{agent_id}/message/stream",
+        "/v1/a2a/{agent_id}/message/stream",
         "/a2a/{agent_id}/.well-known/agent-card.json",
     )
 

litellm/proxy/a2a/agent_card.py

@@ -12,6 +12,8 @@ from collections.abc import Mapping
 from copy import deepcopy
 from typing import Final, Literal
 
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
+
 SupportedA2AVersion = Literal["0.3", "1.0"]
 
 # Protocol versions LiteLLM can serve to A2A clients. The admin pins one per agent;
@@ -21,7 +23,6 @@ SUPPORTED_A2A_PROTOCOL_VERSIONS: Final[tuple[SupportedA2AVersion, ...]] = ("0.3"
 # Default served version when the agent card does not pin one.
 LITELLM_A2A_PROTOCOL_VERSION: Final = "1.0"
 
-
 _PROTOCOL_VERSION_PATTERN: Final = re.compile(
     r"^(\d+\.\d+)(?:\.\d+(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?)?$"
 )
@@ -50,6 +51,24 @@ def resolve_served_protocol_version(card: Mapping[str, object] | None) -> str:
     return normalized if normalized is not None else LITELLM_A2A_PROTOCOL_VERSION
 
 
+def point_agent_card_at_proxy(
+    agent_card: Mapping[str, object],
+    proxy_url: str,
+) -> dict[str, object]:  # mutable-ok: the public helper contract requires a dictionary
+    interfaces: Final = agent_card.get("supportedInterfaces")
+    interface_items: Final = (
+        tuple(iter_items(agent_card.get("supportedInterfaces"))) if isinstance(interfaces, list) else ()
+    )
+    first: Final[object] = interface_items[0] if interface_items else None
+    first_mapping: Final = as_str_mapping(first)
+    pointed_interfaces: Final = (
+        {"supportedInterfaces": [{**first_mapping, "url": proxy_url}, *interface_items[1:]]}
+        if isinstance(first, dict) and first_mapping is not None
+        else {}
+    )
+    return deepcopy({**agent_card, "url": proxy_url, **pointed_interfaces})
+
+
 # Security scheme exposed by the LiteLLM-fronted agent card. Always replaces
 # whatever upstream advertised — the client must authenticate to the proxy,
 # not the upstream agent.

litellm/proxy/agent_endpoints/a2a_endpoints.py

@@ -12,9 +12,8 @@ The A2A SDK can point to LiteLLM's URL and invoke agents registered with LiteLLM
 
 import json
 from collections.abc import AsyncGenerator, Callable, Coroutine, Mapping
-from copy import deepcopy
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Protocol
+from typing import TYPE_CHECKING, Any, Final, Literal, Protocol
 from urllib.parse import urlparse
 
 from fastapi import APIRouter, Depends, HTTPException, Request, Response
@@ -29,6 +28,7 @@ from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
 from litellm.llms.a2a.common_utils import resolve_a2a_hop_auth_header
 from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
 from litellm.proxy._types import ProxyException, UserAPIKeyAuth
+from litellm.proxy.a2a.agent_card import point_agent_card_at_proxy
 from litellm.proxy.a2a.version_convert import (
     A2AVersion,
     normalize_agent_card,
@@ -43,6 +43,7 @@ from litellm.proxy.agent_endpoints.databricks_oauth import (
 )
 from litellm.proxy.agent_endpoints.utils import merge_agent_headers
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body
 from litellm.proxy.common_utils.openai_error_payload import status_typed_proxy_exception
 from litellm.proxy.common_utils.sse_keepalive import (
     SSE_COMMENT_PING,
@@ -59,6 +60,42 @@ if TYPE_CHECKING:
 
 router: Final = APIRouter()
 
+
+class _A2AJsonRpcRequest(BaseModel):
+    model_config = ConfigDict(extra="allow", frozen=True)
+
+    jsonrpc: Literal["2.0"]
+    id: str | int | None = None
+    method: str
+    params: Mapping[str, object]
+
+
+class _A2AJsonRpcResponse(BaseModel):
+    model_config = ConfigDict(extra="allow", frozen=True)
+
+    jsonrpc: Literal["2.0"]
+    id: str | int | None
+    result: object | None = None
+    error: Mapping[str, object] | None = None
+
+
+_A2A_MESSAGE_STREAM_REQUEST_BODY: Final = inline_request_body(
+    _A2AJsonRpcRequest,
+    {
+        "jsonrpc": "2.0",
+        "id": "stream-1",
+        "method": "message/stream",
+        "params": {
+            "message": {
+                "messageId": "message-1",
+                "role": "user",
+                "parts": [{"kind": "text", "text": "Hello"}],
+                "kind": "message",
+            }
+        },
+    },
+)
+
 # Mirrors the native seam's own headers: a reverse proxy that batches the whole
 # stream would swallow the keepalives this route sends to defeat idle timeouts.
 _SSE_KEEPALIVE_HEADERS: Final[Mapping[str, str]] = MappingProxyType(
@@ -206,6 +243,35 @@ def _forwarding_headers(
     return merged or None
 
 
+async def _log_agent_rejection(
+    *,
+    proxy_logging_obj: ProxyLogging,
+    user_api_key_dict: UserAPIKeyAuth,
+    request_id: object,
+    method: str | None,
+    rejection: HTTPException,
+    target_agent_id: str,
+    model: str,
+) -> None:
+    request_data: Final[dict[str, object]] = {  # mutable-ok: the failure hook stamps its logging state onto it
+        "jsonrpc": "2.0",
+        "id": request_id,
+        "method": method,
+        "model": model,
+        "agent_id": target_agent_id,
+        "metadata": {"agent_id": target_agent_id},
+    }
+    try:
+        await proxy_logging_obj.post_call_failure_hook(  # pyright: ignore[reportUnknownMemberType]  # bare dict in sig
+            request_data=request_data,
+            original_exception=rejection,
+            user_api_key_dict=user_api_key_dict,
+            route="/a2a/{agent_id}",
+        )
+    except Exception:  # noqa: BLE001  # a logging failure must not change the rejection the caller gets
+        verbose_proxy_logger.exception("Failed to log rejected A2A request")
+
+
 def _jsonrpc_error(
     request_id: object,
     code: int,
@@ -684,16 +750,12 @@ async def get_agent_card(
             )
 
         proxy_url: Final = get_custom_url(str(request.base_url), route=f"a2a/{agent_id}")
-        agent_card = deepcopy(agent.agent_card_params)
-        agent_card["url"] = proxy_url
-        interfaces: Final = agent_card.get("supportedInterfaces")
-        if isinstance(interfaces, list) and interfaces:
-            interfaces[0]["url"] = proxy_url
+        agent_card: Final = point_agent_card_at_proxy(agent.agent_card_params, proxy_url)
         served_version: Final = _served_version(agent, request)
-        agent_card = normalize_agent_card(agent_card, served_version)
+        normalized_agent_card: Final = normalize_agent_card(agent_card, served_version)
 
         verbose_proxy_logger.debug("Returning agent card for '%s' with proxy URL: %s", agent_id, proxy_url)
-        return JSONResponse(content=agent_card)
+        return JSONResponse(content=normalized_agent_card)
 
     except HTTPException:
         raise
@@ -717,6 +779,20 @@ async def get_agent_card(
     tags=["[beta] A2A Agents"],
     dependencies=[Depends(user_api_key_auth)],
 )
+@router.post(
+    "/a2a/{agent_id}/message/stream",
+    tags=["[beta] A2A Agents"],
+    dependencies=[Depends(user_api_key_auth)],
+    openapi_extra={"requestBody": _A2A_MESSAGE_STREAM_REQUEST_BODY},
+    responses={200: {"model": _A2AJsonRpcResponse}},
+)
+@router.post(
+    "/v1/a2a/{agent_id}/message/stream",
+    tags=["[beta] A2A Agents"],
+    dependencies=[Depends(user_api_key_auth)],
+    openapi_extra={"requestBody": _A2A_MESSAGE_STREAM_REQUEST_BODY},
+    responses={200: {"model": _A2AJsonRpcResponse}},
+)
 async def invoke_agent_a2a(
     agent_id: str,
     request: Request,
@@ -778,6 +854,15 @@ async def invoke_agent_a2a(
         # Find the agent
         agent: Final = await _get_agent(agent_id)
         if agent is None:
+            await _log_agent_rejection(
+                proxy_logging_obj=proxy_logging_obj,
+                user_api_key_dict=user_api_key_dict,
+                request_id=request_id,
+                method=method,
+                rejection=HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found"),
+                target_agent_id=agent_id,
+                model=f"a2a_agent/{agent_id}",
+            )
             return _jsonrpc_error(request_id, -32000, f"Agent '{agent_id}' not found", 404)
 
         served_version: Final = _served_version(agent, request, original_method)
@@ -787,10 +872,20 @@ async def invoke_agent_a2a(
             user_api_key_auth=user_api_key_dict,
         )
         if not is_allowed:
-            raise HTTPException(
+            denial: Final = HTTPException(
                 status_code=403,
                 detail=f"Agent '{agent_id}' is not allowed for your key/team. Contact proxy admin for access.",
             )
+            await _log_agent_rejection(
+                proxy_logging_obj=proxy_logging_obj,
+                user_api_key_dict=user_api_key_dict,
+                request_id=request_id,
+                method=method,
+                rejection=denial,
+                target_agent_id=agent.agent_id,
+                model=f"a2a_agent/{(as_str_mapping(agent.agent_card_params or {}) or {}).get('name', agent_id)}",
+            )
+            raise denial
 
         user_api_key_dict.invoked_agent_id = agent.agent_id
         strict_policy: Final = agent.identity_managed or bool(user_api_key_dict.requires_fresh_policy)

litellm/proxy/agent_endpoints/agent_search.py

@@ -140,7 +140,7 @@ async def search_agents(
     user_api_key_dict: UserAPIKeyAuth,
     proxy_logging_obj: ProxyLogging,
 ) -> AgentSearchOutcome:
-    if embedding_model is None:
+    if embedding_model is None or not embedding_model.strip():
         return AgentSearchNotConfigured(
             reason="agent search needs litellm_settings.agent_search_embedding_model set to an embedding model from model_list"
         )

litellm/proxy/agent_endpoints/endpoints.py

@@ -22,6 +22,7 @@ from typing_extensions import ReadOnly, Required, assert_never
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import AGENT_HEALTH_CHECK_GATHER_TIMEOUT_SECONDS, AGENT_HEALTH_CHECK_TIMEOUT_SECONDS
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
 from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 from litellm.proxy._types import (
     CommonProxyErrors,
@@ -33,6 +34,7 @@ from litellm.proxy.a2a.agent_card import (
     SUPPORTED_A2A_PROTOCOL_VERSIONS,
     merge_agent_card,
     normalize_protocol_version,
+    point_agent_card_at_proxy,
 )
 from litellm.proxy.agent_endpoints.agent_registry import (
     AgentIdWhere,
@@ -180,10 +182,36 @@ def _redact_agent_litellm_params_dict(
     return dict(parse_agent_litellm_params(redact_sensitive_agent_litellm_params(litellm_params)))
 
 
+def _non_admin_agent_card(
+    card: Mapping[str, object], proxy_url: str
+) -> dict[str, object]:  # mutable-ok: AgentResponse.agent_card_params requires a mutable dictionary
+    pointed_card: Final = point_agent_card_at_proxy(card, proxy_url)
+    interfaces: Final = pointed_card.get("supportedInterfaces")
+    interface_items: Final = (
+        tuple(iter_items(pointed_card.get("supportedInterfaces"))) if isinstance(interfaces, list) else ()
+    )
+    first_interface: Final[object] = interface_items[0] if interface_items else None
+    first_mapping: Final = as_str_mapping(first_interface)
+    proxy_interface: Final = (
+        {"supportedInterfaces": [{**first_mapping, "url": proxy_url}]}
+        if isinstance(first_interface, dict) and first_mapping is not None and first_mapping.get("url") == proxy_url
+        else {}
+    )
+    return {
+        **{
+            key: value
+            for key, value in pointed_card.items()
+            if key not in ("additionalInterfaces", "supportedInterfaces")
+        },
+        **proxy_interface,
+    }
+
+
 def _redact_sensitive_agent_fields(
     agents: Sequence[AgentResponse],
     *,
     is_admin: bool,
+    request_base_url: str,
 ) -> list[AgentResponse]:
     """
     Return copies of the given agents with credential-bearing litellm_params
@@ -207,6 +235,11 @@ def _redact_sensitive_agent_fields(
             copy.keys = None
             copy.kill_switch = None
             copy.identity = None
+            if copy.agent_card_params:
+                copy.agent_card_params = _non_admin_agent_card(
+                    copy.agent_card_params,
+                    get_custom_url(request_base_url, route=f"a2a/{copy.agent_id}"),
+                )
         if copy.litellm_params:
             copy.litellm_params = _redact_agent_litellm_params_dict(copy.litellm_params)
         copy.kill_switch = redact_kill_switch(copy.kill_switch)
@@ -461,14 +494,6 @@ async def get_agents(
                 global_agent_registry.ids_for_agent(agent.agent_id).isdisjoint(litellm.public_agent_groups)
             )
 
-        # litellm_params secrets are always redacted; keys/headers stay
-        # admin-only.
-        is_admin: Final = (
-            user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
-            or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value
-        )
-        returned_agents = _redact_sensitive_agent_fields(returned_agents, is_admin=is_admin)
-
         if health_check:
             agents_with_url: Final = [agent for agent in returned_agents if (agent.agent_card_params or {}).get("url")]
             agents_without_url = [agent for agent in returned_agents if not (agent.agent_card_params or {}).get("url")]
@@ -493,6 +518,18 @@ async def get_agents(
             healthy_ids: Final = {result["agent_id"] for result in health_results if result["healthy"]}
             returned_agents = [agent for agent in agents_with_url if agent.agent_id in healthy_ids] + agents_without_url
 
+        # litellm_params secrets are always redacted; keys/headers stay
+        # admin-only.
+        is_admin: Final = (
+            user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
+            or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value
+        )
+        returned_agents = _redact_sensitive_agent_fields(
+            returned_agents,
+            is_admin=is_admin,
+            request_base_url=str(request.base_url),
+        )
+
         if query is None:
             return returned_agents
         return await _rank_agents_by_query(query, returned_agents, top_k, user_api_key_dict)
@@ -691,7 +728,11 @@ async def create_agent(
 
         # The caller is a proxy admin (enforced above); litellm_params
         # secrets are still never echoed back in the response.
-        return _redact_sensitive_agent_fields((result,), is_admin=True)[0]
+        return _redact_sensitive_agent_fields(
+            (result,),
+            is_admin=True,
+            request_base_url=str(http_request.base_url),
+        )[0]
 
     except HTTPException:
         raise
@@ -708,6 +749,7 @@ async def create_agent(
 )
 async def get_agent_by_id(
     agent_id: str,
+    request: Request,
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
 ):
     """
@@ -770,7 +812,11 @@ async def get_agent_by_id(
             user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
             or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value
         )
-        agent = _redact_sensitive_agent_fields((agent,), is_admin=is_admin)[0]
+        agent = _redact_sensitive_agent_fields(
+            (agent,),
+            is_admin=is_admin,
+            request_base_url=str(request.base_url),
+        )[0]
 
         return agent
     except HTTPException:
@@ -879,7 +925,11 @@ async def update_agent(
             "Successfully updated agent '%s' (ID: %s) in memory", existing_agent.get("agent_name"), agent_id
         )
 
-        return _redact_sensitive_agent_fields((result,), is_admin=True)[0]
+        return _redact_sensitive_agent_fields(
+            (result,),
+            is_admin=True,
+            request_base_url=str(http_request.base_url),
+        )[0]
     except HTTPException:
         raise
     except Exception as e:
@@ -987,7 +1037,11 @@ async def patch_agent(
             "Successfully updated agent '%s' (ID: %s) in memory", existing_agent.get("agent_name"), agent_id
         )
 
-        return _redact_sensitive_agent_fields((result,), is_admin=True)[0]
+        return _redact_sensitive_agent_fields(
+            (result,),
+            is_admin=True,
+            request_base_url=str(http_request.base_url),
+        )[0]
     except HTTPException:
         raise
     except Exception as e:

litellm/proxy/dev_reload.py

@@ -0,0 +1,172 @@
+import importlib
+import logging
+import multiprocessing
+import os
+import signal
+import sys
+import threading
+from collections.abc import Callable, Mapping, Sequence
+from functools import partial
+from inspect import signature
+from multiprocessing.process import BaseProcess
+from socket import socket
+from typing import Final, Literal
+
+from pydantic import ConfigDict, ImportString, JsonValue, TypeAdapter, with_config
+from typing_extensions import ReadOnly, Required, TypedDict
+from uvicorn import Config, Server
+from uvicorn import config as uvicorn_config
+from uvicorn.supervisors import ChangeReload, Multiprocess
+
+
+@with_config(ConfigDict(extra="allow"))
+class ServerOptions(TypedDict, total=False):
+    host: ReadOnly[str]
+    port: ReadOnly[int]
+    server_header: ReadOnly[bool]
+    timeout_graceful_shutdown: ReadOnly[float]
+    log_config: ReadOnly[  # mutable-ok: uvicorn.Config requires a concrete logging dict
+        str | dict[str, JsonValue] | None
+    ]
+    log_level: ReadOnly[str | int]
+    timeout_keep_alive: ReadOnly[int]
+    timeout_worker_healthcheck: ReadOnly[int]
+    limit_max_requests: ReadOnly[int]
+    limit_max_requests_jitter: ReadOnly[int]
+    limit_concurrency: ReadOnly[int]
+    ssl_keyfile: ReadOnly[str]
+    ssl_certfile: ReadOnly[str]
+    loop: ReadOnly[Literal["none", "auto", "asyncio", "uvloop"]]
+
+
+@with_config(ConfigDict(extra="allow"))
+class ReloadOptions(ServerOptions):
+    app: Required[ReadOnly[str]]
+    reload: ReadOnly[bool]
+    reload_dirs: ReadOnly[tuple[str, ...]]
+    reload_includes: ReadOnly[tuple[str, ...]]
+
+
+RELOAD_OPTIONS: Final = TypeAdapter(ReloadOptions)
+SERVER_OPTIONS: Final = TypeAdapter(ServerOptions)
+ASGI_APP: Final = TypeAdapter(ImportString[Callable[..., object]])
+LOGGER: Final = logging.getLogger("uvicorn.error")
+RELOAD_WORKER_HEALTHCHECK_SECONDS: Final = 30
+RELOADER_ONLY_KEYS: Final = frozenset(ReloadOptions.__annotations__) - frozenset(ServerOptions.__annotations__)
+
+
+def multi_worker_reload_supported(platform: str = sys.platform) -> bool:
+    return platform != "win32" and hasattr(uvicorn_config, "STARTUP_FAILURE")
+
+
+def server_options(options: ReloadOptions) -> ServerOptions:
+    return SERVER_OPTIONS.validate_python(
+        {key: value for key, value in options.items() if key not in RELOADER_ONLY_KEYS}
+    )
+
+
+def load_app_or_exit(import_path: str, factory: bool = False) -> Callable[..., object]:
+    module_name: Final = import_path.partition(":")[0]
+    try:
+        importlib.import_module(module_name)
+        app: Final = ASGI_APP.validate_python(import_path)
+        return ASGI_APP.validate_python(app()) if factory else app
+    except Exception:
+        LOGGER.exception("Error loading ASGI app %s", import_path)
+        sys.exit(uvicorn_config.STARTUP_FAILURE)
+
+
+def stop_worker_when_pool_exits(pool_parent: BaseProcess) -> None:
+    pool_parent.join()
+    LOGGER.info("Worker pool process [%s] exited, stopping worker [%s].", pool_parent.pid, os.getpid())
+    os.kill(os.getpid(), signal.SIGTERM)
+
+
+def load_worker_app(import_path: str, worker_app_factory: bool) -> Callable[..., object]:
+    pool_parent: Final = multiprocessing.parent_process()
+    if pool_parent is not None:
+        threading.Thread(target=stop_worker_when_pool_exits, args=(pool_parent,), daemon=True).start()
+    return load_app_or_exit(import_path, worker_app_factory)
+
+
+def pool_config(options: ReloadOptions, workers: int, worker_overrides: Mapping[str, object] | None = None) -> Config:
+    overrides: Final = worker_overrides or {}
+    worker_app: Final = TypeAdapter(str).validate_python(overrides.get("app", options["app"]))
+    worker_app_factory: Final = TypeAdapter(bool).validate_python(overrides.get("factory", False))
+    return Config(
+        partial(load_worker_app, worker_app, worker_app_factory),
+        factory=True,
+        workers=workers,
+        **SERVER_OPTIONS.validate_python(
+            {
+                "timeout_worker_healthcheck": RELOAD_WORKER_HEALTHCHECK_SECONDS,
+                **server_options(options),
+                **{key: value for key, value in overrides.items() if key not in ("app", "factory")},
+            }
+        ),  # pyright: ignore[reportArgumentType]  # uvicorn types the float graceful shutdown timeout as int
+    )
+
+
+def stop_pool_when_reloader_exits(reloader: BaseProcess, pool_should_exit: threading.Event) -> None:
+    reloader.join()
+    LOGGER.info("Reloader process [%s] exited, stopping the worker pool.", reloader.pid)
+    pool_should_exit.set()
+
+
+class ReloadWorkerPool(Multiprocess):
+    quit_received: bool = False
+
+    def handle_quit(self) -> None:
+        LOGGER.info("Received SIGQUIT, exiting.")
+        self.quit_received = True
+        self.should_exit.set()
+
+    def terminate_all(self) -> None:
+        quit_signal: Final = getattr(signal, "SIGQUIT", None)
+        if quit_signal is not None and quit_signal in self.signal_queue:
+            self.handle_quit()
+        if not self.quit_received:
+            super().terminate_all()
+            return
+        for process in self.processes:
+            process.kill()
+
+
+def _run_worker_pool(config: Config, sockets: Sequence[socket] | None) -> None:
+    shared_sockets: Final = list(sockets or ())
+    pool: Final = (
+        ReloadWorkerPool(
+            config=config,
+            target=Server(config=config).run,  # pyright: ignore[reportCallIssue]  # Older uvicorn versions require target
+            sockets=shared_sockets,
+        )
+        if "target" in signature(Multiprocess).parameters
+        else ReloadWorkerPool(config=config, sockets=shared_sockets)
+    )
+    reloader: Final = multiprocessing.parent_process()
+    if reloader is not None:
+        threading.Thread(target=stop_pool_when_reloader_exits, args=(reloader, pool.should_exit), daemon=True).start()
+    pool.run()
+
+
+def run_multi_worker_reload(
+    options: ReloadOptions, workers: int, worker_overrides: Mapping[str, object] | None = None
+) -> None:
+    worker_config: Final = pool_config(options, workers, worker_overrides)
+    reload_config: Final = Config(
+        options["app"],
+        workers=1,
+        reload=options["reload"],
+        reload_dirs=list(options["reload_dirs"]),
+        reload_includes=list(options["reload_includes"]),
+        **server_options(options),  # pyright: ignore[reportArgumentType]  # uvicorn types the float graceful shutdown timeout as int
+    )
+    shared_socket: Final = worker_config.bind_socket()
+    try:
+        ChangeReload(
+            config=reload_config,
+            target=partial(_run_worker_pool, worker_config),
+            sockets=[shared_socket],
+        ).run()
+    finally:
+        shared_socket.close()

litellm/proxy/guardrails/guardrail_endpoints.py

@@ -9,7 +9,7 @@ import os
 from collections.abc import Awaitable, Callable, Mapping, Sequence
 from datetime import datetime, timezone
 from types import MappingProxyType, UnionType
-from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, Union, cast, get_args, get_origin
+from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, Union, cast, get_args, get_origin
 from urllib.parse import urlparse
 
 from fastapi import APIRouter, Depends, HTTPException, Request
@@ -43,6 +43,11 @@ from litellm.proxy.guardrails.guardrail_registry import (
     encrypt_guardrail_litellm_params,
     parse_tolerant_litellm_params,
 )
+from litellm.proxy.guardrails.response_handlers import (
+    GUARDRAIL_RESPONSE_HANDLERS,
+    GuardrailEndpointResponse,
+    StandardGuardrailResponseHandler,
+)
 from litellm.proxy.guardrails.usage_endpoints import router as guardrails_usage_router
 from litellm.proxy.management_endpoints.common_utils import (  # noqa: F401  # legacy module exports
     _user_has_admin_view,  # pyright: ignore[reportPrivateUsage,reportUnusedImport]  # backwards-compatible package export
@@ -53,7 +58,6 @@ from litellm.repositories.table_repositories import GuardrailsRepository
 from litellm.types.guardrails import (
     PII_ENTITY_CATEGORIES_MAP,
     ApplyGuardrailRequest,
-    ApplyGuardrailResponse,
     BaseLitellmParams,
     BedrockGuardrailConfigModel,
     BedrockGuardrailStreamingParams,
@@ -2350,13 +2354,7 @@ async def _run_test_guardrail(execute_guardrail: Callable[[], object], timeout:
     return await await_with_timeout(raw_result, remaining, label="test")
 
 
-def _resolve_guardrail_input_type(active_guardrail: CustomGuardrail, input_type: str) -> Literal["request", "response"]:
-    """Return the effective input_type, auto-upgrading to 'response' for post_call guardrails."""
-    if input_type == "request":
-        hook: Final = getattr(active_guardrail, "event_hook", None)
-        if hook == GuardrailEventHooks.post_call or hook == "post_call":
-            return "response"
-    return "response" if input_type == "response" else "request"
+_RESPONSE: Final[TypeAdapter[GuardrailEndpointResponse]] = TypeAdapter(GuardrailEndpointResponse)
 
 
 class _GuardrailLoggingObj(Protocol):
@@ -2384,35 +2382,43 @@ def _patch_logging_obj_for_guardrail(litellm_logging_obj: _GuardrailLoggingObj,
     )
 
 
-async def _emit_guardrail_success_logs(
+async def _guardrail_response_after_success_hook(
+    response_handler: StandardGuardrailResponseHandler,
     proxy_logging_obj: _GuardrailProxyLogging,
-    litellm_logging_obj: _GuardrailLoggingObj | None,
-    data: dict,
+    data: Mapping[str, object],
     user_api_key_dict: UserAPIKeyAuth,
-    response: ApplyGuardrailResponse,
-    start_time: datetime,
-) -> ApplyGuardrailResponse:
-    """Fire proxy and LiteLLM success hooks after a successful guardrail run.
-
-    Each hook is wrapped defensively so a callback failure never prevents the
-    caller from receiving the guardrail response.  Returns the (possibly
-    hook-modified) response.
-    """
+    response: GuardrailEndpointResponse,
+) -> GuardrailEndpointResponse:
     try:
         modified: Final = await proxy_logging_obj.post_call_success_hook(
             data=data,
             user_api_key_dict=user_api_key_dict,
             response=response,
         )
-        if isinstance(modified, ApplyGuardrailResponse):
-            response = modified
+        return response_handler.accept_hook_result(response, modified)
     except Exception:
         verbose_proxy_logger.exception("apply_guardrail: post_call_success_hook failed")
+    return response
+
+
+async def _emit_guardrail_success_logs(
+    response_handler: StandardGuardrailResponseHandler,
+    proxy_logging_obj: _GuardrailProxyLogging,
+    litellm_logging_obj: _GuardrailLoggingObj | None,
+    data: Mapping[str, object],
+    user_api_key_dict: UserAPIKeyAuth,
+    response: GuardrailEndpointResponse,
+    start_time: datetime,
+) -> GuardrailEndpointResponse:
+    """Apply the success hook and log the response returned to the caller."""
+    updated_response: Final = await _guardrail_response_after_success_hook(
+        response_handler, proxy_logging_obj, data, user_api_key_dict, response
+    )
 
     # Build the logging payload after post_call_success_hook so that logged
     # data matches what the caller actually receives if the hook modified
     # the response.
-    response_for_logging: Final = {"response": response.model_dump(exclude_none=True)}
+    response_for_logging: Final = {"response": _RESPONSE.dump_python(updated_response, mode="json")}
 
     if litellm_logging_obj is not None:
         end_time: Final = datetime.now(timezone.utc)
@@ -2426,11 +2432,11 @@ async def _emit_guardrail_success_logs(
         except Exception:
             verbose_proxy_logger.exception("apply_guardrail: async_success_handler failed")
 
-    return response
+    return updated_response
 
 
-@router.post("/guardrails/apply_guardrail", response_model=ApplyGuardrailResponse)
-@router.post("/apply_guardrail", response_model=ApplyGuardrailResponse)
+@router.post("/guardrails/apply_guardrail", response_model=GuardrailEndpointResponse)
+@router.post("/apply_guardrail", response_model=GuardrailEndpointResponse)
 async def apply_guardrail(
     fastapi_request: Request,
     request: ApplyGuardrailRequest,
@@ -2455,7 +2461,7 @@ async def apply_guardrail(
     )
     from litellm.proxy.utils import handle_exception_on_proxy
 
-    data: dict = {
+    data: dict[str, object] = {
         "guardrail_name": request.guardrail_name,
         "input": [request.text],
         "messages": request.messages or [],
@@ -2495,18 +2501,15 @@ async def apply_guardrail(
         if litellm_logging_obj is not None:
             _patch_logging_obj_for_guardrail(litellm_logging_obj, request)
 
-        request_data: Final[dict] = {
-            **({"messages": request.messages} if request.messages is not None else {}),
-            **({"metadata": request.metadata} if request.metadata is not None else {}),
-        }
-        _input_type: Final = _resolve_guardrail_input_type(active_guardrail, request.input_type)
+        response_handler: Final = GUARDRAIL_RESPONSE_HANDLERS[request.response_format]
+        request_data: Final = dict(response_handler.prepare_request_data(request))
+        input_type: Final = response_handler.resolve_input_type(request, active_guardrail)
         guardrailed_inputs: Final = await active_guardrail.apply_guardrail(
             inputs={"texts": [request.text]},
             request_data=request_data,
-            input_type=_input_type,
+            input_type=input_type,
         )
-        response_text: Final = guardrailed_inputs.get("texts", [])
-        response = ApplyGuardrailResponse(response_text=response_text[0] if response_text else request.text)
+        response: Final = response_handler.build_response(request, guardrailed_inputs, request_data)
     except Exception as e:
         if litellm_logging_obj is not None and not isinstance(e, HTTPException):
             try:
@@ -2537,7 +2540,8 @@ async def apply_guardrail(
         raise handle_exception_on_proxy(status_typed_proxy_exception(e) if isinstance(e, HTTPException) else e)
 
     # Success logging outside except so a hook error never triggers failure handlers.
-    response = await _emit_guardrail_success_logs(
+    logged_response: Final = await _emit_guardrail_success_logs(
+        response_handler=response_handler,
         proxy_logging_obj=proxy_logging_obj,
         litellm_logging_obj=litellm_logging_obj,
         data=data,
@@ -2545,7 +2549,7 @@ async def apply_guardrail(
         response=response,
         start_time=start_time,
     )
-    return response
+    return logged_response
 
 
 # Usage (dashboard) endpoints: overview, detail, logs

litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py

@@ -65,6 +65,7 @@ from litellm.proxy.guardrails.anthropic_sse import (
 )
 from litellm.types.guardrails import (
     BedrockChecksConfigModel,
+    BedrockGuardrailOutputScope,
     BedrockGuardrailStreamingParams,
     GuardrailEventHooks,
     LitellmParams,
@@ -289,6 +290,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         streaming_buffer_release_on_scan: bool | None = None,
         streaming_scan_in_background: bool | None = None,
         async_handler: AsyncHTTPHandler | None = None,
+        outputScope: BedrockGuardrailOutputScope | None = None,
         **kwargs,
     ):
         self.async_handler = async_handler or get_async_httpx_client(
@@ -309,6 +311,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         )
         self.guardrailIdentifier = guardrailIdentifier
         self.guardrailVersion = guardrailVersion
+        self.outputScope = outputScope
         self.guardrail_provider = "bedrock"
         self.chunk_budget_chars = chunk_budget_chars
         self.contextual_grounding_from_messages = contextual_grounding_from_messages
@@ -1280,10 +1283,10 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         AWS billed them to ``completed_chunk_usages``, and the attempt log sums those
         with the blocking call's own usage.
         """
-        bedrock_request_data: Final = {
-            **base_request_data,
-            "content": content,
-        }
+        bedrock_request_data: Final = dict(base_request_data)
+        bedrock_request_data["content"] = content
+        if self.outputScope is not None and "outputScope" not in bedrock_request_data:
+            bedrock_request_data["outputScope"] = self.outputScope.value
         prepared_request: Final = await run_aws_signing(
             self._prepare_request,
             credentials=credentials,

litellm/proxy/guardrails/guardrail_hooks/ovalix/__init__.py

@@ -19,6 +19,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
     application_id: Final = getattr(litellm_params, "application_id", None)
     pre_checkpoint_id: Final = getattr(litellm_params, "pre_checkpoint_id", None)
     post_checkpoint_id: Final = getattr(litellm_params, "post_checkpoint_id", None)
+    file_checkpoint_id: Final = getattr(litellm_params, "file_checkpoint_id", None)
 
     _ovalix_callback: Final = OvalixGuardrail(
         guardrail_name=guardrail.get("guardrail_name", ""),
@@ -27,6 +28,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
         application_id=application_id,
         pre_checkpoint_id=pre_checkpoint_id,
         post_checkpoint_id=post_checkpoint_id,
+        file_checkpoint_id=file_checkpoint_id,
         event_hook=litellm_params.mode,
         default_on=litellm_params.default_on,
         timeout=litellm_params.timeout,

litellm/proxy/guardrails/guardrail_hooks/ovalix/ovalix.py

@@ -4,13 +4,19 @@ Use Ovalix Guardrails for your LLM calls. Supports pre_call (user input) and
 post_call (model output) checkpoints with optional correction/blocking.
 """
 
+import asyncio
+import base64
 import datetime
+import gzip
 import hashlib
+import mimetypes
 import os
-from typing import TYPE_CHECKING, Any, Final, Literal
+from collections.abc import Mapping, Sequence
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, Literal
 
 import httpx
-from typing_extensions import ReadOnly, TypedDict, Unpack
+from typing_extensions import TypedDict, Unpack
 
 from litellm._logging import verbose_proxy_logger
 from litellm.exceptions import GuardrailRaisedException
@@ -18,10 +24,23 @@ from litellm.integrations.custom_guardrail import (
     CustomGuardrail,
     log_guardrail_information,
 )
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
 )
+from litellm.proxy.guardrails.guardrail_hooks.ovalix.ovalix_extraction import (
+    FilePart,
+    extract_file_parts_from_images,
+    extract_file_parts_from_messages,
+    extract_tool_calls_from_messages,
+    extract_tool_results,
+    make_tool_data,
+    str_or_none,
+    tool_call_to_tool_data,
+    tool_data_key,
+    tool_result_text_indices,
+)
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.utils import GenericGuardrailAPIInputs
 
@@ -32,13 +51,34 @@ if TYPE_CHECKING:
 
 BLOCKED_BY_OVALIX_FALLBACK_MESSAGE: Final = "This message was blocked by Ovalix"
 BLOCKED_ACTION_TYPE: Final = "block"
+_MODIFY_ACTION_TYPES: Final = ("anonymize", "sanitize")
+_DEFAULT_FILE_SIZE_LIMIT: Final = 64 * 1024 * 1024
+_NO_METADATA: Final[Mapping[str, object]] = MappingProxyType({})
+_FILE_BLOCK_ESCALATION_REASON: Final = (
+    "This message was blocked by Ovalix because file content anonymization isn't possible via LiteLLM"
+)
+_UNDECODABLE_FILE_BLOCK_REASON: Final = (
+    "This message was blocked by Ovalix because an attached file couldn't be decoded for inspection"
+)
+_TOOL_BLOCK_ESCALATION_REASON: Final = (
+    "This message was blocked by Ovalix because tool call anonymization isn't possible via LiteLLM"
+)
+_TOOL_RESULT_BLOCK_ESCALATION_REASON: Final = (
+    "This message was blocked by Ovalix because tool result anonymization isn't possible via LiteLLM"
+)
+
+
+def _encode_file_wire_format(raw: bytes) -> str:
+    return base64.b64encode(gzip.compress(raw)).decode()
+
+
+def _mapping_or_empty(value: object) -> Mapping[str, object]:
+    return as_str_mapping(value) or _NO_METADATA
 
 
 class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object):
     """Base-class constructor options this guardrail forwards untouched to CustomGuardrail."""
 
-    supported_event_hooks: ReadOnly[list[GuardrailEventHooks]]
-
 
 class OvalixGuardrailMissingSecrets(Exception):
     """Raised when required Ovalix config (API base, key, application/checkpoint IDs) is missing."""
@@ -87,6 +127,8 @@ class OvalixGuardrail(CustomGuardrail):
         application_id: str | None = None,
         pre_checkpoint_id: str | None = None,
         post_checkpoint_id: str | None = None,
+        file_checkpoint_id: str | None = None,
+        supported_event_hooks: list[GuardrailEventHooks] | None = None,
         **kwargs: Unpack[_CustomGuardrailOptions],
     ):
         self._tracker_api_base = tracker_api_base or os.environ.get("OVALIX_TRACKER_API_BASE")
@@ -94,23 +136,26 @@ class OvalixGuardrail(CustomGuardrail):
         self._application_id = application_id or os.environ.get("OVALIX_APPLICATION_ID")
         self._pre_checkpoint_id = pre_checkpoint_id or os.environ.get("OVALIX_PRE_CHECKPOINT_ID")
         self._post_checkpoint_id = post_checkpoint_id or os.environ.get("OVALIX_POST_CHECKPOINT_ID")
+        self._file_checkpoint_id = file_checkpoint_id or os.environ.get("OVALIX_FILE_CHECKPOINT_ID")
 
-        supported_event_hooks: Final = kwargs.get("supported_event_hooks", [])
+        event_hooks: Final = self._validated_event_hooks(supported_event_hooks or ())
 
-        self._validate_config(supported_event_hooks)
-
-        self._tracker_headers = httpx.Headers(
-            {
-                "Authorization": f"Bearer {self._tracker_api_key}",
-                "Content-Type": "application/json",
-            },
-            encoding="utf-8",
+        self._tracker_headers = dict(
+            httpx.Headers(
+                MappingProxyType(
+                    {
+                        "Authorization": f"Bearer {self._tracker_api_key}",
+                        "x-api-key": self._tracker_api_key or "",
+                        "Content-Type": "application/json",
+                    }
+                ),
+                encoding="utf-8",
+            )
         )
 
         self._async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback)
 
-        forwarded: Final[_CustomGuardrailOptions] = {**kwargs, "supported_event_hooks": supported_event_hooks}
-        super().__init__(**forwarded)
+        super().__init__(supported_event_hooks=event_hooks, **kwargs)
         verbose_proxy_logger.debug(
             "Ovalix Guardrail initialized: tracker=%s, application_id=%s, pre_checkpoint_id=%s, post_checkpoint_id=%s",
             self._tracker_api_base,
@@ -119,44 +164,53 @@ class OvalixGuardrail(CustomGuardrail):
             self._post_checkpoint_id,
         )
 
-    def _validate_config(self, supported_event_hooks: list[GuardrailEventHooks]) -> None:
-        """Ensure required secrets and checkpoint IDs are set; auto-add hooks when IDs are present."""
-        errors: Final[list[str]] = []
-
-        if not self._tracker_api_base:
-            errors.append("Tracker API base, set OVALIX_TRACKER_API_BASE or pass tracker_api_base")
-        if not self._tracker_api_key:
-            errors.append("Tracker API key, set OVALIX_TRACKER_API_KEY or pass tracker_api_key")
-        if not self._application_id:
-            errors.append("Application ID, set OVALIX_APPLICATION_ID or pass application_id")
-        if not self._pre_checkpoint_id and GuardrailEventHooks.pre_call in supported_event_hooks:
-            errors.append("Pre-checkpoint ID, set OVALIX_PRE_CHECKPOINT_ID or pass pre_checkpoint_id")
-        if not self._post_checkpoint_id and GuardrailEventHooks.post_call in supported_event_hooks:
-            errors.append("Post-checkpoint ID, set OVALIX_POST_CHECKPOINT_ID or pass post_checkpoint_id")
-        if not self._pre_checkpoint_id and not self._post_checkpoint_id:
-            errors.append(
-                "Pre-checkpoint ID or Post-checkpoint ID, set OVALIX_PRE_CHECKPOINT_ID or OVALIX_POST_CHECKPOINT_ID or pass pre_checkpoint_id or post_checkpoint_id"
+    def _validated_event_hooks(self, requested: Sequence[GuardrailEventHooks]) -> list[GuardrailEventHooks]:
+        """Ensure required secrets and checkpoint IDs are set; return the requested hooks plus one per configured checkpoint."""
+        errors: Final = tuple(
+            message
+            for missing, message in (
+                (not self._tracker_api_base, "Tracker API base, set OVALIX_TRACKER_API_BASE or pass tracker_api_base"),
+                (not self._tracker_api_key, "Tracker API key, set OVALIX_TRACKER_API_KEY or pass tracker_api_key"),
+                (not self._application_id, "Application ID, set OVALIX_APPLICATION_ID or pass application_id"),
+                (
+                    not self._pre_checkpoint_id and GuardrailEventHooks.pre_call in requested,
+                    "Pre-checkpoint ID, set OVALIX_PRE_CHECKPOINT_ID or pass pre_checkpoint_id",
+                ),
+                (
+                    not self._post_checkpoint_id and GuardrailEventHooks.post_call in requested,
+                    "Post-checkpoint ID, set OVALIX_POST_CHECKPOINT_ID or pass post_checkpoint_id",
+                ),
+                (
+                    not self._pre_checkpoint_id and not self._post_checkpoint_id,
+                    "Pre-checkpoint ID or Post-checkpoint ID, set OVALIX_PRE_CHECKPOINT_ID or OVALIX_POST_CHECKPOINT_ID or pass pre_checkpoint_id or post_checkpoint_id",
+                ),
             )
+            if missing
+        )
 
         if errors:
             raise OvalixGuardrailMissingSecrets("Missing Ovalix guardrail configuration errors: " + ". ".join(errors))
 
-        # auto-add hooks when checkpoint IDs are present
-        if self._pre_checkpoint_id and GuardrailEventHooks.pre_call not in supported_event_hooks:
-            supported_event_hooks.append(GuardrailEventHooks.pre_call)
-        if self._post_checkpoint_id and GuardrailEventHooks.post_call not in supported_event_hooks:
-            supported_event_hooks.append(GuardrailEventHooks.post_call)
+        auto_added: Final = tuple(
+            hook
+            for configured, hook in (
+                (self._pre_checkpoint_id, GuardrailEventHooks.pre_call),
+                (self._post_checkpoint_id, GuardrailEventHooks.post_call),
+            )
+            if configured and hook not in requested
+        )
+        return [*requested, *auto_added]
 
-    def _get_actor(self, data: dict) -> str:
+    def _get_actor(self, data: Mapping[str, object]) -> str:
         """Return a stable actor identifier from request metadata (e.g. user email or id)."""
-        metadata: Final = data.get("metadata") or data.get("litellm_metadata") or {}
-        if metadata.get("user_api_key_user_email"):
-            return metadata["user_api_key_user_email"]
-        if metadata.get("user_api_key_user_id"):
-            return metadata["user_api_key_user_id"]
-        return "unknown"
+        metadata: Final = _mapping_or_empty(data.get("metadata") or data.get("litellm_metadata"))
+        return (
+            str_or_none(metadata.get("user_api_key_user_email"))
+            or str_or_none(metadata.get("user_api_key_user_id"))
+            or "unknown"
+        )
 
-    def _get_tracker_actor_id(self, data: dict) -> str:
+    def _get_tracker_actor_id(self, data: Mapping[str, object]) -> str:
         """Normalize the actor string into a short, stable id for Tracker API payloads."""
         # NOTE: this hash is purely for normalization — it collapses an arbitrary actor
         # string (email, user id, or "unknown") into a compact, fixed-length, consistent
@@ -166,7 +220,7 @@ class OvalixGuardrail(CustomGuardrail):
         normalized_actor_id: Final = hashlib.sha256(actor_id).hexdigest()[:8]
         return normalized_actor_id
 
-    def _get_session_id(self, data: dict) -> str:
+    def _get_session_id(self, data: Mapping[str, object]) -> str:
         """Return a unique identifier for the chat/session (actor + date + application_id)."""
         actor_hash: Final = self._get_tracker_actor_id(data)
         today: Final = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d")
@@ -174,106 +228,254 @@ class OvalixGuardrail(CustomGuardrail):
 
     async def _call_checkpoint(
         self,
-        content: str,
+        data_type: str,
+        data: Mapping[str, object],
         checkpoint_id: str,
         actor: str,
         session_id: str,
-    ) -> dict[str, Any]:
-        """Call the Ovalix Tracker checkpoint API and return the JSON response."""
+        oversize: bool = False,
+    ) -> Mapping[str, object]:
+        """Call the Ovalix Tracker checkpoint API and return the JSON response.
+
+        Both routes live on the tracker's /beta litellm router, which accepts the api key this
+        guardrail already sends.
+        """
         application_id: Final = self._application_id
         if not application_id or not checkpoint_id:
             raise ValueError("Ovalix: application_id or checkpoint_id not resolved")
 
-        url: Final = f"{self._tracker_api_base}/tracking/custom_application/checkpoint"
-        headers: Final = dict(self._tracker_headers)
+        route: Final = "file_checkpoint" if data_type == "FILE" else "checkpoint"
         payload: Final = {
             "application_id": application_id,
             "checkpoint_id": checkpoint_id,
             "actor": actor,
             "session_id": session_id,
-            "data_type": "TEXT",
-            "data": {"content": content},
+            "data_type": data_type,
+            "data": data,
+            "tool": "LiteLLM",
+            **(MappingProxyType({"oversize": True}) if oversize else _NO_METADATA),
         }
-        response: Final = await self._async_handler.post(url, headers=headers, json=payload, timeout=self.timeout)
+        response: Final = await self._async_handler.post(
+            f"{self._tracker_api_base}/tracking/beta/{route}",
+            headers=self._tracker_headers,
+            json=payload,
+            timeout=self.timeout,
+        )
         response.raise_for_status()
         return response.json()
 
+    def _verdict(self, resp: Mapping[str, object]) -> tuple[str, str | None]:
+        return (str_or_none(resp.get("action_type")) or "").lower(), self._get_trackers_corrected_message(resp)
+
+    async def _block_reason_for_item(
+        self,
+        data_type: str,
+        data: Mapping[str, object],
+        checkpoint_id: str,
+        actor: str,
+        session_id: str,
+        escalation_reason: str,
+        oversize: bool = False,
+    ) -> str | None:
+        try:
+            resp: Final = await self._call_checkpoint(
+                data_type, data, checkpoint_id, actor, session_id, oversize=oversize
+            )
+        except Exception as e:
+            verbose_proxy_logger.exception("Ovalix checkpoint call failed: %s", e)
+            raise GuardrailRaisedException(
+                guardrail_name=self.guardrail_name,
+                message=f"Ovalix guardrail error: {e}",
+                should_wrap_with_default_message=False,
+            ) from e
+        action, corrected = self._verdict(resp)
+        if action == BLOCKED_ACTION_TYPE:
+            return corrected or BLOCKED_BY_OVALIX_FALLBACK_MESSAGE
+        if action in _MODIFY_ACTION_TYPES:
+            return escalation_reason
+        return None
+
+    async def _check_items_block_only(
+        self,
+        items: Sequence[tuple[str, Mapping[str, object]]],
+        checkpoint_id: str,
+        actor: str,
+        session_id: str,
+        escalation_reason: str,
+    ) -> str | None:
+        for data_type, data in items:
+            reason = await self._block_reason_for_item(
+                data_type, data, checkpoint_id, actor, session_id, escalation_reason
+            )
+            if reason is not None:
+                return reason
+        return None
+
+    async def _check_files_for_block(
+        self,
+        file_parts: Sequence[FilePart],
+        checkpoint_id: str,
+        actor: str,
+        session_id: str,
+    ) -> str | None:
+        if any(part.undecodable for part in file_parts):
+            return _UNDECODABLE_FILE_BLOCK_REASON
+        for part in sorted(file_parts, key=lambda p: p.message_index, reverse=True):
+            data = await self._file_part_to_data(part)
+            reason = await self._block_reason_for_item(
+                "FILE",
+                data,
+                checkpoint_id,
+                actor,
+                session_id,
+                _FILE_BLOCK_ESCALATION_REASON,
+                oversize=part.oversize,
+            )
+            if reason is not None:
+                return reason
+        return None
+
     @log_guardrail_information
     async def apply_guardrail(
         self,
         inputs: GenericGuardrailAPIInputs,
-        request_data: dict,
+        request_data: Mapping[str, object],
         input_type: Literal["request", "response"],
         logging_obj: "LiteLLMLoggingObj | None" = None,
     ) -> GenericGuardrailAPIInputs:
-        """
-        Apply Ovalix guardrail to the given inputs (request or response text).
-
-        Used by the unified guardrail flow and the /apply_guardrail API.
-        For "request", uses the pre-checkpoint; for "response", uses the post-checkpoint.
-
-        Args:
-            inputs: Guardrail API inputs (e.g. texts to check).
-            request_data: Full request payload (messages, metadata, response).
-            input_type: "request" (pre_call) or "response" (post_call).
-            logging_obj: Optional logging context.
-
-        Returns:
-            Updated inputs (e.g. with replaced/corrected texts, or unchanged).
-        """
-        if not self._pre_checkpoint_id and not self._post_checkpoint_id:
-            return inputs
-
-        tracker_actor_id: Final = self._get_tracker_actor_id(request_data)
+        actor: Final = self._get_actor(request_data)
         session_id: Final = self._get_session_id(request_data)
-        texts: Final = inputs.get("texts") or []
-        if not texts or not isinstance(texts, list):
+        is_response: Final = input_type == "response"
+
+        prompt_checkpoint: Final = self._post_checkpoint_id if is_response else self._pre_checkpoint_id
+        file_checkpoint: Final = self._file_checkpoint_id or prompt_checkpoint
+        if not file_checkpoint:
+            verbose_proxy_logger.debug(
+                "Ovalix guardrail: application %s has no %s checkpoint, leaving this direction uninspected",
+                self._application_id,
+                input_type,
+            )
             return inputs
 
-        if input_type == "response":
-            if not self._post_checkpoint_id:
-                return inputs
-            corrected_llm_responses: Final = await self._generate_post_guardrail_llm_texts(
-                texts, tracker_actor_id, session_id, self._post_checkpoint_id
+        structured_messages: Final = inputs.get("structured_messages") or ()
+        file_parts: Final = (
+            extract_file_parts_from_images(inputs.get("images"), size_limit=_DEFAULT_FILE_SIZE_LIMIT)
+            if is_response
+            else extract_file_parts_from_messages(structured_messages, size_limit=_DEFAULT_FILE_SIZE_LIMIT)
+        )
+        file_block: Final = await self._check_files_for_block(file_parts, file_checkpoint, actor, session_id)
+        if file_block is not None:
+            self._block_current_message(file_block)
+
+        if not prompt_checkpoint:
+            verbose_proxy_logger.debug(
+                "Ovalix guardrail: application %s has only a %s file checkpoint, skipping text and tool inspection",
+                self._application_id,
+                input_type,
             )
-            return {**inputs, "texts": corrected_llm_responses}
+            return inputs
 
-        if self._pre_checkpoint_id:
-            post_guardrail_texts: Final = await self._generate_post_guardrail_llm_texts(
-                texts, tracker_actor_id, session_id, self._pre_checkpoint_id
-            )
-            return {**inputs, "texts": post_guardrail_texts}
-        return inputs
+        tool_calls: Final = (
+            *(inputs.get("tool_calls") or ()),
+            *extract_tool_calls_from_messages(structured_messages),
+        )
+        unique_tool_data: Final = MappingProxyType(
+            {tool_data_key(data): data for data in (tool_call_to_tool_data(tc) for tc in tool_calls) if data}
+        )
+        tool_call_items: Final = tuple(("TOOL", data) for data in unique_tool_data.values())
+        tool_block: Final = await self._check_items_block_only(
+            tool_call_items,
+            prompt_checkpoint,
+            actor,
+            session_id,
+            _TOOL_BLOCK_ESCALATION_REASON,
+        )
+        if tool_block is not None:
+            self._block_current_message(tool_block)
 
-    async def _generate_post_guardrail_llm_texts(
-        self, texts: list[str], actor: str, session_id: str, checkpoint_id: str
-    ) -> list[str]:
-        """Generate post-guardrail LLM responses for the given LLM responses."""
-        post_guardrail_texts: Final[list[str]] = []
+        tool_results: Final = extract_tool_results(structured_messages)
+        tool_result_items: Final = tuple(("TOOL", make_tool_data(name, content)) for name, content, _ in tool_results)
+        tool_result_block: Final = await self._check_items_block_only(
+            tool_result_items,
+            prompt_checkpoint,
+            actor,
+            session_id,
+            _TOOL_RESULT_BLOCK_ESCALATION_REASON,
+        )
+        if tool_result_block is not None:
+            self._block_current_message(tool_result_block)
 
-        is_first_response = True
-        for llm_response in reversed(texts):
-            try:
-                resp = await self._call_checkpoint(llm_response, checkpoint_id, actor, session_id)
-            except Exception as e:
-                verbose_proxy_logger.exception("Ovalix apply_guardrail checkpoint call failed: %s", e)
-                raise GuardrailRaisedException(
-                    guardrail_name=self.guardrail_name,
-                    message=f"Ovalix guardrail error: {e}",
-                    should_wrap_with_default_message=False,
-                ) from e
+        texts: Final = inputs.get("texts") or ()
+        if not texts:
+            return inputs
+        output_texts: Final = await self._check_texts(
+            texts,
+            prompt_checkpoint,
+            actor,
+            session_id,
+            tool_result_text_indices(structured_messages, texts),
+        )
+        if output_texts is None:
+            return inputs
+        return {**inputs, "texts": output_texts}
 
-            action_type = (resp.get("action_type") or "").lower()
-            blocking_message = self._get_trackers_corrected_message(resp) or BLOCKED_BY_OVALIX_FALLBACK_MESSAGE
-            if action_type == BLOCKED_ACTION_TYPE and is_first_response:
-                self._block_current_message(blocking_message)
-            elif action_type == BLOCKED_ACTION_TYPE:
-                post_guardrail_texts.insert(0, blocking_message)
-            else:
-                corrected_text = self._get_trackers_corrected_message(resp) or llm_response
-                post_guardrail_texts.insert(0, corrected_text)
-            is_first_response = False
-        return post_guardrail_texts
+    async def _file_part_to_data(self, part: FilePart) -> Mapping[str, object]:
+        extension: Final = mimetypes.guess_extension(part.mime_hint) if part.mime_hint else None
+        name: Final = part.name or (f"file{extension}" if extension else "file")
+        content: Final = (
+            await asyncio.get_event_loop().run_in_executor(None, _encode_file_wire_format, part.data)
+            if part.data
+            else None
+        )
+        return {"name": name, "content": content}
+
+    async def _check_texts(
+        self,
+        texts: Sequence[str],
+        checkpoint_id: str,
+        actor: str,
+        session_id: str,
+        skip_indices: frozenset[int],
+    ) -> list[str] | None:
+        newest_first: Final = tuple(index for index in reversed(range(len(texts))) if index not in skip_indices)
+        checked: Final = MappingProxyType(
+            {
+                index: await self._check_text(
+                    texts[index], checkpoint_id, actor, session_id, is_newest=index == len(texts) - 1
+                )
+                for index in newest_first
+            }
+        )
+        output: Final = [checked.get(index, text) for index, text in enumerate(texts)]
+        return output if output != list(texts) else None
+
+    async def _check_text(
+        self,
+        content: str,
+        checkpoint_id: str,
+        actor: str,
+        session_id: str,
+        is_newest: bool,
+    ) -> str:
+        """Return the text that replaces `content` after Ovalix checks it; a blocked newest text raises."""
+        try:
+            resp: Final = await self._call_checkpoint("TEXT", {"content": content}, checkpoint_id, actor, session_id)
+        except Exception as e:
+            verbose_proxy_logger.exception("Ovalix checkpoint call failed: %s", e)
+            raise GuardrailRaisedException(
+                guardrail_name=self.guardrail_name,
+                message=f"Ovalix guardrail error: {e}",
+                should_wrap_with_default_message=False,
+            ) from e
+        action, corrected = self._verdict(resp)
+        if action == BLOCKED_ACTION_TYPE:
+            block_message: Final = corrected or BLOCKED_BY_OVALIX_FALLBACK_MESSAGE
+            if is_newest:
+                self._block_current_message(block_message)
+            return block_message
+        if action in _MODIFY_ACTION_TYPES and corrected is not None:
+            return corrected
+        return content
 
     def _block_current_message(self, blocking_message: str) -> None:
         """Raise OvalixGuardrailBlockedException with the given message (no default wrapper)."""
@@ -283,12 +485,13 @@ class OvalixGuardrail(CustomGuardrail):
             should_wrap_with_default_message=False,
         )
 
-    def _get_trackers_corrected_message(self, resp: dict) -> str | None:
+    def _get_trackers_corrected_message(self, resp: Mapping[str, object]) -> str | None:
         """Extract corrected/blocking message content from Tracker checkpoint response."""
-        modified: Final = resp.get("modified_data")
-        if isinstance(modified, dict) and "content" in modified:
-            return modified["content"]
-        return None
+        modified: Final = as_str_mapping(resp.get("modified_data"))
+        if modified is None:
+            return None
+        content: Final = modified.get("content")
+        return content if isinstance(content, str) else None
 
     @staticmethod
     def get_config_model() -> type["GuardrailConfigModel"] | None:

litellm/proxy/guardrails/guardrail_hooks/ovalix/ovalix_extraction.py

@@ -0,0 +1,396 @@
+import base64
+import json
+import posixpath
+import re
+from bisect import bisect_left
+from collections.abc import Callable, Iterator, Mapping, Sequence
+from itertools import chain, groupby
+from operator import attrgetter
+from types import MappingProxyType
+from typing import Final, NamedTuple
+from urllib.parse import unquote, urlparse
+
+from pydantic import TypeAdapter, ValidationError
+
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
+
+_TOOL_NAME_MAX_LENGTH: Final = 100
+_DEFAULT_TOOL_RESULT_NAME: Final = "tool_result"
+_NO_TOOL_INPUT: Final[Mapping[str, object]] = MappingProxyType({})
+
+_DATA_URL_RE: Final = re.compile(r"^data:(?P<mime>[^;,]+)?(?P<params>(?:;[^;,]+)*?)(?P<b64>;base64)?,", re.IGNORECASE)
+_URLSAFE_TO_STANDARD_B64: Final = str.maketrans("-_", "+/")
+_OBJECT_LIST: Final = TypeAdapter(list[object])
+
+
+def str_or_none(value: object) -> str | None:
+    return value if isinstance(value, str) and value else None
+
+
+def _list_items(value: object) -> Sequence[object]:
+    try:
+        return _OBJECT_LIST.validate_python(value, strict=True)
+    except ValidationError:
+        return ()
+
+
+class FilePart(NamedTuple):
+    name: str | None
+    data: bytes | None
+    mime_hint: str | None
+    inline: bool
+    oversize: bool
+    message_index: int
+    undecodable: bool = False
+
+
+def _split_data_url(value: str) -> tuple[str | None, str | None]:
+    match: Final = _DATA_URL_RE.match(value)
+    if not match:
+        if value.lower().startswith("data:"):
+            return None, None
+        return None, value
+    mime: Final = match.group("mime") or None
+    if not match.group("b64"):
+        return mime, None
+    return mime, value[match.end() :]
+
+
+def _b64decode_or_none(payload: str) -> bytes | None:
+    try:
+        return base64.b64decode(payload, validate=True)
+    except ValueError:
+        return None
+
+
+def _decoded_base64(cleaned: str) -> bytes | None:
+    strict: Final = _b64decode_or_none(cleaned)
+    if strict is not None or ("-" not in cleaned and "_" not in cleaned):
+        return strict
+    return _b64decode_or_none(cleaned.translate(_URLSAFE_TO_STANDARD_B64))
+
+
+def _inline_part(name: str | None, value: str, size_limit: int | None, message_index: int) -> FilePart | None:
+    mime_hint, payload = _split_data_url(value)
+    if payload is None:
+        return FilePart(name, None, mime_hint, True, False, message_index, undecodable=True)
+    cleaned: Final = "".join(payload.split())
+    if not cleaned:
+        return None
+    if size_limit is not None and (len(cleaned) * 3) // 4 - 2 > size_limit:
+        return FilePart(name, None, mime_hint, True, True, message_index)
+    data: Final = _decoded_base64(cleaned)
+    if data is None:
+        return FilePart(name, None, mime_hint, True, False, message_index, undecodable=True)
+    if size_limit is not None and len(data) > size_limit:
+        return FilePart(name, None, mime_hint, True, True, message_index)
+    return FilePart(name, data, mime_hint, True, False, message_index)
+
+
+def _name_from_url(url: str) -> str | None:
+    try:
+        return unquote(posixpath.basename(urlparse(url).path)) or None
+    except ValueError:
+        return None
+
+
+def _part_from_file_block(block: Mapping[str, object], size_limit: int | None, message_index: int) -> FilePart | None:
+    file_obj: Final = as_str_mapping(block.get("file"))
+    if file_obj is None:
+        return None
+    name: Final = str_or_none(file_obj.get("filename")) or str_or_none(file_obj.get("file_id"))
+    file_data: Final = str_or_none(file_obj.get("file_data"))
+    inline: Final = _inline_part(name, file_data, size_limit, message_index) if file_data else None
+    return inline or FilePart(name, None, None, False, False, message_index)
+
+
+def _part_from_image_url_block(
+    block: Mapping[str, object], size_limit: int | None, message_index: int
+) -> FilePart | None:
+    image_url: Final = block.get("image_url")
+    image_url_fields: Final = as_str_mapping(image_url)
+    url: Final = image_url_fields.get("url") if image_url_fields is not None else image_url
+    if not isinstance(url, str) or not url:
+        return None
+    if url.startswith("data:"):
+        return _inline_part(None, url, size_limit, message_index)
+    return FilePart(_name_from_url(url), None, None, False, False, message_index)
+
+
+def _part_from_input_file_block(
+    block: Mapping[str, object], size_limit: int | None, message_index: int
+) -> FilePart | None:
+    declared_name: Final = str_or_none(block.get("filename")) or str_or_none(block.get("file_id"))
+    file_data: Final = str_or_none(block.get("file_data"))
+    inline: Final = _inline_part(declared_name, file_data, size_limit, message_index) if file_data else None
+    if inline is not None:
+        return inline
+    file_url: Final = block.get("file_url")
+    name: Final = (
+        _name_from_url(file_url) if isinstance(file_url, str) and file_url and not declared_name else declared_name
+    )
+    return FilePart(name, None, None, False, False, message_index)
+
+
+def _part_from_input_audio_block(
+    block: Mapping[str, object], size_limit: int | None, message_index: int
+) -> FilePart | None:
+    audio: Final = as_str_mapping(block.get("input_audio"))
+    if audio is None:
+        return None
+    data_b64: Final = audio.get("data")
+    if not isinstance(data_b64, str) or not data_b64:
+        return None
+    name: Final = f"audio.{audio.get('format') or 'bin'}"
+    return _inline_part(name, data_b64, size_limit, message_index) or FilePart(
+        name, None, None, False, False, message_index
+    )
+
+
+_BLOCK_PARSERS: Final[Mapping[str, Callable[[Mapping[str, object], int | None, int], FilePart | None]]] = (
+    MappingProxyType(
+        {
+            "file": _part_from_file_block,
+            "image_url": _part_from_image_url_block,
+            "input_image": _part_from_image_url_block,
+            "input_file": _part_from_input_file_block,
+            "input_audio": _part_from_input_audio_block,
+        }
+    )
+)
+
+
+def _file_parts_of_message(
+    message: Mapping[str, object], size_limit: int | None, message_index: int
+) -> Iterator[FilePart]:
+    for block in map(as_str_mapping, _list_items(message.get("content"))):
+        if block is None:
+            continue
+        block_type = block.get("type")
+        if not isinstance(block_type, str):
+            continue
+        parser = _BLOCK_PARSERS.get(block_type)
+        if parser is None:
+            continue
+        try:
+            part = parser(block, size_limit, message_index)
+        except (TypeError, ValueError, AttributeError, KeyError):
+            continue
+        if part is not None and (part.inline or part.name):
+            yield part
+
+
+def _file_parts_of_messages(structured_messages: Sequence[object], size_limit: int | None) -> Iterator[FilePart]:
+    for message_index, message in enumerate(map(as_str_mapping, structured_messages)):
+        if message is not None:
+            yield from _file_parts_of_message(message, size_limit, message_index)
+
+
+def extract_file_parts_from_messages(
+    structured_messages: Sequence[object] | None, size_limit: int | None = None
+) -> tuple[FilePart, ...]:
+    return tuple(_file_parts_of_messages(structured_messages or (), size_limit))
+
+
+def _file_part_of_image(value: str, size_limit: int | None, index: int) -> FilePart | None:
+    if value.startswith(("http://", "https://")):
+        name: Final = _name_from_url(value)
+        return FilePart(name, None, None, False, False, index) if name else None
+    return _inline_part(None, value, size_limit, index)
+
+
+def extract_file_parts_from_images(
+    images: Sequence[object] | None, size_limit: int | None = None
+) -> tuple[FilePart, ...]:
+    candidates: Final = (
+        _file_part_of_image(value, size_limit, index)
+        for index, value in enumerate(images or ())
+        if isinstance(value, str) and value
+    )
+    return tuple(part for part in candidates if part is not None)
+
+
+def make_tool_data(
+    name: str, content: str | None, tool_input: Mapping[str, object] | None = None
+) -> Mapping[str, object]:
+    action_name: Final = str(name) if str(name).strip() else _DEFAULT_TOOL_RESULT_NAME
+    truncated: Final = action_name[:_TOOL_NAME_MAX_LENGTH]
+    tool_name: Final = truncated if truncated.strip() else _DEFAULT_TOOL_RESULT_NAME
+    return {
+        "content": content,
+        "tool_name": tool_name,
+        "action_name": action_name,
+        "tool_input": dict(tool_input or ()),
+    }
+
+
+def _tool_call_field(tool_call: object, key: str) -> object:
+    fields: Final = as_str_mapping(tool_call)
+    return fields.get(key) if fields is not None else getattr(tool_call, key, None)
+
+
+def _json_or_str(value: object) -> str:
+    try:
+        return json.dumps(value)
+    except (TypeError, ValueError):
+        return str(value)
+
+
+def _parsed_tool_input(raw_arguments: str) -> Mapping[str, object]:
+    if not raw_arguments:
+        return _NO_TOOL_INPUT
+    try:
+        parsed: Final = json.loads(raw_arguments)
+    except (ValueError, TypeError):
+        return _NO_TOOL_INPUT
+    return as_str_mapping(parsed) or _NO_TOOL_INPUT
+
+
+def _tool_content_and_input(raw_arguments: object) -> tuple[str, Mapping[str, object]]:
+    if isinstance(raw_arguments, str):
+        return raw_arguments, _parsed_tool_input(raw_arguments)
+    if raw_arguments is None:
+        return "", _NO_TOOL_INPUT
+    arguments: Final = as_str_mapping(raw_arguments)
+    return _json_or_str(raw_arguments), arguments if arguments is not None else _NO_TOOL_INPUT
+
+
+def tool_call_to_tool_data(tool_call: object) -> Mapping[str, object] | None:
+    function: Final = _tool_call_field(tool_call, "function")
+    name: Final = _tool_call_field(function, "name")
+    if not name or not str(name).strip():
+        return None
+    content, tool_input = _tool_content_and_input(_tool_call_field(function, "arguments"))
+    return make_tool_data(str(name), content, tool_input)
+
+
+def _message_tool_calls(message: object) -> Sequence[object]:
+    fields: Final = as_str_mapping(message)
+    return _list_items(fields.get("tool_calls")) if fields is not None else ()
+
+
+def extract_tool_calls_from_messages(structured_messages: Sequence[object] | None) -> tuple[object, ...]:
+    """Tool calls declared on the messages themselves.
+
+    Surfaces such as the Anthropic request path populate ``structured_messages`` but leave the
+    top-level ``tool_calls`` input empty, so calls made in prior assistant turns are only visible here.
+    """
+    return tuple(chain.from_iterable(_message_tool_calls(message) for message in structured_messages or ()))
+
+
+def tool_data_key(tool_data: Mapping[str, object]) -> str:
+    """Stable identity for a tool payload, so a call reached from two sources is only scanned once."""
+    return json.dumps(tool_data, sort_keys=True, default=str)
+
+
+def _tool_content_block_text(block: object) -> str | None:
+    if isinstance(block, str):
+        return block
+    fields: Final = as_str_mapping(block)
+    return str_or_none(fields.get("text")) if fields is not None else None
+
+
+def _tool_content_text(content: object) -> str:
+    if isinstance(content, str):
+        return content
+    fields: Final = as_str_mapping(content)
+    if fields is not None:
+        return _json_or_str(fields)
+    texts: Final = (_tool_content_block_text(block) for block in _list_items(content))
+    return "\n".join(text for text in texts if text is not None)
+
+
+def _extract_tool_content(content: object) -> str | None:
+    text: Final = _tool_content_text(content)
+    return text if text.strip() else None
+
+
+class _Declaration(NamedTuple):
+    call_id: str
+    message_index: int
+    name: str
+
+
+def _declarations_in(message_index: int, message: object) -> Iterator[_Declaration]:
+    fields: Final = as_str_mapping(message)
+    if fields is None or fields.get("role") != "assistant":
+        return
+    for tool_call in map(as_str_mapping, _list_items(fields.get("tool_calls"))):
+        if tool_call is None:
+            continue
+        call_id = tool_call.get("id")
+        function = as_str_mapping(tool_call.get("function"))
+        name = str_or_none(function.get("name")) if function is not None else None
+        if isinstance(call_id, str) and call_id and name and name.strip():
+            yield _Declaration(call_id, message_index, name)
+
+
+def _declarations_by_call_id(messages: Sequence[object]) -> Mapping[str, tuple[_Declaration, ...]]:
+    """Every named declaration of each tool call id, in message order, built in one pass."""
+    declarations: Final = chain.from_iterable(
+        _declarations_in(index, message) for index, message in enumerate(messages)
+    )
+    ordered: Final = sorted(declarations, key=attrgetter("call_id", "message_index"))
+    return MappingProxyType({call_id: tuple(group) for call_id, group in groupby(ordered, key=attrgetter("call_id"))})
+
+
+def _resolve_tool_name(
+    declarations: Mapping[str, tuple[_Declaration, ...]], tool_index: int, tool_call_id: object
+) -> str:
+    """Name from the nearest declaration of ``tool_call_id`` before ``tool_index``."""
+    if not isinstance(tool_call_id, str) or not tool_call_id:
+        return _DEFAULT_TOOL_RESULT_NAME
+    declared: Final = declarations.get(tool_call_id, ())
+    earlier: Final = bisect_left(declared, tool_index, key=attrgetter("message_index"))
+    return declared[earlier - 1].name if earlier else _DEFAULT_TOOL_RESULT_NAME
+
+
+def extract_tool_results(
+    structured_messages: Sequence[object] | None,
+) -> tuple[tuple[str, str, object], ...]:
+    messages: Final = tuple(structured_messages or ())
+    declarations: Final = _declarations_by_call_id(messages)
+
+    def _results() -> Iterator[tuple[str, str, object]]:
+        for index, message in enumerate(map(as_str_mapping, messages)):
+            if message is None or message.get("role") != "tool":
+                continue
+            content = _extract_tool_content(message.get("content"))
+            if content is None:
+                continue
+            tool_call_id = message.get("tool_call_id")
+            yield _resolve_tool_name(declarations, index, tool_call_id), content, tool_call_id
+
+    return tuple(_results())
+
+
+def _block_texts(content: object) -> Iterator[object]:
+    blocks: Final = (as_str_mapping(block) for block in _list_items(content))
+    return (block.get("text") for block in blocks if block is not None and block.get("text") is not None)
+
+
+def _message_text_origins(structured_messages: Sequence[object] | None) -> Iterator[tuple[object, bool]]:
+    for message in map(as_str_mapping, structured_messages or ()):
+        if message is None:
+            continue
+        content = message.get("content")
+        from_tool_result = message.get("role") == "tool" and _extract_tool_content(content) is not None
+        if isinstance(content, str):
+            yield content, from_tool_result
+        else:
+            yield from ((text, from_tool_result) for text in _block_texts(content))
+
+
+def tool_result_text_indices(structured_messages: Sequence[object] | None, texts: Sequence[str]) -> frozenset[int]:
+    """Positions in ``texts`` that hold content already submitted under the TOOL policy.
+
+    The chat-completions guardrail flow builds ``texts`` and ``structured_messages`` from the
+    same message list, so tool-role content lands in both and would otherwise be checked twice.
+    Other surfaces (e.g. Anthropic messages) build ``texts`` from a differently shaped payload,
+    so the mapping is only trusted when replaying it reproduces ``texts`` exactly; anything else
+    falls back to checking every text.
+    """
+    origins: Final = tuple(_message_text_origins(structured_messages))
+    if tuple(text for text, _ in origins) != tuple(texts):
+        return frozenset()
+    return frozenset(index for index, (_, from_tool_result) in enumerate(origins) if from_tool_result)

litellm/proxy/guardrails/guardrail_initializers.py

@@ -18,6 +18,7 @@ def initialize_bedrock(litellm_params: LitellmParams, guardrail: Guardrail):
         event_hook=litellm_params.mode,
         guardrailIdentifier=litellm_params.guardrailIdentifier,
         guardrailVersion=litellm_params.guardrailVersion,
+        outputScope=litellm_params.outputScope,
         checks=litellm_params.checks if isinstance(litellm_params.checks, BedrockChecksConfigModel) else None,
         content_filter_threshold=litellm_params.content_filter_threshold,
         prompt_attack_threshold=litellm_params.prompt_attack_threshold,

litellm/proxy/guardrails/response_handlers.py

@@ -0,0 +1,117 @@
+"""Response policies for the standalone guardrail endpoint."""
+
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final, Literal, TypeAlias
+
+from fastapi import HTTPException
+from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
+
+from litellm.integrations.custom_guardrail import CustomGuardrail
+from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
+from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.types.guardrails import (
+    ApplyGuardrailRequest,
+    ApplyGuardrailResponse,
+    GuardrailEventHooks,
+    GuardrailResponseFormat,
+    NativeGuardrailResponse,
+)
+from litellm.types.utils import GenericGuardrailAPIInputs
+
+GuardrailEndpointResponse: TypeAlias = ApplyGuardrailResponse | NativeGuardrailResponse
+_NATIVE_RESPONSE: Final[TypeAdapter[NativeGuardrailResponse]] = TypeAdapter(NativeGuardrailResponse)
+
+
+class StandardGuardrailResponseHandler:
+    """Preserve the endpoint's existing text response and hook behavior."""
+
+    def prepare_request_data(self, request: ApplyGuardrailRequest) -> Mapping[str, object]:
+        return request.model_dump(include={"messages", "metadata"}, exclude_none=True)
+
+    def resolve_input_type(
+        self, request: ApplyGuardrailRequest, guardrail: CustomGuardrail
+    ) -> Literal["request", "response"]:
+        if request.input_type == "request" and guardrail.event_hook in (GuardrailEventHooks.post_call, "post_call"):
+            return "response"
+        return "response" if request.input_type == "response" else "request"
+
+    def build_response(
+        self,
+        request: ApplyGuardrailRequest,
+        processed_inputs: GenericGuardrailAPIInputs,
+        request_data: Mapping[str, object],
+    ) -> GuardrailEndpointResponse:
+        texts: Final = processed_inputs.get("texts", [])
+        return ApplyGuardrailResponse(response_text=texts[0] if texts else request.text)
+
+    def accept_hook_result(self, original: GuardrailEndpointResponse, modified: object) -> GuardrailEndpointResponse:
+        return modified if isinstance(modified, ApplyGuardrailResponse) else original
+
+
+class _NativeGuardrailRecord(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    guardrail_name: str | None = None
+    guardrail_response: JsonValue = None
+
+
+class _NativeGuardrailMetadata(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    standard_logging_guardrail_information: tuple[_NativeGuardrailRecord, ...] = ()
+
+
+class NativeGuardrailResponseHandler(StandardGuardrailResponseHandler):
+    """Return structured findings recorded by this guardrail invocation."""
+
+    def prepare_request_data(self, request: ApplyGuardrailRequest) -> Mapping[str, object]:
+        return request.model_dump(
+            include={"messages", "metadata"},
+            exclude={"metadata": {"standard_logging_guardrail_information"}},
+            exclude_none=True,
+        )
+
+    def resolve_input_type(
+        self, request: ApplyGuardrailRequest, guardrail: CustomGuardrail
+    ) -> Literal["request", "response"]:
+        if "input_type" in request.model_fields_set:
+            return "response" if request.input_type == "response" else "request"
+        return super().resolve_input_type(request, guardrail)
+
+    def build_response(
+        self,
+        request: ApplyGuardrailRequest,
+        processed_inputs: GenericGuardrailAPIInputs,
+        request_data: Mapping[str, object],
+    ) -> GuardrailEndpointResponse:
+        metadata: Final = request_data.get(get_metadata_variable_name_from_kwargs(request_data))
+        recorded: Final = _NativeGuardrailMetadata.model_validate_json(
+            safe_dumps(metadata if metadata is not None else {})
+        )
+        findings: Final = tuple(
+            record.guardrail_response
+            for record in recorded.standard_logging_guardrail_information
+            if record.guardrail_name == request.guardrail_name and isinstance(record.guardrail_response, (dict, list))
+        )
+        if not findings:
+            raise HTTPException(
+                status_code=422,
+                detail=f"Guardrail '{request.guardrail_name}' did not report structured native findings.",
+            )
+        return findings[0] if len(findings) == 1 else findings
+
+    def accept_hook_result(self, original: GuardrailEndpointResponse, modified: object) -> GuardrailEndpointResponse:
+        if modified is None:
+            return original
+        return _NATIVE_RESPONSE.validate_python(modified)
+
+
+GUARDRAIL_RESPONSE_HANDLERS: Final[Mapping[GuardrailResponseFormat, StandardGuardrailResponseHandler]] = (
+    MappingProxyType(
+        {
+            GuardrailResponseFormat.STANDARD: StandardGuardrailResponseHandler(),
+            GuardrailResponseFormat.NATIVE: NativeGuardrailResponseHandler(),
+        }
+    )
+)

litellm/proxy/hooks/dynamic_rate_limiter.py

@@ -118,7 +118,7 @@ class _PROXY_DynamicRateLimitHandler(CustomLogger):
             elif priority is not None and litellm.priority_reservation is not None:
                 if os.getenv("LITELLM_LICENSE", None) is None:
                     verbose_proxy_logger.error(
-                        "PREMIUM FEATURE: Reserving tpm/rpm by priority is a premium feature. Please add a 'LITELLM_LICENSE' to your .env to enable this.\nGet a license: https://docs.litellm.ai/docs/proxy/enterprise."
+                        "PREMIUM FEATURE: Reserving tpm/rpm by priority is a premium feature. Please add a 'LITELLM_LICENSE' to your .env to enable this.\nGet a license: https://docs.litellm.ai/docs/enterprise."
                     )
                 else:
                     value: Final = litellm.priority_reservation[priority]

litellm/proxy/hooks/dynamic_rate_limiter_v3.py

@@ -133,7 +133,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
         elif priority is not None and litellm.priority_reservation is not None:
             if os.getenv("LITELLM_LICENSE", None) is None:
                 verbose_proxy_logger.error(
-                    "PREMIUM FEATURE: Reserving tpm/rpm by priority is a premium feature. Please add a 'LITELLM_LICENSE' to your .env to enable this.\nGet a license: https://docs.litellm.ai/docs/proxy/enterprise."
+                    "PREMIUM FEATURE: Reserving tpm/rpm by priority is a premium feature. Please add a 'LITELLM_LICENSE' to your .env to enable this.\nGet a license: https://docs.litellm.ai/docs/enterprise."
                 )
             else:
                 value: Final = litellm.priority_reservation[priority]

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -2405,7 +2405,7 @@ async def generate_key_fn(
     - soft_budget: Optional[float] - Specify soft budget for a given key. Will trigger a slack alert when this soft budget is reached.
     - tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing).
     - prompts: Optional[List[str]] - List of prompts that the key is allowed to use.
-    - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/proxy/enterprise#enforce-required-params-for-llm-requests)
+    - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/enterprise#governance-and-cost)
     - prompts: Optional[List[str]] - List of prompts that the key is allowed to use.
     - allowed_routes: Optional[list] - List of allowed routes for the key. Store the actual route or store a wildcard pattern for a set of routes. Example - ["/chat/completions", "/embeddings", "/keys/*"]
     - allowed_passthrough_routes: Optional[list] - List of allowed pass through endpoints for the key. Store the actual endpoint or store a wildcard pattern for a set of endpoints. Example - ["/my-custom-endpoint"]. Use this instead of allowed_routes, if you just want to specify which pass through endpoints the key can access, without specifying the routes. If allowed_routes is specified, allowed_pass_through_endpoints is ignored.
@@ -2613,7 +2613,7 @@ async def generate_service_account_key_fn(
     - tpd_limit: Optional[int] - Specify tpd limit for a given key (Tokens per day). Charged by batch submissions instead of tpm_limit/rpm_limit.
     - soft_budget: Optional[float] - Specify soft budget for a given key. Will trigger a slack alert when this soft budget is reached.
     - tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing).
-    - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/proxy/enterprise#enforce-required-params-for-llm-requests)
+    - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/enterprise#governance-and-cost)
     - allowed_routes: Optional[list] - List of allowed routes for the key. Store the actual route or store a wildcard pattern for a set of routes. Example - ["/chat/completions", "/embeddings", "/keys/*"]
     - object_permission: Optional[LiteLLM_ObjectPermissionBase] - key-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"], "agents": ["agent_1", "agent_2"], "agent_access_groups": ["dev_group"]}. IF null or {} then no object permission.
     Examples:
@@ -4045,7 +4045,7 @@ async def update_key_fn(
     - models: Optional[list] - Model_name's a user is allowed to call
     - tags: Optional[List[str]] - Tags for organizing keys (Enterprise only)
     - prompts: Optional[List[str]] - List of prompts that the key is allowed to use.
-    - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/proxy/enterprise#enforce-required-params-for-llm-requests)
+    - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/enterprise#governance-and-cost)
     - spend: Optional[float] - Amount spent by key
     - max_budget: Optional[float] - Max budget for key
     - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets keyed by model name, `*` pattern like "claude-opus*", or model access group, e.g. {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}

litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py

@@ -3614,7 +3614,9 @@ async def deepgram_listen_websocket_route(
     if not deepgram_listen_is_priced(target):
         await websocket.close(
             code=1008,
-            reason=_DEEPGRAM_WS_UNPRICED_REASON.format(registry_key=deepgram_listen_registry_key(target)),
+            reason=truncated_close_reason(
+                _DEEPGRAM_WS_UNPRICED_REASON.format(registry_key=deepgram_listen_registry_key(target))
+            ),
         )
         return
 

litellm/proxy/proxy_cli.py

@@ -1193,7 +1193,7 @@ class ProxyInitializationHelpers:
     "--reload",
     is_flag=True,
     default=False,
-    help="Enable uvicorn hot reload (dev only). Also reloads when the --config YAML file changes. Incompatible with --num_workers>1, --run_gunicorn, and --run_hypercorn.",
+    help="Enable uvicorn hot reload (dev only), including every --num_workers worker on uvicorn>=0.50.0 outside Windows. Also reloads when the --config YAML or .env changes. Incompatible with --run_gunicorn, --run_hypercorn, and --run_granian.",
 )
 @click.option(
     "--prometheus_metrics_port",
@@ -1785,9 +1785,28 @@ def run_server(
             if num_workers > 1:
                 start_query_engine_reaper()
             if num_workers > 1 or reload:
-                uvicorn_args.update(
-                    ProxyInitializationHelpers.child_process_uvicorn_overrides(uvicorn_args.get("log_config"))
+                worker_overrides: Final = ProxyInitializationHelpers.child_process_uvicorn_overrides(
+                    uvicorn_args.get("log_config")
                 )
+                if reload and num_workers > 1:
+                    from litellm.proxy.dev_reload import (
+                        RELOAD_OPTIONS,
+                        multi_worker_reload_supported,
+                        run_multi_worker_reload,
+                    )
+
+                    if multi_worker_reload_supported():
+                        run_multi_worker_reload(
+                            RELOAD_OPTIONS.validate_python(uvicorn_args),
+                            workers=TypeAdapter(int).validate_python(num_workers),
+                            worker_overrides=worker_overrides,
+                        )
+                        return
+                    print(
+                        "\033[1;33mLiteLLM Proxy: --reload with --num_workers>1 requires uvicorn>=0.50.0 "
+                        "and an OS other than Windows, so only a single worker reloads.\033[0m"
+                    )
+                uvicorn_args.update(worker_overrides)
             uvicorn.run(
                 **uvicorn_args,
                 workers=num_workers,

litellm/proxy/proxy_server.py

@@ -392,6 +392,7 @@ from litellm.llms.anthropic.messages.convertible_blocks import (
     drop_non_string_tool_result_texts,
     malformed_tool_result_count_error,
 )
+from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import build_pool_wait_trace_config
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.openai.live import parse_live_session_start
 from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS
@@ -1209,7 +1210,7 @@ ui_message += "\n\n💸 [```LiteLLM Model Cost Map```](https://models.litellm.ai
 ui_message += f"\n\n🔎 [```LiteLLM Model Hub```]({model_hub_link}). See available models on the proxy. [**Docs**](https://docs.litellm.ai/docs/proxy/ai_hub)"
 
 custom_swagger_message: Final = (
-    "[**Customize Swagger Docs**](https://docs.litellm.ai/docs/proxy/enterprise#swagger-docs---custom-routes--branding)"
+    "[**Customize Swagger Docs**](https://docs.litellm.ai/docs/enterprise#operations--branding)"
 )
 
 ### CUSTOM BRANDING [ENTERPRISE FEATURE] ###
@@ -1460,7 +1461,11 @@ async def initialize_shared_aiohttp_session() -> "ClientSession | None":
             connector_kwargs["socket_factory"] = socket_factory
 
         connector: Final = TCPConnector(**connector_kwargs)
-        session: Final = ClientSession(connector=connector, cookie_jar=DummyCookieJar())
+        session: Final = ClientSession(
+            connector=connector,
+            cookie_jar=DummyCookieJar(),
+            trace_configs=[build_pool_wait_trace_config()],
+        )
 
         verbose_proxy_logger.info(
             "SESSION REUSE: Created shared aiohttp session for connection pooling (ID: %s, limit=%s, limit_per_host=%s)",
@@ -3070,6 +3075,29 @@ redis_usage_cache: RedisCache | None = None  # redis cache used for tracking spe
 polling_via_cache_enabled: Literal["all"] | list[str] | bool = False
 native_background_mode: list[str] = []  # Models that should use native provider background mode instead of polling
 polling_cache_ttl: int = 3600  # Default 1 hour TTL for polling cache
+
+
+class _ResponseBackgroundSettings(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    polling_via_cache: Literal["all"] | tuple[str, ...] | bool = False
+    native_background_mode: tuple[str, ...] = ()
+    ttl: int = 3600
+
+
+def _apply_background_response_settings(value: SettingsJsonValue) -> None:
+    global polling_via_cache_enabled, native_background_mode, polling_cache_ttl
+
+    responses: Final = _as_settings_mapping(value)
+    settings: Final = _ResponseBackgroundSettings.model_validate(responses.get("background_mode") or {})
+    polling: Final = settings.polling_via_cache
+    polling_via_cache_enabled, native_background_mode, polling_cache_ttl = (  # rebind-ok: Refresh runtime config.
+        list(polling) if isinstance(polling, tuple) else polling,
+        list(settings.native_background_mode),
+        settings.ttl,
+    )
+
+
 user_custom_auth = None
 user_custom_key_generate = None
 # Sentinel: prevents PKCE-no-Redis advisory from re-logging on config hot-reload.
@@ -6932,20 +6960,7 @@ class ProxyConfig:
                     # see usage here: https://docs.litellm.ai/docs/proxy/caching
                     pass
                 elif key == "responses":
-                    # Initialize global polling via cache settings
-                    global polling_via_cache_enabled, native_background_mode, polling_cache_ttl
-                    background_mode = value.get("background_mode", {})
-                    polling_via_cache_enabled = background_mode.get("polling_via_cache", False)
-                    native_background_mode = background_mode.get("native_background_mode", [])
-                    polling_cache_ttl = background_mode.get("ttl", 3600)
-                    verbose_proxy_logger.debug(
-                        "%s Initialized polling via cache: enabled=%s, native_background_mode=%s, ttl=%s%s",
-                        blue_color_code,
-                        polling_via_cache_enabled,
-                        native_background_mode,
-                        polling_cache_ttl,
-                        reset_color_code,
-                    )
+                    _apply_background_response_settings(value)
                 elif key == "max_ui_session_budget":
                     litellm.max_ui_session_budget = float(value) if value is not None else None
                     verbose_proxy_logger.debug(
@@ -8653,6 +8668,7 @@ class ProxyConfig:
 
     def _apply_litellm_settings_db_values(self, db_values: Mapping[str, SettingsJsonValue]) -> None:
         self.litellm_settings.apply_db_row("litellm_settings", db_values)
+        _apply_background_response_settings(self.litellm_settings.get("responses", {}))
         for key in LITELLM_SETTINGS_SAFE_DB_OVERRIDES:
             if key in db_values and (value := self.litellm_settings.get(key)) is not None:
                 setattr(litellm, key, value)
@@ -18110,6 +18126,8 @@ async def model_deprecations(
 def get_model_group_info(
     llm_router: Router, all_models_str: Sequence[str], model_group: str | None, team_id: str | None = None
 ) -> list[ModelGroupInfoProxy]:
+    from litellm.proxy.response_polling.polling_handler import should_use_polling_for_request
+
     model_groups: Final[list[ModelGroupInfoProxy]] = []
 
     unique_models: Final = tuple(dict.fromkeys(all_models_str))
@@ -18119,6 +18137,14 @@ def get_model_group_info(
         if model_group is not None and model_group != model:
             continue
 
+        supports_background_chat = should_use_polling_for_request(
+            background_mode=True,
+            polling_via_cache_enabled=polling_via_cache_enabled,
+            redis_cache=redis_usage_cache,
+            model=model,
+            llm_router=llm_router,
+            native_background_mode=native_background_mode,
+        )
         _model_group_info = registry.group(model, team_id=team_id)
 
         if _model_group_info is not None:
@@ -18126,6 +18152,7 @@ def get_model_group_info(
                 ModelGroupInfoProxy.model_validate(
                     {
                         **_model_group_info.model_dump(),
+                        "supports_background_chat": supports_background_chat,
                         **model_group_custom_fields(
                             registry.deployments(
                                 resolve_model_group_alias(registry.aliases(), model) or model,
@@ -18140,6 +18167,7 @@ def get_model_group_info(
             model_group_info = ModelGroupInfoProxy(
                 model_group=model,
                 providers=[],
+                supports_background_chat=supports_background_chat,
             )
             model_groups.append(model_group_info)
 

litellm/proxy/response_polling/background_streaming.py

@@ -11,6 +11,7 @@ https://platform.openai.com/docs/api-reference/responses-streaming
 import asyncio
 import json
 from collections.abc import AsyncIterable, Callable, Mapping, Sequence
+from time import perf_counter
 from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, runtime_checkable
 
 from fastapi import Request, Response
@@ -128,6 +129,16 @@ async def background_streaming_task(
     https://platform.openai.com/docs/api-reference/responses/object
     """
 
+    started_at: Final = perf_counter()
+    first_token_at: float | None = None  # rebind-ok: first content event closes the TTFT interval
+    finished_at: float | None = None  # rebind-ok: terminal event closes the execution interval
+
+    def timing_snapshot() -> Mapping[str, float]:
+        return {
+            **({"time_to_first_token_ms": (first_token_at - started_at) * 1000} if first_token_at is not None else {}),
+            **({"total_latency_ms": (finished_at - started_at) * 1000} if finished_at is not None else {}),
+        }
+
     try:
         verbose_proxy_logger.info("Starting background streaming for %s", polling_id)
 
@@ -191,7 +202,7 @@ async def background_streaming_task(
         incomplete_details_data = None
 
         state_dirty = False  # Track if state needs to be synced
-        last_update_time = asyncio.get_event_loop().time()
+        last_update_time = perf_counter()
         UPDATE_INTERVAL: Final = 0.150  # 150ms batching interval
 
         # Track the terminal event from the stream (may not be "completed")
@@ -210,13 +221,14 @@ async def background_streaming_task(
             """Flush accumulated state to Redis if interval elapsed or forced"""
             nonlocal state_dirty, last_update_time
 
-            current_time: Final = asyncio.get_event_loop().time()
+            current_time: Final = perf_counter()
             if state_dirty and (force or (current_time - last_update_time) >= UPDATE_INTERVAL):
                 # Convert output_items dict to list for update
                 output_list: Final = list(output_items.values())
                 await polling_handler.update_state(
                     polling_id=polling_id,
                     output=output_list,
+                    timing=timing_snapshot(),
                 )
                 state_dirty = False
                 last_update_time = current_time
@@ -242,6 +254,20 @@ async def background_streaming_task(
                         event: _StreamEvent = _StreamEventParser.parse(chunk_data)
                         event_type = event.get("type", "")
 
+                        if (
+                            first_token_at is None
+                            and event_type
+                            in (
+                                "response.output_text.delta",
+                                "response.reasoning_text.delta",
+                                "response.reasoning_summary_text.delta",
+                                "response.reasoning.delta",
+                            )
+                            and event.get("delta")
+                        ):
+                            first_token_at = perf_counter()
+                            state_dirty = True
+
                         # Process different event types based on OpenAI streaming spec
                         if event_type == "response.output_item.added":
                             # New output item added
@@ -334,6 +360,7 @@ async def background_streaming_task(
                         ):
                             # Terminal event - extract all ResponsesAPIResponse fields
                             # https://platform.openai.com/docs/api-reference/responses-streaming
+                            finished_at = perf_counter()
                             response_data = event.get("response", {})
                             terminal_status = response_data.get(
                                 "status",
@@ -384,10 +411,13 @@ async def background_streaming_task(
 
         # Use the terminal status from the stream, default to "completed"
         final_status: Final = terminal_status or "completed"
+        if finished_at is None:
+            finished_at = perf_counter()  # rebind-ok: close the interval when the stream has no terminal event
 
         await polling_handler.update_state(
             polling_id=polling_id,
             status=final_status,
+            timing=timing_snapshot(),
             usage=usage_data,
             error=terminal_error,
             reasoning=reasoning_data,
@@ -417,6 +447,7 @@ async def background_streaming_task(
         )
 
     except Exception as e:
+        finished_at = perf_counter()  # rebind-ok: capture execution time at failure
         verbose_proxy_logger.error("Error in background streaming task for %s: %s", polling_id, e)
         import traceback
 
@@ -425,6 +456,7 @@ async def background_streaming_task(
         await polling_handler.update_state(
             polling_id=polling_id,
             status="failed",
+            timing=timing_snapshot(),
             error={
                 "type": "internal_error",
                 "message": str(e),

litellm/proxy/response_polling/polling_handler.py

@@ -3,9 +3,9 @@ Response Polling Handler for Background Responses with Cache
 """
 
 import json
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from datetime import datetime, timezone
-from typing import Final
+from typing import TYPE_CHECKING, Final
 
 from pydantic import TypeAdapter
 
@@ -15,6 +15,9 @@ from litellm._uuid import uuid4
 from litellm.caching.redis_cache import RedisCache
 from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStatus
 
+if TYPE_CHECKING:
+    from litellm.router import Router
+
 _RESPONSE_POLLING_TARGET: Final = "response_polling"
 _JSON_OBJECT: Final = TypeAdapter(dict[str, object])
 _JSON_LIST: Final = TypeAdapter(list[object])
@@ -116,6 +119,7 @@ class ResponsePollingHandler:
         parallel_tool_calls: bool | None = None,
         user: str | None = None,
         store: bool | None = None,
+        timing: Mapping[str, float] | None = None,
     ) -> None:
         """
         Update the polling state in Redis
@@ -144,6 +148,7 @@ class ResponsePollingHandler:
             parallel_tool_calls: Whether parallel tool calls are enabled
             user: User identifier
             store: Whether to store the response
+            timing: Server execution timings in milliseconds
         """
         if not self.redis_cache:
             return
@@ -167,6 +172,9 @@ class ResponsePollingHandler:
         if output is not None:
             state["output"] = output
 
+        if timing is not None:
+            state["litellm_response_timing"] = timing
+
         # Update usage
         if usage:
             state["usage"] = usage
@@ -264,10 +272,10 @@ class ResponsePollingHandler:
 
 def should_use_polling_for_request(
     background_mode: bool,
-    polling_via_cache_enabled,  # Can be False, "all", or List[str]
-    redis_cache,  # RedisCache or None
+    polling_via_cache_enabled: bool | str | Sequence[str],
+    redis_cache: RedisCache | None,
     model: str,
-    llm_router,  # Router instance or None
+    llm_router: "Router | None",
     native_background_mode: list[str] | None = None,  # List of models that should use native background mode
 ) -> bool:
     """

litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py

@@ -216,6 +216,20 @@ class _SettingsWithSchema(LiteLLMBaseModel):
     field_schema: dict[str, object]
 
 
+class BackgroundChatSettings(LiteLLMBaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    enabled: bool = Field(description="Enable cache-backed background Responses for Chat and API requests.")
+
+
+class BackgroundChatSettingsResponse(BackgroundChatSettings):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    redis_configured: bool = Field(description="Whether this proxy has coordination Redis configured.")
+    managed_by_config: bool = Field(description="Whether the responses configuration is owned by the config file.")
+    can_save: bool = Field(description="Whether database-backed configuration updates are available.")
+
+
 class SSOSettingsResponse(SettingsResponse):
     """Response model for SSO settings"""
 
@@ -1815,6 +1829,87 @@ async def sync_ui_settings_to_general_settings(
     return apply_runtime_general_settings_flags(parsed)
 
 
+def _background_chat_settings_response(config: Mapping[str, object]) -> BackgroundChatSettingsResponse:
+    from litellm.proxy import proxy_server
+
+    settings: Final = _as_settings_section(config.get("litellm_settings"))
+    responses: Final = _as_settings_section(settings.get("responses"))
+    background: Final = _as_settings_section(responses.get("background_mode"))
+    polling: Final = background.get("polling_via_cache", False)
+    return BackgroundChatSettingsResponse(
+        enabled=bool(polling),
+        redis_configured=proxy_server.redis_usage_cache is not None,
+        managed_by_config=proxy_server.proxy_config.litellm_settings.owned_by_config("responses"),
+        can_save=proxy_server.prisma_client is not None and proxy_server.store_model_in_db is True,
+    )
+
+
+@router.get(
+    "/get/background_chat_settings",
+    tags=["Settings"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=BackgroundChatSettingsResponse,
+)
+async def get_background_chat_settings() -> BackgroundChatSettingsResponse:
+    """Get saved background-response settings and the prerequisites for changing them."""
+    from litellm.proxy.proxy_server import proxy_config
+
+    config: Final = _as_settings_section(_DECODED_JSON.validate_python(await proxy_config.get_config()))
+    return _background_chat_settings_response(config)
+
+
+@router.patch(
+    "/update/background_chat_settings",
+    tags=["Settings"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=BackgroundChatSettingsResponse,
+)
+async def update_background_chat_settings(
+    settings: BackgroundChatSettings,
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+) -> BackgroundChatSettingsResponse:
+    """Save background-response polling settings for the proxy configuration refresh."""
+    from litellm.proxy.proxy_server import (
+        create_config_audit_log,
+        prisma_client,
+        proxy_config,
+        redis_usage_cache,
+        store_model_in_db,
+    )
+
+    if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
+        raise HTTPException(status_code=403, detail="Only proxy admins can update background chat settings.")
+    if prisma_client is None or store_model_in_db is not True:
+        raise HTTPException(status_code=400, detail="Enable database-backed configuration to save this setting.")
+    if settings.enabled and redis_usage_cache is None:
+        raise HTTPException(status_code=400, detail="Configure coordination Redis and restart the proxy first.")
+
+    raw_config: Final = await proxy_config.get_config()
+    config: Final = _as_settings_section(_DECODED_JSON.validate_python(raw_config))
+    litellm_settings: Final = _as_settings_section(config.get("litellm_settings"))
+    responses: Final = _UI_SETTINGS_OBJECT.validate_python(_as_settings_section(litellm_settings.get("responses")))
+    background: Final = _UI_SETTINGS_OBJECT.validate_python(_as_settings_section(responses.get("background_mode")))
+    updated_responses: Final[Mapping[str, JsonValue]] = {
+        **responses,
+        "background_mode": {**background, "polling_via_cache": "all" if settings.enabled else False},
+    }
+    proxy_config.reject_config_owned_writes(
+        section_name="litellm_settings", changed_keys={"responses": dict(updated_responses)}
+    )
+    raw_config["litellm_settings"] = {**litellm_settings, "responses": dict(updated_responses)}
+    await proxy_config.save_config(new_config=config)
+    asyncio.create_task(
+        create_config_audit_log(
+            param_name="responses.background_mode.polling_via_cache",
+            action="updated",
+            before_value=background.get("polling_via_cache", False),
+            after_value="all" if settings.enabled else False,
+            user_api_key_dict=user_api_key_dict,
+        )
+    )
+    return _background_chat_settings_response(config)
+
+
 @router.get(
     "/get/ui_settings",
     tags=["UI Settings"],

litellm/router_strategy/auto_router/litellm_encoder.py

@@ -14,13 +14,13 @@ else:
     RoutingContext = Any
 
 
-def litellm_to_list(embeds: litellm.EmbeddingResponse) -> list[list[float]]:
+def litellm_to_list(embeds: object) -> list[list[float]]:
     """Convert a LiteLLM embedding response to a list of embeddings.
 
     :param embeds: The LiteLLM embedding response.
     :return: A list of embeddings.
     """
-    if not embeds or not embeds.data:
+    if not embeds or not isinstance(embeds, litellm.EmbeddingResponse) or not embeds.data:
         raise ValueError("No embeddings found in LiteLLM embedding response.")
     return [x["embedding"] for x in embeds.data]
 

litellm/router_strategy/complexity_router/jev_classifier.py

@@ -68,6 +68,7 @@ class JevBudgetedRequest:
     telemetry: StandardLoggingRoutingDecision
 
 
+@renamed_router_param("router", "context")
 def jev_input_targets(context: "RoutingContext", config: "OpenSourceClassifierConfig") -> tuple[JevInputTarget, ...]:
     from litellm.types.router import Deployment
 

litellm/router_strategy/tag_based_routing.py

@@ -392,9 +392,11 @@ def _resolve_constraint_only_pool(
 
 def _all_deployments_or_fallback(
     ctx: "RoutingContext",
-    model: str,
+    model: str | None,
     fallback: _DeploymentPool,
 ) -> Sequence[_DeploymentLike | DeploymentTypedDict] | Mapping[_DeploymentLike, object]:
+    if model is None:  # a group of no name has no deployments, it is not a failed lookup
+        return ()
     try:
         return ctx.registry().deployments(model, source="all")
     except Exception:  # noqa: BLE001  # fail safe toward today's healthy-only behavior on lookup errors

litellm/router_utils/handle_error.py

@@ -23,7 +23,7 @@ else:
 
 @renamed_router_param("litellm_router_instance")
 async def send_llm_exception_alert(
-    ctx: "RoutingContext",
+    ctx: "RoutingContext | None",
     request_kwargs: dict,
     error_traceback_str: str,
     original_exception,
@@ -39,6 +39,9 @@ async def send_llm_exception_alert(
     Returns:
         None
     """
+    if ctx is None:
+        return
+
     slack_alerting_logger: Final = ctx.observers().slack_alerting_logger
     if slack_alerting_logger is None:
         return

litellm/types/guardrails.py

@@ -1,10 +1,10 @@
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from datetime import datetime
 from enum import Enum
 from types import MappingProxyType
 from typing import Annotated, Final, Literal, TypeAlias, cast
 
-from pydantic import ConfigDict, Field, field_validator, model_validator
+from pydantic import ConfigDict, Field, JsonValue, field_validator, model_validator
 from pydantic.fields import FieldInfo
 from typing_extensions import ReadOnly, Required, TypedDict
 
@@ -604,6 +604,11 @@ class BedrockChecksConfigModel(LiteLLMBaseModel):
         return self
 
 
+class BedrockGuardrailOutputScope(Enum):
+    INTERVENTIONS = "INTERVENTIONS"
+    FULL = "FULL"
+
+
 class BedrockGuardrailConfigModel(LiteLLMBaseModel):
     """Configuration parameters for the AWS Bedrock guardrail"""
 
@@ -612,6 +617,11 @@ class BedrockGuardrailConfigModel(LiteLLMBaseModel):
         default=None,
         description="The version of your Bedrock guardrail (e.g., DRAFT or version number)",
     )
+    outputScope: BedrockGuardrailOutputScope | None = Field(
+        default=None,
+        description="ApplyGuardrail response scope. FULL includes detected and non-detected findings. "
+        "Omit to retain Bedrock's default of returning detected findings only.",
+    )
     disable_exception_on_block: bool | None = Field(
         default=False,
         description="If True, will not raise an exception when the guardrail is blocked. Useful for OpenWebUI where exceptions can end the chat flow.",
@@ -1540,6 +1550,14 @@ class PresidioPerRequestConfig(LiteLLMBaseModel):
     entities: list[PiiEntityType] | None = None
 
 
+NativeGuardrailResponse: TypeAlias = Mapping[str, JsonValue] | Sequence[JsonValue]
+
+
+class GuardrailResponseFormat(Enum):
+    STANDARD = "standard"
+    NATIVE = "native"
+
+
 class ApplyGuardrailRequest(LiteLLMBaseModel):
     guardrail_name: str
     text: str
@@ -1548,6 +1566,7 @@ class ApplyGuardrailRequest(LiteLLMBaseModel):
     input_type: str = "request"
     messages: list[dict[str, object]] | None = None
     metadata: dict[str, object] | None = None
+    response_format: GuardrailResponseFormat = GuardrailResponseFormat.STANDARD
 
 
 class ApplyGuardrailResponse(LiteLLMBaseModel):

litellm/types/proxy/guardrails/guardrail_hooks/ovalix.py

@@ -28,6 +28,10 @@ class OvalixGuardrailConfigModel(GuardrailConfigModel):
         default=None,
         description="Post-checkpoint ID for the Ovalix Tracker service.",
     )
+    file_checkpoint_id: str | None = Field(
+        default=None,
+        description="File-checkpoint ID for the Ovalix Tracker service (falls back to pre/post).",
+    )
 
     @staticmethod
     def ui_friendly_name() -> str:

litellm/types/proxy/management_endpoints/model_management_endpoints.py

@@ -35,6 +35,9 @@ class PatchModelRequest(updateDeployment):
 
 
 class ModelGroupInfoProxy(ModelGroupInfo):
+    supports_background_chat: bool = Field(
+        default=False, description="Whether cache-backed background Responses runs are enabled for this model group"
+    )
     is_public_model_group: bool = Field(default=False)
     health_status: str | None = Field(default=None)
     health_response_time: float | None = Field(default=None)

litellm/types/utils.py

@@ -1151,6 +1151,8 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = {
     # A2A (Agent-to-Agent)
     "/a2a/{agent_id}": [CallTypes.asend_message, CallTypes.send_message],
     "/a2a/{agent_id}/message/send": [CallTypes.asend_message, CallTypes.send_message],
+    "/a2a/{agent_id}/message/stream": [CallTypes.asend_message, CallTypes.send_message],
+    "/v1/a2a/{agent_id}/message/stream": [CallTypes.asend_message, CallTypes.send_message],
     # Passthrough endpoints
     "/llm_passthrough": [
         CallTypes.allm_passthrough_route,

model_prices_and_context_window.json

@@ -15555,18 +15555,20 @@
         "input_cost_per_token": 2.65e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8000,
-        "max_output_tokens": 2048,
+        "max_output_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 3.5e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md",
         "supports_pdf_input": true
     },
     "bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": {
         "input_cost_per_token": 3e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8000,
-        "max_output_tokens": 2048,
+        "max_output_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 6e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md",
         "supports_pdf_input": true
     },
     "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": {
@@ -48955,6 +48957,7 @@
     },
     "together_ai/Qwen/Qwen3.8-2.4T-A95B": {
         "cache_read_input_token_cost": 2.5e-07,
+        "deprecation_date": "2026-11-04",
         "input_cost_per_token": 2e-06,
         "litellm_provider": "together_ai",
         "max_input_tokens": 1010000,

tests/base_sdk_tests/check_sdk_http.py

@@ -70,7 +70,9 @@ def enqueue(body: object, status: int = 200) -> None:
 
 def enqueue_stream() -> None:
     chunk: Final = {**CHAT, "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": {"content": "pong"}}]}
-    RESPONSES.put((200, f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n".encode()))
+    terminal: Final = {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}
+    stream: Final = f"data: {json.dumps(chunk)}\n\ndata: {json.dumps(terminal)}\n\ndata: [DONE]\n\n"
+    RESPONSES.put((200, stream.encode()))
 
 
 def bedrock_event(event: str, payload: object) -> bytes:

tests/e2e/CONTRIBUTING.md

@@ -90,7 +90,7 @@ The suites run against a live proxy, so bring one up first by running the litell
 
    They also need a proxy whose bundled UI contains the change under test, so run the proxy from your branch (an editable install serves the UI your checkout builds)
 
-Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy. The presidio guardrail tests need a running Presidio analyzer and anonymizer the proxy can reach, addressed by `PRESIDIO_ANALYZER_API_BASE` / `PRESIDIO_ANONYMIZER_API_BASE`
+Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and start the proxy with `LITELLM_OTEL_V2=true`, the `otel` callback, and `OTEL_EXPORTER_OTLP_ENDPOINT` pointing at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy. The presidio guardrail tests need a running Presidio analyzer and anonymizer the proxy can reach, addressed by `PRESIDIO_ANALYZER_API_BASE` / `PRESIDIO_ANONYMIZER_API_BASE`
 
 The Presidio spend-log audit test also requires prompt storage on the proxy, because its assertions inspect the detected entities and their scores. Add the following to the proxy config before starting it:
 

tests/e2e/gateway/stage_mirror_ci_config.yml

@@ -32,7 +32,7 @@ litellm_settings:
       - host: 127.0.0.1
         port: 6379
     ssl: true
-  callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel", "websearch_interception"]
+  callbacks: ["datadog", "smtp_email", "prometheus", "otel", "websearch_interception"]
   websearch_interception_params:
     enabled_providers: ["bedrock"]
     search_tool_name: e2e-search

tests/e2e/logging/test_otel_trace_e2e.py

@@ -54,7 +54,7 @@ def _assert_otel_destination_configured(client: LoggingClient) -> None:
     details = _ReadinessDetails.model_validate_json(readiness_details_body(client))
     assert OTEL_V2_LOGGER_NAME in details.success_callbacks, (
         f"the proxy must report the {OTEL_V2_LOGGER_NAME} callback active "
-        f"(LITELLM_OTEL_V2 + arize_phoenix preset in the compose config); got: {details.success_callbacks}"
+        f"(LITELLM_OTEL_V2=true + the otel callback in the stack config); got: {details.success_callbacks}"
     )
 
 

tests/integration/_support/a2a_denial_probe.py

@@ -0,0 +1,145 @@
+import json
+import os
+from collections.abc import Mapping
+from datetime import datetime
+from typing import Final
+
+import httpx
+from fastapi import HTTPException
+
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.proxy._types import UserAPIKeyAuth
+
+
+def _mapping(value: object) -> Mapping[str, object]:
+    return value if isinstance(value, Mapping) else {}
+
+
+def _logged(hook: str, kwargs: Mapping[str, object]) -> Mapping[str, object]:
+    logged: Final = _mapping(kwargs.get("standard_logging_object"))
+    metadata: Final = _mapping(logged.get("metadata"))
+    return {
+        "hook": hook,
+        "call_type": kwargs.get("call_type"),
+        "litellm_call_id": kwargs.get("litellm_call_id"),
+        "model": logged.get("model"),
+        "status": logged.get("status"),
+        "key_alias": metadata.get("user_api_key_alias"),
+        "key_hash": metadata.get("user_api_key_hash"),
+        "user_id": metadata.get("user_api_key_user_id"),
+        "team_id": metadata.get("user_api_key_team_id"),
+    }
+
+
+def _status(exception: Exception) -> int | None:
+    status: Final = getattr(exception, "status_code", None)
+    if isinstance(status, int):
+        return status
+    code: Final = getattr(exception, "code", None)
+    return int(code) if isinstance(code, str) and code.isdigit() else None
+
+
+def _hooked(
+    request_data: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth, original_exception: Exception
+) -> Mapping[str, object]:
+    return {
+        "hook": "post_call_failure",
+        "call_type": request_data.get("call_type"),
+        "litellm_call_id": request_data.get("litellm_call_id"),
+        "key_alias": user_api_key_dict.key_alias,
+        "key_hash": user_api_key_dict.api_key,
+        "user_id": user_api_key_dict.user_id,
+        "team_id": user_api_key_dict.team_id,
+        "agent_id": request_data.get("agent_id"),
+        "metadata_agent_id": _mapping(request_data.get("metadata")).get("agent_id"),
+        "status_code": _status(original_exception),
+    }
+
+
+def _send(record: Mapping[str, object]) -> None:
+    with httpx.Client(trust_env=False) as client:
+        client.post(os.environ["A2A_DENIAL_PROBE_URL"], content=json.dumps(record, default=str)).raise_for_status()
+
+
+async def _send_async(record: Mapping[str, object]) -> None:
+    async with httpx.AsyncClient(trust_env=False) as client:
+        response: Final = await client.post(os.environ["A2A_DENIAL_PROBE_URL"], content=json.dumps(record, default=str))
+        response.raise_for_status()
+
+
+class A2ADenialProbe(CustomLogger):
+    """Reports the identity, target and outcome every logging and failure hook receives, one record per call."""
+
+    async def async_log_success_event(
+        self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
+    ) -> None:
+        await _send_async(_logged("async_success", kwargs))
+
+    def log_failure_event(
+        self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
+    ) -> None:
+        _send(_logged("sync_failure", kwargs))
+
+    async def async_log_failure_event(
+        self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
+    ) -> None:
+        await _send_async(_logged("async_failure", kwargs))
+
+    async def async_post_call_failure_hook(
+        self,
+        request_data: dict[str, object],
+        original_exception: Exception,
+        user_api_key_dict: UserAPIKeyAuth,
+        traceback_str: str | None = None,
+    ) -> None:
+        await _send_async(_hooked(request_data, user_api_key_dict, original_exception))
+
+
+class RaisingFailureLogger(CustomLogger):
+    """Fails inside both failure hooks."""
+
+    async def async_log_failure_event(
+        self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
+    ) -> None:
+        raise RuntimeError("failure logger is broken")
+
+    async def async_post_call_failure_hook(
+        self,
+        request_data: dict[str, object],
+        original_exception: Exception,
+        user_api_key_dict: UserAPIKeyAuth,
+        traceback_str: str | None = None,
+    ) -> None:
+        raise RuntimeError("failure hook is broken")
+
+
+class ReturningTeapotLogger(CustomLogger):
+    """Hands a replacement 418 back from the failure hook."""
+
+    async def async_post_call_failure_hook(
+        self,
+        request_data: dict[str, object],
+        original_exception: Exception,
+        user_api_key_dict: UserAPIKeyAuth,
+        traceback_str: str | None = None,
+    ) -> HTTPException:
+        return HTTPException(status_code=418, detail="transformed")
+
+
+class RaisingTeapotLogger(CustomLogger):
+    """Raises a replacement 418 from the failure hook."""
+
+    async def async_post_call_failure_hook(
+        self,
+        request_data: dict[str, object],
+        original_exception: Exception,
+        user_api_key_dict: UserAPIKeyAuth,
+        traceback_str: str | None = None,
+    ) -> None:
+        raise HTTPException(status_code=418, detail="transformed")
+
+
+probe: Final = A2ADenialProbe()
+raising_failure_logger: Final = RaisingFailureLogger()
+returning_teapot_logger: Final = ReturningTeapotLogger()
+raising_teapot_logger: Final = RaisingTeapotLogger()

tests/integration/_support/mcp.py

@@ -10,6 +10,7 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Final, Literal
 
+import anyio
 import httpx
 from integration._support.asgi import asgi_server
 from integration._support.client import Gateway, Scenario
@@ -63,7 +64,9 @@ class Confirmation(BaseModel):
     confirmed: bool
 
 
-def math_service(name: str = "integration-math", *, rich: bool = False) -> MCPServer:
+def math_service(
+    name: str = "integration-math", *, rich: bool = False, on_slow: Callable[[], None] | None = None
+) -> MCPServer:
     service: Final = MCPServer(name)
     service.add_tool(add)
     service.add_tool(multiply)
@@ -77,6 +80,8 @@ def math_service(name: str = "integration-math", *, rich: bool = False) -> MCPSe
 
     @service.tool()
     async def slow(seconds: float) -> str:
+        if on_slow is not None:
+            on_slow()
         await asyncio.sleep(seconds)
         return "slept"
 
@@ -114,6 +119,29 @@ def math_service(name: str = "integration-math", *, rich: bool = False) -> MCPSe
     return service
 
 
+def _cancel_on_disconnect(app: ASGIApp) -> ASGIApp:
+    async def connected(scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] != "http":
+            await app(scope, receive, send)
+            return
+        sender, reader = anyio.create_memory_object_stream[Message]()
+        async with sender, reader, anyio.create_task_group() as tasks:
+
+            async def relay() -> None:
+                while True:
+                    message: Final = await receive()
+                    if message["type"] == "http.disconnect":
+                        tasks.cancel_scope.cancel()
+                        return
+                    await sender.send(message)
+
+            tasks.start_soon(relay)
+            await app(scope, reader.receive, send)
+            tasks.cancel_scope.cancel()
+
+    return connected
+
+
 def _capturing(app: Callable[[Scope, Receive, Send], object], observed: queue.Queue[dict[str, object]]):
     async def capture(scope: Scope, receive: Receive, send: Send) -> None:
         if scope["type"] != "http":
@@ -247,8 +275,10 @@ def _draining_sse_watcher(app: Callable[[Scope, Receive, Send], object]):
 
 
 @contextmanager
-def mcp_peer(transport: Literal["http", "sse"] = "http", *, rich: bool = False) -> Iterator[McpPeer]:
-    service: Final = math_service(rich=rich)
+def mcp_peer(
+    transport: Literal["http", "sse"] = "http", *, rich: bool = False, on_slow: Callable[[], None] | None = None
+) -> Iterator[McpPeer]:
+    service: Final = math_service(rich=rich, on_slow=on_slow)
     security: Final = TransportSecuritySettings(enable_dns_rebinding_protection=False)
     app: Final = (
         _draining_sse_watcher(service.sse_app(transport_security=security))
@@ -256,7 +286,11 @@ def mcp_peer(transport: Literal["http", "sse"] = "http", *, rich: bool = False)
         else service.streamable_http_app(stateless_http=True, json_response=True, transport_security=security)
     )
     observed: Final[queue.Queue[dict[str, object]]] = queue.Queue()
-    with asgi_server(_capturing(app, observed), before_stop=_drain_sse_streams if transport == "sse" else None) as url:
+    captured: Final = _capturing(app, observed)
+    with asgi_server(
+        _cancel_on_disconnect(captured) if transport == "http" else captured,
+        before_stop=_drain_sse_streams if transport == "sse" else None,
+    ) as url:
         yield McpPeer(url + ("/sse" if transport == "sse" else "/mcp"), observed, transport)
 
 

tests/integration/authorization/test_agent_access.py

@@ -66,6 +66,13 @@ def _peer_card(url: str, name: str) -> dict[str, JsonValue]:
         "description": "Synthetic peer",
         "version": "1.0.0",
         "url": url + "/",
+        "supportedInterfaces": [
+            {
+                "url": url + "/",
+                "protocolBinding": "JSONRPC",
+                "protocolVersion": "0.3",
+            }
+        ],
         "capabilities": {"streaming": True},
         "defaultInputModes": ["text"],
         "defaultOutputModes": ["text"],
@@ -73,10 +80,15 @@ def _peer_card(url: str, name: str) -> dict[str, JsonValue]:
     }
 
 
-def _register_agent(gateway: Gateway, scenario: Scenario, name: str, url: str) -> str:
-    created: Final = gateway.request(
-        "POST", "/v1/agents", {"agent_name": name, "agent_card_params": _peer_card(url, name)}
-    )
+def _register_agent(
+    gateway: Gateway,
+    scenario: Scenario,
+    name: str,
+    url: str,
+    agent_card: dict[str, JsonValue] | None = None,
+) -> str:
+    card: Final = _peer_card(url, name) if agent_card is None else agent_card
+    created: Final = gateway.request("POST", "/v1/agents", {"agent_name": name, "agent_card_params": card})
     assert created.status_code == 200, created.text
     identity: Final = created.json()["agent_id"]
 
@@ -416,3 +428,68 @@ def test_legacy_agent_json_card_path_applies_agent_permissions_to_non_admin_keys
         )
         assert (denied_legacy.status_code, denied_legacy.text) == (403, refusal(denied)), denied_legacy.text
         assert wire.drain() == ()
+
+
+def test_agent_list_and_detail_cards_use_proxy_urls_for_scoped_keys(gateway: Gateway) -> None:
+    marker: Final = "a2acardurl" + uuid.uuid4().hex[:12]
+
+    def upstream(request: Request) -> Reply:
+        assert request.method == "GET", request.method
+        return Reply(body=json.dumps(_peer_card(wire.url, marker)).encode())
+
+    with wire_server(upstream) as wire, gateway.scenario() as scenario:
+        backend_url: Final = wire.url + "/"
+        peer_card: Final = _peer_card(wire.url, marker)
+        peer_interfaces: Final = peer_card["supportedInterfaces"]
+        assert isinstance(peer_interfaces, list)
+        agent_card: Final = {
+            **peer_card,
+            "additionalInterfaces": [{"url": backend_url, "transport": "GRPC"}],
+            "supportedInterfaces": [
+                *peer_interfaces,
+                {
+                    "url": backend_url,
+                    "protocolBinding": "GRPC",
+                    "protocolVersion": "0.3",
+                },
+            ],
+        }
+        agent_id: Final = _register_agent(gateway, scenario, marker, wire.url, agent_card)
+        user_id: Final = scenario.user(user_role="internal_user")
+        scoped_key: Final = scenario.key(user_id=user_id, object_permission={"agents": [agent_id]})
+
+        public_card: Final = gateway.request(
+            "GET",
+            f"/a2a/{agent_id}/.well-known/agent-card.json",
+            key=scoped_key,
+        )
+        listed: Final = gateway.request("GET", "/v1/agents", key=scoped_key)
+        detailed: Final = gateway.request("GET", f"/v1/agents/{agent_id}", key=scoped_key)
+
+        assert public_card.status_code == 200, public_card.text
+        assert listed.status_code == 200, listed.text
+        assert detailed.status_code == 200, detailed.text
+        assert wire.url not in listed.text
+        assert wire.url not in detailed.text
+        assert "additionalInterfaces" not in listed.text
+        assert "additionalInterfaces" not in detailed.text
+        public_urls: Final = (
+            public_card.json()["url"],
+            public_card.json()["supportedInterfaces"][0]["url"],
+        )
+        listed_agent: Final = next(item for item in listed.json() if item["agent_id"] == agent_id)
+        for body in (listed_agent, detailed.json()):
+            card: Final = body["agent_card_params"]
+            assert (card["url"], card["supportedInterfaces"][0]["url"]) == public_urls
+            assert card["supportedInterfaces"] == [
+                {
+                    "url": public_urls[1],
+                    "protocolBinding": "JSONRPC",
+                    "protocolVersion": "0.3",
+                }
+            ]
+
+        admin_listed: Final = gateway.request("GET", "/v1/agents")
+        assert admin_listed.status_code == 200, admin_listed.text
+        admin_agent: Final = next(item for item in admin_listed.json() if item["agent_id"] == agent_id)
+        assert admin_agent["agent_card_params"]["url"] == backend_url

tests/integration/authorization/test_agent_denial_logging.py

@@ -0,0 +1,809 @@
+import hashlib
+import json
+import uuid
+from collections.abc import Iterator, Mapping
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from types import MappingProxyType
+from typing import Final
+
+import httpx
+import pytest
+import yaml
+from integration._support.client import (
+    JSON_OBJECT,
+    Gateway,
+    Scenario,
+    eventually,
+    gateway_from_environment,
+    held,
+    object_value,
+    string_value,
+)
+from integration._support.process import owned_proxy
+from integration._support.wire import Reply, Request, Wire, wire_server
+from pydantic import BaseModel, ConfigDict, JsonValue
+
+pytestmark = pytest.mark.timeout(900)
+
+_PROBE: Final = "integration._support.a2a_denial_probe."
+_SETTLE: Final = 3.0
+_CONVERGE: Final = 30.0
+_NO_SETTINGS: Final[Mapping[str, JsonValue]] = MappingProxyType({})
+_ROWS: Final = (
+    'SELECT request_id, call_type, status, model, agent_id, spend, api_key, "user", team_id, metadata '
+    'FROM "LiteLLM_SpendLogs" WHERE api_key = %s'
+)
+_ONE_FAILURE: Final = ("async_failure", "post_call_failure")
+_STREAM_METHODS: Final = frozenset({"message/stream", "SendStreamingMessage"})
+
+
+class _Event(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    hook: str
+    call_type: str | None = None
+    litellm_call_id: str | None = None
+    model: str | None = None
+    status: str | None = None
+    key_alias: str | None = None
+    key_hash: str | None = None
+    user_id: str | None = None
+    team_id: str | None = None
+    agent_id: str | None = None
+    metadata_agent_id: str | None = None
+    status_code: int | None = None
+
+
+class _Events:
+    def __init__(self, sink: Wire) -> None:
+        self.sink: Final = sink
+        self.seen: tuple[_Event, ...] = ()
+
+    def of(self, alias: str) -> tuple[_Event, ...]:
+        self.seen = (*self.seen, *(_Event.model_validate_json(request.body) for request in self.sink.drain()))
+        return tuple(event for event in self.seen if event.key_alias == alias)
+
+
+@dataclass(frozen=True, slots=True)
+class _Agent:
+    agent_id: str
+    name: str
+
+
+@dataclass(frozen=True, slots=True)
+class _Rig:
+    proxy: Gateway
+    events: _Events
+    upstream: Wire
+    allowed: _Agent
+    denied: _Agent
+    denied_legacy: _Agent
+    bridge: _Agent
+
+
+@dataclass(frozen=True, slots=True)
+class _Caller:
+    key: str
+    alias: str
+    user_id: str
+    team_id: str | None
+
+    @property
+    def hashed(self) -> str:
+        return hashlib.sha256(self.key.encode()).hexdigest()
+
+
+def _compact(content: Mapping[str, JsonValue]) -> bytes:
+    return json.dumps(content, separators=(",", ":")).encode()
+
+
+def _refusal(agent: str, response: httpx.Response | None = None) -> bytes:
+    message: Final = f"Agent '{agent}' is not allowed for your key/team. Contact proxy admin for access."
+    if response is None:
+        return _compact({"detail": message})
+    return _compact(
+        {
+            "jsonrpc": "2.0",
+            "id": JSON_OBJECT.validate_json(response.request.content)["id"],
+            "error": {
+                "code": -32000,
+                "message": message,
+                "data": {"type": "permission_error", "param": None, "code": "403"},
+            },
+        }
+    )
+
+
+def _not_found(request_id: JsonValue, agent: str) -> bytes:
+    return _compact(
+        {"jsonrpc": "2.0", "id": request_id, "error": {"code": -32000, "message": f"Agent '{agent}' not found"}}
+    )
+
+
+def _card(url: str, name: str, version: str) -> dict[str, JsonValue]:
+    card: Final[dict[str, JsonValue]] = {
+        "protocolVersion": version,
+        "name": name,
+        "description": "Synthetic peer",
+        "version": "1.0.0",
+        "url": url + "/",
+        "capabilities": {"streaming": True},
+        "defaultInputModes": ["text"],
+        "defaultOutputModes": ["text"],
+        "skills": [],
+    }
+    if version == "0.3":
+        return card
+    return {**card, "supportedInterfaces": [{"url": url + "/", "protocolBinding": "JSONRPC", "protocolVersion": "1.0"}]}
+
+
+def _peer(request: Request) -> Reply:
+    if request.method == "GET":
+        return Reply(body=_compact(_card("http://" + request.headers["host"], "peer", "0.3")))
+    body: Final = JSON_OBJECT.validate_json(request.body)
+    if body.get("method") == "message/stream":
+        frame: Final = {
+            "jsonrpc": "2.0",
+            "id": body["id"],
+            "result": {
+                "kind": "status-update",
+                "taskId": "t1",
+                "contextId": "c1",
+                "status": {"state": "completed"},
+                "final": True,
+            },
+        }
+        return Reply(content_type="text/event-stream", chunks=(f"data: {json.dumps(frame)}\n\n".encode(),))
+    return Reply(
+        body=_compact(
+            {
+                "jsonrpc": "2.0",
+                "id": body.get("id"),
+                "result": {
+                    "kind": "message",
+                    "role": "agent",
+                    "messageId": "peer-message",
+                    "parts": [{"kind": "text", "text": "granted"}],
+                },
+            }
+        )
+    )
+
+
+def _register(proxy: Gateway, scenario: Scenario, body: Mapping[str, JsonValue]) -> _Agent:
+    created: Final = proxy.request("POST", "/v1/agents", body)
+    assert created.status_code == 200, created.text
+    agent: Final = _Agent(string_value(JSON_OBJECT.validate_json(created.content)["agent_id"]), str(body["agent_name"]))
+
+    def cleanup() -> None:
+        deleted: Final = proxy.request("DELETE", f"/v1/agents/{agent.agent_id}")
+        assert deleted.status_code == 200, deleted.text
+
+    scenario.cleanups.callback(cleanup)
+    return agent
+
+
+def _config(directory: Path, callbacks: tuple[str, ...], general: Mapping[str, JsonValue]) -> Path:
+    base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
+    config: Final = {
+        **base,
+        "general_settings": {**object_value(base["general_settings"]), **general},
+        "litellm_settings": {
+            **object_value(base["litellm_settings"]),
+            "callbacks": [_PROBE + "probe", *(_PROBE + name for name in callbacks)],
+        },
+    }
+    path: Final = directory / "a2a-denials.yaml"
+    path.write_text(yaml.safe_dump(config))
+    return path
+
+
+@contextmanager
+def _owned_rig(
+    directory: Path,
+    callbacks: tuple[str, ...] = (),
+    general: Mapping[str, JsonValue] = _NO_SETTINGS,
+    environment: Mapping[str, str] = MappingProxyType({}),
+) -> Iterator[_Rig]:
+    marker: Final = "a2adeny" + uuid.uuid4().hex[:10]
+    with (
+        wire_server(_peer) as upstream,
+        wire_server(lambda request: Reply()) as sink,
+        gateway_from_environment() as shared,
+        owned_proxy(
+            shared,
+            directory,
+            {"A2A_DENIAL_PROBE_URL": sink.url, **environment},
+            config=_config(directory, callbacks, general),
+        ) as proxy,
+        proxy.scenario() as scenario,
+    ):
+
+        def agent(suffix: str, version: str, **extra: JsonValue) -> _Agent:
+            name: Final = f"{marker}-{suffix}"
+            return _register(
+                proxy, scenario, {"agent_name": name, "agent_card_params": _card(upstream.url, name, version), **extra}
+            )
+
+        yield _Rig(
+            proxy,
+            _Events(sink),
+            upstream,
+            allowed=agent("allowed", "0.3"),
+            denied=agent("denied", "1.0"),
+            denied_legacy=agent("legacy", "0.3"),
+            bridge=agent(
+                "bridge",
+                "0.3",
+                litellm_params={
+                    "custom_llm_provider": "openai",
+                    "model": "openai/gpt-5.4-mini",
+                    "api_base": upstream.url + "/v1",
+                    "api_key": "synthetic-bridge-key",
+                },
+            ),
+        )
+
+
+@pytest.fixture(scope="module")
+def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
+    with _owned_rig(tmp_path_factory.mktemp("a2a-denials")) as owned:
+        yield owned
+
+
+@pytest.fixture(scope="module", params=["raising_failure_logger,returning_teapot_logger", "raising_teapot_logger"])
+def hostile(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
+    callbacks: Final = tuple(str(request.param).split(","))
+    with _owned_rig(tmp_path_factory.mktemp("a2a-denials-hostile"), callbacks) as owned:
+        yield owned
+
+
+@pytest.fixture(scope="module", params=["disable_pre_routing_error_logs", "disable_error_logs"])
+def flagged(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
+    with _owned_rig(tmp_path_factory.mktemp("a2a-denials-flagged"), general={str(request.param): True}) as owned:
+        yield owned
+
+
+@pytest.fixture(scope="module")
+def alerting(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[_Rig, Wire]]:
+    with wire_server(lambda request: Reply(body=b"ok", content_type="text/plain")) as alerts:
+        with _owned_rig(
+            tmp_path_factory.mktemp("a2a-denials-alerting"),
+            general={"alerting": ["slack"], "alert_types": ["llm_exceptions"]},
+            environment={"SLACK_WEBHOOK_URL": alerts.url, "DEFAULT_FLUSH_INTERVAL_SECONDS": "1"},
+        ) as owned:
+            yield owned, alerts
+
+
+def _caller(rig: _Rig, scenario: Scenario, grant: str = "team") -> _Caller:
+    alias: Final = "a2adeny-" + uuid.uuid4().hex[:12]
+    user: Final = scenario.user()
+    allowed: Final[list[JsonValue]] = [rig.allowed.agent_id]
+    if grant == "personal":
+        return _Caller(
+            scenario.key(user_id=user, key_alias=alias, object_permission={"agents": allowed}), alias, user, None
+        )
+    if grant == "access group":
+        created: Final = rig.proxy.request(
+            "POST", "/v1/access_group", {"access_group_name": alias, "access_agent_ids": allowed}
+        )
+        assert created.status_code == 201, created.text
+        group: Final = string_value(JSON_OBJECT.validate_json(created.content)["access_group_id"])
+        scenario.cleanups.callback(rig.proxy.request, "DELETE", f"/v1/access_group/{group}")
+        return _Caller(scenario.key(user_id=user, key_alias=alias, access_group_ids=[group]), alias, user, None)
+    team: Final = scenario.team(object_permission={"agents": allowed})
+    return _Caller(scenario.key(user_id=user, team_id=team, key_alias=alias), alias, user, team)
+
+
+def _params(method: str, message_id: str, version: str | None) -> dict[str, JsonValue]:
+    if method.startswith("tasks/pushNotificationConfig"):
+        return {"taskId": "t1", "pushNotificationConfig": {"url": "https://callback.example.com/hook"}}
+    if method in {"tasks/get", "tasks/cancel", "GetTask", "CancelTask"}:
+        return {"id": "t1"}
+    if version == "1.0":
+        return {"message": {"messageId": message_id, "role": "ROLE_USER", "parts": [{"text": "x"}]}}
+    return {
+        "message": {
+            "kind": "message",
+            "messageId": message_id,
+            "role": "user",
+            "parts": [{"kind": "text", "text": "x"}],
+        }
+    }
+
+
+def _send(
+    rig: _Rig,
+    key: str,
+    path: str,
+    *,
+    method: str = "SendMessage",
+    version: str | None = "1.0",
+    request_id: JsonValue = None,
+    message_id: str | None = None,
+    extra: Mapping[str, JsonValue] = _NO_SETTINGS,
+) -> httpx.Response:
+    headers: Final = {
+        "Authorization": f"Bearer {key}",
+        **({"A2A-Version": version} if version is not None else {}),
+        **({"Accept": "text/event-stream"} if method in _STREAM_METHODS else {}),
+    }
+    body: Final = {
+        "jsonrpc": "2.0",
+        "id": "r-" + uuid.uuid4().hex if request_id is None else request_id,
+        "method": method,
+        "params": _params(method, message_id or uuid.uuid4().hex, version),
+        **extra,
+    }
+    return rig.proxy.client.post(path, headers=headers, json=body)
+
+
+def _assert_never_dispatched(rig: _Rig, scenario: Scenario, denied_message: str) -> None:
+    control_key: Final = scenario.key(object_permission={"agents": [rig.allowed.agent_id]})
+    control_message: Final = "control-" + uuid.uuid4().hex
+    granted: Final = _send(
+        rig,
+        control_key,
+        f"/a2a/{rig.allowed.agent_id}",
+        method="message/send",
+        version=None,
+        message_id=control_message,
+    )
+    assert granted.status_code == 200, granted.text
+    received: Final = tuple(request.body.decode() for request in rig.upstream.drain())
+    assert any(control_message in body for body in received), received
+    assert not any(denied_message in body for body in received), received
+
+
+def _metadata(row: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
+    value: Final = row["metadata"]
+    return JSON_OBJECT.validate_json(value) if isinstance(value, str) else object_value(value)
+
+
+def _rows(rig: _Rig, caller: _Caller) -> list[dict[str, JsonValue]]:
+    return rig.proxy.rows(_ROWS, (caller.hashed,))
+
+
+def _hooks(events: tuple[_Event, ...]) -> tuple[str, ...]:
+    return tuple(sorted(event.hook for event in events))
+
+
+def _spend(rig: _Rig, caller: _Caller) -> tuple[JsonValue, ...]:
+    key: Final = object_value(rig.proxy.get("/key/info", {"key": caller.key})["info"])["spend"]
+    user: Final = object_value(rig.proxy.get("/user/info", {"user_id": caller.user_id})["user_info"])["spend"]
+    if caller.team_id is None:
+        return key, user
+    team: Final = object_value(rig.proxy.get("/team/info", {"team_id": caller.team_id})["team_info"])["spend"]
+    return key, user, team
+
+
+def _assert_one_failure(rig: _Rig, caller: _Caller, *, status: int, agent_id: str, model: str) -> dict[str, JsonValue]:
+    events: Final = held(
+        lambda: rig.events.of(caller.alias),
+        lambda found: _hooks(found) == _ONE_FAILURE,
+        holding=_SETTLE,
+        seconds=_CONVERGE,
+    )
+    logged: Final = next(event for event in events if event.hook == "async_failure")
+    assert (logged.call_type, logged.status, logged.model) == ("asend_message", "failure", model), logged
+    assert (logged.key_hash, logged.user_id, logged.team_id) == (caller.hashed, caller.user_id, caller.team_id), logged
+    hooked: Final = next(event for event in events if event.hook == "post_call_failure")
+    assert (hooked.status_code, hooked.agent_id, hooked.key_hash) == (status, agent_id, caller.hashed), hooked
+    assert (hooked.user_id, hooked.team_id) == (caller.user_id, caller.team_id), hooked
+    rows: Final = held(lambda: _rows(rig, caller), lambda found: len(found) == 1, holding=_SETTLE, seconds=_CONVERGE)
+    row: Final = rows[0]
+    observed: Final = (row["status"], row["call_type"], row["agent_id"], row["api_key"], row["user"], row["spend"])
+    assert observed == ("failure", "asend_message", agent_id, caller.hashed, caller.user_id, 0.0), row
+    assert (row["team_id"] or None) == caller.team_id, row
+    assert object_value(_metadata(row)["error_information"])["error_code"] == str(status), row
+    assert all(spend == 0.0 for spend in _spend(rig, caller)), _spend(rig, caller)
+    return row
+
+
+@pytest.mark.parametrize(
+    ("route", "method", "version", "legacy"),
+    [
+        pytest.param("/a2a/{}", "SendMessage", "1.0", False, id="regression-send-1.0"),
+        pytest.param("/a2a/{}/message/send", "SendMessage", "1.0", False, id="regression-send-alias"),
+        pytest.param("/v1/a2a/{}/message/send", "SendMessage", "1.0", False, id="regression-send-v1-alias"),
+        pytest.param("/a2a/{}", "SendStreamingMessage", "1.0", False, id="regression-stream-1.0"),
+        pytest.param("/a2a/{}", "GetTask", "1.0", False, id="regression-get-task-1.0"),
+        pytest.param("/a2a/{}", "CancelTask", "1.0", False, id="regression-cancel-task-1.0"),
+        pytest.param("/a2a/{}", "message/send", None, True, id="regression-send-0.3"),
+        pytest.param("/a2a/{}", "message/stream", None, True, id="regression-stream-0.3"),
+        pytest.param("/a2a/{}", "tasks/get", None, True, id="regression-tasks-get-0.3"),
+        pytest.param("/a2a/{}", "tasks/cancel", None, True, id="regression-tasks-cancel-0.3"),
+        pytest.param("/a2a/{}", "tasks/pushNotificationConfig/set", None, True, id="regression-push-set-0.3"),
+    ],
+)
+def test_denied_agent_keeps_its_403_logs_one_zero_spend_failure_for_the_caller_and_never_dispatches(
+    rig: _Rig, route: str, method: str, version: str | None, legacy: bool
+) -> None:
+    target: Final = rig.denied_legacy if legacy else rig.denied
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario)
+        message: Final = "denied-" + uuid.uuid4().hex
+        response: Final = _send(
+            rig, caller.key, route.format(target.agent_id), method=method, version=version, message_id=message
+        )
+        assert (response.status_code, response.content) == (403, _refusal(target.agent_id, response))
+        assert response.headers["content-type"] == "application/json"
+        _assert_never_dispatched(rig, scenario, message)
+        _assert_one_failure(rig, caller, status=403, agent_id=target.agent_id, model=f"a2a_agent/{target.name}")
+
+
+@pytest.mark.parametrize("grant", ["personal", "team", "access group"], ids=lambda grant: "regression-" + grant)
+def test_denied_agent_failure_is_attributed_to_the_authenticated_key_for_every_grant_kind(
+    rig: _Rig, grant: str
+) -> None:
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario, grant)
+        granted: Final = _send(rig, caller.key, f"/a2a/{rig.allowed.agent_id}", method="message/send", version=None)
+        assert granted.status_code == 200, granted.text
+        response: Final = _send(rig, caller.key, f"/a2a/{rig.denied.agent_id}")
+        assert (response.status_code, response.content) == (403, _refusal(rig.denied.agent_id, response))
+        events: Final = held(
+            lambda: rig.events.of(caller.alias),
+            lambda found: _hooks(found) == tuple(sorted((*_ONE_FAILURE, "async_success"))),
+            holding=_SETTLE,
+            seconds=_CONVERGE,
+        )
+        failure: Final = next(event for event in events if event.hook == "async_failure")
+        assert (failure.key_hash, failure.user_id, failure.team_id) == (caller.hashed, caller.user_id, caller.team_id)
+        rows: Final = held(
+            lambda: _rows(rig, caller), lambda found: len(found) == 2, holding=_SETTLE, seconds=_CONVERGE
+        )
+        failed: Final = tuple(row for row in rows if row["status"] == "failure")
+        assert [(row["agent_id"], row["user"], row["spend"]) for row in failed] == [
+            (rig.denied.agent_id, caller.user_id, 0.0)
+        ], rows
+
+
+def test_denied_completion_bridge_agent_logs_one_failure_and_never_reaches_the_bridged_model(rig: _Rig) -> None:
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario)
+        message: Final = "bridge-" + uuid.uuid4().hex
+        response: Final = _send(
+            rig, caller.key, f"/a2a/{rig.bridge.agent_id}", method="message/send", version=None, message_id=message
+        )
+        assert (response.status_code, response.content) == (403, _refusal(rig.bridge.agent_id, response))
+        _assert_never_dispatched(rig, scenario, message)
+        _assert_one_failure(rig, caller, status=403, agent_id=rig.bridge.agent_id, model=f"a2a_agent/{rig.bridge.name}")
+
+
+@pytest.mark.parametrize(
+    ("route", "method", "version"),
+    [
+        pytest.param("/a2a/{}", "SendMessage", "1.0", id="regression-send"),
+        pytest.param("/a2a/{}/message/send", "SendMessage", "1.0", id="regression-send-alias"),
+        pytest.param("/v1/a2a/{}/message/send", "SendMessage", "1.0", id="regression-send-v1-alias"),
+        pytest.param("/a2a/{}", "message/stream", None, id="regression-stream-0.3"),
+    ],
+)
+def test_unknown_agent_keeps_its_jsonrpc_404_and_logs_one_zero_spend_failure_for_the_caller(
+    rig: _Rig, route: str, method: str, version: str | None
+) -> None:
+    unknown: Final = "no-such-" + uuid.uuid4().hex
+    request_id: Final = "u-" + uuid.uuid4().hex
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario)
+        message: Final = "unknown-" + uuid.uuid4().hex
+        response: Final = _send(
+            rig,
+            caller.key,
+            route.format(unknown),
+            method=method,
+            version=version,
+            request_id=request_id,
+            message_id=message,
+        )
+        assert (response.status_code, response.content) == (404, _not_found(request_id, unknown))
+        assert response.headers["content-type"] == "application/json"
+        _assert_never_dispatched(rig, scenario, message)
+        _assert_one_failure(rig, caller, status=404, agent_id=unknown, model=f"a2a_agent/{unknown}")
+
+
+@pytest.mark.parametrize("request_id", [0, 7, "abc"], ids=lambda value: f"regression-id-{value!r}")
+def test_unknown_agent_echoes_the_jsonrpc_id_with_its_type_and_logs_one_failure(
+    rig: _Rig, request_id: JsonValue
+) -> None:
+    unknown: Final = "no-such-" + uuid.uuid4().hex
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario)
+        response: Final = _send(rig, caller.key, f"/a2a/{unknown}", request_id=request_id)
+        assert (response.status_code, response.content) == (404, _not_found(request_id, unknown))
+        assert type(JSON_OBJECT.validate_json(response.content)["id"]) is type(request_id)
+        _assert_one_failure(rig, caller, status=404, agent_id=unknown, model=f"a2a_agent/{unknown}")
+
+
+@pytest.mark.parametrize("outcome", ["denied", "unknown"], ids=lambda outcome: "regression-" + outcome)
+def test_client_supplied_attribution_cannot_rewrite_a_denial_failure_row(rig: _Rig, outcome: str) -> None:
+    unknown: Final = "no-such-" + uuid.uuid4().hex
+    target: Final = rig.denied.agent_id if outcome == "denied" else unknown
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario)
+        other: Final = _caller(rig, scenario)
+        success_id: Final = "s-" + uuid.uuid4().hex
+        granted: Final = _send(
+            rig, other.key, f"/a2a/{rig.allowed.agent_id}", method="message/send", version=None, request_id=success_id
+        )
+        assert granted.status_code == 200, granted.text
+        success_rows: Final = eventually(lambda: _rows(rig, other), lambda found: len(found) == 1, seconds=_CONVERGE)
+        success_request: Final = success_rows[0]["request_id"]
+        request_id: Final = "spoof-" + uuid.uuid4().hex
+        spoof: Final[dict[str, JsonValue]] = {
+            "agent_id": "spoof",
+            "metadata": {
+                "agent_id": "spoof",
+                "user_api_key_alias": "spoof",
+                "user_api_key_team_id": "spoof",
+                "tags": ["x"],
+            },
+            "litellm_call_id": success_request,
+        }
+        body: Final = {
+            "jsonrpc": "2.0",
+            "id": request_id,
+            "method": "SendMessage",
+            "params": {**_params("SendMessage", uuid.uuid4().hex, "1.0"), "litellm_call_id": "spoof2"},
+            **spoof,
+        }
+        response: Final = rig.proxy.client.post(
+            f"/a2a/{target}", headers={"Authorization": f"Bearer {caller.key}", "A2A-Version": "1.0"}, json=body
+        )
+        expected: Final = _refusal(target, response) if outcome == "denied" else _not_found(request_id, unknown)
+        assert (response.status_code, response.content) == (403 if outcome == "denied" else 404, expected)
+        model: Final = f"a2a_agent/{rig.denied.name}" if outcome == "denied" else f"a2a_agent/{unknown}"
+        row: Final = _assert_one_failure(rig, caller, status=response.status_code, agent_id=target, model=model)
+        assert row["request_id"] not in {success_request, "spoof2", request_id}, row
+        metadata: Final = _metadata(row)
+        assert (metadata["user_api_key_alias"], metadata["user_api_key_team_id"]) == (caller.alias, caller.team_id)
+        assert rig.events.of("spoof") == ()
+        after: Final = _rows(rig, other)
+        assert [(item["request_id"], item["status"]) for item in after] == [(success_request, "success")], after
+
+
+def test_two_keys_reusing_one_jsonrpc_id_get_one_denial_row_each(rig: _Rig) -> None:
+    shared_id: Final = "same-" + uuid.uuid4().hex
+    with rig.proxy.scenario() as scenario:
+        callers: Final = (_caller(rig, scenario), _caller(rig, scenario))
+        for caller in callers:
+            response = _send(rig, caller.key, f"/a2a/{rig.denied.agent_id}", request_id=shared_id)
+            assert (response.status_code, response.content) == (403, _refusal(rig.denied.agent_id, response))
+        rows: Final = tuple(
+            _assert_one_failure(
+                rig, caller, status=403, agent_id=rig.denied.agent_id, model=f"a2a_agent/{rig.denied.name}"
+            )
+            for caller in callers
+        )
+        assert rows[0]["request_id"] != rows[1]["request_id"], rows
+
+
+@pytest.mark.parametrize("outcome", ["denied", "unknown"])
+def test_failing_or_transforming_failure_callbacks_never_change_the_denial_response(
+    hostile: _Rig, outcome: str
+) -> None:
+    unknown: Final = "no-such-" + uuid.uuid4().hex
+    request_id: Final = "h-" + uuid.uuid4().hex
+    target: Final = hostile.denied.agent_id if outcome == "denied" else unknown
+    with hostile.proxy.scenario() as scenario:
+        caller: Final = _caller(hostile, scenario)
+        response: Final = _send(hostile, caller.key, f"/a2a/{target}", request_id=request_id)
+        expected: Final = (403, _refusal(target, response)) if outcome == "denied" else (404, _not_found(request_id, unknown))
+        assert (response.status_code, response.content) == expected
+
+
+@pytest.mark.parametrize("outcome", ["denied", "unknown"], ids=lambda outcome: "regression-" + outcome)
+def test_failing_or_transforming_failure_callbacks_do_not_stop_the_denial_from_being_logged(
+    hostile: _Rig, outcome: str
+) -> None:
+    unknown: Final = "no-such-" + uuid.uuid4().hex
+    target: Final = hostile.denied.agent_id if outcome == "denied" else unknown
+    with hostile.proxy.scenario() as scenario:
+        caller: Final = _caller(hostile, scenario)
+        response: Final = _send(hostile, caller.key, f"/a2a/{target}")
+        assert response.status_code == (403 if outcome == "denied" else 404), response.text
+        model: Final = f"a2a_agent/{hostile.denied.name}" if outcome == "denied" else f"a2a_agent/{unknown}"
+        _assert_one_failure(hostile, caller, status=response.status_code, agent_id=target, model=model)
+
+
+def test_denials_follow_the_model_denied_chat_logging_under_error_log_flags(flagged: _Rig) -> None:
+    unknown: Final = "no-such-" + uuid.uuid4().hex
+    request_id: Final = "f-" + uuid.uuid4().hex
+    with flagged.proxy.scenario() as scenario:
+        chat_caller: Final = _caller(flagged, scenario)
+        denied_caller: Final = _caller(flagged, scenario)
+        unknown_caller: Final = _caller(flagged, scenario)
+        chat_alias: Final = "a2adeny-chat-" + uuid.uuid4().hex[:8]
+        chat_key: Final = scenario.key(
+            user_id=chat_caller.user_id, team_id=chat_caller.team_id, key_alias=chat_alias, models=["openai/gpt-5-mini"]
+        )
+        chat: Final = flagged.proxy.request(
+            "POST",
+            "/v1/chat/completions",
+            {"model": "openai/gpt-5.4-mini", "messages": [{"role": "user", "content": "x"}]},
+            key=chat_key,
+        )
+        assert chat.status_code in {401, 403}, chat.text
+        denied: Final = _send(flagged, denied_caller.key, f"/a2a/{flagged.denied.agent_id}")
+        assert (denied.status_code, denied.content) == (403, _refusal(flagged.denied.agent_id, denied))
+        missing: Final = _send(flagged, unknown_caller.key, f"/a2a/{unknown}", request_id=request_id)
+        assert (missing.status_code, missing.content) == (404, _not_found(request_id, unknown))
+        _assert_never_dispatched(flagged, scenario, "never-sent")
+        chat_hooks: Final = held(
+            lambda: _hooks(flagged.events.of(chat_alias)),
+            lambda found: len(found) > 0,
+            holding=_SETTLE,
+            seconds=_CONVERGE,
+        )
+        assert _hooks(flagged.events.of(denied_caller.alias)) == chat_hooks
+        assert _hooks(flagged.events.of(unknown_caller.alias)) == chat_hooks
+        control: Final = _caller(flagged, scenario)
+        granted: Final = _send(
+            flagged, control.key, f"/a2a/{flagged.allowed.agent_id}", method="message/send", version=None
+        )
+        assert granted.status_code == 200, granted.text
+        eventually(lambda: _rows(flagged, control), lambda found: len(found) == 1, seconds=_CONVERGE)
+        assert (_rows(flagged, denied_caller), _rows(flagged, unknown_caller)) == ([], [])
+
+
+def test_client_error_denials_never_raise_an_llm_exceptions_alert(alerting: tuple[_Rig, Wire]) -> None:
+    rig, alerts = alerting
+    unknown: Final = "no-such-" + uuid.uuid4().hex
+    marker: Final = "boom-" + uuid.uuid4().hex
+
+    def failing(request: Request) -> Reply:
+        return Reply(
+            status=500, body=_compact({"error": {"message": f"upstream exploded {marker}", "type": "server_error"}})
+        )
+
+    with rig.proxy.scenario() as scenario, wire_server(failing) as broken:
+        caller: Final = _caller(rig, scenario)
+        assert _send(rig, caller.key, f"/a2a/{rig.denied.agent_id}").status_code == 403
+        assert _send(rig, caller.key, f"/a2a/{unknown}").status_code == 404
+        model: Final = scenario.model(api_base=broken.url + "/v1", num_retries=0)
+        failed: Final = rig.proxy.request(
+            "POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": "x"}]}
+        )
+        assert failed.status_code >= 500, failed.text
+        texts: Final = eventually(
+            lambda: tuple(JSON_OBJECT.validate_json(request.body).get("text") for request in alerts.drain()),
+            lambda found: any(marker in str(text) for text in found),
+            seconds=_CONVERGE,
+        )
+        assert not any(rig.denied.agent_id in str(text) or unknown in str(text) for text in texts), texts
+
+
+def test_guard_model_denied_chat_logs_one_acompletion_failure(rig: _Rig) -> None:
+    with rig.proxy.scenario() as scenario:
+        user: Final = scenario.user()
+        alias: Final = "a2adeny-chat-" + uuid.uuid4().hex[:8]
+        key: Final = scenario.key(user_id=user, key_alias=alias, models=["openai/gpt-5-mini"])
+        caller: Final = _Caller(key, alias, user, None)
+        response: Final = rig.proxy.request(
+            "POST",
+            "/v1/chat/completions",
+            {"model": "openai/gpt-5.4-mini", "messages": [{"role": "user", "content": "x"}]},
+            key=key,
+        )
+        assert response.status_code == 403, response.text
+        assert "key_model_access_denied" in response.text
+        events: Final = held(
+            lambda: rig.events.of(alias),
+            lambda found: _hooks(found) == _ONE_FAILURE,
+            holding=_SETTLE,
+            seconds=_CONVERGE,
+        )
+        assert {event.status_code for event in events if event.hook == "post_call_failure"} == {403}
+        rows: Final = held(
+            lambda: _rows(rig, caller), lambda found: len(found) == 1, holding=_SETTLE, seconds=_CONVERGE
+        )
+        assert (rows[0]["status"], rows[0]["call_type"], rows[0]["spend"]) == ("failure", "acompletion", 0.0), rows
+
+
+@pytest.mark.parametrize(
+    ("method", "call_type"), [("message/send", "asend_message"), ("message/stream", "asend_message_streaming")]
+)
+def test_guard_granted_agent_succeeds_with_one_success_row_and_no_failure(
+    rig: _Rig, method: str, call_type: str
+) -> None:
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario)
+        request_id: Final = "g-" + uuid.uuid4().hex
+        response: Final = _send(
+            rig, caller.key, f"/a2a/{rig.allowed.agent_id}", method=method, version=None, request_id=request_id
+        )
+        assert response.status_code == 200, response.text
+        if method == "message/stream":
+            assert response.text.startswith("data: ") and request_id in response.text, response.text
+        else:
+            assert object_value(JSON_OBJECT.validate_json(response.content)["result"])["parts"] == [
+                {"kind": "text", "text": "granted"}
+            ]
+        held(
+            lambda: rig.events.of(caller.alias),
+            lambda found: _hooks(found) == ("async_success",),
+            holding=_SETTLE,
+            seconds=_CONVERGE,
+        )
+        rows: Final = held(
+            lambda: _rows(rig, caller), lambda found: len(found) == 1, holding=_SETTLE, seconds=_CONVERGE
+        )
+        assert (rows[0]["status"], rows[0]["call_type"], rows[0]["agent_id"]) == (
+            "success",
+            call_type,
+            rig.allowed.agent_id,
+        ), rows
+
+
+def test_guard_zero_budget_key_is_refused_once_without_double_logging(rig: _Rig) -> None:
+    with rig.proxy.scenario() as scenario:
+        user: Final = scenario.user()
+        alias: Final = "a2adeny-budget-" + uuid.uuid4().hex[:8]
+        key: Final = scenario.key(
+            user_id=user, key_alias=alias, max_budget=0, object_permission={"agents": [rig.allowed.agent_id]}
+        )
+        caller: Final = _Caller(key, alias, user, None)
+        message: Final = "budget-" + uuid.uuid4().hex
+        response: Final = _send(
+            rig, key, f"/a2a/{rig.allowed.agent_id}", method="message/send", version=None, message_id=message
+        )
+        assert response.status_code == 422, response.text
+        assert "Budget has been exceeded" in response.text
+        _assert_never_dispatched(rig, scenario, message)
+        held(
+            lambda: rig.events.of(alias),
+            lambda found: _hooks(found) == _ONE_FAILURE,
+            holding=_SETTLE,
+            seconds=_CONVERGE,
+        )
+        rows: Final = held(
+            lambda: _rows(rig, caller), lambda found: len(found) == 1, holding=_SETTLE, seconds=_CONVERGE
+        )
+        assert (rows[0]["status"], rows[0]["call_type"]) == ("failure", "asend_message"), rows
+
+
+def test_guard_agent_card_discovery_keeps_its_refusals_and_writes_no_failure_row(rig: _Rig) -> None:
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario)
+        headers: Final = {"Authorization": f"Bearer {caller.key}"}
+        denied_card: Final = rig.proxy.client.get(
+            f"/a2a/{rig.denied.agent_id}/.well-known/agent-card.json", headers=headers
+        )
+        assert (denied_card.status_code, denied_card.content) == (403, _refusal(rig.denied.agent_id))
+        missing: Final = rig.proxy.client.get("/a2a/no-such-card/.well-known/agent-card.json", headers=headers)
+        assert (missing.status_code, missing.content) == (404, _compact({"detail": "Agent 'no-such-card' not found"}))
+        granted: Final = _send(rig, caller.key, f"/a2a/{rig.allowed.agent_id}", method="message/send", version=None)
+        assert granted.status_code == 200, granted.text
+        rows: Final = held(
+            lambda: _rows(rig, caller), lambda found: len(found) == 1, holding=_SETTLE, seconds=_CONVERGE
+        )
+        assert [row["status"] for row in rows] == ["success"], rows
+        assert _hooks(rig.events.of(caller.alias)) == ("async_success",)
+
+
+def test_guard_jsonrpc_1_0_envelope_keeps_its_invalid_request_body(rig: _Rig) -> None:
+    with rig.proxy.scenario() as scenario:
+        caller: Final = _caller(rig, scenario)
+        request_id: Final = "v-" + uuid.uuid4().hex
+        response: Final = _send(
+            rig,
+            caller.key,
+            f"/a2a/{rig.allowed.agent_id}",
+            method="message/send",
+            version=None,
+            request_id=request_id,
+            extra={"jsonrpc": "1.0"},
+        )
+        assert (response.status_code, response.content) == (
+            400,
+            _compact(
+                {
+                    "jsonrpc": "2.0",
+                    "id": request_id,
+                    "error": {"code": -32600, "message": "Invalid Request: jsonrpc must be '2.0'"},
+                }
+            ),
+        )

tests/integration/compatibility/test_a2a_wire_versions.py

@@ -452,8 +452,13 @@ def test_guard_no_agent_key_or_proxy_caller_key_is_sent(gateway: Gateway) -> Non
     (("0.3", "message/stream", _MESSAGE_V03), ("1.0", "SendStreamingMessage", _MESSAGE_V10)),
     ids=("v0.3", "v1.0"),
 )
+@pytest.mark.parametrize(
+    "path",
+    ("/a2a/{identity}", "/a2a/{identity}/message/stream", "/v1/a2a/{identity}/message/stream"),
+    ids=("bare", "message_stream", "v1_message_stream"),
+)
 def test_a2a_stream_relays_each_upstream_event_as_its_own_sse_frame_in_the_pinned_dialect(
-    gateway: Gateway, version: str, method: str, message: dict[str, JsonValue]
+    gateway: Gateway, version: str, method: str, message: dict[str, JsonValue], path: str
 ) -> None:
     marker: Final = "a2astream" + uuid.uuid4().hex
     first_frame_seen: Final = threading.Event()
@@ -482,7 +487,7 @@ def test_a2a_stream_relays_each_upstream_event_as_its_own_sse_frame_in_the_pinne
         )
         with gateway.client.stream(
             "POST",
-            f"/a2a/{identity}",
+            path.format(identity=identity),
             headers={
                 "Authorization": f"Bearer {key}",
                 "a2a-version": version,

tests/integration/configuration/test_multi_worker_reload.py

@@ -0,0 +1,95 @@
+import json
+import os
+import re
+from pathlib import Path
+from typing import Final
+
+import psutil
+import pytest
+
+from tests.integration._support.client import Gateway, eventually, object_value
+from tests.integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
+
+STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]")
+
+
+def _worker_pids(owned: OwnedProxy) -> frozenset[int]:
+    return frozenset(int(match.group(1)) for match in STARTED_WORKER.finditer(owned.log.read_text()))
+
+
+def _launched_config(owned: OwnedProxy) -> Path:
+    arguments: Final = psutil.Process(owned.process.pid).cmdline()
+    return Path(arguments[arguments.index("--config") + 1])
+
+
+def _config(model: str) -> str:
+    return json.dumps(
+        {
+            "model_list": [{"model_name": model, "litellm_params": {"model": "openai/reload-test"}}],
+            "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY"},
+            "router_settings": {"redis_host": "os.environ/REDIS_HOST", "redis_port": "os.environ/REDIS_PORT"},
+        }
+    )
+
+
+def _assert_replaced(owned: OwnedProxy, previous: frozenset[int], workers: int) -> frozenset[int]:
+    started: Final = eventually(
+        lambda: _worker_pids(owned),
+        lambda pids: len(pids - previous) == workers,
+        seconds=graceful_stop_seconds(),
+    )
+    assert all(not psutil.pid_exists(pid) for pid in previous), owned.log.read_text()
+    assert owned.process.poll() is None, owned.log.read_text()
+    assert len(started) == len(previous) + workers, owned.log.read_text()
+    return started
+
+
+def _assert_models(gateway: Gateway) -> None:
+    data: Final = gateway.get("/v1/models")["data"]
+    assert isinstance(data, list), data
+    assert tuple(object_value(model)["id"] for model in data) == ("after-reload",)
+
+
+@pytest.mark.parametrize("workers", (1, 2))
+@pytest.mark.timeout(6 * graceful_stop_seconds())
+def test_reload_replaces_every_worker_and_shutdown_leaves_none(
+    gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, workers: int
+) -> None:
+    root: Final = Path(__file__).resolve().parents[3]
+    monkeypatch.setenv("INTEGRATION_PROXY_ROOT", str(tmp_path))
+    config: Final = tmp_path / "config.yaml"
+    source: Final = tmp_path / "reload_source.py"
+    env: Final = tmp_path / ".env"
+    config.write_text(_config("before-reload"))
+    source.write_text("VALUE = 1\n")
+    env.write_text("RELOAD_TEST_VALUE=before\n")
+    with owned_proxy_process(
+        gateway,
+        tmp_path,
+        {
+            "PYTHONPATH": os.pathsep.join((str(root), str(root / "tests"), os.environ.get("PYTHONPATH", ""))),
+            "STORE_MODEL_IN_DB": "False",
+        },
+        config=config,
+        remove_environment=("DATABASE_URL", "DATABASE_URL_READ_REPLICA", "PYTHON_DOTENV_DISABLED"),
+        workers=workers,
+        database_setup=(),
+        extra_arguments=("--reload",),
+    ) as owned:
+        initial: Final = eventually(
+            lambda: _worker_pids(owned), lambda pids: len(pids) == workers, seconds=graceful_stop_seconds()
+        )
+        _launched_config(owned).write_text(_config("after-reload"))
+        after_config: Final = _assert_replaced(owned, initial, workers)
+        for _ in range(2 * workers):
+            _assert_models(owned.gateway)
+        source.write_text("VALUE = 2\n")
+        after_source: Final = _assert_replaced(owned, after_config, workers)
+        env.write_text("RELOAD_TEST_VALUE=after\n")
+        after_env: Final = _assert_replaced(owned, after_source, workers)
+        response: Final = owned.gateway.request("GET", "/health/readiness")
+        assert response.status_code == 200, response.text
+        owned.begin_shutdown()
+        owned.process.wait(timeout=graceful_stop_seconds())
+        assert all(not psutil.pid_exists(pid) for pid in after_env), owned.log.read_text()
+        assert owned.process.returncode == 0, owned.log.read_text()

tests/integration/configuration/test_reload_worker_pool.py

@@ -0,0 +1,246 @@
+import multiprocessing
+import re
+import signal
+import socket
+import subprocess
+import sys
+import threading
+import time
+from collections.abc import Callable
+from pathlib import Path
+from typing import Final
+
+import httpx
+import psutil
+import pytest
+
+from litellm.proxy.dev_reload import multi_worker_reload_supported, stop_pool_when_reloader_exits
+from tests.integration._support.client import eventually
+from tests.integration._support.ports import allocate_port
+from tests.integration._support.process import group_members, kill_stray_process, signal_group, stop_processes
+
+POOL_WORKERS: Final = 2
+POOL_TIMEOUT_SECONDS: Final = 120
+WORKER_HEALTHCHECK_SECONDS: Final = 30
+WORKER_STARTED: Final = re.compile(r"Started server process \[(\d+)\]")
+POOL_STOPPED: Final = "failed to start, stopping the parent process"
+APP_LOAD_ERROR: Final = "Error loading ASGI app"
+PID_APP: Final = """import os
+
+
+async def app(scope, receive, send):
+    if scope["type"] != "http":
+        return
+    await send({"type": "http.response.start", "status": 200, "headers": []})
+    await send({"type": "http.response.body", "body": str(os.getpid()).encode()})
+"""
+REQUIRES_UVICORN_STARTUP_FAILURE: Final = pytest.mark.skipif(
+    not multi_worker_reload_supported(), reason="uvicorn<0.50.0 has no startup failure exit code"
+)
+CORE_DUMP_SECONDS: Final = 15
+SLOW_CORE_DUMP_ON_QUIT: Final = f"""
+import os
+import signal
+import subprocess
+
+
+def freeze_until_killed(signum, frame):
+    subprocess.Popen(("sh", "-c", f"sleep {CORE_DUMP_SECONDS}; kill -9 {{os.getpid()}}"))
+    os.kill(os.getpid(), signal.SIGSTOP)
+
+
+signal.signal(signal.SIGQUIT, freeze_until_killed)
+"""
+POOL_RUNNER: Final = """import sys
+
+sys.path.insert(0, sys.argv[1])
+
+from litellm.proxy.dev_reload import run_multi_worker_reload
+
+run_multi_worker_reload(
+    {
+        "app": "reload_pool_app:app",
+        "host": "127.0.0.1",
+        "port": int(sys.argv[2]),
+        "timeout_worker_healthcheck": int(sys.argv[3]),
+        "reload": True,
+        "reload_dirs": (sys.argv[1],),
+        "reload_includes": ("*.py",),
+    },
+    workers=int(sys.argv[4]),
+)
+"""
+
+
+def _started_workers(log_text: str) -> frozenset[int]:
+    return frozenset(int(match.group(1)) for match in WORKER_STARTED.finditer(log_text))
+
+
+def _until(done: Callable[[], bool], log: Path, seconds: float = POOL_TIMEOUT_SECONDS) -> None:
+    eventually(lambda: (done(), log.read_text()), lambda observed: observed[0], seconds=seconds)
+
+
+def _wait_for_log(log: Path, process: subprocess.Popen[bytes], seen: Callable[[str], bool]) -> str:
+    log_text, _ = eventually(
+        lambda: (log.read_text(), process.poll()),
+        lambda observed: seen(observed[0]) or observed[1] is not None,
+        seconds=POOL_TIMEOUT_SECONDS,
+    )
+    assert seen(log_text), log_text
+    return log_text
+
+
+def _start_pool(app_dir: Path, port: int, log: Path, runner: str = POOL_RUNNER) -> subprocess.Popen[bytes]:
+    with log.open("wb") as output:
+        return subprocess.Popen(
+            (
+                sys.executable,
+                "-I",
+                "-c",
+                runner,
+                str(app_dir),
+                str(port),
+                str(WORKER_HEALTHCHECK_SECONDS),
+                str(POOL_WORKERS),
+            ),
+            stdout=output,
+            stderr=subprocess.STDOUT,
+            start_new_session=True,
+        )
+
+
+def _port_refuses_connections(port: int) -> bool:
+    with socket.socket() as sock:
+        return sock.connect_ex(("127.0.0.1", port)) != 0
+
+
+def _next_pool(log: Path, process: subprocess.Popen[bytes], port: int, previous: frozenset[int]) -> frozenset[int]:
+    log_text: Final = _wait_for_log(log, process, lambda text: len(_started_workers(text) - previous) == POOL_WORKERS)
+    pool: Final = _started_workers(log_text) - previous
+    assert not any(psutil.pid_exists(pid) for pid in previous), log_text
+    served: Final = httpx.get(f"http://127.0.0.1:{port}/", timeout=POOL_TIMEOUT_SECONDS)
+    assert int(served.text) in pool, log_text
+    return pool
+
+
+def test_the_pool_is_told_to_stop_once_the_reloader_process_exits() -> None:
+    reloader: Final = multiprocessing.get_context("spawn").Process(target=time.sleep, args=(POOL_TIMEOUT_SECONDS,))
+    reloader.start()
+    pool_should_exit: Final = threading.Event()
+    threading.Thread(target=stop_pool_when_reloader_exits, args=(reloader, pool_should_exit), daemon=True).start()
+    try:
+        assert not pool_should_exit.wait(0.5)
+        kill_stray_process(reloader.pid)
+        assert pool_should_exit.wait(POOL_TIMEOUT_SECONDS)
+    finally:
+        if reloader.is_alive():
+            kill_stray_process(reloader.pid)
+        reloader.join()
+
+
+@REQUIRES_UVICORN_STARTUP_FAILURE
+@pytest.mark.timeout(5 * POOL_TIMEOUT_SECONDS)
+def test_a_save_replaces_every_worker_and_a_broken_save_stops_the_pool_until_the_next_good_one(
+    tmp_path: Path,
+) -> None:
+    app: Final = tmp_path / "reload_pool_app.py"
+    app.write_text(PID_APP)
+    log: Final = tmp_path / "reload.log"
+    port: Final = allocate_port()
+    process: Final = _start_pool(tmp_path, port, log)
+    try:
+        first_pool: Final = _next_pool(log, process, port, frozenset())
+        app.write_text("async def app(\n")
+        _wait_for_log(log, process, lambda text: POOL_STOPPED in text)
+        assert not any(psutil.pid_exists(pid) for pid in first_pool), log.read_text()
+        app.write_text(PID_APP)
+        last_pool: Final = _next_pool(log, process, port, first_pool)
+        signal_group(process.pid, signal.SIGINT)
+        assert process.wait(timeout=POOL_TIMEOUT_SECONDS) == 0, log.read_text()
+        assert not any(psutil.pid_exists(pid) for pid in last_pool), log.read_text()
+        assert 1 <= log.read_text().count(APP_LOAD_ERROR) <= POOL_WORKERS, log.read_text()
+    finally:
+        stop_processes(group_members(process.pid))
+        process.wait()
+
+
+@REQUIRES_UVICORN_STARTUP_FAILURE
+@pytest.mark.timeout(3 * POOL_TIMEOUT_SECONDS)
+@pytest.mark.parametrize(
+    ("kill_reloader", "slow_core_dump", "shutdown_timeout"),
+    (
+        pytest.param(kill_stray_process, False, POOL_TIMEOUT_SECONDS, id="sigkill-reloader"),
+        pytest.param(
+            lambda reloader: signal_group(reloader, signal.SIGHUP), False, POOL_TIMEOUT_SECONDS, id="terminal-closed"
+        ),
+        pytest.param(lambda reloader: signal_group(reloader, signal.SIGQUIT), True, 10, id="ctrl-backslash"),
+    ),
+)
+def test_the_pool_stops_and_frees_the_port_when_the_reloader_dies(
+    tmp_path: Path, kill_reloader: Callable[[int], None], slow_core_dump: bool, shutdown_timeout: float
+) -> None:
+    app_source: Final = PID_APP + (SLOW_CORE_DUMP_ON_QUIT if slow_core_dump else "")
+    (tmp_path / "reload_pool_app.py").write_text(app_source)
+    log: Final = tmp_path / "reload.log"
+    port: Final = allocate_port()
+    process: Final = _start_pool(tmp_path, port, log)
+    try:
+        workers: Final = _next_pool(log, process, port, frozenset())
+        pool_parent: Final = psutil.Process(min(workers)).ppid()
+        assert psutil.Process(pool_parent).ppid() == process.pid, log.read_text()
+        kill_reloader(process.pid)
+        process.wait(timeout=POOL_TIMEOUT_SECONDS)
+        _until(
+            lambda: not psutil.pid_exists(pool_parent) and _port_refuses_connections(port),
+            log,
+            seconds=shutdown_timeout,
+        )
+    finally:
+        stop_processes(group_members(process.pid))
+        process.wait()
+
+
+@REQUIRES_UVICORN_STARTUP_FAILURE
+@pytest.mark.timeout(3 * POOL_TIMEOUT_SECONDS)
+def test_a_killed_pool_parent_takes_its_workers_with_it_so_the_next_save_serves_only_new_workers(
+    tmp_path: Path,
+) -> None:
+    app: Final = tmp_path / "reload_pool_app.py"
+    app.write_text(PID_APP)
+    log: Final = tmp_path / "reload.log"
+    port: Final = allocate_port()
+    process: Final = _start_pool(tmp_path, port, log)
+    try:
+        orphans: Final = _next_pool(log, process, port, frozenset())
+        kill_stray_process(psutil.Process(min(orphans)).ppid())
+        _until(lambda: not any(psutil.pid_exists(pid) for pid in orphans), log)
+        app.write_text(f"{PID_APP}\n")
+        _next_pool(log, process, port, orphans)
+    finally:
+        stop_processes(group_members(process.pid))
+        process.wait()
+
+
+@REQUIRES_UVICORN_STARTUP_FAILURE
+@pytest.mark.timeout(3 * POOL_TIMEOUT_SECONDS)
+def test_ctrl_backslash_frees_the_port_without_waiting_out_the_worker_healthcheck_when_core_dumps_are_slow(
+    tmp_path: Path,
+) -> None:
+    (tmp_path / "reload_pool_app.py").write_text(PID_APP + SLOW_CORE_DUMP_ON_QUIT)
+    log: Final = tmp_path / "reload.log"
+    port: Final = allocate_port()
+    process: Final = _start_pool(tmp_path, port, log)
+    try:
+        workers: Final = _next_pool(log, process, port, frozenset())
+        pool_parent: Final = psutil.Process(min(workers)).ppid()
+        signal_group(process.pid, signal.SIGQUIT)
+        _until(
+            lambda: (
+                process.poll() is not None and not psutil.pid_exists(pool_parent) and _port_refuses_connections(port)
+            ),
+            log,
+            seconds=WORKER_HEALTHCHECK_SECONDS / 2,
+        )
+    finally:
+        stop_processes(group_members(process.pid))
+        process.wait()

tests/integration/configuration/test_search_embedding_model_settings.py

@@ -0,0 +1,299 @@
+from __future__ import annotations
+
+import json
+import uuid
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+from io import BytesIO
+from pathlib import Path
+from typing import Final
+from zipfile import ZipFile
+
+import pytest
+from integration._support.client import (
+    JSON_OBJECT,
+    Gateway,
+    gateway_from_environment,
+    list_value,
+    object_value,
+    string_value,
+)
+from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
+from integration._support.wire import Reply, Request, Wire, wire_server
+from pydantic import JsonValue, TypeAdapter
+
+pytestmark: Final = pytest.mark.timeout(int(2 * graceful_stop_seconds() + 120))
+
+_MODEL_NAME: Final = "search-embedding"
+_AGENT_NOT_CONFIGURED: Final = (
+    "agent search needs litellm_settings.agent_search_embedding_model set to an embedding model from model_list"
+)
+_SKILL_NOT_CONFIGURED: Final = (
+    "skill search needs litellm_settings.skill_search_embedding_model set to an embedding model from model_list"
+)
+_SKILL_MARKDOWN: Final = """---
+name: echo-sample
+description: Echo text for search validation
+---
+
+# Echo sample
+
+Return the supplied text unchanged
+"""
+_JSON_ARRAY: Final = TypeAdapter(list[JsonValue])
+
+
+@dataclass(frozen=True, slots=True)
+class _SearchProxy:
+    owned: OwnedProxy
+    wire: Wire
+    skill_id: str
+    agent_id: str
+
+
+def _embedding_reply(request: Request) -> Reply:
+    if request.method != "POST" or request.target != "/v1/embeddings":
+        return Reply(body=b'{"object":"list","data":[]}')
+    body: Final = JSON_OBJECT.validate_json(request.body)
+    input_value: Final = body.get("input")
+    input_count: Final = len(input_value) if isinstance(input_value, list) else 1
+    embeddings: Final = [
+        {"object": "embedding", "index": index, "embedding": [0.25, 0.75]} for index in range(input_count)
+    ]
+    return Reply(
+        body=json.dumps(
+            {
+                "object": "list",
+                "data": embeddings,
+                "model": "text-embedding-3-small",
+                "usage": {"prompt_tokens": input_count, "total_tokens": input_count},
+            }
+        ).encode()
+    )
+
+
+def _write_config(directory: Path, upstream_url: str, agent_model: str, skill_model: str) -> Path:
+    config: Final = directory / f"search_embedding_{uuid.uuid4().hex}.yaml"
+    contents: Final[dict[str, JsonValue]] = {
+        "model_list": [
+            {
+                "model_name": _MODEL_NAME,
+                "litellm_params": {
+                    "model": "openai/text-embedding-3-small",
+                    "api_base": f"{upstream_url}/v1",
+                    "api_key": "synthetic-openai-key",
+                },
+            }
+        ],
+        "litellm_settings": {
+            "agent_search_embedding_model": agent_model,
+            "skill_search_embedding_model": skill_model,
+        },
+        "general_settings": {
+            "master_key": "os.environ/LITELLM_MASTER_KEY",
+            "database_url": "os.environ/DATABASE_URL",
+            "store_model_in_db": True,
+        },
+        "agents": [
+            {
+                "agent_name": "plain",
+                "agent_card_params": {
+                    "protocolVersion": "1.0",
+                    "name": "plain",
+                    "description": "echo agent",
+                    "url": "http://agent:9000/plain/",
+                    "supportedInterfaces": [
+                        {
+                            "url": "http://agent:9000/plain/",
+                            "protocolBinding": "JSONRPC",
+                            "protocolVersion": "1.0",
+                        }
+                    ],
+                    "version": "1.0.0",
+                    "capabilities": {"streaming": True},
+                    "defaultInputModes": ["text/plain"],
+                    "defaultOutputModes": ["text/plain"],
+                    "skills": [],
+                },
+            }
+        ],
+    }
+    config.write_text(json.dumps(contents))
+    return config
+
+
+def _create_skill(candidate: OwnedProxy) -> str:
+    archive_data: Final = BytesIO()
+    with ZipFile(archive_data, "w") as archive:
+        archive.writestr("echo-sample/SKILL.md", _SKILL_MARKDOWN)
+    response: Final = candidate.gateway.request_multipart(
+        "/v1/skills?beta=true&custom_llm_provider=litellm_proxy",
+        {
+            "display_title": "Echo search fixture",
+            "description": "Echo sample skill for search validation",
+            "instructions": "Return the supplied text unchanged",
+        },
+        {"files[]": ("echo-skill.zip", archive_data.getvalue(), "application/zip")},
+    )
+    assert response.status_code == 200, response.text
+    return string_value(JSON_OBJECT.validate_json(response.content)["id"])
+
+
+def _agent_id(candidate: OwnedProxy) -> str:
+    response: Final = candidate.gateway.request("GET", "/v1/agents")
+    assert response.status_code == 200, response.text
+    agents: Final = _JSON_ARRAY.validate_json(response.content)
+    assert len(agents) == 1
+    return string_value(object_value(agents[0])["agent_id"])
+
+
+@pytest.fixture(scope="module")
+def empty_search_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_SearchProxy]:
+    directory: Final = tmp_path_factory.mktemp("search-empty-model")
+    with wire_server(_embedding_reply) as upstream, gateway_from_environment() as gateway:
+        config: Final = _write_config(
+            directory,
+            upstream.url,
+            "os.environ/AGENT_SEARCH_EMBEDDING_MODEL",
+            "os.environ/SKILL_SEARCH_EMBEDDING_MODEL",
+        )
+        with owned_proxy_process(
+            gateway,
+            directory,
+            {"AGENT_SEARCH_EMBEDDING_MODEL": "", "SKILL_SEARCH_EMBEDDING_MODEL": ""},
+            config=config,
+        ) as owned:
+            yield _SearchProxy(owned, upstream, _create_skill(owned), _agent_id(owned))
+
+
+@pytest.fixture(scope="module")
+def configured_search_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_SearchProxy]:
+    directory: Final = tmp_path_factory.mktemp("search-configured-model")
+    with wire_server(_embedding_reply) as upstream, gateway_from_environment() as gateway:
+        config: Final = _write_config(directory, upstream.url, _MODEL_NAME, _MODEL_NAME)
+        with owned_proxy_process(
+            gateway,
+            directory,
+            {"AGENT_SEARCH_EMBEDDING_MODEL": _MODEL_NAME, "SKILL_SEARCH_EMBEDDING_MODEL": _MODEL_NAME},
+            config=config,
+        ) as owned:
+            yield _SearchProxy(owned, upstream, _create_skill(owned), _agent_id(owned))
+
+
+def _assert_not_configured(response_body: JsonValue, expected_error: str, reason: str) -> None:
+    detail: Final = object_value(object_value(response_body)["detail"])
+    assert detail == {"error": expected_error, "message": reason}
+
+
+def _assert_no_embedding_requests(candidate: _SearchProxy) -> None:
+    embedding_requests: Final = tuple(
+        request for request in candidate.wire.drain() if request.method == "POST" and request.target == "/v1/embeddings"
+    )
+    assert embedding_requests == ()
+
+
+def _assert_mcp_search_not_configured(gateway: Gateway, key: str, tool_name: str, reason: str) -> None:
+    response: Final = gateway.request(
+        "POST",
+        "/mcp-rest/tools/call",
+        {"name": tool_name, "arguments": {"query": "echo"}},
+        key=key,
+    )
+    assert response.status_code == 200, response.text
+    result: Final = JSON_OBJECT.validate_json(response.content)
+    assert result.get("isError") is True, response.text
+    content: Final = object_value(list_value(result["content"])[0])
+    text: Final = string_value(content["text"])
+    assert reason in text
+    assert "You passed in model=" not in text
+
+
+def test_empty_agent_embedding_setting_returns_not_configured(empty_search_proxy: _SearchProxy) -> None:
+    response: Final = empty_search_proxy.owned.gateway.request("GET", "/v1/agents?query=echo")
+    assert response.status_code == 400, response.text
+    _assert_not_configured(
+        JSON_OBJECT.validate_json(response.content),
+        "agent_search_not_configured",
+        _AGENT_NOT_CONFIGURED,
+    )
+    _assert_no_embedding_requests(empty_search_proxy)
+
+
+def test_empty_skill_embedding_setting_returns_not_configured(empty_search_proxy: _SearchProxy) -> None:
+    response: Final = empty_search_proxy.owned.gateway.request(
+        "GET", "/v1/skills?custom_llm_provider=litellm_proxy&query=echo"
+    )
+    assert response.status_code == 400, response.text
+    _assert_not_configured(
+        JSON_OBJECT.validate_json(response.content),
+        "skill_search_not_configured",
+        _SKILL_NOT_CONFIGURED,
+    )
+    _assert_no_embedding_requests(empty_search_proxy)
+
+
+def test_empty_agent_embedding_setting_returns_not_configured_through_mcp(empty_search_proxy: _SearchProxy) -> None:
+    with empty_search_proxy.owned.gateway.scenario() as scenario:
+        key: Final = scenario.key(
+            object_permission={
+                "agents": [empty_search_proxy.agent_id],
+                "mcp_tool_search_enabled": True,
+            },
+        )
+        _assert_mcp_search_not_configured(
+            empty_search_proxy.owned.gateway,
+            key,
+            "agent_search",
+            _AGENT_NOT_CONFIGURED,
+        )
+    _assert_no_embedding_requests(empty_search_proxy)
+
+
+def test_empty_skill_embedding_setting_returns_not_configured_through_mcp(empty_search_proxy: _SearchProxy) -> None:
+    with empty_search_proxy.owned.gateway.scenario() as scenario:
+        user_id: Final = scenario.user(user_role="proxy_admin")
+        key: Final = scenario.key(user_id=user_id, object_permission={"mcp_tool_search_enabled": True})
+        _assert_mcp_search_not_configured(
+            empty_search_proxy.owned.gateway,
+            key,
+            "skill_search",
+            _SKILL_NOT_CONFIGURED,
+        )
+    _assert_no_embedding_requests(empty_search_proxy)
+
+
+def test_configured_agent_embedding_model_returns_search_score(configured_search_proxy: _SearchProxy) -> None:
+    configured_search_proxy.wire.drain()
+    response: Final = configured_search_proxy.owned.gateway.request("GET", "/v1/agents?query=echo")
+    assert response.status_code == 200, response.text
+    agents: Final = _JSON_ARRAY.validate_json(response.content)
+    assert len(agents) == 1
+    agent: Final = object_value(agents[0])
+    assert string_value(agent["agent_name"]) == "plain"
+    assert isinstance(agent["search_score"], (int, float))
+    embedding_requests: Final = tuple(
+        request
+        for request in configured_search_proxy.wire.drain()
+        if request.method == "POST" and request.target == "/v1/embeddings"
+    )
+    assert embedding_requests
+
+
+def test_configured_skill_embedding_model_returns_search_score(configured_search_proxy: _SearchProxy) -> None:
+    configured_search_proxy.wire.drain()
+    response: Final = configured_search_proxy.owned.gateway.request(
+        "GET", "/v1/skills?custom_llm_provider=litellm_proxy&query=echo"
+    )
+    assert response.status_code == 200, response.text
+    skills: Final[Sequence[JsonValue]] = list_value(JSON_OBJECT.validate_json(response.content)["data"])
+    matching_skills: Final = tuple(
+        object_value(skill) for skill in skills if object_value(skill)["id"] == configured_search_proxy.skill_id
+    )
+    assert len(matching_skills) == 1
+    assert isinstance(matching_skills[0]["search_score"], (int, float))
+    embedding_requests: Final = tuple(
+        request
+        for request in configured_search_proxy.wire.drain()
+        if request.method == "POST" and request.target == "/v1/embeddings"
+    )
+    assert embedding_requests

tests/integration/mcp/test_mcp_lifecycle.py

@@ -6,6 +6,7 @@ from collections.abc import Mapping
 from contextlib import ExitStack
 from hashlib import sha256
 from pathlib import Path
+from threading import Event
 from typing import Final, cast
 
 import httpx
@@ -24,6 +25,7 @@ from integration._support.generation import LIFECYCLE_SETTINGS, bounded_http_req
 from integration._support.mcp import (
     JsonRpc,
     McpCaller,
+    McpPeer,
     Outcome,
     ScriptedTool,
     call_tool,
@@ -46,6 +48,34 @@ _OBJECTS: Final = TypeAdapter(Mapping[str, object])
 _STRINGS: Final = TypeAdapter(Mapping[str, str])
 
 
+@pytest.mark.timeout(30)
+def test_owned_http_peer_cleans_up_a_disconnected_tool_call() -> None:
+    started: Final = Event()
+
+    async def hang_up(peer: McpPeer) -> None:
+        async with (
+            streamable_http_client(peer.url, terminate_on_close=False) as streams,
+            ClientSession(streams[0], streams[1]) as session,
+        ):
+            await session.initialize()
+            peer.drain()
+            async with asyncio.TaskGroup() as tasks:
+                running: Final = tasks.create_task(session.call_tool("slow", {"seconds": 60}))
+                received: Final = await asyncio.to_thread(
+                    eventually, lambda: tool_calls(peer.drain()), bool, seconds=30
+                )
+                assert len(received) == 1, received
+                params: Final = _OBJECTS.validate_python(_OBJECTS.validate_python(received[0]["body"])["params"])
+                assert (params["name"], params["arguments"]) == ("slow", {"seconds": 60}), received
+                assert await asyncio.to_thread(started.wait, 5), "The MCP tool handler never started"
+                assert not running.done(), running.result().model_dump_json()
+                running.cancel()
+            assert running.cancelled()
+
+    with mcp_peer(rich=True, on_slow=started.set) as peer:
+        asyncio.run(hang_up(peer))
+
+
 @pytest.mark.covers("mcp.call_tool.saved_headers.reach_actual_transport")
 def test_saved_headers_reach_real_mcp_tool_and_survive_unrelated_edit(gateway: Gateway) -> None:
     with mcp_peer() as peer, gateway.scenario() as scenario:

tests/integration/observability/test_ovalix_guardrail.py

@@ -0,0 +1,338 @@
+"""Ovalix guardrail driven through a real proxy.
+
+The Ovalix tracker is the only guardrail double: an owned wire peer that answers the `/tracking/beta` routes
+the way the live tracker does, including allowing a checkpoint call whose `actor` is empty without inspecting
+it. The provider is a second owned peer. The proxy, Postgres and Redis run for real with two workers.
+"""
+
+import json
+import threading
+import uuid
+from collections.abc import Iterator
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from itertools import product
+from pathlib import Path
+from typing import Final
+from urllib.parse import urlsplit
+
+import httpx
+import pytest
+import yaml
+from integration._support.client import Gateway, JsonValue, gateway_from_environment
+from integration._support.process import owned_proxy_process
+from integration._support.wire import Reply, Request, Wire, wire_server
+
+pytestmark: Final = pytest.mark.shares_worker_state
+
+API_KEY: Final = "synthetic-ovalix-key"
+APP_ID: Final = "synthetic-ovalix-app"
+OTHER_APP: Final = "Synthetic Other App"
+PRE_CHECKPOINT: Final = "synthetic-pre-checkpoint"
+POST_CHECKPOINT: Final = "synthetic-post-checkpoint"
+KEYWORD: Final = "Project Falcon"
+NAME: Final = "David"
+DENIAL: Final = "synthetic ovalix denial"
+REPLY: Final = "synthetic provider reply"
+PINNED: Final = "ovalix-pinned"
+ENDPOINTS: Final = ("/v1/chat/completions", "/v1/messages", "/v1/responses")
+CHECKPOINT: Final = "/tracking/beta/checkpoint"
+OUTAGE_AFTER: Final = 6
+
+
+class _Outage:
+    """Takes the tracker double down once it has answered a set number of checkpoint calls."""
+
+    def __init__(self) -> None:
+        self._lock: Final = threading.Lock()
+        self._remaining: int | None = None
+
+    def arm(self, answered: int) -> None:
+        with self._lock:
+            self._remaining = answered
+
+    def clear(self) -> None:
+        with self._lock:
+            self._remaining = None
+
+    def admits(self, path: str) -> bool:
+        with self._lock:
+            if self._remaining is None:
+                return True
+            if self._remaining == 0:
+                return False
+            if path == CHECKPOINT:
+                self._remaining -= 1
+            return True
+
+
+OUTAGE: Final = _Outage()
+
+
+def _checkpoint_verdict(body: dict[str, object]) -> dict[str, object]:
+    data: Final = body["data"]
+    content: Final = str(data.get("content", "")) if isinstance(data, dict) else ""
+    if not body["actor"]:
+        return {"action_type": "allow"}
+    if KEYWORD in content:
+        return {"action_type": "block", "modified_data": {"content": DENIAL}}
+    if NAME in content:
+        return {"action_type": "anonymize", "modified_data": {"content": content.replace(NAME, "{Name}")}}
+    return {"action_type": "allow"}
+
+
+def _tracker(request: Request) -> Reply:
+    if request.headers.get("x-api-key") != API_KEY:
+        return Reply(status=401, body=b'{"detail": "missing api key"}')
+    path: Final = urlsplit(request.target).path
+    if not OUTAGE.admits(path):
+        return Reply(status=503, body=b'{"detail": "tracker unavailable"}')
+    if path != CHECKPOINT:
+        return Reply(status=404, body=b'{"detail": "not found"}')
+    return Reply(body=json.dumps(_checkpoint_verdict(json.loads(request.body))).encode())
+
+
+def _responses_reply(marker: str) -> dict[str, object]:
+    return {
+        "id": "resp_" + marker,
+        "object": "response",
+        "created_at": 1,
+        "status": "completed",
+        "model": "gpt-4o-mini",
+        "output": [
+            {
+                "type": "message",
+                "id": "msg_" + marker,
+                "status": "completed",
+                "role": "assistant",
+                "content": [{"type": "output_text", "text": REPLY, "annotations": []}],
+            }
+        ],
+        "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
+    }
+
+
+def _chat_reply(marker: str) -> dict[str, object]:
+    return {
+        "id": "chatcmpl-" + marker,
+        "object": "chat.completion",
+        "created": 1,
+        "model": "gpt-4o-mini",
+        "choices": [{"index": 0, "message": {"role": "assistant", "content": REPLY}, "finish_reason": "stop"}],
+        "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+    }
+
+
+def _provider(request: Request) -> Reply:
+    marker: Final = uuid.uuid4().hex
+    is_responses: Final = urlsplit(request.target).path.endswith("/responses")
+    return Reply(body=json.dumps(_responses_reply(marker) if is_responses else _chat_reply(marker)).encode())
+
+
+def _alias() -> str:
+    return f"[{OTHER_APP}] integration key {uuid.uuid4().hex}"
+
+
+def _ovalix(name: str, tracker: str, **routing: str) -> dict[str, object]:
+    return {
+        "guardrail_name": name,
+        "litellm_params": {
+            "guardrail": "ovalix",
+            "mode": "pre_call",
+            "default_on": False,
+            "tracker_api_base": tracker,
+            "tracker_api_key": API_KEY,
+            **routing,
+        },
+    }
+
+
+@dataclass(frozen=True, slots=True)
+class Keys:
+    no_user: str
+    user_id: str
+    with_user_id: str
+    email: str
+    with_email: str
+
+
+@dataclass(frozen=True, slots=True)
+class Rig:
+    proxy: Gateway
+    tracker: Wire
+    provider: Wire
+    model: str
+    keys: Keys
+
+
+@pytest.fixture(scope="module")
+def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
+    root: Final = tmp_path_factory.mktemp("ovalix")
+    with (
+        gateway_from_environment() as gateway,
+        wire_server(_tracker) as tracker,
+        wire_server(_provider) as provider,
+    ):
+        config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
+        config["litellm_settings"]["cache"] = False
+        config["guardrails"] = [
+            _ovalix(
+                PINNED,
+                tracker.url,
+                application_id=APP_ID,
+                pre_checkpoint_id=PRE_CHECKPOINT,
+                post_checkpoint_id=POST_CHECKPOINT,
+            ),
+        ]
+        path: Final = root / "ovalix.yaml"
+        path.write_text(yaml.safe_dump(config))
+        with (
+            owned_proxy_process(gateway, root, {}, config=path, workers=2) as owned,
+            owned.gateway.scenario() as scenario,
+        ):
+            model: Final = scenario.model(
+                model="openai/gpt-4o-mini", api_base=provider.url + "/v1", api_key="synthetic-openai-key"
+            )
+            user_id: Final = scenario.user()
+            email: Final = f"ovalix-{uuid.uuid4().hex}@example.com"
+            keys: Final = Keys(
+                no_user=scenario.key(key_alias=_alias()),
+                user_id=user_id,
+                with_user_id=scenario.key(key_alias=_alias(), user_id=user_id),
+                email=email,
+                with_email=scenario.key(key_alias=_alias(), user_id=scenario.user(user_email=email)),
+            )
+            yield Rig(owned.gateway, tracker, provider, model, keys)
+
+
+def _post(rig: Rig, endpoint: str, key: str, text: str, extra: dict[str, JsonValue] | None = None) -> httpx.Response:
+    prompt: Final[dict[str, JsonValue]] = (
+        {"input": text}
+        if endpoint == "/v1/responses"
+        else {"max_tokens": 64, "messages": [{"role": "user", "content": text}]}
+        if endpoint == "/v1/messages"
+        else {"messages": [{"role": "user", "content": text}]}
+    )
+    return rig.proxy.request(
+        "POST", endpoint, {"model": rig.model, "guardrails": [PINNED], **prompt, **(extra or {})}, key=key
+    )
+
+
+def _send(rig: Rig, endpoint: str, key: str, text: str, extra: dict[str, JsonValue] | None = None) -> httpx.Response:
+    rig.tracker.drain()
+    rig.provider.drain()
+    return _post(rig, endpoint, key, text, extra)
+
+
+def _checkpoint_actors(rig: Rig) -> list[object]:
+    return [
+        json.loads(request.body)["actor"]
+        for request in rig.tracker.drain()
+        if urlsplit(request.target).path == CHECKPOINT
+    ]
+
+
+@pytest.mark.parametrize("stream", (False, True))
+@pytest.mark.parametrize("endpoint", ENDPOINTS)
+def test_a_prompt_from_a_key_with_no_user_is_inspected(rig: Rig, endpoint: str, stream: bool) -> None:
+    response: Final = _send(
+        rig, endpoint, rig.keys.no_user, f"Tell me about {KEYWORD} {uuid.uuid4()}", {"stream": stream}
+    )
+
+    assert response.status_code == 400 and DENIAL in response.text, response.text
+    assert rig.provider.drain() == ()
+    assert _checkpoint_actors(rig) == ["unknown"]
+
+
+@pytest.mark.parametrize(
+    ("endpoint", "client_metadata"),
+    (("/v1/messages", {"user_id": "client-chosen"}), ("/v1/responses", {"client": "chosen"})),
+)
+def test_a_client_metadata_object_does_not_switch_inspection_off(
+    rig: Rig, endpoint: str, client_metadata: dict[str, JsonValue]
+) -> None:
+    response: Final = _send(
+        rig,
+        endpoint,
+        rig.keys.with_user_id,
+        f"Tell me about {KEYWORD} {uuid.uuid4()}",
+        {"metadata": client_metadata},
+    )
+
+    assert response.status_code == 400 and DENIAL in response.text, response.text
+    assert rig.provider.drain() == ()
+    assert _checkpoint_actors(rig) == ["unknown"]
+
+
+def test_a_name_from_a_key_with_no_user_reaches_the_provider_anonymized(rig: Rig) -> None:
+    marker: Final = uuid.uuid4().hex
+    response: Final = _send(rig, "/v1/chat/completions", rig.keys.no_user, f"My name is {NAME} {marker}")
+
+    assert response.status_code == 200, response.text
+    sent: Final = [json.loads(request.body)["messages"][-1]["content"] for request in rig.provider.drain()]
+    assert sent == [f"My name is {{Name}} {marker}"]
+
+
+def test_the_actor_is_the_key_users_email_then_their_id(rig: Rig) -> None:
+    by_id: Final = _send(rig, "/v1/chat/completions", rig.keys.with_user_id, f"hello {uuid.uuid4()}")
+    assert by_id.status_code == 200, by_id.text
+    assert _checkpoint_actors(rig) == [rig.keys.user_id]
+
+    by_email: Final = _send(rig, "/v1/chat/completions", rig.keys.with_email, f"hello {uuid.uuid4()}")
+    assert by_email.status_code == 200, by_email.text
+    assert _checkpoint_actors(rig) == [rig.keys.email]
+
+
+@pytest.mark.parametrize("endpoint", ENDPOINTS)
+def test_a_key_alias_naming_another_app_does_not_change_the_application_checked(rig: Rig, endpoint: str) -> None:
+    response: Final = _send(rig, endpoint, rig.keys.with_email, f"Tell me about {KEYWORD} {uuid.uuid4()}")
+
+    assert response.status_code == 400 and DENIAL in response.text, response.text
+    assert rig.provider.drain() == ()
+    calls: Final = [(urlsplit(request.target).path, json.loads(request.body)) for request in rig.tracker.drain()]
+    assert [
+        (path, body["application_id"], body["checkpoint_id"], "application_name" in body) for path, body in calls
+    ] == [(CHECKPOINT, APP_ID, PRE_CHECKPOINT, False)]
+
+
+def _burst(rig: Rig) -> tuple[httpx.Response, ...]:
+    plan: Final = tuple(product(ENDPOINTS, (False, True), range(4)))
+    with ThreadPoolExecutor(max_workers=12) as pool:
+        return tuple(
+            pool.map(
+                lambda cell: _post(
+                    rig,
+                    cell[0],
+                    rig.keys.no_user,
+                    f"Tell me about {KEYWORD} {uuid.uuid4()}",
+                    {"stream": cell[1]},
+                ),
+                plan,
+            )
+        )
+
+
+def test_a_tracker_outage_mid_burst_fails_closed_and_recovers(rig: Rig) -> None:
+    rig.tracker.drain()
+    rig.provider.drain()
+    OUTAGE.arm(OUTAGE_AFTER)
+    try:
+        during: Final = _burst(rig)
+    finally:
+        OUTAGE.clear()
+
+    denied: Final = [response for response in during if response.status_code == 400 and DENIAL in response.text]
+    failed: Final = [response for response in during if response.status_code >= 400 and DENIAL not in response.text]
+    assert len(denied) == OUTAGE_AFTER, [response.text for response in during]
+    assert len(failed) == len(during) - OUTAGE_AFTER, [response.text for response in during]
+    assert all("Ovalix guardrail error" in response.text for response in failed), [response.text for response in failed]
+    assert rig.provider.drain() == ()
+    assert set(_checkpoint_actors(rig)) == {"unknown"}
+
+    after: Final = _burst(rig)
+
+    assert all(response.status_code == 400 and DENIAL in response.text for response in after), [
+        response.text for response in after
+    ]
+    assert rig.provider.drain() == ()
+    assert _checkpoint_actors(rig) == ["unknown"] * len(after)

tests/integration/sdk/test_router_concurrent_deployment_delete_sdk.py

@@ -8,7 +8,7 @@ import threading
 import uuid
 from collections.abc import AsyncIterable, Awaitable, Callable, Iterable
 from concurrent.futures import ThreadPoolExecutor
-from types import FrameType
+from types import FrameType, MappingProxyType
 from typing import Final, TypeVar
 
 import pytest
@@ -28,12 +28,23 @@ _BURST_THREADS: Final = 8
 _BURST_CALLS: Final = 24
 _CHURN_UPSERTS: Final = 120
 _UNREPORTED_MODEL_ID: Final = frozenset({"aresponses-stream", "aanthropic-messages"})
-_ROUTER_FILE: Final = Router.__init__.__code__.co_filename
-_INDEX_LOCK_LINES: Final = frozenset(
-    number
-    for number, line in enumerate(linecache.getlines(_ROUTER_FILE), start=1)
-    if "with self._deployment_index_lock" in line
-)
+_REINDEX: Final = Router._update_deployment_indices_after_removal.__code__
+
+
+def _router_source_files() -> frozenset[str]:
+    facade_files: Final = (sys.modules[cls.__module__].__file__ for cls in Router.__mro__ if cls is not object)
+    return frozenset({_REINDEX.co_filename, *(path for path in facade_files if path is not None)})
+
+
+def _lock_lines(path: str) -> frozenset[int]:
+    return frozenset(
+        number
+        for number, line in enumerate(linecache.getlines(path), start=1)
+        if "with self._deployment_index_lock" in line
+    )
+
+
+_INDEX_LOCK_LINES: Final = MappingProxyType({path: _lock_lines(path) for path in _router_source_files()})
 
 T = TypeVar("T")
 
@@ -80,7 +91,6 @@ def _read_during_held_write(read: Callable[[], T], write: Callable[[], object])
     """Run `read` on its own thread while `write` waits where the list has shrunk but the index maps have not caught
     up. The write resumes once `read` returns or reaches the router's index lock, so a read that skips the lock meets
     the half-updated list and a read that takes the lock waits for the write"""
-    reindex: Final = Router._update_deployment_indices_after_removal.__code__
     held: Final = threading.Event()
     release: Final = threading.Event()
     settled: Final = threading.Event()
@@ -89,7 +99,7 @@ def _read_during_held_write(read: Callable[[], T], write: Callable[[], object])
     failure: Final[list[BaseException]] = []
 
     def hold_at_reindex(frame: FrameType, event: str, _arg: object) -> None:
-        if event == "call" and frame.f_code is reindex and not held.is_set():
+        if event == "call" and frame.f_code is _REINDEX and not held.is_set():
             held.set()
             released_in_time.append(release.wait(timeout=_SYNC_SECONDS))
 
@@ -102,9 +112,10 @@ def _read_during_held_write(read: Callable[[], T], write: Callable[[], object])
             sys.setprofile(previous)
 
     def watch_read(frame: FrameType, event: str, _arg: object) -> Callable[..., object] | None:
-        if frame.f_code.co_filename != _ROUTER_FILE:
+        lock_lines: Final = _INDEX_LOCK_LINES.get(frame.f_code.co_filename)
+        if lock_lines is None:
             return None
-        if event == "line" and frame.f_lineno in _INDEX_LOCK_LINES:
+        if event == "line" and frame.f_lineno in lock_lines:
             settled.set()
         return watch_read
 

tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py

@@ -152,15 +152,14 @@ def _place_delta(partial_json: str) -> SSEFrame:
     )
 
 
-def _place_chunk(delta: dict[str, JsonValue], *, finish_reason: str | None = None) -> SSEFrame:
-    finish: Final = {} if finish_reason is None else {"finish_reason": finish_reason}
+def _place_chunk(delta: dict[str, JsonValue]) -> SSEFrame:
     return SSEFrame(
         data={
             "id": ANY,
             "object": "chat.completion.chunk",
             "created": ANY,
             "model": "anthropic/claude-haiku-5-5",
-            "choices": [{"index": 0, "delta": delta, **finish}],
+            "choices": [{"index": 0, "delta": delta}],
         }
     )
 
@@ -245,7 +244,19 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationT
         _place_chunk({"content": 'try": "'}),
         _place_chunk({"content": 'France"'}),
         _place_chunk({"content": "}"}),
-        _place_chunk({}, finish_reason="stop"),
+        SSEFrame(
+            data={
+                "error": {
+                    "message": (
+                        "litellm.APIConnectionError: AnthropicException - "
+                        "Stream ended before a finish_reason was received"
+                    ),
+                    "type": "internal_server_error",
+                    "param": None,
+                    "code": "500",
+                }
+            }
+        ),
         SSEFrame(data="[DONE]"),
     ),
     expected_litellm_response={
@@ -257,7 +268,6 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationT
             {
                 "index": 0,
                 "message": {"role": "assistant", "content": '{"city": "Paris", "country": "France"}'},
-                "finish_reason": "stop",
             }
         ],
     },

tests/integration/translation/runner.py

@@ -129,8 +129,9 @@ def _join_stream_frames(endpoint: str, frames: tuple[SSEFrame, ...]) -> Mapping[
         case "/v1/chat/completions":
             state: Final = ChatCompletionStreamState()
             for frame in frames:
-                if frame.data != "[DONE]":
-                    state.handle_chunk(ChatCompletionChunk.model_validate(frame.data))
+                if frame.data == "[DONE]" or (isinstance(frame.data, Mapping) and "error" in frame.data):
+                    continue
+                state.handle_chunk(ChatCompletionChunk.model_validate(frame.data))
             return JSON_OBJECT.validate_python(state.get_final_completion().model_dump(mode="json", exclude_none=True))
         case _:
             raise AssertionError(f"no stream accumulator for {endpoint}")

tests/unit/caching/test_request_redis_batch_pre_call.py

@@ -290,7 +290,11 @@ def _router(redis_cache: FakeRedisCache, routing_strategy: str = "usage-based-ro
 
 
 @pytest.mark.asyncio
-async def test_armed_routing_read_rides_the_admission_pipeline_and_routing_issues_no_read_of_its_own():
+async def test_armed_routing_read_rides_the_admission_pipeline_and_routing_issues_no_read_of_its_own(monkeypatch):
+    monkeypatch.setattr(
+        "litellm.router_strategy.lowest_tpm_rpm_v2.get_utc_datetime",
+        lambda: datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc),
+    )
     client = FakeClient(_lua_ok_replies)
     redis_cache = FakeRedisCache(client)
     router = _router(redis_cache)

tests/unit/enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py

@@ -29,7 +29,7 @@ async def test_apply_guardrail_endpoint_returns_correct_response(
         mock_proxy_logging_ctx(),
     ):
         # Create a mock guardrail
-        mock_guardrail = Mock(spec=CustomGuardrail)
+        mock_guardrail = Mock(spec=CustomGuardrail, event_hook=None)
         # Apply guardrail returns GenericGuardrailAPIInputs (dict with texts key)
         mock_guardrail.apply_guardrail = AsyncMock(
             return_value={"texts": ["Redacted text: [REDACTED] and [REDACTED]"]}
@@ -116,7 +116,7 @@ async def test_apply_guardrail_endpoint_with_presidio_guardrail(mock_proxy_loggi
         mock_proxy_logging_ctx(),
     ):
         # Create a mock guardrail that simulates Presidio behavior
-        mock_guardrail = Mock(spec=CustomGuardrail)
+        mock_guardrail = Mock(spec=CustomGuardrail, event_hook=None)
         # Simulate masking PII entities - returns GenericGuardrailAPIInputs (dict with texts key)
         mock_guardrail.apply_guardrail = AsyncMock(
             return_value={
@@ -168,7 +168,7 @@ async def test_apply_guardrail_endpoint_without_optional_params(mock_proxy_loggi
         mock_proxy_logging_ctx(),
     ):
         # Create a mock guardrail
-        mock_guardrail = Mock(spec=CustomGuardrail)
+        mock_guardrail = Mock(spec=CustomGuardrail, event_hook=None)
         # Returns GenericGuardrailAPIInputs (dict with texts key)
         mock_guardrail.apply_guardrail = AsyncMock(
             return_value={"texts": ["Processed text"]}

tests/unit/integration_support/test_upstream.py

@@ -112,6 +112,39 @@ async def test_mcp_notification_ordering_preserves_other_responses(
     assert outgoing == list(messages)
 
 
+@pytest.mark.asyncio
+async def test_mcp_http_disconnect_cancels_an_in_flight_handler(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2]))
+    from integration._support.mcp import _cancel_on_disconnect
+    from starlette.types import Receive, Send
+
+    started: Final = asyncio.Event()
+    stopped: Final = asyncio.Event()
+    body: Final[Message] = {"type": "http.request", "body": b"tool-call", "more_body": False}
+    pending: Final = iter((body,))
+
+    async def receive() -> Message:
+        message: Final = next(pending, None)
+        if message is not None:
+            return message
+        await started.wait()
+        return {"type": "http.disconnect"}
+
+    async def send(message: Message) -> None:
+        pytest.fail(f"Disconnected handler sent a response: {message}")
+
+    async def app(scope: Scope, receive: Receive, send: Send) -> None:
+        assert await receive() == body
+        started.set()
+        try:
+            await asyncio.Future[None]()
+        finally:
+            stopped.set()
+
+    await asyncio.wait_for(_cancel_on_disconnect(app)({"type": "http"}, receive, send), 2)
+    assert stopped.is_set(), "The disconnected HTTP handler was left running"
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("failure", (RuntimeError("dispatch failed"), asyncio.CancelledError()))
 async def test_mcp_notification_dispatch_failure_is_not_acknowledged(

tests/unit/litellm_core_utils/test_api_route_to_call_types.py

@@ -8,6 +8,8 @@ Regression coverage for the guardrail route table bugs:
   take call_types[0] to a handler-less type
 """
 
+from typing import Final
+
 from litellm.litellm_core_utils.api_route_to_call_types import (
     get_call_types_for_route,
 )
@@ -103,6 +105,16 @@ class TestExistingRouteResolutionUnchanged:
         assert call_types is not None
         assert list(call_types) == [CallTypes.asend_message, CallTypes.send_message]
 
+    def test_single_segment_placeholder_message_stream_resolves(self):
+        call_types: Final = get_call_types_for_route("/a2a/my-agent/message/stream")
+        assert call_types is not None
+        assert list(call_types) == [CallTypes.asend_message, CallTypes.send_message]
+
+    def test_v1_single_segment_placeholder_message_stream_resolves(self):
+        call_types: Final = get_call_types_for_route("/v1/a2a/my-agent/message/stream")
+        assert call_types is not None
+        assert list(call_types) == [CallTypes.asend_message, CallTypes.send_message]
+
     def test_longer_route_does_not_collapse_into_bare_placeholder_pattern(self):
         call_types = get_call_types_for_route("/responses/resp_123/input_items")
         assert call_types is not None

tests/unit/litellm_core_utils/test_json_fragment_accumulator.py

@@ -1,6 +1,5 @@
 import json
 import time
-import tracemalloc
 from collections.abc import Iterator
 from typing import Final
 from unittest.mock import patch
@@ -193,34 +192,15 @@ def _streamed(fragments: tuple[str, ...]) -> JSONFragmentAccumulator:
     return accumulator
 
 
-def _drain_peak_bytes(fragments: tuple[str, ...]) -> int:
-    accumulator: Final = _streamed(fragments)
-    first_found, _ = accumulator.pop_next_value()
-    assert first_found is True
-    already_tracing: Final = tracemalloc.is_tracing()
-    if not already_tracing:
-        tracemalloc.start()
-    tracemalloc.reset_peak()
-    baseline, _ = tracemalloc.get_traced_memory()
-    try:
-        drained: Final = sum(1 for _ in _popped_values(accumulator))
-        _, peak = tracemalloc.get_traced_memory()
-    finally:
-        if not already_tracing:
-            tracemalloc.stop()
-    assert drained == len(fragments) - 1
-    return peak - baseline
-
-
-def test_draining_many_concatenated_values_never_copies_the_remainder():
+def test_draining_many_concatenated_values_yields_every_value_in_order():
     value: Final = '{"text": "' + "x" * 20 + '"}'
     fragments: Final = tuple(value if index % 2 else f" {value}" for index in range(40_000))
-    payload_bytes: Final = sum(len(fragment) for fragment in fragments)
-    peak_bytes: Final = _drain_peak_bytes(fragments)
-    assert peak_bytes < payload_bytes // 4, (
-        f"draining {len(fragments)} values allocated a peak of {peak_bytes} bytes over a {payload_bytes} byte payload, "
-        "copying the remainder on every pop?"
-    )
+    accumulator: Final = _streamed(fragments)
+
+    drained: Final = tuple(_popped_values(accumulator))
+
+    assert drained == (json.loads(value),) * len(fragments)
+    assert accumulator.pop_next_value() == (False, None)
 
 
 def test_could_close_json_after_many_blank_fragments_is_not_quadratic():

tests/unit/litellm_core_utils/test_streaming_handler.py

@@ -7176,6 +7176,80 @@ def test_rebuilding_a_priced_stream_bills_the_margin_once(monkeypatch: pytest.Mo
     )
 
 
+@pytest.mark.parametrize("sync_mode", [True, False])
+@pytest.mark.parametrize("has_served_model", [True, False])
+@pytest.mark.parametrize(
+    ("wrapper_provider", "logging_provider"),
+    [("openrouter", "openrouter"), ("cached_response", "openrouter"),
+     ("openai", "openai"), ("cached_response", "openai")],
+)
+@pytest.mark.asyncio
+async def test_openrouter_served_model_survives_blank_chunks_usage_and_cache_replay(
+    sync_mode: bool, has_served_model: bool, wrapper_provider: str, logging_provider: str
+) -> None:
+    selected_model: Final = "selected-model" if has_served_model else ""
+    expected_model: Final = selected_model if logging_provider == "openrouter" and selected_model else "auto"
+    logging_obj: Final = Logging(
+        model="auto",
+        messages=[{"role": "user", "content": "Say hi"}],
+        stream=True,
+        call_type="completion" if sync_mode else "acompletion",
+        start_time=1.0,
+        litellm_call_id="openrouter-model-identity",
+        function_id="test",
+    )
+    logging_obj.stream_options = {"include_usage": True}
+    logging_obj.model_call_details["custom_llm_provider"] = (
+        None if wrapper_provider == "cached_response" else logging_provider
+    )
+    chunks: Final = [
+        ModelResponseStream(
+            id="served-stream", created=1, model=None,
+            choices=[StreamingChoices(index=0, delta=Delta(role="assistant"))],
+        ),
+        ModelResponseStream(
+            id="served-stream", created=1, model=selected_model,
+            choices=[StreamingChoices(index=0, delta=Delta(content="hello"))],
+            usage=Usage(prompt_tokens=3, completion_tokens=2, total_tokens=5),
+        ),
+        ModelResponseStream(
+            id="served-stream", created=1, model=None,
+            choices=[StreamingChoices(index=0, delta=Delta(content=" world"))],
+        ),
+        ModelResponseStream(
+            id="served-stream", created=1, model="",
+            choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")],
+        ),
+        ModelResponseStream(
+            id="served-stream", created=1, model=None, choices=[],
+            usage=Usage(prompt_tokens=3, completion_tokens=2, total_tokens=5),
+        ),
+    ]
+    wrapper: Final = CustomStreamWrapper(
+        completion_stream=ModelResponseListIterator(model_responses=chunks),
+        model="auto", logging_obj=logging_obj, custom_llm_provider=wrapper_provider,
+        stream_options={"include_usage": True},
+    )
+    logging_obj.update_environment_variables(
+        litellm_params={}, optional_params={}, model="auto", custom_llm_provider=logging_provider,
+    )
+    first: Final = next(wrapper) if sync_mode else await wrapper.__anext__()
+    second: Final = next(wrapper) if sync_mode else await wrapper.__anext__()
+    partial: Final = wrapper.partial_response()
+    emitted: Final = [first, second, *(await _collect(wrapper, sync_mode))]
+    assembled: Final = litellm.stream_chunk_builder(chunks=emitted)
+
+    assert first.model == "auto"
+    assert {chunk.model for chunk in emitted[1:]} == {expected_model}
+    assert partial is not None and partial.model == expected_model
+    assert assembled is not None and assembled.model == expected_model
+    assert assembled.choices[0].message.content == "hello world"
+    assert emitted[-1].choices == []
+    assert emitted[-1].usage is not None and emitted[-1].usage.total_tokens == 5
+    assert second._hidden_params.get("provider_response_model") == (selected_model or None)
+    assert wrapper.model == logging_obj.model == logging_obj.model_call_details["model"] == "auto"
+
+
 def test_chunk_creator_keeps_provider_model_private_across_stream():
     from litellm.litellm_core_utils.add_retry_fallback_headers import (
         get_hidden_params_dict,

tests/unit/llms/custom_httpx/test_aiohttp_handler.py

@@ -307,20 +307,12 @@ class TestBaseLLMAIOHTTPHandler:
         # Should not call aiohttp.ClientSession directly
         mock_client_session.assert_not_called()
 
-    @patch("aiohttp.ClientSession")
-    def test_create_client_session_with_connector(self, mock_client_session):
-        """Test session creation using connector"""
-        mock_connector = Mock(spec=aiohttp.BaseConnector)
-        mock_session_instance = Mock()
-        mock_client_session.return_value = mock_session_instance
-
-        handler = BaseLLMAIOHTTPHandler(connector=mock_connector)
-
-        result = handler._create_client_session_with_transport()
-
-        # Should create session with connector
-        mock_client_session.assert_called_once_with(connector=mock_connector)
-        assert result is mock_session_instance
+    @pytest.mark.asyncio
+    async def test_create_client_session_with_connector(self) -> None:
+        connector: Final = aiohttp.TCPConnector()
+        handler: Final = BaseLLMAIOHTTPHandler(connector=connector)
+        async with handler._create_client_session_with_transport() as session:
+            assert session.connector is connector
 
     @pytest.mark.asyncio
     async def test_create_client_session_default_honors_global_ssl_verify_false(

tests/unit/llms/custom_httpx/test_http_handler.py

@@ -1,13 +1,16 @@
 import asyncio, importlib, json, time
 import gc
 import io
+import logging
 import os
 import pathlib
 import ssl
 import threading
 import weakref
-from collections.abc import Callable, Mapping
+from collections.abc import AsyncIterator, Callable, Mapping
 from concurrent.futures import ThreadPoolExecutor
+from contextlib import asynccontextmanager
+from contextvars import Context
 from typing import Final
 from unittest.mock import MagicMock, patch
 
@@ -15,7 +18,10 @@ import certifi
 import httpx
 import pytest
 import respx
-from aiohttp import ClientSession, TCPConnector
+from aiohttp import ClientResponse, ClientSession, RequestInfo, TCPConnector
+from aiohttp.tracing import Trace
+from multidict import CIMultiDict, CIMultiDictProxy
+from yarl import URL
 
 import litellm
 from litellm.caching.llm_caching_handler import LLMClientCache
@@ -2304,3 +2310,217 @@ async def test_async_timeout_reports_the_per_call_timeout_as_a_float_when_one_is
         await handler.close()
 
     assert "Read timed out. Timeout passed=30.0, time taken=" in str(raised.value), str(raised.value)
+
+
+class _PoolDiagnosticClock:
+    def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
+        self.now = 1000.0
+        self.loop = loop
+        self.schedule = loop.call_later
+        self.timers: list[asyncio.TimerHandle] = []
+
+    def time(self) -> float:
+        return self.now
+
+    def call_later(
+        self, delay: float, callback: Callable[..., object],
+        *args: object, context: Context | None = None,
+    ) -> asyncio.TimerHandle:
+        timer: Final = self.schedule(delay, callback, *args, context=context)
+        self.timers.append(timer)
+        return timer
+
+    async def advance(self, seconds: float) -> None:
+        self.now += seconds
+        checkpoint: Final[asyncio.Future[None]] = self.loop.create_future()
+        self.loop.call_soon(checkpoint.set_result, None)
+        await checkpoint
+
+
+def _pool_diagnostic_clock(monkeypatch: pytest.MonkeyPatch) -> _PoolDiagnosticClock:
+    loop: Final = asyncio.get_running_loop()
+    clock: Final = _PoolDiagnosticClock(loop)
+    monkeypatch.setattr(loop, "time", clock.time)
+    monkeypatch.setattr(loop, "call_later", clock.call_later)
+    return clock
+
+
+@asynccontextmanager
+async def _pool_diagnostic_session(source: str) -> AsyncIterator[ClientSession]:
+    if source == "proxy":
+        from litellm.proxy.proxy_server import initialize_shared_aiohttp_session
+
+        session: Final = await initialize_shared_aiohttp_session()
+        assert session is not None
+        async with session:
+            yield session
+        return
+    if source == "native-connector":
+        from litellm.llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler
+
+        handler: Final = BaseLLMAIOHTTPHandler(connector=TCPConnector(limit=7, limit_per_host=2))
+        try:
+            yield handler._get_async_client_session()
+        finally:
+            await handler.close()
+        return
+    transport: Final = AsyncHTTPHandler.create_aiohttp_transport()
+    if source == "rebuilt":
+        await transport.get_valid_client_session().close()
+    try:
+        yield transport.get_valid_client_session()
+    finally:
+        await transport.aclose()
+
+
+async def _pool_diagnostic_request(session: ClientSession, url: URL) -> tuple[Trace, ...]:
+    traces: Final = tuple(Trace(session, config, config.trace_config_ctx()) for config in session.trace_configs)
+    for trace in traces:
+        await trace.send_request_start("POST", url, CIMultiDict())
+    return traces
+
+
+def _pool_diagnostic_messages(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]:
+    return tuple(record.getMessage() for record in caplog.records if "connection pool" in record.getMessage())
+
+
+@pytest.mark.parametrize("source", ["sdk", "proxy", "native-connector", "rebuilt"])
+@pytest.mark.parametrize("error", [TimeoutError("private timeout"), asyncio.CancelledError("private cancellation")])
+async def test_aiohttp_pool_failure_identifies_queue_wait_and_cleans_up_timer(
+    source: str, error: BaseException, monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    pool_diagnostic_clock: Final = _pool_diagnostic_clock(monkeypatch)
+    caplog.set_level(logging.WARNING, logger="LiteLLM")
+    url: Final = URL("https://private-user:private-password@pool.test:8443/private-path?token=private-query")
+    async with _pool_diagnostic_session(source) as session:
+        connector: Final = session.connector
+        assert connector is not None
+        traces: Final = await _pool_diagnostic_request(session, url)
+        for trace in traces:
+            await trace.send_connection_queued_start()
+        await pool_diagnostic_clock.advance(0.25)
+        for trace in traces:
+            await trace.send_request_exception("POST", url, CIMultiDict(), error)
+        messages: Final = _pool_diagnostic_messages(caplog)
+        assert len(messages) == 1, messages
+        message: Final = messages[0]
+        assert "host=pool.test" in message
+        assert "port=8443" in message
+        assert "0.25" in message
+        assert "acquired_total=0" in message
+        assert f"limit={connector.limit}" in message
+        assert f"limit_per_host={connector.limit_per_host}" in message
+        assert "private" not in message
+        assert all(timer.cancelled() for timer in pool_diagnostic_clock.timers)
+        await pool_diagnostic_clock.advance(2.0)
+        assert _pool_diagnostic_messages(caplog) == messages
+
+
+@pytest.mark.parametrize("queued", [False, True])
+async def test_aiohttp_pool_diagnostics_do_not_label_slow_upstream_as_pool_saturation(
+    queued: bool, monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    pool_diagnostic_clock: Final = _pool_diagnostic_clock(monkeypatch)
+    caplog.set_level(logging.WARNING, logger="LiteLLM")
+    url: Final = URL("https://healthy.test/completions")
+    async with _pool_diagnostic_session("sdk") as session:
+        traces: Final = await _pool_diagnostic_request(session, url)
+        if queued:
+            for trace in traces:
+                await trace.send_connection_queued_start()
+            await pool_diagnostic_clock.advance(0.25)
+            for trace in traces:
+                await trace.send_connection_queued_end()
+        await pool_diagnostic_clock.advance(2.0)
+        for trace in traces:
+            await trace.send_request_exception("POST", url, CIMultiDict(), TimeoutError("slow upstream"))
+        assert _pool_diagnostic_messages(caplog) == ()
+        assert all(timer.cancelled() for timer in pool_diagnostic_clock.timers)
+
+
+async def test_aiohttp_pool_warns_while_queued_once_per_request_with_isolated_concurrent_waits(
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture,
+) -> None:
+    pool_diagnostic_clock: Final = _pool_diagnostic_clock(monkeypatch)
+    caplog.set_level(logging.WARNING, logger="LiteLLM")
+    slow_url: Final = URL("https://blocked.test/completions")
+    fast_url: Final = URL("https://available.test/completions")
+    async with _pool_diagnostic_session("sdk") as session:
+        blocked: Final = await _pool_diagnostic_request(session, slow_url)
+        available: Final = await _pool_diagnostic_request(session, fast_url)
+        for trace in blocked + available:
+            await trace.send_connection_queued_start()
+        await pool_diagnostic_clock.advance(0.25)
+        for trace in available:
+            await trace.send_connection_queued_end()
+        for trace in blocked:
+            await trace.send_connection_queued_end()
+            await trace.send_connection_queued_start()
+        await pool_diagnostic_clock.advance(0.8)
+        messages: Final = _pool_diagnostic_messages(caplog)
+        assert len(messages) == 1, messages
+        assert "host=blocked.test" in messages[0]
+        assert "1.05" in messages[0]
+        assert "available.test" not in messages[0]
+        for trace in blocked:
+            await trace.send_connection_queued_end()
+            await trace.send_connection_queued_start()
+        await pool_diagnostic_clock.advance(2.0)
+        for trace in blocked:
+            await trace.send_request_exception("POST", slow_url, CIMultiDict(), TimeoutError())
+        for trace in available:
+            await trace.send_request_exception("POST", fast_url, CIMultiDict(), TimeoutError())
+        await pool_diagnostic_clock.advance(2.0)
+        assert _pool_diagnostic_messages(caplog) == messages
+        assert all(timer.cancelled() for timer in pool_diagnostic_clock.timers)
+
+
+@pytest.mark.parametrize(("header", "location", "host", "port"), [
+    ("Location", "https://redirected.test:9443/private-path?token=private-query", "redirected.test", 9443),
+    ("Location", "/private-path?token=private-query", "initial.test", 8443),
+    ("URI", "https://redirected.test/private-path", "redirected.test", 443),
+    ("Location", "http://[invalid", None, None),
+])
+async def test_aiohttp_pool_warning_tracks_redirect_destination_without_masking_invalid_redirects(
+    header: str, location: str, host: str | None, port: int | None,
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture,
+) -> None:
+    from aiohttp.client_proto import ResponseHandler
+    from aiohttp.helpers import TimerNoop
+    from aiohttp.http_writer import StreamWriter
+
+    clock: Final = _pool_diagnostic_clock(monkeypatch)
+    caplog.set_level(logging.WARNING, logger="LiteLLM")
+    url: Final = URL("https://initial.test:8443/completions")
+    async with _pool_diagnostic_session("sdk") as session:
+        traces: Final = await _pool_diagnostic_request(session, url)
+        loop: Final = asyncio.get_running_loop()
+        response: Final = ClientResponse(
+            "POST", url, writer=None, continue100=None, timer=TimerNoop(),
+            request_info=RequestInfo(url, "POST", CIMultiDictProxy(CIMultiDict()), url),
+            traces=[], loop=loop, session=session,
+            stream_writer=StreamWriter(ResponseHandler(loop), loop),
+        )
+        response._headers = CIMultiDictProxy(CIMultiDict({header: location}))
+        try:
+            for trace in traces:
+                await trace.send_request_redirect("POST", url, CIMultiDict(), response)
+            if host is None:
+                for trace in traces:
+                    await trace.send_request_exception("POST", url, CIMultiDict(), ValueError("invalid redirect"))
+                assert _pool_diagnostic_messages(caplog) == ()
+                return
+            for trace in traces:
+                await trace.send_connection_queued_start()
+            await clock.advance(0.25)
+            for trace in traces:
+                await trace.send_request_exception("POST", url, CIMultiDict(), TimeoutError())
+            messages: Final = _pool_diagnostic_messages(caplog)
+            assert len(messages) == 1, messages
+            assert f"host={host}" in messages[0]
+            assert f"port={port}" in messages[0]
+            assert "private" not in messages[0]
+        finally:
+            response.close()

tests/unit/llms/litellm_proxy/skills/test_skill_search.py

@@ -281,6 +281,36 @@ class TestSearchSkills:
         assert isinstance(outcome, SkillSearchNotConfigured)
         assert "skill_search_embedding_model" in outcome.reason
 
+    @pytest.mark.parametrize("embedding_model", ("", "   "), ids=("empty", "whitespace"))
+    @pytest.mark.asyncio
+    async def test_empty_embedding_model_is_not_configured(self, embedding_model: str) -> None:
+        router: Final = _embedding_router()
+        not_configured: Final = await search_skills(
+            "language translation",
+            SKILLS,
+            5,
+            router=router,
+            embedding_model=None,
+            index=SkillSearchIndex(),
+            user_api_key_dict=CALLER,
+            proxy_logging_obj=_pass_through_key_limits(),
+        )
+        outcome: Final = await search_skills(
+            "language translation",
+            SKILLS,
+            5,
+            router=router,
+            embedding_model=embedding_model,
+            index=SkillSearchIndex(),
+            user_api_key_dict=CALLER,
+            proxy_logging_obj=_pass_through_key_limits(),
+        )
+
+        assert isinstance(not_configured, SkillSearchNotConfigured)
+        assert isinstance(outcome, SkillSearchNotConfigured)
+        assert outcome.reason == not_configured.reason
+        router.aembedding.assert_not_awaited()
+
     @pytest.mark.asyncio
     async def test_no_router_is_not_configured(self) -> None:
         outcome = await search_skills(

tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py

@@ -11,7 +11,7 @@ import pytest
 import respx
 
 import litellm
-from litellm.llms.custom_httpx.http_handler import HTTPHandler
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
 from litellm.llms.openrouter.chat.transformation import (
     OpenRouterChatCompletionStreamingHandler,
@@ -742,7 +742,11 @@ def test_openrouter_completion_with_non_object_usage_raises_api_connection_error
 
 
 @respx.mock
-def test_openrouter_streaming_completion_assembles_text_and_finish_reason() -> None:
+@pytest.mark.parametrize("async_stream", [False, True])
+@pytest.mark.parametrize("requested_model", ["openrouter/openai/gpt-4o-mini", "openrouter/auto"])
+async def test_openrouter_streaming_completion_assembles_text_and_finish_reason(
+    async_stream: bool, requested_model: str
+) -> None:
     chunks: Final = (
         {
             "id": "gen-stream-model",
@@ -785,14 +789,20 @@ def test_openrouter_streaming_completion_assembles_text_and_finish_reason() -> N
         {"role": "system", "content": "You are a helpful assistant."},
         {"role": "user", "content": "How does a case reach the Supreme Court?"},
     ]
-    stream: Final = litellm.completion(
-        model="openrouter/openai/gpt-4o-mini",
+    kwargs: Final = dict(
+        model=requested_model,
         messages=messages,
         stream=True,
         max_tokens=50,
         api_key="test-openrouter-key",
     )
-    stream_chunks: Final = tuple(stream)
+    client: Final = AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport())
+    try:
+        stream: Final = await litellm.acompletion(**kwargs, client=client) if async_stream else litellm.completion(**kwargs)
+        stream_chunks: Final = tuple([chunk async for chunk in stream]) if async_stream else tuple(stream)
+    finally:
+        await client.client.aclose()
+    assert {chunk.model for chunk in stream_chunks} == {chunks[0]["model"]}
     assert stream_chunks[0].choices[0].delta.role == "assistant"
     assert tuple(
         chunk.choices[0].finish_reason for chunk in stream_chunks if chunk.choices[0].finish_reason is not None
@@ -802,6 +812,7 @@ def test_openrouter_streaming_completion_assembles_text_and_finish_reason() -> N
     ) == "The Supreme Court hears federal cases."
     assembled: Final = litellm.stream_chunk_builder(chunks=list(stream_chunks), messages=messages)
     assert assembled is not None
+    assert assembled.model == chunks[0]["model"]
     assert assembled.choices[0].message.content == "The Supreme Court hears federal cases."
     assert assembled.choices[0].finish_reason == "stop"
 

tests/unit/proxy/a2a/test_agent_card.py

@@ -8,6 +8,7 @@ from litellm.proxy.a2a.agent_card import (
     LITELLM_SECURITY_SCHEMES,
     merge_agent_card,
     normalize_protocol_version,
+    point_agent_card_at_proxy,
     resolve_served_protocol_version,
 )
 
@@ -15,6 +16,32 @@ PROXY_URL = "https://proxy.example/a2a/agent-xyz"
 PROXY_BASE = "https://proxy.example"
 
 
+def test_point_agent_card_at_proxy_returns_a_deep_copy() -> None:
+    card: dict[str, object] = {
+        "url": "http://internal/",
+        "metadata": {"nested": ["original"]},
+        "supportedInterfaces": [
+            {"url": "http://internal/", "nested": {"value": "original"}},
+            {"url": "http://secondary/"},
+        ],
+    }
+
+    pointed: dict[str, object] = point_agent_card_at_proxy(card, PROXY_URL)
+
+    assert pointed["url"] == PROXY_URL
+    assert pointed["metadata"] == card["metadata"]
+    assert pointed["metadata"] is not card["metadata"]
+    assert pointed["supportedInterfaces"] == [
+        {"url": PROXY_URL, "nested": {"value": "original"}},
+        {"url": "http://secondary/"},
+    ]
+    assert card["url"] == "http://internal/"
+    assert card["supportedInterfaces"] == [
+        {"url": "http://internal/", "nested": {"value": "original"}},
+        {"url": "http://secondary/"},
+    ]
+
+
 def _full_upstream_card() -> dict:
     return {
         "protocolVersion": "0.9",

tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py

@@ -12,19 +12,22 @@ import sys
 from collections.abc import AsyncIterator, Awaitable, Callable, Mapping
 from contextlib import AbstractContextManager, ExitStack
 from dataclasses import dataclass
+from datetime import datetime
+from types import MappingProxyType
 from typing import Final
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import httpx
 import pytest
 import respx
+from fastapi import HTTPException
 from fastapi.testclient import TestClient
 from pydantic import BaseModel, ConfigDict
 
 import litellm
 from litellm.integrations.custom_logger import CustomLogger
-from litellm.proxy._types import UserAPIKeyAuth
-from litellm.types.agents import AgentCaller
+from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth
+from litellm.types.agents import AgentCaller, AgentResponse
 
 AddLiteLLMData = Callable[..., Awaitable[dict[str, object]]]
 
@@ -2782,7 +2785,9 @@ async def test_forward_jsonrpc_sse_is_untouched_while_keepalives_are_unconfigure
 
 
 @pytest.mark.asyncio
-async def test_forward_jsonrpc_sse_keeps_unicode_line_separators_inside_an_event(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_forward_jsonrpc_sse_keeps_unicode_line_separators_inside_an_event(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
     monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", None)
     separated_text: Final = "before\u2028between\u2029after"
     event: Final = {
@@ -2944,6 +2949,202 @@ async def test_failed_upstream_stream_relays_its_body_only_when_that_is_a_json_o
     }
 
 
+class _RejectionRecorder(CustomLogger):
+    def __init__(self) -> None:
+        super().__init__()
+        self.failure_payloads: Final[list[Mapping[str, object]]] = []  # mutable-ok: collects callback events
+        self.hook_calls: Final[list[tuple[Mapping[str, object], Exception]]] = []  # mutable-ok: collects hook calls
+
+    async def async_log_failure_event(
+        self,
+        kwargs: dict[str, object],
+        response_obj: object,
+        start_time: datetime | None,
+        end_time: datetime | None,
+    ) -> None:
+        payload: Final = kwargs["standard_logging_object"]
+        assert isinstance(payload, Mapping)
+        self.failure_payloads.append(payload)
+
+    async def async_post_call_failure_hook(
+        self,
+        request_data: dict[str, object],
+        original_exception: Exception,
+        user_api_key_dict: UserAPIKeyAuth,
+        traceback_str: str | None = None,
+    ) -> HTTPException:
+        self.hook_calls.append((dict(request_data), original_exception))
+        return HTTPException(status_code=418, detail="transformed")
+
+
+_SPOOFED_BODY_FIELDS: Final = MappingProxyType(
+    {
+        "agent_id": "spoof",
+        "litellm_call_id": "spoofed-call-id",
+        "metadata": {"agent_id": "spoof", "user_api_key_alias": "spoof", "user_api_key_team_id": "spoof"},
+    }
+)
+
+
+def _rejection_request(path_agent_id: str, request_id: object) -> MagicMock:
+    req: Final = MagicMock()
+    req.headers = {}
+    req.json = AsyncMock(
+        return_value={
+            "jsonrpc": "2.0",
+            "id": request_id,
+            "method": "SendMessage",
+            "params": {**_HELLO_MESSAGE_PARAMS, "litellm_call_id": "spoofed-param-call-id"},
+            **_SPOOFED_BODY_FIELDS,
+        }
+    )
+    return req
+
+
+async def _invoke_rejected(
+    monkeypatch: pytest.MonkeyPatch,
+    *,
+    registered: bool,
+    proxy_logging_obj: object,
+    request_id: object = "rej-1",
+) -> tuple[int, object]:
+    from litellm.proxy import proxy_server
+    from litellm.proxy.agent_endpoints import agent_registry
+    from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a
+
+    path_agent_id: Final = "denied-agent" if registered else "no-such-agent"
+    registry: Final = agent_registry.AgentRegistry()
+    if registered:
+        registry.register_agent(
+            AgentResponse(
+                agent_id="registered-denied-id",
+                agent_name=path_agent_id,
+                agent_card_params={"name": "Test Agent", "url": "https://agent.example.com"},
+            )
+        )
+    monkeypatch.setattr(agent_registry, "global_agent_registry", registry)
+    monkeypatch.setattr(proxy_server, "prisma_client", None)
+    monkeypatch.setattr(proxy_server, "proxy_logging_obj", proxy_logging_obj)
+    auth: Final = UserAPIKeyAuth(
+        api_key="hashed-caller",
+        key_alias="caller-alias",
+        user_id="caller-user",
+        team_id="caller-team",
+        object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="caller-grants", agents=["other-agent"]),
+        request_route=f"/a2a/{path_agent_id}",
+    )
+    with respx.mock as upstream:
+        try:
+            response: Final = await invoke_agent_a2a(
+                agent_id=path_agent_id,
+                request=_rejection_request(path_agent_id, request_id),
+                fastapi_response=MagicMock(),
+                user_api_key_dict=auth,
+            )
+        except HTTPException as e:
+            assert not upstream.calls
+            return e.status_code, e.detail
+        assert not upstream.calls
+    return response.status_code, json.loads(response.body.decode())
+
+
+_EXPECTED_REJECTIONS: Final = (
+    pytest.param(
+        True,
+        403,
+        "Agent 'denied-agent' is not allowed for your key/team. Contact proxy admin for access.",
+        "registered-denied-id",
+        "a2a_agent/Test Agent",
+        id="not-granted",
+    ),
+    pytest.param(
+        False,
+        404,
+        {"jsonrpc": "2.0", "id": "rej-1", "error": {"code": -32000, "message": "Agent 'no-such-agent' not found"}},
+        "no-such-agent",
+        "a2a_agent/no-such-agent",
+        id="unknown",
+    ),
+)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(("registered", "status", "body", "target", "model"), _EXPECTED_REJECTIONS)
+async def test_rejected_agent_call_is_logged_once_as_the_caller_without_dispatch(
+    monkeypatch: pytest.MonkeyPatch, registered: bool, status: int, body: object, target: str, model: str
+) -> None:
+    from litellm.caching.caching import DualCache
+    from litellm.proxy.utils import ProxyLogging
+
+    recorder: Final = _RejectionRecorder()
+    for name in ("callbacks", "failure_callback", "_async_failure_callback"):
+        monkeypatch.setattr(litellm, name, [recorder])
+
+    got_status, got_body = await _invoke_rejected(
+        monkeypatch, registered=registered, proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache())
+    )
+
+    assert (got_status, got_body) == (status, body)
+    assert len(recorder.failure_payloads) == 1
+    payload: Final = recorder.failure_payloads[0]
+    metadata: Final = payload["metadata"]
+    assert (payload["status"], payload["call_type"], payload["model"], payload["response_cost"]) == (
+        "failure",
+        "asend_message",
+        model,
+        0,
+    )
+    assert (
+        metadata["user_api_key_alias"],
+        metadata["user_api_key_user_id"],
+        metadata["user_api_key_team_id"],
+        metadata["user_api_key_hash"],
+    ) == ("caller-alias", "caller-user", "caller-team", "hashed-caller")
+    assert payload["id"] not in {"spoofed-call-id", "spoofed-param-call-id", "rej-1"}
+    assert len(recorder.hook_calls) == 1
+    hook_data, hook_exception = recorder.hook_calls[0]
+    assert (hook_data["agent_id"], hook_data["metadata"]["agent_id"]) == (target, target)
+    assert isinstance(hook_exception, HTTPException) and hook_exception.status_code == status
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(("registered", "status", "body", "target", "model"), _EXPECTED_REJECTIONS)
+async def test_rejected_agent_call_response_survives_a_failing_failure_hook(
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+    registered: bool,
+    status: int,
+    body: object,
+    target: str,
+    model: str,
+) -> None:
+    proxy_logging_obj: Final = MagicMock()
+    proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=RuntimeError("logging down"))
+
+    got_status, got_body = await _invoke_rejected(
+        monkeypatch, registered=registered, proxy_logging_obj=proxy_logging_obj
+    )
+
+    assert (got_status, got_body) == (status, body)
+    proxy_logging_obj.post_call_failure_hook.assert_awaited_once()
+    assert any(record.getMessage() == "Failed to log rejected A2A request" for record in caplog.records)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("request_id", [0, 7, "abc"])
+async def test_unknown_agent_rejection_echoes_the_jsonrpc_id(
+    monkeypatch: pytest.MonkeyPatch, request_id: object
+) -> None:
+    proxy_logging_obj: Final = MagicMock()
+    proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
+
+    _, got_body = await _invoke_rejected(
+        monkeypatch, registered=False, proxy_logging_obj=proxy_logging_obj, request_id=request_id
+    )
+
+    assert isinstance(got_body, dict) and got_body["id"] == request_id and type(got_body["id"]) is type(request_id)
+
+
 def _a2a_app_refusing_with(refusal: Exception) -> TestClient:
     from fastapi import FastAPI
 

tests/unit/proxy/agent_endpoints/test_agent_search.py

@@ -261,6 +261,42 @@ class TestSearchAgents:
         assert isinstance(outcome, AgentSearchNotConfigured)
         assert "agent_search_embedding_model" in outcome.reason
 
+    @pytest.mark.parametrize("embedding_model", ("", "   "), ids=("empty", "whitespace"))
+    @pytest.mark.asyncio
+    async def test_empty_embedding_model_is_not_configured(self, embedding_model: str) -> None:
+        router: Final = MagicMock()
+        router.aembedding = AsyncMock(
+            side_effect=lambda model, input, metadata: litellm.EmbeddingResponse(
+                model=model,
+                data=[{"object": "embedding", "index": i, "embedding": list(VECTORS[t])} for i, t in enumerate(input)],
+            )
+        )
+        not_configured: Final = await search_agents(
+            "language translation",
+            AGENTS,
+            5,
+            router=router,
+            embedding_model=None,
+            index=AgentSearchIndex(),
+            user_api_key_dict=CALLER,
+            proxy_logging_obj=_pass_through_key_limits(),
+        )
+        outcome: Final = await search_agents(
+            "language translation",
+            AGENTS,
+            5,
+            router=router,
+            embedding_model=embedding_model,
+            index=AgentSearchIndex(),
+            user_api_key_dict=CALLER,
+            proxy_logging_obj=_pass_through_key_limits(),
+        )
+
+        assert isinstance(not_configured, AgentSearchNotConfigured)
+        assert isinstance(outcome, AgentSearchNotConfigured)
+        assert outcome.reason == not_configured.reason
+        router.aembedding.assert_not_awaited()
+
     @pytest.mark.asyncio
     async def test_no_router_is_not_configured(self) -> None:
         outcome = await search_agents(

tests/unit/proxy/agent_endpoints/test_endpoints.py

@@ -1,7 +1,6 @@
 import json
 from collections.abc import Mapping
 from datetime import datetime, timezone
-
 from types import SimpleNamespace
 from typing import Final
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -13,7 +12,12 @@ from fastapi.testclient import TestClient
 from prisma.models import LiteLLM_AgentsTable
 
 from litellm.constants import REDACTED_BY_LITELM_STRING
-from litellm.proxy._types import LiteLLM_AuditLogs, LitellmTableNames, LitellmUserRoles, UserAPIKeyAuth
+from litellm.proxy._types import (
+    LiteLLM_AuditLogs,
+    LitellmTableNames,
+    LitellmUserRoles,
+    UserAPIKeyAuth,
+)
 from litellm.proxy.agent_endpoints import endpoints as agent_endpoints
 from litellm.proxy.agent_endpoints.auth.agent_permission_handler import (
     RestrictedAgentAccess,
@@ -43,6 +47,29 @@ def _sample_agent_card_params() -> dict:
     }
 
 
+_AGENT_CARD_BACKEND_URL: Final = "http://agent.internal:9000/plain/"
+
+
+def _sample_agent_card_params_with_backend_url() -> dict[str, object]:
+    return {
+        **_sample_agent_card_params(),
+        "url": _AGENT_CARD_BACKEND_URL,
+        "supportedInterfaces": [
+            {
+                "url": _AGENT_CARD_BACKEND_URL,
+                "protocolBinding": "JSONRPC",
+                "protocolVersion": "1.0",
+            },
+            {
+                "url": _AGENT_CARD_BACKEND_URL,
+                "protocolBinding": "GRPC",
+                "protocolVersion": "1.0",
+            },
+        ],
+        "additionalInterfaces": [{"url": _AGENT_CARD_BACKEND_URL, "transport": "GRPC"}],
+    }
+
+
 def _sample_agent_config() -> dict:
     return {
         "agent_name": "Test Agent",
@@ -641,7 +668,7 @@ class TestAgentRBACProxyAdmin:
         monkeypatch.setattr(agent_endpoints, "AGENT_REGISTRY", self.mock_registry)
 
     def test_should_allow_admin_to_create_agent(self, monkeypatch):
-        with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma:
+        with patch("litellm.proxy.proxy_server.prisma_client"):
             self.mock_registry.get_agent_by_name = MagicMock(return_value=None)
             self.mock_registry.add_agent_to_db = AsyncMock(
                 return_value=_sample_agent_response()
@@ -1015,6 +1042,129 @@ class TestAgentHealthCheck:
         assert len(resp.json()) == 2
 
 
+def _client_with_agent_card(
+    monkeypatch: pytest.MonkeyPatch,
+    role: LitellmUserRoles,
+    agent: AgentResponse,
+) -> TestClient:
+    from litellm.proxy import proxy_server
+    from litellm.proxy.agent_endpoints import agent_registry
+
+    registry: Final = MagicMock()
+    registry.get_agent_by_id = MagicMock(return_value=agent)
+    registry.get_agent_list = MagicMock(return_value=(agent,))
+    registry.ids_for_agent = MagicMock(return_value=frozenset({agent.agent_id}))
+    registry.stable_agent_id = MagicMock(side_effect=lambda agent_id: agent_id)
+    monkeypatch.setattr(agent_endpoints, "AGENT_REGISTRY", registry)
+    monkeypatch.setattr(agent_registry, "global_agent_registry", registry)
+    prisma: Final = MagicMock()
+    prisma.writer_db = prisma.db
+    prisma.db.litellm_agentstable.find_many = AsyncMock(return_value=[])
+    prisma.db.litellm_agentstable.find_unique = AsyncMock(return_value=None)
+    prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[])
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma)
+    test_app: Final = FastAPI()
+    test_app.include_router(router)
+    test_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
+        user_id="test-user",
+        user_role=role,
+        object_permission={"object_permission_id": "agent-card-test", "agents": [agent.agent_id]},
+    )
+    return TestClient(test_app)
+
+
+@pytest.mark.parametrize("path", ["/v1/agents", "/v1/agents/agent-123"])
+def test_non_admin_agent_card_urls_point_at_proxy(
+    monkeypatch: pytest.MonkeyPatch,
+    path: str,
+) -> None:
+    agent: Final = AgentResponse(
+        agent_id="agent-123",
+        agent_name="Test Agent",
+        agent_card_params=_sample_agent_card_params_with_backend_url(),
+    )
+    test_client: Final = _client_with_agent_card(monkeypatch, LitellmUserRoles.INTERNAL_USER, agent)
+
+    response: Final = test_client.get(path, headers={"Authorization": "Bearer k"})
+
+    assert response.status_code == 200, response.text
+    agents: Final = response.json()
+    result: Final = agents[0] if path == "/v1/agents" else agents
+    card: Final = result["agent_card_params"]
+    proxy_url: Final = "http://testserver/a2a/agent-123"
+    assert card["url"] == proxy_url
+    assert "agent.internal" not in response.text
+    assert "additionalInterfaces" not in card
+    assert card["supportedInterfaces"] == [
+        {
+            "url": proxy_url,
+            "protocolBinding": "JSONRPC",
+            "protocolVersion": "1.0",
+        }
+    ]
+
+
+def test_proxy_admin_agent_listing_keeps_backend_card_urls(monkeypatch: pytest.MonkeyPatch) -> None:
+    agent: Final = AgentResponse(
+        agent_id="agent-123",
+        agent_name="Test Agent",
+        agent_card_params=_sample_agent_card_params_with_backend_url(),
+    )
+    test_client: Final = _client_with_agent_card(monkeypatch, LitellmUserRoles.PROXY_ADMIN, agent)
+
+    response: Final = test_client.get("/v1/agents", headers={"Authorization": "Bearer k"})
+
+    assert response.status_code == 200, response.text
+    card: Final = response.json()[0]["agent_card_params"]
+    assert card["url"] == _AGENT_CARD_BACKEND_URL
+    assert card["supportedInterfaces"] == [
+        {
+            "url": _AGENT_CARD_BACKEND_URL,
+            "protocolBinding": "JSONRPC",
+            "protocolVersion": "1.0",
+        },
+        {
+            "url": _AGENT_CARD_BACKEND_URL,
+            "protocolBinding": "GRPC",
+            "protocolVersion": "1.0",
+        },
+    ]
+    assert card["additionalInterfaces"] == [{"url": _AGENT_CARD_BACKEND_URL, "transport": "GRPC"}]
+
+
+def test_non_admin_health_check_probes_backend_and_returns_proxy_card(monkeypatch: pytest.MonkeyPatch) -> None:
+    agent: Final = AgentResponse(
+        agent_id="agent-123",
+        agent_name="Test Agent",
+        agent_card_params=_sample_agent_card_params_with_backend_url(),
+    )
+    test_client: Final = _client_with_agent_card(monkeypatch, LitellmUserRoles.INTERNAL_USER, agent)
+    http_get: Final = AsyncMock(return_value=httpx.Response(200))
+    http_client: Final = MagicMock(get=http_get)
+    monkeypatch.setattr(
+        agent_endpoints,
+        "get_async_httpx_client",
+        MagicMock(return_value=http_client),
+    )
+
+    response: Final = test_client.get("/v1/agents?health_check=true", headers={"Authorization": "Bearer k"})
+
+    assert response.status_code == 200, response.text
+    assert http_get.await_args.args[0] == _AGENT_CARD_BACKEND_URL
+    assert response.json()[0]["agent_card_params"]["url"] == "http://testserver/a2a/agent-123"
+
+
+def test_non_admin_agent_card_without_interfaces_does_not_add_them(monkeypatch: pytest.MonkeyPatch) -> None:
+    agent: Final = _sample_agent_response()
+    test_client: Final = _client_with_agent_card(monkeypatch, LitellmUserRoles.INTERNAL_USER, agent)
+
+    response: Final = test_client.get("/v1/agents/agent-123", headers={"Authorization": "Bearer k"})
+
+    assert response.status_code == 200, response.text
+    assert response.json()["agent_card_params"]["url"] == "http://testserver/a2a/agent-123"
+    assert "supportedInterfaces" not in response.json()["agent_card_params"]
+
+
 class TestCheckAgentUrlHealth:
     """Unit tests for the _check_agent_url_health helper."""
 
@@ -1230,7 +1380,11 @@ def test_jwt_authentication_status_does_not_require_virtual_keys(
     monkeypatch.setattr(proxy_server, "general_settings", {"enable_jwt_auth": enabled})
     monkeypatch.setattr(proxy_server, "jwt_handler", handler)
     agent: Final = _sample_agent_response()
-    response: Final = agent_endpoints._redact_sensitive_agent_fields((agent,), is_admin=True)[0]
+    response: Final = agent_endpoints._redact_sensitive_agent_fields(
+        (agent,),
+        is_admin=True,
+        request_base_url="http://testserver",
+    )[0]
     assert response.jwt_auth_configured is expected
     assert agent.jwt_auth_configured is False
 

tests/unit/proxy/auth/test_route_checks.py

@@ -3925,6 +3925,7 @@ AGENT_INFERENCE_ROUTES = [
     "/a2a/abc-123",
     "/a2a/abc-123/message/send",
     "/a2a/abc-123/message/stream",
+    "/v1/a2a/abc-123/message/stream",
     "/a2a/abc-123/.well-known/agent-card.json",
 ]
 

tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py

@@ -27,8 +27,9 @@ from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import (
     BedrockGuardrail,
     _redact_pii_matches,
 )
+from litellm.proxy.guardrails.guardrail_initializers import initialize_bedrock
 from litellm.proxy.utils import ProxyLogging
-from litellm.types.guardrails import GuardrailEventHooks
+from litellm.types.guardrails import BedrockGuardrailOutputScope, Guardrail, GuardrailEventHooks, LitellmParams
 from litellm.types.proxy.guardrails.guardrail_hooks.bedrock_guardrails import (
     BedrockContentItem,
     BedrockTextContent,
@@ -1020,6 +1021,64 @@ def _anthropic_tool_result_conversation(
     ]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("scope", [None, "INTERVENTIONS", "FULL"])
+async def test_configured_output_scope_reaches_bedrock_and_preserves_healthy_assessments(
+    monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, scope: str | None
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    params: Final = LitellmParams.model_validate(
+        {
+            "guardrail": "bedrock",
+            "mode": "post_call",
+            "guardrailIdentifier": "test-guardrail",
+            "guardrailVersion": "DRAFT",
+            "contextual_grounding_from_messages": True,
+            "aws_access_key_id": "test-access-key",
+            "aws_secret_access_key": "test-secret-key",
+            "aws_region_name": "us-east-1",
+            "aws_bedrock_runtime_endpoint": "https://bedrock.example",
+            **({"outputScope": scope} if scope is not None else {}),
+        }
+    )
+    guardrail: Final = initialize_bedrock(params, Guardrail(guardrail_name="grounding", litellm_params=params))
+    assessment: Final = {
+        "contextualGroundingPolicy": {
+            "filters": [{"type": "GROUNDING", "score": 0.96, "threshold": 0.8, "detected": False, "action": "NONE"}]
+        }
+    }
+
+    def provider(request: httpx.Request) -> httpx.Response:
+        body: Final = json.loads(request.content)
+        return httpx.Response(
+            200,
+            json={"action": "NONE", "assessments": [assessment] if body.get("outputScope") == "FULL" else []},
+        )
+
+    route: Final = respx_mock.post("https://bedrock.example/guardrail/test-guardrail/version/DRAFT/apply").mock(
+        side_effect=provider
+    )
+    request_data: Final = {
+        "messages": [
+            {"role": "system", "content": "The tower is 330 metres tall."},
+            {"role": "user", "content": "How tall is the tower?"},
+        ]
+    }
+    result: Final = await guardrail.apply_guardrail(
+        inputs={"texts": ["The tower is 330 metres tall."]}, request_data=request_data, input_type="response"
+    )
+
+    sent: Final = json.loads(route.calls.last.request.content)
+    if scope is None:
+        assert "outputScope" not in sent
+    else:
+        assert sent["outputScope"] == scope
+    assert sent["source"] == "OUTPUT"
+    assert result["texts"] == ["The tower is 330 metres tall."]
+    recorded: Final = request_data["metadata"]["standard_logging_guardrail_information"][-1]["guardrail_response"]
+    assert recorded["assessments"] == ([assessment] if scope == "FULL" else [])
+
+
 @pytest.mark.asyncio
 async def test_during_call_hook_skips_bedrock_call_for_tool_result_only_turn():
     """A tool_result-only latest user turn must not post an empty content list to Bedrock.
@@ -1805,7 +1864,11 @@ async def test_make_bedrock_api_request_logging_event_type_for_spend_logs():
 
 @pytest.mark.asyncio
 async def test_make_bedrock_api_request_filters_dynamic_evaluation_overrides():
-    guardrail = BedrockGuardrail(guardrailIdentifier="test-guardrail", guardrailVersion="DRAFT")
+    guardrail = BedrockGuardrail(
+        guardrailIdentifier="test-guardrail",
+        guardrailVersion="DRAFT",
+        outputScope=BedrockGuardrailOutputScope.INTERVENTIONS,
+    )
     mock_credentials = MagicMock()
     mock_credentials.access_key = "test-access-key"
     mock_credentials.secret_key = "test-secret-key"

tests/unit/proxy/guardrails/guardrail_hooks/test_ovalix_extraction.py

@@ -0,0 +1,443 @@
+import base64
+from typing import Final
+
+from litellm.proxy.guardrails.guardrail_hooks.ovalix.ovalix_extraction import (
+    extract_file_parts_from_images,
+    extract_file_parts_from_messages,
+    extract_tool_results,
+    make_tool_data,
+    tool_call_to_tool_data,
+    tool_result_text_indices,
+)
+
+
+def _b64(raw: bytes) -> str:
+    return base64.b64encode(raw).decode()
+
+
+def test_file_block_data_url_decoded():
+    msgs = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "file", "file": {"filename": "a.txt", "file_data": f"data:text/plain;base64,{_b64(b'hi')}"}}
+            ],
+        }
+    ]
+    parts = extract_file_parts_from_messages(msgs, size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == b"hi" and parts[0].name == "a.txt" and parts[0].inline
+
+
+def test_image_url_reference_tracked_by_name():
+    msgs = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://x.test/pic.png"}}]}]
+    parts = extract_file_parts_from_messages(msgs, size_limit=1000)
+    assert len(parts) == 1 and parts[0].data is None and parts[0].inline is False and parts[0].name == "pic.png"
+
+
+def test_oversize_file_flagged_no_data():
+    big = _b64(b"x" * 100)
+    msgs = [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "file",
+                    "file": {"filename": "big.bin", "file_data": f"data:application/octet-stream;base64,{big}"},
+                }
+            ],
+        }
+    ]
+    parts = extract_file_parts_from_messages(msgs, size_limit=10)
+    assert len(parts) == 1 and parts[0].oversize is True and parts[0].data is None
+
+
+def test_images_field_data_url():
+    parts = extract_file_parts_from_images([f"data:image/png;base64,{_b64(b'png')}"], size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == b"png" and parts[0].inline
+
+
+def test_tool_call_to_tool_data_parses_arguments():
+    td = tool_call_to_tool_data(
+        {"id": "c1", "type": "function", "function": {"name": "get_weather", "arguments": '{"city": "TLV"}'}}
+    )
+    assert (
+        td["content"] == '{"city": "TLV"}'
+        and td["tool_name"] == "get_weather"
+        and td["action_name"] == "get_weather"
+        and td["tool_input"] == {"city": "TLV"}
+    )
+
+
+def test_tool_call_malformed_dropped():
+    assert tool_call_to_tool_data({"id": "c1", "type": "function", "function": {"name": ""}}) is None
+    assert tool_call_to_tool_data({"id": "c1"}) is None
+
+
+def test_extract_tool_results_correlates_name():
+    msgs = [
+        {"role": "assistant", "tool_calls": [{"id": "c1", "function": {"name": "get_weather"}}]},
+        {"role": "tool", "tool_call_id": "c1", "content": "sunny"},
+        {"role": "tool", "tool_call_id": "unknown", "content": "orphan"},
+    ]
+    results = extract_tool_results(msgs)
+    assert ("get_weather", "sunny", "c1") in results
+    assert ("tool_result", "orphan", "unknown") in results
+
+
+def test_make_tool_data_truncates_and_defaults_name():
+    td = make_tool_data("   ", "content")
+    assert td["tool_name"] == "tool_result" and td["action_name"] == "tool_result"
+    long = "x" * 200
+    td2 = make_tool_data(long, "c")
+    assert len(td2["tool_name"]) == 100 and td2["action_name"] == long
+
+
+def test_unhashable_block_type_skipped_without_raising():
+    msgs = [{"role": "user", "content": [{"type": ["file"], "file": {"filename": "a.txt"}}]}]
+    parts = extract_file_parts_from_messages(msgs, size_limit=1000)
+    assert parts == ()
+
+
+def test_unhashable_tool_call_id_skipped_without_raising():
+    msgs = [
+        {"role": "assistant", "tool_calls": [{"id": ["c1"], "function": {"name": "get_weather"}}]},
+        {"role": "tool", "tool_call_id": ["c1"], "content": "sunny"},
+    ]
+    results = extract_tool_results(msgs)
+    assert results == (("tool_result", "sunny", ["c1"]),)
+
+
+def test_extract_tool_results_list_form_content():
+    msgs = [{"role": "tool", "tool_call_id": "c1", "content": [{"type": "text", "text": "part1"}, "part2"]}]
+    results = extract_tool_results(msgs)
+    assert ("tool_result", "part1\npart2", "c1") in results
+
+
+def test_extract_tool_results_dict_form_content():
+    msgs = [{"role": "tool", "tool_call_id": "c1", "content": {"city": "TLV"}}]
+    results = extract_tool_results(msgs)
+    assert ("tool_result", '{"city": "TLV"}', "c1") in results
+
+
+def test_images_field_oversize_flagged_no_data():
+    big = _b64(b"x" * 100)
+    parts = extract_file_parts_from_images([f"data:image/png;base64,{big}"], size_limit=10)
+    assert len(parts) == 1 and parts[0].oversize is True and parts[0].data is None
+
+
+class _StubFunction:
+    def __init__(self, name, arguments):
+        self.name = name
+        self.arguments = arguments
+
+
+class _StubToolCall:
+    def __init__(self, function):
+        self.function = function
+
+
+def test_tool_call_to_tool_data_accepts_object_style_tool_call():
+    tool_call = _StubToolCall(_StubFunction("get_weather", '{"city": "TLV"}'))
+    td = tool_call_to_tool_data(tool_call)
+    assert td["tool_name"] == "get_weather" and td["tool_input"] == {"city": "TLV"}
+
+
+def _msgs(block):
+    return [{"role": "user", "content": [block]}]
+
+
+def test_image_url_block_data_url_decoded_from_messages():
+    block = {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_b64(b'png')}"}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == b"png" and parts[0].inline and parts[0].name is None
+
+
+def test_image_url_block_non_string_url_skipped():
+    block = {"type": "image_url", "image_url": {"url": 123}}
+    assert extract_file_parts_from_messages(_msgs(block), size_limit=1000) == ()
+
+
+def test_input_image_block_data_url_decoded():
+    block = {"type": "input_image", "image_url": f"data:image/png;base64,{_b64(b'img')}"}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == b"img" and parts[0].inline
+
+
+def test_file_block_non_dict_file_skipped():
+    block = {"type": "file", "file": "not-a-dict"}
+    assert extract_file_parts_from_messages(_msgs(block), size_limit=1000) == ()
+
+
+def test_file_block_reference_without_bytes_is_name_only():
+    block = {"type": "file", "file": {"file_id": "file-abc"}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].name == "file-abc" and parts[0].data is None and parts[0].inline is False
+
+
+def test_file_block_urlsafe_base64_decoded_via_fallback():
+    raw = b"\xff\xff\xfe"  # encodes with url-unsafe chars '+'/'/' in standard b64
+    urlsafe = base64.urlsafe_b64encode(raw).decode()
+    assert "-" in urlsafe or "_" in urlsafe
+    block = {
+        "type": "file",
+        "file": {"filename": "b.bin", "file_data": f"data:application/octet-stream;base64,{urlsafe}"},
+    }
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == raw
+
+
+def test_file_block_data_url_without_base64_marker_is_undecodable():
+    block = {"type": "file", "file": {"filename": "n.txt", "file_data": "data:text/plain,hello"}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].undecodable and parts[0].data is None and parts[0].name == "n.txt"
+
+
+def test_input_file_block_data_url_decoded():
+    block = {"type": "input_file", "filename": "doc.pdf", "file_data": f"data:application/pdf;base64,{_b64(b'pdf')}"}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == b"pdf" and parts[0].name == "doc.pdf" and parts[0].inline
+
+
+def test_input_file_block_file_url_reference_name_only():
+    block = {"type": "input_file", "file_url": "https://x.test/report.csv"}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].name == "report.csv" and parts[0].data is None and parts[0].inline is False
+
+
+def test_input_audio_block_decoded():
+    block = {"type": "input_audio", "input_audio": {"data": _b64(b"wav"), "format": "wav"}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == b"wav" and parts[0].name == "audio.wav" and parts[0].inline
+
+
+def test_input_audio_block_undecodable_is_flagged():
+    block = {"type": "input_audio", "input_audio": {"data": "!!!not-base64!!!"}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].undecodable and parts[0].data is None and parts[0].name == "audio.bin"
+
+
+def test_input_audio_block_non_dict_skipped():
+    block = {"type": "input_audio", "input_audio": "nope"}
+    assert extract_file_parts_from_messages(_msgs(block), size_limit=1000) == ()
+
+
+def test_tool_call_dict_arguments_serialized_and_parsed():
+    td = tool_call_to_tool_data({"function": {"name": "f", "arguments": {"a": 1}}})
+    assert td["content"] == '{"a": 1}' and td["tool_input"] == {"a": 1}
+
+
+def test_tool_call_none_arguments_yields_empty_content():
+    td = tool_call_to_tool_data({"function": {"name": "f", "arguments": None}})
+    assert td["content"] == "" and td["tool_input"] == {}
+
+
+def test_tool_call_non_string_non_dict_arguments_serialized():
+    td = tool_call_to_tool_data({"function": {"name": "f", "arguments": [1, 2]}})
+    assert td["content"] == "[1, 2]" and td["tool_input"] == {}
+
+
+def test_tool_call_invalid_json_string_arguments_kept_as_content():
+    td = tool_call_to_tool_data({"function": {"name": "f", "arguments": "{not json"}})
+    assert td["content"] == "{not json" and td["tool_input"] == {}
+
+
+def test_tool_result_with_non_string_tool_call_id_uses_default_name():
+    msgs = [{"role": "tool", "tool_call_id": ["c1"], "content": "orphan"}]
+    results = extract_tool_results(msgs)
+    assert results == (("tool_result", "orphan", ["c1"]),)
+
+
+def test_images_field_http_url_is_name_only_reference():
+    parts = extract_file_parts_from_images(["https://x.test/pic.png"], size_limit=1000)
+    assert len(parts) == 1 and parts[0].name == "pic.png" and parts[0].data is None and parts[0].inline is False
+
+
+def test_messages_skip_non_dict_and_unknown_blocks():
+    msgs = [
+        "not-a-message",
+        {
+            "role": "user",
+            "content": [
+                "bare-string-block",
+                {"type": "text", "text": "hi"},
+                {"type": "file", "file": {"filename": "a.txt", "file_data": f"data:text/plain;base64,{_b64(b'x')}"}},
+            ],
+        },
+    ]
+    parts = extract_file_parts_from_messages(msgs, size_limit=1000)
+    assert len(parts) == 1 and parts[0].name == "a.txt" and parts[0].data == b"x"
+
+
+def test_image_url_block_invalid_data_url_is_flagged():
+    block = {"type": "image_url", "image_url": {"url": "data:image/png;base64,%%%invalid%%%"}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].undecodable and parts[0].data is None
+
+
+def test_tool_message_with_empty_content_is_skipped():
+    msgs = [{"role": "tool", "tool_call_id": "c1", "content": "   "}]
+    assert extract_tool_results(msgs) == ()
+
+
+def test_extract_tool_results_skips_non_dict_messages_and_tool_calls():
+    msgs = [
+        "junk",
+        {"role": "assistant", "tool_calls": ["not-a-dict", {"id": "c1", "function": {"name": "f"}}]},
+        {"role": "tool", "tool_call_id": "c1", "content": "ok"},
+    ]
+    assert extract_tool_results(msgs) == (("f", "ok", "c1"),)
+
+
+def test_tool_result_takes_its_name_from_the_nearest_earlier_declaration():
+    msgs: Final = [
+        {"role": "assistant", "tool_calls": [{"id": "c1", "function": {"name": "first"}}]},
+        {"role": "tool", "tool_call_id": "c1", "content": "r1"},
+        {
+            "role": "assistant",
+            "tool_calls": [{"id": "c1", "function": {"name": "second"}}, {"id": "c1", "function": {"name": "third"}}],
+        },
+        {"role": "tool", "tool_call_id": "c1", "content": "r2"},
+        {"role": "tool", "tool_call_id": "c2", "content": "r3"},
+        {"role": "assistant", "tool_calls": [{"id": "c2", "function": {"name": "too-late"}}]},
+    ]
+    assert extract_tool_results(msgs) == (("first", "r1", "c1"), ("third", "r2", "c1"), ("tool_result", "r3", "c2"))
+
+
+class _ReadCountingMessage(dict):
+    def __init__(self, **fields):
+        super().__init__(**fields)
+        self.reads = 0
+
+    def get(self, key, default=None):
+        self.reads += 1
+        return super().get(key, default)
+
+
+def _reads_of_a_declaration_followed_by(tool_results):
+    declaration: Final = _ReadCountingMessage(role="assistant", tool_calls=[{"id": "c0", "function": {"name": "f"}}])
+    tool_messages: Final = [
+        {"role": "tool", "tool_call_id": f"c{index}", "content": "x"} for index in range(tool_results)
+    ]
+    extract_tool_results([declaration, *tool_messages])
+    return declaration.reads
+
+
+def test_tool_result_names_resolve_without_rescanning_earlier_messages():
+    assert _reads_of_a_declaration_followed_by(1000) == _reads_of_a_declaration_followed_by(1)
+
+
+def test_malformed_data_url_is_undecodable():
+    block = {"type": "file", "file": {"filename": "x.bin", "file_data": "data:garbage-no-comma"}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].undecodable and parts[0].data is None and parts[0].name == "x.bin"
+
+
+def test_input_audio_block_without_data_skipped():
+    block = {"type": "input_audio", "input_audio": {"format": "wav"}}
+    assert extract_file_parts_from_messages(_msgs(block), size_limit=1000) == ()
+
+
+def test_images_field_non_string_entries_skipped():
+    assert extract_file_parts_from_images([123, None, ""], size_limit=1000) == ()
+
+
+def test_make_tool_data_whitespace_after_truncation_defaults_name():
+    name = " " * 100 + "x"
+    td = make_tool_data(name, "c")
+    assert td["tool_name"] == "tool_result" and td["action_name"] == name
+
+
+def test_raw_base64_file_data_without_data_url_prefix_decoded():
+    block = {"type": "file", "file": {"filename": "a.bin", "file_data": _b64(b"rawbytes")}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == b"rawbytes" and parts[0].mime_hint is None
+
+
+def test_images_field_raw_base64_without_data_url_prefix_decoded():
+    parts = extract_file_parts_from_images([_b64(b"rawimg")], size_limit=1000)
+    assert len(parts) == 1 and parts[0].data == b"rawimg"
+
+
+def test_whitespace_only_base64_payload_yields_no_bytes():
+    block = {"type": "file", "file": {"filename": "n.txt", "file_data": "data:text/plain;base64,    "}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].data is None and parts[0].inline is False and parts[0].name == "n.txt"
+
+
+def test_base64_invalid_even_after_urlsafe_translate_is_undecodable():
+    block = {"type": "file", "file": {"filename": "b.bin", "file_data": "_@@@"}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=1000)
+    assert len(parts) == 1 and parts[0].undecodable and parts[0].data is None and parts[0].name == "b.bin"
+
+
+def test_file_oversize_detected_after_decode_when_estimate_passes():
+    block = {"type": "file", "file": {"filename": "s.bin", "file_data": _b64(b"abc")}}
+    parts = extract_file_parts_from_messages(_msgs(block), size_limit=2)
+    assert len(parts) == 1 and parts[0].oversize is True and parts[0].data is None
+
+
+def test_image_http_url_that_urlparse_rejects_is_dropped():
+    parts = extract_file_parts_from_images(["http://["], size_limit=1000)
+    assert parts == ()
+
+
+def test_message_content_not_a_list_is_skipped():
+    parts = extract_file_parts_from_messages(
+        [{"role": "user", "content": "just a plain string prompt"}], size_limit=1000
+    )
+    assert parts == ()
+
+
+def test_tool_call_dict_arguments_non_serializable_falls_back_to_str():
+    td = tool_call_to_tool_data({"function": {"name": "f", "arguments": {"a": {1, 2}}}})
+    assert isinstance(td["content"], str) and td["tool_input"] == {"a": {1, 2}}
+
+
+def test_tool_call_non_serializable_other_arguments_falls_back_to_str():
+    td = tool_call_to_tool_data({"function": {"name": "f", "arguments": {1, 2}}})
+    assert isinstance(td["content"], str) and td["tool_input"] == {}
+
+
+def test_tool_result_dict_content_non_serializable_falls_back_to_str():
+    results = extract_tool_results([{"role": "tool", "tool_call_id": "c1", "content": {"x": {1, 2}}}])
+    assert len(results) == 1 and isinstance(results[0][1], str) and results[0][1].strip()
+
+
+def test_tool_result_text_indices_marks_only_tool_role_positions():
+    messages = [
+        {"role": "user", "content": "ask"},
+        {"role": "assistant", "tool_calls": [{"id": "c1", "function": {"name": "f"}}]},
+        {"role": "tool", "tool_call_id": "c1", "content": "result"},
+    ]
+    assert tool_result_text_indices(messages, ["ask", "result"]) == frozenset({1})
+
+
+def test_tool_result_text_indices_covers_every_text_block_of_a_tool_message():
+    messages = [
+        {"role": "user", "content": "ask"},
+        {
+            "role": "tool",
+            "tool_call_id": "c1",
+            "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}],
+        },
+    ]
+    assert tool_result_text_indices(messages, ["ask", "a", "b"]) == frozenset({1, 2})
+
+
+def test_tool_result_text_indices_empty_when_texts_do_not_replay_messages():
+    messages = [
+        {"role": "user", "content": "ask"},
+        {"role": "tool", "tool_call_id": "c1", "content": "result"},
+    ]
+    assert tool_result_text_indices(messages, ["result"]) == frozenset()
+    assert tool_result_text_indices(messages, ["ask", "tampered"]) == frozenset()
+
+
+def test_tool_result_text_indices_skips_blank_tool_content_never_submitted_as_tool():
+    messages = [{"role": "tool", "tool_call_id": "c1", "content": "   "}]
+    assert extract_tool_results(messages) == ()
+    assert tool_result_text_indices(messages, ["   "]) == frozenset()
+
+
+def test_tool_result_text_indices_empty_without_structured_messages():
+    assert tool_result_text_indices(None, ["ask"]) == frozenset()

tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py

@@ -2025,6 +2025,28 @@ class TestStreamingHttpErrorFrames:
     def _use_real_mappings(self, monkeypatch):
         _patch_translation_mappings(monkeypatch, load_guardrail_translation_mappings())
 
+    @pytest.mark.asyncio
+    async def test_v1_a2a_stream_route_scans_response_chunks(self):
+        guardrail: Final = RecordingGuardrail()
+        guardrail.streaming_sampling_rate = 1
+        chunk: Final = {
+            "jsonrpc": "2.0",
+            "id": "stream-1",
+            "result": {
+                "kind": "task",
+                "artifacts": [{"parts": [{"kind": "text", "text": "streamed text"}]}],
+            },
+        }
+
+        await _drive_stream(
+            UnifiedLLMGuardrails(),
+            guardrail,
+            [chunk],
+            request_route="/v1/a2a/my-agent/message/stream",
+        )
+
+        assert guardrail.apply_calls[0]["inputs"]["texts"] == ["streamed text"]
+
     @pytest.mark.asyncio
     async def test_chat_eos_block_emits_data_error_frame(self):
         import json as _json

tests/unit/proxy/guardrails/test_guardrail_endpoints.py

@@ -3,17 +3,20 @@ import time
 from collections.abc import Callable
 from datetime import datetime
 from types import MappingProxyType
-from typing import Dict, Final, List, Optional
+from typing import Dict, Final, List, Literal, Optional
 from unittest.mock import AsyncMock
 
+import httpx
 import pytest
 import yaml
-
-
-from fastapi import HTTPException
+from fastapi import FastAPI, HTTPException
+from pydantic import JsonValue
+from pytest_mock import MockerFixture
 
 from litellm.integrations.custom_guardrail import CustomGuardrail
+from litellm.litellm_core_utils.litellm_logging import Logging
 from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth
+from litellm.proxy.guardrails.content_filter_data import DATA_ROOTS
 from litellm.proxy.guardrails.guardrail_endpoints import (
     CreateGuardrailRequest,
     ListGuardrailsResponse,
@@ -37,7 +40,6 @@ from litellm.proxy.guardrails.guardrail_endpoints import (
     reject_guardrail_submission,
     update_guardrail,
 )
-from litellm.proxy.guardrails.content_filter_data import DATA_ROOTS
 from litellm.proxy.guardrails.guardrail_endpoints import (
     test_custom_code_guardrail as run_custom_code_test_endpoint,
 )
@@ -53,8 +55,8 @@ from litellm.proxy.guardrails.guardrail_registry import (
 from litellm.types.guardrails import (
     ApplyGuardrailRequest,
     BaseLitellmParams,
-    GuardrailEventHooks,
     Guardrail,
+    GuardrailEventHooks,
     GuardrailInfoResponse,
     LitellmParams,
 )
@@ -2198,6 +2200,205 @@ def _patch_apply_guardrail_env(mocker, guardrail_result):
     return mock_guardrail
 
 
+class _NativeFindingGuardrail(CustomGuardrail):
+    def __init__(self, findings: tuple[object, ...]):
+        super().__init__(guardrail_name="grounding", event_hook=GuardrailEventHooks.post_call)
+        self.findings = findings
+
+    async def apply_guardrail(
+        self,
+        inputs: GenericGuardrailAPIInputs,
+        request_data: dict[str, object],
+        input_type: Literal["request", "response"],
+        logging_obj: Logging | None = None,
+    ) -> GenericGuardrailAPIInputs:
+        for finding in self.findings:
+            self.add_standard_logging_guardrail_information_to_request_data(
+                guardrail_json_response=finding, request_data=request_data, guardrail_status="success"
+            )
+        return inputs
+
+
+class _InputDirectionGuardrail(_NativeFindingGuardrail):
+    async def apply_guardrail(
+        self,
+        inputs: GenericGuardrailAPIInputs,
+        request_data: dict[str, object],
+        input_type: Literal["request", "response"],
+        logging_obj: Logging | None = None,
+    ) -> GenericGuardrailAPIInputs:
+        self.add_standard_logging_guardrail_information_to_request_data(
+            guardrail_json_response={"input_type": input_type},
+            request_data=request_data,
+            guardrail_status="success",
+        )
+        return {"texts": [input_type]}
+
+
+async def _apply_finding_guardrail_over_http(
+    mocker: MockerFixture,
+    findings: tuple[object, ...],
+    body: dict[str, object],
+    hook_result: object = None,
+    guardrail: CustomGuardrail | None = None,
+) -> httpx.Response:
+    from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+    from litellm.proxy.guardrails.guardrail_endpoints import router
+    from litellm.proxy.proxy_server import openai_exception_handler
+
+    configured: Final = guardrail if guardrail is not None else _NativeFindingGuardrail(findings)
+    patched: Final = _patch_apply_guardrail_env(mocker, {})
+    patched.apply_guardrail = configured.apply_guardrail
+    patched.event_hook = configured.event_hook
+    mocker.patch(
+        "litellm.proxy.proxy_server.proxy_logging_obj.post_call_success_hook", AsyncMock(return_value=hook_result)
+    )
+    app: Final = FastAPI()
+    app.include_router(router)
+    app.add_exception_handler(ProxyException, openai_exception_handler)
+    app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://guardrails.test") as client:
+        return await client.post("/guardrails/apply_guardrail", json=body)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("is_grounded", [True, False])
+async def test_apply_guardrail_native_response_preserves_structured_findings(
+    mocker: MockerFixture, is_grounded: bool
+) -> None:
+    findings: Final = {"is_grounded": is_grounded, "reason": "Compared answer with evidence", "source_ids": ["doc-1"]}
+    result: Final = await _apply_finding_guardrail_over_http(
+        mocker,
+        (findings,),
+        {"guardrail_name": "grounding", "text": "answer", "response_format": "native"},
+    )
+    assert result.status_code == 200
+    assert result.json() == findings
+
+
+@pytest.mark.asyncio
+async def test_apply_guardrail_standard_response_stays_unchanged_with_native_findings(mocker: MockerFixture) -> None:
+    result: Final = await _apply_finding_guardrail_over_http(
+        mocker, ({"is_grounded": False},), {"guardrail_name": "grounding", "text": "answer"}
+    )
+    assert result.status_code == 200
+    assert result.json() == {"response_text": "answer"}
+
+
+@pytest.mark.asyncio
+async def test_apply_guardrail_native_preserves_all_provider_results(mocker: MockerFixture) -> None:
+    findings: Final = ({"is_grounded": False}, {"response_text": "native text", "additional_finding": [1, 2]})
+    result: Final = await _apply_finding_guardrail_over_http(
+        mocker, findings, {"guardrail_name": "grounding", "text": "answer", "response_format": "native"}
+    )
+    assert result.status_code == 200
+    assert result.json() == list(findings)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("finding", "expected"),
+    [
+        ({"is_grounded": True, "source_ids": ("doc-1",)}, {"is_grounded": True, "source_ids": ["doc-1"]}),
+        ({"response_text": "native", "score": 0.2}, {"response_text": "native", "score": 0.2}),
+        ([{"is_grounded": False}], [{"is_grounded": False}]),
+    ],
+)
+async def test_apply_guardrail_native_preserves_provider_json_shape(
+    mocker: MockerFixture, finding: object, expected: JsonValue
+) -> None:
+    result: Final = await _apply_finding_guardrail_over_http(
+        mocker,
+        (finding,),
+        {"guardrail_name": "grounding", "text": "answer", "response_format": "native"},
+    )
+    assert result.status_code == 200
+    assert result.json() == expected
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("findings", [(), ("allow",)])
+async def test_apply_guardrail_native_rejects_missing_findings_and_caller_supplied_verdicts(
+    mocker: MockerFixture, findings: tuple[JsonValue, ...]
+) -> None:
+    result: Final = await _apply_finding_guardrail_over_http(
+        mocker,
+        findings,
+        {
+            "guardrail_name": "grounding",
+            "text": "answer",
+            "response_format": "native",
+            "metadata": {
+                "standard_logging_guardrail_information": [
+                    {"guardrail_name": "grounding", "guardrail_response": {"is_grounded": True}}
+                ]
+            },
+        },
+    )
+    assert result.status_code == 422
+    assert "did not report structured native findings" in result.text
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("response_format", ["standard", "native"])
+async def test_apply_guardrail_omitted_direction_uses_configured_post_call_hook(
+    mocker: MockerFixture, response_format: str
+) -> None:
+    result: Final = await _apply_finding_guardrail_over_http(
+        mocker,
+        (),
+        {"guardrail_name": "grounding", "text": "answer", "response_format": response_format},
+        guardrail=_InputDirectionGuardrail(()),
+    )
+    assert result.status_code == 200
+    assert result.json() == (
+        {"response_text": "response"} if response_format == "standard" else {"input_type": "response"}
+    )
+
+
+@pytest.mark.asyncio
+async def test_apply_guardrail_native_honors_explicit_input_direction(mocker: MockerFixture) -> None:
+    result: Final = await _apply_finding_guardrail_over_http(
+        mocker,
+        (),
+        {"guardrail_name": "grounding", "text": "prompt", "input_type": "request", "response_format": "native"},
+        guardrail=_InputDirectionGuardrail(()),
+    )
+    assert result.status_code == 200
+    assert result.json() == {"input_type": "request"}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("replacement", [{"is_grounded": False, "reason": "Revised"}, [{"score": 0.5}]])
+async def test_apply_guardrail_native_preserves_success_hook_replacements(
+    mocker: MockerFixture, replacement: JsonValue
+) -> None:
+    result: Final = await _apply_finding_guardrail_over_http(
+        mocker,
+        ({"is_grounded": True},),
+        {"guardrail_name": "grounding", "text": "answer", "response_format": "native"},
+        hook_result=replacement,
+    )
+    assert result.status_code == 200
+    assert result.json() == replacement
+
+
+@pytest.mark.asyncio
+async def test_apply_guardrail_native_ignores_malformed_caller_logging_records(mocker: MockerFixture) -> None:
+    result: Final = await _apply_finding_guardrail_over_http(
+        mocker,
+        ({"is_grounded": False},),
+        {
+            "guardrail_name": "grounding",
+            "text": "answer",
+            "response_format": "native",
+            "metadata": {"standard_logging_guardrail_information": "caller-controlled"},
+        },
+    )
+    assert result.status_code == 200
+    assert result.json() == {"is_grounded": False}
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     ("vendor_status", "expected_type"),
@@ -2212,9 +2413,7 @@ async def test_a_guardrail_vendors_rejection_is_labelled_the_way_chat_labels_it(
     mock_guardrail.apply_guardrail.side_effect = HTTPException(
         status_code=vendor_status, detail="vendor refused the request"
     )
-    mocker.patch(
-        "litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook", AsyncMock(return_value=None)
-    )
+    mocker.patch("litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook", AsyncMock(return_value=None))
 
     with pytest.raises(ProxyException) as raised:
         await apply_guardrail(
@@ -3673,6 +3872,7 @@ async def test_guardrail_success_delivers_sync_callback_once_with_overlapping_wo
     from litellm.integrations.custom_logger import CustomLogger
     from litellm.litellm_core_utils import litellm_logging, thread_pool_executor
     from litellm.proxy.guardrails.guardrail_endpoints import _emit_guardrail_success_logs
+    from litellm.proxy.guardrails.response_handlers import StandardGuardrailResponseHandler
     from litellm.types.guardrails import ApplyGuardrailResponse
 
     class OverlappingHook(CustomLogger):
@@ -3711,7 +3911,7 @@ async def test_guardrail_success_delivers_sync_callback_once_with_overlapping_wo
     response: Final = ApplyGuardrailResponse(response_text="masked")
 
     returned: Final = await _emit_guardrail_success_logs(
-        proxy_logging, logging_obj, {}, UserAPIKeyAuth(), response, start
+        StandardGuardrailResponseHandler(), proxy_logging, logging_obj, {}, UserAPIKeyAuth(), response, start
     )
     submissions: Final = tuple(queued_executor.submit.call_args_list)
     assert submissions

tests/unit/proxy/pass_through_endpoints/test_deepgram_ws_passthrough_routes.py

@@ -286,6 +286,20 @@ async def test_deepgram_listen_refuses_sessions_it_cannot_price(query, missing_k
     assert "dg-provider-key" not in websocket.closed[1]
 
 
+@pytest.mark.asyncio
+async def test_deepgram_listen_refusal_for_a_long_unpriced_model_fits_a_close_frame(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.delenv("DEEPGRAM_API_BASE", raising=False)
+    monkeypatch.setenv("DEEPGRAM_API_KEY", "dg-provider-key")
+    websocket: Final = _FakeWebSocket("/deepgram/v1/listen", "model=" + "%C3%A9" * 100)
+
+    relay: Final = await _serve(websocket)
+
+    assert relay.calls == []
+    assert websocket.closed == (1008, "No streaming price for 'deepgram/streaming/" + "é" * 40)
+
+
 @pytest.mark.asyncio
 async def test_deepgram_listen_relays_once_the_operator_prices_the_model(monkeypatch):
     monkeypatch.delenv("DEEPGRAM_API_BASE", raising=False)

tests/unit/proxy/response_polling/test_response_polling_handler.py

@@ -1917,3 +1917,71 @@ async def test_create_initial_state_rejects_request_metadata_that_is_not_an_obje
     assert [(error["loc"], error["type"], repr(error["input"])) for error in rejected.value.errors()] == [
         (("metadata",), "dict_type", repr(metadata))
     ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "delta_type",
+    [
+        "response.output_text.delta",
+        "response.reasoning_text.delta",
+        "response.reasoning_summary_text.delta",
+        "response.reasoning.delta",
+        None,
+    ],
+)
+async def test_background_timings_survive_retrieval_after_completion(delta_type):
+    from litellm.caching.redis_cache import RedisCache
+    from litellm.proxy.response_polling.background_streaming import background_streaming_task
+
+    cache = {}
+    redis = AsyncMock(spec=RedisCache)
+    redis.async_get_cache.side_effect = lambda key: cache.get(key)
+
+    async def save_cache(key, value, **kwargs):
+        cache[key] = value
+
+    redis.async_set_cache.side_effect = save_cache
+    handler = ResponsePollingHandler(redis_cache=redis)
+    await handler.create_initial_state("litellm_poll_timing", {"model": "gpt-4o"})
+    clock = [100.0]
+
+    async def stream():
+        clock[0] = 101.0
+        yield 'data: {"type":"response.output_text.delta","delta":""}\n\n'
+        yield 'data: {"type":"response.mcp_call.in_progress"}\n\n'
+        if delta_type:
+            clock[0] = 102.0
+            yield f"data: {json.dumps({'type': delta_type, 'delta': 'Hello'})}\n\n"
+            partial = await handler.get_state("litellm_poll_timing")
+            assert partial is not None
+            assert partial["status"] == "in_progress"
+            assert partial["litellm_response_timing"] == {"time_to_first_token_ms": 2000.0}
+        clock[0] = 110.0
+        yield 'data: {"type":"response.completed","response":{"status":"completed","output":[]}}\n\n'
+        clock[0] = 5000.0
+        yield "data: [DONE]\n\n"
+
+    from starlette.responses import StreamingResponse
+
+    with (
+        patch(
+            "litellm.proxy.response_polling.background_streaming.perf_counter",
+            create=True,
+            side_effect=lambda: clock[0],
+        ),
+        patch(
+            # test-quality-ok: processor is constructed inside the task; supply the upstream SSE boundary like sibling tests
+            "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
+        ) as processor,
+    ):
+        processor.return_value.base_process_llm_request = AsyncMock(return_value=StreamingResponse(stream()))
+        await background_streaming_task(**_make_background_streaming_kwargs("litellm_poll_timing", handler))
+        state = await handler.get_state("litellm_poll_timing")
+        assert state is not None
+        expected = {"total_latency_ms": 10000.0}
+        if delta_type:
+            expected["time_to_first_token_ms"] = 2000.0
+        assert state["litellm_response_timing"] == expected
+        clock[0] = 10000.0
+        assert (await handler.get_state("litellm_poll_timing"))["litellm_response_timing"] == expected

tests/unit/proxy/test__lazy_features.py

@@ -10,6 +10,7 @@ from fastapi.testclient import TestClient
 from pydantic import BaseModel
 
 from litellm.proxy._lazy_features import (
+    LAZY_FEATURES,
     LazyFeature,
     LazyFeatureMiddleware,
     attach_lazy_features,
@@ -66,6 +67,11 @@ def _has_lazy_middleware(app: FastAPI) -> bool:
     return any(middleware.cls is LazyFeatureMiddleware for middleware in app.user_middleware)
 
 
+def test_a2a_feature_matches_v1_message_stream() -> None:
+    feature: Final = next(feature for feature in LAZY_FEATURES if feature.name == "a2a")
+    assert feature.matches("/v1/a2a/x/message/stream")
+
+
 @pytest.mark.parametrize("disable_lazy_routes", ("false", "true"))
 def test_retired_roi_calculator_cannot_be_loaded_or_called(
     monkeypatch: pytest.MonkeyPatch, disable_lazy_routes: str

tests/unit/proxy/test_dev_reload.py

@@ -0,0 +1,216 @@
+import asyncio
+import importlib
+import logging
+import logging.handlers
+import traceback
+from pathlib import Path
+from typing import Final
+
+import pytest
+from starlette.types import Message, Scope
+from uvicorn import config as uvicorn_config
+
+from litellm.proxy.dev_reload import (
+    RELOAD_OPTIONS,
+    RELOAD_WORKER_HEALTHCHECK_SECONDS,
+    ReloadOptions,
+    load_app_or_exit,
+    load_worker_app,
+    multi_worker_reload_supported,
+    pool_config,
+)
+from litellm.proxy.proxy_cli import ProxyInitializationHelpers
+
+UNCHANGED_LOGGING: Final = {"version": 1, "incremental": True}
+POOL_WORKERS: Final = 2
+REQUIRES_UVICORN_STARTUP_FAILURE: Final = pytest.mark.skipif(
+    not multi_worker_reload_supported(), reason="uvicorn<0.50.0 has no startup failure exit code"
+)
+
+
+def _options(app: str) -> ReloadOptions:
+    return {
+        "app": app,
+        "log_config": UNCHANGED_LOGGING,
+        "reload": True,
+        "reload_dirs": (),
+        "reload_includes": ("*.py",),
+    }
+
+
+@REQUIRES_UVICORN_STARTUP_FAILURE
+@pytest.mark.parametrize(
+    ("module", "source"),
+    (
+        ("reload_syntax_error_app", "def app(\n"),
+        ("reload_name_error_app", "app = undefined_name\n"),
+        ("reload_runtime_error_app", "raise RuntimeError('broken edit')\n"),
+        ("reload_missing_attribute_app", "value = 1\n"),
+    ),
+)
+def test_a_broken_app_exits_the_worker_as_a_startup_failure(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, module: str, source: str
+) -> None:
+    (tmp_path / f"{module}.py").write_text(source)
+    monkeypatch.syspath_prepend(str(tmp_path))
+    with pytest.raises(SystemExit) as exit_info:
+        pool_config(_options(f"{module}:app"), workers=2).load()
+    assert exit_info.value.code == uvicorn_config.STARTUP_FAILURE
+
+
+@REQUIRES_UVICORN_STARTUP_FAILURE
+@pytest.mark.parametrize(
+    ("module", "source"),
+    (
+        (
+            "reload_factory_missing_module_app",
+            "def factory():\n    import a_module_that_does_not_exist\n",
+        ),
+        (
+            "reload_factory_runtime_error_app",
+            "def factory():\n    raise RuntimeError('broken edit')\n",
+        ),
+    ),
+)
+def test_a_broken_factory_exits_the_worker_as_a_startup_failure(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, module: str, source: str
+) -> None:
+    (tmp_path / f"{module}.py").write_text(source)
+    monkeypatch.syspath_prepend(str(tmp_path))
+    with pytest.raises(SystemExit) as exit_info:
+        pool_config(
+            _options(f"{module}:app"),
+            workers=POOL_WORKERS,
+            worker_overrides={"app": f"{module}:factory", "factory": True},
+        ).load()
+    assert exit_info.value.code == uvicorn_config.STARTUP_FAILURE
+
+
+@REQUIRES_UVICORN_STARTUP_FAILURE
+@pytest.mark.parametrize(
+    ("module", "source"),
+    (
+        ("reload_import_error_app", "import os\nfrom os import missing_name\n"),
+        ("reload_value_error_app", "import os\nraise ValueError('broken edit')\n"),
+    ),
+)
+def test_a_broken_import_logs_the_line_that_raised(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, module: str, source: str
+) -> None:
+    app_file: Final = tmp_path / f"{module}.py"
+    app_file.write_text(source)
+    monkeypatch.syspath_prepend(str(tmp_path))
+    uvicorn_logger: Final = logging.getLogger("uvicorn.error")
+    captured: Final = logging.handlers.BufferingHandler(capacity=POOL_WORKERS)
+    uvicorn_logger.addHandler(captured)
+    try:
+        with pytest.raises(SystemExit):
+            load_app_or_exit(f"{module}:app")
+    finally:
+        uvicorn_logger.removeHandler(captured)
+    (record,) = captured.buffer
+    assert record.exc_info is not None
+    raised_at: Final = frozenset((frame.filename, frame.lineno) for frame in traceback.extract_tb(record.exc_info[2]))
+    assert (str(app_file), 2) in raised_at, record.exc_text
+
+
+def test_the_reload_pool_defaults_the_worker_healthcheck_and_keeps_an_explicit_one() -> None:
+    explicit: Final = RELOAD_WORKER_HEALTHCHECK_SECONDS + 7
+    assert pool_config(_options("app:app"), workers=2).timeout_worker_healthcheck == RELOAD_WORKER_HEALTHCHECK_SECONDS
+    assert (
+        pool_config({**_options("app:app"), "timeout_worker_healthcheck": explicit}, workers=2).timeout_worker_healthcheck
+        == explicit
+    )
+
+
+def test_the_reload_pool_accepts_and_keeps_every_uvicorn_arg_the_proxy_cli_builds(tmp_path: Path) -> None:
+    cli_args: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args(
+        "127.0.0.1", 4123, keepalive_timeout=61, timeout_worker_healthcheck=RELOAD_WORKER_HEALTHCHECK_SECONDS + 7
+    )
+    options: Final = RELOAD_OPTIONS.validate_python(
+        {
+            **cli_args,
+            "log_config": UNCHANGED_LOGGING,
+            "reload": True,
+            "reload_dirs": [str(tmp_path)],
+            "reload_includes": ["*.py"],
+        }
+    )
+    config: Final = pool_config(options, workers=POOL_WORKERS)
+    server_args: Final = {key: value for key, value in cli_args.items() if key not in ("app", "log_config")}
+    assert {key: getattr(config, key) for key in server_args} == server_args
+
+
+def test_the_reload_pool_passes_a_uvicorn_arg_it_does_not_list_through_to_the_workers() -> None:
+    config: Final = pool_config(RELOAD_OPTIONS.validate_python({**_options("app:app"), "backlog": 123}), workers=POOL_WORKERS)
+    assert (config.backlog, config.reload) == (123, False)
+
+
+def test_the_reload_pool_serves_the_overridden_worker_app(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    module: Final = "reload_worker_override_app"
+    (tmp_path / f"{module}.py").write_text(
+        "async def base_app(scope, receive, send):\n"
+        "    await receive()\n"
+        "    await send({'type': 'http.response.start', 'status': 200, 'headers': []})\n"
+        "    await send({'type': 'http.response.body', 'body': b'base app', 'more_body': False})\n"
+        "\n"
+        "async def overridden_app(scope, receive, send):\n"
+        "    await receive()\n"
+        "    await send({'type': 'http.response.start', 'status': 200, 'headers': []})\n"
+        "    await send({'type': 'http.response.body', 'body': b'overridden app', 'more_body': False})\n"
+        "\n"
+        "def base_factory():\n"
+        "    return base_app\n"
+        "\n"
+        "def factory():\n"
+        "    return overridden_app\n"
+    )
+    monkeypatch.syspath_prepend(str(tmp_path))
+    config: Final = pool_config(
+        _options(f"{module}:base_factory"),
+        workers=POOL_WORKERS,
+        worker_overrides={"app": f"{module}:factory", "factory": True, "log_config": None},
+    )
+    assert config.log_config is None
+    config.load()
+    loaded_app: Final = config.loaded_app
+    assert loaded_app is not None
+    scope: Final[Scope] = {"type": "http", "method": "GET", "path": "/", "headers": []}
+
+    async def run_app() -> tuple[Message, ...]:
+        messages: Final[asyncio.Queue[Message]] = asyncio.Queue()
+
+        async def receive() -> Message:
+            return {"type": "http.request", "body": b"", "more_body": False}
+
+        async def send(message: Message) -> None:
+            await messages.put(message)
+
+        await loaded_app(scope, receive, send)
+        return tuple(messages.get_nowait() for _ in range(messages.qsize()))
+
+    messages: Final = asyncio.run(run_app())
+    assert messages == (
+        {"type": "http.response.start", "status": 200, "headers": []},
+        {"type": "http.response.body", "body": b"overridden app", "more_body": False},
+    )
+
+
+def test_load_worker_app_calls_the_overridden_factory(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    module: Final = "reload_worker_app_factory"
+    (tmp_path / f"{module}.py").write_text(
+        "def app(scope, receive, send):\n    return None\n\ndef factory():\n    return app\n"
+    )
+    monkeypatch.syspath_prepend(str(tmp_path))
+    app: Final = importlib.import_module(module).app
+    assert load_worker_app(f"{module}:factory", worker_app_factory=True) is app
+
+
+def test_windows_keeps_single_worker_reload_whatever_the_uvicorn_version() -> None:
+    assert not multi_worker_reload_supported("win32")
+
+
+def test_a_working_app_loads_as_the_module_defines_it(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    (tmp_path / "reload_working_app.py").write_text("async def app(scope, receive, send):\n    return None\n")
+    monkeypatch.syspath_prepend(str(tmp_path))
+    assert load_app_or_exit("reload_working_app:app") is importlib.import_module("reload_working_app").app

tests/unit/proxy/test_proxy_server.py

@@ -4561,3 +4561,88 @@ async def test_text_completion_provider_rejection_captures_unknown_usage(
     assert envelope["usage"] is None
     assert envelope["usage_source"] == "unknown"
     assert envelope["pricing"]["reported_cost"] is None
+
+
+@pytest.mark.parametrize(
+    "polling,has_redis,native,expected",
+    [
+        (False, True, False, False),
+        ("all", False, False, False),
+        ("all", True, True, False),
+        ("all", True, False, True),
+        (["openai"], True, False, True),
+        (["anthropic"], True, False, False),
+    ],
+)
+def test_model_groups_advertise_cache_backed_background_chat(
+    monkeypatch: pytest.MonkeyPatch,
+    polling: bool | str | list[str],
+    has_redis: bool,
+    native: bool,
+    expected: bool,
+) -> None:
+    from litellm.caching.redis_cache import RedisCache
+    from litellm.proxy import proxy_server
+
+    router: Final = litellm.Router(
+        model_list=[
+            {
+                "model_name": "research-chat",
+                "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "fake-test-key"},
+            }
+        ]
+    )
+    monkeypatch.setattr(proxy_server, "polling_via_cache_enabled", polling)
+    monkeypatch.setattr(
+        proxy_server, "redis_usage_cache", RedisCache(host="localhost", port=6379) if has_redis else None
+    )
+    monkeypatch.setattr(proxy_server, "native_background_mode", ["research-chat"] if native else [])
+    groups: Final = proxy_server.get_model_group_info(router, ["research-chat"], None)
+    assert groups[0].supports_background_chat is expected
+
+
+@pytest.mark.parametrize("config_owned", [False, True])
+def test_background_response_settings_refresh_and_respect_config(
+    monkeypatch: pytest.MonkeyPatch,
+    config_owned: bool,
+) -> None:
+    from litellm.proxy import proxy_server
+
+    monkeypatch.setattr(proxy_server, "polling_via_cache_enabled", False)
+    monkeypatch.setattr(proxy_server, "native_background_mode", [])
+    monkeypatch.setattr(proxy_server, "polling_cache_ttl", 3600)
+    config: Final = proxy_server.ProxyConfig()
+    if config_owned:
+        config.litellm_settings.load_yaml(
+            {
+                "responses": {
+                    "background_mode": {
+                        "polling_via_cache": ["openai"],
+                        "native_background_mode": ["config-native"],
+                        "ttl": 900,
+                    }
+                },
+            }
+        )
+    config._apply_litellm_settings_db_values(
+        {
+            "responses": {
+                "background_mode": {
+                    "polling_via_cache": "all",
+                    "native_background_mode": ["db-native"],
+                    "ttl": 7200,
+                }
+            },
+        }
+    )
+    assert proxy_server.polling_via_cache_enabled == (["openai"] if config_owned else "all")
+    assert proxy_server.native_background_mode == (["config-native"] if config_owned else ["db-native"])
+    assert proxy_server.polling_cache_ttl == (900 if config_owned else 7200)
+    config._apply_litellm_settings_db_values(
+        {
+            "responses": {"background_mode": {"polling_via_cache": False}},
+        }
+    )
+    assert proxy_server.polling_via_cache_enabled == (["openai"] if config_owned else False)
+    assert proxy_server.native_background_mode == (["config-native"] if config_owned else [])
+    assert proxy_server.polling_cache_ttl == (900 if config_owned else 3600)

tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py

@@ -4488,3 +4488,114 @@ def test_usage_default_range_reports_config_precedence(mock_auth, monkeypatch: p
     assert response.status_code == 200, response.text
     assert response.json()["values"]["usage_default_time_range"] == "month_to_date"
     assert response.json()["source"]["usage_default_time_range"] == "config"
+
+
+@pytest.fixture
+def background_chat_config(monkeypatch: pytest.MonkeyPatch):
+    from copy import deepcopy
+    from unittest.mock import AsyncMock, MagicMock
+
+    from litellm.caching.redis_cache import RedisCache
+    from litellm.proxy import proxy_server
+    from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+    from litellm.proxy.config_resolvers import SettingsStore
+
+    config = {
+        "litellm_settings": {
+            "callbacks": ["prometheus"],
+            "responses": {
+                "background_mode": {
+                    "polling_via_cache": False,
+                    "ttl": 7200,
+                    "native_background_mode": ["native-model"],
+                },
+            },
+        }
+    }
+    store = SettingsStore("litellm_settings")
+    monkeypatch.setattr(proxy_server.proxy_config, "litellm_settings", store)
+    monkeypatch.setattr(proxy_server.proxy_config, "_settings_stores", {"litellm_settings": store})
+    monkeypatch.setattr(proxy_server, "prisma_client", MagicMock())
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+    monkeypatch.setattr(proxy_server, "redis_usage_cache", RedisCache(host="localhost", port=6379))
+    monkeypatch.setattr(proxy_server, "polling_via_cache_enabled", False)
+    monkeypatch.setattr(proxy_server, "native_background_mode", [])
+    monkeypatch.setattr(proxy_server, "polling_cache_ttl", 3600)
+    monkeypatch.setattr(proxy_server.proxy_config, "get_config", AsyncMock(side_effect=lambda: deepcopy(config)))
+
+    async def save_config(*, new_config):
+        config.clear()
+        config.update(deepcopy(new_config))
+
+    save = AsyncMock(side_effect=save_config)
+    audit = AsyncMock()
+    monkeypatch.setattr(proxy_server.proxy_config, "save_config", save)
+    monkeypatch.setattr(proxy_server, "create_config_audit_log", audit)
+    app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
+        user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN
+    )
+    yield config, save, audit
+    app.dependency_overrides.pop(user_api_key_auth, None)
+
+
+def test_background_chat_settings_persist_preserving_existing_options(background_chat_config, monkeypatch):
+    from litellm.proxy import proxy_server
+
+    config, _, _ = background_chat_config
+    response = client.patch("/update/background_chat_settings", json={"enabled": True})
+    assert response.status_code == 200
+    assert response.json()["enabled"] is True
+    assert proxy_server.polling_via_cache_enabled is False
+    assert config["litellm_settings"]["responses"]["background_mode"] == {
+        "polling_via_cache": "all",
+        "ttl": 7200,
+        "native_background_mode": ["native-model"],
+    }
+    assert config["litellm_settings"]["callbacks"] == ["prometheus"]
+    assert client.get("/get/background_chat_settings").json()["enabled"] is True
+    proxy_server.proxy_config._apply_litellm_settings_db_values(config["litellm_settings"])
+    assert proxy_server.polling_via_cache_enabled == "all"
+    disabled = client.patch("/update/background_chat_settings", json={"enabled": False})
+    assert disabled.status_code == 200
+    assert disabled.json()["enabled"] is False
+    proxy_server.proxy_config._apply_litellm_settings_db_values(config["litellm_settings"])
+    assert proxy_server.polling_via_cache_enabled is False
+
+
+@pytest.mark.parametrize("blocker", ["role", "redis", "config", "persistence"])
+def test_background_chat_settings_reject_unavailable_updates(background_chat_config, monkeypatch, blocker):
+    from litellm.proxy import proxy_server
+    from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+
+    config, save, _ = background_chat_config
+    if blocker == "role":
+        app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
+            user_id="viewer", user_role=LitellmUserRoles.INTERNAL_USER
+        )
+    elif blocker == "redis":
+        monkeypatch.setattr(proxy_server, "redis_usage_cache", None)
+        assert client.get("/get/background_chat_settings").json()["redis_configured"] is False
+    elif blocker == "config":
+        proxy_server.proxy_config.litellm_settings.load_yaml(config["litellm_settings"])
+        assert client.get("/get/background_chat_settings").json()["managed_by_config"] is True
+    else:
+        monkeypatch.setattr(proxy_server, "store_model_in_db", False)
+        assert client.get("/get/background_chat_settings").json()["can_save"] is False
+    response = client.patch("/update/background_chat_settings", json={"enabled": True})
+    assert response.status_code == (403 if blocker == "role" else 400)
+    save.assert_not_awaited()
+    assert config["litellm_settings"]["responses"]["background_mode"]["polling_via_cache"] is False
+
+
+def test_background_chat_settings_failed_save_does_not_enable_runtime(background_chat_config):
+    from litellm.proxy import proxy_server
+
+    _, save, audit = background_chat_config
+    save.side_effect = RuntimeError("Database write failed")
+    with pytest.raises(RuntimeError, match="Database write failed"):
+        client.patch("/update/background_chat_settings", json={"enabled": True})
+    assert client.get("/get/background_chat_settings").json()["enabled"] is False
+    assert proxy_server.polling_via_cache_enabled is False
+    audit.assert_not_awaited()

tests/unit/router_strategy/complexity_router/test_jev_classifier.py

@@ -1316,3 +1316,24 @@ async def test_jev_open_breaker_preserves_input_budget_in_fallback_decision(fall
         transport.handle_async_request.assert_awaited_once()
     finally:
         await handler.client.aclose()
+
+
+def test_jev_input_targets_still_takes_a_router_with_a_deprecation_warning() -> None:
+    from litellm.router_strategy.complexity_router.jev_classifier import jev_input_targets
+
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "decision-deployment",
+                "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"},
+                "model_info": {"id": "d1", "max_input_tokens": 1234},
+            }
+        ]
+    )
+    config = JevClassifierConfig.model_validate({"deployment_name": "decision-deployment"})
+
+    with pytest.warns(DeprecationWarning, match="pass context"):
+        old_way = jev_input_targets(router, config)
+
+    assert old_way == jev_input_targets(router._core_ctx, config)
+    assert [target.max_tokens for target in old_way] == [1234]

tests/unit/router_strategy/test_litellm_encoder.py

@@ -134,3 +134,19 @@ class TestEmbeddingInputCap:
         _encoder(router, max_input_chars=20).encode_queries(["route me please, then a huge pasted file"])
 
         assert router.embedded_inputs == [["route me please, the"]]
+
+
+@pytest.mark.parametrize(
+    "embeds",
+    [
+        {"data": [{"embedding": [0.1]}]},
+        [{"embedding": [0.1]}],
+        type("HasData", (), {"data": [{"embedding": [0.1]}]})(),
+    ],
+    ids=["dict", "list", "object-with-data"],
+)
+def test_litellm_to_list_rejects_anything_but_an_embedding_response(embeds: object) -> None:
+    from litellm.router_strategy.auto_router.litellm_encoder import litellm_to_list
+
+    with pytest.raises(ValueError, match="No embeddings found"):
+        litellm_to_list(embeds)

tests/unit/router_strategy/test_router_tag_routing.py

@@ -1225,6 +1225,32 @@ def test_chain_tag_filtering_override_falls_back_to_healthy_deployments_on_looku
     assert _chain_tag_filtering_override(_BrokenCtx(), "gpt-4", healthy_deployments) is False
 
 
+@pytest.mark.asyncio()
+async def test_get_deployments_for_tag_without_a_model_keeps_the_router_wide_setting():
+    # model=None names no group, so no group-level enable_tag_filtering applies: the router-wide
+    # setting (off here) decides, even when a healthy deployment opts in on its own
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-4",
+                "litellm_params": {"model": "gpt-4o", "api_key": "sk-test", "tags": ["teamA"]},
+                "model_info": {"id": "a", "enable_tag_filtering": True},
+            }
+        ]
+    )
+    healthy_deployments = router.registry().deployments("gpt-4", source="all")
+
+    picked = await get_deployments_for_tag(
+        router._core_ctx,
+        None,  # pyright: ignore[reportArgumentType]  # a direct caller passing no model
+        healthy_deployments,
+        request_kwargs={"metadata": {"tags": ["zzz"]}},
+        request_deployments=healthy_deployments,
+    )
+
+    assert [d["model_info"]["id"] for d in picked] == ["a"]
+
+
 @pytest.mark.asyncio()
 async def test_chain_enable_tag_filtering_true_overrides_router_level_false():
     # Router-wide tag filtering is off; this model group opts in on its own via

tests/unit/router_utils/test_handle_error.py

@@ -273,3 +273,10 @@ def setup_and_teardown():
     yield
     loop.close()
     asyncio.set_event_loop(None)
+
+
+@pytest.mark.asyncio
+async def test_send_llm_exception_alert_without_a_router_does_nothing():
+    from litellm.router_utils.handle_error import send_llm_exception_alert
+
+    assert await send_llm_exception_alert(None, {"model": "gpt-5-mini"}, "traceback", Exception("boom")) is None

tests/unit/test_guardrails.py

@@ -0,0 +1,119 @@
+from types import MappingProxyType
+from typing import Final
+
+import httpx
+import pytest
+import respx
+from pydantic import JsonValue, TypeAdapter, ValidationError
+
+import litellm
+
+_REQUEST: Final = TypeAdapter(dict[str, JsonValue])
+
+
+@pytest.fixture(autouse=True)
+def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("base", ["https://gateway.example", "https://gateway.example/v1/"])
+async def test_apply_returns_native_findings_using_shared_gateway_settings(
+    monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, base: str
+) -> None:
+    monkeypatch.setattr(litellm, "api_base", base)
+    monkeypatch.setattr(litellm, "api_key", "test-key")
+    verdict: Final = {"is_grounded": False, "reason": "The source gives a different height.", "source_ids": []}
+    route: Final = respx_mock.post("https://gateway.example/guardrails/apply_guardrail").respond(json=verdict)
+    messages: Final = [{"role": "user", "content": "The tower is 330 metres tall."}]
+
+    result: Final = await litellm.guardrails.apply("grounding", response="450 metres", messages=messages)
+
+    assert result == verdict
+    assert route.calls.last.request.headers["Authorization"] == "Bearer test-key"
+    assert _REQUEST.validate_json(route.calls.last.request.content) == {
+        "guardrail_name": "grounding",
+        "text": "450 metres",
+        "input_type": "response",
+        "response_format": "native",
+        "messages": messages,
+    }
+
+
+@pytest.mark.asyncio
+async def test_apply_infers_input_direction_and_preserves_empty_values(
+    respx_mock: respx.MockRouter,
+) -> None:
+    route: Final = respx_mock.post("https://gateway.example/guardrails/apply_guardrail").respond(json={"safe": True})
+
+    result: Final = await litellm.guardrails.apply(
+        "safety", input="", messages=[], metadata={}, api_base="https://gateway.example", api_key="explicit-key"
+    )
+
+    assert result == {"safe": True}
+    assert route.calls.last.request.headers["Authorization"] == "Bearer explicit-key"
+    assert _REQUEST.validate_json(route.calls.last.request.content) == {
+        "guardrail_name": "safety",
+        "text": "",
+        "input_type": "request",
+        "response_format": "native",
+        "messages": [],
+        "metadata": {},
+    }
+
+
+@pytest.mark.asyncio
+async def test_apply_serializes_read_only_messages_and_metadata(respx_mock: respx.MockRouter) -> None:
+    route: Final = respx_mock.post("https://gateway.example/guardrails/apply_guardrail").respond(
+        json={"is_grounded": True}
+    )
+    messages: Final = (MappingProxyType({"role": "user", "content": "The tower is 330 metres tall."}),)
+    metadata: Final = MappingProxyType({"trace_id": "demo", "source_ids": ["doc-1"], "options": {"report_only": True}})
+
+    result: Final = await litellm.guardrails.apply(
+        "grounding", response="330 metres", messages=messages, metadata=metadata, api_base="https://gateway.example"
+    )
+
+    assert result == {"is_grounded": True}
+    assert _REQUEST.validate_json(route.calls.last.request.content) == {
+        "guardrail_name": "grounding",
+        "text": "330 metres",
+        "input_type": "response",
+        "response_format": "native",
+        "messages": [{"role": "user", "content": "The tower is 330 metres tall."}],
+        "metadata": {"trace_id": "demo", "source_ids": ["doc-1"], "options": {"report_only": True}},
+    }
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("texts", [{}, {"input": "prompt", "response": "answer"}])
+async def test_apply_requires_exactly_one_text(texts: dict[str, str], respx_mock: respx.MockRouter) -> None:
+    with pytest.raises(ValueError, match="exactly one"):
+        await litellm.guardrails.apply(
+            "grounding", input=texts.get("input"), response=texts.get("response"), api_base="https://gateway.example"
+        )
+    assert not respx_mock.calls
+
+
+@pytest.mark.asyncio
+async def test_apply_requires_a_gateway_url(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(litellm, "api_base", None)
+    with pytest.raises(ValueError, match="gateway URL"):
+        await litellm.guardrails.apply("grounding", response="answer")
+
+
+@pytest.mark.asyncio
+async def test_apply_surfaces_guardrail_http_failures(respx_mock: respx.MockRouter) -> None:
+    respx_mock.post("https://gateway.example/guardrails/apply_guardrail").respond(
+        422, json={"error": {"message": "Guardrail did not report structured native findings."}}
+    )
+    with pytest.raises(httpx.HTTPStatusError) as failure:
+        await litellm.guardrails.apply("grounding", response="answer", api_base="https://gateway.example")
+    assert failure.value.response.status_code == 422
+
+
+@pytest.mark.asyncio
+async def test_apply_rejects_an_unstructured_response(respx_mock: respx.MockRouter) -> None:
+    respx_mock.post("https://gateway.example/guardrails/apply_guardrail").respond(json="allow")
+    with pytest.raises(ValidationError):
+        await litellm.guardrails.apply("grounding", response="answer", api_base="https://gateway.example")

tests/unit/test_router/core/test_endpoints.py

@@ -414,6 +414,9 @@ def _scripted_stream(*texts: str) -> Callable[..., CustomStreamWrapper]:
         async def chunks() -> AsyncIterator[ModelResponseStream]:
             for text in texts:
                 yield ModelResponseStream(choices=[{"index": 0, "delta": {"role": "assistant", "content": text}}])
+            # A stream that ends without a finish reason fails since #47665.
+            yield ModelResponseStream(choices=[{"index": 0, "delta": {}, "finish_reason": "stop"}])
+            yield ModelResponseStream(choices=[{"index": 0, "delta": {}, "finish_reason": "stop"}])
 
         logging_obj: Final = LiteLLMLogging(
             model="d1",

ui/litellm-dashboard/eslint-suppressions.json

@@ -1142,11 +1142,6 @@
       "count": 1
     }
   },
-  "src/app/chat/page.tsx": {
-    "no-nested-ternary": {
-      "count": 1
-    }
-  },
   "src/app/login/LoginPage.tsx": {
     "local/no-complex-jsx-arrow": {
       "count": 1
@@ -1737,10 +1732,10 @@
       "count": 5
     },
     "no-restricted-syntax": {
-      "count": 139
+      "count": 133
     },
     "prefer-const": {
-      "count": 28
+      "count": 26
     }
   },
   "src/components/object_permissions_view.tsx": {

ui/litellm-dashboard/src/app/(dashboard)/components/SidebarProvider.test.tsx

@@ -9,7 +9,7 @@ const { getSettings } = vi.hoisted(() => ({
   getSettings: vi.fn().mockResolvedValue({ values: { enable_projects_ui: true } }),
 }));
 
-vi.mock("@/components/networking", () => ({ getUiSettings: getSettings, getUISettings: getSettings }));
+vi.mock("@/components/networking", () => ({ getUiSettings: getSettings }));
 vi.mock("@/components/leftnav", () => ({
   default: ({ enableProjectsUI }: ComponentProps<typeof Sidebar>) => (
     <nav>{enableProjectsUI && <a href="/projects">Projects</a>}</nav>

ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx

@@ -555,7 +555,7 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
         title={`Spend Per ${capitalizedEntityLabel}`}
         action={
           <a
-            href="https://docs.litellm.ai/docs/proxy/enterprise#spend-tracking"
+            href="https://docs.litellm.ai/docs/proxy/cost_tracking"
             className="inline-flex items-center gap-1 text-xs text-muted-foreground hover:text-foreground"
           >
             Track cost per {entityType}

ui/litellm-dashboard/src/app/chat/page.background.integration.test.tsx

@@ -0,0 +1,352 @@
+import { act, fireEvent, screen, waitFor, within } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { afterEach, beforeEach, expect, it, vi } from "vitest";
+import { renderWithProviders, testQueryClient } from "@/../tests/test-utils";
+import { ChatShellProvider } from "@/contexts/ChatShellContext";
+import type { Conversation } from "@/components/chat/types";
+import ChatConversationPage from "./page";
+
+vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn(), replace: vi.fn() }) }));
+
+const storageKey = "litellm_chat_history_v1:background-user";
+const requests: Record<string, unknown>[] = [];
+let supported: boolean;
+let webSearchSupported: boolean;
+let status: "in_progress" | "completed" | "cancelled";
+let retrievalStatus: number;
+let cancellations: number;
+let acceptance: Promise<void> | undefined;
+const saved = (): Conversation[] => JSON.parse(localStorage.getItem(storageKey) ?? "[]");
+const hasMCPTools = () => {
+  const tools = requests.at(-1)?.tools;
+  return Array.isArray(tools) && tools.some((tool) => tool.type === "mcp");
+};
+
+const result = () => ({
+  id: "litellm_poll_research",
+  status,
+  usage: status === "completed" ? { input_tokens: 4, output_tokens: 9, total_tokens: 13, cost: "0.125" } : undefined,
+  litellm_response_timing:
+    status === "completed" ? { time_to_first_token_ms: 250, total_latency_ms: 12500 } : undefined,
+  output:
+    status === "completed"
+      ? [
+          {
+            type: "mcp_call",
+            id: "call_inventory",
+            name: "lookup_inventory",
+            arguments: "{}",
+            output: "12 widgets",
+            server_label: "inventory",
+          },
+          {
+            type: "message",
+            content: [
+              {
+                type: "output_text",
+                text: "There are 12 widgets in stock.",
+                annotations: [
+                  {
+                    type: "url_citation",
+                    url: "https://example.com/inventory",
+                    title: "Inventory report",
+                    start_index: 0,
+                    end_index: 6,
+                  },
+                ],
+              },
+            ],
+          },
+        ].filter((item) => item.type !== "mcp_call" || hasMCPTools())
+      : [],
+});
+
+function renderChat(id?: string) {
+  return renderWithProviders(
+    <ChatShellProvider
+      accessToken="test-token"
+      userId="background-user"
+      userEmail="researcher@example.com"
+      userRole="Internal Viewer"
+      premiumUser={false}
+    >
+      <ChatConversationPage />
+    </ChatShellProvider>,
+    { searchParams: id ? { id } : {} },
+  );
+}
+
+beforeEach(() => {
+  localStorage.clear();
+  testQueryClient.clear();
+  requests.length = 0;
+  cancellations = 0;
+  acceptance = undefined;
+  supported = true;
+  webSearchSupported = true;
+  retrievalStatus = 200;
+  status = "in_progress";
+  vi.stubGlobal(
+    "fetch",
+    vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
+      const request =
+        input instanceof Request ? input : new Request(new URL(String(input), window.location.origin), init);
+      const path = new URL(request.url).pathname;
+      if (path.endsWith("/model_group/info"))
+        return Response.json({
+          data: [
+            {
+              model_group: "research-chat",
+              mode: "chat",
+              supports_background_chat: supported,
+              supports_web_search: webSearchSupported,
+              supports_reasoning: true,
+            },
+          ],
+        });
+      if (path.endsWith("/v1/mcp/server")) return Response.json([{ server_id: "inventory", server_name: "inventory" }]);
+      if (path.endsWith("/mcp-rest/tools/list")) return Response.json({ tools: [{ name: "lookup_inventory" }] });
+      if (path.endsWith("/responses")) {
+        const body = JSON.parse(await request.text());
+        requests.push(body);
+        if (body.background) {
+          await acceptance;
+          request.signal.throwIfAborted();
+          return Response.json({ ...result(), status: "queued" });
+        }
+        return new Response(
+          'data: {"type":"response.output_text.delta","delta":"Foreground answer"}\n\ndata: {"type":"response.completed","response":{"id":"resp_foreground","output":[]}}\n\n',
+          { headers: { "Content-Type": "text/event-stream" } },
+        );
+      }
+      if (path.endsWith("/litellm_poll_research/cancel")) {
+        cancellations++;
+        status = "cancelled";
+        return Response.json(result());
+      }
+      if (path.endsWith("/litellm_poll_research"))
+        return Response.json(retrievalStatus === 200 ? result() : { error: { message: "Unavailable" } }, {
+          status: retrievalStatus,
+        });
+      return Response.json({}, { status: 404 });
+    }),
+  );
+});
+afterEach(() => vi.unstubAllGlobals());
+
+it("should switch the composer placeholder with deep research on home and in a conversation", async () => {
+  const user = userEvent.setup();
+  const home = renderChat();
+  await screen.findByRole("button", { name: "research-chat" });
+  const homeEditor = screen.getByRole("textbox", { name: "Message" });
+  expect(homeEditor).toHaveAttribute("placeholder", "Ask a question, attach a file, or paste something to work on…");
+
+  const enableResearch = async () => {
+    await user.click(screen.getByRole("button", { name: "Connect tools" }));
+    await user.click(await screen.findByRole("button", { name: "Deep research", exact: true }));
+  };
+  await enableResearch();
+  expect(homeEditor).toHaveAttribute("placeholder", "What should I research on the web?");
+  await user.click(screen.getByRole("button", { name: "Turn off deep research" }));
+  expect(homeEditor).toHaveAttribute("placeholder", "Ask a question, attach a file, or paste something to work on…");
+
+  home.unmount();
+  const conversationId = "saved-chat";
+  const history: Conversation[] = [
+    {
+      id: conversationId,
+      title: "Saved chat",
+      model: "research-chat",
+      messages: [{ id: "saved-message", role: "user", content: "Previous question", timestamp: 1 }],
+      mcpServerNames: [],
+      createdAt: 1,
+      updatedAt: 1,
+    },
+  ];
+  localStorage.setItem(storageKey, JSON.stringify(history));
+  renderChat(conversationId);
+  await screen.findByRole("button", { name: "research-chat" });
+  const conversationEditor = screen.getByRole("textbox", { name: "Message" });
+  expect(conversationEditor).toHaveAttribute("placeholder", "Send a message...");
+
+  await enableResearch();
+  expect(conversationEditor).toHaveAttribute("placeholder", "What should I research on the web?");
+  await user.click(screen.getByRole("button", { name: "Turn off deep research" }));
+  expect(conversationEditor).toHaveAttribute("placeholder", "Send a message...");
+});
+
+async function startResearch(withTools = true) {
+  const user = userEvent.setup();
+  await screen.findByRole("button", { name: "research-chat" });
+  if (withTools) {
+    await user.click(screen.getByRole("button", { name: "Connect tools" }));
+    await user.click(await screen.findByRole("switch"));
+    await user.keyboard("{Escape}");
+  }
+  fireEvent.change(screen.getByRole("textbox", { name: "Message", exact: true }), {
+    target: { value: "Check inventory" },
+  });
+  await user.click(screen.getByRole("button", { name: "Send", exact: true }));
+}
+
+it("should retrieve the completed tool run after closing and reopening without submitting it again", async () => {
+  const first = renderChat();
+  await startResearch();
+  await screen.findByText("Running in the background. You can close this tab and return to this chat.");
+  await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.id).toBe("litellm_poll_research"));
+  const id = saved()[0].id;
+  first.unmount();
+  expect(cancellations).toBe(0);
+  status = "completed";
+  renderChat(id);
+  expect(await screen.findByText("There are 12 widgets in stock.")).toBeVisible();
+  expect(screen.getByRole("button", { name: "lookup_inventory" })).toBeVisible();
+  expect(screen.getByLabelText(/^Cost:/)).toHaveTextContent("$0.125");
+  expect(screen.getByLabelText(/^TTFT:/)).toHaveTextContent("0.25s");
+  expect(screen.getByLabelText(/^Total Latency:/)).toHaveTextContent("12.50s");
+  await waitFor(() =>
+    expect(saved()[0]?.messages.at(-1)).toMatchObject({ timeToFirstToken: 250, totalLatency: 12500 }),
+  );
+  expect(requests).toHaveLength(1);
+  expect(requests[0]).toMatchObject({
+    background: true,
+    stream: false,
+    tools: [{ type: "mcp", server_label: "inventory" }],
+  });
+  expect(screen.queryByRole("button", { name: "Stop request" })).not.toBeInTheDocument();
+  fireEvent.change(screen.getByRole("textbox", { name: "Message", exact: true }), {
+    target: { value: "What was the result?" },
+  });
+  fireEvent.click(screen.getByRole("button", { name: "Send", exact: true }));
+  await waitFor(() => expect(requests).toHaveLength(2));
+  expect(requests[1]).toMatchObject({ background: true, stream: false });
+  expect(requests[1]).not.toHaveProperty("previous_response_id");
+  expect(requests[1].input).toEqual(
+    expect.arrayContaining([{ type: "function_call_output", call_id: "call_inventory", output: "12 widgets" }]),
+  );
+});
+
+it("should continue an ordinary chat without MCP tools after closing and reopening", async () => {
+  const first = renderChat();
+  await startResearch(false);
+  await screen.findByText("Running in the background. You can close this tab and return to this chat.");
+  await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.id).toBe("litellm_poll_research"));
+  expect(requests[0]).toMatchObject({ background: true, stream: false });
+  expect(requests[0]).not.toHaveProperty("tools");
+  expect(requests[0]).not.toHaveProperty("instructions");
+  const id = saved()[0].id;
+  first.unmount();
+  status = "completed";
+  renderChat(id);
+  expect(await screen.findByText("There are 12 widgets in stock.")).toBeVisible();
+  expect(screen.getByLabelText(/^TTFT:/)).toHaveTextContent("0.25s");
+  expect(screen.getByLabelText(/^Total Latency:/)).toHaveTextContent("12.50s");
+  await waitFor(() =>
+    expect(saved()[0]?.messages.at(-1)).toMatchObject({ timeToFirstToken: 250, totalLatency: 12500 }),
+  );
+  expect(requests).toHaveLength(1);
+  expect(cancellations).toBe(0);
+});
+
+it("should cancel a background run only when Stop is clicked", async () => {
+  renderChat();
+  await startResearch();
+  await screen.findByText("Running in the background. You can close this tab and return to this chat.");
+  fireEvent.click(screen.getByRole("button", { name: "Stop request" }));
+  expect(await screen.findByText("Cancellation requested")).toBeVisible();
+  expect(cancellations).toBe(1);
+});
+
+it("should retain ordinary streaming when background support is unavailable", async () => {
+  supported = false;
+  renderChat();
+  await startResearch();
+  expect(await screen.findByText("Foreground answer")).toBeVisible();
+  expect(requests[0]).toMatchObject({ stream: true });
+  expect(requests[0]).not.toHaveProperty("background");
+  expect(requests[0]).not.toHaveProperty("instructions");
+  expect(requests[0].tools).not.toContainEqual(expect.objectContaining({ type: "web_search" }));
+  expect(screen.getByText("Keep this tab open. Background runs are not available for this chat.")).toBeVisible();
+});
+
+it("should let the user reconnect after an authentication failure without resubmitting work", async () => {
+  const first = renderChat();
+  await startResearch();
+  await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.id).toBe("litellm_poll_research"));
+  const id = saved()[0].id;
+  first.unmount();
+  retrievalStatus = 401;
+  renderChat(id);
+  await screen.findByRole("button", { name: "Reconnect" });
+  retrievalStatus = 200;
+  status = "completed";
+  fireEvent.click(screen.getByRole("button", { name: "Reconnect" }));
+  expect(await screen.findByText("There are 12 widgets in stock.")).toBeVisible();
+  expect(requests).toHaveLength(1);
+});
+
+it.each([false, true])(
+  "should honor Stop before a background submission returns its ID (tools=%s)",
+  async (withTools) => {
+    let accept!: () => void;
+    acceptance = new Promise<void>((resolve) => {
+      accept = resolve;
+    });
+    renderChat();
+    await startResearch(withTools);
+    await waitFor(() => expect(requests).toHaveLength(1));
+    fireEvent.click(screen.getByRole("button", { name: "Stop request" }));
+    await act(async () => accept());
+    expect(await screen.findByText("Cancellation requested")).toBeVisible();
+    expect(cancellations).toBe(1);
+    expect(requests).toHaveLength(1);
+    await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.status).toBe("cancelled"));
+  },
+);
+
+it("should restore a web research report and its source links after closing and reopening twice", async () => {
+  const first = renderChat();
+  await screen.findByRole("button", { name: "research-chat" });
+  fireEvent.click(screen.getByRole("button", { name: "Connect tools" }));
+  fireEvent.click(await screen.findByRole("button", { name: "Deep research", exact: true }));
+  fireEvent.change(screen.getByRole("textbox", { name: "Message", exact: true }), {
+    target: { value: "Research the inventory" },
+  });
+  fireEvent.click(screen.getByRole("button", { name: "Send", exact: true }));
+  await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.id).toBe("litellm_poll_research"));
+  expect(requests).toHaveLength(1);
+  expect(requests[0]).toMatchObject({ background: true, stream: false, tools: [{ type: "web_search" }] });
+  expect(requests[0].instructions).toEqual(expect.stringContaining("sources you actually consulted"));
+  expect(saved()[0]?.messages.at(-1)?.research).toBe(true);
+  const id = saved()[0].id;
+  first.unmount();
+  expect(cancellations).toBe(0);
+  status = "completed";
+  const second = renderChat(id);
+  const sources = await screen.findByLabelText("Sources");
+  expect(within(sources).getByRole("link", { name: "Inventory report" })).toHaveAttribute(
+    "href",
+    "https://example.com/inventory",
+  );
+  await waitFor(() =>
+    expect(saved()[0]?.messages.at(-1)?.citations).toEqual([
+      { url: "https://example.com/inventory", title: "Inventory report" },
+    ]),
+  );
+  second.unmount();
+  renderChat(id);
+  expect(
+    within(await screen.findByLabelText("Sources")).getByRole("link", { name: "Inventory report" }),
+  ).toHaveAttribute("href", "https://example.com/inventory");
+  expect(requests).toHaveLength(1);
+});
+
+it("should explain when the selected model cannot search instead of sending an unsupported research request", async () => {
+  webSearchSupported = false;
+  renderChat();
+  await screen.findByRole("button", { name: "research-chat" });
+  fireEvent.click(screen.getByRole("button", { name: "Connect tools" }));
+  expect(await screen.findByRole("button", { name: "Deep research", exact: true })).toBeDisabled();
+  expect(screen.getByText("Choose a reasoning model with web search.")).toBeVisible();
+  expect(requests).toHaveLength(0);
+});

ui/litellm-dashboard/src/app/chat/page.integration.test.tsx

@@ -64,7 +64,7 @@ vi.mock("@/contexts/ChatShellContext", () => ({
   },
 }));
 
-const ONE_TURN_ARG_COUNT = 25;
+const ONE_TURN_ARG_COUNT = 29;
 const ON_TIMING_DATA_INDEX = 7;
 const ON_USAGE_DATA_INDEX = 8;
 const ON_TOTAL_LATENCY_INDEX = 24;
@@ -72,7 +72,7 @@ const ON_TOTAL_LATENCY_INDEX = 24;
 async function sendOneMessage(onUrlUpdate?: OnUrlUpdateFunction): Promise<void> {
   renderWithProviders(<ChatConversationPage />, { onUrlUpdate });
   expect(await screen.findByRole("button", { name: /gpt-5\.4-mini/ })).toBeInTheDocument();
-  fireEvent.change(screen.getByPlaceholderText("Describe a change, paste an error, or ask a question…"), {
+  fireEvent.change(screen.getByPlaceholderText("Ask a question, attach a file, or paste something to work on…"), {
     target: { value: "How much did this cost?" },
   });
   fireEvent.click(screen.getByRole("button", { name: "Send" }));

ui/litellm-dashboard/src/app/chat/page.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import React, { useCallback, useEffect, useRef, useState, useLayoutEffect } from "react";
-import { Plus, ChevronDown, Paperclip, ImageIcon, X } from "lucide-react";
+import { Plus, ChevronDown, Paperclip, ImageIcon, BookOpen, X } from "lucide-react";
 import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
 import { Button } from "@/components/ui/button";
 import { toast } from "@/lib/toast";
@@ -15,6 +15,7 @@ import type { AssistantMessageUpdate, ChatAttachment, ChatMessage } from "@/comp
 import ChatModelPicker, { isChatModel, modelChatMode, type ChatMode } from "@/components/chat/ChatModelPicker";
 import MoyaiChatComposer from "@/components/chat/moyai/MoyaiChatComposer";
 import MoyaiChatHome from "@/components/chat/moyai/MoyaiChatHome";
+import { useBackgroundChat } from "@/components/chat/useBackgroundChat";
 import { runChatRequest } from "@/components/chat/runChatRequest";
 import { readChatAttachment, MAX_CHAT_ATTACHMENTS, CHAT_ATTACHMENT_ACCEPT } from "@/components/chat/attachmentHelpers";
 import ChatFileChip from "@/components/chat/ChatFileChip";
@@ -24,6 +25,14 @@ const LOCALSTORAGE_MODEL_KEY = "litellm_chat_selected_model";
 
 type ReplayMessage = Pick<ChatMessage, "role" | "content" | "attachments" | "generatedImages" | "mcpEvents">;
 
+function getPreviousMessages(
+  override: ReplayMessage[] | undefined,
+  previousResponseId: string | null,
+  conversation: { messages: ReplayMessage[] } | undefined | null,
+): ReplayMessage[] {
+  return override ?? (previousResponseId ? [] : conversation?.messages ?? []);
+}
+
 function hasMessageContent(text: string, images: ChatAttachment[], mode: ChatMode): boolean {
   return Boolean(text.trim()) || (mode === "chat" && images.length > 0);
 }
@@ -90,6 +99,7 @@ export default function ChatConversationPage() {
   const [selectedModel, setSelectedModel] = useState<string | null>(null);
   const [models, setModels] = useState<ModelGroup[]>([]);
   const [mode, setMode] = useState<ChatMode>("chat");
+  const [research, setResearch] = useState(false);
   const [attachments, setAttachments] = useState<ChatAttachment[]>([]);
   const [attachmentError, setAttachmentError] = useState<string | null>(null);
   const [isReadingAttachments, setIsReadingAttachments] = useState(false);
@@ -102,7 +112,38 @@ export default function ChatConversationPage() {
   const responsesSessionGenerationRef = useRef(0);
   const previousResponsesConversationRef = useRef(activeConversationId);
   const [prevConversationIdForSessionReset, setPrevConversationIdForSessionReset] = useState(activeConversationId);
-  const [isStreaming, setIsStreaming] = useState(false);
+  const [isRequestInFlight, setIsStreaming] = useState(false);
+  const [cancelOnAcceptance, setCancelOnAcceptance] = useState<string | null>(null);
+  const backgroundRequestRef = useRef(false);
+  const lastMessage = activeConversation?.messages.at(-1);
+  const background = useBackgroundChat(
+    activeConversationId,
+    lastMessage?.backgroundRun,
+    accessToken,
+    updateLastAssistantMessage,
+  );
+  const { pending: backgroundPending, cancelling: backgroundCancelling, cancel: cancelBackground } = background;
+  useEffect(() => {
+    if (!backgroundPending || backgroundCancelling || !activeConversationId) return;
+    if (cancelOnAcceptance !== activeConversationId) return;
+    void cancelBackground().then(() => {
+      setCancelOnAcceptance((current) => (current === activeConversationId ? null : current));
+    });
+  }, [activeConversationId, backgroundPending, backgroundCancelling, cancelBackground, cancelOnAcceptance]);
+
+  const isStreaming = isRequestInFlight || background.pending;
+  const supportsBackground = models.some(
+    (model) => model.model_group === selectedModel && model.supports_background_chat,
+  );
+  const useBackground = mode === "chat" && supportsBackground && !storageUnavailable;
+  const canResearch =
+    useBackground &&
+    models.some(
+      (model) => model.model_group === selectedModel && model.supports_web_search && model.supports_reasoning,
+    );
+  let researchUnavailableReason = "Choose a reasoning model with web search.";
+  if (!useBackground) researchUnavailableReason = "Enable background chats in Admin Settings to use deep research.";
+  if (storageUnavailable) researchUnavailableReason = "Allow chat history storage to use deep research.";
   const [inputText, setInputText] = useState("");
   const [mcpPopoverOpen, setMcpPopoverOpen] = useState(false);
   const [storageBannerDismissed, setStorageBannerDismissed] = useState(false);
@@ -178,6 +219,7 @@ export default function ChatConversationPage() {
         return;
       }
       responsesSessionGenerationRef.current += 1;
+      setResearch(false);
       setSelectedModel(model.model_group);
       setMode(modelChatMode(model));
       setResponsesSessionId(null);
@@ -229,16 +271,28 @@ export default function ChatConversationPage() {
   );
 
   const handleSend = useCallback(
-    async (
-      text: string,
-      historyOverride?: ReplayMessage[],
-      attachmentOverride?: ChatAttachment[],
-      generationModel?: string,
-    ) => {
+    async ({
+      text,
+      historyOverride,
+      attachmentOverride,
+      generationModel,
+      research: researchRequested,
+    }: {
+      text: string;
+      historyOverride?: ReplayMessage[];
+      attachmentOverride?: ChatAttachment[];
+      generationModel?: string;
+      research: boolean;
+    }) => {
       const trimmed = text.trim();
       const images = attachmentOverride ?? attachments;
       const requestMode = generationModel ? "image" : mode;
       const model = generationModel ?? selectedModel;
+      const doResearch = requestMode === "chat" && researchRequested;
+      if (doResearch && !canResearch) {
+        setAttachmentError("Deep research needs a reasoning model with web search and background chats enabled.");
+        return;
+      }
       if (!hasMessageContent(trimmed, images, requestMode) || !model) return;
       if (isStreaming || abortControllerRef.current || readAttachmentsRef.current) return;
       const sessionGeneration = responsesSessionGenerationRef.current;
@@ -257,13 +311,16 @@ export default function ChatConversationPage() {
         content: trimmed,
         attachments: images.length > 0 ? images : undefined,
         imageGeneration,
+        research: doResearch,
       };
       appendMessage(convId, userMessage);
-      appendMessage(convId, { role: "assistant", content: "", imageGeneration });
+      const assistantMessage = { role: "assistant" as const, content: "", imageGeneration, research: doResearch };
+      appendMessage(convId, assistantMessage);
       setAttachments([]);
       setAttachmentError(null);
 
       setIsStreaming(true);
+      setCancelOnAcceptance(null);
       abortControllerRef.current = new AbortController();
       requestConversationRef.current = convId;
 
@@ -284,7 +341,7 @@ export default function ChatConversationPage() {
       const previousResponseId = historyOverride ? null : responsesSessionId;
 
       const history: ReplayMessage[] = [
-        ...(historyOverride ?? (previousResponseId ? [] : activeConversation?.messages ?? [])),
+        ...getPreviousMessages(historyOverride, previousResponseId, activeConversation),
         userMessage,
       ];
       const requestController = abortControllerRef.current;
@@ -297,12 +354,15 @@ export default function ChatConversationPage() {
         signal: requestController.signal,
         messages: history,
         previousResponseId,
+        background: requestMode === "chat" && useBackground,
+        research: doResearch,
         mcpServers: selectedMCPServers,
         onResponseId: (id: string | null) => {
           if (responsesSessionGenerationRef.current === sessionGeneration) setResponsesSessionId(id);
         },
         onUpdate: (updates: AssistantMessageUpdate) => updateLastAssistantMessage(convId, updates),
       };
+      backgroundRequestRef.current = request.background;
       try {
         await runChatRequest(request);
       } finally {
@@ -318,6 +378,7 @@ export default function ChatConversationPage() {
       activeConversation,
       selectedModel,
       selectedMCPServers,
+      canResearch,
       mode,
       attachments,
       accessToken,
@@ -326,13 +387,22 @@ export default function ChatConversationPage() {
       updateLastAssistantMessage,
       isStreaming,
       responsesSessionId,
+      useBackground,
       setConversationIdInUrl,
     ],
   );
 
-  const handleStop = useCallback(() => {
+  const handleStop = () => {
+    if (background.pending) {
+      void background.cancel();
+      return;
+    }
+    if (backgroundRequestRef.current) {
+      setCancelOnAcceptance(activeConversationId);
+      return;
+    }
     abortControllerRef.current?.abort();
-  }, []);
+  };
 
   const handleEditAndResend = useCallback(
     (messageId: string, newContent: string) => {
@@ -349,11 +419,31 @@ export default function ChatConversationPage() {
         return;
       }
       if (!newContent.trim() && !original.attachments?.length) return;
+      if (original.research && !canResearch) {
+        setAttachmentError("Choose a reasoning model with web search and background chats before resending research.");
+        return;
+      }
       const priorMessages = msgs.slice(0, idx);
       truncateFromMessage(activeConversationId, messageId);
-      handleSend(newContent, priorMessages, original.attachments ?? [], original.imageGeneration?.model);
+      const editedRequest = {
+        text: newContent,
+        historyOverride: priorMessages,
+        attachmentOverride: original.attachments ?? [],
+        generationModel: original.imageGeneration?.model,
+        research: original.research ?? false,
+      };
+      handleSend(editedRequest);
     },
-    [activeConversationId, isStreaming, activeConversation, truncateFromMessage, handleSend, mode, selectedModel],
+    [
+      activeConversationId,
+      isStreaming,
+      activeConversation,
+      truncateFromMessage,
+      handleSend,
+      mode,
+      selectedModel,
+      canResearch,
+    ],
   );
 
   // Track scroll position to show/hide scroll-to-bottom button
@@ -403,9 +493,26 @@ export default function ChatConversationPage() {
 
   const showBlankState = !activeConversation || activeConversation.messages.length === 0;
 
+  const backgroundHint = () => {
+    if (background.cancelling || (isRequestInFlight && cancelOnAcceptance === activeConversationId))
+      return "Requesting cancellation…";
+    if (background.pending && storageUnavailable)
+      return "Keep this tab open. This run could not be saved in this browser.";
+    if (background.pending) return "Running in the background. You can close this tab and return to this chat.";
+    if (useBackground) return null;
+    return "Keep this tab open. Background runs are not available for this chat.";
+  };
+
   const inputBar = (inConversation: boolean) => {
+    const hint = mode === "chat" || background.pending ? backgroundHint() : null;
+    const defaultPlaceholder = inConversation
+      ? "Send a message..."
+      : "Ask a question, attach a file, or paste something to work on…";
+    const chatPlaceholder = research ? "What should I research on the web?" : defaultPlaceholder;
     const preparing = isLoadingModels || isReadingAttachments;
-    const submitDisabled = !hasMessageContent(inputText, attachments, mode) || !selectedModel || preparing;
+    const researchBlocked = research && !canResearch;
+    const submitDisabled =
+      !hasMessageContent(inputText, attachments, mode) || !selectedModel || preparing || researchBlocked;
     return (
       <div
         onDragOver={(event) => {
@@ -422,15 +529,9 @@ export default function ChatConversationPage() {
           inConversation={inConversation}
           value={inputText}
           onChange={setInputText}
-          onSubmit={() => void handleSend(inputText)}
+          onSubmit={() => void handleSend({ text: inputText, research })}
           onCancel={handleStop}
-          placeholder={
-            mode === "image"
-              ? "Describe the image you want to create…"
-              : inConversation
-                ? "Send a message..."
-                : "Describe a change, paste an error, or ask a question…"
-          }
+          placeholder={mode === "image" ? "Describe the image you want to create…" : chatPlaceholder}
           isLoading={isStreaming}
           submitLabel={mode === "image" ? "Generate image" : "Send"}
           submitDisabled={submitDisabled}
@@ -500,6 +601,25 @@ export default function ChatConversationPage() {
                     }
                   />
                   <PopoverContent side="top" align="start" className="p-0 w-auto">
+                    <div className="border-b p-2">
+                      <Button
+                        variant="ghost"
+                        size="sm"
+                        className="w-full justify-start"
+                        aria-pressed={research}
+                        disabled={!canResearch}
+                        onClick={() => {
+                          setResearch(!research);
+                          setMcpPopoverOpen(false);
+                        }}
+                      >
+                        <BookOpen className="size-4" />
+                        Deep research
+                      </Button>
+                      {!canResearch && (
+                        <p className="max-w-64 px-2 pb-1 text-xs text-muted-foreground">{researchUnavailableReason}</p>
+                      )}
+                    </div>
                     <MCPConnectPicker
                       accessToken={accessToken}
                       selectedServers={selectedMCPServers}
@@ -507,6 +627,19 @@ export default function ChatConversationPage() {
                     />
                   </PopoverContent>
                 </Popover>
+                {research && (
+                  <Button
+                    variant="secondary"
+                    size="sm"
+                    disabled={isStreaming}
+                    aria-label="Turn off deep research"
+                    onClick={() => setResearch(false)}
+                  >
+                    <BookOpen className="size-3.5" />
+                    Deep research
+                    <X className="size-3.5" />
+                  </Button>
+                )}
               </>
             ) : (
               <span className="flex items-center gap-2 text-xs text-muted-foreground">
@@ -536,6 +669,29 @@ export default function ChatConversationPage() {
               : "No chat models are available for your account."}
           </p>
         )}
+        {background.connectionError && (
+          <div role="alert" className="mt-2 flex items-center gap-2 text-sm text-destructive">
+            <span>{background.connectionError}</span>
+            <Button variant="ghost" size="sm" onClick={background.reconnect}>
+              Reconnect
+            </Button>
+          </div>
+        )}
+        {lastMessage?.backgroundError && (
+          <p role="alert" className="mt-2 text-sm text-destructive">
+            {lastMessage.backgroundError}
+          </p>
+        )}
+        {lastMessage?.backgroundRun?.status === "cancelled" && (
+          <p role="status" className="mt-2 text-xs text-muted-foreground">
+            Cancellation requested
+          </p>
+        )}
+        {hint && (
+          <p role="status" className="mt-2 text-xs text-muted-foreground">
+            {hint}
+          </p>
+        )}
         <p className="moyai-chat-home-composer-hint">
           {mode === "image" ? "Describe an image · Enter to generate" : "Drop files or paste images · Enter to send"}
         </p>
@@ -563,6 +719,7 @@ export default function ChatConversationPage() {
         {showBlankState ? (
           <MoyaiChatHome
             imageMode={mode === "image"}
+            researchMode={research}
             draft={inputText}
             busy={isStreaming}
             conversations={conversations}

ui/litellm-dashboard/src/app/chat/page.url-state.integration.test.tsx

@@ -66,7 +66,7 @@ describe("chat page URL state with ChatShellProvider", () => {
 
     expect(await screen.findByRole("button", { name: /gpt-5\.4-mini/ })).toBeInTheDocument();
     act(() => {
-      fireEvent.change(screen.getByPlaceholderText("Describe a change, paste an error, or ask a question…"), {
+      fireEvent.change(screen.getByPlaceholderText("Ask a question, attach a file, or paste something to work on…"), {
         target: { value: "How much did this cost?" },
       });
     });
@@ -87,9 +87,9 @@ describe("chat page URL state with ChatShellProvider", () => {
     await waitFor(() => expect(window.location.search).toBe(""));
 
     expect(
-      await screen.findByPlaceholderText("Describe a change, paste an error, or ask a question…"),
+      await screen.findByPlaceholderText("Ask a question, attach a file, or paste something to work on…"),
     ).toBeInTheDocument();
-    expect(screen.getByRole("heading", { level: 1, name: "What are we building?" })).toBeInTheDocument();
+    expect(screen.getByRole("heading", { level: 1, name: "What are we working on?" })).toBeInTheDocument();
     expect(screen.queryByText("The estimated cost is one cent.")).not.toBeInTheDocument();
     expect(screen.getByRole("button", { name: "How much did this cost?" })).toBeInTheDocument();
     expect(window.location.pathname).toBe("/chat");