Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_feat/v1.84.0-mcp-gateway-jwt-auth

Co-authored-by: Mateo Wang <mateo-berri@users.noreply.github.com>

.circleci/config.yml

@@ -158,6 +158,8 @@ jobs:
             CHOCOLATEY_CONFIRM_ALL: "true"
       - run:
           name: Install Dependencies
+          environment:
+            UV_HTTP_TIMEOUT: "300"
           command: |
             $installer = Join-Path $env:TEMP "uv-install.ps1"
             Invoke-WebRequest -Uri https://astral.sh/uv/0.10.9/install.ps1 -OutFile $installer

.github/workflows/test-unit-proxy-mgmt-behavior.yml

@@ -0,0 +1,34 @@
+name: "Unit Tests: Proxy Management-Endpoint Behavior Pinning"
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
+
+permissions:
+  contents: read
+  id-token: write
+  pull-requests: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  proxy-mgmt-behavior:
+    uses: ./.github/workflows/_test-unit-services-base.yml
+    with:
+      test-path: tests/proxy_behavior
+      # workers=0 (no xdist): the world seed is a single shared Postgres
+      # state — two xdist workers both call seed_world() and race on the
+      # ``behavior-pin-budget`` row, producing UniqueViolation + cascading
+      # missing-membership FK failures. The whole suite is ~7s sequentially,
+      # so the cost of disabling parallelism here is negligible.
+      workers: 0
+      reruns: 0
+      enable-postgres: true
+      artifact-name: proxy-mgmt-behavior
+      timeout-minutes: 15

litellm/__init__.py

@@ -225,6 +225,10 @@ use_chat_completions_url_for_anthropic_messages: bool = bool(
 route_all_chat_openai_to_responses: bool = (
     os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", "false").lower() == "true"
 )  # When True, routes all OpenAI /chat/completions requests through the Responses API bridge
+use_legacy_interactions_schema: bool = (
+    os.getenv("LITELLM_USE_LEGACY_INTERACTIONS_SCHEMA", "false").lower() == "true"
+)  # When True, sends Api-Revision: 2026-05-07 to Google so responses use the legacy `outputs`
+# schema instead of the new `steps` schema. Remove this flag after June 8, 2026.
 retry = True
 ### AUTH ###
 api_key: Optional[str] = None

litellm/integrations/opentelemetry.py

@@ -1611,11 +1611,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
                     "masked_entity_count", safe_dumps(masked_entity_count)
                 )
 
-            self.safe_set_attribute(
-                span=guardrail_span,
-                key="guardrail_response",
-                value=guardrail_information.get("guardrail_response"),
-            )
+            guardrail_response = guardrail_information.get("guardrail_response")
+            if guardrail_response is not None:
+                guardrail_span.set_attribute(
+                    "guardrail_response", safe_dumps(guardrail_response)
+                )
 
             self._set_team_attributes_from_kwargs(guardrail_span, kwargs)
 

litellm/interactions/litellm_responses_transformation/streaming_iterator.py

@@ -2,7 +2,17 @@
 Streaming iterator for transforming Responses API stream to Interactions API stream.
 """
 
-from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, cast
+from collections import deque
+from typing import (
+    Any,
+    AsyncIterator,
+    Deque,
+    Dict,
+    Iterator,
+    List,
+    Optional,
+    cast,
+)
 
 from litellm.responses.streaming_iterator import (
     BaseResponsesAPIStreamingIterator,
@@ -15,7 +25,6 @@ from litellm.types.interactions import (
     InteractionsAPIStreamingResponse,
 )
 from litellm.types.llms.openai import (
-    ContentPartAddedEvent,
     OutputTextDeltaEvent,
     ResponseCompletedEvent,
     ResponseCreatedEvent,
@@ -30,7 +39,13 @@ class LiteLLMResponsesInteractionsStreamingIterator:
 
     This class handles both sync and async iteration, transforming Responses API
     streaming events (output.text.delta, response.completed, etc.) to Interactions
-    API streaming events (content.delta, interaction.complete, etc.).
+    API streaming events.
+
+    Schema selection:
+    - New schema (default, use_legacy_interactions_schema=False):
+        interaction.created -> step.start -> step.delta ... -> step.stop -> interaction.completed
+    - Legacy schema (use_legacy_interactions_schema=True, remove after June 8 2026):
+        interaction.start -> content.start -> content.delta ... -> content.stop -> interaction.complete
     """
 
     def __init__(
@@ -42,6 +57,8 @@ class LiteLLMResponsesInteractionsStreamingIterator:
         custom_llm_provider: Optional[str] = None,
         litellm_metadata: Optional[Dict[str, Any]] = None,
     ):
+        import litellm
+
         self.model = model
         self.responses_stream_iterator = litellm_custom_stream_wrapper
         self.request_input = request_input
@@ -52,100 +69,156 @@ class LiteLLMResponsesInteractionsStreamingIterator:
         self.collected_text = ""
         self.sent_interaction_start = False
         self.sent_content_start = False
-        self._pending_events: List[InteractionsAPIStreamingResponse] = []
+        # Capture the schema flag once at construction time so all events
+        # emitted by this stream use a consistent schema, even if the global
+        # flag is mutated mid-stream (e.g. by a config reload).
+        self._use_legacy: bool = litellm.use_legacy_interactions_schema
+        # Buffer of events that have been derived from upstream chunks but not
+        # yet returned to the caller. A single Responses API chunk may expand
+        # into multiple Interactions API events (e.g. the first text delta
+        # produces interaction.created + step.start + step.delta), and the
+        # terminal sequence on stream end may also span multiple events
+        # (step.stop + interaction.completed).
+        self._pending_events: Deque[InteractionsAPIStreamingResponse] = deque()
+        # Tracks whether we've already emitted a terminal completion event so
+        # the StopIteration fallback path doesn't double-emit.
+        self._sent_completion_event = False
+        # ID resolved from the first upstream chunk (item_id on a text delta or
+        # response.id on response.created). Persisted so the EOF terminal
+        # events stay correlated with the start events delivered earlier.
+        self._interaction_id: Optional[str] = None
 
-    def _transform_responses_chunk_to_interactions_chunk(
-        self,
-        responses_chunk: ResponsesAPIStreamingResponse,
-    ) -> Optional[InteractionsAPIStreamingResponse]:
+    # ------------------------------------------------------------------
+    # Event builders
+    # ------------------------------------------------------------------
+
+    def _build_interaction_start_event(
+        self, interaction_id: str
+    ) -> InteractionsAPIStreamingResponse:
+        event_type = "interaction.start" if self._use_legacy else "interaction.created"
+        return InteractionsAPIStreamingResponse(
+            event_type=event_type,
+            id=interaction_id,
+            object="interaction",
+            status="in_progress",
+            model=self.model,
+        )
+
+    def _build_content_start_event(
+        self, interaction_id: str
+    ) -> InteractionsAPIStreamingResponse:
+        if self._use_legacy:
+            return InteractionsAPIStreamingResponse(
+                event_type="content.start",
+                id=interaction_id,
+                object="content",
+                delta={"type": "text", "text": ""},
+            )
+        return InteractionsAPIStreamingResponse(
+            event_type="step.start",
+            index=0,
+            step={"type": "model_output", "content": []},
+        )
+
+    def _build_text_delta_event(
+        self, interaction_id: str, delta_text: str
+    ) -> InteractionsAPIStreamingResponse:
+        if self._use_legacy:
+            return InteractionsAPIStreamingResponse(
+                event_type="content.delta",
+                id=interaction_id,
+                object="content",
+                delta={"type": "text", "text": delta_text},
+            )
+        return InteractionsAPIStreamingResponse(
+            event_type="step.delta",
+            index=0,
+            delta={"type": "text", "text": delta_text},
+        )
+
+    def _build_content_stop_event(
+        self, interaction_id: Optional[str]
+    ) -> InteractionsAPIStreamingResponse:
+        if self._use_legacy:
+            return InteractionsAPIStreamingResponse(
+                event_type="content.stop",
+                id=interaction_id,
+                object="content",
+                delta={"type": "text", "text": self.collected_text},
+            )
+        return InteractionsAPIStreamingResponse(
+            event_type="step.stop",
+            index=0,
+        )
+
+    def _build_completion_event(
+        self, response_id: str
+    ) -> InteractionsAPIStreamingResponse:
+        if self._use_legacy:
+            return InteractionsAPIStreamingResponse(
+                event_type="interaction.complete",
+                id=response_id,
+                object="interaction",
+                status="completed",
+                model=self.model,
+                outputs=[{"type": "text", "text": self.collected_text}],
+            )
+        return InteractionsAPIStreamingResponse(
+            event_type="interaction.completed",
+            id=response_id,
+            object="interaction",
+            status="completed",
+            model=self.model,
+            steps=[
+                {
+                    "type": "model_output",
+                    "content": [{"type": "text", "text": self.collected_text}],
+                }
+            ],
+        )
+
+    # ------------------------------------------------------------------
+    # Per-chunk transform (returns a list of events to enqueue)
+    # ------------------------------------------------------------------
+
+    def _events_for_chunk(
+        self, responses_chunk: ResponsesAPIStreamingResponse
+    ) -> List[InteractionsAPIStreamingResponse]:
         """
-        Transform a Responses API streaming chunk to an Interactions API streaming chunk.
+        Translate a single upstream Responses API chunk into the list of
+        Interactions API events it should produce.
 
-        Responses API events:
-        - output.text.delta -> content.delta
-        - response.completed -> interaction.complete
-
-        Interactions API events:
-        - interaction.start
-        - content.start
-        - content.delta
-        - content.stop
-        - interaction.complete
+        Returning a list (rather than a single event) lets a chunk emit any
+        synthetic start events that haven't been sent yet *together with* the
+        actual delta event, so we never silently drop the chunk's payload.
         """
         if not responses_chunk:
-            return None
+            return []
 
-        # Handle OutputTextDeltaEvent -> content.delta
+        # Text delta: emit any missing start events, then the delta itself.
         if isinstance(responses_chunk, OutputTextDeltaEvent):
             delta_text = (
                 responses_chunk.delta if isinstance(responses_chunk.delta, str) else ""
             )
             self.collected_text += delta_text
-
-            # Fallback: emit interaction.start, and queue content.start carrying this
-            # delta so the first token is preserved in the stream.
-            if not self.sent_interaction_start:
-                self.sent_interaction_start = True
-                self.sent_content_start = True
-                self._pending_events.append(
-                    InteractionsAPIStreamingResponse(
-                        event_type="content.start",
-                        id=getattr(responses_chunk, "item_id", None),
-                        object="content",
-                        delta={"type": "text", "text": delta_text},
-                    )
-                )
-                return InteractionsAPIStreamingResponse(
-                    event_type="interaction.start",
-                    id=getattr(responses_chunk, "item_id", None)
-                    or f"interaction_{id(self)}",
-                    object="interaction",
-                    status="in_progress",
-                    model=self.model,
-                )
-
-            # Fallback: emit content.start if ContentPartAddedEvent never arrived
-            if not self.sent_content_start:
-                self.sent_content_start = True
-                return InteractionsAPIStreamingResponse(
-                    event_type="content.start",
-                    id=getattr(responses_chunk, "item_id", None),
-                    object="content",
-                    delta={"type": "text", "text": delta_text},
-                )
-
-            # Normal path: emit content.delta with type field
-            return InteractionsAPIStreamingResponse(
-                event_type="content.delta",
-                id=getattr(responses_chunk, "item_id", None),
-                object="content",
-                delta={"type": "text", "text": delta_text},
+            interaction_id = (
+                getattr(responses_chunk, "item_id", None) or f"interaction_{id(self)}"
             )
+            if self._interaction_id is None:
+                self._interaction_id = interaction_id
 
-        # Handle ContentPartAddedEvent -> content.start (arrives before text deltas)
-        if isinstance(responses_chunk, ContentPartAddedEvent):
-            # Fallback: emit interaction.start if ResponseCreatedEvent never arrived
+            events: List[InteractionsAPIStreamingResponse] = []
             if not self.sent_interaction_start:
                 self.sent_interaction_start = True
-                return InteractionsAPIStreamingResponse(
-                    event_type="interaction.start",
-                    id=getattr(responses_chunk, "item_id", None)
-                    or f"interaction_{id(self)}",
-                    object="interaction",
-                    status="in_progress",
-                    model=self.model,
-                )
+                events.append(self._build_interaction_start_event(interaction_id))
             if not self.sent_content_start:
                 self.sent_content_start = True
-                return InteractionsAPIStreamingResponse(
-                    event_type="content.start",
-                    id=getattr(responses_chunk, "item_id", None),
-                    object="content",
-                    delta={"type": "text", "text": ""},
-                )
-            return None
+                events.append(self._build_content_start_event(interaction_id))
+            events.append(self._build_text_delta_event(interaction_id, delta_text))
+            return events
 
-        # Handle ResponseCreatedEvent or ResponseInProgressEvent -> interaction.start
+        # Response created / in-progress: synthesize interaction start if we
+        # haven't already sent one.
         if isinstance(responses_chunk, (ResponseCreatedEvent, ResponseInProgressEvent)):
             if not self.sent_interaction_start:
                 self.sent_interaction_start = True
@@ -153,177 +226,136 @@ class LiteLLMResponsesInteractionsStreamingIterator:
                     getattr(responses_chunk.response, "id", None)
                     if hasattr(responses_chunk, "response")
                     else None
-                )
-                return InteractionsAPIStreamingResponse(
-                    event_type="interaction.start",
-                    id=response_id or f"interaction_{id(self)}",
-                    object="interaction",
-                    status="in_progress",
-                    model=self.model,
-                )
+                ) or f"interaction_{id(self)}"
+                if self._interaction_id is None:
+                    self._interaction_id = response_id
+                return [self._build_interaction_start_event(response_id)]
+            return []
 
-        # Handle ResponseCompletedEvent -> interaction.complete
+        # Response completed: emit step.stop (if content was started) followed
+        # by the terminal completion event. Prefer the interaction id already
+        # established by earlier events so consumers can correlate the start
+        # and completion events by id (response.id may differ from the item_id
+        # used to derive the initial id when the stream starts directly with a
+        # text delta).
         if isinstance(responses_chunk, ResponseCompletedEvent):
             self.finished = True
             response = responses_chunk.response
-
-            # Send content.stop first if content was started
-            if self.sent_content_start:
-                # Note: We'll send this in the iterator, not here
-                pass
-
-            # Send interaction.complete
-            return InteractionsAPIStreamingResponse(
-                event_type="interaction.complete",
-                id=getattr(response, "id", None) or f"interaction_{id(self)}",
-                object="interaction",
-                status="completed",
-                model=self.model,
-                outputs=[
-                    {
-                        "type": "text",
-                        "text": self.collected_text,
-                    }
-                ],
+            response_id = (
+                self._interaction_id
+                or getattr(response, "id", None)
+                or f"interaction_{id(self)}"
             )
 
-        # For other event types, return None (skip)
-        return None
+            terminal: List[InteractionsAPIStreamingResponse] = []
+            if self.sent_content_start:
+                terminal.append(self._build_content_stop_event(response_id))
+            terminal.append(self._build_completion_event(response_id))
+            self._sent_completion_event = True
+            return terminal
+
+        return []
+
+    def _build_terminal_events_on_eof(
+        self,
+    ) -> List[InteractionsAPIStreamingResponse]:
+        """
+        Build the events to flush when the upstream stream ends without a
+        ResponseCompletedEvent. Ensures consumers always observe a terminal
+        interaction.completed/interaction.complete carrying the full text.
+        """
+        if self._sent_completion_event:
+            return []
+
+        fallback_id = self._interaction_id or f"interaction_{id(self)}"
+        terminal: List[InteractionsAPIStreamingResponse] = []
+        if self.sent_content_start:
+            terminal.append(self._build_content_stop_event(fallback_id))
+        if self.sent_interaction_start or self.collected_text:
+            terminal.append(self._build_completion_event(fallback_id))
+            self._sent_completion_event = True
+        return terminal
+
+    # ------------------------------------------------------------------
+    # Iteration
+    # ------------------------------------------------------------------
 
     def __iter__(self) -> Iterator[InteractionsAPIStreamingResponse]:
-        """Sync iterator implementation."""
         return self
 
     def __next__(self) -> InteractionsAPIStreamingResponse:
-        """Get next chunk in sync mode."""
+        if self._pending_events:
+            return self._pending_events.popleft()
+
         if self.finished:
             raise StopIteration
 
-        # Check if we have a pending interaction.complete to send
-        if hasattr(self, "_pending_interaction_complete"):
-            pending: InteractionsAPIStreamingResponse = getattr(
-                self, "_pending_interaction_complete"
-            )
-            delattr(self, "_pending_interaction_complete")
-            return pending
-
-        # Drain events queued from a prior chunk (e.g. content.start emitted alongside
-        # the interaction.start fallback for the first OutputTextDeltaEvent).
-        if self._pending_events:
-            return self._pending_events.pop(0)
-        # Use a loop instead of recursion to avoid stack overflow
         sync_iterator = cast(
             SyncResponsesAPIStreamingIterator, self.responses_stream_iterator
         )
         while True:
             try:
-                # Get next chunk from responses API stream
                 chunk = next(sync_iterator)
-
-                # Transform chunk (chunk is already a ResponsesAPIStreamingResponse)
-                transformed = self._transform_responses_chunk_to_interactions_chunk(
-                    chunk
-                )
-
-                if transformed:
-                    # If we finished and content was started, send content.stop before interaction.complete
-                    if (
-                        self.finished
-                        and self.sent_content_start
-                        and transformed.event_type == "interaction.complete"
-                    ):
-                        # Send content.stop first
-                        content_stop = InteractionsAPIStreamingResponse(
-                            event_type="content.stop",
-                            id=transformed.id,
-                            object="content",
-                            delta={"type": "text", "text": self.collected_text},
-                        )
-                        # Store the interaction.complete to send next
-                        self._pending_interaction_complete = transformed
-                        return content_stop
-                    return transformed
-
-                # If no transformation, continue to next chunk (loop continues)
-
             except StopIteration:
                 self.finished = True
+                self._pending_events.extend(self._build_terminal_events_on_eof())
+                if self._pending_events:
+                    return self._pending_events.popleft()
+                raise
 
-                # Send final events if needed
-                if self.sent_content_start:
-                    return InteractionsAPIStreamingResponse(
-                        event_type="content.stop",
-                        object="content",
-                        delta={"type": "text", "text": self.collected_text},
-                    )
-
-                raise StopIteration
+            events = self._events_for_chunk(chunk)
+            if events:
+                self._pending_events.extend(events)
+                return self._pending_events.popleft()
 
     def __aiter__(self) -> AsyncIterator[InteractionsAPIStreamingResponse]:
-        """Async iterator implementation."""
         return self
 
     async def __anext__(self) -> InteractionsAPIStreamingResponse:
-        """Get next chunk in async mode."""
+        if self._pending_events:
+            return self._pending_events.popleft()
+
         if self.finished:
             raise StopAsyncIteration
 
-        # Check if we have a pending interaction.complete to send
-        if hasattr(self, "_pending_interaction_complete"):
-            pending: InteractionsAPIStreamingResponse = getattr(
-                self, "_pending_interaction_complete"
-            )
-            delattr(self, "_pending_interaction_complete")
-            return pending
-
-        # Drain events queued from a prior chunk (e.g. content.start emitted alongside
-        # the interaction.start fallback for the first OutputTextDeltaEvent).
-        if self._pending_events:
-            return self._pending_events.pop(0)
-        # Use a loop instead of recursion to avoid stack overflow
         async_iterator = cast(
             ResponsesAPIStreamingIterator, self.responses_stream_iterator
         )
         while True:
             try:
-                # Get next chunk from responses API stream
                 chunk = await async_iterator.__anext__()
-
-                # Transform chunk (chunk is already a ResponsesAPIStreamingResponse)
-                transformed = self._transform_responses_chunk_to_interactions_chunk(
-                    chunk
-                )
-
-                if transformed:
-                    # If we finished and content was started, send content.stop before interaction.complete
-                    if (
-                        self.finished
-                        and self.sent_content_start
-                        and transformed.event_type == "interaction.complete"
-                    ):
-                        # Send content.stop first
-                        content_stop = InteractionsAPIStreamingResponse(
-                            event_type="content.stop",
-                            id=transformed.id,
-                            object="content",
-                            delta={"type": "text", "text": self.collected_text},
-                        )
-                        # Store the interaction.complete to send next
-                        self._pending_interaction_complete = transformed
-                        return content_stop
-                    return transformed
-
-                # If no transformation, continue to next chunk (loop continues)
-
             except StopAsyncIteration:
                 self.finished = True
+                self._pending_events.extend(self._build_terminal_events_on_eof())
+                if self._pending_events:
+                    return self._pending_events.popleft()
+                raise
 
-                # Send final events if needed
-                if self.sent_content_start:
-                    return InteractionsAPIStreamingResponse(
-                        event_type="content.stop",
-                        object="content",
-                        delta={"type": "text", "text": self.collected_text},
-                    )
+            events = self._events_for_chunk(chunk)
+            if events:
+                self._pending_events.extend(events)
+                return self._pending_events.popleft()
 
-                raise StopAsyncIteration
+    # ------------------------------------------------------------------
+    # Backwards-compatible single-chunk transform (used by tests and any
+    # external callers that drove the iterator chunk-by-chunk pre-fix).
+    # ------------------------------------------------------------------
+
+    def _transform_responses_chunk_to_interactions_chunk(
+        self,
+        responses_chunk: ResponsesAPIStreamingResponse,
+    ) -> Optional[InteractionsAPIStreamingResponse]:
+        """
+        Compatibility shim: returns the *first* event produced for this chunk
+        and queues any remaining events on ``self._pending_events`` so they
+        are surfaced on subsequent calls/iterations.
+
+        Prefer ``_events_for_chunk`` in new code.
+        """
+        events = self._events_for_chunk(responses_chunk)
+        if not events:
+            return None
+        first = events[0]
+        if len(events) > 1:
+            self._pending_events.extend(events[1:])
+        return first

litellm/interactions/litellm_responses_transformation/transformation.py

@@ -226,29 +226,37 @@ class LiteLLMResponsesInteractionsConfig:
         - Map status
         - Extract usage
         """
-        # Extract text from outputs
-        outputs = []
+        # Extract text from outputs and build both `outputs` (legacy) and `steps` (new schema).
+        outputs: List[Dict[str, Any]] = []
+        steps: List[Dict[str, Any]] = []
         if hasattr(responses_response, "output") and responses_response.output:
             for output_item in responses_response.output:
                 # Use getattr with None default to safely access content
                 content = getattr(output_item, "content", None)
                 if content is not None:
                     content_items = content if isinstance(content, list) else [content]
+                    model_output_contents: List[Dict[str, Any]] = []
                     for content_item in content_items:
                         # Check if content_item has text attribute
                         text = getattr(content_item, "text", None)
                         if text is not None:
-                            outputs.append(
-                                {
-                                    "type": "text",
-                                    "text": text,
-                                }
-                            )
+                            # Use independent dict instances so mutations to one
+                            # of `outputs` / `steps` don't leak into the other.
+                            outputs.append({"type": "text", "text": text})
+                            model_output_contents.append({"type": "text", "text": text})
                         elif (
                             isinstance(content_item, dict)
                             and content_item.get("type") == "text"
                         ):
-                            outputs.append(content_item)
+                            outputs.append({**content_item})
+                            model_output_contents.append({**content_item})
+                    if model_output_contents:
+                        steps.append(
+                            {
+                                "type": "model_output",
+                                "content": model_output_contents,
+                            }
+                        )
 
         # Convert created_at to ISO string
         created_at = getattr(responses_response, "created_at", None)
@@ -270,12 +278,14 @@ class LiteLLMResponsesInteractionsConfig:
         else:
             interactions_status = status
 
-        # Build interactions response
+        # Build interactions response — populate both `outputs` (legacy schema) and
+        # `steps` (new schema) so callers work regardless of which schema they expect.
         interactions_response_dict: Dict[str, Any] = {
             "id": getattr(responses_response, "id", ""),
             "object": "interaction",
             "status": interactions_status,
             "outputs": outputs,
+            "steps": steps,
             "model": model or getattr(responses_response, "model", ""),
             "created": created,
         }

litellm/interactions/streaming_iterator.py

@@ -101,10 +101,14 @@ class BaseInteractionsAPIStreamingIterator:
                     )
                 )
 
-                # Store the completed response (check for status=completed)
-                if (
-                    streaming_response
-                    and getattr(streaming_response, "status", None) == "completed"
+                # Store the completed response.
+                # Legacy schema signals completion via status="completed".
+                # New schema (Api-Revision: 2026-05-20) uses event_type="interaction.completed".
+                # Remove the legacy check after June 8, 2026.
+                if streaming_response and (
+                    getattr(streaming_response, "status", None) == "completed"
+                    or getattr(streaming_response, "event_type", None)
+                    == "interaction.completed"
                 ):
                     self.completed_response = streaming_response
                     self._handle_logging_completed_response()

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -1344,6 +1344,7 @@ def _get_dummy_thought_signature() -> str:
 def convert_to_gemini_tool_call_invoke(
     message: ChatCompletionAssistantMessage,
     model: Optional[str] = None,
+    custom_llm_provider: Optional[str] = None,
 ) -> List[VertexPartType]:
     """
     OpenAI tool invokes:
@@ -1394,7 +1395,10 @@ def convert_to_gemini_tool_call_invoke(
         )
 
         forward_tool_call_id = bool(
-            model and VertexGeminiConfig._is_gemini_3_or_newer(model)
+            model
+            and VertexGeminiConfig._forward_gemini_function_call_id(
+                model, custom_llm_provider
+            )
         )
 
         if tool_calls is not None:
@@ -1475,6 +1479,7 @@ def convert_to_gemini_tool_call_result(  # noqa: PLR0915
     message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage],
     last_message_with_tool_calls: Optional[dict],
     model: Optional[str] = None,
+    custom_llm_provider: Optional[str] = None,
 ) -> Union[VertexPartType, List[VertexPartType]]:
     """
     OpenAI message with a tool result looks like:
@@ -1616,14 +1621,16 @@ def convert_to_gemini_tool_call_result(  # noqa: PLR0915
                 name = tool.get("function", {}).get("name", "")
 
     # Echo the OpenAI tool_call_id on functionResponse (strip thought-signature suffix).
-    # Only Gemini 3+ accepts (and returns) an `id` on function_response parts;
-    # older Gemini models reject the field with a 400.
+    # Only Google AI Studio Gemini 3+ accepts `id` on function_response parts.
+    # Vertex AI and older Gemini models reject the field with HTTP 400.
     from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
         VertexGeminiConfig,
     )
 
     gemini_call_id: Optional[str] = None
-    if model and VertexGeminiConfig._is_gemini_3_or_newer(model):
+    if model and VertexGeminiConfig._forward_gemini_function_call_id(
+        model, custom_llm_provider
+    ):
         raw_tool_call_id = message.get("tool_call_id")
         if raw_tool_call_id and isinstance(raw_tool_call_id, str):
             stripped_id = raw_tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0]

litellm/llms/anthropic/chat/transformation.py

@@ -1506,9 +1506,21 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 optional_params["metadata"] = {"user_id": value}
             elif param == "thinking":
                 optional_params["thinking"] = value
-            elif param == "reasoning_effort" and isinstance(value, str):
+            elif param == "reasoning_effort":
+                # Accept both string ("low") and dict ({"effort": "low",
+                # "summary": "concise"}). The Responses->Chat parser keeps the
+                # full dict when `summary` is set (see #25359), so a dict here
+                # is the standard shape Otto/OpenAI-Responses-Bridge callers
+                # send. Coerce to the effort string before mapping — same
+                # shape-tolerance the GPT-5 path already implements in
+                # `_normalize_reasoning_effort_for_chat_completion`.
+                effort_value = value
+                if isinstance(effort_value, dict):
+                    effort_value = effort_value.get("effort")
+                if not isinstance(effort_value, str):
+                    continue
                 mapped_thinking = AnthropicConfig._map_reasoning_effort(
-                    reasoning_effort=value,
+                    reasoning_effort=effort_value,
                     model=model,
                     llm_provider=self.custom_llm_provider or "anthropic",
                 )
@@ -1519,12 +1531,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                     optional_params["thinking"] = mapped_thinking
                     if AnthropicConfig._is_adaptive_thinking_model(model):
                         mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(
-                            value
+                            effort_value
                         )
                         if mapped_effort is None:
                             AnthropicConfig._raise_invalid_reasoning_effort(
                                 model=model,
-                                value=value,
+                                value=effort_value,
                                 llm_provider=self.custom_llm_provider or "anthropic",
                             )
                         optional_params["output_config"] = {"effort": mapped_effort}

litellm/llms/azure/containers/transformation.py

@@ -1,9 +1,16 @@
 from typing import Optional
+from urllib.parse import parse_qs, urlparse, urlunparse
 
 from litellm.llms.azure.common_utils import BaseAzureLLM
 from litellm.llms.openai.containers.transformation import OpenAIContainerConfig
 from litellm.types.router import GenericLiteLLMParams
 
+# Endpoint-specific path suffixes that may appear in a deployment's api_base
+# (e.g. the responses endpoint URL is stored as api_base for Azure models).
+# Strip these before building the containers URL so we always start from the
+# resource root (https://resource.cognitiveservices.azure.com).
+_AZURE_ENDPOINT_PATHS = ("/openai/responses",)
+
 
 class AzureContainerConfig(OpenAIContainerConfig):
     """
@@ -27,6 +34,27 @@ class AzureContainerConfig(OpenAIContainerConfig):
             litellm_params=GenericLiteLLMParams(api_key=api_key),
         )
 
+    @staticmethod
+    def _normalize_api_base(api_base: Optional[str]) -> Optional[str]:
+        """Strip endpoint-specific path suffixes from api_base to get the resource root."""
+        if not api_base:
+            return api_base
+        parsed = urlparse(api_base)
+        path = parsed.path.rstrip("/")
+        for ep in _AZURE_ENDPOINT_PATHS:
+            if path.endswith(ep):
+                return urlunparse(
+                    (parsed.scheme, parsed.netloc, path[: -len(ep)], "", "", "")
+                )
+        return api_base
+
+    @staticmethod
+    def _extract_api_version(api_base: Optional[str]) -> Optional[str]:
+        """Return the api-version query param from api_base if present."""
+        if not api_base:
+            return None
+        return parse_qs(urlparse(api_base).query).get("api-version", [None])[0]
+
     def get_complete_url(
         self,
         api_base: Optional[str],
@@ -39,10 +67,19 @@ class AzureContainerConfig(OpenAIContainerConfig):
           {endpoint}/openai/v1/containers
         when api_version is 'v1', 'latest', or 'preview'; otherwise:
           {endpoint}/openai/containers
+
+        The deployment's api_base may be the responses endpoint URL
+        (e.g. .../openai/responses?api-version=2025-04-01-preview). We
+        prefer the api-version embedded there over the deployment's
+        api_version field, which may point to an older chat API version.
         """
+        effective_params = dict(litellm_params)
+        api_version_from_base = self._extract_api_version(api_base)
+        if api_version_from_base:
+            effective_params["api_version"] = api_version_from_base
         return BaseAzureLLM._get_base_azure_url(
-            api_base=api_base,
-            litellm_params=litellm_params,
+            api_base=self._normalize_api_base(api_base),
+            litellm_params=effective_params,
             route="/openai/containers",
             default_api_version="v1",
         )

litellm/llms/custom_httpx/container_handler.py

@@ -257,14 +257,19 @@ class GenericContainerHandler:
         returns_binary = endpoint_config.get("returns_binary", False)
         is_multipart = endpoint_config.get("is_multipart", False)
 
+        # An empty dict passed as `params` to httpx strips any existing query
+        # string from the URL (e.g. ?api-version=...).  Use None instead so
+        # httpx leaves the URL's own query string intact.
+        effective_params = query_params or None
+
         try:
             if method == "GET":
                 response = http_client.get(
-                    url=url, headers=headers, params=query_params
+                    url=url, headers=headers, params=effective_params
                 )
             elif method == "DELETE":
                 response = http_client.delete(
-                    url=url, headers=headers, params=query_params
+                    url=url, headers=headers, params=effective_params
                 )
             elif method == "POST":
                 if is_multipart and "file" in kwargs:
@@ -272,11 +277,11 @@ class GenericContainerHandler:
                         kwargs["file"], headers
                     )
                     response = http_client.post(
-                        url=url, headers=headers, params=query_params, files=files
+                        url=url, headers=headers, params=effective_params, files=files
                     )
                 else:
                     response = http_client.post(
-                        url=url, headers=headers, params=query_params
+                        url=url, headers=headers, params=effective_params
                     )
             else:
                 raise ValueError(f"Unsupported HTTP method: {method}")
@@ -376,14 +381,19 @@ class GenericContainerHandler:
         returns_binary = endpoint_config.get("returns_binary", False)
         is_multipart = endpoint_config.get("is_multipart", False)
 
+        # An empty dict passed as `params` to httpx strips any existing query
+        # string from the URL (e.g. ?api-version=...).  Use None instead so
+        # httpx leaves the URL's own query string intact.
+        effective_params = query_params or None
+
         try:
             if method == "GET":
                 response = await http_client.get(
-                    url=url, headers=headers, params=query_params
+                    url=url, headers=headers, params=effective_params
                 )
             elif method == "DELETE":
                 response = await http_client.delete(
-                    url=url, headers=headers, params=query_params
+                    url=url, headers=headers, params=effective_params
                 )
             elif method == "POST":
                 if is_multipart and "file" in kwargs:
@@ -391,11 +401,11 @@ class GenericContainerHandler:
                         kwargs["file"], headers
                     )
                     response = await http_client.post(
-                        url=url, headers=headers, params=query_params, files=files
+                        url=url, headers=headers, params=effective_params, files=files
                     )
                 else:
                     response = await http_client.post(
-                        url=url, headers=headers, params=query_params
+                        url=url, headers=headers, params=effective_params
                     )
             else:
                 raise ValueError(f"Unsupported HTTP method: {method}")

litellm/llms/custom_httpx/llm_http_handler.py

@@ -7834,7 +7834,7 @@ class BaseLLMHTTPHandler:
             response = sync_httpx_client.get(
                 url=url,
                 headers=headers,
-                params=params,
+                params=params or None,
             )
 
             return container_provider_config.transform_container_list_response(
@@ -7911,7 +7911,7 @@ class BaseLLMHTTPHandler:
             response = await async_httpx_client.get(
                 url=url,
                 headers=headers,
-                params=params,
+                params=params or None,
             )
 
             return container_provider_config.transform_container_list_response(
@@ -8001,7 +8001,7 @@ class BaseLLMHTTPHandler:
             response = sync_httpx_client.get(
                 url=url,
                 headers=headers,
-                params=params,
+                params=params or None,
             )
 
             return container_provider_config.transform_container_retrieve_response(
@@ -8078,7 +8078,7 @@ class BaseLLMHTTPHandler:
             response = await async_httpx_client.get(
                 url=url,
                 headers=headers,
-                params=params,
+                params=params or None,
             )
 
             return container_provider_config.transform_container_retrieve_response(
@@ -8168,7 +8168,7 @@ class BaseLLMHTTPHandler:
             response = sync_httpx_client.delete(
                 url=url,
                 headers=headers,
-                params=params,
+                params=params or None,
             )
 
             return container_provider_config.transform_container_delete_response(
@@ -8245,7 +8245,7 @@ class BaseLLMHTTPHandler:
             response = await async_httpx_client.delete(
                 url=url,
                 headers=headers,
-                params=params,
+                params=params or None,
             )
 
             return container_provider_config.transform_container_delete_response(
@@ -8341,7 +8341,7 @@ class BaseLLMHTTPHandler:
             response = sync_httpx_client.get(
                 url=url,
                 headers=headers,
-                params=params,
+                params=params or None,
             )
 
             return container_provider_config.transform_container_file_list_response(
@@ -8420,7 +8420,7 @@ class BaseLLMHTTPHandler:
             response = await async_httpx_client.get(
                 url=url,
                 headers=headers,
-                params=params,
+                params=params or None,
             )
 
             return container_provider_config.transform_container_file_list_response(
@@ -8508,7 +8508,7 @@ class BaseLLMHTTPHandler:
             response = sync_httpx_client.get(
                 url=url,
                 headers=headers,
-                params=params,
+                params=params or None,
             )
 
             return container_provider_config.transform_container_file_content_response(
@@ -8584,7 +8584,7 @@ class BaseLLMHTTPHandler:
             response = await async_httpx_client.get(
                 url=url,
                 headers=headers,
-                params=params,
+                params=params or None,
             )
 
             return container_provider_config.transform_container_file_content_response(

litellm/llms/gemini/chat/transformation.py

@@ -164,5 +164,8 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
                                 # If conversion fails, leave as is and let the API handle it
                                 pass
         return _gemini_convert_messages_with_history(
-            messages=messages, model=model, litellm_params=litellm_params
+            messages=messages,
+            model=model,
+            litellm_params=litellm_params,
+            custom_llm_provider="gemini",
         )

litellm/llms/gemini/interactions/transformation.py

@@ -6,13 +6,18 @@ Per OpenAPI spec (https://ai.google.dev/static/api/interactions.openapi.json):
 - Get: GET https://generativelanguage.googleapis.com/{api_version}/interactions/{interaction_id}
 - Delete: DELETE https://generativelanguage.googleapis.com/{api_version}/interactions/{interaction_id}
 
-This is a thin wrapper - no transformation needed since we follow the spec directly.
+Schema versioning:
+- Default (Api-Revision: 2026-05-20): new `steps` schema.
+- Legacy (Api-Revision: 2026-05-07): old `outputs` schema, controlled via
+  litellm.use_legacy_interactions_schema = True. Remove flag after June 8, 2026.
 """
 
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
 
 import httpx
 
+import litellm
+
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.core_helpers import process_response_headers
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
@@ -84,6 +89,15 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
             api_key = GeminiModelInfo.get_api_key(litellm_params.get("api_key"))
             if api_key:
                 headers["x-goog-api-key"] = api_key
+
+        # Inject the Api-Revision header to select the response schema.
+        # Default to the new `steps` schema unless the operator has opted out.
+        # Remove this conditional after June 8, 2026 and always use 2026-05-20.
+        if litellm.use_legacy_interactions_schema:
+            headers["Api-Revision"] = "2026-05-07"
+        else:
+            headers["Api-Revision"] = "2026-05-20"
+
         return headers
 
     def get_complete_url(
@@ -119,8 +133,19 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
         headers: dict,
     ) -> Dict:
         """
-        Build request body per OpenAPI spec - minimal transformation.
+        Build request body per OpenAPI spec.
+
+        When on the new schema (use_legacy_interactions_schema=False, the default):
+        - ``response_mime_type`` is folded into ``response_format`` and stripped from
+          the body (the field was removed in Api-Revision 2026-05-20).
+        - ``generation_config.image_config`` is moved to a ``response_format`` entry
+          with ``"type": "image"`` (also removed from generation_config in 2026-05-20).
+
+        When on the legacy schema (use_legacy_interactions_schema=True):
+        - All fields are forwarded as-is.
         """
+        use_legacy: bool = litellm.use_legacy_interactions_schema
+
         request_body: Dict[str, Any] = {}
 
         # Model or Agent (one required)
@@ -135,24 +160,81 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
         if input is not None:
             request_body["input"] = input
 
-        # Pass through optional params directly (they match the spec)
+        # Pass through optional params — legacy schema keeps all fields as-is.
         optional_keys = [
             "tools",
             "system_instruction",
-            "generation_config",
             "stream",
             "store",
             "background",
             "environment",
             "response_modalities",
-            "response_format",
-            "response_mime_type",
             "previous_interaction_id",
         ]
         for key in optional_keys:
             if optional_params.get(key) is not None:
                 request_body[key] = optional_params[key]
 
+        if use_legacy:
+            # Legacy schema: forward response_mime_type and response_format as-is.
+            for key in ("response_format", "response_mime_type", "generation_config"):
+                if optional_params.get(key) is not None:
+                    request_body[key] = optional_params[key]
+        else:
+            # New schema (Api-Revision: 2026-05-20):
+            # response_mime_type is removed — fold it into response_format.
+            response_format = optional_params.get("response_format")
+            response_mime_type = optional_params.get("response_mime_type")
+
+            if (
+                response_mime_type
+                and not isinstance(response_format, list)
+                and (
+                    not isinstance(response_format, dict)
+                    or "mime_type" not in response_format
+                )
+            ):
+                # Wrap the legacy schema into the new polymorphic format.
+                new_rf: Dict[str, Any] = {
+                    "type": "text",
+                    "mime_type": response_mime_type,
+                }
+                if response_format is not None:
+                    new_rf["schema"] = response_format
+                response_format = new_rf
+
+            if response_format is not None:
+                request_body["response_format"] = response_format
+
+            # image_config moves out of generation_config into response_format.
+            generation_config: Optional[Dict[str, Any]] = optional_params.get(
+                "generation_config"
+            )
+            if generation_config is not None:
+                image_config = None
+                if isinstance(generation_config, dict):
+                    generation_config = dict(
+                        generation_config
+                    )  # avoid mutating the caller's dict
+                    image_config = generation_config.pop("image_config", None)
+                    if not generation_config:
+                        generation_config = None
+
+                if generation_config is not None:
+                    request_body["generation_config"] = generation_config
+
+                if image_config is not None:
+                    # Move image_config to response_format with type=image.
+                    image_rf: Dict[str, Any] = {"type": "image", **image_config}
+                    existing_rf = request_body.get("response_format")
+                    if existing_rf is None:
+                        request_body["response_format"] = image_rf
+                    elif isinstance(existing_rf, list):
+                        request_body["response_format"] = [*existing_rf, image_rf]
+                    else:
+                        # Convert single entry to array for multimodal output.
+                        request_body["response_format"] = [existing_rf, image_rf]
+
         return request_body
 
     def transform_response(

litellm/llms/vertex_ai/context_caching/transformation.py

@@ -174,7 +174,9 @@ def transform_openai_messages_to_gemini_context_caching(
     )
 
     transformed_messages = _gemini_convert_messages_with_history(
-        messages=new_messages, model=model
+        messages=new_messages,
+        model=model,
+        custom_llm_provider=custom_llm_provider,
     )
 
     model_name = "models/{}".format(model)

litellm/llms/vertex_ai/gemini/transformation.py

@@ -682,6 +682,7 @@ def _gemini_convert_messages_with_history(  # noqa: PLR0915
     messages: List[AllMessageValues],
     model: Optional[str] = None,
     litellm_params: Optional[dict] = None,
+    custom_llm_provider: Optional[str] = None,
 ) -> List[ContentType]:
     """
     Converts given messages from OpenAI format to Gemini format
@@ -983,7 +984,9 @@ def _gemini_convert_messages_with_history(  # noqa: PLR0915
                     or assistant_msg.get("function_call") is not None
                 ):  # support assistant tool invoke conversion
                     gemini_tool_call_parts = convert_to_gemini_tool_call_invoke(
-                        assistant_msg, model=model
+                        assistant_msg,
+                        model=model,
+                        custom_llm_provider=custom_llm_provider,
                     )
                     ## check if gemini_tool_call already exists in assistant_content
                     for gemini_tool_call_part in gemini_tool_call_parts:
@@ -1045,6 +1048,7 @@ def _gemini_convert_messages_with_history(  # noqa: PLR0915
                     messages[msg_i],  # type: ignore
                     last_message_with_tool_calls,  # type: ignore
                     model=model,
+                    custom_llm_provider=custom_llm_provider,
                 )
                 msg_i += 1
                 # Handle both single part and list of parts (for Computer Use with images)

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -289,6 +289,20 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             return True
         return False
 
+    @staticmethod
+    def _forward_gemini_function_call_id(
+        model: str, custom_llm_provider: Optional[str] = None
+    ) -> bool:
+        """
+        Whether to include `id` on function_call / function_response parts.
+
+        Gemini 3+ on Google AI Studio accepts (and returns) `id` for strict
+        tool-call matching. Vertex AI rejects the field with HTTP 400.
+        """
+        if custom_llm_provider != "gemini":
+            return False
+        return VertexGeminiConfig._is_gemini_3_or_newer(model)
+
     def _supports_penalty_parameters(self, model: str) -> bool:
         # Gemini 3 models do not support penalty parameters
         if VertexGeminiConfig._is_gemini_3_or_newer(model):
@@ -2649,7 +2663,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
         litellm_params: Optional[dict] = None,
     ) -> List[ContentType]:
         return _gemini_convert_messages_with_history(
-            messages=messages, model=model, litellm_params=litellm_params
+            messages=messages,
+            model=model,
+            litellm_params=litellm_params,
+            custom_llm_provider="vertex_ai",
         )
 
     def get_error_class(

litellm/model_prices_and_context_window_backup.json

@@ -27296,6 +27296,58 @@
         "supports_web_search": true,
         "tpm": 800000
     },
+    "openrouter/google/gemini-3.1-flash-lite": {
+        "cache_read_input_token_cost": 2.5e-08,
+        "cache_read_input_token_cost_per_audio_token": 5e-08,
+        "input_cost_per_audio_token": 5e-07,
+        "input_cost_per_token": 2.5e-07,
+        "litellm_provider": "openrouter",
+        "max_audio_length_hours": 8.4,
+        "max_audio_per_prompt": 1,
+        "max_images_per_prompt": 3000,
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_pdf_size_mb": 30,
+        "max_tokens": 65536,
+        "max_video_length": 1,
+        "max_videos_per_prompt": 10,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 1.5e-06,
+        "output_cost_per_token": 1.5e-06,
+        "rpm": 2000,
+        "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": false,
+        "supports_code_execution": true,
+        "supports_file_search": true,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_url_context": true,
+        "supports_video_input": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "tpm": 800000
+    },
     "openrouter/google/gemini-3.1-pro-preview": {
         "cache_read_input_token_cost": 2e-07,
         "cache_read_input_token_cost_above_200k_tokens": 4e-07,

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -1278,6 +1278,7 @@ class MCPServerManager:
                 tools = await self._get_tools_from_server(
                     server=server,
                     mcp_auth_header=server_auth_header,
+                    user_api_key_auth=user_api_key_auth,
                 )
                 return tools
             except Exception as e:
@@ -1458,6 +1459,7 @@ class MCPServerManager:
         extra_headers: Optional[Dict[str, str]] = None,
         add_prefix: bool = True,
         raw_headers: Optional[Dict[str, str]] = None,
+        user_api_key_auth: Optional[UserAPIKeyAuth] = None,
     ) -> List[MCPTool]:
         """
         Helper method to get tools from a single MCP server with prefixed names.
@@ -1484,6 +1486,46 @@ class MCPServerManager:
                     extra_headers = {}
                 extra_headers.update(server.static_headers)
 
+            # MCPJWTSigner: inject signed JWT for tools/list (list path skips pre_call_hook).
+            # Skip entirely when the signer is not configured (avoid an unnecessary
+            # dict copy on every list call), when the server has its own static
+            # Authorization header, when a per-user mcp_auth_header has already
+            # been resolved, or when the caller already supplied an Authorization
+            # entry in extra_headers (e.g. a per-user OAuth token resolved
+            # upstream) — admin-configured static auth and per-user OAuth must
+            # take precedence so the signer doesn't silently overwrite e.g. an
+            # upstream API key or a user's OAuth token (MCPClient._get_auth_headers
+            # applies extra_headers after writing Authorization from auth_value, so
+            # an injected JWT would otherwise clobber the per-user token).
+            if user_api_key_auth is not None and not server.spec_path:
+                from litellm.proxy.guardrails.guardrail_hooks.mcp_jwt_signer.mcp_jwt_signer import (
+                    get_mcp_jwt_signer,
+                    inject_mcp_jwt_headers_for_upstream,
+                )
+
+                static_headers = server.static_headers or {}
+                has_static_authorization = any(
+                    isinstance(k, str) and k.lower() == "authorization"
+                    for k in static_headers.keys()
+                )
+                has_extra_authorization = bool(extra_headers) and any(
+                    isinstance(k, str) and k.lower() == "authorization"
+                    for k in (extra_headers or {}).keys()
+                )
+
+                if (
+                    get_mcp_jwt_signer() is not None
+                    and not has_static_authorization
+                    and not mcp_auth_header
+                    and not has_extra_authorization
+                ):
+                    extra_headers = await inject_mcp_jwt_headers_for_upstream(
+                        user_api_key_dict=user_api_key_auth,
+                        extra_headers=extra_headers,
+                        raw_headers=raw_headers,
+                        for_list_tools=True,
+                    )
+
             stdio_env = self._build_stdio_env(server, raw_headers)
 
             client = await self._create_mcp_client(
@@ -2878,6 +2920,112 @@ class MCPServerManager:
 
         return cast(CallToolResult, result)
 
+    def _resolve_mcp_server_for_tool_call(
+        self,
+        server_name: str,
+        name: str,
+    ) -> MCPServer:
+        """Resolve MCP server for call_tool (prefixed name, registry, fallback)."""
+        prefixed_tool_name = add_server_prefix_to_name(name, server_name)
+        mcp_server = self._get_mcp_server_from_tool_name(prefixed_tool_name)
+        resolved_by_server_name_only = False
+        normalized_server_name = normalize_server_name(server_name)
+
+        def _candidate_matches_server_name(candidate: MCPServer) -> bool:
+            for identifier in (
+                candidate.alias,
+                candidate.server_name,
+                candidate.name,
+            ):
+                if identifier and normalize_server_name(identifier) == (
+                    normalized_server_name
+                ):
+                    return True
+            return False
+
+        if mcp_server is None:
+            for candidate in self.get_registry().values():
+                if _candidate_matches_server_name(candidate):
+                    mcp_server = candidate
+                    resolved_by_server_name_only = True
+                    break
+        if mcp_server is None:
+            fallback = self._get_mcp_server_from_tool_name(name)
+            if fallback is not None and (
+                not server_name or _candidate_matches_server_name(fallback)
+            ):
+                mcp_server = fallback
+        if mcp_server is None:
+            raise ValueError(f"Tool {name} not found")
+
+        if resolved_by_server_name_only:
+            tool_known = (
+                name in self.tool_name_to_mcp_server_name_mapping
+                or prefixed_tool_name in self.tool_name_to_mcp_server_name_mapping
+            )
+            if not tool_known:
+                raise ValueError(f"Tool {name} not found")
+
+        return mcp_server
+
+    async def _resolve_oauth2_headers_for_tool_call(
+        self,
+        mcp_server: MCPServer,
+        oauth2_headers: Optional[Dict[str, str]],
+        user_api_key_auth: Optional[UserAPIKeyAuth],
+    ) -> Optional[Dict[str, str]]:
+        """Look up per-user OAuth headers when the client did not supply a token."""
+        if (
+            not mcp_server.needs_user_oauth_token
+            or oauth2_headers
+            or user_api_key_auth is None
+        ):
+            return oauth2_headers
+
+        user_id = getattr(user_api_key_auth, "user_id", None)
+        if not user_id:
+            return oauth2_headers
+
+        try:
+            from litellm.proxy._experimental.mcp_server.server import (  # noqa: PLC0415
+                _get_user_oauth_extra_headers_from_db,
+            )
+
+            stored_headers = await _get_user_oauth_extra_headers_from_db(
+                server=mcp_server,
+                user_api_key_auth=user_api_key_auth,
+            )
+            if stored_headers:
+                return stored_headers
+        except Exception as _lookup_exc:
+            verbose_logger.debug(
+                "call_tool: per-user token lookup failed for " "user=%s server=%s: %s",
+                user_id,
+                mcp_server.server_id,
+                _lookup_exc,
+            )
+        return oauth2_headers
+
+    async def _gather_openapi_tool_tasks(
+        self,
+        tasks: List[Any],
+        proxy_logging_obj: Optional[ProxyLogging],
+    ) -> CallToolResult:
+        """Await OpenAPI tool tasks and return the tool call result."""
+        try:
+            mcp_responses = await asyncio.gather(*tasks)
+            result_index = 1 if proxy_logging_obj else 0
+            return cast(CallToolResult, mcp_responses[result_index])
+        except (
+            BlockedPiiEntityError,
+            GuardrailRaisedException,
+            HTTPException,
+        ) as e:
+            verbose_logger.error(
+                f"Guardrail blocked MCP tool call during result check: {str(e)}"
+            )
+            raise e
+
     async def call_tool(
         self,
         server_name: str,
@@ -2908,12 +3056,7 @@ class MCPServerManager:
             CallToolResult from the MCP server
         """
         start_time = datetime.datetime.now()
-
-        # Get the MCP server
-        prefixed_tool_name = add_server_prefix_to_name(name, server_name)
-        mcp_server = self._get_mcp_server_from_tool_name(prefixed_tool_name)
-        if mcp_server is None:
-            raise ValueError(f"Tool {name} not found")
+        mcp_server = self._resolve_mcp_server_for_tool_call(server_name, name)
 
         #########################################################
         # Pre MCP Tool Call Hook
@@ -2947,36 +3090,9 @@ class MCPServerManager:
             )
             tasks.append(during_hook_task)
 
-        # For per-user OAuth servers: if the client didn't supply a token in
-        # oauth2_headers, look up the stored token from Redis / DB.  This is the
-        # call_tool equivalent of _get_user_oauth_extra_headers_from_db used in
-        # list_tools.
-        if (
-            mcp_server.needs_user_oauth_token
-            and not oauth2_headers
-            and user_api_key_auth is not None
-        ):
-            user_id = getattr(user_api_key_auth, "user_id", None)
-            if user_id:
-                try:
-                    from litellm.proxy._experimental.mcp_server.server import (  # noqa: PLC0415
-                        _get_user_oauth_extra_headers_from_db,
-                    )
-
-                    stored_headers = await _get_user_oauth_extra_headers_from_db(
-                        server=mcp_server,
-                        user_api_key_auth=user_api_key_auth,
-                    )
-                    if stored_headers:
-                        oauth2_headers = stored_headers
-                except Exception as _lookup_exc:
-                    verbose_logger.debug(
-                        "call_tool: per-user token lookup failed for "
-                        "user=%s server=%s: %s",
-                        user_id,
-                        mcp_server.server_id,
-                        _lookup_exc,
-                    )
+        oauth2_headers = await self._resolve_oauth2_headers_for_tool_call(
+            mcp_server, oauth2_headers, user_api_key_auth
+        )
 
         # For OpenAPI servers, call the tool handler directly instead of via MCP client
         if mcp_server.spec_path:
@@ -3012,26 +3128,7 @@ class MCPServerManager:
                 hook_extra_headers=hook_result.get("extra_headers"),
             )
 
-        # For OpenAPI tools, await outside the client context
-        try:
-            mcp_responses = await asyncio.gather(*tasks)
-
-            # If proxy_logging_obj is None, the tool call result is at index 0
-            # If proxy_logging_obj is not None, the tool call result is at index 1 (after the during hook task)
-            result_index = 1 if proxy_logging_obj else 0
-            result = mcp_responses[result_index]
-
-            return cast(CallToolResult, result)
-        except (
-            BlockedPiiEntityError,
-            GuardrailRaisedException,
-            HTTPException,
-        ) as e:
-            # Re-raise guardrail exceptions to properly fail the MCP call
-            verbose_logger.error(
-                f"Guardrail blocked MCP tool call during result check: {str(e)}"
-            )
-            raise e
+        return await self._gather_openapi_tool_tasks(tasks, proxy_logging_obj)
 
     #########################################################
     # End of Methods that call the upstream MCP servers

litellm/proxy/_experimental/mcp_server/oauth_utils.py

@@ -29,6 +29,16 @@ _DEFAULT_PORTS = {"http": 80, "https": 443}
 # subdomain. HTTPS only.
 _TRUSTED_REDIRECT_ORIGINS_ENV = "MCP_TRUSTED_REDIRECT_ORIGINS"
 
+# Comma-separated private-use URI allowlist for native MCP clients.
+# A trailing ``*`` is a prefix match; end the prefix with ``/`` (e.g.
+# ``myapp://host/oauth/*``) so ``.../oauth/callback*`` does not also
+# match ``.../oauth/callback-2``.
+_TRUSTED_NATIVE_REDIRECT_URIS_ENV = "MCP_TRUSTED_NATIVE_REDIRECT_URIS"
+
+# Default allowlist for trusted native redirect URIs.
+_DEFAULT_NATIVE_REDIRECT_URIS: List[str] = [
+    "cursor://anysphere.cursor-mcp/oauth/callback",
+]
 
 _warned_invalid_proxy_base_url: Optional[str] = None
 
@@ -212,10 +222,82 @@ def _matches_trusted_origin_entry(netloc: str, entry: str) -> bool:
     return netloc == entry
 
 
+def _normalize_native_redirect_uri(
+    parsed,
+) -> str:
+    """Lowercase scheme, netloc, and path for allowlist comparison."""
+    return urlunparse(
+        (
+            (parsed.scheme or "").lower(),
+            (parsed.netloc or "").lower(),
+            (parsed.path or "").lower(),
+            "",
+            "",
+            "",
+        )
+    )
+
+
+def _parse_trusted_native_redirect_uris() -> List[str]:
+    """Built-in native MCP callbacks plus ``MCP_TRUSTED_NATIVE_REDIRECT_URIS``."""
+    entries: List[str] = [uri.lower() for uri in _DEFAULT_NATIVE_REDIRECT_URIS]
+    raw = os.environ.get(_TRUSTED_NATIVE_REDIRECT_URIS_ENV, "").strip()
+    if not raw:
+        return entries
+    for token in raw.split(","):
+        entry = token.strip().lower()
+        if entry and entry not in entries:
+            entries.append(entry)
+    return entries
+
+
+def _native_wildcard_prefix_matches(normalized: str, prefix: str) -> bool:
+    """Prefix match for ``entry*`` allowlist rows.
+
+    When the prefix does not end with ``/``, only exact matches or
+    deeper path segments (``prefix/...``) are accepted — not siblings
+    like ``prefix-2``.
+    """
+    if not normalized.startswith(prefix):
+        return False
+    suffix = normalized[len(prefix) :]
+    if not suffix:
+        return True
+    if prefix.endswith("/"):
+        return True
+    return suffix[0] == "/"
+
+
+def _matches_trusted_native_redirect_uri(parsed) -> bool:
+    """Allowlisted private-use / custom-scheme OAuth callbacks for native MCP clients."""
+    if parsed.fragment:
+        return False
+    # Query strings are not part of registered redirect_uris (RFC 6749 §3.1.2).
+    # Rejecting them prevents allowlist bypass via ``.../callback?injected=...``.
+    if parsed.query:
+        return False
+    if not parsed.netloc:
+        return False
+    if parsed.username is not None or parsed.password is not None:
+        return False
+    if "\\" in parsed.netloc:
+        return False
+
+    normalized = _normalize_native_redirect_uri(parsed)
+    for entry in _parse_trusted_native_redirect_uris():
+        if entry.endswith("*"):
+            if _native_wildcard_prefix_matches(normalized, entry[:-1]):
+                return True
+        elif normalized == entry:
+            return True
+    return False
+
+
 def validate_trusted_redirect_uri(request: Request, redirect_uri: str) -> None:
     """Accept ``redirect_uri`` when it is (a) same-origin with the
-    proxy's own request origin, (b) loopback, or (c) listed in the
-    ``MCP_TRUSTED_REDIRECT_ORIGINS`` ops allowlist.
+    proxy's own request origin, (b) loopback, (c) listed in the
+    ``MCP_TRUSTED_REDIRECT_ORIGINS`` ops allowlist, or (d) a built-in /
+    env-configured native MCP client callback (e.g. ``cursor://``).
 
     Same-origin is VERIA-57's threat-model-safe equivalent of loopback:
     an attacker who can host content on the proxy's own HTTPS origin
@@ -239,6 +321,8 @@ def validate_trusted_redirect_uri(request: Request, redirect_uri: str) -> None:
     except ValueError:
         raise HTTPException(status_code=400, detail="invalid_request")
     if parsed.scheme not in ("http", "https"):
+        if _matches_trusted_native_redirect_uri(parsed):
+            return
         raise HTTPException(status_code=400, detail="invalid_request")
     if parsed.fragment:
         raise HTTPException(status_code=400, detail="invalid_request")
@@ -310,9 +394,12 @@ def validate_trusted_redirect_uri(request: Request, redirect_uri: str) -> None:
         "Inbound headers: X-Forwarded-Proto=%r X-Forwarded-Host=%r "
         "X-Forwarded-Port=%r Host=%r. "
         "Trusted-redirect-origins env=%r. "
+        "Trusted-native-redirect-uris env=%r. "
         "If this should be accepted, either align ingress X-Forwarded-* "
         "with the browser URL, set PROXY_BASE_URL to your public origin, "
-        "or add the redirect_uri host to MCP_TRUSTED_REDIRECT_ORIGINS.",
+        "add the redirect_uri host to MCP_TRUSTED_REDIRECT_ORIGINS, or "
+        "for native MCP clients (cursor://, etc.) add the full redirect_uri "
+        "to MCP_TRUSTED_NATIVE_REDIRECT_URIS.",
         redirect_uri,
         proxy_base,
         os.environ.get("PROXY_BASE_URL"),
@@ -321,5 +408,6 @@ def validate_trusted_redirect_uri(request: Request, redirect_uri: str) -> None:
         request.headers.get("X-Forwarded-Port"),
         request.headers.get("Host"),
         os.environ.get(_TRUSTED_REDIRECT_ORIGINS_ENV),
+        os.environ.get(_TRUSTED_NATIVE_REDIRECT_URIS_ENV),
     )
     raise HTTPException(status_code=400, detail="invalid_request")

litellm/proxy/_experimental/mcp_server/rest_endpoints.py

@@ -1,6 +1,17 @@
 import importlib
 from datetime import datetime
-from typing import Any, Awaitable, Callable, Dict, List, Literal, Optional, Set, Union
+from typing import (
+    Any,
+    Awaitable,
+    Callable,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+)
 
 from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
 
@@ -232,11 +243,32 @@ if MCP_AVAILABLE:
         )
         return mcp_auth_header, mcp_server_auth_headers, raw_headers
 
+    def _resolve_mcp_server_id_for_rest(
+        server_id: str,
+        allowed_server_ids: Union[Set[str], List[str]],
+        client_ip: Optional[str] = None,
+    ) -> str:
+        """
+        Map REST ``server_id`` (UUID, server_name, or alias) to canonical server_id.
+
+        tools/list already did this; tools/call must match so clients can pass
+        server names like ``order_status_mcp`` instead of only UUIDs.
+        """
+        allowed = set(allowed_server_ids)
+        if server_id in allowed:
+            return server_id
+        by_name = global_mcp_server_manager.get_mcp_server_by_name(
+            server_id, client_ip=client_ip
+        )
+        if by_name is not None and by_name.server_id in allowed:
+            return by_name.server_id
+        return server_id
+
     async def _resolve_allowed_mcp_servers_with_ip_filter(
         request: Request,
         user_api_key_dict: UserAPIKeyAuth,
         server_id: str,
-    ) -> List[MCPServer]:
+    ) -> Tuple[List[MCPServer], str]:
         """
         Resolve allowed MCP servers for a tool call with IP filtering.
 
@@ -246,10 +278,10 @@ if MCP_AVAILABLE:
             server_id: The server ID to validate access for
 
         Returns:
-            List of allowed MCPServer objects
+            Tuple of (allowed MCPServer objects, canonical server_id)
 
         Raises:
-            HTTPException: If the server_id is not allowed
+            HTTPException: If the server_id is not allowed or not found
         """
         # Get all auth contexts
         auth_contexts = await build_effective_auth_contexts(user_api_key_dict)
@@ -269,8 +301,41 @@ if MCP_AVAILABLE:
             )
         )
 
-        # Check if the specified server_id is allowed
-        if server_id not in allowed_server_ids_set:
+        canonical_server_id = _resolve_mcp_server_id_for_rest(
+            server_id, allowed_server_ids_set, _rest_client_ip
+        )
+
+        if canonical_server_id not in allowed_server_ids_set:
+            _server = global_mcp_server_manager.get_mcp_server_by_id(
+                server_id
+            ) or global_mcp_server_manager.get_mcp_server_by_name(server_id)
+            if (
+                _server is not None
+                and _rest_client_ip is not None
+                and not global_mcp_server_manager._is_server_accessible_from_ip(
+                    _server, _rest_client_ip
+                )
+            ):
+                raise HTTPException(
+                    status_code=403,
+                    detail={
+                        "error": "ip_filtering",
+                        "message": (
+                            f"MCP server '{server_id}' is not accessible from your IP address "
+                            f"({_rest_client_ip}). This server is restricted to internal "
+                            "networks only. To make it externally accessible, set "
+                            "'available_on_public_internet: true' in the server configuration."
+                        ),
+                    },
+                )
+            if _server is None:
+                raise HTTPException(
+                    status_code=404,
+                    detail={
+                        "error": "server_not_found",
+                        "message": f"MCP server '{server_id}' was not found",
+                    },
+                )
             raise HTTPException(
                 status_code=403,
                 detail={
@@ -286,7 +351,7 @@ if MCP_AVAILABLE:
             if server is not None:
                 allowed_mcp_servers.append(server)
 
-        return allowed_mcp_servers
+        return allowed_mcp_servers, canonical_server_id
 
     async def _get_tools_for_single_server(
         server,
@@ -302,6 +367,7 @@ if MCP_AVAILABLE:
             extra_headers=extra_headers,
             add_prefix=False,
             raw_headers=raw_headers,
+            user_api_key_auth=user_api_key_auth,
         )
 
         # Filter tools based on allowed_tools configuration
@@ -778,7 +844,7 @@ if MCP_AVAILABLE:
                     },
                 )
 
-            tool_arguments = data.get("arguments")
+            tool_arguments = data.get("arguments") or {}
 
             proxy_base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
             (
@@ -811,14 +877,18 @@ if MCP_AVAILABLE:
                 data["user_api_key_auth"] = data["metadata"]["user_api_key_auth"]
 
             # Resolve allowed MCP servers with IP filtering
-            allowed_mcp_servers = await _resolve_allowed_mcp_servers_with_ip_filter(
+            (
+                allowed_mcp_servers,
+                canonical_server_id,
+            ) = await _resolve_allowed_mcp_servers_with_ip_filter(
                 request, user_api_key_dict, server_id
             )
 
             # Look up per-user OAuth headers for this server (mirrors list_tool_rest_api).
             user_oauth_extra_headers: Optional[Dict[str, str]] = None
             target_server = next(
-                (s for s in allowed_mcp_servers if s.server_id == server_id), None
+                (s for s in allowed_mcp_servers if s.server_id == canonical_server_id),
+                None,
             )
             if target_server is not None:
                 user_oauth_extra_headers = await _get_user_oauth_extra_headers(
@@ -837,6 +907,7 @@ if MCP_AVAILABLE:
                 oauth2_headers=user_oauth_extra_headers or data.get("oauth2_headers"),
                 raw_headers=data.get("raw_headers"),
                 litellm_logging_obj=data.get("litellm_logging_obj"),
+                requested_server_id=canonical_server_id,
             )
             return result
         except BlockedPiiEntityError as e:

litellm/proxy/_experimental/mcp_server/server.py

@@ -1424,6 +1424,7 @@ if MCP_AVAILABLE:
                         extra_headers=extra_headers,
                         add_prefix=True,  # Always add server prefix
                         raw_headers=raw_headers,
+                        user_api_key_auth=user_api_key_auth,
                     )
                     filtered_tools = filter_tools_by_allowed_tools(tools, server)
 
@@ -2130,6 +2131,7 @@ if MCP_AVAILABLE:
         """
         # Track resolved MCP server for both permission checks and dispatch
         mcp_server: Optional[MCPServer] = None
+        requested_server_id: Optional[str] = kwargs.get("requested_server_id")
 
         # If the client called with a display-name override (e.g. "Get Pet"),
         # translate it back to the original prefixed name before any routing.
@@ -2138,14 +2140,55 @@ if MCP_AVAILABLE:
         # Remove prefix from tool name for logging and processing
         original_tool_name, server_name = split_server_prefix_from_name(name)
 
+        requested_server: Optional[MCPServer] = None
+        if requested_server_id:
+            requested_server = next(
+                (s for s in allowed_mcp_servers if s.server_id == requested_server_id),
+                None,
+            )
+
         # Resolve the actual MCP server up-front so the permission check uses
         # the canonical server.name even when the tool name is prefixed with a
         # short ID (LITELLM_USE_SHORT_MCP_TOOL_PREFIX) that doesn't match the
         # server's display name directly.
         mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(name)
+        if mcp_server is None and requested_server is not None:
+            # REST callers may pass the raw tool name (no prefix) plus a
+            # ``requested_server_id``. The mapping might only contain the
+            # prefixed form, so retry the lookup with every known prefix of
+            # the requested server before treating the tool as unresolved —
+            # otherwise the tool_server_mismatch guard below is silently
+            # bypassed.
+            for known_prefix in iter_known_server_prefixes(requested_server):
+                candidate = global_mcp_server_manager._get_mcp_server_from_tool_name(
+                    add_server_prefix_to_name(name, known_prefix)
+                )
+                if candidate is not None:
+                    mcp_server = candidate
+                    break
         if mcp_server is not None:
             server_name = mcp_server.name
 
+        # REST /mcp-rest/tools/call passes server_id — tool must belong to that server
+        if requested_server is not None:
+            if (
+                mcp_server is not None
+                and mcp_server.server_id != requested_server.server_id
+            ):
+                raise HTTPException(
+                    status_code=403,
+                    detail={
+                        "error": "tool_server_mismatch",
+                        "message": (
+                            f"Tool '{name}' belongs to MCP server '{mcp_server.name}' "
+                            f"but request specified server_id for '{requested_server.name}'."
+                        ),
+                    },
+                )
+            if mcp_server is None:
+                mcp_server = requested_server
+                server_name = requested_server.name
+
         # Only enforce server-level permissions when we can resolve a server
         if server_name:
             if not MCPRequestHandler.is_tool_allowed(

litellm/proxy/_lazy_openapi_snapshot.json

@@ -3171,7 +3171,7 @@
           ]
         },
         "post": {
-          "description": "Create a new agent\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/agents\" \\\n    -H \"Authorization: Bearer <your_api_key>\" \\\n    -H \"Content-Type: application/json\" \\\n    -d '{\n        \"agent\": {\n            \"agent_name\": \"my-custom-agent\",\n            \"agent_card_params\": {\n                \"protocolVersion\": \"1.0\",\n                \"name\": \"Hello World Agent\",\n                \"description\": \"Just a hello world agent\",\n                \"url\": \"http://localhost:9999/\",\n                \"version\": \"1.0.0\",\n                \"defaultInputModes\": [\"text\"],\n                \"defaultOutputModes\": [\"text\"],\n                \"capabilities\": {\n                    \"streaming\": true\n                },\n                \"skills\": [\n                    {\n                        \"id\": \"hello_world\",\n                        \"name\": \"Returns hello world\",\n                        \"description\": \"just returns hello world\",\n                        \"tags\": [\"hello world\"],\n                        \"examples\": [\"hi\", \"hello world\"]\n                    }\n                ]\n            },\n            \"litellm_params\": {\n                \"make_public\": true\n            }\n        }\n    }'\n```",
+          "description": "Create a new agent\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/v1/agents\" \\\n    -H \"Authorization: Bearer <your_api_key>\" \\\n    -H \"Content-Type: application/json\" \\\n    -d '{\n        \"agent_name\": \"my-custom-agent\",\n            \"agent_card_params\": {\n                \"protocolVersion\": \"1.0\",\n                \"name\": \"Hello World Agent\",\n                \"description\": \"Just a hello world agent\",\n                \"url\": \"http://localhost:9999/\",\n                \"version\": \"1.0.0\",\n                \"defaultInputModes\": [\"text\"],\n                \"defaultOutputModes\": [\"text\"],\n                \"capabilities\": {\n                    \"streaming\": true\n                },\n                \"skills\": [\n                    {\n                        \"id\": \"hello_world\",\n                        \"name\": \"Returns hello world\",\n                        \"description\": \"just returns hello world\",\n                        \"tags\": [\"hello world\"],\n                        \"examples\": [\"hi\", \"hello world\"]\n                    }\n                ]\n            },\n            \"litellm_params\": {\n                \"make_public\": true\n       }\n    }'\n```",
           "operationId": "create_agent_v1_agents_post",
           "requestBody": {
             "content": {

litellm/proxy/_types.py

@@ -2361,6 +2361,30 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
     database_connection_timeout: Optional[float] = Field(
         60, description="default timeout for a connection to the database"
     )
+    database_connect_timeout: Optional[float] = Field(
+        None,
+        description=(
+            "Prisma `connect_timeout` URL param (seconds). Bounds how long the "
+            "engine waits to establish a new connection before failing. Defaults "
+            "to Prisma's built-in value when unset."
+        ),
+    )
+    database_socket_timeout: Optional[float] = Field(
+        None,
+        description=(
+            "Prisma `socket_timeout` URL param (seconds). When set, an idle/slow "
+            "connection that has not produced data within this window is closed. "
+            "This is the main knob for capping idle DB connections from LiteLLM."
+        ),
+    )
+    database_extra_connection_params: Optional[Dict[str, Any]] = Field(
+        None,
+        description=(
+            "Escape hatch: extra key/value pairs appended verbatim to the Prisma "
+            "DATABASE_URL / DIRECT_URL query string (e.g. `sslmode`, `pgbouncer`, "
+            "`statement_cache_size`). Keys here override any default LiteLLM sets."
+        ),
+    )
     database_type: Optional[Literal["dynamo_db"]] = Field(
         None, description="to use dynamodb instead of postgres db"
     )

litellm/proxy/auth/model_checks.py

@@ -4,13 +4,17 @@ from typing import Dict, List, Optional, Set
 
 import litellm
 from litellm._logging import verbose_proxy_logger
+from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
 from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth
 from litellm.router import Router
 from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
-from litellm.types.router import LiteLLM_Params
+from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params
 from litellm.utils import get_valid_models
 
 
+_CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields)
+
+
 def _check_wildcard_routing(model: str) -> bool:
     """
     Returns True if a model is a provider wildcard.
@@ -178,6 +182,7 @@ def get_complete_model_list(
     model_access_groups: Dict[str, List[str]] = {},
     include_model_access_groups: Optional[bool] = False,
     only_model_access_groups: Optional[bool] = False,
+    team_id: Optional[str] = None,
 ) -> List[str]:
     """Logic for returning complete model list for a given key + team pair"""
 
@@ -222,6 +227,7 @@ def get_complete_model_list(
         unique_models=unique_models,
         return_wildcard_routes=return_wildcard_routes,
         llm_router=llm_router,
+        team_id=team_id,
     )
 
     complete_model_list = unique_models + all_wildcard_models
@@ -229,6 +235,29 @@ def get_complete_model_list(
     return complete_model_list
 
 
+def _hydrate_litellm_credential_name(
+    litellm_params: Optional[LiteLLM_Params],
+) -> Optional[LiteLLM_Params]:
+    if litellm_params is None or litellm_params.litellm_credential_name is None:
+        return litellm_params
+
+    credential_values = CredentialAccessor.get_credential_values(
+        litellm_params.litellm_credential_name
+    )
+    if not credential_values:
+        return litellm_params
+
+    litellm_params = litellm_params.model_copy()
+    for key, value in credential_values.items():
+        if (
+            key in _CREDENTIAL_LITELLM_PARAM_FIELDS
+            and getattr(litellm_params, key, None) is None
+        ):
+            setattr(litellm_params, key, value)
+    litellm_params.litellm_credential_name = None
+    return litellm_params
+
+
 def get_known_models_from_wildcard(
     wildcard_model: str, litellm_params: Optional[LiteLLM_Params] = None
 ) -> List[str]:
@@ -247,7 +276,7 @@ def get_known_models_from_wildcard(
     else:
         provider = wildcard_provider_prefix
 
-    # get all known provider models
+    litellm_params = _hydrate_litellm_credential_name(litellm_params)
 
     wildcard_models = get_provider_models(
         provider=provider, litellm_params=litellm_params
@@ -285,6 +314,7 @@ def _get_wildcard_models(
     unique_models: List[str],
     return_wildcard_routes: Optional[bool] = False,
     llm_router: Optional[Router] = None,
+    team_id: Optional[str] = None,
 ) -> List[str]:
     models_to_remove = set()
     all_wildcard_models = []
@@ -297,7 +327,9 @@ def _get_wildcard_models(
 
             ## get litellm params from model
             if llm_router is not None:
-                model_list = llm_router.get_model_list(model_name=model)
+                model_list = llm_router.get_model_list(
+                    model_name=model, team_id=team_id
+                )
                 if model_list:
                     for router_model in model_list:
                         wildcard_models = get_known_models_from_wildcard(

litellm/proxy/auth/user_api_key_auth.py

@@ -12,7 +12,7 @@ import fnmatch
 import re
 import secrets
 from datetime import datetime, timezone
-from typing import Any, Iterator, List, Optional, Tuple, Union, cast
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Union, cast
 
 import fastapi
 from fastapi import HTTPException, Request, WebSocket, status
@@ -333,8 +333,22 @@ def _apply_budget_limits_to_end_user_params(
 async def user_api_key_auth_websocket(websocket: WebSocket):
     # Accept the WebSocket connection
 
-    scope_headers = list(websocket.scope.get("headers") or [])
-    request = Request(scope={"type": "http", "headers": scope_headers})
+    ws_scope = websocket.scope or {}
+    scope_headers = list(ws_scope.get("headers") or [])
+    # ``get_request_route`` falls back to ``request.url.path`` when
+    # ``scope["path"]`` is absent. On WebSockets that fallback reads
+    # ``websocket.url``, which Starlette reconstructs from the (poisonable)
+    # Host header. Carry the ASGI scope's path / root_path so the lookup
+    # never reaches the fallback.
+    synthetic_scope: Dict[str, Any] = {
+        "type": "http",
+        "headers": scope_headers,
+        "path": ws_scope.get("path", ""),
+    }
+    for key in ("root_path", "app_root_path"):
+        if key in ws_scope:
+            synthetic_scope[key] = ws_scope[key]
+    request = Request(scope=synthetic_scope)
 
     request._url = websocket.url
 

litellm/proxy/batches_endpoints/endpoints.py

@@ -523,6 +523,10 @@ async def retrieve_batch(  # noqa: PLR0915
                 custom_llm_provider=custom_llm_provider, **data  # type: ignore
             )
 
+        response = await proxy_logging_obj.post_call_success_hook(
+            data=data, user_api_key_dict=user_api_key_dict, response=response
+        )
+
         # FIX: Update the database with the latest state from provider
         await update_batch_in_database(
             batch_id=batch_id,
@@ -533,19 +537,9 @@ async def retrieve_batch(  # noqa: PLR0915
             verbose_proxy_logger=verbose_proxy_logger,
             db_batch_object=db_batch_object,
             operation="retrieve",
+            user_api_key_dict=user_api_key_dict,
         )
 
-        ### CALL HOOKS ### - modify outgoing data
-        response = await proxy_logging_obj.post_call_success_hook(
-            data=data, user_api_key_dict=user_api_key_dict, response=response
-        )
-
-        # Fix: bug_feb14_batch_retrieve_returns_raw_input_file_id
-        # Resolve raw provider file IDs (input, output, error) to unified IDs.
-        if unified_batch_id:
-            await resolve_input_file_id_to_unified(response, prisma_client)
-            await resolve_output_file_ids_to_unified(response, prisma_client)
-
         ### ALERTING ###
         asyncio.create_task(
             proxy_logging_obj.update_request_status(
@@ -917,10 +911,14 @@ async def cancel_batch(
                 **_cancel_batch_data,
             )
 
-        # FIX: Update the database with the new cancelled state
         managed_files_obj = proxy_logging_obj.get_proxy_hook("managed_files")
         from litellm.proxy.proxy_server import prisma_client
 
+        response = await proxy_logging_obj.post_call_success_hook(
+            data=data, user_api_key_dict=user_api_key_dict, response=response
+        )
+
+        # FIX: Update the database with the new cancelled state
         await update_batch_in_database(
             batch_id=batch_id,
             unified_batch_id=unified_batch_id,
@@ -929,11 +927,7 @@ async def cancel_batch(
             prisma_client=prisma_client,
             verbose_proxy_logger=verbose_proxy_logger,
             operation="cancel",
-        )
-
-        ### CALL HOOKS ### - modify outgoing data
-        response = await proxy_logging_obj.post_call_success_hook(
-            data=data, user_api_key_dict=user_api_key_dict, response=response
+            user_api_key_dict=user_api_key_dict,
         )
 
         ### ALERTING ###

litellm/proxy/common_utils/http_parsing_utils.py

@@ -13,6 +13,34 @@ from litellm.proxy.common_utils.callback_utils import (
 from litellm.types.router import Deployment
 
 
+_FORM_CONTENT_TYPES: frozenset[str] = frozenset(
+    {"application/x-www-form-urlencoded", "multipart/form-data"}
+)
+
+
+def _normalize_media_type(content_type: str) -> str:
+    """Return the bare media type per RFC 7231: strip params, trim, lowercase."""
+    if not content_type:
+        return ""
+    return content_type.split(";", 1)[0].strip().lower()
+
+
+def _is_form_content_type(content_type: str) -> bool:
+    """
+    True iff Starlette's ``request.form()`` will actually parse this body.
+
+    Substring matching ``"form"`` is unsafe: ``request.form()`` returns empty
+    ``FormData`` for non-canonical types without consuming the body, leaving
+    the auth-time pre-read and the handler's read seeing different payloads.
+    """
+    return _normalize_media_type(content_type) in _FORM_CONTENT_TYPES
+
+
+def _is_json_content_type(content_type: str) -> bool:
+    """True iff the body should be parsed as JSON."""
+    return _normalize_media_type(content_type) == "application/json"
+
+
 async def _read_request_body(request: Optional[Request]) -> Dict:
     """
     Safely read the request body and parse it as JSON.
@@ -37,8 +65,24 @@ async def _read_request_body(request: Optional[Request]) -> Dict:
         _request_headers: dict = _safe_get_request_headers(request=request)
         content_type = _request_headers.get("content-type", "")
 
-        if "form" in content_type:
-            parsed_body = dict(await request.form())
+        if _is_form_content_type(content_type):
+            try:
+                form_data = await request.form()
+            except Exception as e:
+                # ``request.form()`` raises on malformed multipart (missing
+                # boundary, malformed chunk encoding, …). Surface as 400 so
+                # the auth-time pre-read does not silently cache ``{}`` while
+                # a later raw-body re-read sees the original payload —
+                # banned-param checks must see the same body the handler
+                # acts on.
+                verbose_proxy_logger.error(f"Invalid form payload: {e}")
+                raise ProxyException(
+                    message=f"Invalid form payload: {e}",
+                    type="invalid_request_error",
+                    param="request_body",
+                    code=status.HTTP_400_BAD_REQUEST,
+                )
+            parsed_body = dict(form_data)
             if "metadata" in parsed_body and isinstance(parsed_body["metadata"], str):
                 parsed_body["metadata"] = json.loads(parsed_body["metadata"])
         else:
@@ -306,18 +350,13 @@ async def get_request_body(request: Request) -> Dict[str, Any]:
     Read the request body and parse it as JSON.
     """
     if request.method == "POST":
-        if request.headers.get("content-type", "") == "application/json":
+        content_type = request.headers.get("content-type", "")
+        if _is_json_content_type(content_type):
             return await _read_request_body(request)
-        elif "multipart/form-data" in request.headers.get(
-            "content-type", ""
-        ) or "application/x-www-form-urlencoded" in request.headers.get(
-            "content-type", ""
-        ):
+        elif _is_form_content_type(content_type):
             return await get_form_data(request)
         else:
-            raise ValueError(
-                f"Unsupported content type: {request.headers.get('content-type')}"
-            )
+            raise ValueError(f"Unsupported content type: {content_type}")
     return {}
 
 

litellm/proxy/container_endpoints/endpoints.py

@@ -328,7 +328,7 @@ async def retrieve_container(
         custom_llm_provider=custom_llm_provider,
     )
     data.update(
-        get_container_forwarding_params(
+        await get_container_forwarding_params(
             container_id,
             original_container_id,
             custom_llm_provider,
@@ -433,7 +433,7 @@ async def delete_container(
         custom_llm_provider=custom_llm_provider,
     )
     data.update(
-        get_container_forwarding_params(
+        await get_container_forwarding_params(
             container_id,
             original_container_id,
             custom_llm_provider,

litellm/proxy/container_endpoints/handler_factory.py

@@ -196,10 +196,12 @@ async def _process_binary_request(
     )
     data: Dict[str, Any] = {
         "file_id": file_id,
-        **get_container_forwarding_params(
-            container_id=container_id,
-            original_container_id=original_container_id,
-            custom_llm_provider=resolved_provider,
+        **(
+            await get_container_forwarding_params(
+                container_id=container_id,
+                original_container_id=original_container_id,
+                custom_llm_provider=resolved_provider,
+            )
         ),
     }
     processor = ProxyBaseLLMRequestProcessing(data=data)
@@ -316,7 +318,7 @@ async def _process_multipart_upload_request(
     )
 
     data.update(
-        get_container_forwarding_params(
+        await get_container_forwarding_params(
             container_id=container_id,
             original_container_id=original_container_id,
             custom_llm_provider=resolved_provider,
@@ -396,7 +398,7 @@ async def _process_request(
             )
         )
         data.update(
-            get_container_forwarding_params(
+            await get_container_forwarding_params(
                 container_id=path_params["container_id"],
                 original_container_id=original_container_id,
                 custom_llm_provider=resolved_provider,

litellm/proxy/container_endpoints/ownership.py

@@ -23,6 +23,13 @@ CONTAINER_OBJECT_PURPOSE = "container"
 _NEGATIVE_OWNER_SENTINEL = "__litellm_container_no_owner__"
 _CONTAINER_OWNER_CACHE = InMemoryCache(max_size_in_memory=10000, default_ttl=60)
 
+# Caches the stored ``unified_object_id`` (the encoded container ID
+# captured at create time) so ``get_container_forwarding_params`` can
+# recover the deployment ``model_id`` for native upstream IDs without
+# re-hitting Prisma on every retrieve/delete.
+_NEGATIVE_STORED_ID_SENTINEL = "__litellm_container_no_stored_id__"
+_CONTAINER_STORED_ID_CACHE = InMemoryCache(max_size_in_memory=10000, default_ttl=60)
+
 # Per-caller-scope cache for ``GET /v1/containers`` list filtering. Without
 # this, every list call issues a fresh ``find_many`` against
 # ``litellm_managedobjecttable``. The cache key is the sorted owner-scope
@@ -56,7 +63,7 @@ def decode_container_id_for_ownership(
     return original_container_id, custom_llm_provider
 
 
-def get_container_forwarding_params(
+async def get_container_forwarding_params(
     container_id: str, original_container_id: str, custom_llm_provider: str
 ) -> Dict[str, str]:
     params = {
@@ -65,6 +72,20 @@ def get_container_forwarding_params(
     }
     decoded = ResponsesAPIRequestUtils._decode_container_id(container_id)
     model_id = decoded.get("model_id")
+    if not (isinstance(model_id, str) and model_id):
+        # Native upstream IDs (e.g. Azure ``cntr_<hex>``) carry no LiteLLM
+        # routing payload, so decoding the user-supplied id yields no
+        # ``model_id``. Recover it from the encoded ``unified_object_id``
+        # captured on the ownership row at create time — when the router
+        # selected a specific deployment that ID embeds the model_id.
+        stored_id = await _get_stored_container_id(
+            original_container_id, custom_llm_provider
+        )
+        if stored_id and stored_id != container_id:
+            stored_decoded = ResponsesAPIRequestUtils._decode_container_id(stored_id)
+            stored_model_id = stored_decoded.get("model_id")
+            if isinstance(stored_model_id, str) and stored_model_id:
+                model_id = stored_model_id
     if isinstance(model_id, str) and model_id:
         params["model_id"] = model_id
     return params
@@ -168,6 +189,7 @@ async def record_container_owner(
         )
 
     _CONTAINER_OWNER_CACHE.set_cache(model_object_id, owner)
+    _CONTAINER_STORED_ID_CACHE.set_cache(model_object_id, container_id)
     # Drop the caller's own list-cache entry so the just-created container
     # shows up on their next ``GET /v1/containers``. Other callers with
     # disjoint scope tuples have their own entries; intersecting-scope
@@ -207,9 +229,60 @@ async def _get_container_owner(
     _CONTAINER_OWNER_CACHE.set_cache(
         model_object_id, owner if owner is not None else _NEGATIVE_OWNER_SENTINEL
     )
+    stored_id = getattr(row, "unified_object_id", None) if row is not None else None
+    _CONTAINER_STORED_ID_CACHE.set_cache(
+        model_object_id,
+        (
+            stored_id
+            if isinstance(stored_id, str) and stored_id
+            else _NEGATIVE_STORED_ID_SENTINEL
+        ),
+    )
     return owner
 
 
+async def _get_stored_container_id(
+    original_container_id: str, custom_llm_provider: str
+) -> Optional[str]:
+    """Return the ``unified_object_id`` stored at create time, if any.
+
+    Used by :func:`get_container_forwarding_params` to recover the
+    deployment ``model_id`` for native upstream container IDs: the stored
+    value is the encoded form produced by ``encode_container_id_in_response``
+    when the router selected a specific deployment.
+    """
+    model_object_id = _container_model_object_id(
+        original_container_id, custom_llm_provider
+    )
+
+    cached = _CONTAINER_STORED_ID_CACHE.get_cache(model_object_id)
+    if cached == _NEGATIVE_STORED_ID_SENTINEL:
+        return None
+    if isinstance(cached, str) and cached:
+        return cached
+
+    prisma_client = await _get_prisma_client()
+    if prisma_client is None:
+        return None
+
+    row = await prisma_client.db.litellm_managedobjecttable.find_first(
+        where={
+            "model_object_id": model_object_id,
+            "file_purpose": CONTAINER_OBJECT_PURPOSE,
+        }
+    )
+    stored_id = getattr(row, "unified_object_id", None) if row is not None else None
+    _CONTAINER_STORED_ID_CACHE.set_cache(
+        model_object_id,
+        (
+            stored_id
+            if isinstance(stored_id, str) and stored_id
+            else _NEGATIVE_STORED_ID_SENTINEL
+        ),
+    )
+    return stored_id if isinstance(stored_id, str) and stored_id else None
+
+
 async def assert_user_can_access_container(
     container_id: str,
     user_api_key_dict: UserAPIKeyAuth,

litellm/proxy/db/spend_counter_reseed.py

@@ -178,15 +178,28 @@ class SpendCounterReseed:
             if db_spend is None:
                 return None
             # Warm even when 0 so subsequent reads hit cache, not DB.
+            #
+            # Seed via SET NX (cross-pod safe): only one pod initializes the
+            # Redis key with db_spend; concurrent seeders read the winner's
+            # value. INCRBYFLOAT-of-db_spend from N pods would multiply the
+            # counter (N x db_spend) and trigger spurious budget alerts.
+            current_value: float = float(db_spend)
             try:
                 if spend_counter_cache.redis_cache is not None:
-                    current_value = (
-                        await spend_counter_cache.redis_cache.async_increment(
-                            key=counter_key,
-                            value=db_spend,
-                            refresh_ttl=True,
-                        )
+                    seeded = await spend_counter_cache.redis_cache.async_set_cache(
+                        key=counter_key,
+                        value=db_spend,
+                        nx=True,
                     )
+                    if seeded:
+                        current_value = float(db_spend)
+                    else:
+                        cached = await spend_counter_cache.redis_cache.async_get_cache(
+                            key=counter_key
+                        )
+                        current_value = (
+                            float(cached) if cached is not None else float(db_spend)
+                        )
                     spend_counter_cache.in_memory_cache.set_cache(
                         key=counter_key,
                         value=current_value,
@@ -202,7 +215,7 @@ class SpendCounterReseed:
                 )
                 if require_cache_warm:
                     raise
-            return db_spend
+            return current_value
 
     @staticmethod
     async def window_from_spend_logs(

litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py

@@ -92,6 +92,8 @@ from litellm.types.utils import CallTypesLiteral
 # Module-level singleton for the JWKS discovery endpoint to access.
 _mcp_jwt_signer_instance: Optional["MCPJWTSigner"] = None
 
+_MCP_JWT_CALL_TYPES = frozenset({"call_mcp_tool", "list_mcp_tools"})
+
 # Simple in-memory JWKS cache: keyed by JWKS URI → (keys_list, fetched_at).
 _jwks_cache: Dict[str, tuple] = {}
 _JWKS_CACHE_TTL = 3600  # 1 hour
@@ -603,17 +605,23 @@ class MCPJWTSigner(CustomGuardrail):
     # FR-10: Scope building
     # ------------------------------------------------------------------
 
-    def _build_scope(self, raw_tool_name: str) -> str:
+    def _build_scope(
+        self,
+        raw_tool_name: str,
+        call_type: Optional[CallTypesLiteral] = None,
+    ) -> str:
         """
         Build the JWT scope string.
 
         When allowed_scopes is configured: join them verbatim.
         Otherwise auto-generate minimal, least-privilege scopes:
           - Tool call   → mcp:tools/call  mcp:tools/<name>:call
-          - No tool     → mcp:tools/call  mcp:tools/list
+          - No tool     → mcp:tools/list
 
         NOTE: tools/list is intentionally NOT granted on tool-call JWTs to
         prevent callers from enumerating tools they didn't ask to use.
+        Conversely, tools/call is NOT granted on tools/list-only JWTs so an
+        intercepted list token cannot be replayed to invoke tools.
         """
         if self.allowed_scopes is not None:
             return " ".join(self.allowed_scopes)
@@ -623,8 +631,14 @@ class MCPJWTSigner(CustomGuardrail):
         )
         if tool_name:
             scopes = ["mcp:tools/call", f"mcp:tools/{tool_name}:call"]
+        elif call_type == "call_mcp_tool":
+            # Tool-call request reached the signer without a tool name (e.g.
+            # missing mcp_tool_name in hook data). Fall back to a generic
+            # tools/call scope so the upstream server still accepts the
+            # invocation rather than rejecting it as a tools/list-only token.
+            scopes = ["mcp:tools/call"]
         else:
-            scopes = ["mcp:tools/call", "mcp:tools/list"]
+            scopes = ["mcp:tools/list"]
         return " ".join(scopes)
 
     # ------------------------------------------------------------------
@@ -673,6 +687,7 @@ class MCPJWTSigner(CustomGuardrail):
         user_api_key_dict: UserAPIKeyAuth,
         data: dict,
         jwt_claims: Optional[Dict[str, Any]] = None,
+        call_type: Optional[CallTypesLiteral] = None,
     ) -> Dict[str, Any]:
         """
         Build JWT claims for the outbound MCP access token.
@@ -713,7 +728,7 @@ class MCPJWTSigner(CustomGuardrail):
 
         # scope (FR-10)
         raw_tool_name: str = data.get("mcp_tool_name", "")
-        claims["scope"] = self._build_scope(raw_tool_name)
+        claims["scope"] = self._build_scope(raw_tool_name, call_type=call_type)
 
         # optional_claims passthrough (FR-15)
         claims = self._passthrough_optional_claims(claims, jwt_claims)
@@ -779,16 +794,20 @@ class MCPJWTSigner(CustomGuardrail):
         Verifies the incoming token (when configured), validates required claims,
         then signs an outbound JWT and injects it as the Authorization header.
 
-        All non-MCP call types pass through unchanged.
+        Signs outbound MCP tool calls and tools/list requests.
         """
-        if call_type != "call_mcp_tool":
+        if call_type not in _MCP_JWT_CALL_TYPES:
             return data
 
+        hook_data = dict(data)
+        if call_type == "list_mcp_tools":
+            hook_data["mcp_tool_name"] = ""
+
         # ------------------------------------------------------------------
         # FR-5: Verify incoming token before re-signing
         # ------------------------------------------------------------------
         jwt_claims: Optional[Dict[str, Any]] = None
-        raw_token: Optional[str] = data.get("incoming_bearer_token")
+        raw_token: Optional[str] = hook_data.get("incoming_bearer_token")
 
         if self.access_token_discovery_uri and raw_token:
             # Three-dot pattern → JWT;  otherwise opaque.
@@ -837,7 +856,9 @@ class MCPJWTSigner(CustomGuardrail):
         # ------------------------------------------------------------------
         # Build outbound access token
         # ------------------------------------------------------------------
-        claims = self._build_claims(user_api_key_dict, data, jwt_claims)
+        claims = self._build_claims(
+            user_api_key_dict, hook_data, jwt_claims, call_type=call_type
+        )
 
         signed_token = jwt.encode(
             claims,
@@ -848,7 +869,7 @@ class MCPJWTSigner(CustomGuardrail):
 
         # Merge into existing extra_headers — a prior guardrail in the chain may
         # have already injected tracing headers or correlation IDs.
-        existing_headers: Dict[str, str] = data.get("extra_headers") or {}
+        existing_headers: Dict[str, str] = hook_data.get("extra_headers") or {}
         new_headers: Dict[str, str] = {
             **existing_headers,
             "Authorization": f"Bearer {signed_token}",
@@ -875,17 +896,74 @@ class MCPJWTSigner(CustomGuardrail):
                 claims, self._kid
             )
 
-        data["extra_headers"] = new_headers
+        hook_data["extra_headers"] = new_headers
 
         verbose_proxy_logger.debug(
             "MCPJWTSigner: signed JWT sub=%s act=%s tool=%s exp=%d "
-            "verified=%s channel=%s",
+            "verified=%s channel=%s call_type=%s",
             claims.get("sub"),
             claims.get("act", {}).get("sub"),
-            data.get("mcp_tool_name"),
+            hook_data.get("mcp_tool_name"),
             claims["exp"],
             jwt_claims is not None,
             bool(self.channel_token_audience),
+            call_type,
         )
 
-        return data
+        return hook_data
+
+
+async def inject_mcp_jwt_headers_for_upstream(
+    user_api_key_dict: Optional[UserAPIKeyAuth],
+    extra_headers: Optional[Dict[str, str]] = None,
+    raw_headers: Optional[Dict[str, str]] = None,
+    *,
+    for_list_tools: bool = False,
+    mcp_tool_name: str = "",
+) -> Dict[str, str]:
+    """
+    Sign outbound MCP headers when MCPJWTSigner is configured.
+
+    Used by tools/list paths that do not go through proxy pre_call_hook.
+    """
+    merged = dict(extra_headers or {})
+    signer = get_mcp_jwt_signer()
+    if signer is None or user_api_key_dict is None:
+        return merged
+
+    normalized_raw = {k.lower(): v for k, v in (raw_headers or {}).items()}
+    incoming_bearer_token: Optional[str] = None
+    auth_hdr = normalized_raw.get("authorization", "")
+    if auth_hdr.lower().startswith("bearer "):
+        incoming_bearer_token = auth_hdr[len("bearer ") :]
+
+    hook_data: Dict[str, Any] = {
+        "mcp_tool_name": "" if for_list_tools else mcp_tool_name,
+        "incoming_bearer_token": incoming_bearer_token,
+        "extra_headers": merged,
+    }
+    call_type: CallTypesLiteral = (
+        "list_mcp_tools" if for_list_tools else "call_mcp_tool"
+    )
+    try:
+        from litellm.proxy.proxy_server import (  # noqa: PLC0415
+            proxy_logging_obj as _proxy_logging,
+        )
+
+        shared_cache = (
+            _proxy_logging.internal_usage_cache.dual_cache
+            if _proxy_logging is not None
+            else DualCache()
+        )
+    except Exception:
+        shared_cache = DualCache()
+
+    result = await signer.async_pre_call_hook(
+        user_api_key_dict=user_api_key_dict,
+        cache=shared_cache,
+        data=hook_data,
+        call_type=call_type,
+    )
+    if isinstance(result, dict) and result.get("extra_headers"):
+        merged.update(result["extra_headers"])
+    return merged

litellm/proxy/management_endpoints/ui_sso.py

@@ -1798,7 +1798,10 @@ async def cli_sso_callback(
 
         from fastapi.responses import HTMLResponse
 
-        verify_url = str(request.url_for("cli_sso_complete", login_id=key))
+        verify_url = get_custom_url(
+            request_base_url=str(request.base_url),
+            route=f"sso/cli/complete/{key}",
+        )
         html_content = _render_cli_sso_verification_page(
             verify_url=verify_url,
             browser_complete_token=browser_complete_token,

litellm/proxy/openai_files_endpoints/common_utils.py

@@ -727,6 +727,76 @@ async def resolve_output_file_ids_to_unified(response, prisma_client) -> None:
             pass
 
 
+async def ensure_batch_response_managed_file_ids(
+    response,
+    managed_files_obj,
+    prisma_client,
+    verbose_proxy_logger,
+    user_api_key_dict=None,
+    db_batch_object=None,
+) -> None:
+    """Normalize batch file IDs to managed unified IDs before DB persistence."""
+    await resolve_input_file_id_to_unified(response, prisma_client)
+    await resolve_output_file_ids_to_unified(response, prisma_client)
+
+    if managed_files_obj is None:
+        return
+
+    hidden_params = getattr(response, "_hidden_params", None) or {}
+    model_id = hidden_params.get("model_id")
+    if not model_id:
+        return
+
+    model_name = hidden_params.get("model_name")
+    unified_file_id = hidden_params.get("unified_file_id")
+    if not model_name and isinstance(unified_file_id, str):
+        decoded_unified_file_id = (
+            _is_base64_encoded_unified_file_id(unified_file_id) or unified_file_id
+        )
+        target_model_names = get_models_from_unified_file_id(decoded_unified_file_id)
+        if target_model_names:
+            model_name = ",".join(target_model_names)
+
+    if user_api_key_dict is None and db_batch_object is not None:
+        from litellm.proxy._types import UserAPIKeyAuth
+
+        user_api_key_dict = UserAPIKeyAuth(
+            user_id=getattr(db_batch_object, "created_by", None) or "default-user-id",
+            team_id=getattr(db_batch_object, "team_id", None),
+        )
+    if user_api_key_dict is None:
+        return
+
+    for file_attr in ("output_file_id", "error_file_id"):
+        raw_file_id = getattr(response, file_attr, None)
+        if not raw_file_id or _is_base64_encoded_unified_file_id(raw_file_id):
+            continue
+        try:
+            new_unified_file_id = managed_files_obj.get_unified_output_file_id(
+                output_file_id=raw_file_id,
+                model_id=model_id,
+                model_name=model_name,
+            )
+            await managed_files_obj.store_unified_file_id(
+                file_id=new_unified_file_id,
+                file_object=None,
+                litellm_parent_otel_span=getattr(
+                    user_api_key_dict, "parent_otel_span", None
+                ),
+                model_mappings={model_id: raw_file_id},
+                user_api_key_dict=user_api_key_dict,
+            )
+            setattr(response, file_attr, new_unified_file_id)
+            verbose_proxy_logger.debug(
+                f"Converted batch {file_attr} {raw_file_id!r} to managed ID before DB write"
+            )
+        except Exception as e:
+            verbose_proxy_logger.warning(
+                f"Failed to convert batch {file_attr}={raw_file_id!r} to managed ID "
+                f"before DB write: {e}"
+            )
+
+
 async def get_batch_from_database(
     batch_id: str,
     unified_batch_id: Union[str, Literal[False]],
@@ -800,6 +870,7 @@ async def update_batch_in_database(
     verbose_proxy_logger,
     db_batch_object=None,
     operation: str = "update",
+    user_api_key_dict=None,
 ):
     """
     Update batch status and object in ManagedObjectTable.
@@ -813,6 +884,7 @@ async def update_batch_in_database(
         verbose_proxy_logger: Logger instance
         db_batch_object: Optional existing database object (for comparison)
         operation: Description of operation ("update", "cancel", etc.)
+        user_api_key_dict: Optional auth context for creating managed file IDs
     """
     import litellm.utils
 
@@ -823,6 +895,18 @@ async def update_batch_in_database(
         if not prisma_client:
             return
 
+        # Always normalize the response's file IDs to unified managed IDs
+        # (mutates in place) so the caller returns unified IDs to the user
+        # even when we skip the DB update below for an unchanged status.
+        await ensure_batch_response_managed_file_ids(
+            response=response,
+            managed_files_obj=managed_files_obj,
+            prisma_client=prisma_client,
+            verbose_proxy_logger=verbose_proxy_logger,
+            user_api_key_dict=user_api_key_dict,
+            db_batch_object=db_batch_object,
+        )
+
         # Only update if status has changed (when db_batch_object is provided)
         if db_batch_object and response.status == db_batch_object.status:
             return

litellm/proxy/proxy_cli.py

@@ -38,6 +38,35 @@ class LiteLLMDatabaseConnectionPool(Enum):
     database_connection_pool_timeout = 60
 
 
+def _build_db_connection_url_params(
+    connection_limit: int,
+    pool_timeout: Optional[Union[int, float]],
+    connect_timeout: Optional[Union[int, float]] = None,
+    socket_timeout: Optional[Union[int, float]] = None,
+    extra_params: Optional[dict] = None,
+) -> dict:
+    """Build the Prisma DATABASE_URL query params controlling connection pool behavior.
+
+    `connect_timeout` / `socket_timeout` map to the Prisma URL params of the same
+    name (https://www.prisma.io/docs/orm/overview/databases/postgresql) and are
+    omitted when None so Prisma's defaults apply. `extra_params` is an
+    untyped passthrough — keys it provides win over the named arguments above,
+    so it can be used to override any default we set here.
+    """
+    params: dict = {
+        "connection_limit": connection_limit,
+    }
+    if pool_timeout is not None:
+        params["pool_timeout"] = pool_timeout
+    if connect_timeout is not None:
+        params["connect_timeout"] = connect_timeout
+    if socket_timeout is not None:
+        params["socket_timeout"] = socket_timeout
+    if extra_params:
+        params.update(extra_params)
+    return params
+
+
 def append_query_params(url: Optional[str], params: dict) -> str:
     from litellm._logging import verbose_proxy_logger
 
@@ -807,6 +836,9 @@ def run_server(  # noqa: PLR0915
         db_connection_pool_limit = 100
         # Starts optional due to config fallback checks; guaranteed non-None before use.
         db_connection_timeout: Optional[Union[int, float]] = 60
+        db_connect_timeout: Optional[Union[int, float]] = None
+        db_socket_timeout: Optional[Union[int, float]] = None
+        db_extra_connection_params: Optional[dict] = None
         general_settings = {}
         ### GET DB TOKEN FOR IAM AUTH ###
 
@@ -924,6 +956,11 @@ def run_server(  # noqa: PLR0915
                 db_connection_timeout = (
                     LiteLLMDatabaseConnectionPool.database_connection_pool_timeout.value
                 )
+            db_connect_timeout = general_settings.get("database_connect_timeout")
+            db_socket_timeout = general_settings.get("database_socket_timeout")
+            db_extra_connection_params = general_settings.get(
+                "database_extra_connection_params"
+            )
             if database_url and database_url.startswith("os.environ/"):
                 original_dir = os.getcwd()
                 # set the working directory to where this script is
@@ -963,27 +1000,26 @@ def run_server(  # noqa: PLR0915
             try:
                 from litellm.secret_managers.main import get_secret
 
+                connection_url_params = _build_db_connection_url_params(
+                    connection_limit=db_connection_pool_limit,
+                    pool_timeout=db_connection_timeout,
+                    connect_timeout=db_connect_timeout,
+                    socket_timeout=db_socket_timeout,
+                    extra_params=db_extra_connection_params,
+                )
                 if os.getenv("DATABASE_URL", None) is not None:
-                    ### add connection pool + pool timeout args
-                    params = {
-                        "connection_limit": db_connection_pool_limit,
-                        "pool_timeout": db_connection_timeout,
-                    }
                     database_url = get_secret("DATABASE_URL", default_value=None)
                     modified_url = append_query_params(
-                        str(database_url) if database_url else None, params
+                        str(database_url) if database_url else None,
+                        connection_url_params,
                     )
                     os.environ["DATABASE_URL"] = modified_url
                 if os.getenv("DIRECT_URL", None) is not None:
-                    ### add connection pool + pool timeout args
-                    params = {
-                        "connection_limit": db_connection_pool_limit,
-                        "pool_timeout": db_connection_timeout,
-                    }
                     database_url = os.getenv("DIRECT_URL")
-                    modified_url = append_query_params(database_url, params)
+                    modified_url = append_query_params(
+                        database_url, connection_url_params
+                    )
                     os.environ["DIRECT_URL"] = modified_url
-                    ###
                 subprocess.run(["prisma"], capture_output=True)
                 is_prisma_runnable = True
             except FileNotFoundError:

litellm/proxy/proxy_server.py

@@ -4328,6 +4328,19 @@ class ProxyConfig:
                 "health_check_concurrency", None
             )
             health_check_details = general_settings.get("health_check_details", True)
+            ### INTERACTIONS API SCHEMA ###
+            _use_legacy_interactions_schema = general_settings.get(
+                "use_legacy_interactions_schema"
+            )
+            if _use_legacy_interactions_schema is not None:
+                if isinstance(_use_legacy_interactions_schema, str):
+                    litellm.use_legacy_interactions_schema = (
+                        _use_legacy_interactions_schema.lower() == "true"
+                    )
+                else:
+                    litellm.use_legacy_interactions_schema = bool(
+                        _use_legacy_interactions_schema
+                    )
             # Health-check-driven routing (opt-in, passes through to Router later)
             _enable_hc_routing = general_settings.get(
                 "enable_health_check_routing", False

litellm/proxy/utils.py

@@ -6068,6 +6068,8 @@ async def get_available_models_for_user(
         include_model_access_groups=include_model_access_groups,
     )
 
+    effective_team_id = team_id or user_api_key_dict.team_id
+
     # Get complete model list
     all_models = get_complete_model_list(
         key_models=key_models,
@@ -6080,6 +6082,7 @@ async def get_available_models_for_user(
         model_access_groups=model_access_groups,
         include_model_access_groups=include_model_access_groups,
         only_model_access_groups=only_model_access_groups,
+        team_id=effective_team_id,
     )
 
     return all_models

litellm/responses/litellm_completion_transformation/handler.py

@@ -65,8 +65,7 @@ class LiteLLMCompletionTransformationHandler:
         litellm_completion_response: Union[
             ModelResponse, litellm.CustomStreamWrapper
         ] = litellm.completion(
-            **litellm_completion_request,
-            **kwargs,
+            **completion_args,
         )
 
         if isinstance(litellm_completion_response, ModelResponse):

litellm/responses/main.py

@@ -1115,6 +1115,7 @@ def responses(
                 stream=stream,
                 extra_headers=extra_headers,
                 extra_body=extra_body,
+                timeout=timeout if timeout is not None else request_timeout,
                 **kwargs,
             )
 

litellm/router.py

@@ -208,6 +208,15 @@ if TYPE_CHECKING:
     from litellm.router_strategy.quality_router.quality_router import (
         QualityRouter,
     )
+    from litellm.responses.streaming_iterator import (
+        BaseResponsesAPIStreamingIterator,
+    )
+    from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject
+    from litellm.types.llms.openai import (
+        ResponseAPIUsage,
+        ResponseInputParam,
+        ResponsesAPIResponse,
+    )
 
     Span = Union[_Span, Any]
 else:
@@ -2246,6 +2255,388 @@ class Router:
 
         return FallbackStreamWrapper(stream_with_fallbacks())
 
+    @staticmethod
+    def _extract_partial_responses_usage(
+        source_iterator: "BaseResponsesAPIStreamingIterator",
+    ) -> Optional["ResponseAPIUsage"]:
+        """
+        Best-effort: pull partial token usage from a Responses-API streaming
+        iterator that errored mid-stream, normalized to ResponseAPIUsage so
+        the caller can combine without crossing token-naming conventions.
+
+        Two sources, in priority order:
+          1. The bridge path (LiteLLMCompletionStreamingIterator) accumulates
+             chat-completion chunks while streaming — feed them through
+             stream_chunk_builder to recover chat Usage, then translate
+             (prompt_tokens → input_tokens, completion_tokens → output_tokens).
+          2. The native path (ResponsesAPIStreamingIterator) only has a
+             completed_response object if the stream reached
+             RESPONSE_COMPLETED before erroring — uncommon mid-stream but
+             worth checking. Already ResponseAPIUsage-shaped.
+
+        Returns None when no partial usage is recoverable.
+        """
+        from litellm.responses.litellm_completion_transformation.streaming_iterator import (
+            LiteLLMCompletionStreamingIterator,
+        )
+        from litellm.types.llms.openai import (
+            ResponseAPIUsage,
+            ResponseCompletedEvent,
+            ResponseFailedEvent,
+            ResponseIncompleteEvent,
+        )
+
+        # Bridge subclass is the only iterator that accumulates chat-completion
+        # chunks. isinstance narrows the type so we can read the attribute
+        # directly instead of getattr-ing on the base class.
+        if isinstance(source_iterator, LiteLLMCompletionStreamingIterator):
+            chunks = source_iterator.collected_chat_completion_chunks
+            if chunks:
+                try:
+                    from litellm.main import stream_chunk_builder
+
+                    built = stream_chunk_builder(chunks=chunks)
+                    # stream_chunk_builder returns ModelResponse |
+                    # TextCompletionResponse | None. ModelResponse sets .usage
+                    # in __init__ rather than declaring it as a class field, so
+                    # static narrowing doesn't expose it. Mirror the sync path
+                    # (_completion_streaming_iterator) and pull via getattr.
+                    chat = getattr(built, "usage", None) if built is not None else None
+                    if chat is not None:
+                        # getattr-with-default because the test path may
+                        # substitute a SimpleNamespace lacking some fields;
+                        # real Usage instances always have them.
+                        prompt = int(getattr(chat, "prompt_tokens", 0) or 0)
+                        completion = int(getattr(chat, "completion_tokens", 0) or 0)
+                        total = int(
+                            getattr(chat, "total_tokens", prompt + completion)
+                            or (prompt + completion)
+                        )
+                        return ResponseAPIUsage(
+                            input_tokens=prompt,
+                            output_tokens=completion,
+                            total_tokens=total,
+                        )
+                except Exception:
+                    # Builder is best-effort — fall through to native path.
+                    pass
+
+        # Native path: completed_response is set only if RESPONSE_COMPLETED
+        # arrived before the error (uncommon mid-stream but worth checking).
+        # Already ResponseAPIUsage-shaped — return as-is.
+        completed = source_iterator.completed_response
+        if isinstance(
+            completed,
+            (ResponseCompletedEvent, ResponseFailedEvent, ResponseIncompleteEvent),
+        ):
+            return completed.response.usage
+        return None
+
+    @staticmethod
+    def _combine_responses_fallback_usage(
+        fallback_item: "BaseLiteLLMOpenAIResponseObject",
+        partial_usage: "ResponseAPIUsage",
+    ) -> None:
+        """
+        Merge partial-stream usage with fallback-stream usage on a
+        Responses-API streaming event.
+
+        Only mutates events that carry a `response` with a `usage` field
+        (response.completed / response.failed / response.incomplete). Other
+        events pass through unchanged.
+
+        Both inputs are ResponseAPIUsage-shaped (see
+        _extract_partial_responses_usage which normalizes the bridge path),
+        so we can sum input_tokens / output_tokens / total_tokens directly
+        and produce a clean ResponseAPIUsage — no token-naming split, no
+        setattr bypass.
+        """
+        from litellm.types.llms.openai import (
+            ResponseAPIUsage,
+            ResponseCompletedEvent,
+            ResponseFailedEvent,
+            ResponseIncompleteEvent,
+        )
+
+        if not isinstance(
+            fallback_item,
+            (ResponseCompletedEvent, ResponseFailedEvent, ResponseIncompleteEvent),
+        ):
+            return
+        response = fallback_item.response
+        if response.usage is None:
+            return
+
+        fb = response.usage
+        response.usage = ResponseAPIUsage(
+            input_tokens=(partial_usage.input_tokens or 0) + (fb.input_tokens or 0),
+            output_tokens=(partial_usage.output_tokens or 0) + (fb.output_tokens or 0),
+            total_tokens=(partial_usage.total_tokens or 0) + (fb.total_tokens or 0),
+        )
+
+    @staticmethod
+    def _build_responses_continuation_input(
+        input_val: Optional[Union[str, "ResponseInputParam"]],
+        generated_content: str,
+    ) -> "ResponseInputParam":
+        """
+        Convert Responses-API input + partial assistant output into a
+        continuation input that asks the fallback model to pick up where the
+        prior assistant message stopped.
+
+        Best effort across providers. The chat-completions path uses
+        Anthropic's `prefix: True` prefill trick on the assistant message;
+        the Responses-API input schema has no direct equivalent, so we
+        append an instruction (developer role) plus a prior assistant
+        message containing the partial output. Providers without prefill
+        semantics (OpenAI, Vertex) treat this as conversational context
+        and may regenerate — same trade-off as the chat-completions path
+        for non-Anthropic fallbacks.
+        """
+        # base/continuation are List[Any] because ResponseInputParam items
+        # are a wide Union of TypedDicts (EasyInputMessageParam, Message,
+        # ResponseOutputMessageParam, ...) — annotating as List[Dict[str, Any]]
+        # rejects the list() spread of input_val. We cast the combined list to
+        # ResponseInputParam at the return.
+        base: List[Any]
+        if isinstance(input_val, str):
+            base = [
+                {
+                    "type": "message",
+                    "role": "user",
+                    "content": [{"type": "input_text", "text": input_val}],
+                }
+            ]
+        elif isinstance(input_val, list):
+            base = list(input_val)
+        else:
+            base = []
+        continuation: List[Any] = [
+            {
+                "type": "message",
+                "role": "developer",
+                "content": [
+                    {
+                        "type": "input_text",
+                        "text": (
+                            "The previous assistant response was interrupted "
+                            "mid-stream. Continue exactly where it stopped — "
+                            "do not repeat any of its content. Your response "
+                            "must read as a seamless continuation."
+                        ),
+                    }
+                ],
+            },
+            {
+                "type": "message",
+                "role": "assistant",
+                "content": [{"type": "output_text", "text": generated_content}],
+            },
+        ]
+        return cast("ResponseInputParam", base + continuation)
+
+    async def _aresponses_streaming_iterator(
+        self,
+        response: "BaseResponsesAPIStreamingIterator",
+        initial_kwargs: Dict[str, Any],
+    ) -> "BaseResponsesAPIStreamingIterator":
+        """
+        Wrap a Responses-API streaming iterator so MidStreamFallbackError
+        triggers the Router's fallback chain (parity with
+        _acompletion_streaming_iterator for the chat-completions path).
+
+        The Responses-API streaming path goes through
+        _ageneric_api_call_with_fallbacks rather than _acompletion, so the
+        returned iterator is never wrapped by the chat completions
+        fallback handler. Without this wrapper, MidStreamFallbackError
+        raised mid-stream from the underlying CustomStreamWrapper (used by
+        LiteLLMCompletionStreamingIterator when the Responses API is
+        served via the completion bridge) propagates unhandled and the
+        configured cross-provider fallback never fires.
+
+        Full parity with the chat-completions path:
+          - Pre-first-chunk: retry with the original input unchanged.
+          - Partial content: inject a developer instruction + prior
+            assistant message carrying the generated text so the fallback
+            model continues rather than restarts.
+          - Usage combining: merge partial-stream usage onto the fallback's
+            response.completed event so accounting reflects both attempts.
+          - Stream cleanup: shielded aclose() on both source and fallback
+            iterators on terminate.
+        """
+        from litellm.exceptions import MidStreamFallbackError
+        from litellm.responses.streaming_iterator import (
+            BaseResponsesAPIStreamingIterator,
+        )
+
+        source_iterator = response
+
+        class FallbackResponsesStreamWrapper(BaseResponsesAPIStreamingIterator):
+            """
+            Subclasses BaseResponsesAPIStreamingIterator only for isinstance
+            compatibility (proxy + interactions code paths check the type).
+            Bypasses the parent constructor and delegates iteration to an
+            async generator.
+            """
+
+            def __init__(self, async_generator: AsyncGenerator):
+                import time
+                from datetime import datetime
+
+                self._async_generator = async_generator
+                # Mirror every attribute BaseResponsesAPIStreamingIterator.__init__
+                # would have set. The wrapper bypasses super().__init__ (it has no
+                # httpx.Response of its own and no provider config to drive), so
+                # we copy from source_iterator where applicable and use safe
+                # defaults elsewhere. This keeps inherited methods (e.g.
+                # _check_max_streaming_duration, _handle_failure) safe to call.
+                #
+                # The bridge path (LiteLLMCompletionStreamingIterator used by
+                # Anthropic/Bedrock/Vertex) does not call super().__init__ and
+                # is missing many of these attributes — use getattr fallbacks
+                # so wrapper construction never raises AttributeError. The
+                # bridge stores the logging object as `litellm_logging_obj`.
+                self.response = getattr(source_iterator, "response", None)
+                self.model = getattr(source_iterator, "model", None)
+                self.logging_obj = getattr(
+                    source_iterator,
+                    "logging_obj",
+                    getattr(source_iterator, "litellm_logging_obj", None),
+                )
+                self.finished = False
+                self.responses_api_provider_config = getattr(
+                    source_iterator, "responses_api_provider_config", None
+                )
+                self.completed_response = None
+                self.start_time = getattr(source_iterator, "start_time", datetime.now())
+                self._failure_handled = False
+                self._completed_response_cached = False
+                self._completed_response_logged = False
+                self._completed_response_cache_hit = None
+                self._persist_completed_response_before_logging = True
+                self._stream_created_time = time.time()
+                self.litellm_metadata = getattr(
+                    source_iterator, "litellm_metadata", None
+                )
+                self.custom_llm_provider = getattr(
+                    source_iterator, "custom_llm_provider", None
+                )
+                self.request_data = getattr(source_iterator, "request_data", {}) or {}
+                self.call_type = getattr(source_iterator, "call_type", None)
+                # Preserve hidden params so response headers (model_id,
+                # api_base, additional_headers) keep flowing.
+                self._hidden_params = dict(
+                    getattr(source_iterator, "_hidden_params", None) or {}
+                )
+
+            def __aiter__(self):
+                return self
+
+            async def __anext__(self):
+                return await self._async_generator.__anext__()
+
+            async def aclose(self):
+                # async generators always expose aclose — no defensive check needed.
+                await self._async_generator.aclose()
+
+        async def stream_with_fallbacks():
+            fallback_response = None
+            try:
+                async for item in source_iterator:
+                    yield item
+            except MidStreamFallbackError as e:
+                partial_usage = Router._extract_partial_responses_usage(source_iterator)
+                try:
+                    model_group = cast(str, initial_kwargs.get("model"))
+                    fallbacks: Optional[List] = initial_kwargs.get(
+                        "fallbacks", self.fallbacks
+                    )
+                    context_window_fallbacks: Optional[List] = initial_kwargs.get(
+                        "context_window_fallbacks", self.context_window_fallbacks
+                    )
+                    content_policy_fallbacks: Optional[List] = initial_kwargs.get(
+                        "content_policy_fallbacks", self.content_policy_fallbacks
+                    )
+                    # Re-enter via the per-attempt helper so the fallback chain
+                    # picks deployments through
+                    # _ageneric_api_call_with_fallbacks_helper.
+                    # original_generic_function is preserved by the caller so
+                    # the helper knows what underlying API to invoke per attempt.
+                    initial_kwargs["original_function"] = (
+                        self._ageneric_api_call_with_fallbacks_helper
+                    )
+                    if e.is_pre_first_chunk or not e.generated_content:
+                        # No content generated before the error — retry with the
+                        # original input. Adding a continuation prompt would
+                        # waste tokens and confuse the model.
+                        pass
+                    else:
+                        initial_kwargs["input"] = (
+                            Router._build_responses_continuation_input(
+                                initial_kwargs.get("input"),
+                                e.generated_content,
+                            )
+                        )
+                    # The Responses-API path stores observability metadata
+                    # under "litellm_metadata" (not the default "metadata") —
+                    # see _ageneric_api_call_with_fallbacks. Mirroring that
+                    # here ensures model_group, model_group_alias, and trace
+                    # ids land in the same key litellm.aresponses reads from.
+                    self._update_kwargs_before_fallbacks(
+                        model=model_group,
+                        kwargs=initial_kwargs,
+                        metadata_variable_name="litellm_metadata",
+                    )
+                    fallback_response = (
+                        await self.async_function_with_fallbacks_common_utils(
+                            e=e,
+                            disable_fallbacks=False,
+                            fallbacks=fallbacks,
+                            context_window_fallbacks=context_window_fallbacks,
+                            content_policy_fallbacks=content_policy_fallbacks,
+                            model_group=model_group,
+                            args=(),
+                            kwargs=initial_kwargs,
+                        )
+                    )
+
+                    if hasattr(fallback_response, "__aiter__"):
+                        async for fallback_item in fallback_response:  # type: ignore
+                            if partial_usage is not None:
+                                Router._combine_responses_fallback_usage(
+                                    fallback_item, partial_usage
+                                )
+                            yield fallback_item
+                    else:
+                        yield fallback_response
+                except Exception as fallback_error:
+                    verbose_router_logger.error(
+                        f"Responses streaming fallback also failed: {fallback_error}"
+                    )
+                    raise fallback_error
+            finally:
+                with anyio.CancelScope(shield=True):
+                    if hasattr(source_iterator, "aclose"):
+                        try:
+                            await source_iterator.aclose()  # type: ignore[func-returns-value]
+                        except BaseException as exc:
+                            verbose_router_logger.debug(
+                                "stream_with_fallbacks(aresponses): error closing source: %s",
+                                exc,
+                            )
+                    if fallback_response is not None and hasattr(
+                        fallback_response, "aclose"
+                    ):
+                        try:
+                            await fallback_response.aclose()
+                        except BaseException as exc:
+                            verbose_router_logger.debug(
+                                "stream_with_fallbacks(aresponses): error closing fallback: %s",
+                                exc,
+                            )
+
+        return FallbackResponsesStreamWrapper(stream_with_fallbacks())
+
     def _completion_streaming_iterator(  # noqa: PLR0915
         self,
         model_response: CustomStreamWrapper,
@@ -4292,6 +4683,61 @@ class Router:
                 self.fail_calls[model] += 1
             raise e
 
+    async def _aresponses_with_streaming_fallbacks(
+        self, original_function: Callable, **kwargs: Any
+    ) -> Union["ResponsesAPIResponse", "BaseResponsesAPIStreamingIterator"]:
+        """
+        _ageneric_api_call_with_fallbacks for the Responses API, with the
+        addition of mid-stream fallback handling.
+
+        When stream=True and the underlying call returns a
+        BaseResponsesAPIStreamingIterator, wrap it with
+        _aresponses_streaming_iterator so MidStreamFallbackError raised
+        during iteration triggers the Router's cross-provider fallback chain.
+        """
+        from litellm.responses.streaming_iterator import (
+            BaseResponsesAPIStreamingIterator,
+        )
+
+        from litellm.litellm_core_utils.core_helpers import safe_deep_copy
+
+        # Snapshot the request kwargs before _ageneric_api_call_with_fallbacks
+        # mutates them. A shallow copy alone is not enough: the primary
+        # attempt mutates nested dicts in place — notably `litellm_metadata`,
+        # which `_update_kwargs_with_deployment` populates with
+        # deployment-specific fields (`deployment`, `model_info`, `api_base`,
+        # tags, etc.). Without an explicit copy of that dict, the shallow
+        # copy would still share its reference, leaking primary-deployment
+        # metadata into the mid-stream fallback request.
+        #
+        # We avoid deep-copying the full kwargs because it can contain
+        # non-deepcopyable objects (logging handles, async clients, etc.);
+        # `safe_deep_copy` deep-copies the metadata dicts key-by-key with a
+        # fallback to the original reference for any non-picklable value.
+        # The original_generic_function is preserved so the per-attempt
+        # helper knows which underlying API to call on fallback.
+        fallback_kwargs: Dict[str, Any] = kwargs.copy()
+        if isinstance(fallback_kwargs.get("litellm_metadata"), dict):
+            fallback_kwargs["litellm_metadata"] = safe_deep_copy(
+                fallback_kwargs["litellm_metadata"]
+            )
+        if isinstance(fallback_kwargs.get("metadata"), dict):
+            fallback_kwargs["metadata"] = safe_deep_copy(fallback_kwargs["metadata"])
+        fallback_kwargs["original_generic_function"] = original_function
+
+        response = await self._ageneric_api_call_with_fallbacks(
+            original_function=original_function, **kwargs
+        )
+
+        if kwargs.get("stream") and isinstance(
+            response, BaseResponsesAPIStreamingIterator
+        ):
+            return await self._aresponses_streaming_iterator(
+                response=response,
+                initial_kwargs=fallback_kwargs,
+            )
+        return response
+
     def _generic_api_call_with_fallbacks(
         self, model: str, original_function: Callable, **kwargs
     ):
@@ -5511,9 +5957,13 @@ class Router:
                     custom_llm_provider=custom_llm_provider,
                     **kwargs,
                 )
+            elif call_type == "aresponses":
+                return await self._aresponses_with_streaming_fallbacks(
+                    original_function=original_function,
+                    **kwargs,
+                )
             elif call_type in (
                 "anthropic_messages",
-                "aresponses",
                 "_arealtime",
                 "_aresponses_websocket",
                 "acreate_fine_tuning_job",
@@ -5670,6 +6120,7 @@ class Router:
         from litellm.responses.utils import ResponsesAPIRequestUtils
 
         container_id = kwargs.get("container_id")
+        _forwarded_model_id = kwargs.get("model_id")
         if isinstance(container_id, str):
             decoded = ResponsesAPIRequestUtils._decode_container_id(container_id)
             original_id = decoded.get("response_id", container_id)
@@ -5678,7 +6129,14 @@ class Router:
             decoded_provider = decoded.get("custom_llm_provider")
             if decoded_provider and kwargs.get("custom_llm_provider") == "openai":
                 kwargs["custom_llm_provider"] = decoded_provider
-            model_id = decoded.get("model_id")
+            # Fall back to the model_id forwarded by the proxy when the container_id
+            # is a native upstream ID (e.g. Azure hex cntr_) that carries no LiteLLM
+            # routing payload, so deployment credentials (api_base, api_key) are applied.
+            model_id = decoded.get("model_id") or (
+                _forwarded_model_id.strip()
+                if isinstance(_forwarded_model_id, str) and _forwarded_model_id.strip()
+                else None
+            )
             if model_id:
                 kwargs["model"] = model_id
                 return await self._ageneric_api_call_with_fallbacks(

litellm/types/interactions/__init__.py

@@ -36,9 +36,13 @@ from litellm.types.interactions.generated import (
     GoogleSearchResultContent,
     ImageContent,
     Interaction,
+    InteractionCompleted,
+    InteractionCreated,
     InteractionEvent,
     InteractionEnvironment,
+    InteractionInProgress,
     InteractionInput,
+    InteractionRequiresAction,
     InteractionsAPIOptionalRequestParams,
     InteractionsAPIResponse,
     InteractionsAPIStreamingResponse,
@@ -50,6 +54,9 @@ from litellm.types.interactions.generated import (
     McpServerToolResultContent,
     ModelOption,
     ResponseModality,
+    StepDelta,
+    StepStart,
+    StepStop,
 )
 from litellm.types.interactions.generated import (
     Status3 as InteractionStatus,  # Main request/response types; Content types; Turn for multi-turn conversations; Tool types; Config types; Usage; Status enum; Events for streaming; Agent configs; Model/Agent options; Response modality; Annotation; LiteLLM types; Backwards compat aliases
@@ -115,6 +122,14 @@ __all__ = [
     "AgentOption",
     "ResponseModality",
     "Annotation",
+    # New schema SSE event types (Api-Revision: 2026-05-20)
+    "StepStart",
+    "StepDelta",
+    "StepStop",
+    "InteractionCreated",
+    "InteractionInProgress",
+    "InteractionCompleted",
+    "InteractionRequiresAction",
     # LiteLLM types
     "InteractionEnvironment",
     "InteractionInput",

litellm/types/interactions/generated.py

@@ -1153,9 +1153,114 @@ class InteractionEvent(BaseModel):
     )
 
 
+# ---------------------------------------------------------------
+# New schema SSE event types (Api-Revision: 2026-05-20)
+# These replace the legacy content.* / interaction.start|complete
+# events and will become the only events after June 8, 2026.
+# ---------------------------------------------------------------
+
+
+class StepStart(BaseModel):
+    """Emitted when a new step begins (replaces content.start)."""
+
+    event_type: Literal["step.start"] = "step.start"
+    index: Optional[int] = None
+    step: Optional[Dict[str, Any]] = Field(
+        None,
+        description="The initial step data (type, content, signature, etc.).",
+    )
+    event_id: Optional[str] = Field(
+        None,
+        description="The event_id token to be used to resume the interaction stream.",
+    )
+
+
+class StepDelta(BaseModel):
+    """Emitted for incremental step content (replaces content.delta)."""
+
+    event_type: Literal["step.delta"] = "step.delta"
+    index: Optional[int] = None
+    delta: Optional[Dict[str, Any]] = Field(
+        None,
+        description="Incremental content delta (e.g. text, arguments_delta for function calls).",
+    )
+    event_id: Optional[str] = Field(
+        None,
+        description="The event_id token to be used to resume the interaction stream.",
+    )
+
+
+class StepStop(BaseModel):
+    """Emitted when a step finishes (replaces content.stop)."""
+
+    event_type: Literal["step.stop"] = "step.stop"
+    index: Optional[int] = None
+    status: Optional[str] = Field(
+        None,
+        description="Step completion status (e.g. 'done').",
+    )
+    event_id: Optional[str] = Field(
+        None,
+        description="The event_id token to be used to resume the interaction stream.",
+    )
+
+
+class InteractionCreated(BaseModel):
+    """Emitted when the interaction is first created (replaces interaction.start)."""
+
+    event_type: Literal["interaction.created"] = "interaction.created"
+    interaction: Optional[Dict[str, Any]] = None
+    event_id: Optional[str] = Field(
+        None,
+        description="The event_id token to be used to resume the interaction stream.",
+    )
+
+
+class InteractionInProgress(BaseModel):
+    """Emitted while the interaction is running."""
+
+    event_type: Literal["interaction.in_progress"] = "interaction.in_progress"
+    interaction_id: Optional[str] = None
+    event_id: Optional[str] = Field(
+        None,
+        description="The event_id token to be used to resume the interaction stream.",
+    )
+
+
+class InteractionCompleted(BaseModel):
+    """Emitted when the interaction finishes (replaces interaction.complete)."""
+
+    event_type: Literal["interaction.completed"] = "interaction.completed"
+    interaction: Optional[Dict[str, Any]] = None
+    event_id: Optional[str] = Field(
+        None,
+        description="The event_id token to be used to resume the interaction stream.",
+    )
+
+
+class InteractionRequiresAction(BaseModel):
+    """Emitted when the interaction is paused waiting for a tool result."""
+
+    event_type: Literal["interaction.requires_action"] = "interaction.requires_action"
+    interaction_id: Optional[str] = None
+    event_id: Optional[str] = Field(
+        None,
+        description="The event_id token to be used to resume the interaction stream.",
+    )
+
+
 class InteractionSseEvent(
     RootModel[
         Union[
+            # New schema events (Api-Revision: 2026-05-20)
+            StepStart,
+            StepDelta,
+            StepStop,
+            InteractionCreated,
+            InteractionInProgress,
+            InteractionCompleted,
+            InteractionRequiresAction,
+            # Legacy schema events (Api-Revision: 2026-05-07, removed June 8 2026)
             InteractionEvent,
             InteractionStatusUpdate,
             ContentStart,
@@ -1166,6 +1271,15 @@ class InteractionSseEvent(
     ]
 ):
     root: Union[
+        # New schema events (Api-Revision: 2026-05-20)
+        StepStart,
+        StepDelta,
+        StepStop,
+        InteractionCreated,
+        InteractionInProgress,
+        InteractionCompleted,
+        InteractionRequiresAction,
+        # Legacy schema events (Api-Revision: 2026-05-07, removed June 8 2026)
         InteractionEvent,
         InteractionStatusUpdate,
         ContentStart,
@@ -1195,6 +1309,11 @@ class InteractionsAPIResponse(BaseLiteLLMOpenAIResponseObject):
     Response from the Interactions API.
 
     Wraps the API response with LiteLLM-specific hidden params.
+
+    Schema notes:
+    - New schema (Api-Revision: 2026-05-20, default): response contains ``steps``.
+    - Legacy schema (Api-Revision: 2026-05-07, removed June 8 2026): response contains ``outputs``.
+    Both fields are kept here so callers work with either schema.
     """
 
     id: Optional[str] = None
@@ -1205,7 +1324,10 @@ class InteractionsAPIResponse(BaseLiteLLMOpenAIResponseObject):
     created: Optional[str] = None
     updated: Optional[str] = None
     role: Optional[str] = None
+    # Legacy schema field (Api-Revision: 2026-05-07). Remove after June 8, 2026.
     outputs: Optional[List[Dict[str, Any]]] = None
+    # New schema field (Api-Revision: 2026-05-20).
+    steps: Optional[List[Dict[str, Any]]] = None
     usage: Optional[Dict[str, Any]] = None
 
     _hidden_params: dict = PrivateAttr(default_factory=dict)
@@ -1215,7 +1337,12 @@ class InteractionsAPIStreamingResponse(BaseLiteLLMOpenAIResponseObject):
     """
     Streaming response chunk from the Interactions API.
 
-    Event types per OpenAPI spec:
+    New schema event types (Api-Revision: 2026-05-20):
+    - interaction.created, interaction.in_progress, interaction.completed,
+      interaction.requires_action
+    - step.start, step.delta, step.stop
+
+    Legacy event types (Api-Revision: 2026-05-07, removed June 8 2026):
     - interaction.start, interaction.status_update, interaction.complete
     - content.start, content.delta, content.stop
     - error
@@ -1230,9 +1357,17 @@ class InteractionsAPIStreamingResponse(BaseLiteLLMOpenAIResponseObject):
     created: Optional[str] = None
     updated: Optional[str] = None
     role: Optional[str] = None
+    # Legacy schema field (Api-Revision: 2026-05-07). Remove after June 8, 2026.
     outputs: Optional[List[Dict[str, Any]]] = None
+    # New schema field (Api-Revision: 2026-05-20).
+    steps: Optional[List[Dict[str, Any]]] = None
     usage: Optional[Dict[str, Any]] = None
     delta: Optional[Dict[str, Any]] = None
+    # New schema streaming fields
+    index: Optional[int] = None
+    step: Optional[Dict[str, Any]] = None
+    interaction_id: Optional[str] = None
+    interaction: Optional[Dict[str, Any]] = None
 
     _hidden_params: dict = PrivateAttr(default_factory=dict)
 

litellm/types/llms/vertex_ai.py

@@ -16,15 +16,15 @@ GeminiEmbeddingInput = Union[EmbeddingInput, List[List[str]]]
 
 class FunctionResponse(TypedDict, total=False):
     # `id` correlates this response with the originating `functionCall` part.
-    # Required by Gemini 3.5+ for strict function-calling response matching.
+    # Supported on Google AI Studio Gemini 3.5+; Vertex AI rejects this field.
     id: str
     name: Required[str]
     response: Optional[dict]
 
 
 class FunctionCall(TypedDict, total=False):
-    # `id` is returned by Gemini 3.5+ to correlate the corresponding
-    # `functionResponse`. Older Gemini models omit this field.
+    # `id` correlates the corresponding `functionResponse` on Google AI Studio
+    # Gemini 3.5+. Vertex AI and older Gemini models omit/reject this field.
     id: str
     name: Required[str]
     args: Optional[dict]
@@ -52,8 +52,8 @@ class PartType(TypedDict, total=False):
 
 
 class HttpxFunctionCall(TypedDict, total=False):
-    # `id` is returned by Gemini 3.5+ to correlate the corresponding
-    # `functionResponse`. Older Gemini models omit this field.
+    # `id` correlates the corresponding `functionResponse` on Google AI Studio
+    # Gemini 3.5+. Vertex AI and older Gemini models omit/reject this field.
     id: str
     name: Required[str]
     args: dict

model_prices_and_context_window.json

@@ -27296,6 +27296,58 @@
         "supports_web_search": true,
         "tpm": 800000
     },
+    "openrouter/google/gemini-3.1-flash-lite": {
+        "cache_read_input_token_cost": 2.5e-08,
+        "cache_read_input_token_cost_per_audio_token": 5e-08,
+        "input_cost_per_audio_token": 5e-07,
+        "input_cost_per_token": 2.5e-07,
+        "litellm_provider": "openrouter",
+        "max_audio_length_hours": 8.4,
+        "max_audio_per_prompt": 1,
+        "max_images_per_prompt": 3000,
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_pdf_size_mb": 30,
+        "max_tokens": 65536,
+        "max_video_length": 1,
+        "max_videos_per_prompt": 10,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 1.5e-06,
+        "output_cost_per_token": 1.5e-06,
+        "rpm": 2000,
+        "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": false,
+        "supports_code_execution": true,
+        "supports_file_search": true,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_url_context": true,
+        "supports_video_input": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "tpm": 800000
+    },
     "openrouter/google/gemini-3.1-pro-preview": {
         "cache_read_input_token_cost": 2e-07,
         "cache_read_input_token_cost_above_200k_tokens": 4e-07,
@@ -28105,10 +28157,10 @@
         "supports_tool_choice": true
     },
     "openrouter/xiaomi/mimo-v2-flash": {
-        "input_cost_per_token": 9e-08,
-        "output_cost_per_token": 2.9e-07,
+        "input_cost_per_token": 1e-07,
+        "output_cost_per_token": 3e-07,
         "cache_creation_input_token_cost": 0.0,
-        "cache_read_input_token_cost": 0.0,
+        "cache_read_input_token_cost": 1e-08,
         "litellm_provider": "openrouter",
         "max_input_tokens": 262144,
         "max_output_tokens": 16384,
@@ -28118,7 +28170,43 @@
         "supports_tool_choice": true,
         "supports_reasoning": true,
         "supports_vision": false,
-        "supports_prompt_caching": false
+        "supports_prompt_caching": true
+    },
+    "openrouter/xiaomi/mimo-v2.5-pro": {
+        "input_cost_per_token": 1e-06,
+        "output_cost_per_token": 3e-06,
+        "cache_creation_input_token_cost": 0.0,
+        "cache_read_input_token_cost": 2e-07,
+        "litellm_provider": "openrouter",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_reasoning": true,
+        "supports_vision": false,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true
+    },
+    "openrouter/xiaomi/mimo-v2.5": {
+        "input_cost_per_token": 4e-07,
+        "output_cost_per_token": 2e-06,
+        "cache_creation_input_token_cost": 0.0,
+        "cache_read_input_token_cost": 8e-08,
+        "litellm_provider": "openrouter",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_reasoning": true,
+        "supports_vision": true,
+        "supports_audio_input": true,
+        "supports_video_input": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true
     },
     "openrouter/z-ai/glm-4.7": {
         "input_cost_per_token": 4e-07,

pyproject.toml

@@ -287,6 +287,12 @@ paths_to_mutate = [
 ]
 tests_dir = [
     "tests/test_litellm/proxy/management_endpoints/",
+    # PR1 (key Tier-1) behavior-pinning suite. Manual mutmut runs
+    # (.github/workflows/mutation-test.yml) include this directory so the
+    # behavior matrix contributes to mutation-score signal alongside the
+    # legacy mock suite. See tests/proxy_behavior/management/README.md
+    # for the G5 triage protocol.
+    "tests/proxy_behavior/management/",
 ]
 also_copy = [
     "litellm/",

tests/llm_responses_api_testing/test_anthropic_responses_api.py

@@ -3,7 +3,7 @@ import sys
 import pytest
 import asyncio
 from typing import Optional
-from unittest.mock import patch, AsyncMock
+from unittest.mock import patch, AsyncMock, MagicMock
 from litellm.responses.litellm_completion_transformation.handler import (
     LiteLLMCompletionTransformationHandler,
 )
@@ -130,6 +130,26 @@ def test_multiturn_tool_calls():
     print("follow_up_response=", follow_up_response)
 
 
+def test_response_api_handler_merges_metadata_and_service_tier_without_error():
+    """Sync path must merge kwargs like async; double-splat raises TypeError."""
+    handler = LiteLLMCompletionTransformationHandler()
+
+    with patch("litellm.completion", new_callable=MagicMock) as mock_completion:
+        mock_completion.return_value = ModelResponse(
+            id="id", created=0, model="test", object="chat.completion", choices=[]
+        )
+        handler.response_api_handler(
+            model="test",
+            input="hi",
+            responses_api_request={},
+            metadata={"trace": "abc"},
+            service_tier="auto",
+        )
+        assert mock_completion.call_count == 1
+        assert mock_completion.call_args.kwargs["metadata"] == {"trace": "abc"}
+        assert mock_completion.call_args.kwargs["service_tier"] == "auto"
+
+
 @pytest.mark.asyncio
 async def test_async_response_api_handler_merges_trace_id_without_error():
     handler = LiteLLMCompletionTransformationHandler()
@@ -158,3 +178,39 @@ async def test_async_response_api_handler_merges_trace_id_without_error():
             assert (
                 mock_acompletion.call_args.kwargs["litellm_trace_id"] == "session-trace"
             )
+
+
+@pytest.mark.asyncio
+async def test_aresponses_forwards_timeout_to_acompletion():
+    """Regression test: timeout passed to aresponses() must reach acompletion()
+    on the completion transformation path (Anthropic, Bedrock, Vertex etc.).
+
+    Previously, `timeout` was a named param of `responses()` but was NOT
+    forwarded to `litellm_completion_transformation_handler.response_api_handler`,
+    so it was silently dropped — `Router(timeout=N)` was a no-op for Anthropic
+    and similar providers, with calls falling back to the provider SDK default
+    (~600s for Anthropic).
+    """
+    with patch("litellm.acompletion", new_callable=AsyncMock) as mock_acompletion:
+        mock_acompletion.return_value = ModelResponse(
+            id="id",
+            created=0,
+            model="anthropic/claude-sonnet-4-5",
+            object="chat.completion",
+            choices=[],
+        )
+
+        await litellm.aresponses(
+            model="anthropic/claude-sonnet-4-5",
+            input="hello",
+            timeout=42,
+            api_key="sk-ant-fake",
+        )
+
+    assert mock_acompletion.call_count == 1
+    forwarded_timeout = mock_acompletion.call_args.kwargs.get("timeout")
+    assert forwarded_timeout == 42, (
+        f"timeout was not forwarded to acompletion (got {forwarded_timeout!r}); "
+        "this means Router(timeout=N) silently fails for providers on the "
+        "completion transformation path."
+    )

tests/llm_translation/realtime/base_realtime_tests.py

@@ -79,7 +79,7 @@ class RealTimeWebSocketClient:
 
     def _is_initial_event(self, msg_type: str) -> bool:
         """Check if message type is an initial connection event"""
-        # OpenAI sends "session.created", xAI sends "conversation.created"
+        # OpenAI and xAI send "session.created"; some providers send "conversation.created"
         return msg_type in ["session.created", "conversation.created"]
 
     async def receive_text(self):

tests/llm_translation/realtime/test_xai_realtime.py

@@ -19,8 +19,8 @@ class TestXAIRealtime(BaseRealtimeTest):
     """
     E2E tests for xAI Realtime API.
 
-    xAI's Grok Voice Agent API is OpenAI-compatible but uses:
-    - Different initial event: "conversation.created" instead of "session.created"
+    xAI's Grok Voice Agent API is OpenAI-compatible:
+    - Initial event: "session.created" (matches OpenAI)
     - Different endpoint: wss://api.x.ai/v1/realtime
     - Model: grok-4-1-fast-non-reasoning
     """
@@ -32,4 +32,4 @@ class TestXAIRealtime(BaseRealtimeTest):
         return "XAI_API_KEY"
 
     def get_initial_event_type(self) -> str:
-        return "conversation.created"
+        return "session.created"

tests/local_testing/conftest.py

@@ -31,10 +31,8 @@ import litellm
 # cassette state the branch is being tested with.
 from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
 
-_local_cost_map = GetModelCostMap.load_local_model_cost_map()
-for _k, _v in _local_cost_map.items():
+for _k, _v in GetModelCostMap.load_local_model_cost_map().items():
     litellm.model_cost.setdefault(_k, _v)
-del _local_cost_map
 
 from tests._vcr_conftest_common import (  # noqa: E402,F401
     VerboseReporterState,

tests/mcp_tests/test_mcp_server.py

@@ -382,6 +382,11 @@ async def test_mcp_http_transport_tool_not_found():
         }
     )
 
+    # Mapping populated for this server but not for the requested tool
+    test_manager.tool_name_to_mcp_server_name_mapping["gmail_send_email"] = (
+        "test_http_server"
+    )
+
     # Try to call a tool that doesn't exist in mapping
     with pytest.raises(ValueError, match="Tool nonexistent_tool not found"):
         await test_manager.call_tool(
@@ -944,6 +949,7 @@ async def test_get_tools_from_mcp_servers():
                 extra_headers=None,
                 add_prefix=False,
                 raw_headers=None,
+                user_api_key_auth=None,
             ):
                 if server.server_id == "server1_id":
                     return [mock_tool_1]
@@ -1919,6 +1925,7 @@ async def test_get_tools_for_single_server():
             extra_headers=None,
             add_prefix=False,
             raw_headers=None,
+            user_api_key_auth=None,
         )
 
         # Verify the result

tests/proxy_behavior/management/actors.py

@@ -0,0 +1,257 @@
+"""8-actor read-world seed for the authz matrix tests."""
+
+import enum
+import uuid
+from dataclasses import dataclass
+from typing import Any, Dict
+
+from prisma import Json
+
+from litellm.proxy._types import LitellmUserRoles
+from litellm.proxy.utils import PrismaClient, hash_token
+
+
+class Actor(str, enum.Enum):
+    PROXY_ADMIN = "proxy_admin"
+    ORG_ADMIN = "org_admin"
+    TEAM_ADMIN = "team_admin"
+    INTERNAL_USER = "internal_user"
+    OWNER = "owner"
+    UNRELATED_SAME_ORG = "unrelated_same_org"
+    CROSS_ORG_USER = "cross_org_user"
+    SERVICE_ACCOUNT = "service_account"
+
+
+PREFIX = "behavior-pin-"
+ORG_A = PREFIX + "org-a"
+ORG_B = PREFIX + "org-b"
+TEAM_ALPHA = PREFIX + "team-alpha"
+TEAM_BETA = PREFIX + "team-beta"
+BUDGET_ID = PREFIX + "budget"
+
+
+@dataclass(frozen=True)
+class SeededKey:
+    user_id: str
+    cleartext: str
+    hashed: str
+
+
+@dataclass(frozen=True)
+class World:
+    org_a_id: str
+    org_b_id: str
+    team_alpha_id: str
+    team_beta_id: str
+    keys: Dict[Actor, SeededKey]
+
+
+def _new_clear_key() -> str:
+    return "sk-" + uuid.uuid4().hex
+
+
+def _actor_profile() -> Dict[Actor, Dict[str, Any]]:
+    return {
+        Actor.PROXY_ADMIN: {
+            "user_role": LitellmUserRoles.PROXY_ADMIN.value,
+            "team_id": None,
+            "organization_id": None,
+        },
+        Actor.ORG_ADMIN: {
+            "user_role": LitellmUserRoles.ORG_ADMIN.value,
+            "team_id": None,
+            "organization_id": ORG_A,
+        },
+        Actor.TEAM_ADMIN: {
+            "user_role": LitellmUserRoles.INTERNAL_USER.value,
+            "team_id": TEAM_ALPHA,
+            "organization_id": ORG_A,
+        },
+        Actor.INTERNAL_USER: {
+            "user_role": LitellmUserRoles.INTERNAL_USER.value,
+            "team_id": TEAM_ALPHA,
+            "organization_id": ORG_A,
+        },
+        Actor.OWNER: {
+            "user_role": LitellmUserRoles.INTERNAL_USER.value,
+            "team_id": TEAM_ALPHA,
+            "organization_id": ORG_A,
+        },
+        Actor.UNRELATED_SAME_ORG: {
+            "user_role": LitellmUserRoles.INTERNAL_USER.value,
+            "team_id": TEAM_ALPHA,
+            "organization_id": ORG_A,
+        },
+        Actor.CROSS_ORG_USER: {
+            "user_role": LitellmUserRoles.INTERNAL_USER.value,
+            "team_id": TEAM_BETA,
+            "organization_id": ORG_B,
+        },
+        Actor.SERVICE_ACCOUNT: {
+            "user_role": LitellmUserRoles.INTERNAL_USER.value,
+            "team_id": TEAM_ALPHA,
+            "organization_id": ORG_A,
+        },
+    }
+
+
+async def _wipe_world(prisma: PrismaClient) -> None:
+    await prisma.db.litellm_verificationtoken.delete_many(
+        where={"user_id": {"startswith": PREFIX}}
+    )
+    await prisma.db.litellm_organizationmembership.delete_many(
+        where={"user_id": {"startswith": PREFIX}}
+    )
+    await prisma.db.litellm_teammembership.delete_many(
+        where={"user_id": {"startswith": PREFIX}}
+    )
+    await prisma.db.litellm_usertable.delete_many(
+        where={"user_id": {"startswith": PREFIX}}
+    )
+    await prisma.db.litellm_teamtable.delete_many(
+        where={"team_id": {"startswith": PREFIX}}
+    )
+    await prisma.db.litellm_organizationtable.delete_many(
+        where={"organization_id": {"startswith": PREFIX}}
+    )
+    await prisma.db.litellm_budgettable.delete_many(where={"budget_id": BUDGET_ID})
+
+
+async def seed_world(prisma: PrismaClient) -> World:
+    await _wipe_world(prisma)
+
+    await prisma.db.litellm_budgettable.create(
+        data={
+            "budget_id": BUDGET_ID,
+            "created_by": "behavior-pin-seeder",
+            "updated_by": "behavior-pin-seeder",
+        }
+    )
+
+    for org_id, alias in [(ORG_A, "alpha"), (ORG_B, "beta")]:
+        await prisma.db.litellm_organizationtable.create(
+            data={
+                "organization_id": org_id,
+                "organization_alias": alias,
+                "budget_id": BUDGET_ID,
+                "created_by": "behavior-pin-seeder",
+                "updated_by": "behavior-pin-seeder",
+            }
+        )
+
+    profiles = _actor_profile()
+    user_ids: Dict[Actor, str] = {actor: PREFIX + actor.value for actor in Actor}
+
+    for actor, profile in profiles.items():
+        teams_list = [profile["team_id"]] if profile["team_id"] else []
+        await prisma.db.litellm_usertable.create(
+            data={
+                "user_id": user_ids[actor],
+                "user_role": profile["user_role"],
+                "team_id": profile["team_id"],
+                "organization_id": profile["organization_id"],
+                "teams": teams_list,
+            }
+        )
+
+    # _get_user_in_team in key_management_endpoints.py walks members_with_roles
+    # (a JSON list of {user_id, role}), not the String[] members column —
+    # populate both to match what /team/new produces.
+    await prisma.db.litellm_teamtable.create(
+        data={
+            "team_id": TEAM_ALPHA,
+            "team_alias": "alpha-1",
+            "organization_id": ORG_A,
+            "admins": [user_ids[Actor.TEAM_ADMIN]],
+            "members": [
+                user_ids[Actor.TEAM_ADMIN],
+                user_ids[Actor.INTERNAL_USER],
+                user_ids[Actor.OWNER],
+                user_ids[Actor.UNRELATED_SAME_ORG],
+                user_ids[Actor.SERVICE_ACCOUNT],
+            ],
+            "members_with_roles": Json(
+                [
+                    {"user_id": user_ids[Actor.TEAM_ADMIN], "role": "admin"},
+                    {"user_id": user_ids[Actor.INTERNAL_USER], "role": "user"},
+                    {"user_id": user_ids[Actor.OWNER], "role": "user"},
+                    {"user_id": user_ids[Actor.UNRELATED_SAME_ORG], "role": "user"},
+                    {"user_id": user_ids[Actor.SERVICE_ACCOUNT], "role": "user"},
+                ]
+            ),
+        }
+    )
+    await prisma.db.litellm_teamtable.create(
+        data={
+            "team_id": TEAM_BETA,
+            "team_alias": "beta-1",
+            "organization_id": ORG_B,
+            "admins": [],
+            "members": [user_ids[Actor.CROSS_ORG_USER]],
+            "members_with_roles": Json(
+                [
+                    {"user_id": user_ids[Actor.CROSS_ORG_USER], "role": "user"},
+                ]
+            ),
+        }
+    )
+
+    for actor, org_id, role in [
+        (Actor.ORG_ADMIN, ORG_A, "org_admin"),
+        (Actor.TEAM_ADMIN, ORG_A, "internal_user"),
+        (Actor.INTERNAL_USER, ORG_A, "internal_user"),
+        (Actor.OWNER, ORG_A, "internal_user"),
+        (Actor.UNRELATED_SAME_ORG, ORG_A, "internal_user"),
+        (Actor.SERVICE_ACCOUNT, ORG_A, "internal_user"),
+        (Actor.CROSS_ORG_USER, ORG_B, "internal_user"),
+    ]:
+        await prisma.db.litellm_organizationmembership.create(
+            data={
+                "user_id": user_ids[actor],
+                "organization_id": org_id,
+                "user_role": role,
+            }
+        )
+
+    for actor, team_id in [
+        (Actor.TEAM_ADMIN, TEAM_ALPHA),
+        (Actor.INTERNAL_USER, TEAM_ALPHA),
+        (Actor.OWNER, TEAM_ALPHA),
+        (Actor.UNRELATED_SAME_ORG, TEAM_ALPHA),
+        (Actor.SERVICE_ACCOUNT, TEAM_ALPHA),
+        (Actor.CROSS_ORG_USER, TEAM_BETA),
+    ]:
+        await prisma.db.litellm_teammembership.create(
+            data={"user_id": user_ids[actor], "team_id": team_id}
+        )
+
+    keys: Dict[Actor, SeededKey] = {}
+    for actor, profile in profiles.items():
+        cleartext = _new_clear_key()
+        hashed = hash_token(cleartext)
+        token_data: Dict[str, Any] = {
+            "token": hashed,
+            "key_name": PREFIX + actor.value + "-key",
+            "user_id": user_ids[actor],
+            # LiteLLM_VerificationTokenView's models field rejects NULL even
+            # though the column is nullable in Postgres.
+            "models": [],
+        }
+        if profile["team_id"]:
+            token_data["team_id"] = profile["team_id"]
+        if profile["organization_id"]:
+            token_data["organization_id"] = profile["organization_id"]
+        if actor == Actor.SERVICE_ACCOUNT:
+            token_data["metadata"] = Json({"service_account_id": user_ids[actor]})
+        await prisma.db.litellm_verificationtoken.create(data=token_data)
+        keys[actor] = SeededKey(
+            user_id=user_ids[actor], cleartext=cleartext, hashed=hashed
+        )
+
+    return World(
+        org_a_id=ORG_A,
+        org_b_id=ORG_B,
+        team_alpha_id=TEAM_ALPHA,
+        team_beta_id=TEAM_BETA,
+        keys=keys,
+    )

tests/proxy_behavior/management/conftest.py

@@ -0,0 +1,156 @@
+"""Session-scoped async ASGI client for HTTP-boundary behavior tests."""
+
+import os
+import tempfile
+import uuid
+from dataclasses import dataclass
+from typing import Any, AsyncIterator, Dict, Optional
+
+import httpx
+import pytest_asyncio
+import yaml
+
+
+MASTER_KEY = "sk-1234"
+SCRATCH_PREFIX = "scratch-"
+
+
+def _write_minimal_proxy_config() -> str:
+    config = {
+        "general_settings": {"master_key": MASTER_KEY},
+        "litellm_settings": {},
+    }
+    database_url = os.environ.get("DATABASE_URL")
+    if database_url:
+        config["general_settings"]["database_url"] = database_url
+    f = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False)
+    yaml.dump(config, f)
+    f.close()
+    return f.name
+
+
+@pytest_asyncio.fixture(scope="session")
+async def proxy_app():
+    from litellm.proxy import proxy_server
+    from litellm.proxy.proxy_server import (
+        app,
+        cleanup_router_config_variables,
+        initialize,
+        proxy_startup_event,
+    )
+
+    cleanup_router_config_variables()
+    config_path = _write_minimal_proxy_config()
+
+    # proxy_startup_event re-reads master_key from LITELLM_MASTER_KEY and
+    # unconditionally overwrites the global, even when initialize() already
+    # set it from the config YAML. Force (not setdefault) both vars: an
+    # ambient LITELLM_MASTER_KEY with a different value would make the proxy
+    # authenticate on that key while the tests still send MASTER_KEY.
+    os.environ["LITELLM_MASTER_KEY"] = MASTER_KEY
+    os.environ["CONFIG_FILE_PATH"] = config_path
+
+    await initialize(config=config_path)
+
+    # /key/regenerate is gated behind premium_user; flipping it lets the matrix
+    # pin authz behavior instead of the licensing gate.
+    proxy_server.premium_user = True
+
+    async with proxy_startup_event(app):
+        proxy_server.premium_user = True  # lifespan re-runs _license_check
+        # The lifespan fires check_view_exists() as a background task; on a
+        # fresh DB the first auth call races it and resolves user_id=None.
+        if proxy_server.prisma_client is not None:
+            await proxy_server.prisma_client.check_view_exists()
+        yield app
+
+
+@pytest_asyncio.fixture(scope="session")
+async def proxy_client(proxy_app) -> AsyncIterator[httpx.AsyncClient]:
+    transport = httpx.ASGITransport(app=proxy_app)
+    async with httpx.AsyncClient(
+        transport=transport, base_url="http://testserver"
+    ) as client:
+        yield client
+
+
+@pytest_asyncio.fixture(scope="session")
+async def prisma(proxy_app):
+    from litellm.proxy import proxy_server
+
+    assert proxy_server.prisma_client is not None
+    return proxy_server.prisma_client
+
+
+@pytest_asyncio.fixture(scope="session")
+async def world(prisma):
+    from .actors import seed_world
+
+    return await seed_world(prisma)
+
+
+@dataclass(frozen=True)
+class Scratch:
+    prefix: str
+
+    def tag(self, suffix: str = "") -> str:
+        return f"{self.prefix}-{suffix}" if suffix else self.prefix
+
+
+async def create_scratch_key(
+    proxy_client,
+    seeder_cleartext: str,
+    scratch_prefix: str,
+    *,
+    user_id: str,
+    team_id: Optional[str] = None,
+    organization_id: Optional[str] = None,
+) -> str:
+    """Seed a scratch-tagged key via /key/generate; returns its cleartext.
+
+    Shared by the write-scenario matrices (key update/regenerate/delete).
+    """
+    body: Dict[str, Any] = {"key_alias": scratch_prefix, "user_id": user_id}
+    if team_id is not None:
+        body["team_id"] = team_id
+    if organization_id is not None:
+        body["organization_id"] = organization_id
+    resp = await proxy_client.post(
+        "/key/generate",
+        headers={"Authorization": f"Bearer {seeder_cleartext}"},
+        json=body,
+    )
+    assert resp.status_code == 200, f"setup failed: {resp.text}"
+    return resp.json()["key"]
+
+
+@pytest_asyncio.fixture
+async def scratch(prisma):
+    handle = Scratch(prefix=f"{SCRATCH_PREFIX}{uuid.uuid4().hex[:12]}")
+    try:
+        yield handle
+    finally:
+        # Children before parents to avoid FK violations.
+        await prisma.db.litellm_verificationtoken.delete_many(
+            where={
+                "OR": [
+                    {"key_alias": {"startswith": handle.prefix}},
+                    {"key_name": {"startswith": handle.prefix}},
+                ]
+            }
+        )
+        await prisma.db.litellm_teammembership.delete_many(
+            where={"team_id": {"startswith": handle.prefix}}
+        )
+        await prisma.db.litellm_organizationmembership.delete_many(
+            where={"user_id": {"startswith": handle.prefix}}
+        )
+        await prisma.db.litellm_teamtable.delete_many(
+            where={"team_id": {"startswith": handle.prefix}}
+        )
+        await prisma.db.litellm_usertable.delete_many(
+            where={"user_id": {"startswith": handle.prefix}}
+        )
+        await prisma.db.litellm_budgettable.delete_many(
+            where={"budget_id": {"startswith": handle.prefix}}
+        )

tests/proxy_behavior/management/test_key_delete.py

@@ -0,0 +1,101 @@
+import pytest
+
+from litellm.proxy.utils import hash_token
+
+from .actors import TEAM_ALPHA, TEAM_BETA, Actor
+from .conftest import create_scratch_key
+
+pytestmark = pytest.mark.asyncio(loop_scope="session")
+
+
+# Same-team peers can READ each other's keys (see test_key_info) but cannot
+# DELETE them — delete is stricter than read.
+_SCENARIOS = [
+    ("self/proxy_admin", Actor.PROXY_ADMIN, "self", 200),
+    ("self/org_admin", Actor.ORG_ADMIN, "self", 401),
+    ("self/team_admin", Actor.TEAM_ADMIN, "self", 200),
+    ("self/internal_user", Actor.INTERNAL_USER, "self", 200),
+    ("self/owner", Actor.OWNER, "self", 200),
+    ("self/unrelated_same_org", Actor.UNRELATED_SAME_ORG, "self", 200),
+    ("self/cross_org_user", Actor.CROSS_ORG_USER, "self", 200),
+    ("self/service_account", Actor.SERVICE_ACCOUNT, "self", 200),
+    ("owner_target/proxy_admin", Actor.PROXY_ADMIN, "owner", 200),
+    ("owner_target/org_admin", Actor.ORG_ADMIN, "owner", 401),
+    ("owner_target/team_admin", Actor.TEAM_ADMIN, "owner", 200),
+    ("owner_target/internal_user", Actor.INTERNAL_USER, "owner", 403),
+    ("owner_target/unrelated_same_org", Actor.UNRELATED_SAME_ORG, "owner", 403),
+    ("owner_target/cross_org_user", Actor.CROSS_ORG_USER, "owner", 403),
+    ("owner_target/service_account", Actor.SERVICE_ACCOUNT, "owner", 403),
+    ("cross_org_target/proxy_admin", Actor.PROXY_ADMIN, "cross_org", 200),
+    ("cross_org_target/org_admin", Actor.ORG_ADMIN, "cross_org", 401),
+    ("cross_org_target/team_admin", Actor.TEAM_ADMIN, "cross_org", 403),
+    ("cross_org_target/owner", Actor.OWNER, "cross_org", 403),
+    ("cross_org_target/cross_org_user", Actor.CROSS_ORG_USER, "cross_org", 200),
+    ("cross_org_target/service_account", Actor.SERVICE_ACCOUNT, "cross_org", 403),
+]
+
+
+@pytest.mark.parametrize(
+    "actor,target_shape,expected_status",
+    [(a, t, s) for (_id, a, t, s) in _SCENARIOS],
+    ids=[s[0] for s in _SCENARIOS],
+)
+async def test_key_delete_authz_matrix(
+    actor: Actor,
+    target_shape: str,
+    expected_status: int,
+    proxy_client,
+    prisma,
+    scratch,
+    world,
+):
+    caller = world.keys[actor]
+    seeder = world.keys[Actor.PROXY_ADMIN].cleartext
+
+    if target_shape == "self":
+        target_cleartext = await create_scratch_key(
+            proxy_client, seeder, scratch.prefix, user_id=caller.user_id
+        )
+    elif target_shape == "owner":
+        target_cleartext = await create_scratch_key(
+            proxy_client,
+            seeder,
+            scratch.prefix,
+            user_id=world.keys[Actor.OWNER].user_id,
+            team_id=TEAM_ALPHA,
+        )
+    elif target_shape == "cross_org":
+        target_cleartext = await create_scratch_key(
+            proxy_client,
+            seeder,
+            scratch.prefix,
+            user_id=world.keys[Actor.CROSS_ORG_USER].user_id,
+            team_id=TEAM_BETA,
+        )
+    else:
+        pytest.fail(f"unknown target_shape={target_shape}")
+
+    target_hashed = hash_token(target_cleartext)
+
+    resp = await proxy_client.post(
+        "/key/delete",
+        headers={"Authorization": f"Bearer {caller.cleartext}"},
+        json={"keys": [target_cleartext]},
+    )
+    assert (
+        resp.status_code == expected_status
+    ), f"{actor.value} {target_shape}: {resp.status_code} {resp.text}"
+
+    row = await prisma.db.litellm_verificationtoken.find_unique(
+        where={"token": target_hashed}
+    )
+    auth_check = await proxy_client.get(
+        "/key/info", headers={"Authorization": f"Bearer {target_cleartext}"}
+    )
+
+    if expected_status == 200:
+        # Hard- or soft-delete both produce a 401 on subsequent auth.
+        assert auth_check.status_code == 401
+    else:
+        assert row is not None, f"{actor.value}: denied but row vanished"
+        assert auth_check.status_code == 200

tests/proxy_behavior/management/test_key_generate.py

@@ -0,0 +1,70 @@
+from typing import Any, Dict
+
+import pytest
+
+from .actors import TEAM_ALPHA, TEAM_BETA, Actor
+
+pytestmark = pytest.mark.asyncio(loop_scope="session")
+
+
+# (id, actor, body_extras, expected_status). Status codes pinned to observed
+# handler behavior — heterogeneous (200, 400, 401) because the handler routes
+# denials through three different gates (role gate, user_id mismatch, team
+# member permission).
+_SCENARIOS = [
+    ("self/proxy_admin", Actor.PROXY_ADMIN, {}, 200),
+    ("self/org_admin", Actor.ORG_ADMIN, {}, 401),
+    ("self/team_admin", Actor.TEAM_ADMIN, {}, 200),
+    ("self/internal_user", Actor.INTERNAL_USER, {}, 200),
+    ("self/owner", Actor.OWNER, {}, 200),
+    ("self/unrelated_same_org", Actor.UNRELATED_SAME_ORG, {}, 200),
+    ("self/cross_org_user", Actor.CROSS_ORG_USER, {}, 200),
+    ("self/service_account", Actor.SERVICE_ACCOUNT, {}, 200),
+    ("team_alpha/proxy_admin", Actor.PROXY_ADMIN, {"team_id": TEAM_ALPHA}, 200),
+    ("team_alpha/org_admin", Actor.ORG_ADMIN, {"team_id": TEAM_ALPHA}, 401),
+    ("team_alpha/team_admin", Actor.TEAM_ADMIN, {"team_id": TEAM_ALPHA}, 200),
+    ("team_alpha/internal_user", Actor.INTERNAL_USER, {"team_id": TEAM_ALPHA}, 401),
+    ("team_alpha/cross_org_user", Actor.CROSS_ORG_USER, {"team_id": TEAM_ALPHA}, 400),
+    ("team_beta/proxy_admin", Actor.PROXY_ADMIN, {"team_id": TEAM_BETA}, 200),
+    ("team_beta/org_admin", Actor.ORG_ADMIN, {"team_id": TEAM_BETA}, 401),
+    ("team_beta/team_admin", Actor.TEAM_ADMIN, {"team_id": TEAM_BETA}, 400),
+    ("team_beta/internal_user", Actor.INTERNAL_USER, {"team_id": TEAM_BETA}, 400),
+    ("team_beta/cross_org_user", Actor.CROSS_ORG_USER, {"team_id": TEAM_BETA}, 401),
+]
+
+
+@pytest.mark.parametrize(
+    "actor,body_extras,expected_status",
+    [(actor, body, expected) for (_id, actor, body, expected) in _SCENARIOS],
+    ids=[s[0] for s in _SCENARIOS],
+)
+async def test_key_generate_authz_matrix(
+    actor: Actor,
+    body_extras: Dict[str, Any],
+    expected_status: int,
+    proxy_client,
+    prisma,
+    scratch,
+    world,
+):
+    seeded = world.keys[actor]
+    body: Dict[str, Any] = {"key_alias": scratch.prefix, **body_extras}
+
+    resp = await proxy_client.post(
+        "/key/generate",
+        headers={"Authorization": f"Bearer {seeded.cleartext}"},
+        json=body,
+    )
+    assert (
+        resp.status_code == expected_status
+    ), f"{actor.value} {body!r} → {resp.status_code}: {resp.text}"
+
+    rows = await prisma.db.litellm_verificationtoken.find_many(
+        where={"key_alias": scratch.prefix}
+    )
+    if expected_status == 200:
+        cleartext = resp.json()["key"]
+        assert cleartext.startswith("sk-")
+        assert len(rows) == 1
+    else:
+        assert rows == [], f"{actor.value}: denied but row leaked"

tests/proxy_behavior/management/test_key_info.py

@@ -0,0 +1,74 @@
+import pytest
+
+from .actors import Actor
+
+pytestmark = pytest.mark.asyncio(loop_scope="session")
+
+
+# (id, actor, target_actor, expected_status). Targets are 3 fixed seeded keys
+# representing the canonical relations: own, OWNER (same org_a/team_alpha),
+# and CROSS_ORG_USER (org_b/team_beta).
+#
+# Notable pinned behaviors (intentionally surfaced, not endorsed):
+#   - ORG_ADMIN 403s on individual key info even within its own org —
+#     visibility is "your own keys" + "your team's keys", not "your org's keys".
+#   - Same-team peers (internal_user, unrelated_same_org, service_account) DO
+#     see each other's keys.
+_SCENARIOS = [
+    ("own/proxy_admin", Actor.PROXY_ADMIN, Actor.PROXY_ADMIN, 200),
+    ("own/org_admin", Actor.ORG_ADMIN, Actor.ORG_ADMIN, 200),
+    ("own/team_admin", Actor.TEAM_ADMIN, Actor.TEAM_ADMIN, 200),
+    ("own/internal_user", Actor.INTERNAL_USER, Actor.INTERNAL_USER, 200),
+    ("own/owner", Actor.OWNER, Actor.OWNER, 200),
+    ("own/unrelated_same_org", Actor.UNRELATED_SAME_ORG, Actor.UNRELATED_SAME_ORG, 200),
+    ("own/cross_org_user", Actor.CROSS_ORG_USER, Actor.CROSS_ORG_USER, 200),
+    ("own/service_account", Actor.SERVICE_ACCOUNT, Actor.SERVICE_ACCOUNT, 200),
+    ("owner_key/proxy_admin", Actor.PROXY_ADMIN, Actor.OWNER, 200),
+    ("owner_key/org_admin", Actor.ORG_ADMIN, Actor.OWNER, 403),
+    ("owner_key/team_admin", Actor.TEAM_ADMIN, Actor.OWNER, 200),
+    ("owner_key/internal_user", Actor.INTERNAL_USER, Actor.OWNER, 200),
+    ("owner_key/owner", Actor.OWNER, Actor.OWNER, 200),
+    ("owner_key/unrelated_same_org", Actor.UNRELATED_SAME_ORG, Actor.OWNER, 200),
+    ("owner_key/cross_org_user", Actor.CROSS_ORG_USER, Actor.OWNER, 403),
+    ("owner_key/service_account", Actor.SERVICE_ACCOUNT, Actor.OWNER, 200),
+    ("cross_org/proxy_admin", Actor.PROXY_ADMIN, Actor.CROSS_ORG_USER, 200),
+    ("cross_org/org_admin", Actor.ORG_ADMIN, Actor.CROSS_ORG_USER, 403),
+    ("cross_org/team_admin", Actor.TEAM_ADMIN, Actor.CROSS_ORG_USER, 403),
+    ("cross_org/internal_user", Actor.INTERNAL_USER, Actor.CROSS_ORG_USER, 403),
+    ("cross_org/owner", Actor.OWNER, Actor.CROSS_ORG_USER, 403),
+    (
+        "cross_org/unrelated_same_org",
+        Actor.UNRELATED_SAME_ORG,
+        Actor.CROSS_ORG_USER,
+        403,
+    ),
+    ("cross_org/cross_org_user", Actor.CROSS_ORG_USER, Actor.CROSS_ORG_USER, 200),
+    ("cross_org/service_account", Actor.SERVICE_ACCOUNT, Actor.CROSS_ORG_USER, 403),
+]
+
+
+@pytest.mark.parametrize(
+    "actor,target_actor,expected_status",
+    [(a, t, s) for (_id, a, t, s) in _SCENARIOS],
+    ids=[s[0] for s in _SCENARIOS],
+)
+async def test_key_info_authz_matrix(
+    actor: Actor, target_actor: Actor, expected_status: int, proxy_client, world
+):
+    caller = world.keys[actor]
+    target = world.keys[target_actor]
+
+    resp = await proxy_client.get(
+        f"/key/info?key={target.cleartext}",
+        headers={"Authorization": f"Bearer {caller.cleartext}"},
+    )
+    assert (
+        resp.status_code == expected_status
+    ), f"{actor.value} → {target_actor.value}: {resp.status_code} {resp.text}"
+
+    if expected_status == 200:
+        body = resp.json()
+        # The handler echoes back whatever ?key was passed (cleartext here),
+        # so accept either form — info.user_id is the canonical identity check.
+        assert body.get("key") in (target.cleartext, target.hashed)
+        assert body["info"].get("user_id") == target.user_id

tests/proxy_behavior/management/test_key_list.py

@@ -0,0 +1,63 @@
+from typing import FrozenSet
+
+import pytest
+
+from .actors import Actor
+
+pytestmark = pytest.mark.asyncio(loop_scope="session")
+
+
+# Pinned default visibility for /key/list (no filter params): each actor's
+# expected set of seeded actor keys.
+_VISIBILITY = {
+    Actor.PROXY_ADMIN: frozenset(Actor),
+    Actor.ORG_ADMIN: frozenset({Actor.ORG_ADMIN}),
+    Actor.TEAM_ADMIN: frozenset({Actor.TEAM_ADMIN}),
+    Actor.INTERNAL_USER: frozenset({Actor.INTERNAL_USER}),
+    Actor.OWNER: frozenset({Actor.OWNER}),
+    Actor.UNRELATED_SAME_ORG: frozenset({Actor.UNRELATED_SAME_ORG}),
+    Actor.CROSS_ORG_USER: frozenset({Actor.CROSS_ORG_USER}),
+    Actor.SERVICE_ACCOUNT: frozenset({Actor.SERVICE_ACCOUNT}),
+}
+
+
+async def _all_visible_hashes(proxy_client, caller_cleartext) -> set:
+    """Walk every /key/list page — size is capped at 100 by the endpoint, so a
+    single request can truncate PROXY_ADMIN's view on a non-fresh DB."""
+    hashes: set = set()
+    page = 1
+    while True:
+        resp = await proxy_client.get(
+            f"/key/list?page={page}&size=100",
+            headers={"Authorization": f"Bearer {caller_cleartext}"},
+        )
+        assert resp.status_code == 200, resp.text
+        body = resp.json()
+        for entry in body.get("keys", []):
+            tok = entry.get("token") if isinstance(entry, dict) else entry
+            if tok:
+                hashes.add(tok)
+        if page >= (body.get("total_pages") or 1):
+            return hashes
+        page += 1
+
+
+@pytest.mark.parametrize(
+    "actor,expected_visible",
+    list(_VISIBILITY.items()),
+    ids=[a.value for a in _VISIBILITY],
+)
+async def test_key_list_visibility(
+    actor: Actor, expected_visible: FrozenSet[Actor], proxy_client, world
+):
+    caller = world.keys[actor]
+    hashed_to_actor = {world.keys[a].hashed: a for a in Actor}
+
+    returned_hashes = await _all_visible_hashes(proxy_client, caller.cleartext)
+    visible_seeded = {
+        hashed_to_actor[h] for h in returned_hashes if h in hashed_to_actor
+    }
+    assert visible_seeded == set(expected_visible), (
+        f"{actor.value}: expected {sorted(a.value for a in expected_visible)}, "
+        f"got {sorted(a.value for a in visible_seeded)}"
+    )

tests/proxy_behavior/management/test_key_regenerate.py

@@ -0,0 +1,117 @@
+import pytest
+
+from .actors import TEAM_ALPHA, TEAM_BETA, Actor
+from .conftest import create_scratch_key
+
+pytestmark = pytest.mark.asyncio(loop_scope="session")
+
+
+# Most denials route through team_member_permission (401), unlike /key/update
+# which goes through user_id-mismatch (403). The matrix surfaces that
+# divergence between the two endpoints.
+_SCENARIOS = [
+    ("self/proxy_admin", Actor.PROXY_ADMIN, "self", 200),
+    ("self/org_admin", Actor.ORG_ADMIN, "self", 401),
+    ("self/team_admin", Actor.TEAM_ADMIN, "self", 200),
+    ("self/internal_user", Actor.INTERNAL_USER, "self", 200),
+    ("self/owner", Actor.OWNER, "self", 200),
+    ("self/unrelated_same_org", Actor.UNRELATED_SAME_ORG, "self", 200),
+    ("self/cross_org_user", Actor.CROSS_ORG_USER, "self", 200),
+    ("self/service_account", Actor.SERVICE_ACCOUNT, "self", 200),
+    ("owner_target/proxy_admin", Actor.PROXY_ADMIN, "owner", 200),
+    ("owner_target/org_admin", Actor.ORG_ADMIN, "owner", 401),
+    ("owner_target/team_admin", Actor.TEAM_ADMIN, "owner", 200),
+    ("owner_target/internal_user", Actor.INTERNAL_USER, "owner", 401),
+    ("owner_target/unrelated_same_org", Actor.UNRELATED_SAME_ORG, "owner", 401),
+    ("owner_target/cross_org_user", Actor.CROSS_ORG_USER, "owner", 401),
+    ("owner_target/service_account", Actor.SERVICE_ACCOUNT, "owner", 401),
+    ("cross_org_target/proxy_admin", Actor.PROXY_ADMIN, "cross_org", 200),
+    ("cross_org_target/org_admin", Actor.ORG_ADMIN, "cross_org", 401),
+    ("cross_org_target/team_admin", Actor.TEAM_ADMIN, "cross_org", 401),
+    ("cross_org_target/owner", Actor.OWNER, "cross_org", 401),
+    ("cross_org_target/cross_org_user", Actor.CROSS_ORG_USER, "cross_org", 401),
+    ("cross_org_target/service_account", Actor.SERVICE_ACCOUNT, "cross_org", 401),
+]
+
+
+async def _info(proxy_client, cleartext: str):
+    return await proxy_client.get(
+        "/key/info", headers={"Authorization": f"Bearer {cleartext}"}
+    )
+
+
+@pytest.mark.parametrize(
+    "actor,target_shape,expected_status",
+    [(a, t, s) for (_id, a, t, s) in _SCENARIOS],
+    ids=[s[0] for s in _SCENARIOS],
+)
+async def test_key_regenerate_authz_matrix(
+    actor: Actor,
+    target_shape: str,
+    expected_status: int,
+    proxy_client,
+    scratch,
+    world,
+):
+    caller = world.keys[actor]
+    seeder = world.keys[Actor.PROXY_ADMIN].cleartext
+
+    if target_shape == "self":
+        target_cleartext = await create_scratch_key(
+            proxy_client, seeder, scratch.prefix, user_id=caller.user_id
+        )
+    elif target_shape == "owner":
+        target_cleartext = await create_scratch_key(
+            proxy_client,
+            seeder,
+            scratch.prefix,
+            user_id=world.keys[Actor.OWNER].user_id,
+            team_id=TEAM_ALPHA,
+        )
+    elif target_shape == "cross_org":
+        target_cleartext = await create_scratch_key(
+            proxy_client,
+            seeder,
+            scratch.prefix,
+            user_id=world.keys[Actor.CROSS_ORG_USER].user_id,
+            team_id=TEAM_BETA,
+        )
+    else:
+        pytest.fail(f"unknown target_shape={target_shape}")
+
+    resp = await proxy_client.post(
+        "/key/regenerate",
+        headers={"Authorization": f"Bearer {caller.cleartext}"},
+        json={"key": target_cleartext},
+    )
+    assert (
+        resp.status_code == expected_status
+    ), f"{actor.value} {target_shape}: {resp.status_code} {resp.text}"
+
+    if expected_status == 200:
+        new_cleartext = resp.json()["key"]
+        assert new_cleartext.startswith("sk-") and new_cleartext != target_cleartext
+        assert (await _info(proxy_client, target_cleartext)).status_code == 401
+        assert (await _info(proxy_client, new_cleartext)).status_code == 200
+    else:
+        # Denied: rotation must not have leaked — old cleartext still works.
+        assert (await _info(proxy_client, target_cleartext)).status_code == 200
+
+
+async def test_key_path_regenerate_smoke(proxy_client, scratch, world):
+    """Pins that POST /key/{key:path}/regenerate shares the same handler."""
+    caller = world.keys[Actor.PROXY_ADMIN]
+    target_cleartext = await create_scratch_key(
+        proxy_client, caller.cleartext, scratch.prefix, user_id=caller.user_id
+    )
+
+    resp = await proxy_client.post(
+        f"/key/{target_cleartext}/regenerate",
+        headers={"Authorization": f"Bearer {caller.cleartext}"},
+        json={},
+    )
+    assert resp.status_code == 200, resp.text
+    new_cleartext = resp.json()["key"]
+    assert new_cleartext.startswith("sk-") and new_cleartext != target_cleartext
+    assert (await _info(proxy_client, target_cleartext)).status_code == 401
+    assert (await _info(proxy_client, new_cleartext)).status_code == 200

tests/proxy_behavior/management/test_key_update.py

@@ -0,0 +1,100 @@
+import pytest
+
+from litellm.proxy.utils import hash_token
+
+from .actors import TEAM_ALPHA, TEAM_BETA, Actor
+from .conftest import create_scratch_key
+
+pytestmark = pytest.mark.asyncio(loop_scope="session")
+
+
+# (id, actor, target_shape, expected_status). Pinned against current gating:
+# proxy_admin bypasses; org_admin is blocked by an early role gate (401);
+# every other (INTERNAL_USER-roled) actor hits user_id-mismatch 403, no-team-
+# admin 403, or team_member_permission 401 depending on target / membership.
+_SCENARIOS = [
+    ("self/proxy_admin", Actor.PROXY_ADMIN, "self", 200),
+    ("self/org_admin", Actor.ORG_ADMIN, "self", 401),
+    ("self/team_admin", Actor.TEAM_ADMIN, "self", 403),
+    ("self/internal_user", Actor.INTERNAL_USER, "self", 403),
+    ("self/owner", Actor.OWNER, "self", 403),
+    ("self/unrelated_same_org", Actor.UNRELATED_SAME_ORG, "self", 403),
+    ("self/cross_org_user", Actor.CROSS_ORG_USER, "self", 403),
+    ("self/service_account", Actor.SERVICE_ACCOUNT, "self", 403),
+    ("owner_target/proxy_admin", Actor.PROXY_ADMIN, "owner", 200),
+    ("owner_target/org_admin", Actor.ORG_ADMIN, "owner", 401),
+    ("owner_target/team_admin", Actor.TEAM_ADMIN, "owner", 403),
+    ("owner_target/internal_user", Actor.INTERNAL_USER, "owner", 403),
+    ("owner_target/unrelated_same_org", Actor.UNRELATED_SAME_ORG, "owner", 403),
+    ("owner_target/cross_org_user", Actor.CROSS_ORG_USER, "owner", 403),
+    ("owner_target/service_account", Actor.SERVICE_ACCOUNT, "owner", 403),
+    ("cross_org_target/proxy_admin", Actor.PROXY_ADMIN, "cross_org", 200),
+    ("cross_org_target/org_admin", Actor.ORG_ADMIN, "cross_org", 401),
+    ("cross_org_target/team_admin", Actor.TEAM_ADMIN, "cross_org", 403),
+    ("cross_org_target/owner", Actor.OWNER, "cross_org", 403),
+    ("cross_org_target/cross_org_user", Actor.CROSS_ORG_USER, "cross_org", 401),
+    ("cross_org_target/service_account", Actor.SERVICE_ACCOUNT, "cross_org", 403),
+]
+
+MARKER_MODEL = "behavior-pin-update-marker-model"
+
+
+@pytest.mark.parametrize(
+    "actor,target_shape,expected_status",
+    [(a, t, s) for (_id, a, t, s) in _SCENARIOS],
+    ids=[s[0] for s in _SCENARIOS],
+)
+async def test_key_update_authz_matrix(
+    actor: Actor,
+    target_shape: str,
+    expected_status: int,
+    proxy_client,
+    prisma,
+    scratch,
+    world,
+):
+    caller = world.keys[actor]
+    seeder = world.keys[Actor.PROXY_ADMIN].cleartext
+
+    if target_shape == "self":
+        target_cleartext = await create_scratch_key(
+            proxy_client, seeder, scratch.prefix, user_id=caller.user_id
+        )
+    elif target_shape == "owner":
+        target_cleartext = await create_scratch_key(
+            proxy_client,
+            seeder,
+            scratch.prefix,
+            user_id=world.keys[Actor.OWNER].user_id,
+            team_id=TEAM_ALPHA,
+        )
+    elif target_shape == "cross_org":
+        target_cleartext = await create_scratch_key(
+            proxy_client,
+            seeder,
+            scratch.prefix,
+            user_id=world.keys[Actor.CROSS_ORG_USER].user_id,
+            team_id=TEAM_BETA,
+        )
+    else:
+        pytest.fail(f"unknown target_shape={target_shape}")
+
+    target_hashed = hash_token(target_cleartext)
+
+    resp = await proxy_client.post(
+        "/key/update",
+        headers={"Authorization": f"Bearer {caller.cleartext}"},
+        json={"key": target_cleartext, "models": [MARKER_MODEL]},
+    )
+    assert (
+        resp.status_code == expected_status
+    ), f"{actor.value} {target_shape}: {resp.status_code} {resp.text}"
+
+    row = await prisma.db.litellm_verificationtoken.find_unique(
+        where={"token": target_hashed}
+    )
+    assert row is not None
+    if expected_status == 200:
+        assert row.models == [MARKER_MODEL]
+    else:
+        assert row.models != [MARKER_MODEL], "denied but row mutated"

tests/proxy_behavior/management/test_no_management_imports.py

@@ -0,0 +1,46 @@
+import pathlib
+import re
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
+BEHAVIOR_DIR = REPO_ROOT / "tests" / "proxy_behavior"
+
+FORBIDDEN_IMPORT = re.compile(r"^\s*from\s+litellm\.proxy\.management_endpoints\b")
+FORBIDDEN_AUTH_MOCK = re.compile(
+    r"(?:mock\.[A-Za-z_]+|patch[a-z_]*)\([^)]*user_api_key_auth"
+)
+# This file is the only place the forbidden patterns appear as regex source;
+# exclude it so it can describe what it forbids.
+SELF = pathlib.Path(__file__).resolve()
+
+
+def _iter_py_files():
+    for path in BEHAVIOR_DIR.rglob("*.py"):
+        if path.resolve() != SELF:
+            yield path
+
+
+def _scan(pattern):
+    violations = []
+    for path in _iter_py_files():
+        for lineno, line in enumerate(path.read_text().splitlines(), start=1):
+            if pattern.search(line):
+                violations.append(
+                    f"{path.relative_to(REPO_ROOT)}:{lineno}: {line.strip()}"
+                )
+    return violations
+
+
+def test_no_management_endpoint_imports():
+    violations = _scan(FORBIDDEN_IMPORT)
+    assert not violations, (
+        "tests/proxy_behavior/ must not import from litellm.proxy.management_endpoints. "
+        "Violations:\n  " + "\n  ".join(violations)
+    )
+
+
+def test_no_user_api_key_auth_mocking():
+    violations = _scan(FORBIDDEN_AUTH_MOCK)
+    assert not violations, (
+        "tests/proxy_behavior/ must not mock user_api_key_auth. "
+        "Violations:\n  " + "\n  ".join(violations)
+    )

tests/proxy_behavior/management/test_scratch_teardown.py

@@ -0,0 +1,31 @@
+import pytest
+
+from .conftest import MASTER_KEY, SCRATCH_PREFIX
+
+pytestmark = pytest.mark.asyncio(loop_scope="session")
+
+
+# The two tests run in file order: _a writes a scratch-tagged key and asserts
+# it lands; _b runs after _a's fixture teardown and asserts no scratch row
+# survived. A leak in either direction fails _b on the next collection.
+
+
+async def test_a_scratch_key_lands_in_db(proxy_client, prisma, scratch):
+    resp = await proxy_client.post(
+        "/key/generate",
+        headers={"Authorization": f"Bearer {MASTER_KEY}"},
+        json={"key_alias": scratch.prefix},
+    )
+    assert resp.status_code == 200, resp.text
+
+    rows = await prisma.db.litellm_verificationtoken.find_many(
+        where={"key_alias": scratch.prefix}
+    )
+    assert len(rows) == 1
+
+
+async def test_b_scratch_namespace_is_clean(prisma):
+    rows = await prisma.db.litellm_verificationtoken.find_many(
+        where={"key_alias": {"startswith": SCRATCH_PREFIX}}
+    )
+    assert rows == []

tests/proxy_behavior/management/test_smoke.py

@@ -0,0 +1,28 @@
+import pytest
+
+from .conftest import MASTER_KEY
+
+pytestmark = pytest.mark.asyncio(loop_scope="session")
+
+
+async def test_liveliness(proxy_client):
+    resp = await proxy_client.get("/health/liveliness")
+    assert resp.status_code == 200
+
+
+async def test_key_generate_lands_in_db(proxy_client, prisma, scratch):
+    from litellm.proxy.utils import hash_token
+
+    resp = await proxy_client.post(
+        "/key/generate",
+        headers={"Authorization": f"Bearer {MASTER_KEY}"},
+        json={"key_alias": scratch.prefix},
+    )
+    assert resp.status_code == 200, resp.text
+    cleartext = resp.json()["key"]
+    assert cleartext.startswith("sk-")
+
+    hashed = hash_token(cleartext)
+    row = await prisma.db.litellm_verificationtoken.find_unique(where={"token": hashed})
+    assert row is not None
+    assert row.token == hashed != cleartext